diff options
| -rw-r--r-- | Include/cpython/optimizer.h | 51 | ||||
| -rw-r--r-- | Include/internal/pycore_opcode_metadata.h | 116 | ||||
| -rw-r--r-- | Include/internal/pycore_optimizer.h | 2 | ||||
| -rw-r--r-- | Include/internal/pycore_uop_ids.h | 195 | ||||
| -rw-r--r-- | Include/internal/pycore_uop_metadata.h | 536 | ||||
| -rw-r--r-- | Python/bytecodes.c | 139 | ||||
| -rw-r--r-- | Python/ceval.c | 57 | ||||
| -rw-r--r-- | Python/ceval_macros.h | 6 | ||||
| -rw-r--r-- | Python/executor_cases.c.h | 738 | ||||
| -rw-r--r-- | Python/generated_cases.c.h | 122 | ||||
| -rw-r--r-- | Python/jit.c | 37 | ||||
| -rw-r--r-- | Python/optimizer.c | 379 | ||||
| -rw-r--r-- | Python/optimizer_analysis.c | 50 | ||||
| -rw-r--r-- | Python/optimizer_cases.c.h | 57 | ||||
| -rw-r--r-- | Tools/cases_generator/analyzer.py | 66 | ||||
| -rw-r--r-- | Tools/cases_generator/generators_common.py | 17 | ||||
| -rw-r--r-- | Tools/cases_generator/opcode_metadata_generator.py | 1 | ||||
| -rw-r--r-- | Tools/cases_generator/tier2_generator.py | 38 | ||||
| -rw-r--r-- | Tools/cases_generator/uop_metadata_generator.py | 17 | ||||
| -rw-r--r-- | Tools/jit/_stencils.py | 6 | ||||
| -rw-r--r-- | Tools/jit/template.c | 31 |
21 files changed, 1660 insertions, 1001 deletions
diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index df83e6d..bc960c5 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -30,16 +30,63 @@ typedef struct { PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). } _PyVMData; +#define UOP_FORMAT_TARGET 0 +#define UOP_FORMAT_EXIT 1 +#define UOP_FORMAT_JUMP 2 +#define UOP_FORMAT_UNUSED 3 + +/* Depending on the format, + * the 32 bits between the oparg and operand are: + * UOP_FORMAT_TARGET: + * uint32_t target; + * UOP_FORMAT_EXIT + * uint16_t exit_index; + * uint16_t error_target; + * UOP_FORMAT_JUMP + * uint16_t jump_target; + * uint16_t error_target; + */ typedef struct { - uint16_t opcode; + uint16_t opcode:14; + uint16_t format:2; uint16_t oparg; union { uint32_t target; - uint32_t exit_index; + struct { + union { + uint16_t exit_index; + uint16_t jump_target; + }; + uint16_t error_target; + }; }; uint64_t operand; // A cache entry } _PyUOpInstruction; +static inline uint32_t uop_get_target(const _PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_TARGET); + return inst->target; +} + +static inline uint16_t uop_get_exit_index(const _PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_EXIT); + return inst->exit_index; +} + +static inline uint16_t uop_get_jump_target(const _PyUOpInstruction *inst) +{ + assert(inst->format == UOP_FORMAT_JUMP); + return inst->jump_target; +} + +static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) +{ + assert(inst->format != UOP_FORMAT_TARGET); + return inst->error_target; +} + typedef struct _exit_data { uint32_t target; int16_t temperature; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index f754de3..de525f7 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -921,6 +921,7 @@ enum InstructionFormat { #define HAS_PURE_FLAG (2048) #define HAS_PASSTHROUGH_FLAG (4096) #define HAS_OPARG_AND_1_FLAG (8192) +#define HAS_ERROR_NO_POP_FLAG (16384) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -935,6 +936,7 @@ enum InstructionFormat { #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) #define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) +#define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -954,17 +956,17 @@ struct opcode_metadata { extern const struct opcode_metadata _PyOpcode_opcode_metadata[268]; #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { - [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BEFORE_ASYNC_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [BEFORE_WITH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -975,25 +977,25 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [BUILD_CONST_KEY_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1005,11 +1007,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1017,40 +1019,40 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, - [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, - [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1059,10 +1061,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1076,10 +1078,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, @@ -1089,20 +1091,20 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1117,15 +1119,15 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1133,7 +1135,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1147,12 +1149,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, @@ -1188,8 +1190,6 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { - [BEFORE_ASYNC_WITH] = { .nuops = 1, .uops = { { _BEFORE_ASYNC_WITH, 0, 0 } } }, - [BEFORE_WITH] = { .nuops = 1, .uops = { { _BEFORE_WITH, 0, 0 } } }, [BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, 0, 0 } } }, [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, [BINARY_OP_ADD_INT] = { .nuops = 2, .uops = { { _GUARD_BOTH_INT, 0, 0 }, { _BINARY_OP_ADD_INT, 0, 0 } } }, @@ -1207,7 +1207,6 @@ _PyOpcode_macro_expansion[256] = { [BUILD_CONST_KEY_MAP] = { .nuops = 1, .uops = { { _BUILD_CONST_KEY_MAP, 0, 0 } } }, [BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, 0, 0 } } }, [BUILD_MAP] = { .nuops = 1, .uops = { { _BUILD_MAP, 0, 0 } } }, - [BUILD_SET] = { .nuops = 1, .uops = { { _BUILD_SET, 0, 0 } } }, [BUILD_SLICE] = { .nuops = 1, .uops = { { _BUILD_SLICE, 0, 0 } } }, [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, @@ -1291,7 +1290,6 @@ _PyOpcode_macro_expansion[256] = { [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } }, - [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, 0, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, [MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, 0, 0 } } }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index fcead4d..44cafe6 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -12,7 +12,7 @@ extern "C" { #include <stdbool.h> // This is the length of the trace we project initially. -#define UOP_MAX_TRACE_LENGTH 512 +#define UOP_MAX_TRACE_LENGTH 800 #define TRACE_STACK_SIZE 5 diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index b569b80..bcb10ab 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -87,45 +87,47 @@ extern "C" { #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR +#define _DEOPT 341 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE #define _END_SEND END_SEND +#define _ERROR_POP_N 342 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _FATAL_ERROR 341 +#define _FATAL_ERROR 343 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 342 +#define _FOR_ITER 344 #define _FOR_ITER_GEN FOR_ITER_GEN -#define _FOR_ITER_TIER_TWO 343 +#define _FOR_ITER_TIER_TWO 345 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 344 -#define _GUARD_BOTH_INT 345 -#define _GUARD_BOTH_UNICODE 346 -#define _GUARD_BUILTINS_VERSION 347 -#define _GUARD_DORV_VALUES 348 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 349 -#define _GUARD_GLOBALS_VERSION 350 -#define _GUARD_IS_FALSE_POP 351 -#define _GUARD_IS_NONE_POP 352 -#define _GUARD_IS_NOT_NONE_POP 353 -#define _GUARD_IS_TRUE_POP 354 -#define _GUARD_KEYS_VERSION 355 -#define _GUARD_NOT_EXHAUSTED_LIST 356 -#define _GUARD_NOT_EXHAUSTED_RANGE 357 -#define _GUARD_NOT_EXHAUSTED_TUPLE 358 -#define _GUARD_TYPE_VERSION 359 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 360 -#define _INIT_CALL_PY_EXACT_ARGS 361 -#define _INIT_CALL_PY_EXACT_ARGS_0 362 -#define _INIT_CALL_PY_EXACT_ARGS_1 363 -#define _INIT_CALL_PY_EXACT_ARGS_2 364 -#define _INIT_CALL_PY_EXACT_ARGS_3 365 -#define _INIT_CALL_PY_EXACT_ARGS_4 366 +#define _GUARD_BOTH_FLOAT 346 +#define _GUARD_BOTH_INT 347 +#define _GUARD_BOTH_UNICODE 348 +#define _GUARD_BUILTINS_VERSION 349 +#define _GUARD_DORV_VALUES 350 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 351 +#define _GUARD_GLOBALS_VERSION 352 +#define _GUARD_IS_FALSE_POP 353 +#define _GUARD_IS_NONE_POP 354 +#define _GUARD_IS_NOT_NONE_POP 355 +#define _GUARD_IS_TRUE_POP 356 +#define _GUARD_KEYS_VERSION 357 +#define _GUARD_NOT_EXHAUSTED_LIST 358 +#define _GUARD_NOT_EXHAUSTED_RANGE 359 +#define _GUARD_NOT_EXHAUSTED_TUPLE 360 +#define _GUARD_TYPE_VERSION 361 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 362 +#define _INIT_CALL_PY_EXACT_ARGS 363 +#define _INIT_CALL_PY_EXACT_ARGS_0 364 +#define _INIT_CALL_PY_EXACT_ARGS_1 365 +#define _INIT_CALL_PY_EXACT_ARGS_2 366 +#define _INIT_CALL_PY_EXACT_ARGS_3 367 +#define _INIT_CALL_PY_EXACT_ARGS_4 368 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -142,65 +144,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 367 -#define _IS_NONE 368 +#define _INTERNAL_INCREMENT_OPT_COUNTER 369 +#define _IS_NONE 370 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 369 -#define _ITER_CHECK_RANGE 370 -#define _ITER_CHECK_TUPLE 371 -#define _ITER_JUMP_LIST 372 -#define _ITER_JUMP_RANGE 373 -#define _ITER_JUMP_TUPLE 374 -#define _ITER_NEXT_LIST 375 -#define _ITER_NEXT_RANGE 376 -#define _ITER_NEXT_TUPLE 377 -#define _JUMP_TO_TOP 378 +#define _ITER_CHECK_LIST 371 +#define _ITER_CHECK_RANGE 372 +#define _ITER_CHECK_TUPLE 373 +#define _ITER_JUMP_LIST 374 +#define _ITER_JUMP_RANGE 375 +#define _ITER_JUMP_TUPLE 376 +#define _ITER_NEXT_LIST 377 +#define _ITER_NEXT_RANGE 378 +#define _ITER_NEXT_TUPLE 379 +#define _JUMP_TO_TOP 380 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 379 -#define _LOAD_ATTR_CLASS 380 -#define _LOAD_ATTR_CLASS_0 381 -#define _LOAD_ATTR_CLASS_1 382 +#define _LOAD_ATTR 381 +#define _LOAD_ATTR_CLASS 382 +#define _LOAD_ATTR_CLASS_0 383 +#define _LOAD_ATTR_CLASS_1 384 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 383 -#define _LOAD_ATTR_INSTANCE_VALUE_0 384 -#define _LOAD_ATTR_INSTANCE_VALUE_1 385 -#define _LOAD_ATTR_METHOD_LAZY_DICT 386 -#define _LOAD_ATTR_METHOD_NO_DICT 387 -#define _LOAD_ATTR_METHOD_WITH_VALUES 388 -#define _LOAD_ATTR_MODULE 389 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 390 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 391 +#define _LOAD_ATTR_INSTANCE_VALUE 385 +#define _LOAD_ATTR_INSTANCE_VALUE_0 386 +#define _LOAD_ATTR_INSTANCE_VALUE_1 387 +#define _LOAD_ATTR_METHOD_LAZY_DICT 388 +#define _LOAD_ATTR_METHOD_NO_DICT 389 +#define _LOAD_ATTR_METHOD_WITH_VALUES 390 +#define _LOAD_ATTR_MODULE 391 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 392 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 393 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 392 -#define _LOAD_ATTR_SLOT_0 393 -#define _LOAD_ATTR_SLOT_1 394 -#define _LOAD_ATTR_WITH_HINT 395 +#define _LOAD_ATTR_SLOT 394 +#define _LOAD_ATTR_SLOT_0 395 +#define _LOAD_ATTR_SLOT_1 396 +#define _LOAD_ATTR_WITH_HINT 397 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 396 -#define _LOAD_CONST_INLINE_BORROW 397 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 398 -#define _LOAD_CONST_INLINE_WITH_NULL 399 +#define _LOAD_CONST_INLINE 398 +#define _LOAD_CONST_INLINE_BORROW 399 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 400 +#define _LOAD_CONST_INLINE_WITH_NULL 401 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 400 -#define _LOAD_FAST_0 401 -#define _LOAD_FAST_1 402 -#define _LOAD_FAST_2 403 -#define _LOAD_FAST_3 404 -#define _LOAD_FAST_4 405 -#define _LOAD_FAST_5 406 -#define _LOAD_FAST_6 407 -#define _LOAD_FAST_7 408 +#define _LOAD_FAST 402 +#define _LOAD_FAST_0 403 +#define _LOAD_FAST_1 404 +#define _LOAD_FAST_2 405 +#define _LOAD_FAST_3 406 +#define _LOAD_FAST_4 407 +#define _LOAD_FAST_5 408 +#define _LOAD_FAST_6 409 +#define _LOAD_FAST_7 410 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 409 -#define _LOAD_GLOBAL_BUILTINS 410 -#define _LOAD_GLOBAL_MODULE 411 +#define _LOAD_GLOBAL 411 +#define _LOAD_GLOBAL_BUILTINS 412 +#define _LOAD_GLOBAL_MODULE 413 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -214,48 +216,49 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 412 -#define _POP_JUMP_IF_FALSE 413 -#define _POP_JUMP_IF_TRUE 414 +#define _POP_FRAME 414 +#define _POP_JUMP_IF_FALSE 415 +#define _POP_JUMP_IF_TRUE 416 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 415 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 417 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 416 +#define _PUSH_FRAME 418 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 417 +#define _REPLACE_WITH_TRUE 419 #define _RESUME_CHECK RESUME_CHECK -#define _SAVE_RETURN_OFFSET 418 -#define _SEND 419 +#define _SAVE_RETURN_OFFSET 420 +#define _SEND 421 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 420 -#define _STORE_ATTR 421 -#define _STORE_ATTR_INSTANCE_VALUE 422 -#define _STORE_ATTR_SLOT 423 +#define _SIDE_EXIT 422 +#define _START_EXECUTOR 423 +#define _STORE_ATTR 424 +#define _STORE_ATTR_INSTANCE_VALUE 425 +#define _STORE_ATTR_SLOT 426 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 424 -#define _STORE_FAST_0 425 -#define _STORE_FAST_1 426 -#define _STORE_FAST_2 427 -#define _STORE_FAST_3 428 -#define _STORE_FAST_4 429 -#define _STORE_FAST_5 430 -#define _STORE_FAST_6 431 -#define _STORE_FAST_7 432 +#define _STORE_FAST 427 +#define _STORE_FAST_0 428 +#define _STORE_FAST_1 429 +#define _STORE_FAST_2 430 +#define _STORE_FAST_3 431 +#define _STORE_FAST_4 432 +#define _STORE_FAST_5 433 +#define _STORE_FAST_6 434 +#define _STORE_FAST_7 435 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 433 +#define _STORE_SUBSCR 436 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 434 +#define _TO_BOOL 437 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -265,12 +268,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 435 +#define _UNPACK_SEQUENCE 438 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 435 +#define MAX_UOP_ID 438 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 507bd27..51206cd 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -15,11 +15,13 @@ extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1]; extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; +extern int _PyUop_num_popped(int opcode, int oparg); + #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, - [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, + [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, @@ -49,22 +51,22 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_TO_BOOL_BOOL] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, - [_TO_BOOL_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_TO_BOOL_LIST] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_TO_BOOL_NONE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_TO_BOOL_STR] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_TO_BOOL_BOOL] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_TO_BOOL_INT] = HAS_EXIT_FLAG, + [_TO_BOOL_LIST] = HAS_EXIT_FLAG, + [_TO_BOOL_NONE] = HAS_EXIT_FLAG, + [_TO_BOOL_STR] = HAS_EXIT_FLAG, [_REPLACE_WITH_TRUE] = 0, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_INT] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_GUARD_BOTH_FLOAT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_FLOAT] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_MULTIPLY_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_PURE_FLAG, - [_GUARD_BOTH_UNICODE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_BOTH_UNICODE] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -83,13 +85,13 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_POP_FRAME] = HAS_ESCAPES_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_ASSERTION_ERROR] = 0, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, @@ -98,19 +100,18 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FROM_DICT_OR_GLOBALS] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_GLOBALS_VERSION] = HAS_DEOPT_FLAG, [_GUARD_BUILTINS_VERSION] = HAS_DEOPT_FLAG, [_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, - [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, [_COPY_FREE_VARS] = HAS_ARG_FLAG, @@ -119,7 +120,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SETUP_ANNOTATIONS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_CONST_KEY_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -129,7 +129,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_TYPE_VERSION] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, + [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_LOAD_ATTR_INSTANCE_VALUE_0] = HAS_DEOPT_FLAG, [_LOAD_ATTR_INSTANCE_VALUE_1] = HAS_DEOPT_FLAG, @@ -165,8 +165,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MATCH_SEQUENCE] = 0, [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_FOR_ITER_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_ITER_NEXT_LIST] = 0, @@ -176,8 +176,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ITER_CHECK_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, - [_BEFORE_ASYNC_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BEFORE_WITH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG, @@ -204,18 +202,18 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG, [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_LEN] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, @@ -224,14 +222,14 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_GUARD_IS_TRUE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_GUARD_IS_FALSE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_GUARD_IS_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_GUARD_IS_NOT_NONE_POP] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, + [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG, [_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG, [_SET_IP] = 0, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, - [_EXIT_TRACE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_EXIT_TRACE] = HAS_EXIT_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, @@ -240,10 +238,13 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, - [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_START_EXECUTOR] = 0, [_FATAL_ERROR] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, + [_DEOPT] = 0, + [_SIDE_EXIT] = 0, + [_ERROR_POP_N] = HAS_ARG_FLAG, }; const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { @@ -253,8 +254,6 @@ const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { }; const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { - [_BEFORE_ASYNC_WITH] = "_BEFORE_ASYNC_WITH", - [_BEFORE_WITH] = "_BEFORE_WITH", [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", @@ -272,7 +271,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BUILD_CONST_KEY_MAP] = "_BUILD_CONST_KEY_MAP", [_BUILD_LIST] = "_BUILD_LIST", [_BUILD_MAP] = "_BUILD_MAP", - [_BUILD_SET] = "_BUILD_SET", [_BUILD_SLICE] = "_BUILD_SLICE", [_BUILD_STRING] = "_BUILD_STRING", [_BUILD_TUPLE] = "_BUILD_TUPLE", @@ -323,9 +321,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DELETE_GLOBAL] = "_DELETE_GLOBAL", [_DELETE_NAME] = "_DELETE_NAME", [_DELETE_SUBSCR] = "_DELETE_SUBSCR", + [_DEOPT] = "_DEOPT", [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", [_END_SEND] = "_END_SEND", + [_ERROR_POP_N] = "_ERROR_POP_N", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", [_FATAL_ERROR] = "_FATAL_ERROR", @@ -416,7 +416,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", [_LOAD_LOCALS] = "_LOAD_LOCALS", - [_LOAD_NAME] = "_LOAD_NAME", [_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR", [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", [_MAKE_CELL] = "_MAKE_CELL", @@ -442,6 +441,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", [_SET_IP] = "_SET_IP", [_SET_UPDATE] = "_SET_UPDATE", + [_SIDE_EXIT] = "_SIDE_EXIT", [_START_EXECUTOR] = "_START_EXECUTOR", [_STORE_ATTR] = "_STORE_ATTR", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", @@ -481,6 +481,466 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", }; +int _PyUop_num_popped(int opcode, int oparg) +{ + switch(opcode) { + case _NOP: + return 0; + case _RESUME_CHECK: + return 0; + case _LOAD_FAST_CHECK: + return 0; + case _LOAD_FAST_0: + return 0; + case _LOAD_FAST_1: + return 0; + case _LOAD_FAST_2: + return 0; + case _LOAD_FAST_3: + return 0; + case _LOAD_FAST_4: + return 0; + case _LOAD_FAST_5: + return 0; + case _LOAD_FAST_6: + return 0; + case _LOAD_FAST_7: + return 0; + case _LOAD_FAST: + return 0; + case _LOAD_FAST_AND_CLEAR: + return 0; + case _LOAD_FAST_LOAD_FAST: + return 0; + case _LOAD_CONST: + return 0; + case _STORE_FAST_0: + return 1; + case _STORE_FAST_1: + return 1; + case _STORE_FAST_2: + return 1; + case _STORE_FAST_3: + return 1; + case _STORE_FAST_4: + return 1; + case _STORE_FAST_5: + return 1; + case _STORE_FAST_6: + return 1; + case _STORE_FAST_7: + return 1; + case _STORE_FAST: + return 1; + case _STORE_FAST_LOAD_FAST: + return 1; + case _STORE_FAST_STORE_FAST: + return 2; + case _POP_TOP: + return 1; + case _PUSH_NULL: + return 0; + case _END_SEND: + return 2; + case _UNARY_NEGATIVE: + return 1; + case _UNARY_NOT: + return 1; + case _TO_BOOL: + return 1; + case _TO_BOOL_BOOL: + return 1; + case _TO_BOOL_INT: + return 1; + case _TO_BOOL_LIST: + return 1; + case _TO_BOOL_NONE: + return 1; + case _TO_BOOL_STR: + return 1; + case _REPLACE_WITH_TRUE: + return 1; + case _UNARY_INVERT: + return 1; + case _GUARD_BOTH_INT: + return 2; + case _BINARY_OP_MULTIPLY_INT: + return 2; + case _BINARY_OP_ADD_INT: + return 2; + case _BINARY_OP_SUBTRACT_INT: + return 2; + case _GUARD_BOTH_FLOAT: + return 2; + case _BINARY_OP_MULTIPLY_FLOAT: + return 2; + case _BINARY_OP_ADD_FLOAT: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT: + return 2; + case _GUARD_BOTH_UNICODE: + return 2; + case _BINARY_OP_ADD_UNICODE: + return 2; + case _BINARY_SUBSCR: + return 2; + case _BINARY_SLICE: + return 3; + case _STORE_SLICE: + return 4; + case _BINARY_SUBSCR_LIST_INT: + return 2; + case _BINARY_SUBSCR_STR_INT: + return 2; + case _BINARY_SUBSCR_TUPLE_INT: + return 2; + case _BINARY_SUBSCR_DICT: + return 2; + case _LIST_APPEND: + return 2 + (oparg-1); + case _SET_ADD: + return 2 + (oparg-1); + case _STORE_SUBSCR: + return 3; + case _STORE_SUBSCR_LIST_INT: + return 3; + case _STORE_SUBSCR_DICT: + return 3; + case _DELETE_SUBSCR: + return 2; + case _CALL_INTRINSIC_1: + return 1; + case _CALL_INTRINSIC_2: + return 2; + case _POP_FRAME: + return 1; + case _GET_AITER: + return 1; + case _GET_ANEXT: + return 1; + case _GET_AWAITABLE: + return 1; + case _POP_EXCEPT: + return 1; + case _LOAD_ASSERTION_ERROR: + return 0; + case _LOAD_BUILD_CLASS: + return 0; + case _STORE_NAME: + return 1; + case _DELETE_NAME: + return 0; + case _UNPACK_SEQUENCE: + return 1; + case _UNPACK_SEQUENCE_TWO_TUPLE: + return 1; + case _UNPACK_SEQUENCE_TUPLE: + return 1; + case _UNPACK_SEQUENCE_LIST: + return 1; + case _UNPACK_EX: + return 1; + case _STORE_ATTR: + return 2; + case _DELETE_ATTR: + return 1; + case _STORE_GLOBAL: + return 1; + case _DELETE_GLOBAL: + return 0; + case _LOAD_LOCALS: + return 0; + case _LOAD_FROM_DICT_OR_GLOBALS: + return 1; + case _LOAD_GLOBAL: + return 0; + case _GUARD_GLOBALS_VERSION: + return 0; + case _GUARD_BUILTINS_VERSION: + return 0; + case _LOAD_GLOBAL_MODULE: + return 0; + case _LOAD_GLOBAL_BUILTINS: + return 0; + case _DELETE_FAST: + return 0; + case _MAKE_CELL: + return 0; + case _DELETE_DEREF: + return 0; + case _LOAD_FROM_DICT_OR_DEREF: + return 1; + case _LOAD_DEREF: + return 0; + case _STORE_DEREF: + return 1; + case _COPY_FREE_VARS: + return 0; + case _BUILD_STRING: + return oparg; + case _BUILD_TUPLE: + return oparg; + case _BUILD_LIST: + return oparg; + case _LIST_EXTEND: + return 2 + (oparg-1); + case _SET_UPDATE: + return 2 + (oparg-1); + case _BUILD_MAP: + return oparg*2; + case _SETUP_ANNOTATIONS: + return 0; + case _BUILD_CONST_KEY_MAP: + return 1 + oparg; + case _DICT_UPDATE: + return 2 + (oparg - 1); + case _DICT_MERGE: + return 5 + (oparg - 1); + case _MAP_ADD: + return 3 + (oparg - 1); + case _LOAD_SUPER_ATTR_ATTR: + return 3; + case _LOAD_SUPER_ATTR_METHOD: + return 3; + case _LOAD_ATTR: + return 1; + case _GUARD_TYPE_VERSION: + return 1; + case _CHECK_MANAGED_OBJECT_HAS_VALUES: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE_0: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE_1: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE: + return 1; + case _CHECK_ATTR_MODULE: + return 1; + case _LOAD_ATTR_MODULE: + return 1; + case _CHECK_ATTR_WITH_HINT: + return 1; + case _LOAD_ATTR_WITH_HINT: + return 1; + case _LOAD_ATTR_SLOT_0: + return 1; + case _LOAD_ATTR_SLOT_1: + return 1; + case _LOAD_ATTR_SLOT: + return 1; + case _CHECK_ATTR_CLASS: + return 1; + case _LOAD_ATTR_CLASS_0: + return 1; + case _LOAD_ATTR_CLASS_1: + return 1; + case _LOAD_ATTR_CLASS: + return 1; + case _GUARD_DORV_VALUES: + return 1; + case _STORE_ATTR_INSTANCE_VALUE: + return 2; + case _STORE_ATTR_SLOT: + return 2; + case _COMPARE_OP: + return 2; + case _COMPARE_OP_FLOAT: + return 2; + case _COMPARE_OP_INT: + return 2; + case _COMPARE_OP_STR: + return 2; + case _IS_OP: + return 2; + case _CONTAINS_OP: + return 2; + case _CONTAINS_OP_SET: + return 2; + case _CONTAINS_OP_DICT: + return 2; + case _CHECK_EG_MATCH: + return 2; + case _CHECK_EXC_MATCH: + return 2; + case _IS_NONE: + return 1; + case _GET_LEN: + return 1; + case _MATCH_CLASS: + return 3; + case _MATCH_MAPPING: + return 1; + case _MATCH_SEQUENCE: + return 1; + case _MATCH_KEYS: + return 2; + case _GET_ITER: + return 1; + case _GET_YIELD_FROM_ITER: + return 1; + case _FOR_ITER_TIER_TWO: + return 1; + case _ITER_CHECK_LIST: + return 1; + case _GUARD_NOT_EXHAUSTED_LIST: + return 1; + case _ITER_NEXT_LIST: + return 1; + case _ITER_CHECK_TUPLE: + return 1; + case _GUARD_NOT_EXHAUSTED_TUPLE: + return 1; + case _ITER_NEXT_TUPLE: + return 1; + case _ITER_CHECK_RANGE: + return 1; + case _GUARD_NOT_EXHAUSTED_RANGE: + return 1; + case _ITER_NEXT_RANGE: + return 1; + case _WITH_EXCEPT_START: + return 4; + case _PUSH_EXC_INFO: + return 1; + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: + return 1; + case _GUARD_KEYS_VERSION: + return 1; + case _LOAD_ATTR_METHOD_WITH_VALUES: + return 1; + case _LOAD_ATTR_METHOD_NO_DICT: + return 1; + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: + return 1; + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: + return 1; + case _CHECK_ATTR_METHOD_LAZY_DICT: + return 1; + case _LOAD_ATTR_METHOD_LAZY_DICT: + return 1; + case _CHECK_PERIODIC: + return 0; + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: + return 2 + oparg; + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: + return 2 + oparg; + case _CHECK_PEP_523: + return 0; + case _CHECK_FUNCTION_EXACT_ARGS: + return 2 + oparg; + case _CHECK_STACK_SPACE: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_0: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_1: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_2: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_3: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS_4: + return 2 + oparg; + case _INIT_CALL_PY_EXACT_ARGS: + return 2 + oparg; + case _PUSH_FRAME: + return 1; + case _CALL_TYPE_1: + return 3; + case _CALL_STR_1: + return 3; + case _CALL_TUPLE_1: + return 3; + case _EXIT_INIT_CHECK: + return 1; + case _CALL_BUILTIN_CLASS: + return 2 + oparg; + case _CALL_BUILTIN_O: + return 2 + oparg; + case _CALL_BUILTIN_FAST: + return 2 + oparg; + case _CALL_BUILTIN_FAST_WITH_KEYWORDS: + return 2 + oparg; + case _CALL_LEN: + return 2 + oparg; + case _CALL_ISINSTANCE: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_O: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_NOARGS: + return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_FAST: + return 2 + oparg; + case _MAKE_FUNCTION: + return 1; + case _SET_FUNCTION_ATTRIBUTE: + return 2; + case _BUILD_SLICE: + return 2 + ((oparg == 3) ? 1 : 0); + case _CONVERT_VALUE: + return 1; + case _FORMAT_SIMPLE: + return 1; + case _FORMAT_WITH_SPEC: + return 2; + case _COPY: + return 1 + (oparg-1); + case _BINARY_OP: + return 2; + case _SWAP: + return 2 + (oparg-2); + case _GUARD_IS_TRUE_POP: + return 1; + case _GUARD_IS_FALSE_POP: + return 1; + case _GUARD_IS_NONE_POP: + return 1; + case _GUARD_IS_NOT_NONE_POP: + return 1; + case _JUMP_TO_TOP: + return 0; + case _SET_IP: + return 0; + case _SAVE_RETURN_OFFSET: + return 0; + case _EXIT_TRACE: + return 0; + case _CHECK_VALIDITY: + return 0; + case _LOAD_CONST_INLINE: + return 0; + case _LOAD_CONST_INLINE_BORROW: + return 0; + case _POP_TOP_LOAD_CONST_INLINE_BORROW: + return 1; + case _LOAD_CONST_INLINE_WITH_NULL: + return 0; + case _LOAD_CONST_INLINE_BORROW_WITH_NULL: + return 0; + case _CHECK_FUNCTION: + return 0; + case _INTERNAL_INCREMENT_OPT_COUNTER: + return 1; + case _COLD_EXIT: + return 0; + case _START_EXECUTOR: + return 0; + case _FATAL_ERROR: + return 0; + case _CHECK_VALIDITY_AND_SET_IP: + return 0; + case _DEOPT: + return 0; + case _SIDE_EXIT: + return 0; + case _ERROR_POP_N: + return oparg; + default: + return -1; + } +} + #endif // NEED_OPCODE_METADATA diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 476975d..5cd9db9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -179,7 +179,7 @@ dummy_func( uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - GOTO_ERROR(error); + ERROR_NO_POP(); } next_instr = this_instr; } @@ -206,7 +206,13 @@ dummy_func( inst(LOAD_FAST_CHECK, (-- value)) { value = GETLOCAL(oparg); - ERROR_IF(value == NULL, unbound_local_error); + if (value == NULL) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + ERROR_IF(1, error); + } Py_INCREF(value); } @@ -275,7 +281,7 @@ dummy_func( if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); + ERROR_NO_POP(); } PyErr_SetRaisedException(NULL); } @@ -290,7 +296,7 @@ dummy_func( if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); + ERROR_NO_POP(); } PyErr_SetRaisedException(NULL); } @@ -826,7 +832,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); - if (err) GOTO_ERROR(error); + if (err) ERROR_NO_POP(); STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -850,7 +856,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); - if (err) GOTO_ERROR(error); + if (err) ERROR_NO_POP(); Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -906,7 +912,7 @@ dummy_func( if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { - GOTO_ERROR(error); + ERROR_NO_POP(); } } else { if (type->tp_as_async != NULL){ @@ -916,7 +922,7 @@ dummy_func( if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { - GOTO_ERROR(error); + ERROR_NO_POP(); } } else { @@ -924,7 +930,7 @@ dummy_func( "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); - GOTO_ERROR(error); + ERROR_NO_POP(); } awaitable = _PyCoro_GetAwaitableIter(next_iter); @@ -936,7 +942,7 @@ dummy_func( Py_TYPE(next_iter)->tp_name); Py_DECREF(next_iter); - GOTO_ERROR(error); + ERROR_NO_POP(); } else { Py_DECREF(next_iter); } @@ -1018,7 +1024,7 @@ dummy_func( JUMPBY(oparg); } else { - GOTO_ERROR(error); + ERROR_NO_POP(); } } Py_DECREF(v); @@ -1054,7 +1060,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, retval); - if (err) GOTO_ERROR(error); + if (err) ERROR_NO_POP(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -1108,7 +1114,7 @@ dummy_func( else { assert(PyLong_Check(lasti)); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - GOTO_ERROR(error); + ERROR_NO_POP(); } } assert(exc && PyExceptionInstance_Check(exc)); @@ -1184,7 +1190,7 @@ dummy_func( if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - GOTO_ERROR(error); + ERROR_NO_POP(); } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. @@ -1192,7 +1198,7 @@ dummy_func( _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + ERROR_NO_POP(); } } @@ -1312,12 +1318,12 @@ dummy_func( int err = PyDict_Pop(GLOBALS(), name, NULL); // Can't use ERROR_IF here. if (err < 0) { - GOTO_ERROR(error); + ERROR_NO_POP(); } if (err == 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + ERROR_NO_POP(); } } @@ -1334,21 +1340,21 @@ dummy_func( inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); + ERROR_NO_POP(); } if (v == NULL) { if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); + ERROR_NO_POP(); } if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); + ERROR_NO_POP(); } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + ERROR_NO_POP(); } } } @@ -1364,21 +1370,21 @@ dummy_func( } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); + ERROR_NO_POP(); } if (v == NULL) { if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); + ERROR_NO_POP(); } if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); + ERROR_NO_POP(); } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + ERROR_NO_POP(); } } } @@ -1494,7 +1500,13 @@ dummy_func( inst(DELETE_FAST, (--)) { PyObject *v = GETLOCAL(oparg); - ERROR_IF(v == NULL, unbound_local_error); + if (v == NULL) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + ERROR_IF(1, error); + } SETLOCAL(oparg, NULL); } @@ -1504,7 +1516,7 @@ dummy_func( PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - GOTO_ERROR(error); + ERROR_NO_POP(); } SETLOCAL(oparg, cell); } @@ -1516,7 +1528,7 @@ dummy_func( // Fortunately we don't need its superpower. if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + ERROR_NO_POP(); } PyCell_SET(cell, NULL); Py_DECREF(oldobj); @@ -1528,14 +1540,14 @@ dummy_func( assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - GOTO_ERROR(error); + ERROR_NO_POP(); } if (!value) { PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + ERROR_NO_POP(); } Py_INCREF(value); } @@ -1615,7 +1627,7 @@ dummy_func( inst(BUILD_SET, (values[oparg] -- set)) { set = PySet_New(NULL); if (set == NULL) - GOTO_ERROR(error); + ERROR_NO_POP(); int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; @@ -1662,12 +1674,8 @@ dummy_func( } inst(BUILD_CONST_KEY_MAP, (values[oparg], keys -- map)) { - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } + assert(PyTuple_CheckExact(keys)); + assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); @@ -2502,7 +2510,7 @@ dummy_func( _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - GOTO_ERROR(error); + ERROR_NO_POP(); } iter = iterable; } @@ -2513,7 +2521,7 @@ dummy_func( /* `iterable` is not a generator. */ iter = PyObject_GetIter(iterable); if (iter == NULL) { - GOTO_ERROR(error); + ERROR_NO_POP(); } DECREF_INPUTS(); } @@ -2550,7 +2558,7 @@ dummy_func( if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + ERROR_NO_POP(); } monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); @@ -2573,7 +2581,7 @@ dummy_func( if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + ERROR_NO_POP(); } _PyErr_Clear(tstate); } @@ -2599,7 +2607,7 @@ dummy_func( else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + ERROR_NO_POP(); } monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); @@ -2779,7 +2787,7 @@ dummy_func( "asynchronous context manager protocol", Py_TYPE(mgr)->tp_name); } - GOTO_ERROR(error); + ERROR_NO_POP(); } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); if (exit == NULL) { @@ -2791,7 +2799,7 @@ dummy_func( Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - GOTO_ERROR(error); + ERROR_NO_POP(); } DECREF_INPUTS(); res = PyObject_CallNoArgs(enter); @@ -2814,7 +2822,7 @@ dummy_func( "context manager protocol", Py_TYPE(mgr)->tp_name); } - GOTO_ERROR(error); + ERROR_NO_POP(); } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); if (exit == NULL) { @@ -2826,7 +2834,7 @@ dummy_func( Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - GOTO_ERROR(error); + ERROR_NO_POP(); } DECREF_INPUTS(); res = PyObject_CallNoArgs(enter); @@ -3075,7 +3083,7 @@ dummy_func( // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - GOTO_ERROR(error); + ERROR_NO_POP(); } frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); @@ -3298,7 +3306,7 @@ dummy_func( STAT_INC(CALL, hit); PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { - GOTO_ERROR(error); + ERROR_NO_POP(); } Py_DECREF(tp); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( @@ -3335,7 +3343,7 @@ dummy_func( PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(should_be_none)->tp_name); - GOTO_ERROR(error); + ERROR_NO_POP(); } } @@ -3472,7 +3480,7 @@ dummy_func( PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { - GOTO_ERROR(error); + ERROR_NO_POP(); } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3498,7 +3506,7 @@ dummy_func( PyObject *inst = args[0]; int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { - GOTO_ERROR(error); + ERROR_NO_POP(); } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3712,7 +3720,7 @@ dummy_func( // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - GOTO_ERROR(error); + ERROR_NO_POP(); } assert(next_instr - this_instr == 1); frame->return_offset = 1; @@ -3760,11 +3768,11 @@ dummy_func( assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { - GOTO_ERROR(error); + ERROR_NO_POP(); } PyObject *tuple = PySequence_Tuple(callargs); if (tuple == NULL) { - GOTO_ERROR(error); + ERROR_NO_POP(); } Py_SETREF(callargs, tuple); } @@ -3776,7 +3784,7 @@ dummy_func( int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - if (err) GOTO_ERROR(error); + if (err) ERROR_NO_POP(); result = PyObject_Call(func, callargs, kwargs); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { @@ -3810,7 +3818,7 @@ dummy_func( // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { - GOTO_ERROR(error); + ERROR_NO_POP(); } assert(next_instr - this_instr == 1); frame->return_offset = 1; @@ -3831,7 +3839,7 @@ dummy_func( Py_DECREF(codeobj); if (func_obj == NULL) { - GOTO_ERROR(error); + ERROR_NO_POP(); } _PyFunction_SetVersion( @@ -3871,7 +3879,7 @@ dummy_func( PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); if (gen == NULL) { - GOTO_ERROR(error); + ERROR_NO_POP(); } assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4169,7 +4177,7 @@ dummy_func( if (optimized < 0) { Py_DECREF(previous); tstate->previous_executor = Py_None; - ERROR_IF(1, error); + GOTO_UNWIND(); } GOTO_TIER_ONE(target); } @@ -4199,6 +4207,19 @@ dummy_func( frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; } + tier2 op(_DEOPT, (--)) { + EXIT_TO_TIER1(); + } + + tier2 op(_SIDE_EXIT, (--)) { + EXIT_TO_TRACE(); + } + + tier2 op(_ERROR_POP_N, (unused[oparg] --)) { + SYNC_SP(); + GOTO_UNWIND(); + } + // END BYTECODES // } diff --git a/Python/ceval.c b/Python/ceval.c index b35a321..cd51011 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -642,7 +642,6 @@ int _Py_CheckRecursiveCallPy( return 0; } - static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { /* Put a NOP at the start, so that the IP points into * the code, rather than before it */ @@ -850,15 +849,6 @@ resume_frame: or goto error. */ Py_UNREACHABLE(); -unbound_local_error: - { - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - goto error; - } - pop_4_error: STACK_SHRINK(1); pop_3_error: @@ -980,12 +970,6 @@ enter_tier_two: #undef GOTO_ERROR #define GOTO_ERROR(LABEL) goto LABEL ## _tier_two -#undef DEOPT_IF -#define DEOPT_IF(COND, INSTNAME) \ - if ((COND)) { \ - goto deoptimize;\ - } - #ifdef Py_STATS // Disable these macros that apply to Tier 1 stats when we are in Tier 2 #undef STAT_INC @@ -1013,6 +997,7 @@ enter_tier_two: #endif assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); +tier2_dispatch: for (;;) { uopcode = next_uop->opcode; #ifdef Py_DEBUG @@ -1054,24 +1039,7 @@ enter_tier_two: } } -// Jump here from ERROR_IF(..., unbound_local_error) -unbound_local_error_tier_two: - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - goto error_tier_two; - -// JUMP to any of these from ERROR_IF(..., error) -pop_4_error_tier_two: - STACK_SHRINK(1); -pop_3_error_tier_two: - STACK_SHRINK(1); -pop_2_error_tier_two: - STACK_SHRINK(1); -pop_1_error_tier_two: - STACK_SHRINK(1); -error_tier_two: +jump_to_error_target: #ifdef Py_DEBUG if (lltrace >= 2) { printf("Error: [UOp "); @@ -1081,15 +1049,28 @@ error_tier_two: _PyOpcode_OpName[frame->instr_ptr->op.code]); } #endif + assert (next_uop[-1].format == UOP_FORMAT_JUMP); + uint16_t target = uop_get_error_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + +error_tier_two: OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + assert(next_uop[-1].format == UOP_FORMAT_TARGET); frame->return_offset = 0; // Don't leave this random _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(current_executor); tstate->previous_executor = NULL; goto resume_with_error; -// Jump here from DEOPT_IF() -deoptimize: +jump_to_jump_target: + assert(next_uop[-1].format == UOP_FORMAT_JUMP); + target = uop_get_jump_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + +exit_to_tier1: + assert(next_uop[-1].format == UOP_FORMAT_TARGET); next_instr = next_uop[-1].target + _PyCode_CODE(_PyFrame_GetCode(frame)); #ifdef Py_DEBUG if (lltrace >= 2) { @@ -1105,8 +1086,8 @@ deoptimize: tstate->previous_executor = NULL; DISPATCH(); -// Jump here from EXIT_IF() -side_exit: +exit_to_trace: + assert(next_uop[-1].format == UOP_FORMAT_EXIT); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); UOP_STAT_INC(uopcode, miss); uint32_t exit_index = next_uop[-1].exit_index; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 22992aa..f2536ed 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -423,3 +423,9 @@ do { \ #define CURRENT_OPARG() (next_uop[-1].oparg) #define CURRENT_OPERAND() (next_uop[-1].operand) + +#define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target +#define JUMP_TO_ERROR() goto jump_to_error_target +#define GOTO_UNWIND() goto error_tier_two +#define EXIT_TO_TRACE() goto exit_to_trace +#define EXIT_TO_TIER1() goto exit_to_tier1 diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a55daa2..224b600 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -14,23 +14,29 @@ case _RESUME_CHECK: { #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) goto deoptimize; + if (_Py_emscripten_signal_clock == 0) JUMP_TO_JUMP_TARGET(); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) goto deoptimize; + if (eval_breaker != version) JUMP_TO_JUMP_TARGET(); break; } - /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_RESUME is not a viable micro-op for tier 2 because it is instrumented */ case _LOAD_FAST_CHECK: { PyObject *value; oparg = CURRENT_OPARG(); value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error_tier_two; + if (value == NULL) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + if (1) JUMP_TO_ERROR(); + } Py_INCREF(value); stack_pointer[0] = value; stack_pointer += 1; @@ -287,7 +293,7 @@ value = stack_pointer[-1]; res = PyNumber_Negative(value); Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = res; break; } @@ -308,7 +314,7 @@ value = stack_pointer[-1]; int err = PyObject_IsTrue(value); Py_DECREF(value); - if (err < 0) goto pop_1_error_tier_two; + if (err < 0) JUMP_TO_ERROR(); res = err ? Py_True : Py_False; stack_pointer[-1] = res; break; @@ -317,7 +323,7 @@ case _TO_BOOL_BOOL: { PyObject *value; value = stack_pointer[-1]; - if (!PyBool_Check(value)) goto side_exit; + if (!PyBool_Check(value)) JUMP_TO_JUMP_TARGET(); STAT_INC(TO_BOOL, hit); break; } @@ -326,7 +332,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyLong_CheckExact(value)) goto side_exit; + if (!PyLong_CheckExact(value)) JUMP_TO_JUMP_TARGET(); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -344,7 +350,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyList_CheckExact(value)) goto side_exit; + if (!PyList_CheckExact(value)) JUMP_TO_JUMP_TARGET(); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); @@ -357,7 +363,7 @@ PyObject *res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - if (!Py_IsNone(value)) goto side_exit; + if (!Py_IsNone(value)) JUMP_TO_JUMP_TARGET(); STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; @@ -368,7 +374,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyUnicode_CheckExact(value)) goto side_exit; + if (!PyUnicode_CheckExact(value)) JUMP_TO_JUMP_TARGET(); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -399,7 +405,7 @@ value = stack_pointer[-1]; res = PyNumber_Invert(value); Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = res; break; } @@ -409,8 +415,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyLong_CheckExact(left)) goto side_exit; - if (!PyLong_CheckExact(right)) goto side_exit; + if (!PyLong_CheckExact(left)) JUMP_TO_JUMP_TARGET(); + if (!PyLong_CheckExact(right)) JUMP_TO_JUMP_TARGET(); break; } @@ -424,7 +430,7 @@ res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -440,7 +446,7 @@ res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -456,7 +462,7 @@ res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -467,8 +473,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyFloat_CheckExact(left)) goto side_exit; - if (!PyFloat_CheckExact(right)) goto side_exit; + if (!PyFloat_CheckExact(left)) JUMP_TO_JUMP_TARGET(); + if (!PyFloat_CheckExact(right)) JUMP_TO_JUMP_TARGET(); break; } @@ -525,8 +531,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyUnicode_CheckExact(left)) goto side_exit; - if (!PyUnicode_CheckExact(right)) goto side_exit; + if (!PyUnicode_CheckExact(left)) JUMP_TO_JUMP_TARGET(); + if (!PyUnicode_CheckExact(right)) JUMP_TO_JUMP_TARGET(); break; } @@ -540,7 +546,7 @@ res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -555,7 +561,7 @@ res = PyObject_GetItem(container, sub); Py_DECREF(container); Py_DECREF(sub); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -580,7 +586,7 @@ Py_DECREF(slice); } Py_DECREF(container); - if (res == NULL) goto pop_3_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = res; stack_pointer += -2; break; @@ -606,7 +612,7 @@ } Py_DECREF(v); Py_DECREF(container); - if (err) goto pop_4_error_tier_two; + if (err) JUMP_TO_ERROR(); stack_pointer += -4; break; } @@ -617,12 +623,12 @@ PyObject *res; sub = stack_pointer[-1]; list = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) goto deoptimize; - if (!PyList_CheckExact(list)) goto deoptimize; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); + if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET(); // Deopt unless 0 <= sub < PyList_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyList_GET_SIZE(list)) goto deoptimize; + if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET(); STAT_INC(BINARY_SUBSCR, hit); res = PyList_GET_ITEM(list, index); assert(res != NULL); @@ -640,14 +646,14 @@ PyObject *res; sub = stack_pointer[-1]; str = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) goto deoptimize; - if (!PyUnicode_CheckExact(str)) goto deoptimize; - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); + if (!PyUnicode_CheckExact(str)) JUMP_TO_JUMP_TARGET(); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) goto deoptimize; + if (PyUnicode_GET_LENGTH(str) <= index) JUMP_TO_JUMP_TARGET(); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) goto deoptimize; + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) JUMP_TO_JUMP_TARGET(); STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); @@ -663,12 +669,12 @@ PyObject *res; sub = stack_pointer[-1]; tuple = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) goto deoptimize; - if (!PyTuple_CheckExact(tuple)) goto deoptimize; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); + if (!PyTuple_CheckExact(tuple)) JUMP_TO_JUMP_TARGET(); // Deopt unless 0 <= sub < PyTuple_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) goto deoptimize; + if (index >= PyTuple_GET_SIZE(tuple)) JUMP_TO_JUMP_TARGET(); STAT_INC(BINARY_SUBSCR, hit); res = PyTuple_GET_ITEM(tuple, index); assert(res != NULL); @@ -686,7 +692,7 @@ PyObject *res; sub = stack_pointer[-1]; dict = stack_pointer[-2]; - if (!PyDict_CheckExact(dict)) goto deoptimize; + if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); STAT_INC(BINARY_SUBSCR, hit); int rc = PyDict_GetItemRef(dict, sub, &res); if (rc == 0) { @@ -694,14 +700,14 @@ } Py_DECREF(dict); Py_DECREF(sub); - if (rc <= 0) goto pop_2_error_tier_two; + if (rc <= 0) JUMP_TO_ERROR(); // not found or error stack_pointer[-2] = res; stack_pointer += -1; break; } - /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 */ + /* _BINARY_SUBSCR_GETITEM is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _LIST_APPEND: { PyObject *v; @@ -709,7 +715,7 @@ oparg = CURRENT_OPARG(); v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; - if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error_tier_two; + if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) JUMP_TO_ERROR(); stack_pointer += -1; break; } @@ -722,7 +728,7 @@ set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(set, v); Py_DECREF(v); - if (err) goto pop_1_error_tier_two; + if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; } @@ -739,7 +745,7 @@ Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - if (err) goto pop_3_error_tier_two; + if (err) JUMP_TO_ERROR(); stack_pointer += -3; break; } @@ -751,13 +757,13 @@ sub = stack_pointer[-1]; list = stack_pointer[-2]; value = stack_pointer[-3]; - if (!PyLong_CheckExact(sub)) goto deoptimize; - if (!PyList_CheckExact(list)) goto deoptimize; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); + if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET(); // Ensure nonnegative, zero-or-one-digit ints. - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) goto deoptimize; + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) goto deoptimize; + if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET(); STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); @@ -776,11 +782,11 @@ sub = stack_pointer[-1]; dict = stack_pointer[-2]; value = stack_pointer[-3]; - if (!PyDict_CheckExact(dict)) goto deoptimize; + if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); - if (err) goto pop_3_error_tier_two; + if (err) JUMP_TO_ERROR(); stack_pointer += -3; break; } @@ -794,7 +800,7 @@ int err = PyObject_DelItem(container, sub); Py_DECREF(container); Py_DECREF(sub); - if (err) goto pop_2_error_tier_two; + if (err) JUMP_TO_ERROR(); stack_pointer += -2; break; } @@ -807,7 +813,7 @@ assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = res; break; } @@ -823,7 +829,7 @@ res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); Py_DECREF(value2); Py_DECREF(value1); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -855,9 +861,9 @@ break; } - /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_RETURN_VALUE is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 because it is instrumented */ case _GET_AITER: { PyObject *obj; @@ -874,11 +880,11 @@ "__aiter__ method, got %.100s", type->tp_name); Py_DECREF(obj); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR(); } iter = (*getter)(obj); Py_DECREF(obj); - if (iter == NULL) goto pop_1_error_tier_two; + if (iter == NULL) JUMP_TO_ERROR(); if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, @@ -886,7 +892,7 @@ "that does not implement __anext__: %.100s", Py_TYPE(iter)->tp_name); Py_DECREF(iter); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR(); } stack_pointer[-1] = iter; break; @@ -902,7 +908,7 @@ if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } } else { if (type->tp_as_async != NULL){ @@ -911,7 +917,7 @@ if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } } else { @@ -919,7 +925,7 @@ "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); - GOTO_ERROR(error); + JUMP_TO_ERROR(); } awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { @@ -929,7 +935,7 @@ "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); Py_DECREF(next_iter); - GOTO_ERROR(error); + JUMP_TO_ERROR(); } else { Py_DECREF(next_iter); } @@ -962,16 +968,16 @@ /* The code below jumps to `error` if `iter` is NULL. */ } } - if (iter == NULL) goto pop_1_error_tier_two; + if (iter == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = iter; break; } - /* _SEND is not a viable micro-op for tier 2 */ + /* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - /* _SEND_GEN is not a viable micro-op for tier 2 */ + /* _SEND_GEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 because it is instrumented */ case _POP_EXCEPT: { PyObject *exc_value; @@ -992,11 +998,11 @@ case _LOAD_BUILD_CLASS: { PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error_tier_two; + if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) JUMP_TO_ERROR(); if (bc == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - if (true) goto error_tier_two; + if (true) JUMP_TO_ERROR(); } stack_pointer[0] = bc; stack_pointer += 1; @@ -1014,14 +1020,14 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); Py_DECREF(v); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR(); } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); Py_DECREF(v); - if (err) goto pop_1_error_tier_two; + if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; } @@ -1034,7 +1040,7 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - GOTO_ERROR(error); + JUMP_TO_ERROR(); } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. @@ -1042,7 +1048,7 @@ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + JUMP_TO_ERROR(); } break; } @@ -1054,7 +1060,7 @@ PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); Py_DECREF(seq); - if (res == 0) goto pop_1_error_tier_two; + if (res == 0) JUMP_TO_ERROR(); stack_pointer += -1 + oparg; break; } @@ -1066,8 +1072,8 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; assert(oparg == 2); - if (!PyTuple_CheckExact(seq)) goto deoptimize; - if (PyTuple_GET_SIZE(seq) != 2) goto deoptimize; + if (!PyTuple_CheckExact(seq)) JUMP_TO_JUMP_TARGET(); + if (PyTuple_GET_SIZE(seq) != 2) JUMP_TO_JUMP_TARGET(); STAT_INC(UNPACK_SEQUENCE, hit); val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); @@ -1084,8 +1090,8 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyTuple_CheckExact(seq)) goto deoptimize; - if (PyTuple_GET_SIZE(seq) != oparg) goto deoptimize; + if (!PyTuple_CheckExact(seq)) JUMP_TO_JUMP_TARGET(); + if (PyTuple_GET_SIZE(seq) != oparg) JUMP_TO_JUMP_TARGET(); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1102,8 +1108,8 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyList_CheckExact(seq)) goto deoptimize; - if (PyList_GET_SIZE(seq) != oparg) goto deoptimize; + if (!PyList_CheckExact(seq)) JUMP_TO_JUMP_TARGET(); + if (PyList_GET_SIZE(seq) != oparg) JUMP_TO_JUMP_TARGET(); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1122,7 +1128,7 @@ PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq); - if (res == 0) goto pop_1_error_tier_two; + if (res == 0) JUMP_TO_ERROR(); stack_pointer += (oparg >> 8) + (oparg & 0xFF); break; } @@ -1137,7 +1143,7 @@ int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); Py_DECREF(owner); - if (err) goto pop_2_error_tier_two; + if (err) JUMP_TO_ERROR(); stack_pointer += -2; break; } @@ -1149,7 +1155,7 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); - if (err) goto pop_1_error_tier_two; + if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; } @@ -1161,7 +1167,7 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); - if (err) goto pop_1_error_tier_two; + if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; } @@ -1172,12 +1178,12 @@ int err = PyDict_Pop(GLOBALS(), name, NULL); // Can't use ERROR_IF here. if (err < 0) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } if (err == 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + JUMP_TO_ERROR(); } break; } @@ -1188,7 +1194,7 @@ if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - if (true) goto error_tier_two; + if (true) JUMP_TO_ERROR(); } Py_INCREF(locals); stack_pointer[0] = locals; @@ -1203,21 +1209,21 @@ mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } if (v == NULL) { if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + JUMP_TO_ERROR(); } } } @@ -1226,39 +1232,7 @@ break; } - case _LOAD_NAME: { - PyObject *v; - oparg = CURRENT_OPARG(); - PyObject *mod_or_class_dict = LOCALS(); - if (mod_or_class_dict == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error_tier_two; - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - GOTO_ERROR(error); - } - } - } - stack_pointer[0] = v; - stack_pointer += 1; - break; - } + /* _LOAD_NAME is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _LOAD_GLOBAL: { PyObject *res; @@ -1278,22 +1252,22 @@ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); } - if (true) goto error_tier_two; + if (true) JUMP_TO_ERROR(); } Py_INCREF(res); } else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error_tier_two; + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) JUMP_TO_ERROR(); if (res == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error_tier_two; + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) JUMP_TO_ERROR(); if (res == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - if (true) goto error_tier_two; + if (true) JUMP_TO_ERROR(); } } } @@ -1307,8 +1281,8 @@ case _GUARD_GLOBALS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) goto deoptimize; - if (dict->ma_keys->dk_version != version) goto deoptimize; + if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); + if (dict->ma_keys->dk_version != version) JUMP_TO_JUMP_TARGET(); assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1316,8 +1290,8 @@ case _GUARD_BUILTINS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) goto deoptimize; - if (dict->ma_keys->dk_version != version) goto deoptimize; + if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); + if (dict->ma_keys->dk_version != version) JUMP_TO_JUMP_TARGET(); assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1330,7 +1304,7 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - if (res == NULL) goto deoptimize; + if (res == NULL) JUMP_TO_JUMP_TARGET(); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1348,7 +1322,7 @@ PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - if (res == NULL) goto deoptimize; + if (res == NULL) JUMP_TO_JUMP_TARGET(); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1361,7 +1335,13 @@ case _DELETE_FAST: { oparg = CURRENT_OPARG(); PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error_tier_two; + if (v == NULL) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + if (1) JUMP_TO_ERROR(); + } SETLOCAL(oparg, NULL); break; } @@ -1373,7 +1353,7 @@ PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } SETLOCAL(oparg, cell); break; @@ -1387,7 +1367,7 @@ // Fortunately we don't need its superpower. if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + JUMP_TO_ERROR(); } PyCell_SET(cell, NULL); Py_DECREF(oldobj); @@ -1404,14 +1384,14 @@ assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } if (!value) { PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + JUMP_TO_ERROR(); } Py_INCREF(value); } @@ -1427,7 +1407,7 @@ value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - if (true) goto error_tier_two; + if (true) JUMP_TO_ERROR(); } Py_INCREF(value); stack_pointer[0] = value; @@ -1471,7 +1451,7 @@ for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - if (str == NULL) { stack_pointer += -oparg; goto error_tier_two; } + if (str == NULL) JUMP_TO_ERROR(); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; break; @@ -1483,7 +1463,7 @@ oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { stack_pointer += -oparg; goto error_tier_two; } + if (tup == NULL) JUMP_TO_ERROR(); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; break; @@ -1495,7 +1475,7 @@ oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { stack_pointer += -oparg; goto error_tier_two; } + if (list == NULL) JUMP_TO_ERROR(); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; break; @@ -1518,7 +1498,7 @@ Py_TYPE(iterable)->tp_name); } Py_DECREF(iterable); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR(); } assert(Py_IsNone(none_val)); Py_DECREF(iterable); @@ -1534,34 +1514,12 @@ set = stack_pointer[-2 - (oparg-1)]; int err = _PySet_Update(set, iterable); Py_DECREF(iterable); - if (err < 0) goto pop_1_error_tier_two; + if (err < 0) JUMP_TO_ERROR(); stack_pointer += -1; break; } - case _BUILD_SET: { - PyObject **values; - PyObject *set; - oparg = CURRENT_OPARG(); - values = &stack_pointer[-oparg]; - set = PySet_New(NULL); - if (set == NULL) - GOTO_ERROR(error); - int err = 0; - for (int i = 0; i < oparg; i++) { - PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); - } - if (err != 0) { - Py_DECREF(set); - if (true) { stack_pointer += -oparg; goto error_tier_two; } - } - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - break; - } + /* _BUILD_SET is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _BUILD_MAP: { PyObject **values; @@ -1575,7 +1533,7 @@ for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - if (map == NULL) { stack_pointer += -oparg*2; goto error_tier_two; } + if (map == NULL) JUMP_TO_ERROR(); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; break; @@ -1587,17 +1545,17 @@ if (LOCALS() == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - if (true) goto error_tier_two; + if (true) JUMP_TO_ERROR(); } /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error_tier_two; + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) JUMP_TO_ERROR(); if (ann_dict == NULL) { ann_dict = PyDict_New(); - if (ann_dict == NULL) goto error_tier_two; + if (ann_dict == NULL) JUMP_TO_ERROR(); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); Py_DECREF(ann_dict); - if (err) goto error_tier_two; + if (err) JUMP_TO_ERROR(); } else { Py_DECREF(ann_dict); @@ -1612,12 +1570,8 @@ oparg = CURRENT_OPARG(); keys = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } + assert(PyTuple_CheckExact(keys)); + assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); @@ -1625,7 +1579,7 @@ Py_DECREF(values[_i]); } Py_DECREF(keys); - if (map == NULL) { stack_pointer += -1 - oparg; goto error_tier_two; } + if (map == NULL) JUMP_TO_ERROR(); stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; break; @@ -1644,7 +1598,7 @@ Py_TYPE(update)->tp_name); } Py_DECREF(update); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR(); } Py_DECREF(update); stack_pointer += -1; @@ -1662,7 +1616,7 @@ if (_PyDict_MergeEx(dict, update, 2) < 0) { _PyEval_FormatKwargsError(tstate, callable, update); Py_DECREF(update); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR(); } Py_DECREF(update); stack_pointer += -1; @@ -1680,12 +1634,12 @@ assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error_tier_two; + if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) JUMP_TO_ERROR(); stack_pointer += -2; break; } - /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 because it is instrumented */ case _LOAD_SUPER_ATTR_ATTR: { PyObject *self; @@ -1697,15 +1651,15 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) goto deoptimize; - if (!PyType_Check(class)) goto deoptimize; + if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET(); + if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - if (attr == NULL) goto pop_3_error_tier_two; + if (attr == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = attr; stack_pointer += -2; break; @@ -1722,8 +1676,8 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) goto deoptimize; - if (!PyType_Check(class)) goto deoptimize; + if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET(); + if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -1734,7 +1688,7 @@ Py_DECREF(class); if (attr == NULL) { Py_DECREF(self); - if (true) goto pop_3_error_tier_two; + if (true) JUMP_TO_ERROR(); } if (method_found) { self_or_null = self; // transfer ownership @@ -1774,7 +1728,7 @@ meth | NULL | arg1 | ... | argN */ Py_DECREF(owner); - if (attr == NULL) goto pop_1_error_tier_two; + if (attr == NULL) JUMP_TO_ERROR(); self_or_null = NULL; } } @@ -1782,7 +1736,7 @@ /* Classic, pushes one value. */ attr = PyObject_GetAttr(owner, name); Py_DECREF(owner); - if (attr == NULL) goto pop_1_error_tier_two; + if (attr == NULL) JUMP_TO_ERROR(); } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; @@ -1796,7 +1750,7 @@ uint32_t type_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - if (tp->tp_version_tag != type_version) goto side_exit; + if (tp->tp_version_tag != type_version) JUMP_TO_JUMP_TARGET(); break; } @@ -1806,7 +1760,7 @@ assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) goto deoptimize; + if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET(); break; } @@ -1819,7 +1773,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - if (attr == NULL) goto deoptimize; + if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1837,7 +1791,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - if (attr == NULL) goto deoptimize; + if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1854,10 +1808,10 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)CURRENT_OPERAND(); - if (!PyModule_CheckExact(owner)) goto deoptimize; + if (!PyModule_CheckExact(owner)) JUMP_TO_JUMP_TARGET(); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - if (dict->ma_keys->dk_version != dict_version) goto deoptimize; + if (dict->ma_keys->dk_version != dict_version) JUMP_TO_JUMP_TARGET(); break; } @@ -1873,7 +1827,7 @@ assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; attr = ep->me_value; - if (attr == NULL) goto deoptimize; + if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1889,9 +1843,9 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (_PyDictOrValues_IsValues(dorv)) goto deoptimize; + if (_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET(); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - if (dict == NULL) goto deoptimize; + if (dict == NULL) JUMP_TO_JUMP_TARGET(); assert(PyDict_CheckExact((PyObject *)dict)); break; } @@ -1905,19 +1859,19 @@ uint16_t hint = (uint16_t)CURRENT_OPERAND(); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - if (hint >= (size_t)dict->ma_keys->dk_nentries) goto deoptimize; + if (hint >= (size_t)dict->ma_keys->dk_nentries) JUMP_TO_JUMP_TARGET(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) goto deoptimize; + if (ep->me_key != name) JUMP_TO_JUMP_TARGET(); attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) goto deoptimize; + if (ep->me_key != name) JUMP_TO_JUMP_TARGET(); attr = ep->me_value; } - if (attr == NULL) goto deoptimize; + if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1937,7 +1891,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - if (attr == NULL) goto deoptimize; + if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1955,7 +1909,7 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - if (attr == NULL) goto deoptimize; + if (attr == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1972,9 +1926,9 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - if (!PyType_Check(owner)) goto deoptimize; + if (!PyType_Check(owner)) JUMP_TO_JUMP_TARGET(); assert(type_version != 0); - if (((PyTypeObject *)owner)->tp_version_tag != type_version) goto deoptimize; + if (((PyTypeObject *)owner)->tp_version_tag != type_version) JUMP_TO_JUMP_TARGET(); break; } @@ -2014,16 +1968,16 @@ /* _LOAD_ATTR_CLASS is split on (oparg & 1) */ - /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */ + /* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ + /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _GUARD_DORV_VALUES: { PyObject *owner; owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(dorv)) goto deoptimize; + if (!_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET(); break; } @@ -2049,7 +2003,7 @@ break; } - /* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 */ + /* _STORE_ATTR_WITH_HINT is not a viable micro-op for tier 2 because it has unused cache entries */ case _STORE_ATTR_SLOT: { PyObject *owner; @@ -2078,11 +2032,11 @@ res = PyObject_RichCompare(left, right, oparg >> 5); Py_DECREF(left); Py_DECREF(right); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); if (oparg & 16) { int res_bool = PyObject_IsTrue(res); Py_DECREF(res); - if (res_bool < 0) goto pop_2_error_tier_two; + if (res_bool < 0) JUMP_TO_ERROR(); res = res_bool ? Py_True : Py_False; } stack_pointer[-2] = res; @@ -2118,8 +2072,8 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!_PyLong_IsCompact((PyLongObject *)left)) goto deoptimize; - if (!_PyLong_IsCompact((PyLongObject *)right)) goto deoptimize; + if (!_PyLong_IsCompact((PyLongObject *)left)) JUMP_TO_JUMP_TARGET(); + if (!_PyLong_IsCompact((PyLongObject *)right)) JUMP_TO_JUMP_TARGET(); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); @@ -2184,7 +2138,7 @@ int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); - if (res < 0) goto pop_2_error_tier_two; + if (res < 0) JUMP_TO_ERROR(); b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -2198,13 +2152,13 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) goto deoptimize; + if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) JUMP_TO_JUMP_TARGET(); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! int res = _PySet_Contains((PySetObject *)right, left); Py_DECREF(left); Py_DECREF(right); - if (res < 0) goto pop_2_error_tier_two; + if (res < 0) JUMP_TO_ERROR(); b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -2218,12 +2172,12 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyDict_CheckExact(right)) goto deoptimize; + if (!PyDict_CheckExact(right)) JUMP_TO_JUMP_TARGET(); STAT_INC(CONTAINS_OP, hit); int res = PyDict_Contains(right, left); Py_DECREF(left); Py_DECREF(right); - if (res < 0) goto pop_2_error_tier_two; + if (res < 0) JUMP_TO_ERROR(); b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -2240,7 +2194,7 @@ if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); - if (true) goto pop_2_error_tier_two; + if (true) JUMP_TO_ERROR(); } match = NULL; rest = NULL; @@ -2248,9 +2202,9 @@ &match, &rest); Py_DECREF(exc_value); Py_DECREF(match_type); - if (res < 0) goto pop_2_error_tier_two; + if (res < 0) JUMP_TO_ERROR(); assert((match == NULL) == (rest == NULL)); - if (match == NULL) goto pop_2_error_tier_two; + if (match == NULL) JUMP_TO_ERROR(); if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } @@ -2268,7 +2222,7 @@ assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { Py_DECREF(right); - if (true) goto pop_1_error_tier_two; + if (true) JUMP_TO_ERROR(); } int res = PyErr_GivenExceptionMatches(left, right); Py_DECREF(right); @@ -2277,9 +2231,9 @@ break; } - /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is replaced */ - /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */ case _IS_NONE: { PyObject *value; @@ -2302,9 +2256,9 @@ obj = stack_pointer[-1]; // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); - if (len_i < 0) goto error_tier_two; + if (len_i < 0) JUMP_TO_ERROR(); len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) goto error_tier_two; + if (len_o == NULL) JUMP_TO_ERROR(); stack_pointer[0] = len_o; stack_pointer += 1; break; @@ -2330,7 +2284,7 @@ assert(PyTuple_CheckExact(attrs)); // Success! } else { - if (_PyErr_Occurred(tstate)) goto pop_3_error_tier_two; + if (_PyErr_Occurred(tstate)) JUMP_TO_ERROR(); // Error! attrs = Py_None; // Failure! } @@ -2369,7 +2323,7 @@ subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); - if (values_or_none == NULL) goto error_tier_two; + if (values_or_none == NULL) JUMP_TO_ERROR(); stack_pointer[0] = values_or_none; stack_pointer += 1; break; @@ -2382,7 +2336,7 @@ /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); - if (iter == NULL) goto pop_1_error_tier_two; + if (iter == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = iter; break; } @@ -2400,7 +2354,7 @@ _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - GOTO_ERROR(error); + JUMP_TO_ERROR(); } iter = iterable; } @@ -2411,7 +2365,7 @@ /* `iterable` is not a generator. */ iter = PyObject_GetIter(iterable); if (iter == NULL) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } Py_DECREF(iterable); } @@ -2419,7 +2373,7 @@ break; } - /* _FOR_ITER is not a viable micro-op for tier 2 */ + /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */ case _FOR_ITER_TIER_TWO: { PyObject *iter; @@ -2430,7 +2384,7 @@ if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } _PyErr_Clear(tstate); } @@ -2438,7 +2392,7 @@ Py_DECREF(iter); STACK_SHRINK(1); /* The translator sets the deopt target just past END_FOR */ - if (true) goto deoptimize; + if (true) JUMP_TO_JUMP_TARGET(); } // Common case: no jump, leave it to the code generator stack_pointer[0] = next; @@ -2446,16 +2400,16 @@ break; } - /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */ case _ITER_CHECK_LIST: { PyObject *iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyListIter_Type) goto deoptimize; + if (Py_TYPE(iter) != &PyListIter_Type) JUMP_TO_JUMP_TARGET(); break; } - /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 */ + /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_LIST: { PyObject *iter; @@ -2463,8 +2417,8 @@ _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; - if (seq == NULL) goto deoptimize; - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) goto deoptimize; + if (seq == NULL) JUMP_TO_JUMP_TARGET(); + if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) JUMP_TO_JUMP_TARGET(); break; } @@ -2486,11 +2440,11 @@ case _ITER_CHECK_TUPLE: { PyObject *iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyTupleIter_Type) goto deoptimize; + if (Py_TYPE(iter) != &PyTupleIter_Type) JUMP_TO_JUMP_TARGET(); break; } - /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 */ + /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_TUPLE: { PyObject *iter; @@ -2498,8 +2452,8 @@ _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; - if (seq == NULL) goto deoptimize; - if (it->it_index >= PyTuple_GET_SIZE(seq)) goto deoptimize; + if (seq == NULL) JUMP_TO_JUMP_TARGET(); + if (it->it_index >= PyTuple_GET_SIZE(seq)) JUMP_TO_JUMP_TARGET(); break; } @@ -2522,18 +2476,18 @@ PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - if (Py_TYPE(r) != &PyRangeIter_Type) goto deoptimize; + if (Py_TYPE(r) != &PyRangeIter_Type) JUMP_TO_JUMP_TARGET(); break; } - /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 */ + /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_RANGE: { PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); - if (r->len <= 0) goto deoptimize; + if (r->len <= 0) JUMP_TO_JUMP_TARGET(); break; } @@ -2548,96 +2502,17 @@ r->start = value + r->step; r->len--; next = PyLong_FromLong(value); - if (next == NULL) goto error_tier_two; + if (next == NULL) JUMP_TO_ERROR(); stack_pointer[0] = next; stack_pointer += 1; break; } - /* _FOR_ITER_GEN is not a viable micro-op for tier 2 */ + /* _FOR_ITER_GEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _BEFORE_ASYNC_WITH: { - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "asynchronous context manager protocol " - "(missed __aexit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = PyObject_CallNoArgs(enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error_tier_two; - } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - break; - } + /* _BEFORE_ASYNC_WITH is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - case _BEFORE_WITH: { - PyObject *mgr; - PyObject *exit; - PyObject *res; - mgr = stack_pointer[-1]; - /* pop the context manager, push its __exit__ and the - * value returned from calling its __enter__ - */ - PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); - if (enter == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol", - Py_TYPE(mgr)->tp_name); - } - GOTO_ERROR(error); - } - exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); - if (exit == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object does not support the " - "context manager protocol " - "(missed __exit__ method)", - Py_TYPE(mgr)->tp_name); - } - Py_DECREF(enter); - GOTO_ERROR(error); - } - Py_DECREF(mgr); - res = PyObject_CallNoArgs(enter); - Py_DECREF(enter); - if (res == NULL) { - Py_DECREF(exit); - if (true) goto pop_1_error_tier_two; - } - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - break; - } + /* _BEFORE_WITH is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _WITH_EXCEPT_START: { PyObject *val; @@ -2670,7 +2545,7 @@ PyObject *stack[4] = {NULL, exc, val, tb}; res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) goto error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[0] = res; stack_pointer += 1; break; @@ -2700,7 +2575,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) goto deoptimize; + if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET(); break; } @@ -2710,7 +2585,7 @@ uint32_t keys_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - if (owner_heap_type->ht_cached_keys->dk_version != keys_version) goto deoptimize; + if (owner_heap_type->ht_cached_keys->dk_version != keys_version) JUMP_TO_JUMP_TARGET(); break; } @@ -2792,7 +2667,7 @@ assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ - if (dict != NULL) goto deoptimize; + if (dict != NULL) JUMP_TO_JUMP_TARGET(); break; } @@ -2815,9 +2690,9 @@ break; } - /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_CALL is not a viable micro-op for tier 2 because it is instrumented */ - /* _CALL is not a viable micro-op for tier 2 */ + /* _CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _CHECK_PERIODIC: { CHECK_EVAL_BREAKER(); @@ -2830,8 +2705,8 @@ oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - if (null != NULL) goto deoptimize; - if (Py_TYPE(callable) != &PyMethod_Type) goto deoptimize; + if (null != NULL) JUMP_TO_JUMP_TARGET(); + if (Py_TYPE(callable) != &PyMethod_Type) JUMP_TO_JUMP_TARGET(); break; } @@ -2853,7 +2728,7 @@ } case _CHECK_PEP_523: { - if (tstate->interp->eval_frame) goto deoptimize; + if (tstate->interp->eval_frame) JUMP_TO_JUMP_TARGET(); break; } @@ -2864,11 +2739,11 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - if (!PyFunction_Check(callable)) goto deoptimize; + if (!PyFunction_Check(callable)) JUMP_TO_JUMP_TARGET(); PyFunctionObject *func = (PyFunctionObject *)callable; - if (func->func_version != func_version) goto deoptimize; + if (func->func_version != func_version) JUMP_TO_JUMP_TARGET(); PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (self_or_null != NULL)) goto deoptimize; + if (code->co_argcount != oparg + (self_or_null != NULL)) JUMP_TO_JUMP_TARGET(); break; } @@ -2878,8 +2753,8 @@ callable = stack_pointer[-2 - oparg]; PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) goto deoptimize; - if (tstate->py_recursion_remaining <= 1) goto deoptimize; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) JUMP_TO_JUMP_TARGET(); + if (tstate->py_recursion_remaining <= 1) JUMP_TO_JUMP_TARGET(); break; } @@ -3049,7 +2924,7 @@ break; } - /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ + /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _CALL_TYPE_1: { PyObject *arg; @@ -3061,8 +2936,8 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) goto deoptimize; - if (callable != (PyObject *)&PyType_Type) goto deoptimize; + if (null != NULL) JUMP_TO_JUMP_TARGET(); + if (callable != (PyObject *)&PyType_Type) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(arg)); Py_DECREF(arg); @@ -3081,12 +2956,12 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) goto deoptimize; - if (callable != (PyObject *)&PyUnicode_Type) goto deoptimize; + if (null != NULL) JUMP_TO_JUMP_TARGET(); + if (callable != (PyObject *)&PyUnicode_Type) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); res = PyObject_Str(arg); Py_DECREF(arg); - if (res == NULL) goto pop_3_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = res; stack_pointer += -2; break; @@ -3102,18 +2977,18 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) goto deoptimize; - if (callable != (PyObject *)&PyTuple_Type) goto deoptimize; + if (null != NULL) JUMP_TO_JUMP_TARGET(); + if (callable != (PyObject *)&PyTuple_Type) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); res = PySequence_Tuple(arg); Py_DECREF(arg); - if (res == NULL) goto pop_3_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = res; stack_pointer += -2; break; } - /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */ + /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _EXIT_INIT_CHECK: { PyObject *should_be_none; @@ -3123,7 +2998,7 @@ PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(should_be_none)->tp_name); - GOTO_ERROR(error); + JUMP_TO_ERROR(); } stack_pointer += -1; break; @@ -3143,9 +3018,9 @@ args--; total_args++; } - if (!PyType_Check(callable)) goto deoptimize; + if (!PyType_Check(callable)) JUMP_TO_JUMP_TARGET(); PyTypeObject *tp = (PyTypeObject *)callable; - if (tp->tp_vectorcall == NULL) goto deoptimize; + if (tp->tp_vectorcall == NULL) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); /* Free the arguments. */ @@ -3153,7 +3028,7 @@ Py_DECREF(args[i]); } Py_DECREF(tp); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3174,11 +3049,11 @@ args--; total_args++; } - if (total_args != 1) goto deoptimize; - if (!PyCFunction_CheckExact(callable)) goto deoptimize; - if (PyCFunction_GET_FLAGS(callable) != METH_O) goto deoptimize; + if (total_args != 1) JUMP_TO_JUMP_TARGET(); + if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); + if (PyCFunction_GET_FLAGS(callable) != METH_O) JUMP_TO_JUMP_TARGET(); // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) goto deoptimize; + if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); PyObject *arg = args[0]; @@ -3188,7 +3063,7 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3209,8 +3084,8 @@ args--; total_args++; } - if (!PyCFunction_CheckExact(callable)) goto deoptimize; - if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) goto deoptimize; + if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); + if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ @@ -3224,7 +3099,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3245,8 +3120,8 @@ args--; total_args++; } - if (!PyCFunction_CheckExact(callable)) goto deoptimize; - if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) goto deoptimize; + if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); + if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ PyCFunctionFastWithKeywords cfunc = @@ -3259,7 +3134,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3280,14 +3155,14 @@ args--; total_args++; } - if (total_args != 1) goto deoptimize; + if (total_args != 1) JUMP_TO_JUMP_TARGET(); PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.len) goto deoptimize; + if (callable != interp->callable_cache.len) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3316,15 +3191,15 @@ args--; total_args++; } - if (total_args != 2) goto deoptimize; + if (total_args != 2) JUMP_TO_JUMP_TARGET(); PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.isinstance) goto deoptimize; + if (callable != interp->callable_cache.isinstance) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3354,15 +3229,15 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (total_args != 2) goto deoptimize; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; + if (total_args != 2) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) goto deoptimize; + if (meth->ml_flags != METH_O) JUMP_TO_JUMP_TARGET(); // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) goto deoptimize; + if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); PyObject *arg = args[1]; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; + if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3372,7 +3247,7 @@ Py_DECREF(self); Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3393,12 +3268,12 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) goto deoptimize; + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) JUMP_TO_JUMP_TARGET(); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, d_type)) goto deoptimize; + if (!Py_IS_TYPE(self, d_type)) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = @@ -3410,7 +3285,7 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3431,15 +3306,15 @@ args--; total_args++; } - if (total_args != 1) goto deoptimize; + if (total_args != 1) JUMP_TO_JUMP_TARGET(); PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; - if (meth->ml_flags != METH_NOARGS) goto deoptimize; + if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); + if (meth->ml_flags != METH_NOARGS) JUMP_TO_JUMP_TARGET(); // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) goto deoptimize; + if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3448,7 +3323,7 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3470,11 +3345,11 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) goto deoptimize; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) goto deoptimize; + if (meth->ml_flags != METH_FASTCALL) JUMP_TO_JUMP_TARGET(); PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) goto deoptimize; + if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); STAT_INC(CALL, hit); PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; @@ -3486,19 +3361,19 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { stack_pointer += -2 - oparg; goto error_tier_two; } + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; } - /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 because it is instrumented */ - /* _CALL_KW is not a viable micro-op for tier 2 */ + /* _CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it is instrumented */ - /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + /* _CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _MAKE_FUNCTION: { PyObject *codeobj; @@ -3508,7 +3383,7 @@ PyFunction_New(codeobj, GLOBALS()); Py_DECREF(codeobj); if (func_obj == NULL) { - GOTO_ERROR(error); + JUMP_TO_ERROR(); } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); @@ -3565,7 +3440,7 @@ Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - if (slice == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error_tier_two; } + if (slice == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); break; @@ -3581,7 +3456,7 @@ conv_fn = _PyEval_ConversionFuncs[oparg]; result = conv_fn(value); Py_DECREF(value); - if (result == NULL) goto pop_1_error_tier_two; + if (result == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = result; break; } @@ -3595,7 +3470,7 @@ if (!PyUnicode_CheckExact(value)) { res = PyObject_Format(value, NULL); Py_DECREF(value); - if (res == NULL) goto pop_1_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); } else { res = value; @@ -3613,7 +3488,7 @@ res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -3642,7 +3517,7 @@ res = _PyEval_BinaryOps[oparg](lhs, rhs); Py_DECREF(lhs); Py_DECREF(rhs); - if (res == NULL) goto pop_2_error_tier_two; + if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -3660,25 +3535,25 @@ break; } - /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_JUMP_BACKWARD is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 because it is instrumented */ - /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ + /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 because it is instrumented */ case _GUARD_IS_TRUE_POP: { PyObject *flag; flag = stack_pointer[-1]; stack_pointer += -1; - if (!Py_IsTrue(flag)) goto side_exit; + if (!Py_IsTrue(flag)) JUMP_TO_JUMP_TARGET(); assert(Py_IsTrue(flag)); break; } @@ -3687,7 +3562,7 @@ PyObject *flag; flag = stack_pointer[-1]; stack_pointer += -1; - if (!Py_IsFalse(flag)) goto side_exit; + if (!Py_IsFalse(flag)) JUMP_TO_JUMP_TARGET(); assert(Py_IsFalse(flag)); break; } @@ -3698,7 +3573,7 @@ stack_pointer += -1; if (!Py_IsNone(val)) { Py_DECREF(val); - if (1) goto side_exit; + if (1) JUMP_TO_JUMP_TARGET(); } break; } @@ -3707,7 +3582,7 @@ PyObject *val; val = stack_pointer[-1]; stack_pointer += -1; - if (Py_IsNone(val)) goto side_exit; + if (Py_IsNone(val)) JUMP_TO_JUMP_TARGET(); Py_DECREF(val); break; } @@ -3738,12 +3613,12 @@ } case _EXIT_TRACE: { - if (1) goto side_exit; + if (1) JUMP_TO_JUMP_TARGET(); break; } case _CHECK_VALIDITY: { - if (!current_executor->vm_data.valid) goto deoptimize; + if (!current_executor->vm_data.valid) JUMP_TO_JUMP_TARGET(); break; } @@ -3803,7 +3678,7 @@ case _CHECK_FUNCTION: { uint32_t func_version = (uint32_t)CURRENT_OPERAND(); assert(PyFunction_Check(frame->f_funcobj)); - if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) goto deoptimize; + if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) JUMP_TO_JUMP_TARGET(); break; } @@ -3838,7 +3713,7 @@ if (optimized < 0) { Py_DECREF(previous); tstate->previous_executor = Py_None; - if (1) goto error_tier_two; + GOTO_UNWIND(); } GOTO_TIER_ONE(target); } @@ -3869,9 +3744,26 @@ case _CHECK_VALIDITY_AND_SET_IP: { PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); - if (!current_executor->vm_data.valid) goto deoptimize; + if (!current_executor->vm_data.valid) JUMP_TO_JUMP_TARGET(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; break; } + case _DEOPT: { + EXIT_TO_TIER1(); + break; + } + + case _SIDE_EXIT: { + EXIT_TO_TRACE(); + break; + } + + case _ERROR_POP_N: { + oparg = CURRENT_OPARG(); + stack_pointer += -oparg; + GOTO_UNWIND(); + break; + } + #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2996ee7..c66eb67 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -25,7 +25,7 @@ "asynchronous context manager protocol", Py_TYPE(mgr)->tp_name); } - GOTO_ERROR(error); + goto error; } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); if (exit == NULL) { @@ -37,7 +37,7 @@ Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - GOTO_ERROR(error); + goto error; } Py_DECREF(mgr); res = PyObject_CallNoArgs(enter); @@ -71,7 +71,7 @@ "context manager protocol", Py_TYPE(mgr)->tp_name); } - GOTO_ERROR(error); + goto error; } exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); if (exit == NULL) { @@ -83,7 +83,7 @@ Py_TYPE(mgr)->tp_name); } Py_DECREF(enter); - GOTO_ERROR(error); + goto error; } Py_DECREF(mgr); res = PyObject_CallNoArgs(enter); @@ -605,12 +605,8 @@ PyObject *map; keys = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString(tstate, PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - GOTO_ERROR(error); // Pop the keys and values. - } + assert(PyTuple_CheckExact(keys)); + assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); @@ -667,7 +663,7 @@ values = &stack_pointer[-oparg]; set = PySet_New(NULL); if (set == NULL) - GOTO_ERROR(error); + goto error; int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; @@ -808,7 +804,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - GOTO_ERROR(error); + goto error; } frame->return_offset = (uint16_t)(next_instr - this_instr); DISPATCH_INLINED(new_frame); @@ -882,7 +878,7 @@ STAT_INC(CALL, hit); PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { - GOTO_ERROR(error); + goto error; } Py_DECREF(tp); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( @@ -1213,11 +1209,11 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { - GOTO_ERROR(error); + goto error; } PyObject *tuple = PySequence_Tuple(callargs); if (tuple == NULL) { - GOTO_ERROR(error); + goto error; } Py_SETREF(callargs, tuple); } @@ -1229,7 +1225,7 @@ int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - if (err) GOTO_ERROR(error); + if (err) goto error; result = PyObject_Call(func, callargs, kwargs); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result == NULL) { @@ -1261,7 +1257,7 @@ // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { - GOTO_ERROR(error); + goto error; } assert(next_instr - this_instr == 1); frame->return_offset = 1; @@ -1342,7 +1338,7 @@ PyObject *inst = args[0]; int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { - GOTO_ERROR(error); + goto error; } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1407,7 +1403,7 @@ // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { - GOTO_ERROR(error); + goto error; } assert(next_instr - this_instr == 1); frame->return_offset = 1; @@ -1475,7 +1471,7 @@ PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { - GOTO_ERROR(error); + goto error; } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -2329,7 +2325,7 @@ // Fortunately we don't need its superpower. if (oldobj == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + goto error; } PyCell_SET(cell, NULL); Py_DECREF(oldobj); @@ -2341,7 +2337,13 @@ next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); PyObject *v = GETLOCAL(oparg); - if (v == NULL) goto unbound_local_error; + if (v == NULL) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + if (1) goto error; + } SETLOCAL(oparg, NULL); DISPATCH(); } @@ -2354,12 +2356,12 @@ int err = PyDict_Pop(GLOBALS(), name, NULL); // Can't use ERROR_IF here. if (err < 0) { - GOTO_ERROR(error); + goto error; } if (err == 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + goto error; } DISPATCH(); } @@ -2374,7 +2376,7 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - GOTO_ERROR(error); + goto error; } err = PyObject_DelItem(ns, name); // Can't use ERROR_IF here. @@ -2382,7 +2384,7 @@ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + goto error; } DISPATCH(); } @@ -2523,7 +2525,7 @@ PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(should_be_none)->tp_name); - GOTO_ERROR(error); + goto error; } stack_pointer += -1; DISPATCH(); @@ -2610,7 +2612,7 @@ if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + goto error; } monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); @@ -2841,7 +2843,7 @@ if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { - GOTO_ERROR(error); + goto error; } } else { if (type->tp_as_async != NULL){ @@ -2850,7 +2852,7 @@ if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { - GOTO_ERROR(error); + goto error; } } else { @@ -2858,7 +2860,7 @@ "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); - GOTO_ERROR(error); + goto error; } awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { @@ -2868,7 +2870,7 @@ "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); Py_DECREF(next_iter); - GOTO_ERROR(error); + goto error; } else { Py_DECREF(next_iter); } @@ -2956,7 +2958,7 @@ _PyErr_SetString(tstate, PyExc_TypeError, "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); - GOTO_ERROR(error); + goto error; } iter = iterable; } @@ -2967,7 +2969,7 @@ /* `iterable` is not a generator. */ iter = PyObject_GetIter(iterable); if (iter == NULL) { - GOTO_ERROR(error); + goto error; } Py_DECREF(iterable); } @@ -3066,7 +3068,7 @@ if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); + goto error; } PyErr_SetRaisedException(NULL); } @@ -3087,7 +3089,7 @@ if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, this_instr)) { - GOTO_ERROR(error); + goto error; } PyErr_SetRaisedException(NULL); } @@ -3113,7 +3115,7 @@ else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - GOTO_ERROR(error); + goto error; } monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); @@ -3268,7 +3270,7 @@ uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { - GOTO_ERROR(error); + goto error; } next_instr = this_instr; } @@ -3299,7 +3301,7 @@ int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); - if (err) GOTO_ERROR(error); + if (err) goto error; Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3324,7 +3326,7 @@ int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); - if (err) GOTO_ERROR(error); + if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3356,7 +3358,7 @@ int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, retval); - if (err) GOTO_ERROR(error); + if (err) goto error; tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -4138,7 +4140,13 @@ INSTRUCTION_STATS(LOAD_FAST_CHECK); PyObject *value; value = GETLOCAL(oparg); - if (value == NULL) goto unbound_local_error; + if (value == NULL) { + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + if (1) goto error; + } Py_INCREF(value); stack_pointer[0] = value; stack_pointer += 1; @@ -4175,14 +4183,14 @@ assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { - GOTO_ERROR(error); + goto error; } if (!value) { PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - GOTO_ERROR(error); + goto error; } Py_INCREF(value); } @@ -4200,21 +4208,21 @@ mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); + goto error; } if (v == NULL) { if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); + goto error; } if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); + goto error; } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + goto error; } } } @@ -4398,21 +4406,21 @@ } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - GOTO_ERROR(error); + goto error; } if (v == NULL) { if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - GOTO_ERROR(error); + goto error; } if (v == NULL) { if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - GOTO_ERROR(error); + goto error; } if (v == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - GOTO_ERROR(error); + goto error; } } } @@ -4574,7 +4582,7 @@ PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - GOTO_ERROR(error); + goto error; } SETLOCAL(oparg, cell); DISPATCH(); @@ -4591,7 +4599,7 @@ PyFunction_New(codeobj, GLOBALS()); Py_DECREF(codeobj); if (func_obj == NULL) { - GOTO_ERROR(error); + goto error; } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); @@ -4910,7 +4918,7 @@ else { assert(PyLong_Check(lasti)); _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - GOTO_ERROR(error); + goto error; } } assert(exc && PyExceptionInstance_Check(exc)); @@ -5017,7 +5025,7 @@ PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); if (gen == NULL) { - GOTO_ERROR(error); + goto error; } assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5126,7 +5134,7 @@ JUMPBY(oparg); } else { - GOTO_ERROR(error); + goto error; } } Py_DECREF(v); diff --git a/Python/jit.c b/Python/jit.c index f67d641..03bcf11 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -381,11 +381,13 @@ int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length) { // Loop once to find the total compiled size: - size_t code_size = 0; - size_t data_size = 0; + uint32_t instruction_starts[UOP_MAX_TRACE_LENGTH]; + uint32_t code_size = 0; + uint32_t data_size = 0; for (size_t i = 0; i < length; i++) { _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; const StencilGroup *group = &stencil_groups[instruction->opcode]; + instruction_starts[i] = code_size; code_size += group->code.body_size; data_size += group->data.body_size; } @@ -403,11 +405,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size // Loop again to emit the code: unsigned char *code = memory; unsigned char *data = memory + code_size; - unsigned char *top = code; - if (trace[0].opcode == _START_EXECUTOR) { - // Don't want to execute this more than once: - top += stencil_groups[_START_EXECUTOR].code.body_size; - } + assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT); for (size_t i = 0; i < length; i++) { _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; const StencilGroup *group = &stencil_groups[instruction->opcode]; @@ -419,8 +417,29 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size patches[HoleValue_EXECUTOR] = (uint64_t)executor; patches[HoleValue_OPARG] = instruction->oparg; patches[HoleValue_OPERAND] = instruction->operand; - patches[HoleValue_TARGET] = instruction->target; - patches[HoleValue_TOP] = (uint64_t)top; + switch (instruction->format) { + case UOP_FORMAT_TARGET: + patches[HoleValue_TARGET] = instruction->target; + break; + case UOP_FORMAT_EXIT: + assert(instruction->exit_index < executor->exit_count); + patches[HoleValue_EXIT_INDEX] = instruction->exit_index; + if (instruction->error_target < length) { + patches[HoleValue_ERROR_TARGET] = (uint64_t)memory + instruction_starts[instruction->error_target]; + } + break; + case UOP_FORMAT_JUMP: + assert(instruction->jump_target < length); + patches[HoleValue_JUMP_TARGET] = (uint64_t)memory + instruction_starts[instruction->jump_target]; + if (instruction->error_target < length) { + patches[HoleValue_ERROR_TARGET] = (uint64_t)memory + instruction_starts[instruction->error_target]; + } + break; + default: + assert(0); + Py_FatalError("Illegal instruction format"); + } + patches[HoleValue_TOP] = (uint64_t)memory + instruction_starts[1]; patches[HoleValue_ZERO] = 0; emit(group, patches); code += group->code.body_size; diff --git a/Python/optimizer.c b/Python/optimizer.c index f8c1390..38ab6d3 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -154,13 +154,19 @@ PyUnstable_GetOptimizer(void) } static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies); +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies); static int init_cold_exit_executor(_PyExecutorObject *executor, int oparg); +/* It is impossible for the number of exits to reach 1/4 of the total length, + * as the number of exits cannot reach 1/3 of the number of non-exits, due to + * the presence of CHECK_VALIDITY checks and instructions to produce the values + * being checked in exits. */ +#define COLD_EXIT_COUNT (UOP_MAX_TRACE_LENGTH/4) + static int cold_exits_initialized = 0; -static _PyExecutorObject COLD_EXITS[UOP_MAX_TRACE_LENGTH] = { 0 }; +static _PyExecutorObject COLD_EXITS[COLD_EXIT_COUNT] = { 0 }; static const _PyBloomFilter EMPTY_FILTER = { 0 }; @@ -172,7 +178,7 @@ _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) } else if (cold_exits_initialized == 0) { cold_exits_initialized = 1; - for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { + for (int i = 0; i < COLD_EXIT_COUNT; i++) { if (init_cold_exit_executor(&COLD_EXITS[i], i)) { return NULL; } @@ -313,10 +319,33 @@ _PyUOpPrint(const _PyUOpInstruction *uop) else { printf("%s", name); } - printf(" (%d, target=%d, operand=%#" PRIx64 ")", - uop->oparg, - uop->target, - (uint64_t)uop->operand); + switch(uop->format) { + case UOP_FORMAT_TARGET: + printf(" (%d, target=%d, operand=%#" PRIx64, + uop->oparg, + uop->target, + (uint64_t)uop->operand); + break; + case UOP_FORMAT_JUMP: + printf(" (%d, jump_target=%d, operand=%#" PRIx64, + uop->oparg, + uop->jump_target, + (uint64_t)uop->operand); + break; + case UOP_FORMAT_EXIT: + printf(" (%d, exit_index=%d, operand=%#" PRIx64, + uop->oparg, + uop->exit_index, + (uint64_t)uop->operand); + break; + default: + printf(" (%d, Unknown format)", uop->oparg); + } + if (_PyUop_Flags[uop->opcode] & HAS_ERROR_FLAG) { + printf(", error_target=%d", uop->error_target); + } + + printf(")"); } #endif @@ -432,28 +461,36 @@ BRANCH_TO_GUARD[4][2] = { #endif -// Beware: Macro arg order differs from struct member order +static inline int +add_to_trace( + _PyUOpInstruction *trace, + int trace_length, + uint16_t opcode, + uint16_t oparg, + uint64_t operand, + uint32_t target) +{ + trace[trace_length].opcode = opcode; + trace[trace_length].format = UOP_FORMAT_TARGET; + trace[trace_length].target = target; + trace[trace_length].oparg = oparg; + trace[trace_length].operand = operand; + return trace_length + 1; +} + #ifdef Py_DEBUG #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ assert(trace_length < max_length); \ - trace[trace_length].opcode = (OPCODE); \ - trace[trace_length].oparg = (OPARG); \ - trace[trace_length].target = (TARGET); \ - trace[trace_length].operand = (OPERAND); \ + trace_length = add_to_trace(trace, trace_length, (OPCODE), (OPARG), (OPERAND), (TARGET)); \ if (lltrace >= 2) { \ printf("%4d ADD_TO_TRACE: ", trace_length); \ - _PyUOpPrint(&trace[trace_length]); \ + _PyUOpPrint(&trace[trace_length-1]); \ printf("\n"); \ - } \ - trace_length++; + } #else #define ADD_TO_TRACE(OPCODE, OPARG, OPERAND, TARGET) \ assert(trace_length < max_length); \ - trace[trace_length].opcode = (OPCODE); \ - trace[trace_length].oparg = (OPARG); \ - trace[trace_length].target = (TARGET); \ - trace[trace_length].operand = (OPERAND); \ - trace_length++; + trace_length = add_to_trace(trace, trace_length, (OPCODE), (OPARG), (OPERAND), (TARGET)); #endif #define INSTR_IP(INSTR, CODE) \ @@ -476,8 +513,7 @@ BRANCH_TO_GUARD[4][2] = { if (trace_stack_depth >= TRACE_STACK_SIZE) { \ DPRINTF(2, "Trace stack overflow\n"); \ OPT_STAT_INC(trace_stack_overflow); \ - ADD_TO_TRACE(uop, oparg, operand, target); \ - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); \ + trace_length = 0; \ goto done; \ } \ assert(func == NULL || func->func_code == (PyObject *)code); \ @@ -495,7 +531,7 @@ BRANCH_TO_GUARD[4][2] = { assert(func == NULL || func->func_code == (PyObject *)code); \ instr = trace_stack[trace_stack_depth].instr; -/* Returns 1 on success, +/* Returns the length of the trace on success, * 0 if it failed to produce a worthwhile trace, * and -1 on an error. */ @@ -515,7 +551,8 @@ translate_bytecode_to_trace( _Py_BloomFilter_Add(dependencies, initial_code); _Py_CODEUNIT *initial_instr = instr; int trace_length = 0; - int max_length = buffer_size; + // Leave space for possible trailing _EXIT_TRACE + int max_length = buffer_size-2; struct { PyFunctionObject *func; PyCodeObject *code; @@ -538,13 +575,16 @@ translate_bytecode_to_trace( PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); + ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); uint32_t target = 0; top: // Jump here after _PUSH_FRAME or likely branches for (;;) { target = INSTR_IP(instr, code); - RESERVE_RAW(2, "epilogue"); // Always need space for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE + RESERVE_RAW(2, "_CHECK_VALIDITY_AND_SET_IP"); ADD_TO_TRACE(_CHECK_VALIDITY_AND_SET_IP, 0, (uintptr_t)instr, target); + // Need space for _DEOPT + max_length--; uint32_t opcode = instr->op.code; uint32_t oparg = instr->op.arg; @@ -582,13 +622,22 @@ top: // Jump here after _PUSH_FRAME or likely branches continue; } else { - if (OPCODE_HAS_DEOPT(opcode)) { + if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { opcode = _PyOpcode_Deopt[opcode]; } + assert(!OPCODE_HAS_EXIT(opcode)); assert(!OPCODE_HAS_DEOPT(opcode)); } } + if (OPCODE_HAS_EXIT(opcode)) { + // Make space for exit code + max_length--; + } + if (OPCODE_HAS_ERROR(opcode)) { + // Make space for error code + max_length--; + } switch (opcode) { case POP_JUMP_IF_NONE: case POP_JUMP_IF_NOT_NONE: @@ -624,10 +673,10 @@ top: // Jump here after _PUSH_FRAME or likely branches DPRINTF(2, "Jump likely (%04x = %d bits), continue at byte offset %d\n", instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code)); instr = target_instr; - ADD_TO_TRACE(uopcode, max_length, 0, INSTR_IP(next_instr, code)); + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(next_instr, code)); goto top; } - ADD_TO_TRACE(uopcode, max_length, 0, INSTR_IP(target_instr, code)); + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, code)); break; } @@ -849,7 +898,9 @@ done: progress_needed ? "no progress" : "too short"); return 0; } - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + if (trace[trace_length-1].opcode != _JUMP_TO_TOP) { + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + } DPRINTF(1, "Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n", PyUnicode_AsUTF8(code->co_qualname), @@ -857,8 +908,8 @@ done: code->co_firstlineno, 2 * INSTR_IP(initial_instr, code), trace_length); - OPT_HIST(trace_length + buffer_size - max_length, trace_length_hist); - return 1; + OPT_HIST(trace_length, trace_length_hist); + return trace_length; } #undef RESERVE @@ -871,43 +922,86 @@ done: #define SET_BIT(array, bit) (array[(bit)>>5] |= (1<<((bit)&31))) #define BIT_IS_SET(array, bit) (array[(bit)>>5] & (1<<((bit)&31))) -/* Count the number of used uops, and mark them in the bit vector `used`. - * This can be done in a single pass using simple reachability analysis, - * as there are no backward jumps. - * NOPs are excluded from the count. +/* Count the number of unused uops and exits */ static int -compute_used(_PyUOpInstruction *buffer, uint32_t *used, int *exit_count_ptr) +count_exits(_PyUOpInstruction *buffer, int length) { - int count = 0; int exit_count = 0; - SET_BIT(used, 0); - for (int i = 0; i < UOP_MAX_TRACE_LENGTH; i++) { - if (!BIT_IS_SET(used, i)) { - continue; - } - count++; + for (int i = 0; i < length; i++) { int opcode = buffer[i].opcode; - if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { + if (opcode == _SIDE_EXIT) { exit_count++; } - if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE) { - continue; + } + return exit_count; +} + +static void make_exit(_PyUOpInstruction *inst, int opcode, int target) +{ + inst->opcode = opcode; + inst->oparg = 0; + inst->format = UOP_FORMAT_TARGET; + inst->target = target; +} + +/* Convert implicit exits, errors and deopts + * into explicit ones. */ +static int +prepare_for_execution(_PyUOpInstruction *buffer, int length) +{ + int32_t current_jump = -1; + int32_t current_jump_target = -1; + int32_t current_error = -1; + int32_t current_error_target = -1; + int32_t current_popped = -1; + /* Leaving in NOPs slows down the interpreter and messes up the stats */ + _PyUOpInstruction *copy_to = &buffer[0]; + for (int i = 0; i < length; i++) { + _PyUOpInstruction *inst = &buffer[i]; + if (inst->opcode != _NOP) { + if (copy_to != inst) { + *copy_to = *inst; + } + copy_to++; } - /* All other micro-ops fall through, so i+1 is reachable */ - SET_BIT(used, i+1); - assert(opcode <= MAX_UOP_ID); - if (_PyUop_Flags[opcode] & HAS_JUMP_FLAG) { - /* Mark target as reachable */ - SET_BIT(used, buffer[i].oparg); + } + length = (int)(copy_to - buffer); + int next_spare = length; + for (int i = 0; i < length; i++) { + _PyUOpInstruction *inst = &buffer[i]; + int opcode = inst->opcode; + int32_t target = (int32_t)uop_get_target(inst); + if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { + if (target != current_jump_target) { + uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? _SIDE_EXIT : _DEOPT; + make_exit(&buffer[next_spare], exit_op, target); + current_jump_target = target; + current_jump = next_spare; + next_spare++; + } + buffer[i].jump_target = current_jump; + buffer[i].format = UOP_FORMAT_JUMP; } - if (opcode == NOP) { - count--; - UNSET_BIT(used, i); + if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { + int popped = (_PyUop_Flags[opcode] & HAS_ERROR_NO_POP_FLAG) ? + 0 : _PyUop_num_popped(opcode, inst->oparg); + if (target != current_error_target || popped != current_popped) { + current_popped = popped; + current_error = next_spare; + current_error_target = target; + make_exit(&buffer[next_spare], _ERROR_POP_N, 0); + buffer[next_spare].oparg = popped; + next_spare++; + } + buffer[i].error_target = current_error; + if (buffer[i].format == UOP_FORMAT_TARGET) { + buffer[i].format = UOP_FORMAT_JUMP; + buffer[i].jump_target = 0; + } } } - *exit_count_ptr = exit_count; - return count; + return next_spare; } /* Executor side exits */ @@ -926,61 +1020,118 @@ allocate_executor(int exit_count, int length) return res; } +#ifdef Py_DEBUG + +#define CHECK(PRED) \ +if (!(PRED)) { \ + printf(#PRED " at %d\n", i); \ + assert(0); \ +} + +static int +target_unused(int opcode) +{ + return (_PyUop_Flags[opcode] & (HAS_ERROR_FLAG | HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) == 0; +} + +static void +sanity_check(_PyExecutorObject *executor) +{ + for (uint32_t i = 0; i < executor->exit_count; i++) { + _PyExitData *exit = &executor->exits[i]; + CHECK(exit->target < (1 << 25)); + } + bool ended = false; + uint32_t i = 0; + CHECK(executor->trace[0].opcode == _START_EXECUTOR || executor->trace[0].opcode == _COLD_EXIT); + for (; i < executor->code_size; i++) { + const _PyUOpInstruction *inst = &executor->trace[i]; + uint16_t opcode = inst->opcode; + CHECK(opcode <= MAX_UOP_ID); + CHECK(_PyOpcode_uop_name[opcode] != NULL); + switch(inst->format) { + case UOP_FORMAT_TARGET: + CHECK(target_unused(opcode)); + break; + case UOP_FORMAT_EXIT: + CHECK(opcode == _SIDE_EXIT); + CHECK(inst->exit_index < executor->exit_count); + break; + case UOP_FORMAT_JUMP: + CHECK(inst->jump_target < executor->code_size); + break; + case UOP_FORMAT_UNUSED: + CHECK(0); + break; + } + if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { + CHECK(inst->format == UOP_FORMAT_JUMP); + CHECK(inst->error_target < executor->code_size); + } + if (opcode == _JUMP_TO_TOP || opcode == _EXIT_TRACE || opcode == _COLD_EXIT) { + ended = true; + i++; + break; + } + } + CHECK(ended); + for (; i < executor->code_size; i++) { + const _PyUOpInstruction *inst = &executor->trace[i]; + uint16_t opcode = inst->opcode; + CHECK( + opcode == _DEOPT || + opcode == _SIDE_EXIT || + opcode == _ERROR_POP_N); + if (opcode == _SIDE_EXIT) { + CHECK(inst->format == UOP_FORMAT_EXIT); + } + } +} + +#undef CHECK +#endif + /* Makes an executor from a buffer of uops. * Account for the buffer having gaps and NOPs by computing a "used" * bit vector and only copying the used uops. Here "used" means reachable * and not a NOP. */ static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *dependencies) +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies) { - uint32_t used[(UOP_MAX_TRACE_LENGTH + 31)/32] = { 0 }; - int exit_count; - int length = compute_used(buffer, used, &exit_count); - length += 1; // For _START_EXECUTOR + int exit_count = count_exits(buffer, length); _PyExecutorObject *executor = allocate_executor(exit_count, length); if (executor == NULL) { return NULL; } - OPT_HIST(length, optimized_trace_length_hist); /* Initialize exits */ + assert(exit_count < COLD_EXIT_COUNT); for (int i = 0; i < exit_count; i++) { executor->exits[i].executor = &COLD_EXITS[i]; executor->exits[i].temperature = 0; } int next_exit = exit_count-1; - _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length-1]; - /* Scan backwards, so that we see the destinations of jumps before the jumps themselves. */ - for (int i = UOP_MAX_TRACE_LENGTH-1; i >= 0; i--) { - if (!BIT_IS_SET(used, i)) { - continue; - } - *dest = buffer[i]; + _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; + assert(buffer[0].opcode == _START_EXECUTOR); + buffer[0].operand = (uint64_t)executor; + for (int i = length-1; i >= 0; i--) { int opcode = buffer[i].opcode; - if (opcode == _POP_JUMP_IF_FALSE || - opcode == _POP_JUMP_IF_TRUE) - { - /* The oparg of the target will already have been set to its new offset */ - int oparg = dest->oparg; - dest->oparg = buffer[oparg].oparg; - } - if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { + dest--; + *dest = buffer[i]; + assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); + if (opcode == _SIDE_EXIT) { executor->exits[next_exit].target = buffer[i].target; dest->exit_index = next_exit; + dest->format = UOP_FORMAT_EXIT; next_exit--; } - /* Set the oparg to be the destination offset, - * so that we can set the oparg of earlier jumps correctly. */ - buffer[i].oparg = (uint16_t)(dest - executor->trace); - dest--; } assert(next_exit == -1); assert(dest == executor->trace); - dest->opcode = _START_EXECUTOR; + assert(dest->opcode == _START_EXECUTOR); dest->oparg = 0; dest->target = 0; - dest->operand = (uintptr_t)executor; _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -996,6 +1147,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende printf("\n"); } } + sanity_check(executor); #endif #ifdef _Py_JIT executor->jit_code = NULL; @@ -1024,6 +1176,9 @@ init_cold_exit_executor(_PyExecutorObject *executor, int oparg) for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { assert(executor->vm_data.bloom.bits[i] == 0); } +#ifdef Py_DEBUG + sanity_check(executor); +#endif #ifdef _Py_JIT executor->jit_code = NULL; executor->jit_size = 0; @@ -1034,6 +1189,28 @@ init_cold_exit_executor(_PyExecutorObject *executor, int oparg) return 0; } +#ifdef Py_STATS +/* Returns the effective trace length. + * Ignores NOPs and trailing exit and error handling.*/ +int effective_trace_length(_PyUOpInstruction *buffer, int length) +{ + int nop_count = 0; + for (int i = 0; i < length; i++) { + int opcode = buffer[i].opcode; + if (opcode == _NOP) { + nop_count++; + } + if (opcode == _EXIT_TRACE || + opcode == _JUMP_TO_TOP || + opcode == _COLD_EXIT) { + return i+1-nop_count; + } + } + Py_FatalError("No terminating instruction"); + Py_UNREACHABLE(); +} +#endif + static int uop_optimize( _PyOptimizerObject *self, @@ -1046,24 +1223,26 @@ uop_optimize( _Py_BloomFilter_Init(&dependencies); _PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH]; OPT_STAT_INC(attempts); - int err = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies); - if (err <= 0) { + int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies); + if (length <= 0) { // Error or nothing translated - return err; + return length; } + assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); if (env_var == NULL || *env_var == '\0' || *env_var > '0') { - err = _Py_uop_analyze_and_optimize(frame, buffer, - UOP_MAX_TRACE_LENGTH, + length = _Py_uop_analyze_and_optimize(frame, buffer, + length, curr_stackentries, &dependencies); - if (err <= 0) { - return err; + if (length <= 0) { + return length; } } - assert(err == 1); + assert(length < UOP_MAX_TRACE_LENGTH); + assert(length >= 1); /* Fix up */ - for (int pc = 0; pc < UOP_MAX_TRACE_LENGTH; pc++) { + for (int pc = 0; pc < length; pc++) { int opcode = buffer[pc].opcode; int oparg = buffer[pc].oparg; if (_PyUop_Flags[opcode] & HAS_OPARG_AND_1_FLAG) { @@ -1078,10 +1257,14 @@ uop_optimize( assert(_PyOpcode_uop_name[buffer[pc].opcode]); assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0); } - _PyExecutorObject *executor = make_executor_from_uops(buffer, &dependencies); + OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); + length = prepare_for_execution(buffer, length); + assert(length <= UOP_MAX_TRACE_LENGTH); + _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); if (executor == NULL) { return -1; } + assert(length <= UOP_MAX_TRACE_LENGTH); *exec_ptr = executor; return 1; } @@ -1156,12 +1339,14 @@ counter_optimize( return 0; } _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg; - _PyUOpInstruction buffer[3] = { + _PyUOpInstruction buffer[5] = { + { .opcode = _START_EXECUTOR }, { .opcode = _LOAD_CONST_INLINE_BORROW, .operand = (uintptr_t)self }, { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, - { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)) } + { .opcode = _EXIT_TRACE, .jump_target = 4, .format=UOP_FORMAT_JUMP }, + { .opcode = _SIDE_EXIT, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET } }; - _PyExecutorObject *executor = make_executor_from_uops(buffer, &EMPTY_FILTER); + _PyExecutorObject *executor = make_executor_from_uops(buffer, 5, &EMPTY_FILTER); if (executor == NULL) { return -1; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 95924a5..6f553f8 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -387,9 +387,9 @@ optimize_uops( ctx->curr_frame_depth++; ctx->frame = frame; - for (_PyUOpInstruction *this_instr = trace; - this_instr < trace + trace_len && !op_is_end(this_instr->opcode); - this_instr++) { + _PyUOpInstruction *this_instr = NULL; + for (int i = 0; i < trace_len; i++) { + this_instr = &trace[i]; int oparg = this_instr->oparg; opcode = this_instr->opcode; @@ -416,9 +416,8 @@ optimize_uops( ctx->frame->stack_pointer = stack_pointer; assert(STACK_LEVEL() >= 0); } - _Py_uop_abstractcontext_fini(ctx); - return 1; + return trace_len; out_of_space: DPRINTF(3, "\n"); @@ -447,11 +446,11 @@ done: /* Cannot optimize further, but there would be no benefit * in retrying later */ _Py_uop_abstractcontext_fini(ctx); - return 1; + return trace_len; } -static void +static int remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) { /* Remove _SET_IP and _CHECK_VALIDITY where possible. @@ -506,7 +505,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } case _JUMP_TO_TOP: case _EXIT_TRACE: - return; + return pc + 1; default: { bool needs_ip = false; @@ -530,6 +529,8 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } } } + Py_FatalError("No terminating instruction"); + Py_UNREACHABLE(); } static void @@ -582,43 +583,36 @@ peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_s // 0 - failure, no error raised, just fall back to Tier 1 // -1 - failure, and raise error -// 1 - optimizer success +// > 0 - length of optimized trace int _Py_uop_analyze_and_optimize( _PyInterpreterFrame *frame, _PyUOpInstruction *buffer, - int buffer_size, + int length, int curr_stacklen, _PyBloomFilter *dependencies ) { OPT_STAT_INC(optimizer_attempts); - int err = remove_globals(frame, buffer, buffer_size, dependencies); - if (err == 0) { - goto not_ready; - } - if (err < 0) { - goto error; + int err = remove_globals(frame, buffer, length, dependencies); + if (err <= 0) { + return err; } - peephole_opt(frame, buffer, buffer_size); + peephole_opt(frame, buffer, length); - err = optimize_uops( + length = optimize_uops( _PyFrame_GetCode(frame), buffer, - buffer_size, curr_stacklen, dependencies); + length, curr_stacklen, dependencies); - if (err == 0) { - goto not_ready; + if (length <= 0) { + return length; } - assert(err == 1); - remove_unneeded_uops(buffer, buffer_size); + length = remove_unneeded_uops(buffer, length); + assert(length > 0); OPT_STAT_INC(optimizer_successes); - return 1; -not_ready: - return 0; -error: - return -1; + return length; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 6aeea51..df73cc0 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -769,14 +769,7 @@ break; } - case _LOAD_NAME: { - _Py_UopsSymbol *v; - v = sym_new_not_null(ctx); - if (v == NULL) goto out_of_space; - stack_pointer[0] = v; - stack_pointer += 1; - break; - } + /* _LOAD_NAME is not a viable micro-op for tier 2 */ case _LOAD_GLOBAL: { _Py_UopsSymbol *res; @@ -900,14 +893,7 @@ break; } - case _BUILD_SET: { - _Py_UopsSymbol *set; - set = sym_new_not_null(ctx); - if (set == NULL) goto out_of_space; - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - break; - } + /* _BUILD_SET is not a viable micro-op for tier 2 */ case _BUILD_MAP: { _Py_UopsSymbol *map; @@ -1408,31 +1394,9 @@ /* _FOR_ITER_GEN is not a viable micro-op for tier 2 */ - case _BEFORE_ASYNC_WITH: { - _Py_UopsSymbol *exit; - _Py_UopsSymbol *res; - exit = sym_new_not_null(ctx); - if (exit == NULL) goto out_of_space; - res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - break; - } + /* _BEFORE_ASYNC_WITH is not a viable micro-op for tier 2 */ - case _BEFORE_WITH: { - _Py_UopsSymbol *exit; - _Py_UopsSymbol *res; - exit = sym_new_not_null(ctx); - if (exit == NULL) goto out_of_space; - res = sym_new_not_null(ctx); - if (res == NULL) goto out_of_space; - stack_pointer[-1] = exit; - stack_pointer[0] = res; - stack_pointer += 1; - break; - } + /* _BEFORE_WITH is not a viable micro-op for tier 2 */ case _WITH_EXCEPT_START: { _Py_UopsSymbol *res; @@ -2029,3 +1993,16 @@ break; } + case _DEOPT: { + break; + } + + case _SIDE_EXIT: { + break; + } + + case _ERROR_POP_N: { + stack_pointer += -oparg; + break; + } + diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 27e6ba2..2329205 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -8,7 +8,8 @@ from typing import Optional @dataclass class Properties: escapes: bool - infallible: bool + error_with_pop: bool + error_without_pop: bool deopts: bool oparg: bool jumps: bool @@ -37,7 +38,8 @@ class Properties: def from_list(properties: list["Properties"]) -> "Properties": return Properties( escapes=any(p.escapes for p in properties), - infallible=all(p.infallible for p in properties), + error_with_pop=any(p.error_with_pop for p in properties), + error_without_pop=any(p.error_without_pop for p in properties), deopts=any(p.deopts for p in properties), oparg=any(p.oparg for p in properties), jumps=any(p.jumps for p in properties), @@ -55,10 +57,16 @@ class Properties: passthrough=all(p.passthrough for p in properties), ) + @property + def infallible(self) -> bool: + return not self.error_with_pop and not self.error_without_pop + + SKIP_PROPERTIES = Properties( escapes=False, - infallible=True, + error_with_pop=False, + error_without_pop=False, deopts=False, oparg=False, jumps=False, @@ -157,20 +165,32 @@ class Uop: self._size = sum(c.size for c in self.caches) return self._size - def is_viable(self) -> bool: + def why_not_viable(self) -> str | None: if self.name == "_SAVE_RETURN_OFFSET": - return True # Adjusts next_instr, but only in tier 1 code - if self.properties.needs_this: - return False + return None # Adjusts next_instr, but only in tier 1 code if "INSTRUMENTED" in self.name: - return False + return "is instrumented" if "replaced" in self.annotations: - return False + return "is replaced" if self.name in ("INTERPRETER_EXIT", "JUMP_BACKWARD"): - return False + return "has tier 1 control flow" + if self.properties.needs_this: + return "uses the 'this_instr' variable" if len([c for c in self.caches if c.name != "unused"]) > 1: - return False - return True + return "has unused cache entries" + if self.properties.error_with_pop and self.properties.error_without_pop: + return "has both popping and not-popping errors" + if self.properties.eval_breaker: + if self.properties.error_with_pop or self.properties.error_without_pop: + return "has error handling and eval-breaker check" + if self.properties.side_exit: + return "exits and eval-breaker check" + if self.properties.deopts: + return "deopts and eval-breaker check" + return None + + def is_viable(self) -> bool: + return self.why_not_viable() is None def is_super(self) -> bool: for tkn in self.body: @@ -320,10 +340,17 @@ def tier_variable(node: parser.InstDef) -> int | None: return int(token.text[-1]) return None -def is_infallible(op: parser.InstDef) -> bool: - return not ( +def has_error_with_pop(op: parser.InstDef) -> bool: + return ( variable_used(op, "ERROR_IF") - or variable_used(op, "error") + or variable_used(op, "pop_1_error") + or variable_used(op, "exception_unwind") + or variable_used(op, "resume_with_error") + ) + +def has_error_without_pop(op: parser.InstDef) -> bool: + return ( + variable_used(op, "ERROR_NO_POP") or variable_used(op, "pop_1_error") or variable_used(op, "exception_unwind") or variable_used(op, "resume_with_error") @@ -507,12 +534,15 @@ def compute_properties(op: parser.InstDef) -> Properties: tkn.column, op.name, ) - infallible = is_infallible(op) + error_with_pop = has_error_with_pop(op) + error_without_pop = has_error_without_pop(op) + infallible = not error_with_pop and not error_without_pop passthrough = stack_effect_only_peeks(op) and infallible return Properties( escapes=makes_escaping_api_call(op), - infallible=infallible, - deopts=deopts_if or exits_if, + error_with_pop=error_with_pop, + error_without_pop=error_without_pop, + deopts=deopts_if, side_exit=exits_if, oparg=variable_used(op, "oparg"), jumps=variable_used(op, "JUMPBY"), diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 0b4b99c..0addcf0 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -99,6 +99,20 @@ def replace_error( out.emit(close) +def replace_error_no_pop( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) # LPAREN + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit_at("goto error;", tkn) + + def replace_decrefs( out: CWriter, tkn: Token, @@ -160,6 +174,7 @@ REPLACEMENT_FUNCTIONS = { "EXIT_IF": replace_deopt, "DEOPT_IF": replace_deopt, "ERROR_IF": replace_error, + "ERROR_NO_POP": replace_error_no_pop, "DECREF_INPUTS": replace_decrefs, "CHECK_EVAL_BREAKER": replace_check_eval_breaker, "SYNC_SP": replace_sync_sp, @@ -213,6 +228,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_EXIT_FLAG") if not p.infallible: flags.append("HAS_ERROR_FLAG") + if p.error_without_pop: + flags.append("HAS_ERROR_NO_POP_FLAG") if p.escapes: flags.append("HAS_ESCAPES_FLAG") if p.pure: diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index ab59783..04fecb2 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -54,6 +54,7 @@ FLAGS = [ "PURE", "PASSTHROUGH", "OPARG_AND_1", + "ERROR_NO_POP", ] diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index d8eed10..114d28e 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -72,21 +72,21 @@ def tier2_replace_error( label = next(tkn_iter).text next(tkn_iter) # RPAREN next(tkn_iter) # Semi colon - out.emit(") ") - c_offset = stack.peek_offset.to_c() - try: - offset = -int(c_offset) - close = ";\n" - except ValueError: - offset = None - out.emit(f"{{ stack_pointer += {c_offset}; ") - close = "; }\n" - out.emit("goto ") - if offset: - out.emit(f"pop_{offset}_") - out.emit(label + "_tier_two") - out.emit(close) + out.emit(") JUMP_TO_ERROR();\n") + +def tier2_replace_error_no_pop( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + stack: Stack, + inst: Instruction | None, +) -> None: + next(tkn_iter) # LPAREN + next(tkn_iter) # RPAREN + next(tkn_iter) # Semi colon + out.emit_at("JUMP_TO_ERROR();", tkn) def tier2_replace_deopt( out: CWriter, @@ -100,7 +100,7 @@ def tier2_replace_deopt( out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon - out.emit(") goto deoptimize;\n") + out.emit(") JUMP_TO_JUMP_TARGET();\n") def tier2_replace_exit_if( @@ -115,7 +115,7 @@ def tier2_replace_exit_if( out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon - out.emit(") goto side_exit;\n") + out.emit(") JUMP_TO_JUMP_TARGET();\n") def tier2_replace_oparg( @@ -141,6 +141,7 @@ def tier2_replace_oparg( TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy() TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error +TIER2_REPLACEMENT_FUNCTIONS["ERROR_NO_POP"] = tier2_replace_error_no_pop TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt TIER2_REPLACEMENT_FUNCTIONS["oparg"] = tier2_replace_oparg TIER2_REPLACEMENT_FUNCTIONS["EXIT_IF"] = tier2_replace_exit_if @@ -201,8 +202,9 @@ def generate_tier2( continue if uop.is_super(): continue - if not uop.is_viable(): - out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n") + why_not_viable = uop.why_not_viable() + if why_not_viable is not None: + out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 because it {why_not_viable} */\n\n") continue out.emit(f"case {uop.name}: {{\n") declare_variables(uop, out) diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 72eed30..7b3325a 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -15,10 +15,10 @@ from generators_common import ( write_header, cflags, ) +from stack import Stack from cwriter import CWriter from typing import TextIO - DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_metadata.h" @@ -26,6 +26,7 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") out.emit("extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1];\n") out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") + out.emit("extern int _PyUop_num_popped(int opcode, int oparg);\n\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") for uop in analysis.uops.values(): @@ -44,6 +45,20 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: if uop.is_viable() and uop.properties.tier != 1: out.emit(f'[{uop.name}] = "{uop.name}",\n') out.emit("};\n") + out.emit("int _PyUop_num_popped(int opcode, int oparg)\n{\n") + out.emit("switch(opcode) {\n") + for uop in analysis.uops.values(): + if uop.is_viable() and uop.properties.tier != 1: + stack = Stack() + for var in reversed(uop.stack.inputs): + stack.pop(var) + popped = (-stack.base_offset).to_c() + out.emit(f"case {uop.name}:\n") + out.emit(f" return {popped};\n") + out.emit("default:\n") + out.emit(" return -1;\n") + out.emit("}\n") + out.emit("}\n\n") out.emit("#endif // NEED_OPCODE_METADATA\n\n") diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 05c4ce8..601ea0b 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -31,6 +31,12 @@ class HoleValue(enum.Enum): OPERAND = enum.auto() # The current uop's target (exposed as _JIT_TARGET): TARGET = enum.auto() + # The base address of the machine code for the jump target (exposed as _JIT_JUMP_TARGET): + JUMP_TARGET = enum.auto() + # The base address of the machine code for the error jump target (exposed as _JIT_ERROR_TARGET): + ERROR_TARGET = enum.auto() + # The index of the exit to be jumped through (exposed as _JIT_EXIT_INDEX): + EXIT_INDEX = enum.auto() # The base address of the machine code for the first uop (exposed as _JIT_TOP): TOP = enum.auto() # A hardcoded value of zero (used for symbol lookups): diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 504e6c8..9b4fc2a 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -64,9 +64,17 @@ do { \ TYPE NAME = (TYPE)(uint64_t)&ALIAS; #define PATCH_JUMP(ALIAS) \ +do { \ PyAPI_DATA(void) ALIAS; \ __attribute__((musttail)) \ - return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); + return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); \ +} while (0) + +#undef JUMP_TO_JUMP_TARGET +#define JUMP_TO_JUMP_TARGET() PATCH_JUMP(_JIT_JUMP_TARGET) + +#undef JUMP_TO_ERROR +#define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET) _Py_CODEUNIT * _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate) @@ -79,6 +87,7 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) PATCH_VALUE(uint32_t, _target, _JIT_TARGET) + PATCH_VALUE(uint16_t, _exit_index, _JIT_EXIT_INDEX) // The actual instruction definitions (only one will be used): if (opcode == _JUMP_TO_TOP) { CHECK_EVAL_BREAKER(); @@ -91,28 +100,16 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * } PATCH_JUMP(_JIT_CONTINUE); // Labels that the instruction implementations expect to exist: -unbound_local_error_tier_two: - _PyEval_FormatExcCheckArg( - tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)); - goto error_tier_two; -pop_4_error_tier_two: - STACK_SHRINK(1); -pop_3_error_tier_two: - STACK_SHRINK(1); -pop_2_error_tier_two: - STACK_SHRINK(1); -pop_1_error_tier_two: - STACK_SHRINK(1); + error_tier_two: tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_ONE(NULL); -deoptimize: +exit_to_tier1: tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target); -side_exit: +exit_to_trace: { - _PyExitData *exit = ¤t_executor->exits[_target]; + _PyExitData *exit = ¤t_executor->exits[_exit_index]; Py_INCREF(exit->executor); tstate->previous_executor = (PyObject *)current_executor; GOTO_TIER_TWO(exit->executor); |