diff options
author | Irit Katriel <1055913+iritkatriel@users.noreply.github.com> | 2022-09-01 20:36:47 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-09-01 20:36:47 (GMT) |
commit | 4c72517cada147b215cf30ff8dac70ea0f08f1e0 (patch) | |
tree | 1012522409ed719a8d8ba0b5eab4e5f3bce77bd6 /Python | |
parent | a91f25577c71ab8797a4b42f22c43bbaffc2604d (diff) | |
download | cpython-4c72517cada147b215cf30ff8dac70ea0f08f1e0.zip cpython-4c72517cada147b215cf30ff8dac70ea0f08f1e0.tar.gz cpython-4c72517cada147b215cf30ff8dac70ea0f08f1e0.tar.bz2 |
gh-93554: Conditional jump opcodes only jump forward (GH-96318)
Diffstat (limited to 'Python')
-rw-r--r-- | Python/ceval.c | 147 | ||||
-rw-r--r-- | Python/compile.c | 154 | ||||
-rw-r--r-- | Python/opcode_targets.h | 16 | ||||
-rw-r--r-- | Python/specialize.c | 15 |
4 files changed, 112 insertions, 220 deletions
diff --git a/Python/ceval.c b/Python/ceval.c index c61ccd7..76a8118 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3307,7 +3307,7 @@ handle_eval_breaker: TARGET(COMPARE_OP_FLOAT_JUMP) { assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (float ? float) + POP_JUMP_(direction)_IF_(true/false) + // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; int when_to_jump_mask = cache->mask; PyObject *right = TOP(); @@ -3325,23 +3325,12 @@ handle_eval_breaker: STACK_SHRINK(2); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - assert(opcode == POP_JUMP_FORWARD_IF_FALSE || - opcode == POP_JUMP_BACKWARD_IF_FALSE || - opcode == POP_JUMP_FORWARD_IF_TRUE || - opcode == POP_JUMP_BACKWARD_IF_TRUE); - int jump = (9 << (sign + 1)) & when_to_jump_mask; + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); + int jump = (1 << (sign + 1)) & when_to_jump_mask; if (!jump) { next_instr++; } - else if (jump >= 8) { - assert(opcode == POP_JUMP_BACKWARD_IF_TRUE || - opcode == POP_JUMP_BACKWARD_IF_FALSE); - JUMPBY(1 - oparg); - CHECK_EVAL_BREAKER(); - } else { - assert(opcode == POP_JUMP_FORWARD_IF_TRUE || - opcode == POP_JUMP_FORWARD_IF_FALSE); JUMPBY(1 + oparg); } NOTRACE_DISPATCH(); @@ -3349,7 +3338,7 @@ handle_eval_breaker: TARGET(COMPARE_OP_INT_JUMP) { assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (int ? int) + POP_JUMP_(direction)_IF_(true/false) + // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; int when_to_jump_mask = cache->mask; PyObject *right = TOP(); @@ -3368,23 +3357,12 @@ handle_eval_breaker: STACK_SHRINK(2); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - assert(opcode == POP_JUMP_FORWARD_IF_FALSE || - opcode == POP_JUMP_BACKWARD_IF_FALSE || - opcode == POP_JUMP_FORWARD_IF_TRUE || - opcode == POP_JUMP_BACKWARD_IF_TRUE); - int jump = (9 << (sign + 1)) & when_to_jump_mask; + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); + int jump = (1 << (sign + 1)) & when_to_jump_mask; if (!jump) { next_instr++; } - else if (jump >= 8) { - assert(opcode == POP_JUMP_BACKWARD_IF_TRUE || - opcode == POP_JUMP_BACKWARD_IF_FALSE); - JUMPBY(1 - oparg); - CHECK_EVAL_BREAKER(); - } else { - assert(opcode == POP_JUMP_FORWARD_IF_TRUE || - opcode == POP_JUMP_FORWARD_IF_FALSE); JUMPBY(1 + oparg); } NOTRACE_DISPATCH(); @@ -3392,9 +3370,9 @@ handle_eval_breaker: TARGET(COMPARE_OP_STR_JUMP) { assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_(direction)_IF_(true/false) + // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - int when_to_jump_mask = cache->mask; + int invert = cache->mask; PyObject *right = TOP(); PyObject *left = SECOND(); DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); @@ -3407,28 +3385,17 @@ handle_eval_breaker: assert(oparg == Py_EQ || oparg == Py_NE); JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); NEXTOPARG(); - assert(opcode == POP_JUMP_FORWARD_IF_FALSE || - opcode == POP_JUMP_BACKWARD_IF_FALSE || - opcode == POP_JUMP_FORWARD_IF_TRUE || - opcode == POP_JUMP_BACKWARD_IF_TRUE); + assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); STACK_SHRINK(2); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); assert(res == 0 || res == 1); - int sign = 1 - res; - int jump = (9 << (sign + 1)) & when_to_jump_mask; + assert(invert == 0 || invert == 1); + int jump = res ^ invert; if (!jump) { next_instr++; } - else if (jump >= 8) { - assert(opcode == POP_JUMP_BACKWARD_IF_TRUE || - opcode == POP_JUMP_BACKWARD_IF_FALSE); - JUMPBY(1 - oparg); - CHECK_EVAL_BREAKER(); - } else { - assert(opcode == POP_JUMP_FORWARD_IF_TRUE || - opcode == POP_JUMP_FORWARD_IF_FALSE); JUMPBY(1 + oparg); } NOTRACE_DISPATCH(); @@ -3575,34 +3542,8 @@ handle_eval_breaker: JUMP_TO_INSTRUCTION(JUMP_BACKWARD_QUICK); } - TARGET(POP_JUMP_BACKWARD_IF_FALSE) { - PREDICTED(POP_JUMP_BACKWARD_IF_FALSE); - PyObject *cond = POP(); - if (Py_IsTrue(cond)) { - _Py_DECREF_NO_DEALLOC(cond); - DISPATCH(); - } - if (Py_IsFalse(cond)) { - _Py_DECREF_NO_DEALLOC(cond); - JUMPBY(-oparg); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - int err = PyObject_IsTrue(cond); - Py_DECREF(cond); - if (err > 0) - ; - else if (err == 0) { - JUMPBY(-oparg); - CHECK_EVAL_BREAKER(); - } - else - goto error; - DISPATCH(); - } - - TARGET(POP_JUMP_FORWARD_IF_FALSE) { - PREDICTED(POP_JUMP_FORWARD_IF_FALSE); + TARGET(POP_JUMP_IF_FALSE) { + PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = POP(); if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); @@ -3625,32 +3566,7 @@ handle_eval_breaker: DISPATCH(); } - TARGET(POP_JUMP_BACKWARD_IF_TRUE) { - PyObject *cond = POP(); - if (Py_IsFalse(cond)) { - _Py_DECREF_NO_DEALLOC(cond); - DISPATCH(); - } - if (Py_IsTrue(cond)) { - _Py_DECREF_NO_DEALLOC(cond); - JUMPBY(-oparg); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - int err = PyObject_IsTrue(cond); - Py_DECREF(cond); - if (err > 0) { - JUMPBY(-oparg); - CHECK_EVAL_BREAKER(); - } - else if (err == 0) - ; - else - goto error; - DISPATCH(); - } - - TARGET(POP_JUMP_FORWARD_IF_TRUE) { + TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = POP(); if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); @@ -3673,19 +3589,7 @@ handle_eval_breaker: DISPATCH(); } - TARGET(POP_JUMP_BACKWARD_IF_NOT_NONE) { - PyObject *value = POP(); - if (!Py_IsNone(value)) { - Py_DECREF(value); - JUMPBY(-oparg); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - _Py_DECREF_NO_DEALLOC(value); - DISPATCH(); - } - - TARGET(POP_JUMP_FORWARD_IF_NOT_NONE) { + TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = POP(); if (!Py_IsNone(value)) { JUMPBY(oparg); @@ -3694,20 +3598,7 @@ handle_eval_breaker: DISPATCH(); } - TARGET(POP_JUMP_BACKWARD_IF_NONE) { - PyObject *value = POP(); - if (Py_IsNone(value)) { - _Py_DECREF_NO_DEALLOC(value); - JUMPBY(-oparg); - CHECK_EVAL_BREAKER(); - } - else { - Py_DECREF(value); - } - DISPATCH(); - } - - TARGET(POP_JUMP_FORWARD_IF_NONE) { + TARGET(POP_JUMP_IF_NONE) { PyObject *value = POP(); if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); @@ -3834,8 +3725,7 @@ handle_eval_breaker: PyObject *res = match ? Py_True : Py_False; Py_INCREF(res); PUSH(res); - PREDICT(POP_JUMP_FORWARD_IF_FALSE); - PREDICT(POP_JUMP_BACKWARD_IF_FALSE); + PREDICT(POP_JUMP_IF_FALSE); DISPATCH(); } @@ -3845,8 +3735,7 @@ handle_eval_breaker: PyObject *res = match ? Py_True : Py_False; Py_INCREF(res); PUSH(res); - PREDICT(POP_JUMP_FORWARD_IF_FALSE); - PREDICT(POP_JUMP_BACKWARD_IF_FALSE); + PREDICT(POP_JUMP_IF_FALSE); DISPATCH(); } diff --git a/Python/compile.c b/Python/compile.c index 857fca4..862999d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -96,23 +96,11 @@ #define IS_ASSEMBLER_OPCODE(opcode) \ ((opcode) == JUMP_FORWARD || \ (opcode) == JUMP_BACKWARD || \ - (opcode) == JUMP_BACKWARD_NO_INTERRUPT || \ - (opcode) == POP_JUMP_FORWARD_IF_NONE || \ - (opcode) == POP_JUMP_BACKWARD_IF_NONE || \ - (opcode) == POP_JUMP_FORWARD_IF_NOT_NONE || \ - (opcode) == POP_JUMP_BACKWARD_IF_NOT_NONE || \ - (opcode) == POP_JUMP_FORWARD_IF_TRUE || \ - (opcode) == POP_JUMP_BACKWARD_IF_TRUE || \ - (opcode) == POP_JUMP_FORWARD_IF_FALSE || \ - (opcode) == POP_JUMP_BACKWARD_IF_FALSE) + (opcode) == JUMP_BACKWARD_NO_INTERRUPT) #define IS_BACKWARDS_JUMP_OPCODE(opcode) \ ((opcode) == JUMP_BACKWARD || \ - (opcode) == JUMP_BACKWARD_NO_INTERRUPT || \ - (opcode) == POP_JUMP_BACKWARD_IF_NONE || \ - (opcode) == POP_JUMP_BACKWARD_IF_NOT_NONE || \ - (opcode) == POP_JUMP_BACKWARD_IF_TRUE || \ - (opcode) == POP_JUMP_BACKWARD_IF_FALSE) + (opcode) == JUMP_BACKWARD_NO_INTERRUPT) #define IS_UNCONDITIONAL_JUMP_OPCODE(opcode) \ ((opcode) == JUMP || \ @@ -1146,17 +1134,9 @@ stack_effect(int opcode, int oparg, int jump) case JUMP_IF_FALSE_OR_POP: return jump ? 0 : -1; - case POP_JUMP_BACKWARD_IF_NONE: - case POP_JUMP_FORWARD_IF_NONE: case POP_JUMP_IF_NONE: - case POP_JUMP_BACKWARD_IF_NOT_NONE: - case POP_JUMP_FORWARD_IF_NOT_NONE: case POP_JUMP_IF_NOT_NONE: - case POP_JUMP_FORWARD_IF_FALSE: - case POP_JUMP_BACKWARD_IF_FALSE: case POP_JUMP_IF_FALSE: - case POP_JUMP_FORWARD_IF_TRUE: - case POP_JUMP_BACKWARD_IF_TRUE: case POP_JUMP_IF_TRUE: return -1; @@ -7747,63 +7727,91 @@ assemble_emit(struct assembler *a, struct instr *i) return 1; } -static void -normalize_jumps(basicblock *entryblock) +static int +normalize_jumps_in_block(cfg_builder *g, basicblock *b) { + struct instr *last = basicblock_last_instr(b); + if (last == NULL || !is_jump(last)) { + return 0; + } + assert(!IS_ASSEMBLER_OPCODE(last->i_opcode)); + bool is_forward = last->i_target->b_visited == 0; + switch(last->i_opcode) { + case JUMP: + last->i_opcode = is_forward ? JUMP_FORWARD : JUMP_BACKWARD; + return 0; + case JUMP_NO_INTERRUPT: + last->i_opcode = is_forward ? + JUMP_FORWARD : JUMP_BACKWARD_NO_INTERRUPT; + return 0; + } + int reversed_opcode = 0; + switch(last->i_opcode) { + case POP_JUMP_IF_NOT_NONE: + reversed_opcode = POP_JUMP_IF_NONE; + break; + case POP_JUMP_IF_NONE: + reversed_opcode = POP_JUMP_IF_NOT_NONE; + break; + case POP_JUMP_IF_FALSE: + reversed_opcode = POP_JUMP_IF_TRUE; + break; + case POP_JUMP_IF_TRUE: + reversed_opcode = POP_JUMP_IF_FALSE; + break; + case JUMP_IF_TRUE_OR_POP: + case JUMP_IF_FALSE_OR_POP: + if (!is_forward) { + /* As far as we can tell, the compiler never emits + * these jumps with a backwards target. If/when this + * exception is raised, we have found a use case for + * a backwards version of this jump (or to replace + * it with the sequence (COPY 1, POP_JUMP_IF_T/F, POP) + */ + PyErr_Format(PyExc_SystemError, + "unexpected %s jumping backwards", + last->i_opcode == JUMP_IF_TRUE_OR_POP ? + "JUMP_IF_TRUE_OR_POP" : "JUMP_IF_FALSE_OR_POP"); + } + return 0; + } + if (is_forward) { + return 0; + } + + /* transform 'conditional jump T' to + * 'reversed_jump b_next' followed by 'jump_backwards T' + */ + + basicblock *target = last->i_target; + basicblock *backwards_jump = cfg_builder_new_block(g); + if (backwards_jump == NULL) { + return -1; + } + basicblock_addop(backwards_jump, JUMP, target->b_label, NO_LOCATION); + backwards_jump->b_instr[0].i_target = target; + last->i_opcode = reversed_opcode; + last->i_target = b->b_next; + + backwards_jump->b_cold = b->b_cold; + backwards_jump->b_next = b->b_next; + b->b_next = backwards_jump; + return 0; +} + +static int +normalize_jumps(cfg_builder *g) { + basicblock *entryblock = g->g_entryblock; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { b->b_visited = 0; } for (basicblock *b = entryblock; b != NULL; b = b->b_next) { b->b_visited = 1; - struct instr *last = basicblock_last_instr(b); - if (last == NULL) { - continue; - } - assert(!IS_ASSEMBLER_OPCODE(last->i_opcode)); - if (is_jump(last)) { - bool is_forward = last->i_target->b_visited == 0; - switch(last->i_opcode) { - case JUMP: - last->i_opcode = is_forward ? JUMP_FORWARD : JUMP_BACKWARD; - break; - case JUMP_NO_INTERRUPT: - last->i_opcode = is_forward ? - JUMP_FORWARD : JUMP_BACKWARD_NO_INTERRUPT; - break; - case POP_JUMP_IF_NOT_NONE: - last->i_opcode = is_forward ? - POP_JUMP_FORWARD_IF_NOT_NONE : POP_JUMP_BACKWARD_IF_NOT_NONE; - break; - case POP_JUMP_IF_NONE: - last->i_opcode = is_forward ? - POP_JUMP_FORWARD_IF_NONE : POP_JUMP_BACKWARD_IF_NONE; - break; - case POP_JUMP_IF_FALSE: - last->i_opcode = is_forward ? - POP_JUMP_FORWARD_IF_FALSE : POP_JUMP_BACKWARD_IF_FALSE; - break; - case POP_JUMP_IF_TRUE: - last->i_opcode = is_forward ? - POP_JUMP_FORWARD_IF_TRUE : POP_JUMP_BACKWARD_IF_TRUE; - break; - case JUMP_IF_TRUE_OR_POP: - case JUMP_IF_FALSE_OR_POP: - if (!is_forward) { - /* As far as we can tell, the compiler never emits - * these jumps with a backwards target. If/when this - * exception is raised, we have found a use case for - * a backwards version of this jump (or to replace - * it with the sequence (COPY 1, POP_JUMP_IF_T/F, POP) - */ - PyErr_Format(PyExc_SystemError, - "unexpected %s jumping backwards", - last->i_opcode == JUMP_IF_TRUE_OR_POP ? - "JUMP_IF_TRUE_OR_POP" : "JUMP_IF_FALSE_OR_POP"); - } - break; - } + if (normalize_jumps_in_block(g, b) < 0) { + return -1; } } + return 0; } static void @@ -8638,7 +8646,9 @@ assemble(struct compiler *c, int addNone) } /* Order of basic blocks must have been determined by now */ - normalize_jumps(g->g_entryblock); + if (normalize_jumps(g) < 0) { + goto error; + } if (add_checks_for_loads_of_unknown_variables(g->g_entryblock, c) < 0) { goto error; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 7c782d1..c1ff367 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -113,8 +113,8 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, - &&TARGET_POP_JUMP_FORWARD_IF_FALSE, - &&TARGET_POP_JUMP_FORWARD_IF_TRUE, + &&TARGET_POP_JUMP_IF_FALSE, + &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, &&TARGET_IS_OP, &&TARGET_CONTAINS_OP, @@ -127,8 +127,8 @@ static void *opcode_targets[256] = { &&TARGET_STORE_FAST, &&TARGET_DELETE_FAST, &&TARGET_LOAD_FAST_CHECK, - &&TARGET_POP_JUMP_FORWARD_IF_NOT_NONE, - &&TARGET_POP_JUMP_FORWARD_IF_NONE, + &&TARGET_POP_JUMP_IF_NOT_NONE, + &&TARGET_POP_JUMP_IF_NONE, &&TARGET_RAISE_VARARGS, &&TARGET_GET_AWAITABLE, &&TARGET_MAKE_FUNCTION, @@ -172,10 +172,6 @@ static void *opcode_targets[256] = { &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_CALL, &&TARGET_KW_NAMES, - &&TARGET_POP_JUMP_BACKWARD_IF_NOT_NONE, - &&TARGET_POP_JUMP_BACKWARD_IF_NONE, - &&TARGET_POP_JUMP_BACKWARD_IF_FALSE, - &&TARGET_POP_JUMP_BACKWARD_IF_TRUE, &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_ADAPTIVE, &&TARGET_STORE_SUBSCR_DICT, @@ -254,5 +250,9 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_DO_TRACING }; diff --git a/Python/specialize.c b/Python/specialize.c index 8a2f905..e8c3f46 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1999,10 +1999,8 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1); int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]); - if (next_opcode != POP_JUMP_FORWARD_IF_FALSE && - next_opcode != POP_JUMP_BACKWARD_IF_FALSE && - next_opcode != POP_JUMP_FORWARD_IF_TRUE && - next_opcode != POP_JUMP_BACKWARD_IF_TRUE) { + if (next_opcode != POP_JUMP_IF_FALSE && + next_opcode != POP_JUMP_IF_TRUE) { // Can't ever combine, so don't don't bother being adaptive (unless // we're collecting stats, where it's more important to get accurate hit // counts for the unadaptive version and each of the different failure @@ -2021,14 +2019,9 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, } assert(oparg <= Py_GE); int when_to_jump_mask = compare_masks[oparg]; - if (next_opcode == POP_JUMP_FORWARD_IF_FALSE || - next_opcode == POP_JUMP_BACKWARD_IF_FALSE) { + if (next_opcode == POP_JUMP_IF_FALSE) { when_to_jump_mask = (1 | 2 | 4) & ~when_to_jump_mask; } - if (next_opcode == POP_JUMP_BACKWARD_IF_TRUE || - next_opcode == POP_JUMP_BACKWARD_IF_FALSE) { - when_to_jump_mask <<= 3; - } if (Py_TYPE(lhs) != Py_TYPE(rhs)) { SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); goto failure; @@ -2056,7 +2049,7 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, } else { _Py_SET_OPCODE(*instr, COMPARE_OP_STR_JUMP); - cache->mask = when_to_jump_mask; + cache->mask = (when_to_jump_mask & 2) == 0; goto success; } } |