diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 97f934727a85a36..0939dc9534f9fbd 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -190,6 +190,10 @@ enum { _GUARD_NOT_EXHAUSTED_RANGE, _GUARD_NOT_EXHAUSTED_TUPLE, _GUARD_THIRD_NULL, + _GUARD_TOS_AND_NOS_FLOAT, + _GUARD_TOS_AND_NOS_INT, + _GUARD_TOS_AND_NOS_OVERFLOWED, + _GUARD_TOS_AND_NOS_UNICODE, _GUARD_TOS_ANY_DICT, _GUARD_TOS_ANY_SET, _GUARD_TOS_DICT, @@ -364,7 +368,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 652 +#define MAX_UOP_ID 656 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -914,6 +918,22 @@ enum { _GUARD_THIRD_NULL_r13, _GUARD_THIRD_NULL_r23, _GUARD_THIRD_NULL_r33, + _GUARD_TOS_AND_NOS_FLOAT_r02, + _GUARD_TOS_AND_NOS_FLOAT_r12, + _GUARD_TOS_AND_NOS_FLOAT_r22, + _GUARD_TOS_AND_NOS_FLOAT_r33, + _GUARD_TOS_AND_NOS_INT_r02, + _GUARD_TOS_AND_NOS_INT_r12, + _GUARD_TOS_AND_NOS_INT_r22, + _GUARD_TOS_AND_NOS_INT_r33, + _GUARD_TOS_AND_NOS_OVERFLOWED_r02, + _GUARD_TOS_AND_NOS_OVERFLOWED_r12, + _GUARD_TOS_AND_NOS_OVERFLOWED_r22, + _GUARD_TOS_AND_NOS_OVERFLOWED_r33, + _GUARD_TOS_AND_NOS_UNICODE_r02, + _GUARD_TOS_AND_NOS_UNICODE_r12, + _GUARD_TOS_AND_NOS_UNICODE_r22, + _GUARD_TOS_AND_NOS_UNICODE_r33, _GUARD_TOS_ANY_DICT_r01, _GUARD_TOS_ANY_DICT_r11, _GUARD_TOS_ANY_DICT_r22, @@ -1434,7 +1454,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1645 +#define MAX_UOP_REGS_ID 1665 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 990706c0b329223..131994569ea51c8 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -116,6 +116,10 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, + [_GUARD_TOS_AND_NOS_INT] = HAS_EXIT_FLAG, + [_GUARD_TOS_AND_NOS_OVERFLOWED] = HAS_EXIT_FLAG, + [_GUARD_TOS_AND_NOS_FLOAT] = HAS_EXIT_FLAG, + [_GUARD_TOS_AND_NOS_UNICODE] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, @@ -1173,6 +1177,42 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _GUARD_TOS_FLOAT_r33 }, }, }, + [_GUARD_TOS_AND_NOS_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_TOS_AND_NOS_INT_r02 }, + { 2, 1, _GUARD_TOS_AND_NOS_INT_r12 }, + { 2, 2, _GUARD_TOS_AND_NOS_INT_r22 }, + { 3, 3, _GUARD_TOS_AND_NOS_INT_r33 }, + }, + }, + [_GUARD_TOS_AND_NOS_OVERFLOWED] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_TOS_AND_NOS_OVERFLOWED_r02 }, + { 2, 1, _GUARD_TOS_AND_NOS_OVERFLOWED_r12 }, + { 2, 2, _GUARD_TOS_AND_NOS_OVERFLOWED_r22 }, + { 3, 3, _GUARD_TOS_AND_NOS_OVERFLOWED_r33 }, + }, + }, + [_GUARD_TOS_AND_NOS_FLOAT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_TOS_AND_NOS_FLOAT_r02 }, + { 2, 1, _GUARD_TOS_AND_NOS_FLOAT_r12 }, + { 2, 2, _GUARD_TOS_AND_NOS_FLOAT_r22 }, + { 3, 3, _GUARD_TOS_AND_NOS_FLOAT_r33 }, + }, + }, + [_GUARD_TOS_AND_NOS_UNICODE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_TOS_AND_NOS_UNICODE_r02 }, + { 2, 1, _GUARD_TOS_AND_NOS_UNICODE_r12 }, + { 2, 2, _GUARD_TOS_AND_NOS_UNICODE_r22 }, + { 3, 3, _GUARD_TOS_AND_NOS_UNICODE_r33 }, + }, + }, [_BINARY_OP_MULTIPLY_FLOAT] = { .best = { 0, 1, 2, 2 }, .entries = { @@ -4138,6 +4178,22 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_FLOAT_r11] = _GUARD_TOS_FLOAT, [_GUARD_TOS_FLOAT_r22] = _GUARD_TOS_FLOAT, [_GUARD_TOS_FLOAT_r33] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_AND_NOS_INT_r02] = _GUARD_TOS_AND_NOS_INT, + [_GUARD_TOS_AND_NOS_INT_r12] = _GUARD_TOS_AND_NOS_INT, + [_GUARD_TOS_AND_NOS_INT_r22] = _GUARD_TOS_AND_NOS_INT, + [_GUARD_TOS_AND_NOS_INT_r33] = _GUARD_TOS_AND_NOS_INT, + [_GUARD_TOS_AND_NOS_OVERFLOWED_r02] = _GUARD_TOS_AND_NOS_OVERFLOWED, + [_GUARD_TOS_AND_NOS_OVERFLOWED_r12] = _GUARD_TOS_AND_NOS_OVERFLOWED, + [_GUARD_TOS_AND_NOS_OVERFLOWED_r22] = _GUARD_TOS_AND_NOS_OVERFLOWED, + [_GUARD_TOS_AND_NOS_OVERFLOWED_r33] = _GUARD_TOS_AND_NOS_OVERFLOWED, + [_GUARD_TOS_AND_NOS_FLOAT_r02] = _GUARD_TOS_AND_NOS_FLOAT, + [_GUARD_TOS_AND_NOS_FLOAT_r12] = _GUARD_TOS_AND_NOS_FLOAT, + [_GUARD_TOS_AND_NOS_FLOAT_r22] = _GUARD_TOS_AND_NOS_FLOAT, + [_GUARD_TOS_AND_NOS_FLOAT_r33] = _GUARD_TOS_AND_NOS_FLOAT, + [_GUARD_TOS_AND_NOS_UNICODE_r02] = _GUARD_TOS_AND_NOS_UNICODE, + [_GUARD_TOS_AND_NOS_UNICODE_r12] = _GUARD_TOS_AND_NOS_UNICODE, + [_GUARD_TOS_AND_NOS_UNICODE_r22] = _GUARD_TOS_AND_NOS_UNICODE, + [_GUARD_TOS_AND_NOS_UNICODE_r33] = _GUARD_TOS_AND_NOS_UNICODE, [_BINARY_OP_MULTIPLY_FLOAT_r03] = _BINARY_OP_MULTIPLY_FLOAT, [_BINARY_OP_MULTIPLY_FLOAT_r13] = _BINARY_OP_MULTIPLY_FLOAT, [_BINARY_OP_MULTIPLY_FLOAT_r23] = _BINARY_OP_MULTIPLY_FLOAT, @@ -5491,6 +5547,26 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_THIRD_NULL_r13] = "_GUARD_THIRD_NULL_r13", [_GUARD_THIRD_NULL_r23] = "_GUARD_THIRD_NULL_r23", [_GUARD_THIRD_NULL_r33] = "_GUARD_THIRD_NULL_r33", + [_GUARD_TOS_AND_NOS_FLOAT] = "_GUARD_TOS_AND_NOS_FLOAT", + [_GUARD_TOS_AND_NOS_FLOAT_r02] = "_GUARD_TOS_AND_NOS_FLOAT_r02", + [_GUARD_TOS_AND_NOS_FLOAT_r12] = "_GUARD_TOS_AND_NOS_FLOAT_r12", + [_GUARD_TOS_AND_NOS_FLOAT_r22] = "_GUARD_TOS_AND_NOS_FLOAT_r22", + [_GUARD_TOS_AND_NOS_FLOAT_r33] = "_GUARD_TOS_AND_NOS_FLOAT_r33", + [_GUARD_TOS_AND_NOS_INT] = "_GUARD_TOS_AND_NOS_INT", + [_GUARD_TOS_AND_NOS_INT_r02] = "_GUARD_TOS_AND_NOS_INT_r02", + [_GUARD_TOS_AND_NOS_INT_r12] = "_GUARD_TOS_AND_NOS_INT_r12", + [_GUARD_TOS_AND_NOS_INT_r22] = "_GUARD_TOS_AND_NOS_INT_r22", + [_GUARD_TOS_AND_NOS_INT_r33] = "_GUARD_TOS_AND_NOS_INT_r33", + [_GUARD_TOS_AND_NOS_OVERFLOWED] = "_GUARD_TOS_AND_NOS_OVERFLOWED", + [_GUARD_TOS_AND_NOS_OVERFLOWED_r02] = "_GUARD_TOS_AND_NOS_OVERFLOWED_r02", + [_GUARD_TOS_AND_NOS_OVERFLOWED_r12] = "_GUARD_TOS_AND_NOS_OVERFLOWED_r12", + [_GUARD_TOS_AND_NOS_OVERFLOWED_r22] = "_GUARD_TOS_AND_NOS_OVERFLOWED_r22", + [_GUARD_TOS_AND_NOS_OVERFLOWED_r33] = "_GUARD_TOS_AND_NOS_OVERFLOWED_r33", + [_GUARD_TOS_AND_NOS_UNICODE] = "_GUARD_TOS_AND_NOS_UNICODE", + [_GUARD_TOS_AND_NOS_UNICODE_r02] = "_GUARD_TOS_AND_NOS_UNICODE_r02", + [_GUARD_TOS_AND_NOS_UNICODE_r12] = "_GUARD_TOS_AND_NOS_UNICODE_r12", + [_GUARD_TOS_AND_NOS_UNICODE_r22] = "_GUARD_TOS_AND_NOS_UNICODE_r22", + [_GUARD_TOS_AND_NOS_UNICODE_r33] = "_GUARD_TOS_AND_NOS_UNICODE_r33", [_GUARD_TOS_ANY_DICT] = "_GUARD_TOS_ANY_DICT", [_GUARD_TOS_ANY_DICT_r01] = "_GUARD_TOS_ANY_DICT_r01", [_GUARD_TOS_ANY_DICT_r11] = "_GUARD_TOS_ANY_DICT_r11", @@ -6309,6 +6385,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_TOS_FLOAT: return 0; + case _GUARD_TOS_AND_NOS_INT: + return 0; + case _GUARD_TOS_AND_NOS_OVERFLOWED: + return 0; + case _GUARD_TOS_AND_NOS_FLOAT: + return 0; + case _GUARD_TOS_AND_NOS_UNICODE: + return 0; case _BINARY_OP_MULTIPLY_FLOAT: return 2; case _BINARY_OP_ADD_FLOAT: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 25b2c393e6773de..c4a3883e9ba8197 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -640,7 +640,6 @@ def _run_with_optimizer(self, testfunc, arg): ex = get_first_executor(testfunc) return res, ex - def test_int_type_propagation(self): def testfunc(loops): num = 0 @@ -656,9 +655,10 @@ def testfunc(loops): binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] guard_tos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] guard_nos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] + combined_guard_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_AND_NOS_INT"] self.assertGreaterEqual(len(binop_count), 3) - self.assertLessEqual(len(guard_tos_int_count), 1) - self.assertLessEqual(len(guard_nos_int_count), 1) + self.assertLessEqual(len(guard_tos_int_count) + len(combined_guard_count), 1) + self.assertLessEqual(len(guard_nos_int_count) + len(combined_guard_count), 1) def test_int_type_propagation_through_frame(self): def double(x): @@ -679,9 +679,10 @@ def testfunc(loops): binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] guard_tos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] guard_nos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] + combined_guard_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_AND_NOS_INT"] self.assertGreaterEqual(len(binop_count), 3) - self.assertLessEqual(len(guard_tos_int_count), 1) - self.assertLessEqual(len(guard_nos_int_count), 1) + self.assertLessEqual(len(guard_tos_int_count) + len(combined_guard_count), 1) + self.assertLessEqual(len(guard_nos_int_count) + len(combined_guard_count), 1) def test_int_type_propagation_from_frame(self): def double(x): @@ -702,9 +703,10 @@ def testfunc(loops): binop_count = [opname for opname in iter_opnames(ex) if opname == "_BINARY_OP_ADD_INT"] guard_tos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] guard_nos_int_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_NOS_INT"] + combined_guard_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_AND_NOS_INT"] self.assertGreaterEqual(len(binop_count), 3) - self.assertLessEqual(len(guard_tos_int_count), 1) - self.assertLessEqual(len(guard_nos_int_count), 1) + self.assertLessEqual(len(guard_tos_int_count) + len(combined_guard_count), 1) + self.assertLessEqual(len(guard_nos_int_count) + len(combined_guard_count), 1) def test_int_impure_region(self): def testfunc(loops): @@ -751,6 +753,7 @@ def testfunc(n): uops = get_opnames(ex) self.assertNotIn("_GUARD_TOS_INT", uops) self.assertNotIn("_GUARD_NOS_INT", uops) + self.assertNotIn("_GUARD_TOS_AND_NOS_INT", uops) def test_int_value_numbering(self): def testfunc(n): @@ -770,6 +773,7 @@ def testfunc(n): uops = get_opnames(ex) self.assertIn("_GUARD_TOS_INT", uops) self.assertNotIn("_GUARD_NOS_INT", uops) + self.assertNotIn("_GUARD_TOS_AND_NOS_INT", uops) guard_tos_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_TOS_INT"] self.assertEqual(len(guard_tos_count), 1) @@ -4041,7 +4045,7 @@ def test_float_truediv_speculative_guards_from_tracing(self): # a, b are locals with no statically known type. _RECORD_TOS_TYPE / # _RECORD_NOS_TYPE (added to the BINARY_OP macro) capture the observed # operand types during tracing, and the optimizer then speculatively - # emits _GUARD_{TOS,NOS}_FLOAT and specializes the division. + # emits a combined float guard and specializes the division. def testfunc(args): a, b, n = args total = 0.0 @@ -4053,14 +4057,13 @@ def testfunc(args): self.assertAlmostEqual(res, TIER2_THRESHOLD * (10.0 / 3.0)) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_GUARD_TOS_FLOAT", uops) - self.assertIn("_GUARD_NOS_FLOAT", uops) + self.assertIn("_GUARD_TOS_AND_NOS_FLOAT", uops) self.assertIn("_BINARY_OP_TRUEDIV_FLOAT", uops) def test_float_remainder_speculative_guards_from_tracing(self): # a, b are locals with no statically known type. Tracing records # them as floats; the optimizer then speculatively emits - # _GUARD_{TOS,NOS}_FLOAT for NB_REMAINDER. That narrows both + # _GUARD_TOS_AND_NOS_FLOAT for NB_REMAINDER. That narrows both # operands to float, and the _BINARY_OP handler marks the result # as a unique float. Downstream, `* 2.0` therefore specializes # to _BINARY_OP_MULTIPLY_FLOAT_INPLACE. @@ -4075,8 +4078,7 @@ def testfunc(args): self.assertAlmostEqual(res, TIER2_THRESHOLD * (10.0 % 3.0) * 2.0) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_GUARD_TOS_FLOAT", uops) - self.assertIn("_GUARD_NOS_FLOAT", uops) + self.assertIn("_GUARD_TOS_AND_NOS_FLOAT", uops) self.assertIn("_BINARY_OP_MULTIPLY_FLOAT_INPLACE", uops) def test_float_truediv_type_propagation(self): @@ -5582,8 +5584,7 @@ def testfunc(*args): uops = get_opnames(ex) self.assertIn("_BINARY_OP_SUBSCR_INIT_CALL", uops) - # _POP_TOP_NOP is a sign the optimizer ran and didn't hit contradiction. - self.assertGreaterEqual(count_ops(ex, "_POP_TOP_NOP"), 1) + self.assertNotIn("_POP_TOP_NOP", uops) def test_load_attr_property_frame(self): class B: @@ -6215,6 +6216,109 @@ def __exit__(self, e, v, t): ... f1() """), PYTHON_JIT="1") + def test_peephole_cancels_adjacent_push_pop(self): + def negated_constant(n): + x = 0 + for i in range(n): + a = 1 + result = -a + if result < 0: + x += 1 + return x + + res, ex = self._run_with_optimizer(negated_constant, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_SWAP", uops) + self.assertNotIn("_SWAP_2", uops) + + def test_peephole_removes_redundant_rrot(self): + def add(a, b): + return a + b + + def testfunc(n): + x = 0 + for _ in range(n): + x += add(1, 2) + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 3 * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_RROT_3", uops) + + def test_combine_adjacent_int_guards(self): + def testfunc(left, right, loops): + for _ in range(loops): + result = left + right + return result + + result = testfunc(40, 2, TIER2_THRESHOLD) + + self.assertEqual(result, 42) + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_TOS_AND_NOS_INT", uops) + self.assertNotIn("_GUARD_TOS_INT", uops) + self.assertNotIn("_GUARD_NOS_INT", uops) + self.assertEqual(testfunc(40.0, 2.0, 1), 42.0) + self.assertEqual(testfunc(1 << 100, 2, 1), (1 << 100) + 2) + + def test_combine_adjacent_overflowed_guards(self): + def testfunc(start, loops): + for value in range(start, start + loops): + inverted = ~value + result = value + inverted + return result + + result = testfunc(0, TIER2_THRESHOLD) + + self.assertEqual(result, -1) + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_TOS_AND_NOS_OVERFLOWED", uops) + self.assertNotIn("_GUARD_TOS_OVERFLOWED", uops) + self.assertNotIn("_GUARD_NOS_OVERFLOWED", uops) + self.assertEqual(testfunc(1 << 100, 1), -1) + + def test_combine_adjacent_float_guards(self): + def testfunc(left, right, loops): + for _ in range(loops): + result = left + right + return result + + result = testfunc(1.5, 2.5, TIER2_THRESHOLD) + + self.assertEqual(result, 4.0) + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_TOS_AND_NOS_FLOAT", uops) + self.assertNotIn("_GUARD_TOS_FLOAT", uops) + self.assertNotIn("_GUARD_NOS_FLOAT", uops) + self.assertEqual(testfunc(1, 2, 1), 3) + + def test_combine_adjacent_unicode_guards(self): + def testfunc(left, right, loops): + for _ in range(loops): + result = left + right + return result + + result = testfunc("a", "b", TIER2_THRESHOLD) + + self.assertEqual(result, "ab") + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_TOS_AND_NOS_UNICODE", uops) + self.assertNotIn("_GUARD_TOS_UNICODE", uops) + self.assertNotIn("_GUARD_NOS_UNICODE", uops) + self.assertEqual(testfunc(b"a", b"b", 1), b"ab") + def global_identity(x): return x diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4d7b338e2dbd4c3..bd3e07acea7edf0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -776,6 +776,38 @@ dummy_func( EXIT_IF(!PyFloat_CheckExact(value_o)); } + tier2 op(_GUARD_TOS_AND_NOS_INT, (left, right -- left, right)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + EXIT_IF(!PyLong_CheckExact(left_o) || + !PyLong_CheckExact(right_o) || + !_PyLong_BothAreCompact( + (PyLongObject *)left_o, (PyLongObject *)right_o)); + } + + tier2 op(_GUARD_TOS_AND_NOS_OVERFLOWED, (left, right -- left, right)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + EXIT_IF(!_PyLong_BothAreCompact( + (PyLongObject *)left_o, (PyLongObject *)right_o)); + } + + tier2 op(_GUARD_TOS_AND_NOS_FLOAT, (left, right -- left, right)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + EXIT_IF(!PyFloat_CheckExact(left_o) || + !PyFloat_CheckExact(right_o)); + } + + tier2 op(_GUARD_TOS_AND_NOS_UNICODE, (left, right -- left, right)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + EXIT_IF(!PyUnicode_CheckExact(left_o) || + !PyUnicode_CheckExact(right_o)); + } + pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e45bbd7cceb295f..b76c9e93db550da 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -5229,6 +5229,442 @@ break; } + case _GUARD_TOS_AND_NOS_INT_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o) || + !PyLong_CheckExact(right_o) || + !_PyLong_BothAreCompact( + (PyLongObject *)left_o, (PyLongObject *)right_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_INT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o) || + !PyLong_CheckExact(right_o) || + !_PyLong_BothAreCompact( + (PyLongObject *)left_o, (PyLongObject *)right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_INT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o) || + !PyLong_CheckExact(right_o) || + !_PyLong_BothAreCompact( + (PyLongObject *)left_o, (PyLongObject *)right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_INT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + right = _stack_item_2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o) || + !PyLong_CheckExact(right_o) || + !_PyLong_BothAreCompact( + (PyLongObject *)left_o, (PyLongObject *)right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_OVERFLOWED_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact( + (PyLongObject *)left_o, (PyLongObject *)right_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_OVERFLOWED_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact( + (PyLongObject *)left_o, (PyLongObject *)right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_OVERFLOWED_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact( + (PyLongObject *)left_o, (PyLongObject *)right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_OVERFLOWED_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + right = _stack_item_2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact( + (PyLongObject *)left_o, (PyLongObject *)right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_FLOAT_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyFloat_CheckExact(left_o) || + !PyFloat_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_FLOAT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyFloat_CheckExact(left_o) || + !PyFloat_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_FLOAT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyFloat_CheckExact(left_o) || + !PyFloat_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_FLOAT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + right = _stack_item_2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyFloat_CheckExact(left_o) || + !PyFloat_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_UNICODE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyUnicode_CheckExact(left_o) || + !PyUnicode_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_UNICODE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyUnicode_CheckExact(left_o) || + !PyUnicode_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_UNICODE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyUnicode_CheckExact(left_o) || + !PyUnicode_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_AND_NOS_UNICODE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + right = _stack_item_2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyUnicode_CheckExact(left_o) || + !PyUnicode_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _BINARY_OP_MULTIPLY_FLOAT_r03: { CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index e726dc0e6fd1114..396d83ef0a890f2 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -719,6 +719,123 @@ const uint16_t op_without_pop[MAX_UOP_ID + 1] = { [_POP_TOP_UNICODE] = _NOP, }; +/* Type analysis has already processed both guards before this cleanup pass. + * Require the same side exit so removing the NOS guard preserves deoptimization + * behavior. */ +static bool +combine_matching_tos_nos_guards( + _PyUOpInstruction *buffer, int pc, int buffer_size) +{ + if (pc + 1 >= buffer_size) { + return false; + } + int tos_opcode = buffer[pc].opcode; + int expected_nos_opcode; + int combined_opcode; + switch (tos_opcode) { + case _GUARD_TOS_INT: + expected_nos_opcode = _GUARD_NOS_INT; + combined_opcode = _GUARD_TOS_AND_NOS_INT; + break; + case _GUARD_TOS_OVERFLOWED: + expected_nos_opcode = _GUARD_NOS_OVERFLOWED; + combined_opcode = _GUARD_TOS_AND_NOS_OVERFLOWED; + break; + case _GUARD_TOS_FLOAT: + expected_nos_opcode = _GUARD_NOS_FLOAT; + combined_opcode = _GUARD_TOS_AND_NOS_FLOAT; + break; + case _GUARD_TOS_UNICODE: + expected_nos_opcode = _GUARD_NOS_UNICODE; + combined_opcode = _GUARD_TOS_AND_NOS_UNICODE; + break; + default: + return false; + } + if (buffer[pc + 1].opcode != expected_nos_opcode || + uop_get_target(&buffer[pc]) != uop_get_target(&buffer[pc + 1])) + { + return false; + } + buffer[pc].opcode = combined_opcode; + buffer[pc + 1].opcode = _NOP; + return true; +} + +static int +previous_non_skip_uop(_PyUOpInstruction *buffer, int pc) +{ + while (pc >= 0 && op_skip[buffer[pc].opcode]) { + pc--; + } + return pc; +} + +/* Remove redundant stack shuffles left by constant-folding rewrites: + * push push _SWAP(2) pop + * push push push _RROT_3 pop pop + * In both forms, only the push closest to the shuffle survives. */ +static bool +remove_folded_stack_shuffle(_PyUOpInstruction *buffer, int pc, int arity) +{ + int pops[2]; + int idx = pc; + for (int i = 0; i < arity - 1; i++) { + if (idx < 0 || !op_without_pop[buffer[idx].opcode]) { + return false; + } + pops[i] = idx; + idx = previous_non_skip_uop(buffer, idx - 1); + } + int shuf = idx; + if (shuf < 0) { + return false; + } + uint16_t shuf_op = buffer[shuf].opcode; + bool is_shuffle = + (arity == 2 && shuf_op == _SWAP && buffer[shuf].oparg == 2) || + (arity == 3 && shuf_op == _RROT_3); + if (!is_shuffle) { + return false; + } + + int dead_pushes[2]; + for (int i = 0; i < arity; i++) { + idx = previous_non_skip_uop(buffer, idx - 1); + if (idx < 0) { + return false; + } + uint16_t push_op = buffer[idx].opcode; + if (push_op == _COPY || !op_without_push[push_op]) { + return false; + } + if (i > 0) { + dead_pushes[i - 1] = idx; + } + } + + for (int i = 0; i < arity - 1; i++) { + buffer[dead_pushes[i]].opcode = _NOP; + } + buffer[shuf].opcode = _NOP; + for (int i = 0; i < arity - 1; i++) { + buffer[pops[i]].opcode = _NOP; + } + return true; +} + +static bool +remove_adjacent_push_pop(_PyUOpInstruction *buffer, int pc) +{ + int last = previous_non_skip_uop(buffer, pc - 1); + if (last < 0 || !op_without_push[buffer[last].opcode]) { + return false; + } + buffer[last].opcode = _NOP; + buffer[pc].opcode = _NOP; + return true; +} + static int remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) @@ -747,6 +864,14 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) buffer[pc].opcode = _NOP; } break; + case _GUARD_TOS_INT: + case _GUARD_TOS_OVERFLOWED: + case _GUARD_TOS_FLOAT: + case _GUARD_TOS_UNICODE: + if (combine_matching_tos_nos_guards(buffer, pc, buffer_size)) { + opcode = buffer[pc].opcode; + } + _Py_FALLTHROUGH; case _EXIT_TRACE: default: { @@ -755,21 +880,14 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) // ...becomes: // _NOP + _NOP + _POP_TOP + _NOP + _NOP while (op_without_pop[opcode]) { - _PyUOpInstruction *last = &buffer[pc - 1]; - while (op_skip[last->opcode]) { - last--; - } - if (op_without_push[last->opcode] && op_without_pop[opcode]) { - last->opcode = op_without_push[last->opcode]; - opcode = buffer[pc].opcode = op_without_pop[opcode]; - if (op_without_pop[last->opcode]) { - opcode = last->opcode; - pc = (int)(last - buffer); - } - } - else { - break; + if (remove_folded_stack_shuffle(buffer, pc, 2) || + remove_folded_stack_shuffle(buffer, pc, 3) || + remove_adjacent_push_pop(buffer, pc)) + { + opcode = buffer[pc].opcode; + continue; } + break; } /* _PUSH_FRAME doesn't escape or error, but it * does need the IP for the return address */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 5e110360b81b445..7ea148b69199180 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -918,6 +918,22 @@ break; } + case _GUARD_TOS_AND_NOS_INT: { + break; + } + + case _GUARD_TOS_AND_NOS_OVERFLOWED: { + break; + } + + case _GUARD_TOS_AND_NOS_FLOAT: { + break; + } + + case _GUARD_TOS_AND_NOS_UNICODE: { + break; + } + case _BINARY_OP_MULTIPLY_FLOAT: { JitOptRef right; JitOptRef left;