From 06d0a8ce0041e4edc4d379e5ad176c38f3a71798 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Fri, 10 Jul 2026 18:49:15 -0500 Subject: [PATCH 1/2] gh-151289: specialize shrinking int subtraction --- Include/internal/pycore_long.h | 23 ++ Include/internal/pycore_opcode_metadata.h | 4 +- Include/internal/pycore_uop_ids.h | 16 +- Include/internal/pycore_uop_metadata.h | 54 +++- Lib/test/test_capi/test_opt.py | 16 ++ Lib/test/test_long.py | 19 ++ ...-06-10-17-40-41.gh-issue-151289.xZhM7W.rst | 2 + Modules/_testinternalcapi/test_cases.c.h | 19 +- Objects/longobject.c | 66 +++++ Python/bytecodes.c | 20 +- Python/executor_cases.c.h | 260 +++++++++++++----- Python/generated_cases.c.h | 19 +- Python/optimizer_cases.c.h | 16 +- Python/specialize.c | 4 +- 14 files changed, 434 insertions(+), 104 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-17-40-41.gh-issue-151289.xZhM7W.rst diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index fb5622c99f7a135..157fe871e71a0ad 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -116,6 +116,7 @@ PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, int64_t); PyAPI_FUNC(_PyStackRef) _PyCompactLong_Add(PyLongObject *left, PyLongObject *right); PyAPI_FUNC(_PyStackRef) _PyCompactLong_Multiply(PyLongObject *left, PyLongObject *right); PyAPI_FUNC(_PyStackRef) _PyCompactLong_Subtract(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(_PyStackRef) _PyLong_SubtractShrink(PyLongObject *left, PyLongObject *right); // Export for 'binascii' shared extension. PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; @@ -346,6 +347,28 @@ _PyLong_CheckExactAndCompact(PyObject *op) return PyLong_CheckExact(op) && _PyLong_IsCompact((const PyLongObject *)op); } +static inline bool +_PyLong_IsShrinkSubtractPair(const PyLongObject *left, const PyLongObject *right) +{ +#if PyLong_SHIFT == 30 + if (_PyLong_IsCompact((PyLongObject *)left) || + _PyLong_IsCompact((PyLongObject *)right) || + !_PyLong_SameSign(left, right)) { + return false; + } + Py_ssize_t left_size = _PyLong_DigitCount(left); + if (left_size != _PyLong_DigitCount(right) || + left_size < 2 || + left_size > 3) { + return false; + } + return left->long_value.ob_digit[left_size - 1] == + right->long_value.ob_digit[left_size - 1]; +#else + return false; +#endif +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 31d2e31573d8c53..2086602ef095f62 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1116,7 +1116,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_USTR_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_INTERPOLATION] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1371,7 +1371,7 @@ _PyOpcode_macro_expansion[256] = { [BINARY_OP_SUBSCR_TUPLE_INT] = { .nuops = 6, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_TUPLE, OPARG_SIMPLE, 0 }, { _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_TUPLE_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBSCR_USTR_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_USTR_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBTRACT_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBTRACT_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_EXACT_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_EXACT_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, [BINARY_SLICE] = { .nuops = 1, .uops = { { _BINARY_SLICE, OPARG_SIMPLE, 0 } } }, [BUILD_INTERPOLATION] = { .nuops = 1, .uops = { { _BUILD_INTERPOLATION, OPARG_SIMPLE, 0 } } }, [BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 97f934727a85a36..54dfc9fc86be931 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -176,6 +176,7 @@ enum { _GUARD_NOS_COMPACT_ASCII, _GUARD_NOS_DICT_STORE_SUBSCRIPT, _GUARD_NOS_DICT_SUBSCRIPT, + _GUARD_NOS_EXACT_INT, _GUARD_NOS_FLOAT, _GUARD_NOS_INT, _GUARD_NOS_ITER_VIRTUAL, @@ -193,6 +194,7 @@ enum { _GUARD_TOS_ANY_DICT, _GUARD_TOS_ANY_SET, _GUARD_TOS_DICT, + _GUARD_TOS_EXACT_INT, _GUARD_TOS_FLOAT, _GUARD_TOS_FROZENDICT, _GUARD_TOS_FROZENSET, @@ -364,7 +366,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 652 +#define MAX_UOP_ID 654 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -506,8 +508,6 @@ enum { _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23, - _BINARY_OP_SUBTRACT_INT_r03, - _BINARY_OP_SUBTRACT_INT_r13, _BINARY_OP_SUBTRACT_INT_r23, _BINARY_OP_SUBTRACT_INT_INPLACE_r03, _BINARY_OP_SUBTRACT_INT_INPLACE_r13, @@ -858,6 +858,10 @@ enum { _GUARD_NOS_DICT_SUBSCRIPT_r12, _GUARD_NOS_DICT_SUBSCRIPT_r22, _GUARD_NOS_DICT_SUBSCRIPT_r33, + _GUARD_NOS_EXACT_INT_r02, + _GUARD_NOS_EXACT_INT_r12, + _GUARD_NOS_EXACT_INT_r22, + _GUARD_NOS_EXACT_INT_r33, _GUARD_NOS_FLOAT_r02, _GUARD_NOS_FLOAT_r12, _GUARD_NOS_FLOAT_r22, @@ -926,6 +930,10 @@ enum { _GUARD_TOS_DICT_r11, _GUARD_TOS_DICT_r22, _GUARD_TOS_DICT_r33, + _GUARD_TOS_EXACT_INT_r01, + _GUARD_TOS_EXACT_INT_r11, + _GUARD_TOS_EXACT_INT_r22, + _GUARD_TOS_EXACT_INT_r33, _GUARD_TOS_FLOAT_r01, _GUARD_TOS_FLOAT_r11, _GUARD_TOS_FLOAT_r22, @@ -1434,7 +1442,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1645 +#define MAX_UOP_REGS_ID 1653 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 990706c0b329223..e6fb2d0730b322b 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -103,11 +103,13 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, [_GUARD_TOS_INT] = HAS_EXIT_FLAG, + [_GUARD_NOS_EXACT_INT] = HAS_EXIT_FLAG, + [_GUARD_TOS_EXACT_INT] = HAS_EXIT_FLAG, [_GUARD_NOS_OVERFLOWED] = HAS_EXIT_FLAG, [_GUARD_TOS_OVERFLOWED] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_INT] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT_INPLACE] = HAS_EXIT_FLAG, [_BINARY_OP_SUBTRACT_INT_INPLACE] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_INT_INPLACE] = HAS_EXIT_FLAG, @@ -1056,6 +1058,24 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _GUARD_TOS_INT_r33 }, }, }, + [_GUARD_NOS_EXACT_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_EXACT_INT_r02 }, + { 2, 1, _GUARD_NOS_EXACT_INT_r12 }, + { 2, 2, _GUARD_NOS_EXACT_INT_r22 }, + { 3, 3, _GUARD_NOS_EXACT_INT_r33 }, + }, + }, + [_GUARD_TOS_EXACT_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_EXACT_INT_r01 }, + { 1, 1, _GUARD_TOS_EXACT_INT_r11 }, + { 2, 2, _GUARD_TOS_EXACT_INT_r22 }, + { 3, 3, _GUARD_TOS_EXACT_INT_r33 }, + }, + }, [_GUARD_NOS_OVERFLOWED] = { .best = { 0, 1, 2, 3 }, .entries = { @@ -1093,10 +1113,10 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_BINARY_OP_SUBTRACT_INT] = { - .best = { 0, 1, 2, 2 }, + .best = { 2, 2, 2, 2 }, .entries = { - { 3, 0, _BINARY_OP_SUBTRACT_INT_r03 }, - { 3, 1, _BINARY_OP_SUBTRACT_INT_r13 }, + { -1, -1, -1 }, + { -1, -1, -1 }, { 3, 2, _BINARY_OP_SUBTRACT_INT_r23 }, { -1, -1, -1 }, }, @@ -4095,6 +4115,14 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_INT_r11] = _GUARD_TOS_INT, [_GUARD_TOS_INT_r22] = _GUARD_TOS_INT, [_GUARD_TOS_INT_r33] = _GUARD_TOS_INT, + [_GUARD_NOS_EXACT_INT_r02] = _GUARD_NOS_EXACT_INT, + [_GUARD_NOS_EXACT_INT_r12] = _GUARD_NOS_EXACT_INT, + [_GUARD_NOS_EXACT_INT_r22] = _GUARD_NOS_EXACT_INT, + [_GUARD_NOS_EXACT_INT_r33] = _GUARD_NOS_EXACT_INT, + [_GUARD_TOS_EXACT_INT_r01] = _GUARD_TOS_EXACT_INT, + [_GUARD_TOS_EXACT_INT_r11] = _GUARD_TOS_EXACT_INT, + [_GUARD_TOS_EXACT_INT_r22] = _GUARD_TOS_EXACT_INT, + [_GUARD_TOS_EXACT_INT_r33] = _GUARD_TOS_EXACT_INT, [_GUARD_NOS_OVERFLOWED_r02] = _GUARD_NOS_OVERFLOWED, [_GUARD_NOS_OVERFLOWED_r12] = _GUARD_NOS_OVERFLOWED, [_GUARD_NOS_OVERFLOWED_r22] = _GUARD_NOS_OVERFLOWED, @@ -4109,8 +4137,6 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_ADD_INT_r03] = _BINARY_OP_ADD_INT, [_BINARY_OP_ADD_INT_r13] = _BINARY_OP_ADD_INT, [_BINARY_OP_ADD_INT_r23] = _BINARY_OP_ADD_INT, - [_BINARY_OP_SUBTRACT_INT_r03] = _BINARY_OP_SUBTRACT_INT, - [_BINARY_OP_SUBTRACT_INT_r13] = _BINARY_OP_SUBTRACT_INT, [_BINARY_OP_SUBTRACT_INT_r23] = _BINARY_OP_SUBTRACT_INT, [_BINARY_OP_ADD_INT_INPLACE_r03] = _BINARY_OP_ADD_INT_INPLACE, [_BINARY_OP_ADD_INT_INPLACE_r13] = _BINARY_OP_ADD_INT_INPLACE, @@ -4922,8 +4948,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13", [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", - [_BINARY_OP_SUBTRACT_INT_r03] = "_BINARY_OP_SUBTRACT_INT_r03", - [_BINARY_OP_SUBTRACT_INT_r13] = "_BINARY_OP_SUBTRACT_INT_r13", [_BINARY_OP_SUBTRACT_INT_r23] = "_BINARY_OP_SUBTRACT_INT_r23", [_BINARY_OP_SUBTRACT_INT_INPLACE] = "_BINARY_OP_SUBTRACT_INT_INPLACE", [_BINARY_OP_SUBTRACT_INT_INPLACE_r03] = "_BINARY_OP_SUBTRACT_INT_INPLACE_r03", @@ -5421,6 +5445,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_NOS_DICT_SUBSCRIPT_r12] = "_GUARD_NOS_DICT_SUBSCRIPT_r12", [_GUARD_NOS_DICT_SUBSCRIPT_r22] = "_GUARD_NOS_DICT_SUBSCRIPT_r22", [_GUARD_NOS_DICT_SUBSCRIPT_r33] = "_GUARD_NOS_DICT_SUBSCRIPT_r33", + [_GUARD_NOS_EXACT_INT] = "_GUARD_NOS_EXACT_INT", + [_GUARD_NOS_EXACT_INT_r02] = "_GUARD_NOS_EXACT_INT_r02", + [_GUARD_NOS_EXACT_INT_r12] = "_GUARD_NOS_EXACT_INT_r12", + [_GUARD_NOS_EXACT_INT_r22] = "_GUARD_NOS_EXACT_INT_r22", + [_GUARD_NOS_EXACT_INT_r33] = "_GUARD_NOS_EXACT_INT_r33", [_GUARD_NOS_FLOAT] = "_GUARD_NOS_FLOAT", [_GUARD_NOS_FLOAT_r02] = "_GUARD_NOS_FLOAT_r02", [_GUARD_NOS_FLOAT_r12] = "_GUARD_NOS_FLOAT_r12", @@ -5506,6 +5535,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_DICT_r11] = "_GUARD_TOS_DICT_r11", [_GUARD_TOS_DICT_r22] = "_GUARD_TOS_DICT_r22", [_GUARD_TOS_DICT_r33] = "_GUARD_TOS_DICT_r33", + [_GUARD_TOS_EXACT_INT] = "_GUARD_TOS_EXACT_INT", + [_GUARD_TOS_EXACT_INT_r01] = "_GUARD_TOS_EXACT_INT_r01", + [_GUARD_TOS_EXACT_INT_r11] = "_GUARD_TOS_EXACT_INT_r11", + [_GUARD_TOS_EXACT_INT_r22] = "_GUARD_TOS_EXACT_INT_r22", + [_GUARD_TOS_EXACT_INT_r33] = "_GUARD_TOS_EXACT_INT_r33", [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", [_GUARD_TOS_FLOAT_r01] = "_GUARD_TOS_FLOAT_r01", [_GUARD_TOS_FLOAT_r11] = "_GUARD_TOS_FLOAT_r11", @@ -6283,6 +6317,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_TOS_INT: return 0; + case _GUARD_NOS_EXACT_INT: + return 0; + case _GUARD_TOS_EXACT_INT: + return 0; case _GUARD_NOS_OVERFLOWED: return 0; case _GUARD_TOS_OVERFLOWED: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 25b2c393e6773de..c5d5d3f013ec298 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -4455,6 +4455,22 @@ def testfunc(n): self.assertIn("_POP_TOP_NOP", uops) self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + def test_int_subtract_shrink_specialization(self): + def testfunc(args): + a, b, n = args + res = 0 + for _ in range(n): + res = a - b + return res + + res, ex = self._run_with_optimizer( + testfunc, (10_000_000_001, 10_000_000_000, TIER2_THRESHOLD) + ) + self.assertEqual(res, 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_SUBTRACT_INT", uops) + def test_int_mul_op_refcount_elimination(self): def testfunc(n): c = 1 diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index b48a8812a1a2d13..1628d1038c41246 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -1140,6 +1140,25 @@ def test_small_ints(self): self.assertIs(i - i, 0) self.assertIs(0 * i, 0) + @support.cpython_only + def test_int64_boundary_sub(self): + INT64_MAX = (1 << 63) - 1 + INT64_MIN = -(1 << 63) + + self.assertEqual(INT64_MIN - 0, INT64_MIN) + self.assertEqual(INT64_MIN - (-1), INT64_MIN + 1) + + self.assertEqual(INT64_MAX - 0, INT64_MAX) + self.assertEqual(INT64_MAX - 1, INT64_MAX - 1) + + self.assertEqual(INT64_MIN - 1, INT64_MIN - 1) + self.assertEqual(INT64_MAX - (-1), 1 << 63) + + beyond_max = INT64_MAX + 2 + beyond_min = INT64_MIN - 2 + self.assertEqual(beyond_max - 1, INT64_MAX + 1) + self.assertEqual(beyond_min - (-1), INT64_MIN - 1) + def test_bit_length(self): tiny = 1e-10 for x in range(-65000, 65000): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-17-40-41.gh-issue-151289.xZhM7W.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-17-40-41.gh-issue-151289.xZhM7W.rst new file mode 100644 index 000000000000000..8285ad1e0b3064f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-17-40-41.gh-issue-151289.xZhM7W.rst @@ -0,0 +1,2 @@ +Improve performance of binary ``int`` subtraction for shrinking wide exact +integers. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index a17648a33d4fe4a..8c72c820d184d9c 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -1346,21 +1346,21 @@ _PyStackRef res; _PyStackRef l; _PyStackRef r; - // _GUARD_TOS_INT + // _GUARD_TOS_EXACT_INT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + if (!PyLong_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } - // _GUARD_NOS_INT + // _GUARD_NOS_EXACT_INT { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { + if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1374,9 +1374,16 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res = _PyLong_SubtractShrink((PyLongObject *)left_o, (PyLongObject *)right_o); + _PyFrame_StackPointerInvalidate(frame); + } if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); diff --git a/Objects/longobject.c b/Objects/longobject.c index 7a38ae8ea5a36f0..e85c6e2f0c1aac0 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3916,6 +3916,72 @@ _PyCompactLong_Subtract(PyLongObject *a, PyLongObject *b) return medium_from_stwodigits(v); } +static inline _PyStackRef +shrink_subtract_result(int sign, const digit *digits, Py_ssize_t size) +{ + while (size > 0 && digits[size - 1] == 0) { + --size; + } + if (size == 0) { + return PyStackRef_FromPyObjectBorrow(_PyLong_GetZero()); + } + if (size == 1) { + stwodigits value = (stwodigits)digits[0]; + if (sign < 0) { + value = -value; + } + return medium_from_stwodigits(value); + } + PyLongObject *result = PyObject_Malloc( + offsetof(PyLongObject, long_value.ob_digit) + size * sizeof(digit)); + if (result == NULL) { + return PyStackRef_NULL; + } + _PyObject_Init((PyObject *)result, &PyLong_Type); + _PyLong_InitTag(result); + _PyLong_SetSignAndDigitCount(result, sign, size); + memcpy(result->long_value.ob_digit, digits, size * sizeof(digit)); + return PyStackRef_FromPyObjectStealMortal((PyObject *)result); +} + +_PyStackRef +_PyLong_SubtractShrink(PyLongObject *left, PyLongObject *right) +{ + if (!_PyLong_IsShrinkSubtractPair(left, right)) { + return PyStackRef_NULL; + } + Py_ssize_t size = _PyLong_DigitCount(left); + assert(size >= 2 && size <= 3); + + int sign = _PyLong_IsNegative(left) ? -1 : 1; + int cmp = 0; + for (Py_ssize_t i = size - 2; i >= 0; i--) { + digit left_digit = left->long_value.ob_digit[i]; + digit right_digit = right->long_value.ob_digit[i]; + if (left_digit != right_digit) { + cmp = left_digit > right_digit ? 1 : -1; + break; + } + } + if (cmp == 0) { + return PyStackRef_FromPyObjectBorrow(_PyLong_GetZero()); + } + + PyLongObject *larger = cmp > 0 ? left : right; + PyLongObject *smaller = cmp > 0 ? right : left; + digit out[3]; + twodigits borrow = 0; + for (Py_ssize_t i = 0; i < size; i++) { + twodigits lhs = (twodigits)larger->long_value.ob_digit[i]; + twodigits rhs = (twodigits)smaller->long_value.ob_digit[i]; + twodigits diff = lhs - rhs - borrow; + out[i] = (digit)(diff & PyLong_MASK); + borrow = lhs < rhs + borrow; + } + assert(borrow == 0); + return shrink_subtract_result(cmp > 0 ? sign : -sign, out, size); +} + static PyObject * long_sub_method(PyObject *a, PyObject *b) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4d7b338e2dbd4c3..80f4371968a58ec 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -643,6 +643,16 @@ dummy_func( EXIT_IF(!_PyLong_CheckExactAndCompact(value_o)); } + op(_GUARD_NOS_EXACT_INT, (left, unused -- left, unused)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + EXIT_IF(!PyLong_CheckExact(left_o)); + } + + op(_GUARD_TOS_EXACT_INT, (value -- value)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyLong_CheckExact(value_o)); + } + op(_GUARD_NOS_OVERFLOWED, (left, unused -- left, unused)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); assert(Py_TYPE(left_o) == &PyLong_Type); @@ -690,10 +700,14 @@ dummy_func( PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + } + else { + res = _PyLong_SubtractShrink((PyLongObject *)left_o, (PyLongObject *)right_o); + } EXIT_IF(PyStackRef_IsNull(res)); l = left; r = right; @@ -707,7 +721,7 @@ dummy_func( _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_ADD_INT + _POP_TOP_INT + _POP_TOP_INT; macro(BINARY_OP_SUBTRACT_INT) = - _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT + _POP_TOP_INT + _POP_TOP_INT; + _GUARD_TOS_EXACT_INT + _GUARD_NOS_EXACT_INT + unused/5 + _BINARY_OP_SUBTRACT_INT + _POP_TOP_INT + _POP_TOP_INT; // Inplace compact int ops: mutate the uniquely-referenced operand // when possible. The op handles decref of TARGET internally so diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e45bbd7cceb295f..95b3be81e0bdac2 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4009,6 +4009,180 @@ break; } + case _GUARD_NOS_EXACT_INT_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef left; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyLong_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_EXACT_INT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyLong_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_EXACT_INT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyLong_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_EXACT_INT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyLong_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_EXACT_INT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_EXACT_INT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_EXACT_INT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_EXACT_INT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _GUARD_NOS_OVERFLOWED_r02: { CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -4403,76 +4577,6 @@ break; } - case _BINARY_OP_SUBTRACT_INT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - break; - } - - case _BINARY_OP_SUBTRACT_INT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = right; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - break; - } - case _BINARY_OP_SUBTRACT_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -4489,9 +4593,21 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + } + else { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res = _PyLong_SubtractShrink((PyLongObject *)left_o, (PyLongObject *)right_o); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + } if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = right; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6178dc70c1b80e7..52dcc07cc1e439a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1346,21 +1346,21 @@ _PyStackRef res; _PyStackRef l; _PyStackRef r; - // _GUARD_TOS_INT + // _GUARD_TOS_EXACT_INT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + if (!PyLong_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } - // _GUARD_NOS_INT + // _GUARD_NOS_EXACT_INT { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { + if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1374,9 +1374,16 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res = _PyLong_SubtractShrink((PyLongObject *)left_o, (PyLongObject *)right_o); + _PyFrame_StackPointerInvalidate(frame); + } if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 5e110360b81b445..5e4b3a9db6f22b6 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -581,6 +581,14 @@ break; } + case _GUARD_NOS_EXACT_INT: { + break; + } + + case _GUARD_TOS_EXACT_INT: { + break; + } + case _GUARD_NOS_OVERFLOWED: { break; } @@ -764,9 +772,13 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - res_stackref = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + res_stackref = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + } + else { + res_stackref = _PyLong_SubtractShrink((PyLongObject *)left_o, (PyLongObject *)right_o); + } if (PyStackRef_IsNull(res_stackref )) { ctx->done = true; break; diff --git a/Python/specialize.c b/Python/specialize.c index 773b55c329c7720..f3d20fea75c6730 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2388,7 +2388,9 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { break; } - if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs)) { + if (PyLong_CheckExact(lhs) && PyLong_CheckExact(rhs) && + (_PyLong_BothAreCompact((PyLongObject *)lhs, (PyLongObject *)rhs) || + _PyLong_IsShrinkSubtractPair((PyLongObject *)lhs, (PyLongObject *)rhs))) { specialize(instr, BINARY_OP_SUBTRACT_INT); return; } From cd51e7df78e478c2afe87078b7f0be00d9c221f0 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Fri, 10 Jul 2026 19:11:12 -0500 Subject: [PATCH 2/2] gh-151289: reduce subtract specialization uop churn --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_ids.h | 14 +- Include/internal/pycore_uop_metadata.h | 42 ----- Modules/_testinternalcapi/test_cases.c.h | 44 ++--- Python/bytecodes.c | 24 +-- Python/executor_cases.c.h | 212 ++++------------------ Python/generated_cases.c.h | 44 ++--- Python/optimizer_cases.c.h | 29 +-- 8 files changed, 112 insertions(+), 299 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 2086602ef095f62..ed2bf1781890e66 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1371,7 +1371,7 @@ _PyOpcode_macro_expansion[256] = { [BINARY_OP_SUBSCR_TUPLE_INT] = { .nuops = 6, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_TUPLE, OPARG_SIMPLE, 0 }, { _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_TUPLE_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBSCR_USTR_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_USTR_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBTRACT_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_EXACT_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_EXACT_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBTRACT_INT] = { .nuops = 3, .uops = { { _BINARY_OP_SUBTRACT_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, [BINARY_SLICE] = { .nuops = 1, .uops = { { _BINARY_SLICE, OPARG_SIMPLE, 0 } } }, [BUILD_INTERPOLATION] = { .nuops = 1, .uops = { { _BUILD_INTERPOLATION, OPARG_SIMPLE, 0 } } }, [BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 54dfc9fc86be931..13bf37595122ca7 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -176,7 +176,6 @@ enum { _GUARD_NOS_COMPACT_ASCII, _GUARD_NOS_DICT_STORE_SUBSCRIPT, _GUARD_NOS_DICT_SUBSCRIPT, - _GUARD_NOS_EXACT_INT, _GUARD_NOS_FLOAT, _GUARD_NOS_INT, _GUARD_NOS_ITER_VIRTUAL, @@ -194,7 +193,6 @@ enum { _GUARD_TOS_ANY_DICT, _GUARD_TOS_ANY_SET, _GUARD_TOS_DICT, - _GUARD_TOS_EXACT_INT, _GUARD_TOS_FLOAT, _GUARD_TOS_FROZENDICT, _GUARD_TOS_FROZENSET, @@ -366,7 +364,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 654 +#define MAX_UOP_ID 652 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -858,10 +856,6 @@ enum { _GUARD_NOS_DICT_SUBSCRIPT_r12, _GUARD_NOS_DICT_SUBSCRIPT_r22, _GUARD_NOS_DICT_SUBSCRIPT_r33, - _GUARD_NOS_EXACT_INT_r02, - _GUARD_NOS_EXACT_INT_r12, - _GUARD_NOS_EXACT_INT_r22, - _GUARD_NOS_EXACT_INT_r33, _GUARD_NOS_FLOAT_r02, _GUARD_NOS_FLOAT_r12, _GUARD_NOS_FLOAT_r22, @@ -930,10 +924,6 @@ enum { _GUARD_TOS_DICT_r11, _GUARD_TOS_DICT_r22, _GUARD_TOS_DICT_r33, - _GUARD_TOS_EXACT_INT_r01, - _GUARD_TOS_EXACT_INT_r11, - _GUARD_TOS_EXACT_INT_r22, - _GUARD_TOS_EXACT_INT_r33, _GUARD_TOS_FLOAT_r01, _GUARD_TOS_FLOAT_r11, _GUARD_TOS_FLOAT_r22, @@ -1442,7 +1432,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1653 +#define MAX_UOP_REGS_ID 1643 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index e6fb2d0730b322b..4eb12273560d4b3 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -103,8 +103,6 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, [_GUARD_TOS_INT] = HAS_EXIT_FLAG, - [_GUARD_NOS_EXACT_INT] = HAS_EXIT_FLAG, - [_GUARD_TOS_EXACT_INT] = HAS_EXIT_FLAG, [_GUARD_NOS_OVERFLOWED] = HAS_EXIT_FLAG, [_GUARD_TOS_OVERFLOWED] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, @@ -1058,24 +1056,6 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _GUARD_TOS_INT_r33 }, }, }, - [_GUARD_NOS_EXACT_INT] = { - .best = { 0, 1, 2, 3 }, - .entries = { - { 2, 0, _GUARD_NOS_EXACT_INT_r02 }, - { 2, 1, _GUARD_NOS_EXACT_INT_r12 }, - { 2, 2, _GUARD_NOS_EXACT_INT_r22 }, - { 3, 3, _GUARD_NOS_EXACT_INT_r33 }, - }, - }, - [_GUARD_TOS_EXACT_INT] = { - .best = { 0, 1, 2, 3 }, - .entries = { - { 1, 0, _GUARD_TOS_EXACT_INT_r01 }, - { 1, 1, _GUARD_TOS_EXACT_INT_r11 }, - { 2, 2, _GUARD_TOS_EXACT_INT_r22 }, - { 3, 3, _GUARD_TOS_EXACT_INT_r33 }, - }, - }, [_GUARD_NOS_OVERFLOWED] = { .best = { 0, 1, 2, 3 }, .entries = { @@ -4115,14 +4095,6 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_INT_r11] = _GUARD_TOS_INT, [_GUARD_TOS_INT_r22] = _GUARD_TOS_INT, [_GUARD_TOS_INT_r33] = _GUARD_TOS_INT, - [_GUARD_NOS_EXACT_INT_r02] = _GUARD_NOS_EXACT_INT, - [_GUARD_NOS_EXACT_INT_r12] = _GUARD_NOS_EXACT_INT, - [_GUARD_NOS_EXACT_INT_r22] = _GUARD_NOS_EXACT_INT, - [_GUARD_NOS_EXACT_INT_r33] = _GUARD_NOS_EXACT_INT, - [_GUARD_TOS_EXACT_INT_r01] = _GUARD_TOS_EXACT_INT, - [_GUARD_TOS_EXACT_INT_r11] = _GUARD_TOS_EXACT_INT, - [_GUARD_TOS_EXACT_INT_r22] = _GUARD_TOS_EXACT_INT, - [_GUARD_TOS_EXACT_INT_r33] = _GUARD_TOS_EXACT_INT, [_GUARD_NOS_OVERFLOWED_r02] = _GUARD_NOS_OVERFLOWED, [_GUARD_NOS_OVERFLOWED_r12] = _GUARD_NOS_OVERFLOWED, [_GUARD_NOS_OVERFLOWED_r22] = _GUARD_NOS_OVERFLOWED, @@ -5445,11 +5417,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_NOS_DICT_SUBSCRIPT_r12] = "_GUARD_NOS_DICT_SUBSCRIPT_r12", [_GUARD_NOS_DICT_SUBSCRIPT_r22] = "_GUARD_NOS_DICT_SUBSCRIPT_r22", [_GUARD_NOS_DICT_SUBSCRIPT_r33] = "_GUARD_NOS_DICT_SUBSCRIPT_r33", - [_GUARD_NOS_EXACT_INT] = "_GUARD_NOS_EXACT_INT", - [_GUARD_NOS_EXACT_INT_r02] = "_GUARD_NOS_EXACT_INT_r02", - [_GUARD_NOS_EXACT_INT_r12] = "_GUARD_NOS_EXACT_INT_r12", - [_GUARD_NOS_EXACT_INT_r22] = "_GUARD_NOS_EXACT_INT_r22", - [_GUARD_NOS_EXACT_INT_r33] = "_GUARD_NOS_EXACT_INT_r33", [_GUARD_NOS_FLOAT] = "_GUARD_NOS_FLOAT", [_GUARD_NOS_FLOAT_r02] = "_GUARD_NOS_FLOAT_r02", [_GUARD_NOS_FLOAT_r12] = "_GUARD_NOS_FLOAT_r12", @@ -5535,11 +5502,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_DICT_r11] = "_GUARD_TOS_DICT_r11", [_GUARD_TOS_DICT_r22] = "_GUARD_TOS_DICT_r22", [_GUARD_TOS_DICT_r33] = "_GUARD_TOS_DICT_r33", - [_GUARD_TOS_EXACT_INT] = "_GUARD_TOS_EXACT_INT", - [_GUARD_TOS_EXACT_INT_r01] = "_GUARD_TOS_EXACT_INT_r01", - [_GUARD_TOS_EXACT_INT_r11] = "_GUARD_TOS_EXACT_INT_r11", - [_GUARD_TOS_EXACT_INT_r22] = "_GUARD_TOS_EXACT_INT_r22", - [_GUARD_TOS_EXACT_INT_r33] = "_GUARD_TOS_EXACT_INT_r33", [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", [_GUARD_TOS_FLOAT_r01] = "_GUARD_TOS_FLOAT_r01", [_GUARD_TOS_FLOAT_r11] = "_GUARD_TOS_FLOAT_r11", @@ -6317,10 +6279,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_TOS_INT: return 0; - case _GUARD_NOS_EXACT_INT: - return 0; - case _GUARD_TOS_EXACT_INT: - return 0; case _GUARD_NOS_OVERFLOWED: return 0; case _GUARD_TOS_OVERFLOWED: diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 8c72c820d184d9c..949c3d182f9e439 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -1340,42 +1340,44 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; - // _GUARD_TOS_EXACT_INT + _PyStackRef value; + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_INT { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - } - // _GUARD_NOS_EXACT_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + bool both_compact = _PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o); + bool shrink_pair = false; + if (!both_compact) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + shrink_pair = _PyLong_IsShrinkSubtractPair((PyLongObject *)left_o, (PyLongObject *)right_o); + _PyFrame_StackPointerInvalidate(frame); + } + if (!(both_compact || shrink_pair)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); - if (_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + if (both_compact) { res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); } else { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 80f4371968a58ec..e4daf98a194b155 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -643,16 +643,6 @@ dummy_func( EXIT_IF(!_PyLong_CheckExactAndCompact(value_o)); } - op(_GUARD_NOS_EXACT_INT, (left, unused -- left, unused)) { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - EXIT_IF(!PyLong_CheckExact(left_o)); - } - - op(_GUARD_TOS_EXACT_INT, (value -- value)) { - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - EXIT_IF(!PyLong_CheckExact(value_o)); - } - op(_GUARD_NOS_OVERFLOWED, (left, unused -- left, unused)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); assert(Py_TYPE(left_o) == &PyLong_Type); @@ -698,11 +688,17 @@ dummy_func( pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); + EXIT_IF(!PyLong_CheckExact(left_o)); + EXIT_IF(!PyLong_CheckExact(right_o)); + bool both_compact = _PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o); + bool shrink_pair = false; + if (!both_compact) { + shrink_pair = _PyLong_IsShrinkSubtractPair((PyLongObject *)left_o, (PyLongObject *)right_o); + } + EXIT_IF(!(both_compact || shrink_pair)); STAT_INC(BINARY_OP, hit); - if (_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + if (both_compact) { res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); } else { @@ -721,7 +717,7 @@ dummy_func( _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_ADD_INT + _POP_TOP_INT + _POP_TOP_INT; macro(BINARY_OP_SUBTRACT_INT) = - _GUARD_TOS_EXACT_INT + _GUARD_NOS_EXACT_INT + unused/5 + _BINARY_OP_SUBTRACT_INT + _POP_TOP_INT + _POP_TOP_INT; + unused/5 + _BINARY_OP_SUBTRACT_INT + _POP_TOP_INT + _POP_TOP_INT; // Inplace compact int ops: mutate the uniquely-referenced operand // when possible. The op handles decref of TARGET internally so diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 95b3be81e0bdac2..59ce53c911436a5 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4009,180 +4009,6 @@ break; } - case _GUARD_NOS_EXACT_INT_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef left; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - break; - } - - case _GUARD_NOS_EXACT_INT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - break; - } - - case _GUARD_NOS_EXACT_INT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - left = _stack_item_0; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = left; - SET_CURRENT_CACHED_VALUES(2); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - break; - } - - case _GUARD_NOS_EXACT_INT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef left; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - left = _stack_item_1; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - break; - } - - case _GUARD_TOS_EXACT_INT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef value; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - break; - } - - case _GUARD_TOS_EXACT_INT_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - value = _stack_item_0; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = value; - SET_CURRENT_CACHED_VALUES(1); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - break; - } - - case _GUARD_TOS_EXACT_INT_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - value = _stack_item_1; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - break; - } - - case _GUARD_TOS_EXACT_INT_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - value = _stack_item_2; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - break; - } - case _GUARD_NOS_OVERFLOWED_r02: { CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -4591,10 +4417,42 @@ left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); + if (!PyLong_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (!PyLong_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + bool both_compact = _PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o); + bool shrink_pair = false; + if (!both_compact) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + shrink_pair = _PyLong_IsShrinkSubtractPair((PyLongObject *)left_o, (PyLongObject *)right_o); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + } + if (!(both_compact || shrink_pair)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_OP, hit); - if (_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + if (both_compact) { res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); } else { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 52dcc07cc1e439a..b6da5bba405f761 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1340,42 +1340,44 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; - // _GUARD_TOS_EXACT_INT + _PyStackRef value; + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_INT { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - } - // _GUARD_NOS_EXACT_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + bool both_compact = _PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o); + bool shrink_pair = false; + if (!both_compact) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + shrink_pair = _PyLong_IsShrinkSubtractPair((PyLongObject *)left_o, (PyLongObject *)right_o); + _PyFrame_StackPointerInvalidate(frame); + } + if (!(both_compact || shrink_pair)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); - if (_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + if (both_compact) { res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); } else { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 5e4b3a9db6f22b6..e785a52e3c5d5bb 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -581,14 +581,6 @@ break; } - case _GUARD_NOS_EXACT_INT: { - break; - } - - case _GUARD_TOS_EXACT_INT: { - break; - } - case _GUARD_NOS_OVERFLOWED: { break; } @@ -770,10 +762,25 @@ /* Start of uop copied from bytecodes for constant evaluation */ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); + if (!PyLong_CheckExact(left_o)) { + ctx->done = true; + break; + } + if (!PyLong_CheckExact(right_o)) { + ctx->done = true; + break; + } + bool both_compact = _PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o); + bool shrink_pair = false; + if (!both_compact) { + shrink_pair = _PyLong_IsShrinkSubtractPair((PyLongObject *)left_o, (PyLongObject *)right_o); + } + if (!(both_compact || shrink_pair)) { + ctx->done = true; + break; + } STAT_INC(BINARY_OP, hit); - if (_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + if (both_compact) { res_stackref = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); } else {