From b4b2106e4f5569df15a883c2c215a3926beb5811 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 25 Jul 2025 21:27:41 -0700 Subject: [PATCH 01/14] _COMPARE_OP and _CONTAINS_OP --- Lib/test/test_capi/test_opt.py | 34 ++++++++++++++ Python/optimizer_bytecodes.c | 2 + Python/optimizer_cases.c.h | 84 +++++++++++++++++++++++++++++++++- 3 files changed, 119 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index c796b0dd4b5b7e..9f5e0a489121a6 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1614,6 +1614,23 @@ def f(n): # But all of the appends we care about are still there: self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) + def test_compare_op_pop_two_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for _ in range(n): + a = 10 + b = 10.0 + if a == b: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_COMPARE_OP", uops) + self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) + def test_compare_op_int_pop_two_load_const_inline_borrow(self): def testfunc(n): x = 0 @@ -1665,6 +1682,23 @@ def testfunc(n): self.assertNotIn("_COMPARE_OP_FLOAT", uops) self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) + def test_contains_op_pop_two_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for _ in range(n): + a = "foo" + s = "foo bar baz" + if a in s: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_CONTAINS_OP", uops) + self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) + def test_to_bool_bool_contains_op_set(self): """ Test that _TO_BOOL_BOOL is removed from code like: diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 77759f67532f80..34b394ddc5be86 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -422,6 +422,7 @@ dummy_func(void) { op(_COMPARE_OP, (left, right -- res)) { if (oparg & 16) { + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); res = sym_new_type(ctx, &PyBool_Type); } else { @@ -449,6 +450,7 @@ dummy_func(void) { } op(_CONTAINS_OP, (left, right -- b)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); b = sym_new_type(ctx, &PyBool_Type); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 99206f01618d79..ba06e65333acb5 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1640,8 +1640,54 @@ } case _COMPARE_OP: { + JitOptRef right; + JitOptRef left; JitOptRef res; - if (oparg & 16) { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (oparg & 16) {if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + if (res_o == NULL) { + goto error; + } + if (oparg & 16) { + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + if (res_bool < 0) { + goto error; + } + res_stackref = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res_stackref = PyStackRef_FromPyObjectSteal(res_o); + } + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } res = sym_new_type(ctx, &PyBool_Type); } else { @@ -1810,7 +1856,43 @@ } case _CONTAINS_OP: { + JitOptRef right; + JitOptRef left; JitOptRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef b_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + int res = PySequence_Contains(right_o, left_o); + if (res < 0) { + goto error; + } + b_stackref = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + /* End of uop copied from bytecodes for constant evaluation */ + b = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(b_stackref)); + + if (sym_is_const(ctx, b)) { + PyObject *result = sym_get_const(ctx, b); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; From cc4b9cec6fe135305e3dddab4116b7bae2d774bc Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 28 Jul 2025 20:58:32 -0700 Subject: [PATCH 02/14] add pop_top --- Tools/cases_generator/optimizer_generator.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index b9985eaf48309d..664c20a3ce3c68 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -233,6 +233,18 @@ def replace_opcode_if_evaluates_pure( else: emitter.emit(f"{outp.name} = sym_new_const(ctx, PyStackRef_AsPyObjectBorrow({outp.name}_stackref));\n") + if len(used_stack_inputs) == 1 and len(self.original_uop.stack.outputs) == 1: + outp = self.original_uop.stack.outputs[0] + if not outp.peek: + emitter.emit(f""" + if (sym_is_const(ctx, {used_stack_inputs[0].name})) {{ + PyObject *result = sym_get_const(ctx, {used_stack_inputs[0].name}); + if (_Py_IsImmortal(result)) {{ + // Replace with _POP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + }} + }}""") + if len(used_stack_inputs) == 2 and len(self.original_uop.stack.outputs) == 1: outp = self.original_uop.stack.outputs[0] if not outp.peek: From 2d5f428f13eab2737c9901c910fd194cc5fb1e40 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 30 Jul 2025 20:16:50 -0700 Subject: [PATCH 03/14] progress --- Tools/cases_generator/optimizer_generator.py | 22 ++++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 664c20a3ce3c68..573b66eb8be0d2 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -233,17 +233,17 @@ def replace_opcode_if_evaluates_pure( else: emitter.emit(f"{outp.name} = sym_new_const(ctx, PyStackRef_AsPyObjectBorrow({outp.name}_stackref));\n") - if len(used_stack_inputs) == 1 and len(self.original_uop.stack.outputs) == 1: - outp = self.original_uop.stack.outputs[0] - if not outp.peek: - emitter.emit(f""" - if (sym_is_const(ctx, {used_stack_inputs[0].name})) {{ - PyObject *result = sym_get_const(ctx, {used_stack_inputs[0].name}); - if (_Py_IsImmortal(result)) {{ - // Replace with _POP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - }} - }}""") + # if len(used_stack_inputs) == 1 and len(self.original_uop.stack.outputs) == 1: + # outp = self.original_uop.stack.outputs[0] + # if not outp.peek: + # emitter.emit(f""" + # if (sym_is_const(ctx, {used_stack_inputs[0].name})) {{ + # PyObject *result = sym_get_const(ctx, {used_stack_inputs[0].name}); + # if (_Py_IsImmortal(result)) {{ + # // Replace with _POP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + # REPLACE_OP(this_instr, _POP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + # }} + # }}""") if len(used_stack_inputs) == 2 and len(self.original_uop.stack.outputs) == 1: outp = self.original_uop.stack.outputs[0] From e75ca80cff4fac57cebd1aee8ee2f84db93620cb Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Mon, 4 Aug 2025 21:23:10 -0700 Subject: [PATCH 04/14] POP TOP --- Lib/test/test_capi/test_opt.py | 49 +++++ Python/optimizer_bytecodes.c | 1 + Python/optimizer_cases.c.h | 190 +++++++++++-------- Tools/cases_generator/optimizer_generator.py | 42 ++-- 4 files changed, 182 insertions(+), 100 deletions(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 9f5e0a489121a6..a7e5f2ef87adbd 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1614,6 +1614,55 @@ def f(n): # But all of the appends we care about are still there: self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) + def test_replace_with_true_pop_top_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for _ in range(n): + a = 42 + result = bool(a) + if result: + x += 1 + return x + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_REPLACE_WITH_TRUE", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + + def test_to_bool_none_pop_top_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for _ in range(n): + a = None + result = bool(a) + if result: + x += 1 + return x + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_TO_BOOL_NONE", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + + def test_get_len_pop_top_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for _ in range(n): + a = "foo" + result = len(a) + if result: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GET_LEN", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + def test_compare_op_pop_two_load_const_inline_borrow(self): def testfunc(n): x = 0 diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 34b394ddc5be86..e7fe1c70255a0d 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -362,6 +362,7 @@ dummy_func(void) { } op(_TO_BOOL_NONE, (value -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(value); int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { sym_set_const(value, Py_None); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index ba06e65333acb5..ebca6e254f4673 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -218,6 +218,14 @@ ? PyStackRef_True : PyStackRef_False; /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-1] = res; break; } @@ -304,6 +312,32 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, value) + ) { + JitOptRef value_sym = value; + _PyStackRef value = sym_get_const_as_stackref(ctx, value_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + if (!PyStackRef_IsNone(value)) { + ctx->done = true; + break; + } + STAT_INC(TO_BOOL, hit); + res_stackref = PyStackRef_False; + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + stack_pointer[-1] = res; + break; + } int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { sym_set_const(value, Py_None); @@ -437,13 +471,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -488,13 +522,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -539,13 +573,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -609,13 +643,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -662,13 +696,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -715,13 +749,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -795,13 +829,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1676,13 +1710,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1727,13 +1761,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1778,13 +1812,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1827,13 +1861,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1881,13 +1915,13 @@ /* End of uop copied from bytecodes for constant evaluation */ b = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(b_stackref)); - if (sym_is_const(ctx, b)) { - PyObject *result = sym_get_const(ctx, b); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, b)) { + PyObject *result = sym_get_const(ctx, b); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -2966,13 +3000,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 573b66eb8be0d2..5a0fb8c536ad4b 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -233,29 +233,27 @@ def replace_opcode_if_evaluates_pure( else: emitter.emit(f"{outp.name} = sym_new_const(ctx, PyStackRef_AsPyObjectBorrow({outp.name}_stackref));\n") - # if len(used_stack_inputs) == 1 and len(self.original_uop.stack.outputs) == 1: - # outp = self.original_uop.stack.outputs[0] - # if not outp.peek: - # emitter.emit(f""" - # if (sym_is_const(ctx, {used_stack_inputs[0].name})) {{ - # PyObject *result = sym_get_const(ctx, {used_stack_inputs[0].name}); - # if (_Py_IsImmortal(result)) {{ - # // Replace with _POP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - # REPLACE_OP(this_instr, _POP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - # }} - # }}""") - - if len(used_stack_inputs) == 2 and len(self.original_uop.stack.outputs) == 1: - outp = self.original_uop.stack.outputs[0] - if not outp.peek: + if len(used_stack_inputs) >= 1 and len(self.original_uop.stack.outputs) == 1: + outp = self.original_uop.stack.outputs[0] + if not outp.peek: + if len(used_stack_inputs) == 1: + emitter.emit(f""" + if (sym_is_const(ctx, {outp.name})) {{ + PyObject *result = sym_get_const(ctx, {outp.name}); + if (_Py_IsImmortal(result)) {{ + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + }} + }}""") + elif len(used_stack_inputs) == 2: emitter.emit(f""" - if (sym_is_const(ctx, {outp.name})) {{ - PyObject *result = sym_get_const(ctx, {outp.name}); - if (_Py_IsImmortal(result)) {{ - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - }} - }}""") + if (sym_is_const(ctx, {outp.name})) {{ + PyObject *result = sym_get_const(ctx, {outp.name}); + if (_Py_IsImmortal(result)) {{ + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + }} + }}""") storage.flush(self.out) emitter.emit("break;\n") From 3b37082d649c45c165b13a8712f20e63c5b23775 Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Tue, 5 Aug 2025 19:13:13 -0700 Subject: [PATCH 05/14] UNARY_NOT --- Lib/test/test_capi/test_opt.py | 17 +++++++++++++++++ Python/optimizer_bytecodes.c | 1 + Python/optimizer_cases.c.h | 26 ++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index a7e5f2ef87adbd..e3899bbf322e60 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1614,6 +1614,23 @@ def f(n): # But all of the appends we care about are still there: self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) + def test_unary_not_pop_top_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for i in range(n): + a = 42 + result = not a + if result: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_UNARY_NOT", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + def test_replace_with_true_pop_top_load_const_inline_borrow(self): def testfunc(n): x = 0 diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e7fe1c70255a0d..f16c2676ddce53 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -398,6 +398,7 @@ dummy_func(void) { } op(_UNARY_NEGATIVE, (value -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(value); if (sym_is_compact_int(value)) { res = sym_new_compact_int(ctx); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index ebca6e254f4673..941fda84c37cbe 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -186,6 +186,32 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, value) + ) { + JitOptRef value_sym = value; + _PyStackRef value = sym_get_const_as_stackref(ctx, value_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) { + goto error; + } + res_stackref = PyStackRef_FromPyObjectSteal(res_o); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + stack_pointer[-1] = res; + break; + } if (sym_is_compact_int(value)) { res = sym_new_compact_int(ctx); } From e7a1e7ea473219b2add60e7b6f7551797c23e51a Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Tue, 5 Aug 2025 19:34:19 -0700 Subject: [PATCH 06/14] UNARY_NEGATIVE --- Lib/test/test_capi/test_opt.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index e3899bbf322e60..df9f99ddb4ba33 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1614,6 +1614,23 @@ def f(n): # But all of the appends we care about are still there: self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) + def test_unary_negative_pop_top_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for i in range(n): + a = 1 # This will make -1, which is definitely immortal + result = -a + if result < 0: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_UNARY_NEGATIVE", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + def test_unary_not_pop_top_load_const_inline_borrow(self): def testfunc(n): x = 0 From f69228a4f561f1320e5095604fd6bc5eaf6a45ad Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Tue, 5 Aug 2025 19:57:28 -0700 Subject: [PATCH 07/14] UNARY_INVERT --- Lib/test/test_capi/test_opt.py | 19 ++++++++++++++++++- Python/optimizer_bytecodes.c | 1 + Python/optimizer_cases.c.h | 26 ++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index df9f99ddb4ba33..20e79fe3be78ae 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1618,7 +1618,7 @@ def test_unary_negative_pop_top_load_const_inline_borrow(self): def testfunc(n): x = 0 for i in range(n): - a = 1 # This will make -1, which is definitely immortal + a = 1 result = -a if result < 0: x += 1 @@ -1648,6 +1648,23 @@ def testfunc(n): self.assertNotIn("_UNARY_NOT", uops) self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + def test_unary_invert_pop_top_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for i in range(n): + a = 0 + result = ~a + if result < 0: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_UNARY_INVERT", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + def test_replace_with_true_pop_top_load_const_inline_borrow(self): def testfunc(n): x = 0 diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index f16c2676ddce53..435c578abc417a 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -414,6 +414,7 @@ dummy_func(void) { } op(_UNARY_INVERT, (value -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(value); if (sym_matches_type(value, &PyLong_Type)) { res = sym_new_type(ctx, &PyLong_Type); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 941fda84c37cbe..131da71fe84cd1 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -417,6 +417,32 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, value) + ) { + JitOptRef value_sym = value; + _PyStackRef value = sym_get_const_as_stackref(ctx, value_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) { + goto error; + } + res_stackref = PyStackRef_FromPyObjectSteal(res_o); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + stack_pointer[-1] = res; + break; + } if (sym_matches_type(value, &PyLong_Type)) { res = sym_new_type(ctx, &PyLong_Type); } From 01a8e2a27f4683de1b0aef6cd9f590b13e9869d3 Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Fri, 8 Aug 2025 19:41:28 -0700 Subject: [PATCH 08/14] Remove test --- Lib/test/test_capi/test_opt.py | 17 ----------------- Python/optimizer_bytecodes.c | 2 +- Python/optimizer_cases.c.h | 25 ++++++++++++++++++++++++- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 20e79fe3be78ae..8f87e6cfad9a8a 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1697,23 +1697,6 @@ def testfunc(n): self.assertNotIn("_TO_BOOL_NONE", uops) self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) - def test_get_len_pop_top_load_const_inline_borrow(self): - def testfunc(n): - x = 0 - for _ in range(n): - a = "foo" - result = len(a) - if result: - x += 1 - return x - - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_GET_LEN", uops) - self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) - def test_compare_op_pop_two_load_const_inline_borrow(self): def testfunc(n): x = 0 diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 435c578abc417a..85a7ba31c56eb5 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1028,7 +1028,7 @@ dummy_func(void) { } op(_REPLACE_WITH_TRUE, (value -- res)) { - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); + REPLACE_OPCODE_IF_EVALUATES_PURE(value); res = sym_new_const(ctx, Py_True); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 131da71fe84cd1..bf1dc166c3bc18 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -406,8 +406,31 @@ } case _REPLACE_WITH_TRUE: { + JitOptRef value; JitOptRef res; - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); + value = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, value) + ) { + JitOptRef value_sym = value; + _PyStackRef value = sym_get_const_as_stackref(ctx, value_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyStackRef_CLOSE(value); + res_stackref = PyStackRef_True; + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + stack_pointer[-1] = res; + break; + } res = sym_new_const(ctx, Py_True); stack_pointer[-1] = res; break; From 49bd7c92f6a3791dbd1f3eed9f8871eb85a292e6 Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Fri, 8 Aug 2025 20:34:51 -0700 Subject: [PATCH 09/14] Add check for _ --- Python/optimizer_cases.c.h | 216 +++++++++---------- Tools/cases_generator/optimizer_generator.py | 39 ++-- 2 files changed, 128 insertions(+), 127 deletions(-) diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b43b7cb99b7d33..355d278c7b8720 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -202,13 +202,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-1] = res; break; } @@ -245,13 +245,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-1] = res; break; } @@ -354,13 +354,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-1] = res; break; } @@ -421,13 +421,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-1] = res; break; } @@ -456,13 +456,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-1] = res; break; } @@ -546,13 +546,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -597,13 +597,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -648,13 +648,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -718,13 +718,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -771,13 +771,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -824,13 +824,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -904,13 +904,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1785,13 +1785,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1836,13 +1836,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1887,13 +1887,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1936,13 +1936,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1990,13 +1990,13 @@ /* End of uop copied from bytecodes for constant evaluation */ b = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(b_stackref)); - if (sym_is_const(ctx, b)) { - PyObject *result = sym_get_const(ctx, b); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, b)) { + PyObject *result = sym_get_const(ctx, b); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3075,13 +3075,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 5a0fb8c536ad4b..0c71cd45021ebe 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -233,27 +233,28 @@ def replace_opcode_if_evaluates_pure( else: emitter.emit(f"{outp.name} = sym_new_const(ctx, PyStackRef_AsPyObjectBorrow({outp.name}_stackref));\n") - if len(used_stack_inputs) >= 1 and len(self.original_uop.stack.outputs) == 1: + if len(self.original_uop.stack.outputs) == 1: outp = self.original_uop.stack.outputs[0] if not outp.peek: - if len(used_stack_inputs) == 1: - emitter.emit(f""" - if (sym_is_const(ctx, {outp.name})) {{ - PyObject *result = sym_get_const(ctx, {outp.name}); - if (_Py_IsImmortal(result)) {{ - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - }} - }}""") - elif len(used_stack_inputs) == 2: - emitter.emit(f""" - if (sym_is_const(ctx, {outp.name})) {{ - PyObject *result = sym_get_const(ctx, {outp.name}); - if (_Py_IsImmortal(result)) {{ - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - }} - }}""") + if self.original_uop.name.startswith('_'): + if len(used_stack_inputs) == 1: + emitter.emit(f""" + if (sym_is_const(ctx, {outp.name})) {{ + PyObject *result = sym_get_const(ctx, {outp.name}); + if (_Py_IsImmortal(result)) {{ + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + }} + }}""") + elif len(used_stack_inputs) == 2: + emitter.emit(f""" + if (sym_is_const(ctx, {outp.name})) {{ + PyObject *result = sym_get_const(ctx, {outp.name}); + if (_Py_IsImmortal(result)) {{ + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + }} + }}""") storage.flush(self.out) emitter.emit("break;\n") From 2ada46bff0316753b73011b46e06d0c42c2160b4 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 9 Aug 2025 04:07:08 +0000 Subject: [PATCH 10/14] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst new file mode 100644 index 00000000000000..1a6f6cc4cf01be --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst @@ -0,0 +1 @@ +Optimize ``_COMPARE_OP``, ``_CONTAINS_OP``, ``_GET_LEN``, ``_REPLACE_WITH_TRUE``, ``_TO_BOOL_NONE``, `_UNARY_NEGATIVE``, ``_UNARY_NOT``, and ``_UNARY_INVERT`` in JIT builds with constant-loading uops (``_POP_TWO_LOAD_CONST_INLINE_BORROW`` and ``_POP_TOP_LOAD_CONST_INLINE_BORROW``), and then remove both to reduce instruction count. From 024690c3bee3c6e0dc47f8dd72f2dcca394e7b4b Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Fri, 8 Aug 2025 21:49:58 -0700 Subject: [PATCH 11/14] Address PR comments --- Lib/test/test_capi/test_opt.py | 32 ----- ...-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst | 2 +- Python/optimizer_bytecodes.c | 4 +- Python/optimizer_cases.c.h | 117 +++++------------- Tools/cases_generator/optimizer_generator.py | 35 +++--- 5 files changed, 53 insertions(+), 137 deletions(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 8f87e6cfad9a8a..86c59b4fd76a9c 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1665,38 +1665,6 @@ def testfunc(n): self.assertNotIn("_UNARY_INVERT", uops) self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) - def test_replace_with_true_pop_top_load_const_inline_borrow(self): - def testfunc(n): - x = 0 - for _ in range(n): - a = 42 - result = bool(a) - if result: - x += 1 - return x - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, TIER2_THRESHOLD) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_REPLACE_WITH_TRUE", uops) - self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) - - def test_to_bool_none_pop_top_load_const_inline_borrow(self): - def testfunc(n): - x = 0 - for _ in range(n): - a = None - result = bool(a) - if result: - x += 1 - return x - res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) - self.assertEqual(res, 0) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_TO_BOOL_NONE", uops) - self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) - def test_compare_op_pop_two_load_const_inline_borrow(self): def testfunc(n): x = 0 diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst index 1a6f6cc4cf01be..c1fa14e0566e15 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst @@ -1 +1 @@ -Optimize ``_COMPARE_OP``, ``_CONTAINS_OP``, ``_GET_LEN``, ``_REPLACE_WITH_TRUE``, ``_TO_BOOL_NONE``, `_UNARY_NEGATIVE``, ``_UNARY_NOT``, and ``_UNARY_INVERT`` in JIT builds with constant-loading uops (``_POP_TWO_LOAD_CONST_INLINE_BORROW`` and ``_POP_TOP_LOAD_CONST_INLINE_BORROW``), and then remove both to reduce instruction count. +Optimize ``_COMPARE_OP``, ``_CONTAINS_OP``, ``_UNARY_NEGATIVE``, ``_UNARY_NOT``, and ``_UNARY_INVERT`` in JIT builds with constant-loading uops (``_POP_TWO_LOAD_CONST_INLINE_BORROW`` and ``_POP_TOP_LOAD_CONST_INLINE_BORROW``), and then remove both to reduce instruction count. diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 85a7ba31c56eb5..474df8aa2e9fe9 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -362,7 +362,6 @@ dummy_func(void) { } op(_TO_BOOL_NONE, (value -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(value); int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { sym_set_const(value, Py_None); @@ -424,8 +423,8 @@ dummy_func(void) { } op(_COMPARE_OP, (left, right -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); if (oparg & 16) { - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); res = sym_new_type(ctx, &PyBool_Type); } else { @@ -1028,7 +1027,6 @@ dummy_func(void) { } op(_REPLACE_WITH_TRUE, (value -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(value); res = sym_new_const(ctx, Py_True); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 355d278c7b8720..8659fc49f2fb3c 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -338,32 +338,6 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; - if ( - sym_is_safe_const(ctx, value) - ) { - JitOptRef value_sym = value; - _PyStackRef value = sym_get_const_as_stackref(ctx, value_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - if (!PyStackRef_IsNone(value)) { - ctx->done = true; - break; - } - STAT_INC(TO_BOOL, hit); - res_stackref = PyStackRef_False; - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } - stack_pointer[-1] = res; - break; - } int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { sym_set_const(value, Py_None); @@ -406,31 +380,7 @@ } case _REPLACE_WITH_TRUE: { - JitOptRef value; JitOptRef res; - value = stack_pointer[-1]; - if ( - sym_is_safe_const(ctx, value) - ) { - JitOptRef value_sym = value; - _PyStackRef value = sym_get_const_as_stackref(ctx, value_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - PyStackRef_CLOSE(value); - res_stackref = PyStackRef_True; - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } - stack_pointer[-1] = res; - break; - } res = sym_new_const(ctx, Py_True); stack_pointer[-1] = res; break; @@ -1754,36 +1704,36 @@ JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (oparg & 16) {if ( - sym_is_safe_const(ctx, left) && - sym_is_safe_const(ctx, right) - ) { - JitOptRef left_sym = left; - JitOptRef right_sym = right; - _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); - _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert((oparg >> 5) <= Py_GE); - PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); - if (res_o == NULL) { + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + if (res_o == NULL) { + goto error; + } + if (oparg & 16) { + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + if (res_bool < 0) { goto error; } - if (oparg & 16) { - int res_bool = PyObject_IsTrue(res_o); - Py_DECREF(res_o); - if (res_bool < 0) { - goto error; - } - res_stackref = res_bool ? PyStackRef_True : PyStackRef_False; - } - else { - res_stackref = PyStackRef_FromPyObjectSteal(res_o); - } - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + res_stackref = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res_stackref = PyStackRef_FromPyObjectSteal(res_o); + } + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); @@ -1792,11 +1742,12 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - break; - } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + if (oparg & 16) { res = sym_new_type(ctx, &PyBool_Type); } else { diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 0c71cd45021ebe..ad88282cf7ebc4 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -232,29 +232,28 @@ def replace_opcode_if_evaluates_pure( emitter.emit(f"{outp.name} = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal({outp.name}_stackref));\n") else: emitter.emit(f"{outp.name} = sym_new_const(ctx, PyStackRef_AsPyObjectBorrow({outp.name}_stackref));\n") - if len(self.original_uop.stack.outputs) == 1: outp = self.original_uop.stack.outputs[0] if not outp.peek: if self.original_uop.name.startswith('_'): - if len(used_stack_inputs) == 1: - emitter.emit(f""" - if (sym_is_const(ctx, {outp.name})) {{ - PyObject *result = sym_get_const(ctx, {outp.name}); - if (_Py_IsImmortal(result)) {{ - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - }} - }}""") - elif len(used_stack_inputs) == 2: + # Map input count to the appropriate constant-loading uop + input_count_to_uop = { + 1: "_POP_TOP_LOAD_CONST_INLINE_BORROW", + 2: "_POP_TWO_LOAD_CONST_INLINE_BORROW" + } + + input_count = len(used_stack_inputs) + if input_count in input_count_to_uop: + replacement_uop = input_count_to_uop[input_count] + input_desc = "one input" if input_count == 1 else "two inputs" emitter.emit(f""" - if (sym_is_const(ctx, {outp.name})) {{ - PyObject *result = sym_get_const(ctx, {outp.name}); - if (_Py_IsImmortal(result)) {{ - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - }} - }}""") + if (sym_is_const(ctx, {outp.name})) {{ + PyObject *result = sym_get_const(ctx, {outp.name}); + if (_Py_IsImmortal(result)) {{ + // Replace with {replacement_uop} since we have {input_desc} and an immortal result + REPLACE_OP(this_instr, {replacement_uop}, 0, (uintptr_t)result); + }} + }}""") storage.flush(self.out) emitter.emit("break;\n") From 7945c196a3673148df6acc1f1820d38d0e3f1649 Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Fri, 8 Aug 2025 21:50:44 -0700 Subject: [PATCH 12/14] Add back replace_op --- Python/optimizer_bytecodes.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 474df8aa2e9fe9..dd275a910846f3 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1027,6 +1027,7 @@ dummy_func(void) { } op(_REPLACE_WITH_TRUE, (value -- res)) { + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); } From 55d098f20c70cab7f90d84bb38fc71903d129eb4 Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Fri, 8 Aug 2025 21:51:04 -0700 Subject: [PATCH 13/14] Update cases --- Python/optimizer_cases.c.h | 225 +++++++++++++++++++------------------ 1 file changed, 113 insertions(+), 112 deletions(-) diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 8659fc49f2fb3c..bfc2bb6257d12d 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -202,13 +202,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-1] = res; break; } @@ -245,13 +245,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-1] = res; break; } @@ -381,6 +381,7 @@ case _REPLACE_WITH_TRUE: { JitOptRef res; + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); stack_pointer[-1] = res; break; @@ -406,13 +407,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-1] = res; break; } @@ -496,13 +497,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -547,13 +548,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -598,13 +599,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -668,13 +669,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -721,13 +722,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -774,13 +775,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -854,13 +855,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1735,13 +1736,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1787,13 +1788,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1838,13 +1839,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1887,13 +1888,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -1941,13 +1942,13 @@ /* End of uop copied from bytecodes for constant evaluation */ b = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(b_stackref)); - if (sym_is_const(ctx, b)) { - PyObject *result = sym_get_const(ctx, b); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, b)) { + PyObject *result = sym_get_const(ctx, b); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-2] = b; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3026,13 +3027,13 @@ /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-2] = res; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); From 4c614a085d584a36dc014ed5b433dc84ed65c0a2 Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Sat, 9 Aug 2025 19:17:16 -0700 Subject: [PATCH 14/14] Guard against booleans in unary_invert --- Python/optimizer_bytecodes.c | 4 +++- Python/optimizer_cases.c.h | 35 ++++++++++++++++++----------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index dd275a910846f3..6dba289f40fe6b 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -413,7 +413,9 @@ dummy_func(void) { } op(_UNARY_INVERT, (value -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(value); + if (!sym_matches_type(value, &PyBool_Type)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(value); + } if (sym_matches_type(value, &PyLong_Type)) { res = sym_new_type(ctx, &PyLong_Type); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index bfc2bb6257d12d..c7041ac39cc31e 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -391,21 +391,21 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; - if ( - sym_is_safe_const(ctx, value) - ) { - JitOptRef value_sym = value; - _PyStackRef value = sym_get_const_as_stackref(ctx, value_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - PyStackRef_CLOSE(value); - if (res_o == NULL) { - goto error; - } - res_stackref = PyStackRef_FromPyObjectSteal(res_o); - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (!sym_matches_type(value, &PyBool_Type)) {if ( + sym_is_safe_const(ctx, value) + ) { + JitOptRef value_sym = value; + _PyStackRef value = sym_get_const_as_stackref(ctx, value_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) { + goto error; + } + res_stackref = PyStackRef_FromPyObjectSteal(res_o); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); @@ -414,8 +414,9 @@ REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } - stack_pointer[-1] = res; - break; + stack_pointer[-1] = res; + break; + } } if (sym_matches_type(value, &PyLong_Type)) { res = sym_new_type(ctx, &PyLong_Type);