diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 355178d9d51ed..4fb373ebed1c7 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -102,7 +102,8 @@ static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_runtime_jit(ZEND_OPCODE_HANDLE static int zend_jit_trace_op_len(const zend_op *opline); static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline); -static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags); + +static uint32_t zend_jit_trace_get_exit_point(zend_jit_ctx *ctx, const zend_op *to_opline, uint32_t flags); static const void *zend_jit_trace_get_exit_addr(uint32_t n); static void zend_jit_trace_add_code(const void *start, uint32_t size); static zend_string *zend_jit_func_name(const zend_op_array *op_array); @@ -802,6 +803,9 @@ static bool zend_jit_may_be_modified(const zend_function *func, const zend_op_ar # pragma clang diagnostic ignored "-Wstring-compare" #endif +static bool zend_jit_inc_call_level(uint8_t opcode); +static bool zend_jit_dec_call_level(uint8_t opcode); + #include "jit/zend_jit_ir.c" #if defined(__clang__) @@ -1608,6 +1612,18 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op call_level++; } +#if ZEND_DEBUG && 0 + { + const void *handler; + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + handler = zend_get_opcode_handler_func(opline); + } else { + handler = opline->handler; + } + ir_RSTORE(8, jit_CONST_FUNC(&ctx, (uintptr_t)handler, IR_FASTCALL_FUNC)); + } +#endif + if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { switch (opline->opcode) { case ZEND_PRE_INC: @@ -1675,10 +1691,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op && zend_jit_next_is_send_result(opline)) { i++; res_use_info = -1; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } else { res_use_info = -1; @@ -1729,10 +1742,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op && zend_jit_next_is_send_result(opline)) { i++; res_use_info = -1; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } else { res_use_info = -1; @@ -1785,10 +1795,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if ((i + 1) <= end && zend_jit_next_is_send_result(opline)) { i++; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } if (!zend_jit_concat(&ctx, opline, op1_info, op2_info, res_addr, @@ -2039,10 +2046,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op && zend_jit_next_is_send_result(opline) && (!(op1_info & MAY_HAVE_DTOR) || !(op1_info & MAY_BE_RC1))) { i++; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } } if (!zend_jit_assign(&ctx, opline, diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 3ec8ae6041a53..539f75b3ae9f3 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -27,6 +27,8 @@ #include "Zend/Optimizer/zend_func_info.h" #include "Zend/Optimizer/zend_call_graph.h" +typedef struct _zend_jit_ctx zend_jit_ctx; + /* Address Encoding */ typedef uintptr_t zend_jit_addr; @@ -62,6 +64,14 @@ typedef uintptr_t zend_jit_addr; #define Z_SSA_VAR(addr) ((addr)>>_ZEND_ADDR_REG_SHIFT) #define Z_IR_REF(addr) ((addr)>>_ZEND_ADDR_REG_SHIFT) +#define ZEND_ADDR_ARG(call, var) ZEND_ADDR_REF_ZVAL(ir_ADD_OFFSET(call, var)) + +#define Z_IS_ARG_ADDR(addr) \ + (Z_MODE(addr) == IS_REF_ZVAL && ( \ + Z_IR_REF(addr) == jit->call \ + || (jit->ctx.ir_base[Z_IR_REF(addr)].op == IR_ADD \ + && jit->ctx.ir_base[Z_IR_REF(addr)].op1 == jit->call))) + #define Z_STORE(addr) \ ((jit->ra && jit->ra[Z_SSA_VAR(addr)].ref) ? \ (jit->ra[Z_SSA_VAR(addr)].flags & ZREG_STORE) : \ @@ -240,9 +250,9 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_H ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_copy_extra_args_helper(ZEND_OPCODE_HANDLER_ARGS); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_copy_extra_args_helper_no_skip_recv(ZEND_OPCODE_HANDLER_ARGS); -bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D); -bool ZEND_FASTCALL zend_jit_nodiscard_helper(OPLINE_D); -bool ZEND_FASTCALL zend_jit_deprecated_nodiscard_helper(OPLINE_D); +bool ZEND_FASTCALL zend_jit_deprecated_helper(zend_execute_data *call); +bool ZEND_FASTCALL zend_jit_nodiscard_helper(zend_execute_data *call); +bool ZEND_FASTCALL zend_jit_deprecated_nodiscard_helper(zend_execute_data *call); void ZEND_FASTCALL zend_jit_undefined_long_key(EXECUTE_DATA_D); void ZEND_FASTCALL zend_jit_undefined_long_key_ex(zend_long key EXECUTE_DATA_DC); void ZEND_FASTCALL zend_jit_undefined_string_key(EXECUTE_DATA_D); @@ -446,6 +456,8 @@ typedef struct _zend_jit_trace_exit_info { int32_t poly_this_ref; int8_t poly_func_reg; int8_t poly_this_reg; + int32_t call_ref; + int8_t call_reg; } zend_jit_trace_exit_info; typedef struct _zend_jit_trace_stack { diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index 60fcdff8a61f3..40bb65078307b 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -18,6 +18,7 @@ #include "jit/ir/ir.h" #include "jit/ir/ir_builder.h" +#include "jit/zend_jit_internal.h" #if defined(IR_TARGET_X86) # define IR_REG_SP 4 /* IR_REG_RSP */ @@ -55,8 +56,6 @@ # error "Unknown IR target" #endif -#define ZREG_RX ZREG_IP - #define OPTIMIZE_FOR_SIZE 0 /* IR builder defines */ @@ -209,9 +208,6 @@ static size_t tsrm_tls_offset = 0; #define jit_EX(_field) \ jit_CALL(jit_FP(jit), _field) -#define jit_RX(_field) \ - jit_CALL(jit_IP(jit), _field) - #define JIT_STUBS(_) \ _(exception_handler, IR_SKIP_PROLOGUE) \ _(exception_handler_undef, IR_SKIP_PROLOGUE) \ @@ -270,13 +266,15 @@ typedef struct _zend_jit_ctx { const zend_op *last_valid_opline; bool use_last_valid_opline; bool track_last_valid_opline; - bool reuse_ip; uint32_t delayed_call_level; int b; /* current basic block number or -1 */ #ifdef ZTS ir_ref tls; #endif ir_ref fp; + ir_ref call; /* EX(call) cache */ + ir_ref poly_func_ref; /* restored from parent trace snapshot */ + ir_ref poly_this_ref; /* restored from parent trace snapshot */ ir_ref trace_loop_ref; ir_ref return_inputs; const zend_op_array *op_array; @@ -319,6 +317,9 @@ typedef struct _zend_jit_registers_buf { static uint32_t zend_jit_exit_point_by_addr(const void *addr); int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs); +static ir_ref zend_jit_deopt_rload(zend_jit_ctx *jit, ir_type type, int32_t reg); +static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit); + static int zend_jit_assign_to_variable(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_use_addr, @@ -622,18 +623,19 @@ static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr) /* Check if we need snapshot entries for polymorphic method call */ zend_jit_trace_info *t = jit->trace; uint32_t exit_point = 0, n = 0; + bool save_call = false; if (addr < 0) { - if (t->exit_count > 0 - && jit->ctx.ir_base[addr].val.u64 == (uintptr_t)zend_jit_trace_get_exit_addr(t->exit_count - 1)) { - exit_point = t->exit_count - 1; - if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) { - n = 2; - } + exit_point = zend_jit_exit_point_by_addr((void*)(uintptr_t) jit->ctx.ir_base[addr].val.u64); + if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) { + n = 2; + } + if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_RESTORE_CALL) { + save_call = true; } } - if (stack_size || n) { + if (stack_size || n || save_call) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint32_t snapshot_size, i; @@ -647,10 +649,10 @@ static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr) break; } } - if (snapshot_size || n) { + if (snapshot_size || n || save_call) { ir_ref snapshot; - snapshot = ir_SNAPSHOT(snapshot_size + n); + snapshot = ir_SNAPSHOT(snapshot_size + n + save_call); for (i = 0; i < snapshot_size; i++) { ir_ref ref = STACK_REF(stack, i); @@ -663,6 +665,10 @@ static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr) ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func_ref); ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this_ref); } + if (save_call) { + ZEND_ASSERT(t->exit_info[exit_point].call_ref != IR_UNUSED); + ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + n + 1, t->exit_info[exit_point].call_ref); + } } } } @@ -721,19 +727,62 @@ void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snaps ZEND_ASSERT(exit_point < t->exit_count); exit_flags = t->exit_info[exit_point].flags; - if (exit_flags & ZEND_JIT_EXIT_METHOD_CALL) { + if (exit_flags & ZEND_JIT_EXIT_RESTORE_CALL) { int8_t *reg_ops = ctx->regs[snapshot_ref]; + ZEND_ASSERT(reg_ops[n] != -1); + + int8_t reg = reg_ops[n]; + ir_ref ref = ir_insn_op(snapshot, n); + if (IR_REG_SPILLED(reg)) { + reg = ((ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP) | IR_REG_SPILL_LOAD; + ref = ir_get_spill_slot_offset(ctx, ref); + } + + if ((exit_flags & ZEND_JIT_EXIT_FIXED) + && (t->exit_info[exit_point].call_reg != reg + || (IR_REG_SPILLED(reg) && t->exit_info[exit_point].call_ref != ref))) { + exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref); + addr = (void*)zend_jit_trace_get_exit_addr(exit_point); + exit_flags &= ~ZEND_JIT_EXIT_FIXED; + } + t->exit_info[exit_point].call_ref = ref; + t->exit_info[exit_point].call_reg = reg; + n--; + } + if (exit_flags & ZEND_JIT_EXIT_METHOD_CALL) { + int8_t *reg_ops = ctx->regs[snapshot_ref]; ZEND_ASSERT(reg_ops[n - 1] != -1 && reg_ops[n] != -1); + + int8_t func_reg = reg_ops[n - 1]; + ir_ref func_ref = ir_insn_op(snapshot, n - 1); + if (IR_REG_SPILLED(func_reg)) { + func_reg = ((ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP) | IR_REG_SPILL_LOAD; + func_ref = ir_get_spill_slot_offset(ctx, func_ref); + } + + int8_t this_reg = reg_ops[n]; + ir_ref this_ref = ir_insn_op(snapshot, n); + if (IR_REG_SPILLED(this_reg)) { + this_reg = ((ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP) | IR_REG_SPILL_LOAD; + this_ref = ir_get_spill_slot_offset(ctx, this_ref); + } + if ((exit_flags & ZEND_JIT_EXIT_FIXED) - && (t->exit_info[exit_point].poly_func_reg != reg_ops[n - 1] - || t->exit_info[exit_point].poly_this_reg != reg_ops[n])) { + && ((t->exit_info[exit_point].poly_func_reg != func_reg + || (IR_REG_SPILLED(func_reg) + && t->exit_info[exit_point].poly_func_ref != func_ref)) + || (t->exit_info[exit_point].poly_this_reg != this_reg + || (IR_REG_SPILLED(this_reg) + && t->exit_info[exit_point].poly_this_ref != this_ref)))) { exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref); addr = (void*)zend_jit_trace_get_exit_addr(exit_point); exit_flags &= ~ZEND_JIT_EXIT_FIXED; } - t->exit_info[exit_point].poly_func_reg = reg_ops[n - 1]; - t->exit_info[exit_point].poly_this_reg = reg_ops[n]; + t->exit_info[exit_point].poly_func_reg = func_reg; + t->exit_info[exit_point].poly_func_ref = func_ref; + t->exit_info[exit_point].poly_this_reg = this_reg; + t->exit_info[exit_point].poly_this_ref = this_ref; n -= 2; } @@ -901,11 +950,17 @@ static ir_ref jit_IP(zend_jit_ctx *jit) return ir_RLOAD_A(ZREG_IP); } -static void jit_STORE_IP(zend_jit_ctx *jit, ir_ref ref) +static void jit_STORE_IP_no_reset(zend_jit_ctx *jit, ir_ref ref) { ir_RSTORE(ZREG_IP, ref); } +static void jit_STORE_IP(zend_jit_ctx *jit, ir_ref ref) +{ + jit_STORE_IP_no_reset(jit, ref); + zend_jit_reset_last_valid_opline(jit); +} + static ir_ref jit_IP32(zend_jit_ctx *jit) { return ir_RLOAD_U32(ZREG_IP); @@ -937,10 +992,8 @@ static bool zend_jit_trace_uses_initial_ip(zend_jit_ctx *jit) static void zend_jit_set_last_valid_opline(zend_jit_ctx *jit, const zend_op *opline) { - if (!jit->reuse_ip) { - jit->track_last_valid_opline = 1; - jit->last_valid_opline = opline; - } + jit->track_last_valid_opline = 1; + jit->last_valid_opline = opline; } static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit) @@ -949,25 +1002,13 @@ static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit) jit->last_valid_opline = NULL; } -static void zend_jit_start_reuse_ip(zend_jit_ctx *jit) -{ - zend_jit_reset_last_valid_opline(jit); - jit->reuse_ip = 1; -} - -static int zend_jit_reuse_ip(zend_jit_ctx *jit) +static ir_ref jit_EX_CALL(zend_jit_ctx *jit) { - if (!jit->reuse_ip) { - zend_jit_start_reuse_ip(jit); - // RX = EX(call); - jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call))); + if (!jit->call) { + jit->call = ir_LOAD_A(jit_EX(call)); } - return 1; -} -static void zend_jit_stop_reuse_ip(zend_jit_ctx *jit) -{ - jit->reuse_ip = 0; + return jit->call; } static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level) @@ -982,7 +1023,8 @@ static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level) call = ir_LOAD_A(jit_EX(call)); } - rx = jit_IP(jit); + ZEND_ASSERT(jit->call != IR_UNUSED || call_level == 1); + rx = jit_EX_CALL(jit); // JIT: call->prev_execute_data = call; ir_STORE(jit_CALL(rx, prev_execute_data), call); @@ -1015,12 +1057,11 @@ static int zend_jit_set_ip(zend_jit_ctx *jit, const zend_op *target) } else { ref = ir_SUB_A(ref, ir_CONST_ADDR((uintptr_t)jit->last_valid_opline - (uintptr_t)target)); } - jit_STORE_IP(jit, ref); + jit_STORE_IP_no_reset(jit, ref); } } else { - jit_STORE_IP(jit, ir_CONST_ADDR(target)); + jit_STORE_IP_no_reset(jit, ir_CONST_ADDR(target)); } - jit->reuse_ip = 0; zend_jit_set_last_valid_opline(jit, target); return 1; } @@ -1043,8 +1084,6 @@ static ir_ref jit_ZVAL_ADDR(zend_jit_ctx *jit, zend_jit_addr addr) if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); - } else if (Z_REG(addr) == ZREG_RX) { - reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } @@ -1072,8 +1111,6 @@ static ir_ref jit_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr) ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); - } else if (Z_REG(addr) == ZREG_RX) { - reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } @@ -1098,8 +1135,6 @@ static ir_ref jit_Z_TYPE_FLAGS(zend_jit_ctx *jit, zend_jit_addr addr) ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); - } else if (Z_REG(addr) == ZREG_RX) { - reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } @@ -1124,8 +1159,6 @@ static ir_ref jit_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr) ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); - } else if (Z_REG(addr) == ZREG_RX) { - reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } @@ -1148,8 +1181,6 @@ static void jit_set_Z_TYPE_INFO_ex(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); - } else if (Z_REG(addr) == ZREG_RX) { - reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } @@ -2019,7 +2050,7 @@ static int zend_jit_leave_function_handler_stub(zend_jit_ctx *jit) if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info); - jit_STORE_IP(jit, + jit_STORE_IP_no_reset(jit, ir_LOAD_A(jit_EX(opline))); ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); } else if (GCC_GLOBAL_REGS) { @@ -2701,7 +2732,6 @@ static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags) jit->last_valid_opline = NULL; jit->use_last_valid_opline = 0; jit->track_last_valid_opline = 0; - jit->reuse_ip = 0; jit->delayed_call_level = 0; delayed_call_chain = 0; jit->b = -1; @@ -2709,6 +2739,9 @@ static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags) jit->tls = IR_UNUSED; #endif jit->fp = IR_UNUSED; + jit->call = IR_UNUSED; + jit->poly_func_ref = IR_UNUSED; + jit->poly_this_ref = IR_UNUSED; jit->trace_loop_ref = IR_UNUSED; jit->return_inputs = IR_UNUSED; jit->bb_start_ref = NULL; @@ -4180,6 +4213,15 @@ static int zend_jit_handler(zend_jit_ctx *jit, const zend_op *opline, int may_th zend_jit_set_last_valid_opline(jit, opline + 1); break; } + + /* Op handler invalidates jit->call by changing EX(call) */ + if (zend_jit_inc_call_level(opline->opcode) + || zend_jit_dec_call_level(opline->opcode) + || opline->opcode == ZEND_SEND_UNPACK + || opline->opcode == ZEND_SEND_ARRAY) { + jit->call = IR_UNUSED; + } + return 1; } @@ -4254,6 +4296,7 @@ static int zend_jit_tail_handler(zend_jit_ctx *jit, const zend_op *opline) jit->b = -1; zend_jit_reset_last_valid_opline(jit); } + jit->call = IR_UNUSED; return 1; } @@ -4797,7 +4840,7 @@ static int zend_jit_inc_dec(zend_jit_ctx *jit, const zend_op *opline, uint32_t o } } - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr)); @@ -4831,7 +4874,7 @@ static int zend_jit_inc_dec(zend_jit_ctx *jit, const zend_op *opline, uint32_t o SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); } } - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { @@ -5091,17 +5134,17 @@ static int zend_jit_math_long_long(zend_jit_ctx *jit, old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0)); - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); } else if (opline->opcode == ZEND_SUB && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) { old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0)); - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); } else { - exit_point = zend_jit_trace_get_exit_point(opline, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); } exit_addr = zend_jit_trace_get_exit_addr(exit_point); @@ -5111,7 +5154,7 @@ static int zend_jit_math_long_long(zend_jit_ctx *jit, ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr)); may_overflow = 0; } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -5591,7 +5634,7 @@ static int zend_jit_math_helper(zend_jit_ctx *jit, if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); - } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { + } else if (Z_IS_ARG_ADDR(res_addr)) { zend_jit_check_exception_undef_result(jit, opline); } else { zend_jit_check_exception(jit); @@ -5976,7 +6019,7 @@ static int zend_jit_long_math_helper(zend_jit_ctx *jit, if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); - } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { + } else if (Z_IS_ARG_ADDR(res_addr)) { zend_jit_check_exception_undef_result(jit, opline); } else { zend_jit_check_exception(jit); @@ -6103,7 +6146,7 @@ static int zend_jit_concat_helper(zend_jit_ctx *jit, if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); - } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { + } else if (Z_IS_ARG_ADDR(res_addr)) { zend_jit_check_exception_undef_result(jit, opline); } else { zend_jit_check_exception(jit); @@ -6404,7 +6447,7 @@ static int zend_jit_assign_to_variable_call(zend_jit_ctx *jit, if (val_info & MAY_BE_UNDEF) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -8037,7 +8080,7 @@ static zend_jit_addr zend_jit_guard_fetch_result_type(zend_jit_ctx *jit, ir_IF_FALSE_cold(if_type); SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY); - exit_point = zend_jit_trace_get_exit_point(opline+1, flags); + exit_point = zend_jit_trace_get_exit_point(jit, opline+1, flags); res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!res_exit_addr) { return 0; @@ -8049,7 +8092,7 @@ static zend_jit_addr zend_jit_guard_fetch_result_type(zend_jit_ctx *jit, } SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY); - exit_point = zend_jit_trace_get_exit_point(opline+1, flags); + exit_point = zend_jit_trace_get_exit_point(jit, opline+1, flags); res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!res_exit_addr) { return 0; @@ -8422,7 +8465,7 @@ typedef struct _zend_closure { static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t used_stack) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -8439,10 +8482,9 @@ static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline, uint32 return 1; } -static int zend_jit_free_trampoline(zend_jit_ctx *jit, int8_t func_reg) +static int zend_jit_free_trampoline(zend_jit_ctx *jit, ir_ref func) { // JIT: if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) - ir_ref func = ir_RLOAD_A(func_reg); ir_ref if_trampoline = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, common.fn_flags))), ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE))); @@ -8459,7 +8501,7 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co uint32_t used_stack; ir_ref used_stack_ref = IR_UNUSED; bool stack_check = 1; - ir_ref rx, ref, top, if_enough_stack, cold_path = IR_UNUSED; + ir_ref rx, ref, top, if_enough_stack, cold_path = IR_UNUSED, cold_call = IR_UNUSED; ZEND_ASSERT(func_ref != IR_NULL); if (func) { @@ -8511,22 +8553,20 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co used_stack_ref = ir_PHI_2(IR_ADDR, ref, used_stack_ref); } - zend_jit_start_reuse_ip(jit); - // JIT: if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { - jit_STORE_IP(jit, ir_LOAD_A(jit_EG(vm_stack_top))); + rx = ir_LOAD_A(jit_EG(vm_stack_top)); if (stack_check) { // JIT: Check Stack Overflow ref = ir_UGE( ir_SUB_A( ir_LOAD_A(jit_EG(vm_stack_end)), - jit_IP(jit)), + rx), used_stack_ref); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { bool may_be_trampoline = !func && (opline->opcode == ZEND_INIT_METHOD_CALL); - int32_t exit_point = zend_jit_trace_get_exit_point(opline, + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, may_be_trampoline ? (ZEND_JIT_EXIT_TO_VM | ZEND_JIT_EXIT_METHOD_CALL) : ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); @@ -8551,7 +8591,7 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { #endif jit_SET_EX_OPLINE(jit, opline); - ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_int_extend_stack_helper), used_stack_ref); + cold_call = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_int_extend_stack_helper), used_stack_ref); } else { if (!is_closure) { ref = func_ref; @@ -8559,10 +8599,9 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co ref = ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func)); } jit_SET_EX_OPLINE(jit, opline); - ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_extend_stack_helper), + cold_call = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_extend_stack_helper), used_stack_ref, ref); } - jit_STORE_IP(jit, ref); cold_path = ir_END(); ir_IF_TRUE(if_enough_stack); @@ -8570,7 +8609,6 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co } ref = jit_EG(vm_stack_top); - rx = jit_IP(jit); #if !OPTIMIZE_FOR_SIZE /* JIT: EG(vm_stack_top) = (zval*)((char*)call + used_stack); * This vesions is longer but faster @@ -8601,7 +8639,7 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co #endif if (cold_path) { ir_MERGE_WITH(cold_path); - rx = jit_IP(jit); + rx = ir_PHI_2(IR_ADDR, rx, cold_call); } // JIT: call->func = func; @@ -8616,7 +8654,7 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co } if (cold_path) { ir_MERGE_WITH(cold_path); - rx = jit_IP(jit); + rx = ir_PHI_2(IR_ADDR, rx, cold_call); } } if (opline->opcode == ZEND_INIT_METHOD_CALL) { @@ -8714,6 +8752,8 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co // JIT: ZEND_CALL_NUM_ARGS(call) = num_args; ir_STORE(jit_CALL(rx, This.u2.num_args), ir_CONST_U32(opline->extended_value)); + jit->call = rx; + return 1; } @@ -8761,7 +8801,7 @@ static int zend_jit_init_fcall_guard(zend_jit_ctx *jit, uint32_t level, const ze return 0; } - exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); + exit_point = zend_jit_trace_get_exit_point(jit, to_opline, ZEND_JIT_EXIT_POLYMORPHISM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -8873,7 +8913,7 @@ static int zend_jit_init_fcall(zend_jit_ctx *jit, const zend_op *opline, uint32_ ZEND_UNREACHABLE(); } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); @@ -8935,15 +8975,16 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, zend_class_entry *trace_ce, zend_jit_trace_rec *trace, int checked_stack, - int8_t func_reg, - int8_t this_reg, + ir_ref func_ref, + ir_ref this_ref, bool polymorphic_side_trace) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; zend_function *func = NULL; zval *function_name; - ir_ref if_static = IR_UNUSED, cold_path, this_ref = IR_NULL, func_ref = IR_NULL; + ir_ref if_static = IR_UNUSED, cold_path; + ir_ref call_ref = IR_UNUSED, static_call_ref = IR_UNUSED; ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); @@ -8962,9 +9003,7 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, if (polymorphic_side_trace) { /* function is passed in r0 from parent_trace */ - ZEND_ASSERT(func_reg >= 0 && this_reg >= 0); - func_ref = zend_jit_deopt_rload(jit, IR_ADDR, func_reg); - this_ref = zend_jit_deopt_rload(jit, IR_ADDR, this_reg); + ZEND_ASSERT(func_ref != IR_UNUSED && this_ref != IR_UNUSED); } else { ir_ref ref, ref2, if_found, fast_path, run_time_cache, this_ref2; @@ -8993,7 +9032,7 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -9104,7 +9143,7 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, int32_t exit_point; const void *exit_addr; - exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); + exit_point = zend_jit_trace_get_exit_point(jit, opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -9129,24 +9168,23 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, } if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { - ir_ref ret; - if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { - ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_method_call_frame_tmp), + static_call_ref = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_method_call_frame_tmp), this_ref, func_ref, ir_CONST_U32(opline->extended_value)); } else { - ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_method_call_frame), + static_call_ref = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_method_call_frame), this_ref, func_ref, ir_CONST_U32(opline->extended_value)); } if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) { - ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler)); + ir_GUARD(static_call_ref, jit_STUB_ADDR(jit, jit_stub_exception_handler)); } - jit_STORE_IP(jit, ret); + + jit->call = static_call_ref; } if (!func) { @@ -9158,12 +9196,13 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, if (!zend_jit_push_call_frame(jit, opline, NULL, func, 0, delayed_fetch_this, checked_stack, func_ref, this_ref)) { return 0; } + call_ref = jit->call; } if (!func) { ir_MERGE_WITH(cold_path); + jit->call = ir_PHI_2(IR_ADDR, call_ref, static_call_ref); } - zend_jit_start_reuse_ip(jit); if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { if (!zend_jit_save_call_chain(jit, call_level)) { @@ -9200,8 +9239,9 @@ static int zend_jit_init_static_method_call(zend_jit_ctx *jit, zend_call_info *call_info = NULL; zend_class_entry *ce; zend_function *func = NULL; - ir_ref func_ref, func_ref2, scope_ref, scope_ref2, if_cached, cold_path, ref; + ir_ref func_ref, func_ref2, scope_ref, scope_ref2, if_cached, cold_path; ir_ref if_static = IR_UNUSED; + ir_ref call_ref = IR_UNUSED, this_call_ref = IR_UNUSED; if (info) { call_info = info->callee_info; @@ -9274,7 +9314,7 @@ static int zend_jit_init_static_method_call(zend_jit_ctx *jit, int32_t exit_point; const void *exit_addr; - exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline, func ? ZEND_JIT_EXIT_INVALIDATE : 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -9300,12 +9340,13 @@ static int zend_jit_init_static_method_call(zend_jit_ctx *jit, } jit_SET_EX_OPLINE(jit, opline); - ref = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_this_method_call_frame), + this_call_ref = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_this_method_call_frame), scope_ref, func_ref, ir_CONST_U32(opline->extended_value)); - ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler)); - jit_STORE_IP(jit, ref); + ir_GUARD(this_call_ref, jit_STUB_ADDR(jit, jit_stub_exception_handler)); + + jit->call = this_call_ref; if (!func) { cold_path = ir_END(); @@ -9337,12 +9378,14 @@ static int zend_jit_init_static_method_call(zend_jit_ctx *jit, return 0; } + call_ref = jit->call; + if (!func) { ir_MERGE_2(cold_path, ir_END()); + jit->call = ir_PHI_2(IR_ADDR, this_call_ref, call_ref); } } - zend_jit_start_reuse_ip(jit); if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { if (!zend_jit_save_call_chain(jit, call_level)) { return 0; @@ -9382,7 +9425,7 @@ static int zend_jit_init_closure_call(zend_jit_ctx *jit, if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -9412,7 +9455,7 @@ static int zend_jit_init_closure_call(zend_jit_ctx *jit, func = (zend_function*)trace->func; opcodes = func->op_array.opcodes; - exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); + exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_CLOSURE_CALL); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -9463,10 +9506,6 @@ static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); - if (!zend_jit_reuse_ip(jit)) { - return 0; - } - if (opline->opcode == ZEND_SEND_VAL_EX) { uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); @@ -9482,11 +9521,12 @@ static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t } } else { ir_ref cond = ir_AND_U32( - ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))), + ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(jit_EX_CALL(jit), func)), offsetof(zend_function, quick_arg_flags))), ir_CONST_U32(mask)); + ir_ref call = jit_EX_CALL(jit); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -9504,6 +9544,7 @@ static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t jit_set_Z_TYPE_INFO(jit, addr, IS_UNDEF); } jit_SET_EX_OPLINE(jit, opline); + jit_STORE_IP_no_reset(jit, call); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_throw_cannot_pass_by_ref)); ir_IF_FALSE(if_pass_by_ref); @@ -9511,7 +9552,7 @@ static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t } } - arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + arg_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), opline->result.var); if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); @@ -9536,11 +9577,7 @@ static int zend_jit_send_ref(zend_jit_ctx *jit, const zend_op *opline, const zen ir_ref ref_path = IR_UNUSED; op1_addr = OP1_ADDR(); - arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); - - if (!zend_jit_reuse_ip(jit)) { - return 0; - } + arg_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), opline->result.var); if (opline->op1_type == IS_VAR) { if (op1_info & MAY_BE_INDIRECT) { @@ -9628,11 +9665,7 @@ static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zen opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || arg_num <= MAX_ARG_FLAG_NUM); - arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); - - if (!zend_jit_reuse_ip(jit)) { - return 0; - } + arg_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), opline->result.var); if (opline->opcode == ZEND_SEND_VAR_EX) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE @@ -9650,7 +9683,7 @@ static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zen // JIT: if (RX->func->quick_arg_flags & mask) ir_ref if_send_by_ref = ir_IF(ir_AND_U32( - ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))), + ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(jit_EX_CALL(jit), func)), offsetof(zend_function, quick_arg_flags))), ir_CONST_U32(mask))); ir_IF_TRUE_cold(if_send_by_ref); @@ -9679,7 +9712,7 @@ static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zen /* Don't generate code that always throws exception */ return 0; } else { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -9697,7 +9730,7 @@ static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zen ir_ref func, if_send_by_ref, if_prefer_ref; // JIT: if (RX->func->quick_arg_flags & mask) - func = ir_LOAD_A(jit_RX(func)); + func = ir_LOAD_A(jit_CALL(jit_EX_CALL(jit), func)); if_send_by_ref = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, quick_arg_flags))), ir_CONST_U32(mask))); @@ -9727,7 +9760,7 @@ static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zen ir_IF_FALSE(if_prefer_ref); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -9757,7 +9790,7 @@ static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zen } else { // JIT: if (RX->This.u1.type_info & ZEND_CALL_SEND_ARG_BY_REF) ir_ref if_send_by_ref = ir_IF(ir_AND_U32( - ir_LOAD_U32(jit_RX(This.u1.type_info)), + ir_LOAD_U32(jit_CALL(jit_EX_CALL(jit), This.u1.type_info)), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF))); ir_IF_TRUE_cold(if_send_by_ref); @@ -9815,7 +9848,7 @@ static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zen ir_IF_FALSE(if_ref); } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -9928,24 +9961,14 @@ static int zend_jit_check_func_arg(zend_jit_ctx *jit, const zend_op *opline) if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); // JIT: ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); - if (jit->reuse_ip) { - ref = jit_IP(jit); - } else { - ref = ir_LOAD_A(jit_EX(call)); - } - ref = jit_CALL(ref, This.u1.type_info); + ref = jit_CALL(jit_EX_CALL(jit), This.u1.type_info); ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF))); } } else { if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); // JIT: ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); - if (jit->reuse_ip) { - ref = jit_IP(jit); - } else { - ref = ir_LOAD_A(jit_EX(call)); - } - ref = jit_CALL(ref, This.u1.type_info); + ref = jit_CALL(jit_EX_CALL(jit), This.u1.type_info); ir_STORE(ref, ir_AND_U32(ir_LOAD_U32(ref), ir_CONST_U32(~ZEND_CALL_SEND_ARG_BY_REF))); } } @@ -9954,11 +9977,7 @@ static int zend_jit_check_func_arg(zend_jit_ctx *jit, const zend_op *opline) uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); ir_ref rx, if_ref, cold_path; - if (!zend_jit_reuse_ip(jit)) { - return 0; - } - - rx = jit_IP(jit); + rx = jit_EX_CALL(jit); ref = ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(rx, func)), offsetof(zend_function, quick_arg_flags))), @@ -9987,11 +10006,7 @@ static int zend_jit_check_undef_args(zend_jit_ctx *jit, const zend_op *opline) { ir_ref call, if_may_have_undef, ret; - if (jit->reuse_ip) { - call = jit_IP(jit); - } else { - call = ir_LOAD_A(jit_EX(call)); - } + call = jit_EX_CALL(jit); if_may_have_undef = ir_IF(ir_AND_U8( ir_LOAD_U8(ir_ADD_OFFSET(call, offsetof(zend_execute_data, This.u1.type_info) + 3)), @@ -10018,6 +10033,8 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen const zend_op *prev_opline; ir_ref rx, func_ref = IR_UNUSED, if_user = IR_UNUSED, user_path = IR_UNUSED; + rx = jit_EX_CALL(jit); + prev_opline = opline - 1; while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { prev_opline--; @@ -10087,20 +10104,12 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); - if (!jit->reuse_ip) { - zend_jit_start_reuse_ip(jit); - // JIT: call = EX(call); - jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call))); - } - rx = jit_IP(jit); - zend_jit_stop_reuse_ip(jit); - jit_SET_EX_OPLINE(jit, opline); if (opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME) { if (!func) { if (trace) { - uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + uint32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -10137,7 +10146,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen if (opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME) { if (!func) { if (!trace) { - ir_ref if_deprecated_nodiscard, ret; + ir_ref if_deprecated_nodiscard; uint32_t no_discard = RETURN_VALUE_USED(opline) ? 0 : ZEND_ACC_NODISCARD; @@ -10147,34 +10156,18 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen ir_IF_TRUE_cold(if_deprecated_nodiscard); ir_ref helper = ir_CONST_FC_FUNC(no_discard ? zend_jit_deprecated_nodiscard_helper : zend_jit_deprecated_helper); - if (GCC_GLOBAL_REGS) { - ret = ir_CALL(IR_BOOL, helper); - } else { - ret = ir_CALL_1(IR_BOOL, helper, rx); - } + ir_ref ret = ir_CALL_1(IR_BOOL, helper, rx); ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler)); ir_MERGE_WITH_EMPTY_FALSE(if_deprecated_nodiscard); } } else { if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { - ir_ref ret; - - if (GCC_GLOBAL_REGS) { - ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper)); - } else { - ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx); - } + ir_ref ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx); ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler)); } if ((func->common.fn_flags & ZEND_ACC_NODISCARD) && !RETURN_VALUE_USED(opline)) { - ir_ref ret; - - if (GCC_GLOBAL_REGS) { - ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_nodiscard_helper)); - } else { - ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_nodiscard_helper), rx); - } + ir_ref ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_nodiscard_helper), rx); ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler)); } } @@ -10411,7 +10404,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen if (ZEND_OBSERVER_ENABLED && (!func || (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) == 0)) { ir_ref observer_handler; - ir_ref rx = jit_FP(jit); + ir_ref rx = jit_EX_CALL(jit); struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref); if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) { ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END); @@ -10423,7 +10416,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen jit_observer_fcall_begin(jit, rx, observer_handler); if (trace) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -10595,7 +10588,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen for (i = 0; i < call_num_args; i++ ) { if (zend_jit_needs_arg_dtor(func, i, call_info)) { uint32_t offset = EX_NUM_TO_VAR(i); - zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset); + zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(ir_ADD_OFFSET(rx, offset)); jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, opline); } @@ -10702,7 +10695,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen if (!zend_interrupt_function) { // TODO: Can we avoid checking for interrupts after each call ??? if (trace && jit->last_valid_opline != opline) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -10728,6 +10721,8 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen ir_MERGE_WITH(user_path); } + jit->call = IR_UNUSED; + return 1; } @@ -10846,7 +10841,7 @@ static int zend_jit_recv(zend_jit_ctx *jit, const zend_op *opline, const zend_op if (!JIT_G(current_frame) || TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 || arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -11946,7 +11941,7 @@ static int zend_jit_fetch_dimension_address_inner(zend_jit_ctx *jit, if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R && !exit_addr) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -11967,7 +11962,7 @@ static int zend_jit_fetch_dimension_address_inner(zend_jit_ctx *jit, ir_IF_TRUE(if_type); } if (op1_info & MAY_BE_PACKED_GUARD) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_PACKED_GUARD); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -12543,7 +12538,7 @@ static int zend_jit_fetch_dim_read(zend_jit_ctx *jit, if (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && !has_concrete_type(op1_info)) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -12597,7 +12592,7 @@ static int zend_jit_fetch_dim_read(zend_jit_ctx *jit, old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_TYPE_ONLY); - exit_point = zend_jit_trace_get_exit_point(opline+1, flags); + exit_point = zend_jit_trace_get_exit_point(jit, opline+1, flags); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!not_found_exit_addr) { @@ -13376,7 +13371,7 @@ static int zend_jit_assign_dim(zend_jit_ctx *jit, } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -13603,7 +13598,7 @@ static int zend_jit_assign_dim_op(zend_jit_ctx *jit, && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY && (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!not_found_exit_addr) { return 0; @@ -13775,7 +13770,7 @@ static int zend_jit_fe_reset(zend_jit_ctx *jit, const zend_op *opline, uint32_t static int zend_jit_packed_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint32_t op_info) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_PACKED_GUARD); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); ir_ref ref; @@ -14153,7 +14148,7 @@ static int zend_jit_fetch_this(zend_jit_ctx *jit, const zend_op *opline, const z !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -14189,7 +14184,7 @@ static int zend_jit_fetch_this(zend_jit_ctx *jit, const zend_op *opline, const z static int zend_jit_class_guard(zend_jit_ctx *jit, const zend_op *opline, ir_ref obj_ref, zend_class_entry *ce) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -14254,7 +14249,7 @@ static int zend_jit_fetch_obj(zend_jit_ctx *jit, } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -14474,7 +14469,7 @@ static int zend_jit_fetch_obj(zend_jit_ctx *jit, if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { /* perform IS_UNDEF check only after result type guard (during deoptimization) */ - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -14795,7 +14790,7 @@ static int zend_jit_assign_obj(zend_jit_ctx *jit, } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -14933,7 +14928,7 @@ static int zend_jit_assign_obj(zend_jit_ctx *jit, } // Undefined property with potential magic __get()/__set() or lazy object if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -15140,7 +15135,7 @@ static int zend_jit_assign_obj_op(zend_jit_ctx *jit, } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -15254,7 +15249,7 @@ static int zend_jit_assign_obj_op(zend_jit_ctx *jit, if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -15336,7 +15331,7 @@ static int zend_jit_assign_obj_op(zend_jit_ctx *jit, uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; if (use_prop_guard) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -15575,7 +15570,7 @@ static int zend_jit_incdec_obj(zend_jit_ctx *jit, } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -15677,7 +15672,7 @@ static int zend_jit_incdec_obj(zend_jit_ctx *jit, if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -15758,7 +15753,7 @@ static int zend_jit_incdec_obj(zend_jit_ctx *jit, ir_ref if_overflow = IR_UNUSED; if (use_prop_guard) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -15932,7 +15927,7 @@ static int zend_jit_incdec_obj(zend_jit_ctx *jit, const void *exit_addr; SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -16219,7 +16214,7 @@ static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_ if (next_opline) { if (next_opline != default_opline) { - exit_point = zend_jit_trace_get_exit_point(default_opline, 0); + exit_point = zend_jit_trace_get_exit_point(jit, default_opline, 0); default_label = zend_jit_trace_get_exit_addr(exit_point); if (!default_label) { return 0; @@ -16233,7 +16228,7 @@ static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_ if (next_opline) { if (next_opline != opline + 1) { - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, 0); fallback_label = zend_jit_trace_get_exit_addr(exit_point); if (!fallback_label) { return 0; @@ -16315,7 +16310,7 @@ static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_ if (target == next_opline) { ir_END_list(continue_list); } else { - exit_point = zend_jit_trace_get_exit_point(target, 0); + exit_point = zend_jit_trace_get_exit_point(jit, target, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -16360,7 +16355,7 @@ static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_ if (next_opline) { if (next_opline != opline + 1) { - exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline + 1, 0); fallback_label = zend_jit_trace_get_exit_addr(exit_point); if (!fallback_label) { return 0; @@ -16440,7 +16435,7 @@ static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_ if (target == next_opline) { ir_END_list(continue_list); } else { - exit_point = zend_jit_trace_get_exit_point(target, 0); + exit_point = zend_jit_trace_get_exit_point(jit, target, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -16576,7 +16571,7 @@ static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_ if (target == next_opline) { ir_END_list(continue_list); } else { - exit_point = zend_jit_trace_get_exit_point(target, 0); + exit_point = zend_jit_trace_get_exit_point(jit, target, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; @@ -16853,7 +16848,7 @@ static const void *zend_jit_trace_allocate_exit_group(uint32_t n) static int zend_jit_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint8_t type) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); @@ -16867,7 +16862,7 @@ static int zend_jit_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_ static int zend_jit_scalar_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); @@ -16881,7 +16876,7 @@ static int zend_jit_scalar_type_guard(zend_jit_ctx *jit, const zend_op *opline, static bool zend_jit_noref_guard(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_addr) { - uint32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + uint32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -16894,7 +16889,7 @@ static bool zend_jit_noref_guard(zend_jit_ctx *jit, const zend_op *opline, zend_ static int zend_jit_trace_opline_guard(zend_jit_ctx *jit, const zend_op *opline) { - uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); + uint32_t exit_point = zend_jit_trace_get_exit_point(jit, NULL, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -16918,7 +16913,7 @@ static bool zend_jit_guard_reference(zend_jit_ctx *jit, ir_ref ref; if (add_ref_guard) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -16952,7 +16947,7 @@ static bool zend_jit_fetch_reference(zend_jit_ctx *jit, ir_ref ref; if (add_ref_guard || add_type_guard) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -17013,7 +17008,7 @@ static bool zend_jit_fetch_indirect_var(zend_jit_ctx *jit, const zend_op *opline ir_ref ref = IR_UNUSED; if (add_indirect_guard) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + int32_t exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -17036,7 +17031,7 @@ static bool zend_jit_fetch_indirect_var(zend_jit_ctx *jit, const zend_op *opline if (!(var_type & IS_TRACE_REFERENCE) && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { - exit_point = zend_jit_trace_get_exit_point(opline, 0); + exit_point = zend_jit_trace_get_exit_point(jit, opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -17194,7 +17189,7 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); } - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(jit, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { @@ -17223,12 +17218,21 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr zend_jit_set_last_valid_opline(jit, trace->opline); + /* Op handler invalidates jit->call by changing EX(call) */ + if (zend_jit_inc_call_level(opline->opcode) + || zend_jit_dec_call_level(opline->opcode) + || opline->opcode == ZEND_SEND_UNPACK + || opline->opcode == ZEND_SEND_ARRAY) { + jit->call = IR_UNUSED; + } + return 1; } static int zend_jit_deoptimizer_start(zend_jit_ctx *jit, zend_string *name, uint32_t trace_num, + zend_jit_trace_info *trace, uint32_t exit_num) { zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET); @@ -17319,8 +17323,19 @@ static int zend_jit_trace_start(zend_jit_ctx *jit, if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { ZEND_ASSERT(parent->exit_info[exit_num].poly_func_reg >= 0 && parent->exit_info[exit_num].poly_this_reg >= 0); - ir_RLOAD_A(parent->exit_info[exit_num].poly_func_reg); - ir_RLOAD_A(parent->exit_info[exit_num].poly_this_reg); + if (!IR_REG_SPILLED(parent->exit_info[exit_num].poly_func_reg)) { + ir_RLOAD_A(parent->exit_info[exit_num].poly_func_reg); + } + if (!IR_REG_SPILLED(parent->exit_info[exit_num].poly_this_reg)) { + ir_RLOAD_A(parent->exit_info[exit_num].poly_this_reg); + } + } + + if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) { + ZEND_ASSERT(parent->exit_info[exit_num].call_reg >= 0); + if (!IR_REG_SPILLED(parent->exit_info[exit_num].call_reg)) { + ir_RLOAD_A(parent->exit_info[exit_num].call_reg); + } } ir_STORE(jit_EG(jit_trace_num), ir_CONST_U32(trace_num)); diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 68c096d9d3df2..198f792274707 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -131,7 +131,7 @@ static uint32_t zend_jit_exit_point_by_addr(const void *addr) return (uint32_t)-1; } -static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags) +static uint32_t zend_jit_trace_get_exit_point(zend_jit_ctx *jit, const zend_op *to_opline, uint32_t flags) { zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; uint32_t exit_point; @@ -140,7 +140,7 @@ static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t uint32_t stack_size; zend_jit_trace_stack *stack = NULL; - if (delayed_call_chain) { + if (jit->delayed_call_level) { assert(to_opline != NULL); /* CALL and IP share the same register */ flags |= ZEND_JIT_EXIT_RESTORE_CALL; } @@ -166,7 +166,7 @@ static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t /* Try to reuse exit points */ if (to_opline != NULL - && !(flags & ZEND_JIT_EXIT_METHOD_CALL) + && !(flags & (ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_RESTORE_CALL)) && t->exit_count > 0) { uint32_t i = t->exit_count; @@ -193,6 +193,7 @@ static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack)); memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)); } + ZEND_ASSERT(!(flags & ZEND_JIT_EXIT_RESTORE_CALL) || jit->call); t->exit_count++; t->exit_info[exit_point].opline = to_opline; t->exit_info[exit_point].op_array = op_array; @@ -203,6 +204,8 @@ static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t t->exit_info[exit_point].poly_this_ref = 0; t->exit_info[exit_point].poly_func_reg = ZREG_NONE; t->exit_info[exit_point].poly_this_reg = ZREG_NONE; + t->exit_info[exit_point].call_ref = jit->call; + t->exit_info[exit_point].call_reg = ZREG_NONE; } return exit_point; @@ -3513,17 +3516,18 @@ static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t } static int zend_jit_trace_deoptimization( - zend_jit_ctx *jit, - uint32_t flags, - const zend_op *opline, - zend_jit_trace_stack *parent_stack, - int parent_vars_count, - zend_ssa *ssa, - zend_jit_trace_stack *stack, - zend_jit_exit_const *constants, - int8_t func_reg, - bool polymorphic_side_trace) + zend_jit_ctx *jit, + const zend_jit_trace_exit_info *exit_info, + zend_jit_trace_stack *parent_stack, + int parent_vars_count, + zend_ssa *ssa, + zend_jit_trace_stack *stack, + zend_jit_exit_const *constants, + bool polymorphic_side_trace) { + uint32_t flags = exit_info->flags; + const zend_op *opline = exit_info->opline; + int i; int check2 = -1; @@ -3628,6 +3632,17 @@ static int zend_jit_trace_deoptimization( } } + if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { + int8_t call_reg = exit_info->call_reg; + ir_ref call_ref = exit_info->call_ref; + ZEND_ASSERT(call_reg >= 0); + if (IR_REG_SPILLED(exit_info->call_reg)) { + jit->call = ir_LOAD_A(ir_ADD_OFFSET(zend_jit_deopt_rload(jit, IR_ADDR, IR_REG_NUM(call_reg)), call_ref)); + } else { + jit->call = zend_jit_deopt_rload(jit, IR_ADDR, call_reg); + } + } + if (check2 != -1) { int8_t reg = STACK_REG(parent_stack, check2); @@ -3667,9 +3682,27 @@ static int zend_jit_trace_deoptimization( zend_jit_check_exception(jit); } - if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) { - if (!zend_jit_free_trampoline(jit, func_reg)) { - return 0; + if (flags & ZEND_JIT_EXIT_METHOD_CALL) { + int8_t func_reg = exit_info->poly_func_reg; + ir_ref func_ref = exit_info->poly_func_ref; + if (IR_REG_SPILLED(func_reg)) { + jit->poly_func_ref = ir_LOAD_A(ir_ADD_OFFSET(zend_jit_deopt_rload(jit, IR_ADDR, IR_REG_NUM(func_reg)), func_ref)); + } else { + jit->poly_func_ref = zend_jit_deopt_rload(jit, IR_ADDR, func_reg); + } + + int8_t this_reg = exit_info->poly_this_reg; + ir_ref this_ref = exit_info->poly_this_ref; + if (IR_REG_SPILLED(this_reg)) { + jit->poly_this_ref = ir_LOAD_A(ir_ADD_OFFSET(zend_jit_deopt_rload(jit, IR_ADDR, IR_REG_NUM(this_reg)), this_ref)); + } else { + jit->poly_this_ref = zend_jit_deopt_rload(jit, IR_ADDR, this_reg); + } + + if (!polymorphic_side_trace) { + if (!zend_jit_free_trampoline(jit, jit->poly_func_ref)) { + return 0; + } } } @@ -4264,11 +4297,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (parent_trace) { /* Deoptimization */ if (!zend_jit_trace_deoptimization(&ctx, - zend_jit_traces[parent_trace].exit_info[exit_num].flags, - zend_jit_traces[parent_trace].exit_info[exit_num].opline, + &zend_jit_traces[parent_trace].exit_info[exit_num], parent_stack, parent_vars_count, ssa, stack, zend_jit_traces[parent_trace].constants, - zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg, polymorphic_side_trace)) { goto jit_failure; } @@ -4362,7 +4393,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par // if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { // if (ra && dzend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) { -// uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); +// uint32_t exit_point = zend_jit_trace_get_exit_point(&ctx, opline, ZEND_JIT_EXIT_TO_VM); // // timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point); // if (!timeout_exit_addr) { @@ -4388,6 +4419,14 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par bool gen_handler = false; opline = p->opline; + +#if ZEND_DEBUG && 0 + { + const void *handler = ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->call_handler; + ir_RSTORE(8, jit_CONST_FUNC(&ctx, (uintptr_t)handler, IR_FASTCALL_FUNC)); + } +#endif + if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) { op1_type = IS_UNKNOWN; } @@ -4503,10 +4542,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && zend_jit_trace_next_is_send_result(opline, p, frame)) { send_result = 1; res_use_info = -1; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } else { res_use_info = zend_jit_trace_type_to_info( STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))); @@ -4575,10 +4611,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par && zend_jit_trace_next_is_send_result(opline, p, frame)) { send_result = 1; res_use_info = -1; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } else { res_use_info = zend_jit_trace_type_to_info( STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))); @@ -4640,10 +4673,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par res_addr = RES_REG_ADDR(); if (zend_jit_trace_next_is_send_result(opline, p, frame)) { send_result = 1; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } if (!zend_jit_concat(&ctx, opline, op1_info, op2_info, res_addr, @@ -5132,10 +5162,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (Z_MODE(res_addr) != IS_REG && zend_jit_trace_next_is_send_result(opline, p, frame)) { send_result = 1; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } } op1_def_addr = op1_addr; @@ -5157,10 +5184,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (Z_MODE(res_addr) != IS_REG && zend_jit_trace_next_is_send_result(opline, p, frame)) { send_result = 1; - res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); - if (!zend_jit_reuse_ip(&ctx)) { - goto jit_failure; - } + res_addr = ZEND_ADDR_ARG(jit_EX_CALL(jit), (opline+1)->result.var); } } if (!zend_jit_assign(&ctx, opline, @@ -5411,7 +5435,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (ra) { zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes); } - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { goto jit_failure; @@ -5459,7 +5483,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (ra) { zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes); } - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { goto jit_failure; @@ -5494,7 +5518,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { bool exit_if_true = 0; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); - uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + uint32_t exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -5524,7 +5548,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if (ra) { zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes); } - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { goto jit_failure; @@ -5669,14 +5693,14 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { goto jit_failure; } } else { - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { goto jit_failure; @@ -5716,7 +5740,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } else { ZEND_UNREACHABLE(); } - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { goto jit_failure; @@ -5748,7 +5772,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { bool exit_if_true = 0; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); - uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + uint32_t exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -5778,7 +5802,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { bool exit_if_true = 0; const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); - uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + uint32_t exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -5954,10 +5978,10 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info); } else { - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); } exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -6298,7 +6322,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } else { ZEND_UNREACHABLE(); } - exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_point = zend_jit_trace_get_exit_point(&ctx, exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { goto jit_failure; @@ -6380,8 +6404,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par op_array, ssa, ssa_op, frame->call_level, op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce, p + 1, peek_checked_stack - checked_stack, - polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg : -1, - polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_this_reg : -1, + polymorphic_side_trace ? jit->poly_func_ref : -1, + polymorphic_side_trace ? jit->poly_this_ref : -1, polymorphic_side_trace)) { goto jit_failure; } @@ -7000,6 +7024,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par TRACE_FRAME_SET_CLOSURE_CALL(call); } } + if (init_opline) { if (init_opline->opcode != ZEND_NEW && (init_opline->opcode != ZEND_INIT_METHOD_CALL @@ -7196,7 +7221,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par || (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) { /* Deoptimize to the first instruction of the loop */ - uint32_t exit_point = zend_jit_trace_get_exit_point(trace_buffer[1].opline, ZEND_JIT_EXIT_TO_VM); + uint32_t exit_point = zend_jit_trace_get_exit_point(&ctx, trace_buffer[1].opline, ZEND_JIT_EXIT_TO_VM); timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!timeout_exit_addr) { @@ -7267,7 +7292,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par for (i = 0; i < op_array->last_var + op_array->T; i++) { SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1); } - exit_point = zend_jit_trace_get_exit_point(zend_jit_traces[t->link].opline, ZEND_JIT_EXIT_TO_VM); + exit_point = zend_jit_trace_get_exit_point(&ctx, zend_jit_traces[t->link].opline, ZEND_JIT_EXIT_TO_VM); timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!timeout_exit_addr) { goto jit_failure; @@ -7404,7 +7429,7 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n name = zend_jit_trace_escape_name(trace_num, exit_num); - if (!zend_jit_deoptimizer_start(&ctx, name, trace_num, exit_num)) { + if (!zend_jit_deoptimizer_start(&ctx, name, trace_num, &zend_jit_traces[trace_num], exit_num)) { zend_string_release(name); return NULL; } @@ -7418,11 +7443,9 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n NULL; if (!zend_jit_trace_deoptimization(&ctx, - zend_jit_traces[trace_num].exit_info[exit_num].flags, - zend_jit_traces[trace_num].exit_info[exit_num].opline, + &zend_jit_traces[trace_num].exit_info[exit_num], stack, stack_size, NULL, NULL, zend_jit_traces[trace_num].constants, - zend_jit_traces[trace_num].exit_info[exit_num].poly_func_reg, 0)) { goto jit_failure; } @@ -7997,7 +8020,8 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t) fprintf(stderr, "/VM"); } if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) { - fprintf(stderr, "/CALL"); + fprintf(stderr, "/CALL(%s)", + zend_reg_name(t->exit_info[i].call_reg)); } if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) { fprintf(stderr, "/POLY"); @@ -8624,7 +8648,15 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf zend_jit_trace_stack *stack = stack_size ? t->stack_map + t->exit_info[exit_num].stack_offset : NULL; if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) { - zend_execute_data *call = (zend_execute_data *)regs->gpr[ZREG_RX]; + int8_t reg = t->exit_info[exit_num].call_reg; + ZEND_ASSERT(reg != ZREG_NONE); + + zend_execute_data *call; + if (IR_REG_SPILLED(reg)) { + call = *(zend_execute_data**)(regs->gpr[IR_REG_NUM(reg)] + t->exit_info[exit_num].call_ref); + } else { + call = (zend_execute_data*)regs->gpr[reg]; + } call->prev_execute_data = EX(call); EX(call) = call; } @@ -8730,9 +8762,15 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf } } if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { - ZEND_ASSERT(t->exit_info[exit_num].poly_func_reg >= 0); - zend_function *func = (zend_function*)regs->gpr[t->exit_info[exit_num].poly_func_reg]; + int8_t reg = t->exit_info[exit_num].poly_func_reg; + ZEND_ASSERT(reg >= 0); + zend_function *func; + if (IR_REG_SPILLED(reg)) { + func = *(zend_function**)(regs->gpr[IR_REG_NUM(reg)] + t->exit_info[exit_num].poly_func_ref); + } else { + func = (zend_function*)regs->gpr[reg]; + } if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) { zend_string_release_ex(func->common.function_name, 0); zend_free_trampoline(func); diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 0ce7f082ae15f..e98e8eefe89a8 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -176,9 +176,8 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_copy_extra_args_helper_no_skip_re return zend_jit_copy_extra_args_helper_ex(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_EX false); } -bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D) +bool ZEND_FASTCALL zend_jit_deprecated_helper(zend_execute_data *call) { - zend_execute_data *call = (zend_execute_data *) opline; zend_function *fbc = call->func; zend_deprecated_function(fbc); @@ -204,9 +203,8 @@ bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D) return 1; } -bool ZEND_FASTCALL zend_jit_nodiscard_helper(OPLINE_D) +bool ZEND_FASTCALL zend_jit_nodiscard_helper(zend_execute_data *call) { - zend_execute_data *call = (zend_execute_data *) opline; zend_function *fbc = call->func; zend_nodiscard_function(fbc); @@ -232,19 +230,18 @@ bool ZEND_FASTCALL zend_jit_nodiscard_helper(OPLINE_D) return 1; } -bool ZEND_FASTCALL zend_jit_deprecated_nodiscard_helper(OPLINE_D) +bool ZEND_FASTCALL zend_jit_deprecated_nodiscard_helper(zend_execute_data *call) { - zend_execute_data *call = (zend_execute_data *) opline; zend_function *fbc = call->func; if (fbc->common.fn_flags & ZEND_ACC_DEPRECATED) { - if (zend_jit_deprecated_helper(OPLINE_C) == 0) { + if (zend_jit_deprecated_helper(call) == 0) { return 0; } } if (fbc->common.fn_flags & ZEND_ACC_NODISCARD) { - if (zend_jit_nodiscard_helper(OPLINE_C) == 0) { + if (zend_jit_nodiscard_helper(call) == 0) { return 0; } }