From 7f7b3cdb90cd38026e5c5c031903777722cbdd72 Mon Sep 17 00:00:00 2001 From: Arnaud Le Blanc Date: Wed, 2 Jul 2025 09:23:16 +0200 Subject: [PATCH 1/3] Introduce zend_vm_opcode_handler_t / zend_vm_opcode_handler_func_t This reduces the chances of confusion between opcode handlers used by the VM, and opcode handler functions used for tracing or debugging. Depending on the VM, zend_vm_opcode_handler_t may not be a function. For instance in the HYBRID VM this is a label pointer. Closes GH-19006 --- Zend/zend_compile.h | 3 +- Zend/zend_vm_execute.h | 22 ++++++---- Zend/zend_vm_execute.skl | 2 +- Zend/zend_vm_gen.php | 48 ++++++++++++++------- Zend/zend_vm_opcodes.h | 14 ++++++ ext/opcache/jit/zend_jit.c | 62 +++++++++++++-------------- ext/opcache/jit/zend_jit_internal.h | 12 +++--- ext/opcache/jit/zend_jit_ir.c | 30 +++++-------- ext/opcache/jit/zend_jit_trace.c | 28 ++++++------ ext/opcache/jit/zend_jit_vm_helpers.c | 8 +++- 10 files changed, 130 insertions(+), 99 deletions(-) diff --git a/Zend/zend_compile.h b/Zend/zend_compile.h index 8460c0fac3d45..39a159236f035 100644 --- a/Zend/zend_compile.h +++ b/Zend/zend_compile.h @@ -24,6 +24,7 @@ #include "zend_types.h" #include "zend_map_ptr.h" #include "zend_alloc.h" +#include "zend_vm_opcodes.h" #include #include @@ -135,7 +136,7 @@ void zend_const_expr_to_zval(zval *result, zend_ast **ast_ptr, bool allow_dynami typedef int (*user_opcode_handler_t) (zend_execute_data *execute_data); struct _zend_op { - const void *handler; + zend_vm_opcode_handler_t handler; znode_op op1; znode_op op2; znode_op result; diff --git a/Zend/zend_vm_execute.h b/Zend/zend_vm_execute.h index 0107369ffa714..0d282e39e9cf2 100644 --- a/Zend/zend_vm_execute.h +++ b/Zend/zend_vm_execute.h @@ -321,14 +321,14 @@ static uint8_t zend_user_opcodes[256] = {0, #define SPEC_RULE_OBSERVER 0x02000000 static const uint32_t *zend_spec_handlers; -static const void * const *zend_opcode_handlers; +static zend_vm_opcode_handler_t const *zend_opcode_handlers; static int zend_handlers_count; #if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) -static const void * const * zend_opcode_handler_funcs; +static zend_vm_opcode_handler_func_t const * zend_opcode_handler_funcs; static zend_op hybrid_halt_op; #endif #if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID) || !ZEND_VM_SPEC -static const void *zend_vm_get_opcode_handler(uint8_t opcode, const zend_op* op); +static zend_vm_opcode_handler_t zend_vm_get_opcode_handler(uint8_t opcode, const zend_op* op); #endif #if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) @@ -58666,8 +58666,8 @@ ZEND_API void execute_ex(zend_execute_data *ex) (void*)&&ZEND_FE_FETCH_R_SIMPLE_SPEC_VAR_CV_RETVAL_USED_LABEL, (void*)&&ZEND_NULL_LABEL }; - zend_opcode_handlers = (const void **) labels; - zend_handlers_count = sizeof(labels) / sizeof(void*); + zend_opcode_handlers = (zend_vm_opcode_handler_t*) labels; + zend_handlers_count = sizeof(labels) / sizeof(labels[0]); memset(&hybrid_halt_op, 0, sizeof(hybrid_halt_op)); hybrid_halt_op.handler = (void*)&&HYBRID_HALT_LABEL; #ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE @@ -64398,7 +64398,11 @@ ZEND_API void zend_execute(zend_op_array *op_array, zval *return_value) void zend_vm_init(void) { - static const void * const labels[] = { +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) + static zend_vm_opcode_handler_func_t const labels[] = { +#else + static zend_vm_opcode_handler_t const labels[] = { +#endif ZEND_NOP_SPEC_HANDLER, ZEND_ADD_SPEC_CONST_CONST_HANDLER, ZEND_ADD_SPEC_CONST_TMPVARCV_HANDLER, @@ -68158,7 +68162,7 @@ void zend_vm_init(void) execute_ex(NULL); #else zend_opcode_handlers = labels; - zend_handlers_count = sizeof(labels) / sizeof(void*); + zend_handlers_count = sizeof(labels) / sizeof(labels[0]); zend_spec_handlers = specs; #endif VM_TRACE_START(); @@ -68200,7 +68204,7 @@ ZEND_API void ZEND_FASTCALL zend_serialize_opcode_handler(zend_op *op) } zv = zend_hash_index_find(zend_handlers_table, (zend_long)(uintptr_t)op->handler); ZEND_ASSERT(zv != NULL); - op->handler = (const void *)(uintptr_t)Z_LVAL_P(zv); + op->handler = (zend_vm_opcode_handler_t)(uintptr_t)Z_LVAL_P(zv); } ZEND_API void ZEND_FASTCALL zend_deserialize_opcode_handler(zend_op *op) @@ -68286,7 +68290,7 @@ static uint32_t ZEND_FASTCALL zend_vm_get_opcode_handler_idx(uint32_t spec, cons } #if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID) || !ZEND_VM_SPEC -static const void *zend_vm_get_opcode_handler(uint8_t opcode, const zend_op* op) +static zend_vm_opcode_handler_t zend_vm_get_opcode_handler(uint8_t opcode, const zend_op* op) { return zend_opcode_handlers[zend_vm_get_opcode_handler_idx(zend_spec_handlers[opcode], op)]; } diff --git a/Zend/zend_vm_execute.skl b/Zend/zend_vm_execute.skl index 2417b858b5b18..3eabb4c46bedf 100644 --- a/Zend/zend_vm_execute.skl +++ b/Zend/zend_vm_execute.skl @@ -126,7 +126,7 @@ ZEND_API void ZEND_FASTCALL zend_serialize_opcode_handler(zend_op *op) } zv = zend_hash_index_find(zend_handlers_table, (zend_long)(uintptr_t)op->handler); ZEND_ASSERT(zv != NULL); - op->handler = (const void *)(uintptr_t)Z_LVAL_P(zv); + op->handler = (zend_vm_opcode_handler_t)(uintptr_t)Z_LVAL_P(zv); } ZEND_API void ZEND_FASTCALL zend_deserialize_opcode_handler(zend_op *op) diff --git a/Zend/zend_vm_gen.php b/Zend/zend_vm_gen.php index dbd7da0430f1c..813e629fea008 100755 --- a/Zend/zend_vm_gen.php +++ b/Zend/zend_vm_gen.php @@ -1186,7 +1186,7 @@ function gen_null_label($f, $kind, $prolog) { out($f,$prolog."ZEND_NULL_HANDLER,\n"); break; case ZEND_VM_KIND_SWITCH: - out($f,$prolog."(void*)(uintptr_t)-1,\n"); + out($f,$prolog."-1,\n"); break; case ZEND_VM_KIND_GOTO: out($f,$prolog."(void*)&&ZEND_NULL_LABEL,\n"); @@ -1388,7 +1388,7 @@ function gen_labels($f, $spec, $kind, $prolog, &$specs, $switch_labels = array() out($f,"$prolog{$spec_name}_HANDLER,\n"); break; case ZEND_VM_KIND_SWITCH: - out($f,$prolog."(void*)(uintptr_t)$switch_labels[$spec_name],\n"); + out($f,$prolog."$switch_labels[$spec_name],\n"); break; case ZEND_VM_KIND_GOTO: out($f,$prolog."(void*)&&{$spec_name}_LABEL,\n"); @@ -1436,7 +1436,7 @@ function gen_labels($f, $spec, $kind, $prolog, &$specs, $switch_labels = array() out($f,$prolog."ZEND_NULL_HANDLER,\n"); break; case ZEND_VM_KIND_SWITCH: - out($f,$prolog."(void*)(uintptr_t)-1,\n"); + out($f,$prolog."-1,\n"); break; case ZEND_VM_KIND_GOTO: out($f,$prolog."(void*)&&ZEND_NULL_LABEL,\n"); @@ -1467,7 +1467,7 @@ function gen_labels($f, $spec, $kind, $prolog, &$specs, $switch_labels = array() out($f,$prolog.$dsc["op"]."_HANDLER,\n"); break; case ZEND_VM_KIND_SWITCH: - out($f,$prolog."(void*)(uintptr_t)".((string)$num).",\n"); + out($f,$prolog.((string)$num).",\n"); break; case ZEND_VM_KIND_GOTO: out($f,$prolog."(void*)&&".$dsc["op"]."_LABEL,\n"); @@ -1480,7 +1480,7 @@ function gen_labels($f, $spec, $kind, $prolog, &$specs, $switch_labels = array() out($f,$prolog."ZEND_NULL_HANDLER,\n"); break; case ZEND_VM_KIND_SWITCH: - out($f,$prolog."(void*)(uintptr_t)-1,\n"); + out($f,$prolog."-1,\n"); break; case ZEND_VM_KIND_GOTO: out($f,$prolog."(void*)&&ZEND_NULL_LABEL,\n"); @@ -1497,7 +1497,7 @@ function gen_labels($f, $spec, $kind, $prolog, &$specs, $switch_labels = array() out($f,$prolog."ZEND_NULL_HANDLER\n"); break; case ZEND_VM_KIND_SWITCH: - out($f,$prolog."(void*)(uintptr_t)-1\n"); + out($f,$prolog."-1\n"); break; case ZEND_VM_KIND_GOTO: out($f,$prolog."(void*)&&ZEND_NULL_LABEL\n"); @@ -1813,16 +1813,16 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name) out($f,"#define SPEC_RULE_OBSERVER 0x02000000\n"); out($f,"\n"); out($f,"static const uint32_t *zend_spec_handlers;\n"); - out($f,"static const void * const *zend_opcode_handlers;\n"); + out($f,"static zend_vm_opcode_handler_t const *zend_opcode_handlers;\n"); out($f,"static int zend_handlers_count;\n"); if ($kind == ZEND_VM_KIND_HYBRID) { out($f,"#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID)\n"); - out($f,"static const void * const * zend_opcode_handler_funcs;\n"); + out($f,"static zend_vm_opcode_handler_func_t const * zend_opcode_handler_funcs;\n"); out($f,"static zend_op hybrid_halt_op;\n"); out($f,"#endif\n"); } out($f,"#if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID) || !ZEND_VM_SPEC\n"); - out($f,"static const void *zend_vm_get_opcode_handler(uint8_t opcode, const zend_op* op);\n"); + out($f,"static zend_vm_opcode_handler_t zend_vm_get_opcode_handler(uint8_t opcode, const zend_op* op);\n"); out($f,"#endif\n\n"); if ($kind == ZEND_VM_KIND_HYBRID) { out($f,"#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID)\n"); @@ -2040,7 +2040,7 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name) break; case "HELPER_VARS": if ($kind == ZEND_VM_KIND_SWITCH) { - out($f,$m[1]."const void *dispatch_handler;\n"); + out($f,$m[1]."zend_vm_opcode_handler_t dispatch_handler;\n"); } if ($kind != ZEND_VM_KIND_CALL && count($params)) { if ($kind == ZEND_VM_KIND_HYBRID) { @@ -2097,8 +2097,8 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name) out($f,$prolog."\tstatic const void * const labels[] = {\n"); gen_labels($f, $spec, ($kind == ZEND_VM_KIND_HYBRID) ? ZEND_VM_KIND_GOTO : $kind, $prolog."\t\t", $specs); out($f,$prolog."\t};\n"); - out($f,$prolog."\tzend_opcode_handlers = (const void **) labels;\n"); - out($f,$prolog."\tzend_handlers_count = sizeof(labels) / sizeof(void*);\n"); + out($f,$prolog."\tzend_opcode_handlers = (zend_vm_opcode_handler_t*) labels;\n"); + out($f,$prolog."\tzend_handlers_count = sizeof(labels) / sizeof(labels[0]);\n"); if ($kind == ZEND_VM_KIND_HYBRID) { out($f,$prolog."\tmemset(&hybrid_halt_op, 0, sizeof(hybrid_halt_op));\n"); out($f,$prolog."\thybrid_halt_op.handler = (void*)&&HYBRID_HALT_LABEL;\n"); @@ -2212,7 +2212,11 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name) out($f,$prolog."zend_spec_handlers = specs;\n"); out($f,$prolog.$executor_name."_ex(NULL);\n"); } else { - out($f,$prolog."static const void * const labels[] = {\n"); + out($f,"#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID)\n"); + out($f,$prolog."static zend_vm_opcode_handler_func_t const labels[] = {\n"); + out($f,"#else\n"); + out($f,$prolog."static zend_vm_opcode_handler_t const labels[] = {\n"); + out($f,"#endif\n"); gen_labels($f, $spec, ($kind == ZEND_VM_KIND_HYBRID) ? ZEND_VM_KIND_CALL : $kind, $prolog."\t", $specs, $switch_labels); out($f,$prolog."};\n"); out($f,$prolog."static const uint32_t specs[] = {\n"); @@ -2226,7 +2230,7 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name) out($f,"#else\n"); } out($f,$prolog."zend_opcode_handlers = labels;\n"); - out($f,$prolog."zend_handlers_count = sizeof(labels) / sizeof(void*);\n"); + out($f,$prolog."zend_handlers_count = sizeof(labels) / sizeof(labels[0]);\n"); out($f,$prolog."zend_spec_handlers = specs;\n"); if ($kind == ZEND_VM_KIND_HYBRID) { out($f,"#endif\n"); @@ -2359,6 +2363,20 @@ function gen_vm_opcodes_header( $str .= "# endif\n"; $str .= "#endif\n"; $str .= "\n"; + $str .= "#if ZEND_VM_KIND == ZEND_VM_KIND_HYBRID\n"; + $str .= "typedef const void* zend_vm_opcode_handler_t;\n"; + $str .= "typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_func_t)(void);\n"; + $str .= "#elif ZEND_VM_KIND == ZEND_VM_KIND_CALL\n"; + $str .= "typedef const struct _zend_op *(ZEND_FASTCALL *zend_vm_opcode_handler_t)(struct _zend_execute_data *execute_data, const struct _zend_op *opline);\n"; + $str .= "typedef const struct _zend_op *(ZEND_FASTCALL *zend_vm_opcode_handler_func_t)(struct _zend_execute_data *execute_data, const struct _zend_op *opline);\n"; + $str .= "#elif ZEND_VM_KIND == ZEND_VM_KIND_SWITCH\n"; + $str .= "typedef int zend_vm_opcode_handler_t;\n"; + $str .= "#elif ZEND_VM_KIND == ZEND_VM_KIND_GOTO\n"; + $str .= "typedef const void* zend_vm_opcode_handler_t;\n"; + $str .= "#else\n"; + $str .= "# error\n"; + $str .= "#endif\n"; + $str .= "\n"; foreach ($vm_op_flags as $name => $val) { $str .= sprintf("#define %-24s 0x%08x\n", $name, $val); } @@ -2840,7 +2858,7 @@ function gen_vm($def, $skel) { } out($f, "}\n\n"); out($f, "#if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID) || !ZEND_VM_SPEC\n"); - out($f, "static const void *zend_vm_get_opcode_handler(uint8_t opcode, const zend_op* op)\n"); + out($f, "static zend_vm_opcode_handler_t zend_vm_get_opcode_handler(uint8_t opcode, const zend_op* op)\n"); out($f, "{\n"); if (!ZEND_VM_SPEC) { out($f, "\treturn zend_opcode_handlers[zend_vm_get_opcode_handler_idx(opcode, op)];\n"); diff --git a/Zend/zend_vm_opcodes.h b/Zend/zend_vm_opcodes.h index 29469bb5f7dca..e8e03a534b02a 100644 --- a/Zend/zend_vm_opcodes.h +++ b/Zend/zend_vm_opcodes.h @@ -42,6 +42,20 @@ # endif #endif +#if ZEND_VM_KIND == ZEND_VM_KIND_HYBRID +typedef const void* zend_vm_opcode_handler_t; +typedef void (ZEND_FASTCALL *zend_vm_opcode_handler_func_t)(void); +#elif ZEND_VM_KIND == ZEND_VM_KIND_CALL +typedef const struct _zend_op *(ZEND_FASTCALL *zend_vm_opcode_handler_t)(struct _zend_execute_data *execute_data, const struct _zend_op *opline); +typedef const struct _zend_op *(ZEND_FASTCALL *zend_vm_opcode_handler_func_t)(struct _zend_execute_data *execute_data, const struct _zend_op *opline); +#elif ZEND_VM_KIND == ZEND_VM_KIND_SWITCH +typedef int zend_vm_opcode_handler_t; +#elif ZEND_VM_KIND == ZEND_VM_KIND_GOTO +typedef const void* zend_vm_opcode_handler_t; +#else +# error +#endif + #define ZEND_VM_OP_SPEC 0x00000001 #define ZEND_VM_OP_CONST 0x00000002 #define ZEND_VM_OP_TMPVAR 0x00000004 diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index fa2cc10e8b3a0..b46a8792fb779 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -90,13 +90,13 @@ static size_t dasm_size = 0; static zend_long jit_bisect_pos = 0; -static const void *zend_jit_runtime_jit_handler = NULL; -static const void *zend_jit_profile_jit_handler = NULL; -static const void *zend_jit_func_hot_counter_handler = NULL; -static const void *zend_jit_loop_hot_counter_handler = NULL; -static const void *zend_jit_func_trace_counter_handler = NULL; -static const void *zend_jit_ret_trace_counter_handler = NULL; -static const void *zend_jit_loop_trace_counter_handler = NULL; +static zend_vm_opcode_handler_t zend_jit_runtime_jit_handler = NULL; +static zend_vm_opcode_handler_t zend_jit_profile_jit_handler = NULL; +static zend_vm_opcode_handler_t zend_jit_func_hot_counter_handler = NULL; +static zend_vm_opcode_handler_t zend_jit_loop_hot_counter_handler = NULL; +static zend_vm_opcode_handler_t zend_jit_func_trace_counter_handler = NULL; +static zend_vm_opcode_handler_t zend_jit_ret_trace_counter_handler = NULL; +static zend_vm_opcode_handler_t zend_jit_loop_trace_counter_handler = NULL; static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_runtime_jit(ZEND_OPCODE_HANDLER_ARGS); @@ -1417,7 +1417,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op zend_jit_ctx ctx; zend_jit_ctx *jit = &ctx; zend_jit_reg_var *ra = NULL; - void *handler; + zend_vm_opcode_handler_t handler; int call_level = 0; void *checkpoint = NULL; bool recv_emitted = 0; /* emitted at least one RECV opcode */ @@ -3213,7 +3213,7 @@ static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cf } } - opline->handler = (const void*)zend_jit_func_hot_counter_handler; + opline->handler = zend_jit_func_hot_counter_handler; } if (JIT_G(hot_loop)) { @@ -3223,7 +3223,7 @@ static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cf if ((cfg->blocks[i].flags & ZEND_BB_REACHABLE) && (cfg->blocks[i].flags & ZEND_BB_LOOP_HEADER)) { op_array->opcodes[cfg->blocks[i].start].handler = - (const void*)zend_jit_loop_hot_counter_handler; + zend_jit_loop_hot_counter_handler; } } } @@ -3316,7 +3316,7 @@ int zend_jit_op_array(zend_op_array *op_array, zend_script *script) jit_extension->op_array = op_array; jit_extension->orig_handler = (void*)opline->handler; ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); - opline->handler = (const void*)zend_jit_runtime_jit_handler; + opline->handler = zend_jit_runtime_jit_handler; zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension); return SUCCESS; @@ -3346,7 +3346,7 @@ int zend_jit_op_array(zend_op_array *op_array, zend_script *script) jit_extension->op_array = op_array; jit_extension->orig_handler = (void*)opline->handler; ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); - opline->handler = (const void*)zend_jit_profile_jit_handler; + opline->handler = zend_jit_profile_jit_handler; zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension); } @@ -3566,23 +3566,23 @@ void zend_jit_protect(void) static void zend_jit_init_handlers(void) { - if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - zend_jit_runtime_jit_handler = zend_jit_stub_handlers[jit_stub_hybrid_runtime_jit]; - zend_jit_profile_jit_handler = zend_jit_stub_handlers[jit_stub_hybrid_profile_jit]; - zend_jit_func_hot_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_func_hot_counter]; - zend_jit_loop_hot_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_loop_hot_counter]; - zend_jit_func_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_func_trace_counter]; - zend_jit_ret_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_ret_trace_counter]; - zend_jit_loop_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_loop_trace_counter]; - } else { - zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit; - zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper; - zend_jit_func_hot_counter_handler = (const void*)zend_jit_func_counter_helper; - zend_jit_loop_hot_counter_handler = (const void*)zend_jit_loop_counter_helper; - zend_jit_func_trace_counter_handler = (const void*)zend_jit_func_trace_helper; - zend_jit_ret_trace_counter_handler = (const void*)zend_jit_ret_trace_helper; - zend_jit_loop_trace_counter_handler = (const void*)zend_jit_loop_trace_helper; - } +#if ZEND_VM_KIND == ZEND_VM_KIND_HYBRID + zend_jit_runtime_jit_handler = (zend_vm_opcode_handler_t)zend_jit_stub_handlers[jit_stub_hybrid_runtime_jit]; + zend_jit_profile_jit_handler = (zend_vm_opcode_handler_t)zend_jit_stub_handlers[jit_stub_hybrid_profile_jit]; + zend_jit_func_hot_counter_handler = (zend_vm_opcode_handler_t)zend_jit_stub_handlers[jit_stub_hybrid_func_hot_counter]; + zend_jit_loop_hot_counter_handler = (zend_vm_opcode_handler_t)zend_jit_stub_handlers[jit_stub_hybrid_loop_hot_counter]; + zend_jit_func_trace_counter_handler = (zend_vm_opcode_handler_t)zend_jit_stub_handlers[jit_stub_hybrid_func_trace_counter]; + zend_jit_ret_trace_counter_handler = (zend_vm_opcode_handler_t)zend_jit_stub_handlers[jit_stub_hybrid_ret_trace_counter]; + zend_jit_loop_trace_counter_handler = (zend_vm_opcode_handler_t)zend_jit_stub_handlers[jit_stub_hybrid_loop_trace_counter]; +#else + zend_jit_runtime_jit_handler = zend_runtime_jit; + zend_jit_profile_jit_handler = zend_jit_profile_helper; + zend_jit_func_hot_counter_handler = zend_jit_func_counter_helper; + zend_jit_loop_hot_counter_handler = zend_jit_loop_counter_helper; + zend_jit_func_trace_counter_handler = zend_jit_func_trace_helper; + zend_jit_ret_trace_counter_handler = zend_jit_ret_trace_helper; + zend_jit_loop_trace_counter_handler = zend_jit_loop_trace_helper; +#endif } static void zend_jit_globals_ctor(zend_jit_globals *jit_globals) @@ -3945,9 +3945,9 @@ static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array) } } if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) { - opline->handler = (const void*)zend_jit_runtime_jit_handler; + opline->handler = zend_jit_runtime_jit_handler; } else { - opline->handler = (const void*)zend_jit_profile_jit_handler; + opline->handler = zend_jit_profile_jit_handler; } #endif } diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 3623689f05aae..9ff011d0f9962 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -26,6 +26,7 @@ #include "Zend/zend_constants.h" #include "Zend/Optimizer/zend_func_info.h" #include "Zend/Optimizer/zend_call_graph.h" +#include "zend_vm_opcodes.h" /* Address Encoding */ typedef uintptr_t zend_jit_addr; @@ -130,7 +131,7 @@ static zend_always_inline bool zend_jit_same_addr(zend_jit_addr addr1, zend_jit_ typedef struct _zend_jit_op_array_extension { zend_func_info func_info; const zend_op_array *op_array; - const void *orig_handler; + zend_vm_opcode_handler_t orig_handler; } zend_jit_op_array_extension; /* Profiler */ @@ -169,7 +170,7 @@ typedef struct _zend_jit_op_array_hot_extension { zend_func_info func_info; const zend_op_array *op_array; int16_t *counter; - const void *orig_handlers[1]; + zend_vm_opcode_handler_t orig_handlers[1]; } zend_jit_op_array_hot_extension; #define zend_jit_op_array_hash(op_array) \ @@ -225,9 +226,6 @@ extern const zend_op *zend_jit_halt_op; # define ZEND_VM_ENTER_BIT 1ULL #endif -/* VM handlers */ -typedef ZEND_OPCODE_HANDLER_RET (ZEND_FASTCALL *zend_vm_opcode_handler_t)(ZEND_OPCODE_HANDLER_ARGS); - /* VM helpers */ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(ZEND_OPCODE_HANDLER_ARGS_EX uint32_t call_info); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(ZEND_OPCODE_HANDLER_ARGS_EX uint32_t call_info); @@ -339,8 +337,8 @@ typedef enum _zend_jit_trace_stop { typedef union _zend_op_trace_info { zend_op dummy; /* the size of this structure must be the same as zend_op */ struct { - const void *orig_handler; - const void *call_handler; + zend_vm_opcode_handler_t orig_handler; + zend_vm_opcode_handler_func_t call_handler; int16_t *counter; uint8_t trace_flags; }; diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index 6aa7896554920..c65fe5e3fbf7f 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -1918,15 +1918,13 @@ static void zend_jit_vm_leave(zend_jit_ctx *jit, ir_ref to_opline) static int zend_jit_exception_handler_stub(zend_jit_ctx *jit) { - const void *handler; - if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - handler = zend_get_opcode_handler_func(EG(exception_op)); + zend_vm_opcode_handler_func_t handler = (zend_vm_opcode_handler_func_t)zend_get_opcode_handler_func(EG(exception_op)); ir_CALL(IR_VOID, ir_CONST_FUNC(handler)); ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); } else { - handler = EG(exception_op)->handler; + zend_vm_opcode_handler_t handler = EG(exception_op)->handler; if (GCC_GLOBAL_REGS) { ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler)); @@ -4176,17 +4174,12 @@ static ir_ref zend_jit_continue_entry(zend_jit_ctx *jit, ir_ref src, unsigned in static int zend_jit_handler(zend_jit_ctx *jit, const zend_op *opline, int may_throw) { - const void *handler; - zend_jit_set_ip(jit, opline); - if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - handler = zend_get_opcode_handler_func(opline); - } else { - handler = opline->handler; - } if (GCC_GLOBAL_REGS) { + zend_vm_opcode_handler_func_t handler = (zend_vm_opcode_handler_func_t)zend_get_opcode_handler_func(opline); ir_CALL(IR_VOID, ir_CONST_FUNC(handler)); } else { + zend_vm_opcode_handler_t handler = opline->handler; ir_ref ip = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(handler), jit_FP(jit), jit_IP(jit)); jit_STORE_IP(jit, ip); } @@ -4216,7 +4209,6 @@ static int zend_jit_handler(zend_jit_ctx *jit, const zend_op *opline, int may_th static int zend_jit_tail_handler(zend_jit_ctx *jit, const zend_op *opline) { - const void *handler; ir_ref ref; zend_basic_block *bb; @@ -4228,16 +4220,16 @@ static int zend_jit_tail_handler(zend_jit_ctx *jit, const zend_op *opline) opline->opcode == ZEND_RETURN) { /* Use inlined HYBRID VM handler */ - handler = opline->handler; + zend_vm_opcode_handler_t handler = opline->handler; ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler)); } else { - handler = zend_get_opcode_handler_func(opline); + zend_vm_opcode_handler_func_t handler = (zend_vm_opcode_handler_func_t)zend_get_opcode_handler_func(opline); ir_CALL(IR_VOID, ir_CONST_FUNC(handler)); ref = ir_LOAD_A(jit_IP(jit)); ir_TAILCALL(IR_VOID, ref); } } else { - handler = opline->handler; + zend_vm_opcode_handler_t handler = opline->handler; if (GCC_GLOBAL_REGS) { ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler)); } else if ((jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY) @@ -16741,7 +16733,7 @@ static int zend_jit_start(zend_jit_ctx *jit, const zend_op_array *op_array, zend return 1; } -static void *zend_jit_finish(zend_jit_ctx *jit) +static zend_vm_opcode_handler_t zend_jit_finish(zend_jit_ctx *jit) { void *entry; size_t size; @@ -16825,14 +16817,14 @@ static void *zend_jit_finish(zend_jit_ctx *jit) opline++; } } - opline->handler = entry; + opline->handler = (zend_vm_opcode_handler_t)entry; if (jit->ctx.entries_count) { /* For all entries */ int i = jit->ctx.entries_count; do { ir_insn *insn = &jit->ctx.ir_base[jit->ctx.entries[--i]]; - op_array->opcodes[insn->op2].handler = (char*)entry + insn->op3; + op_array->opcodes[insn->op2].handler = (zend_vm_opcode_handler_t)((char*)entry + insn->op3); } while (i != 0); } } else { @@ -16859,7 +16851,7 @@ static void *zend_jit_finish(zend_jit_ctx *jit) zend_string_release(str); } - return entry; + return (zend_vm_opcode_handler_t)entry; } static const void *zend_jit_trace_allocate_exit_group(uint32_t n) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index aeec674d60f3e..4965193f786c6 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -3409,7 +3409,7 @@ static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offse ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; } ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN; - next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler; + next_opline->handler = zend_jit_ret_trace_counter_handler; } } @@ -4079,9 +4079,9 @@ static bool zend_jit_trace_may_throw(const zend_op *opline, return zend_may_throw_ex(opline, ssa_op, op_array, ssa, t1, t2); } -static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num) +static zend_vm_opcode_handler_t zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num) { - const void *handler = NULL; + zend_vm_opcode_handler_t handler = NULL; zend_jit_ctx ctx; zend_jit_ctx *jit = &ctx; zend_jit_reg_var *ra = NULL; @@ -7392,9 +7392,9 @@ static zend_string *zend_jit_trace_escape_name(uint32_t trace_num, uint32_t exit return buf.s; } -static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num) +static zend_vm_opcode_handler_t zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num) { - const void *handler = NULL; + zend_vm_opcode_handler_t handler = NULL; zend_jit_ctx ctx; zend_string *name; void *checkpoint; @@ -7458,7 +7458,7 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset) { zend_jit_trace_stop ret; - const void *handler; + zend_vm_opcode_handler_t handler; uint8_t orig_trigger; zend_jit_trace_info *t = NULL; zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS]; @@ -8832,11 +8832,11 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf zend_jit_unprotect(); if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_LOOP) { - ((zend_op*)(t->opline))->handler = (const void*)zend_jit_loop_trace_counter_handler; + ((zend_op*)(t->opline))->handler = zend_jit_loop_trace_counter_handler; } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_ENTER) { - ((zend_op*)(t->opline))->handler = (const void*)zend_jit_func_trace_counter_handler; + ((zend_op*)(t->opline))->handler = zend_jit_func_trace_counter_handler; } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_RETURN) { - ((zend_op*)(t->opline))->handler = (const void*)zend_jit_ret_trace_counter_handler; + ((zend_op*)(t->opline))->handler = zend_jit_ret_trace_counter_handler; } ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &= ZEND_JIT_TRACE_START_LOOP|ZEND_JIT_TRACE_START_ENTER|ZEND_JIT_TRACE_START_RETURN; @@ -8889,9 +8889,9 @@ static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array) jit_extension->trace_info[i].trace_flags &= ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_UNSUPPORTED; if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_LOOP) { - op_array->opcodes[i].handler = (const void*)zend_jit_loop_trace_counter_handler; + op_array->opcodes[i].handler = zend_jit_loop_trace_counter_handler; } else if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_ENTER) { - op_array->opcodes[i].handler = (const void*)zend_jit_func_trace_counter_handler; + op_array->opcodes[i].handler = zend_jit_func_trace_counter_handler; } else { op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler; } @@ -8917,7 +8917,7 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes; for (i = 0; i < op_array->last; i++) { jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler; - jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]); + jit_extension->trace_info[i].call_handler = (zend_vm_opcode_handler_func_t)zend_get_opcode_handler_func(&op_array->opcodes[i]); jit_extension->trace_info[i].counter = NULL; jit_extension->trace_info[i].trace_flags = zend_jit_trace_supported(&op_array->opcodes[i]); @@ -8939,7 +8939,7 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) /* loop header */ opline = op_array->opcodes + cfg.blocks[i].start; if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) { - opline->handler = (const void*)zend_jit_loop_trace_counter_handler; + opline->handler = zend_jit_loop_trace_counter_handler; if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) { ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; @@ -8964,7 +8964,7 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags) { /* function entry */ - opline->handler = (const void*)zend_jit_func_trace_counter_handler; + opline->handler = zend_jit_func_trace_counter_handler; ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 4348fbd53ad48..b19de612c4439 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -298,6 +298,7 @@ void ZEND_FASTCALL zend_jit_undefined_string_key(EXECUTE_DATA_D) ZVAL_NULL(result); } +#if ZEND_VM_KIND != ZEND_VM_KIND_HYBRID ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS) { zend_op_array *op_array = (zend_op_array*)EX(func); @@ -341,6 +342,7 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_H ZEND_OPCODE_TAIL_CALL(handler); } } +#endif static zend_always_inline zend_constant* _zend_quick_get_constant( const zval *key, uint32_t flags, int check_defined_only) @@ -402,6 +404,7 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key) return _zend_quick_get_constant(key, 0, 1); } +#if ZEND_VM_KIND != ZEND_VM_KIND_HYBRID static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_counter_helper(ZEND_OPCODE_HANDLER_ARGS_EX uint32_t cost) { zend_jit_op_array_trace_extension *jit_extension = @@ -450,6 +453,7 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HAN ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); } +#endif #define TRACE_RECORD(_op, _info, _ptr) \ trace_buffer[idx].info = _op | (_info); \ @@ -657,7 +661,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, zend_jit_trace_stop halt = 0; int level = 0; int ret_level = 0; - zend_vm_opcode_handler_t handler; + zend_vm_opcode_handler_func_t handler; const zend_op_array *op_array; zend_jit_op_array_trace_extension *jit_extension; size_t offset; @@ -983,7 +987,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, break; } - handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; + handler = ZEND_OP_TRACE_INFO(opline, offset)->call_handler; #ifdef HAVE_GCC_GLOBAL_REGS handler(); if (UNEXPECTED(opline == zend_jit_halt_op)) { From b63372058552fd7cae24a8684d8b2b8276b3a2fc Mon Sep 17 00:00:00 2001 From: Arnaud Le Blanc Date: Sat, 26 Jul 2025 10:03:34 +0200 Subject: [PATCH 2/3] Add unique entry point for extra tests We are adding extra (non-phpt) test suites in [1] and [2]. In order to avoid touching CI files too often (which are maintained in 8.1 and merged in upper branches), we add a single entry point to call the extra tests. The entry point can be updated in branches without synchronizing all the way from 8.1. CI files still need to be touched to install dependencies of these tests, but this should be manageable as these do not change often and are the same in every branch. Closes GH-19242. [1] https://github.com/php/php-src/pull/16987 [2] https://github.com/php/php-src/pull/18939 --- .github/actions/apt-x32/action.yml | 5 +- .github/actions/apt-x64/action.yml | 5 +- .github/actions/extra-tests/action.yml | 7 ++ .github/actions/freebsd/action.yml | 13 ++- .github/workflows/nightly.yml | 11 +++ run-extra-tests.php | 123 +++++++++++++++++++++++++ 6 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 .github/actions/extra-tests/action.yml create mode 100755 run-extra-tests.php diff --git a/.github/actions/apt-x32/action.yml b/.github/actions/apt-x32/action.yml index 879300f992747..0147b15cfcf5f 100644 --- a/.github/actions/apt-x32/action.yml +++ b/.github/actions/apt-x32/action.yml @@ -6,6 +6,8 @@ runs: run: | set -x + OPCACHE_TLS_TESTS_DEPS="gcc clang lld" + export DEBIAN_FRONTEND=noninteractive dpkg --add-architecture i386 apt-get update -y | true @@ -50,4 +52,5 @@ runs: re2c \ unzip \ wget \ - zlib1g-dev:i386 + zlib1g-dev:i386 \ + $OPCACHE_TLS_TESTS_DEPS diff --git a/.github/actions/apt-x64/action.yml b/.github/actions/apt-x64/action.yml index 4e1a03dd58cc5..15697ed9f2fa4 100644 --- a/.github/actions/apt-x64/action.yml +++ b/.github/actions/apt-x64/action.yml @@ -6,6 +6,8 @@ runs: run: | set -x + OPCACHE_TLS_TESTS_DEPS="gcc clang lld" + sudo apt-get update sudo apt-get install \ bison \ @@ -58,4 +60,5 @@ runs: libqdbm-dev \ libjpeg-dev \ libpng-dev \ - libfreetype6-dev + libfreetype6-dev \ + $OPCACHE_TLS_TESTS_DEPS diff --git a/.github/actions/extra-tests/action.yml b/.github/actions/extra-tests/action.yml new file mode 100644 index 0000000000000..0537496064008 --- /dev/null +++ b/.github/actions/extra-tests/action.yml @@ -0,0 +1,7 @@ +name: Extra tests +runs: + using: composite + steps: + - shell: sh + run: | + sapi/cli/php run-extra-tests.php diff --git a/.github/actions/freebsd/action.yml b/.github/actions/freebsd/action.yml index 17446311bb864..ce133925e2521 100644 --- a/.github/actions/freebsd/action.yml +++ b/.github/actions/freebsd/action.yml @@ -3,6 +3,9 @@ inputs: configurationParameters: default: '' required: false + runExtraTests: + default: false + required: false runs: using: composite steps: @@ -17,6 +20,8 @@ runs: prepare: | cd $GITHUB_WORKSPACE + OPCACHE_TLS_TESTS_DEPS="gcc" + kldload accf_http pkg install -y \ autoconf \ @@ -41,9 +46,11 @@ runs: webp \ libavif \ `#sqlite3` \ - curl + curl \ + $OPCACHE_TLS_TESTS_DEPS ./buildconf -f + CC=clang CXX=clang++ \ ./configure \ --prefix=/usr/local \ --enable-debug \ @@ -106,3 +113,7 @@ runs: --show-slow 1000 \ --set-timeout 120 \ -d zend_extension=opcache.so + + if test "${{ inputs.runExtraTests }}" = "true"; then + sapi/cli/php run-extra-tests.php + fi diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5817c647a871a..38ccb05e09928 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -85,6 +85,8 @@ jobs: with: runTestsParameters: >- --asan -x + - name: Extra tests + uses: ./.github/actions/extra-tests ALPINE: if: inputs.run_alpine name: ALPINE_X64_ASAN_UBSAN_DEBUG_ZTS @@ -134,6 +136,8 @@ jobs: --asan -x -d zend_extension=opcache.so -d opcache.enable_cli=1 + - name: Extra tests + uses: ./.github/actions/extra-tests - name: Notify Slack if: failure() uses: ./.github/actions/notify-slack @@ -266,6 +270,8 @@ jobs: ${{ matrix.run_tests_parameters }} -d zend_extension=opcache.so -d opcache.enable_cli=1 + - name: Extra tests + uses: ./.github/actions/extra-tests - name: Verify generated files are up to date uses: ./.github/actions/verify-generated-files - name: Notify Slack @@ -355,6 +361,8 @@ jobs: ${{ matrix.run_tests_parameters }} -d zend_extension=opcache.so -d opcache.enable_cli=1 + - name: Extra tests + uses: ./.github/actions/extra-tests - name: Notify Slack if: failure() uses: ./.github/actions/notify-slack @@ -414,6 +422,8 @@ jobs: runTestsParameters: >- -d zend_extension=opcache.so -d opcache.enable_cli=1 + - name: Extra tests + uses: ./.github/actions/extra-tests - name: Verify generated files are up to date uses: ./.github/actions/verify-generated-files - name: Notify Slack @@ -1076,3 +1086,4 @@ jobs: with: configurationParameters: >- --${{ matrix.zts && 'enable' || 'disable' }}-zts + runExtraTests: true diff --git a/run-extra-tests.php b/run-extra-tests.php new file mode 100755 index 0000000000000..a299addf4044c --- /dev/null +++ b/run-extra-tests.php @@ -0,0 +1,123 @@ +#!/usr/bin/env php +os}\n"; + echo "CPU Arch: {$environment->cpuArch}\n"; + echo "ZTS: " . ($environment->zts ? 'Yes' : 'No') . "\n"; + echo "DEBUG: " . ($environment->debug ? 'Yes' : 'No') . "\n"; + echo "=====================================================================\n"; + + echo "No tests in this branch yet.\n"; + + echo "All OK\n"; +} + +function output_group_start(Environment $environment, string $name): void +{ + if ($environment->githubAction) { + printf("::group::%s\n", $name); + } else { + printf("%s\n", $name); + } +} + +function output_group_end(Environment $environment): void +{ + if ($environment->githubAction) { + printf("::endgroup::\n"); + } +} + +/** + * Returns getenv('TEST_PHP_OS') if defined, otherwise returns one of + * 'Windows NT', 'Linux', 'FreeBSD', 'Darwin', ... + */ +function detect_os(): string +{ + $os = (string) getenv('TEST_PHP_OS'); + if ($os !== '') { + return $os; + } + + return php_uname('s'); +} + +/** + * Returns getenv('TEST_PHP_CPU_ARCH') if defined, otherwise returns one of + * 'x86', 'x86_64', 'aarch64', ... + */ +function detect_cpu_arch(): string +{ + $cpu = (string) getenv('TEST_PHP_CPU_ARCH'); + if ($cpu !== '') { + return $cpu; + } + + $cpu = php_uname('m'); + if (strtolower($cpu) === 'amd64') { + $cpu = 'x86_64'; + } else if (in_array($cpu, ['i386', 'i686'])) { + $cpu = 'x86'; + } else if ($cpu === 'arm64') { + $cpu = 'aarch64'; + } + + return $cpu; +} + +main($argc, $argv); From 73b1ebfa203e7c391fcccc5171e1d800e17f013c Mon Sep 17 00:00:00 2001 From: Arnaud Le Blanc Date: Tue, 24 Jun 2025 11:29:23 +0200 Subject: [PATCH 3/3] Fix linker failure when building Opcache statically We use linker relocations to fetch the TLS index and offset of _tsrm_ls_cache. When building Opcache statically, linkers may attempt to optimize that into a more efficient code sequence (relaxing from "General Dynamic" to "Local Exec" model [1]). Unfortunately, linkers will fail, rather than ignore our relocations, when they don't recognize the exact code sequence they are expecting. This results in errors as reported by GH-15074: TLS transition from R_X86_64_TLSGD to R_X86_64_GOTTPOFF against `_tsrm_ls_cache' at 0x12fc3 in section `.text' failed" Here I take a different approach: * Emit the exact full code sequence expected by linkers * Extract the TLS index/offset by inspecting the linked ASM code, rather than executing it (execution would give us the thread-local address). * We detect when the code was relaxed, in which case we can extract the TCB offset instead. * This is done in a conservative way so that if the linker did something we didn't expect, we fallback to a safer (but slower) mechanism. One additional benefit of that is we are now able to use the Local Exec model in more cases, in JIT'ed code. This makes non-glibc builds faster in these cases. Closes GH-18939. Related RFC: https://wiki.php.net/rfc/make_opcache_required. [1] https://www.akkadia.org/drepper/tls.pdf --- ext/opcache/config.m4 | 8 + ext/opcache/config.w32 | 6 + ext/opcache/jit/tls/testing/.gitignore | 4 + ext/opcache/jit/tls/testing/def-vars.h | 1030 ++++++++++++++++++++ ext/opcache/jit/tls/testing/def.c | 33 + ext/opcache/jit/tls/testing/main.c | 48 + ext/opcache/jit/tls/testing/test.sh | 232 +++++ ext/opcache/jit/tls/testing/user.c | 28 + ext/opcache/jit/tls/zend_jit_tls.h | 40 + ext/opcache/jit/tls/zend_jit_tls_aarch64.c | 255 +++++ ext/opcache/jit/tls/zend_jit_tls_darwin.c | 82 ++ ext/opcache/jit/tls/zend_jit_tls_win.c | 64 ++ ext/opcache/jit/tls/zend_jit_tls_x86.c | 239 +++++ ext/opcache/jit/tls/zend_jit_tls_x86_64.c | 222 +++++ ext/opcache/jit/zend_jit_ir.c | 200 +--- ext/opcache/tests/zzz_basic_logging.phpt | 1 + run-extra-tests.php | 81 +- 17 files changed, 2402 insertions(+), 171 deletions(-) create mode 100644 ext/opcache/jit/tls/testing/.gitignore create mode 100644 ext/opcache/jit/tls/testing/def-vars.h create mode 100644 ext/opcache/jit/tls/testing/def.c create mode 100644 ext/opcache/jit/tls/testing/main.c create mode 100755 ext/opcache/jit/tls/testing/test.sh create mode 100644 ext/opcache/jit/tls/testing/user.c create mode 100644 ext/opcache/jit/tls/zend_jit_tls.h create mode 100644 ext/opcache/jit/tls/zend_jit_tls_aarch64.c create mode 100644 ext/opcache/jit/tls/zend_jit_tls_darwin.c create mode 100644 ext/opcache/jit/tls/zend_jit_tls_win.c create mode 100644 ext/opcache/jit/tls/zend_jit_tls_x86.c create mode 100644 ext/opcache/jit/tls/zend_jit_tls_x86_64.c diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 8f6d5ab711b28..a7b35351c6ed6 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -73,19 +73,23 @@ if test "$PHP_OPCACHE" != "no"; then IR_TARGET=IR_TARGET_X64 DASM_FLAGS="-D X64APPLE=1 -D X64=1" DASM_ARCH="x86" + TLS_TARGET="darwin" ], [*x86_64*|amd64-*-freebsd*], [ IR_TARGET=IR_TARGET_X64 DASM_FLAGS="-D X64=1" DASM_ARCH="x86" + TLS_TARGET="x86_64" ], [[i[34567]86*|x86*]], [ IR_TARGET=IR_TARGET_X86 DASM_ARCH="x86" + TLS_TARGET="x86" ], [aarch64*], [ IR_TARGET=IR_TARGET_AARCH64 DASM_ARCH="aarch64" + TLS_TARGET="aarch64" ]) AS_VAR_IF([PHP_CAPSTONE], [yes], @@ -102,6 +106,10 @@ if test "$PHP_OPCACHE" != "no"; then JIT_CFLAGS="-I@ext_builddir@/jit/ir -D$IR_TARGET -DIR_PHP" AS_VAR_IF([ZEND_DEBUG], [yes], [JIT_CFLAGS="$JIT_CFLAGS -DIR_DEBUG"]) + + AS_VAR_IF([PHP_THREAD_SAFETY], [yes], [ + ZEND_JIT_SRC="$ZEND_JIT_SRC jit/tls/zend_jit_tls_$TLS_TARGET.c" + ]) ]) AC_CHECK_FUNCS([mprotect shm_create_largepage]) diff --git a/ext/opcache/config.w32 b/ext/opcache/config.w32 index fa89ca1f18a39..2ae2c66474c3e 100644 --- a/ext/opcache/config.w32 +++ b/ext/opcache/config.w32 @@ -33,6 +33,7 @@ if (PHP_OPCACHE != "no") { DEFINE("IR_TARGET", ir_target); DEFINE("DASM_FLAGS", dasm_flags); DEFINE("DASM_ARCH", "x86"); + DEFINE("TLS_TARGET", "win"); AC_DEFINE('HAVE_JIT', 1, 'Define to 1 to enable JIT.'); @@ -52,6 +53,11 @@ if (PHP_OPCACHE != "no") { ADD_SOURCES(configure_module_dirname + "\\jit", "zend_jit.c zend_jit_vm_helpers.c", "opcache", "ext\\opcache\\jit"); + if (PHP_ZTS == "yes") { + ADD_SOURCES(configure_module_dirname + "\\jit\\tls", + "zend_jit_tls_win.c", + "opcache", "ext\\opcache\\jit\\tls"); + } ADD_SOURCES(configure_module_dirname + "\\jit\\ir", "ir.c", "opcache", "ext\\opcache\\jit\\ir"); ADD_SOURCES(configure_module_dirname + "\\jit\\ir", diff --git a/ext/opcache/jit/tls/testing/.gitignore b/ext/opcache/jit/tls/testing/.gitignore new file mode 100644 index 0000000000000..3873646140f91 --- /dev/null +++ b/ext/opcache/jit/tls/testing/.gitignore @@ -0,0 +1,4 @@ +*.so +*.o +main +main.dSYM diff --git a/ext/opcache/jit/tls/testing/def-vars.h b/ext/opcache/jit/tls/testing/def-vars.h new file mode 100644 index 0000000000000..66cdc2442b8cd --- /dev/null +++ b/ext/opcache/jit/tls/testing/def-vars.h @@ -0,0 +1,1030 @@ +/* Declare a few additional TLS variables to fill any surplus space, + * so _tsrm_ls_cache is allocated in the dynamic section. */ + +#define DEF_VAR(prefix, num) __thread void* prefix##num +#define DEF_VARS(prefix) \ + DEF_VAR(prefix, 0000); \ + DEF_VAR(prefix, 0001); \ + DEF_VAR(prefix, 0002); \ + DEF_VAR(prefix, 0003); \ + DEF_VAR(prefix, 0004); \ + DEF_VAR(prefix, 0005); \ + DEF_VAR(prefix, 0006); \ + DEF_VAR(prefix, 0007); \ + DEF_VAR(prefix, 0008); \ + DEF_VAR(prefix, 0009); \ + DEF_VAR(prefix, 0010); \ + DEF_VAR(prefix, 0011); \ + DEF_VAR(prefix, 0012); \ + DEF_VAR(prefix, 0013); \ + DEF_VAR(prefix, 0014); \ + DEF_VAR(prefix, 0015); \ + DEF_VAR(prefix, 0016); \ + DEF_VAR(prefix, 0017); \ + DEF_VAR(prefix, 0018); \ + DEF_VAR(prefix, 0019); \ + DEF_VAR(prefix, 0020); \ + DEF_VAR(prefix, 0021); \ + DEF_VAR(prefix, 0022); \ + DEF_VAR(prefix, 0023); \ + DEF_VAR(prefix, 0024); \ + DEF_VAR(prefix, 0025); \ + DEF_VAR(prefix, 0026); \ + DEF_VAR(prefix, 0027); \ + DEF_VAR(prefix, 0028); \ + DEF_VAR(prefix, 0029); \ + DEF_VAR(prefix, 0030); \ + DEF_VAR(prefix, 0031); \ + DEF_VAR(prefix, 0032); \ + DEF_VAR(prefix, 0033); \ + DEF_VAR(prefix, 0034); \ + DEF_VAR(prefix, 0035); \ + DEF_VAR(prefix, 0036); \ + DEF_VAR(prefix, 0037); \ + DEF_VAR(prefix, 0038); \ + DEF_VAR(prefix, 0039); \ + DEF_VAR(prefix, 0040); \ + DEF_VAR(prefix, 0041); \ + DEF_VAR(prefix, 0042); \ + DEF_VAR(prefix, 0043); \ + DEF_VAR(prefix, 0044); \ + DEF_VAR(prefix, 0045); \ + DEF_VAR(prefix, 0046); \ + DEF_VAR(prefix, 0047); \ + DEF_VAR(prefix, 0048); \ + DEF_VAR(prefix, 0049); \ + DEF_VAR(prefix, 0050); \ + DEF_VAR(prefix, 0051); \ + DEF_VAR(prefix, 0052); \ + DEF_VAR(prefix, 0053); \ + DEF_VAR(prefix, 0054); \ + DEF_VAR(prefix, 0055); \ + DEF_VAR(prefix, 0056); \ + DEF_VAR(prefix, 0057); \ + DEF_VAR(prefix, 0058); \ + DEF_VAR(prefix, 0059); \ + DEF_VAR(prefix, 0060); \ + DEF_VAR(prefix, 0061); \ + DEF_VAR(prefix, 0062); \ + DEF_VAR(prefix, 0063); \ + DEF_VAR(prefix, 0064); \ + DEF_VAR(prefix, 0065); \ + DEF_VAR(prefix, 0066); \ + DEF_VAR(prefix, 0067); \ + DEF_VAR(prefix, 0068); \ + DEF_VAR(prefix, 0069); \ + DEF_VAR(prefix, 0070); \ + DEF_VAR(prefix, 0071); \ + DEF_VAR(prefix, 0072); \ + DEF_VAR(prefix, 0073); \ + DEF_VAR(prefix, 0074); \ + DEF_VAR(prefix, 0075); \ + DEF_VAR(prefix, 0076); \ + DEF_VAR(prefix, 0077); \ + DEF_VAR(prefix, 0078); \ + DEF_VAR(prefix, 0079); \ + DEF_VAR(prefix, 0080); \ + DEF_VAR(prefix, 0081); \ + DEF_VAR(prefix, 0082); \ + DEF_VAR(prefix, 0083); \ + DEF_VAR(prefix, 0084); \ + DEF_VAR(prefix, 0085); \ + DEF_VAR(prefix, 0086); \ + DEF_VAR(prefix, 0087); \ + DEF_VAR(prefix, 0088); \ + DEF_VAR(prefix, 0089); \ + DEF_VAR(prefix, 0090); \ + DEF_VAR(prefix, 0091); \ + DEF_VAR(prefix, 0092); \ + DEF_VAR(prefix, 0093); \ + DEF_VAR(prefix, 0094); \ + DEF_VAR(prefix, 0095); \ + DEF_VAR(prefix, 0096); \ + DEF_VAR(prefix, 0097); \ + DEF_VAR(prefix, 0098); \ + DEF_VAR(prefix, 0099); \ + DEF_VAR(prefix, 0100); \ + DEF_VAR(prefix, 0101); \ + DEF_VAR(prefix, 0102); \ + DEF_VAR(prefix, 0103); \ + DEF_VAR(prefix, 0104); \ + DEF_VAR(prefix, 0105); \ + DEF_VAR(prefix, 0106); \ + DEF_VAR(prefix, 0107); \ + DEF_VAR(prefix, 0108); \ + DEF_VAR(prefix, 0109); \ + DEF_VAR(prefix, 0110); \ + DEF_VAR(prefix, 0111); \ + DEF_VAR(prefix, 0112); \ + DEF_VAR(prefix, 0113); \ + DEF_VAR(prefix, 0114); \ + DEF_VAR(prefix, 0115); \ + DEF_VAR(prefix, 0116); \ + DEF_VAR(prefix, 0117); \ + DEF_VAR(prefix, 0118); \ + DEF_VAR(prefix, 0119); \ + DEF_VAR(prefix, 0120); \ + DEF_VAR(prefix, 0121); \ + DEF_VAR(prefix, 0122); \ + DEF_VAR(prefix, 0123); \ + DEF_VAR(prefix, 0124); \ + DEF_VAR(prefix, 0125); \ + DEF_VAR(prefix, 0126); \ + DEF_VAR(prefix, 0127); \ + DEF_VAR(prefix, 0128); \ + DEF_VAR(prefix, 0129); \ + DEF_VAR(prefix, 0130); \ + DEF_VAR(prefix, 0131); \ + DEF_VAR(prefix, 0132); \ + DEF_VAR(prefix, 0133); \ + DEF_VAR(prefix, 0134); \ + DEF_VAR(prefix, 0135); \ + DEF_VAR(prefix, 0136); \ + DEF_VAR(prefix, 0137); \ + DEF_VAR(prefix, 0138); \ + DEF_VAR(prefix, 0139); \ + DEF_VAR(prefix, 0140); \ + DEF_VAR(prefix, 0141); \ + DEF_VAR(prefix, 0142); \ + DEF_VAR(prefix, 0143); \ + DEF_VAR(prefix, 0144); \ + DEF_VAR(prefix, 0145); \ + DEF_VAR(prefix, 0146); \ + DEF_VAR(prefix, 0147); \ + DEF_VAR(prefix, 0148); \ + DEF_VAR(prefix, 0149); \ + DEF_VAR(prefix, 0150); \ + DEF_VAR(prefix, 0151); \ + DEF_VAR(prefix, 0152); \ + DEF_VAR(prefix, 0153); \ + DEF_VAR(prefix, 0154); \ + DEF_VAR(prefix, 0155); \ + DEF_VAR(prefix, 0156); \ + DEF_VAR(prefix, 0157); \ + DEF_VAR(prefix, 0158); \ + DEF_VAR(prefix, 0159); \ + DEF_VAR(prefix, 0160); \ + DEF_VAR(prefix, 0161); \ + DEF_VAR(prefix, 0162); \ + DEF_VAR(prefix, 0163); \ + DEF_VAR(prefix, 0164); \ + DEF_VAR(prefix, 0165); \ + DEF_VAR(prefix, 0166); \ + DEF_VAR(prefix, 0167); \ + DEF_VAR(prefix, 0168); \ + DEF_VAR(prefix, 0169); \ + DEF_VAR(prefix, 0170); \ + DEF_VAR(prefix, 0171); \ + DEF_VAR(prefix, 0172); \ + DEF_VAR(prefix, 0173); \ + DEF_VAR(prefix, 0174); \ + DEF_VAR(prefix, 0175); \ + DEF_VAR(prefix, 0176); \ + DEF_VAR(prefix, 0177); \ + DEF_VAR(prefix, 0178); \ + DEF_VAR(prefix, 0179); \ + DEF_VAR(prefix, 0180); \ + DEF_VAR(prefix, 0181); \ + DEF_VAR(prefix, 0182); \ + DEF_VAR(prefix, 0183); \ + DEF_VAR(prefix, 0184); \ + DEF_VAR(prefix, 0185); \ + DEF_VAR(prefix, 0186); \ + DEF_VAR(prefix, 0187); \ + DEF_VAR(prefix, 0188); \ + DEF_VAR(prefix, 0189); \ + DEF_VAR(prefix, 0190); \ + DEF_VAR(prefix, 0191); \ + DEF_VAR(prefix, 0192); \ + DEF_VAR(prefix, 0193); \ + DEF_VAR(prefix, 0194); \ + DEF_VAR(prefix, 0195); \ + DEF_VAR(prefix, 0196); \ + DEF_VAR(prefix, 0197); \ + DEF_VAR(prefix, 0198); \ + DEF_VAR(prefix, 0199); \ + DEF_VAR(prefix, 0200); \ + DEF_VAR(prefix, 0201); \ + DEF_VAR(prefix, 0202); \ + DEF_VAR(prefix, 0203); \ + DEF_VAR(prefix, 0204); \ + DEF_VAR(prefix, 0205); \ + DEF_VAR(prefix, 0206); \ + DEF_VAR(prefix, 0207); \ + DEF_VAR(prefix, 0208); \ + DEF_VAR(prefix, 0209); \ + DEF_VAR(prefix, 0210); \ + DEF_VAR(prefix, 0211); \ + DEF_VAR(prefix, 0212); \ + DEF_VAR(prefix, 0213); \ + DEF_VAR(prefix, 0214); \ + DEF_VAR(prefix, 0215); \ + DEF_VAR(prefix, 0216); \ + DEF_VAR(prefix, 0217); \ + DEF_VAR(prefix, 0218); \ + DEF_VAR(prefix, 0219); \ + DEF_VAR(prefix, 0220); \ + DEF_VAR(prefix, 0221); \ + DEF_VAR(prefix, 0222); \ + DEF_VAR(prefix, 0223); \ + DEF_VAR(prefix, 0224); \ + DEF_VAR(prefix, 0225); \ + DEF_VAR(prefix, 0226); \ + DEF_VAR(prefix, 0227); \ + DEF_VAR(prefix, 0228); \ + DEF_VAR(prefix, 0229); \ + DEF_VAR(prefix, 0230); \ + DEF_VAR(prefix, 0231); \ + DEF_VAR(prefix, 0232); \ + DEF_VAR(prefix, 0233); \ + DEF_VAR(prefix, 0234); \ + DEF_VAR(prefix, 0235); \ + DEF_VAR(prefix, 0236); \ + DEF_VAR(prefix, 0237); \ + DEF_VAR(prefix, 0238); \ + DEF_VAR(prefix, 0239); \ + DEF_VAR(prefix, 0240); \ + DEF_VAR(prefix, 0241); \ + DEF_VAR(prefix, 0242); \ + DEF_VAR(prefix, 0243); \ + DEF_VAR(prefix, 0244); \ + DEF_VAR(prefix, 0245); \ + DEF_VAR(prefix, 0246); \ + DEF_VAR(prefix, 0247); \ + DEF_VAR(prefix, 0248); \ + DEF_VAR(prefix, 0249); \ + DEF_VAR(prefix, 0250); \ + DEF_VAR(prefix, 0251); \ + DEF_VAR(prefix, 0252); \ + DEF_VAR(prefix, 0253); \ + DEF_VAR(prefix, 0254); \ + DEF_VAR(prefix, 0255); \ + DEF_VAR(prefix, 0256); \ + DEF_VAR(prefix, 0257); \ + DEF_VAR(prefix, 0258); \ + DEF_VAR(prefix, 0259); \ + DEF_VAR(prefix, 0260); \ + DEF_VAR(prefix, 0261); \ + DEF_VAR(prefix, 0262); \ + DEF_VAR(prefix, 0263); \ + DEF_VAR(prefix, 0264); \ + DEF_VAR(prefix, 0265); \ + DEF_VAR(prefix, 0266); \ + DEF_VAR(prefix, 0267); \ + DEF_VAR(prefix, 0268); \ + DEF_VAR(prefix, 0269); \ + DEF_VAR(prefix, 0270); \ + DEF_VAR(prefix, 0271); \ + DEF_VAR(prefix, 0272); \ + DEF_VAR(prefix, 0273); \ + DEF_VAR(prefix, 0274); \ + DEF_VAR(prefix, 0275); \ + DEF_VAR(prefix, 0276); \ + DEF_VAR(prefix, 0277); \ + DEF_VAR(prefix, 0278); \ + DEF_VAR(prefix, 0279); \ + DEF_VAR(prefix, 0280); \ + DEF_VAR(prefix, 0281); \ + DEF_VAR(prefix, 0282); \ + DEF_VAR(prefix, 0283); \ + DEF_VAR(prefix, 0284); \ + DEF_VAR(prefix, 0285); \ + DEF_VAR(prefix, 0286); \ + DEF_VAR(prefix, 0287); \ + DEF_VAR(prefix, 0288); \ + DEF_VAR(prefix, 0289); \ + DEF_VAR(prefix, 0290); \ + DEF_VAR(prefix, 0291); \ + DEF_VAR(prefix, 0292); \ + DEF_VAR(prefix, 0293); \ + DEF_VAR(prefix, 0294); \ + DEF_VAR(prefix, 0295); \ + DEF_VAR(prefix, 0296); \ + DEF_VAR(prefix, 0297); \ + DEF_VAR(prefix, 0298); \ + DEF_VAR(prefix, 0299); \ + DEF_VAR(prefix, 0300); \ + DEF_VAR(prefix, 0301); \ + DEF_VAR(prefix, 0302); \ + DEF_VAR(prefix, 0303); \ + DEF_VAR(prefix, 0304); \ + DEF_VAR(prefix, 0305); \ + DEF_VAR(prefix, 0306); \ + DEF_VAR(prefix, 0307); \ + DEF_VAR(prefix, 0308); \ + DEF_VAR(prefix, 0309); \ + DEF_VAR(prefix, 0310); \ + DEF_VAR(prefix, 0311); \ + DEF_VAR(prefix, 0312); \ + DEF_VAR(prefix, 0313); \ + DEF_VAR(prefix, 0314); \ + DEF_VAR(prefix, 0315); \ + DEF_VAR(prefix, 0316); \ + DEF_VAR(prefix, 0317); \ + DEF_VAR(prefix, 0318); \ + DEF_VAR(prefix, 0319); \ + DEF_VAR(prefix, 0320); \ + DEF_VAR(prefix, 0321); \ + DEF_VAR(prefix, 0322); \ + DEF_VAR(prefix, 0323); \ + DEF_VAR(prefix, 0324); \ + DEF_VAR(prefix, 0325); \ + DEF_VAR(prefix, 0326); \ + DEF_VAR(prefix, 0327); \ + DEF_VAR(prefix, 0328); \ + DEF_VAR(prefix, 0329); \ + DEF_VAR(prefix, 0330); \ + DEF_VAR(prefix, 0331); \ + DEF_VAR(prefix, 0332); \ + DEF_VAR(prefix, 0333); \ + DEF_VAR(prefix, 0334); \ + DEF_VAR(prefix, 0335); \ + DEF_VAR(prefix, 0336); \ + DEF_VAR(prefix, 0337); \ + DEF_VAR(prefix, 0338); \ + DEF_VAR(prefix, 0339); \ + DEF_VAR(prefix, 0340); \ + DEF_VAR(prefix, 0341); \ + DEF_VAR(prefix, 0342); \ + DEF_VAR(prefix, 0343); \ + DEF_VAR(prefix, 0344); \ + DEF_VAR(prefix, 0345); \ + DEF_VAR(prefix, 0346); \ + DEF_VAR(prefix, 0347); \ + DEF_VAR(prefix, 0348); \ + DEF_VAR(prefix, 0349); \ + DEF_VAR(prefix, 0350); \ + DEF_VAR(prefix, 0351); \ + DEF_VAR(prefix, 0352); \ + DEF_VAR(prefix, 0353); \ + DEF_VAR(prefix, 0354); \ + DEF_VAR(prefix, 0355); \ + DEF_VAR(prefix, 0356); \ + DEF_VAR(prefix, 0357); \ + DEF_VAR(prefix, 0358); \ + DEF_VAR(prefix, 0359); \ + DEF_VAR(prefix, 0360); \ + DEF_VAR(prefix, 0361); \ + DEF_VAR(prefix, 0362); \ + DEF_VAR(prefix, 0363); \ + DEF_VAR(prefix, 0364); \ + DEF_VAR(prefix, 0365); \ + DEF_VAR(prefix, 0366); \ + DEF_VAR(prefix, 0367); \ + DEF_VAR(prefix, 0368); \ + DEF_VAR(prefix, 0369); \ + DEF_VAR(prefix, 0370); \ + DEF_VAR(prefix, 0371); \ + DEF_VAR(prefix, 0372); \ + DEF_VAR(prefix, 0373); \ + DEF_VAR(prefix, 0374); \ + DEF_VAR(prefix, 0375); \ + DEF_VAR(prefix, 0376); \ + DEF_VAR(prefix, 0377); \ + DEF_VAR(prefix, 0378); \ + DEF_VAR(prefix, 0379); \ + DEF_VAR(prefix, 0380); \ + DEF_VAR(prefix, 0381); \ + DEF_VAR(prefix, 0382); \ + DEF_VAR(prefix, 0383); \ + DEF_VAR(prefix, 0384); \ + DEF_VAR(prefix, 0385); \ + DEF_VAR(prefix, 0386); \ + DEF_VAR(prefix, 0387); \ + DEF_VAR(prefix, 0388); \ + DEF_VAR(prefix, 0389); \ + DEF_VAR(prefix, 0390); \ + DEF_VAR(prefix, 0391); \ + DEF_VAR(prefix, 0392); \ + DEF_VAR(prefix, 0393); \ + DEF_VAR(prefix, 0394); \ + DEF_VAR(prefix, 0395); \ + DEF_VAR(prefix, 0396); \ + DEF_VAR(prefix, 0397); \ + DEF_VAR(prefix, 0398); \ + DEF_VAR(prefix, 0399); \ + DEF_VAR(prefix, 0400); \ + DEF_VAR(prefix, 0401); \ + DEF_VAR(prefix, 0402); \ + DEF_VAR(prefix, 0403); \ + DEF_VAR(prefix, 0404); \ + DEF_VAR(prefix, 0405); \ + DEF_VAR(prefix, 0406); \ + DEF_VAR(prefix, 0407); \ + DEF_VAR(prefix, 0408); \ + DEF_VAR(prefix, 0409); \ + DEF_VAR(prefix, 0410); \ + DEF_VAR(prefix, 0411); \ + DEF_VAR(prefix, 0412); \ + DEF_VAR(prefix, 0413); \ + DEF_VAR(prefix, 0414); \ + DEF_VAR(prefix, 0415); \ + DEF_VAR(prefix, 0416); \ + DEF_VAR(prefix, 0417); \ + DEF_VAR(prefix, 0418); \ + DEF_VAR(prefix, 0419); \ + DEF_VAR(prefix, 0420); \ + DEF_VAR(prefix, 0421); \ + DEF_VAR(prefix, 0422); \ + DEF_VAR(prefix, 0423); \ + DEF_VAR(prefix, 0424); \ + DEF_VAR(prefix, 0425); \ + DEF_VAR(prefix, 0426); \ + DEF_VAR(prefix, 0427); \ + DEF_VAR(prefix, 0428); \ + DEF_VAR(prefix, 0429); \ + DEF_VAR(prefix, 0430); \ + DEF_VAR(prefix, 0431); \ + DEF_VAR(prefix, 0432); \ + DEF_VAR(prefix, 0433); \ + DEF_VAR(prefix, 0434); \ + DEF_VAR(prefix, 0435); \ + DEF_VAR(prefix, 0436); \ + DEF_VAR(prefix, 0437); \ + DEF_VAR(prefix, 0438); \ + DEF_VAR(prefix, 0439); \ + DEF_VAR(prefix, 0440); \ + DEF_VAR(prefix, 0441); \ + DEF_VAR(prefix, 0442); \ + DEF_VAR(prefix, 0443); \ + DEF_VAR(prefix, 0444); \ + DEF_VAR(prefix, 0445); \ + DEF_VAR(prefix, 0446); \ + DEF_VAR(prefix, 0447); \ + DEF_VAR(prefix, 0448); \ + DEF_VAR(prefix, 0449); \ + DEF_VAR(prefix, 0450); \ + DEF_VAR(prefix, 0451); \ + DEF_VAR(prefix, 0452); \ + DEF_VAR(prefix, 0453); \ + DEF_VAR(prefix, 0454); \ + DEF_VAR(prefix, 0455); \ + DEF_VAR(prefix, 0456); \ + DEF_VAR(prefix, 0457); \ + DEF_VAR(prefix, 0458); \ + DEF_VAR(prefix, 0459); \ + DEF_VAR(prefix, 0460); \ + DEF_VAR(prefix, 0461); \ + DEF_VAR(prefix, 0462); \ + DEF_VAR(prefix, 0463); \ + DEF_VAR(prefix, 0464); \ + DEF_VAR(prefix, 0465); \ + DEF_VAR(prefix, 0466); \ + DEF_VAR(prefix, 0467); \ + DEF_VAR(prefix, 0468); \ + DEF_VAR(prefix, 0469); \ + DEF_VAR(prefix, 0470); \ + DEF_VAR(prefix, 0471); \ + DEF_VAR(prefix, 0472); \ + DEF_VAR(prefix, 0473); \ + DEF_VAR(prefix, 0474); \ + DEF_VAR(prefix, 0475); \ + DEF_VAR(prefix, 0476); \ + DEF_VAR(prefix, 0477); \ + DEF_VAR(prefix, 0478); \ + DEF_VAR(prefix, 0479); \ + DEF_VAR(prefix, 0480); \ + DEF_VAR(prefix, 0481); \ + DEF_VAR(prefix, 0482); \ + DEF_VAR(prefix, 0483); \ + DEF_VAR(prefix, 0484); \ + DEF_VAR(prefix, 0485); \ + DEF_VAR(prefix, 0486); \ + DEF_VAR(prefix, 0487); \ + DEF_VAR(prefix, 0488); \ + DEF_VAR(prefix, 0489); \ + DEF_VAR(prefix, 0490); \ + DEF_VAR(prefix, 0491); \ + DEF_VAR(prefix, 0492); \ + DEF_VAR(prefix, 0493); \ + DEF_VAR(prefix, 0494); \ + DEF_VAR(prefix, 0495); \ + DEF_VAR(prefix, 0496); \ + DEF_VAR(prefix, 0497); \ + DEF_VAR(prefix, 0498); \ + DEF_VAR(prefix, 0499); \ + DEF_VAR(prefix, 0500); \ + DEF_VAR(prefix, 0501); \ + DEF_VAR(prefix, 0502); \ + DEF_VAR(prefix, 0503); \ + DEF_VAR(prefix, 0504); \ + DEF_VAR(prefix, 0505); \ + DEF_VAR(prefix, 0506); \ + DEF_VAR(prefix, 0507); \ + DEF_VAR(prefix, 0508); \ + DEF_VAR(prefix, 0509); \ + DEF_VAR(prefix, 0510); \ + DEF_VAR(prefix, 0511); \ + DEF_VAR(prefix, 0512); \ + DEF_VAR(prefix, 0513); \ + DEF_VAR(prefix, 0514); \ + DEF_VAR(prefix, 0515); \ + DEF_VAR(prefix, 0516); \ + DEF_VAR(prefix, 0517); \ + DEF_VAR(prefix, 0518); \ + DEF_VAR(prefix, 0519); \ + DEF_VAR(prefix, 0520); \ + DEF_VAR(prefix, 0521); \ + DEF_VAR(prefix, 0522); \ + DEF_VAR(prefix, 0523); \ + DEF_VAR(prefix, 0524); \ + DEF_VAR(prefix, 0525); \ + DEF_VAR(prefix, 0526); \ + DEF_VAR(prefix, 0527); \ + DEF_VAR(prefix, 0528); \ + DEF_VAR(prefix, 0529); \ + DEF_VAR(prefix, 0530); \ + DEF_VAR(prefix, 0531); \ + DEF_VAR(prefix, 0532); \ + DEF_VAR(prefix, 0533); \ + DEF_VAR(prefix, 0534); \ + DEF_VAR(prefix, 0535); \ + DEF_VAR(prefix, 0536); \ + DEF_VAR(prefix, 0537); \ + DEF_VAR(prefix, 0538); \ + DEF_VAR(prefix, 0539); \ + DEF_VAR(prefix, 0540); \ + DEF_VAR(prefix, 0541); \ + DEF_VAR(prefix, 0542); \ + DEF_VAR(prefix, 0543); \ + DEF_VAR(prefix, 0544); \ + DEF_VAR(prefix, 0545); \ + DEF_VAR(prefix, 0546); \ + DEF_VAR(prefix, 0547); \ + DEF_VAR(prefix, 0548); \ + DEF_VAR(prefix, 0549); \ + DEF_VAR(prefix, 0550); \ + DEF_VAR(prefix, 0551); \ + DEF_VAR(prefix, 0552); \ + DEF_VAR(prefix, 0553); \ + DEF_VAR(prefix, 0554); \ + DEF_VAR(prefix, 0555); \ + DEF_VAR(prefix, 0556); \ + DEF_VAR(prefix, 0557); \ + DEF_VAR(prefix, 0558); \ + DEF_VAR(prefix, 0559); \ + DEF_VAR(prefix, 0560); \ + DEF_VAR(prefix, 0561); \ + DEF_VAR(prefix, 0562); \ + DEF_VAR(prefix, 0563); \ + DEF_VAR(prefix, 0564); \ + DEF_VAR(prefix, 0565); \ + DEF_VAR(prefix, 0566); \ + DEF_VAR(prefix, 0567); \ + DEF_VAR(prefix, 0568); \ + DEF_VAR(prefix, 0569); \ + DEF_VAR(prefix, 0570); \ + DEF_VAR(prefix, 0571); \ + DEF_VAR(prefix, 0572); \ + DEF_VAR(prefix, 0573); \ + DEF_VAR(prefix, 0574); \ + DEF_VAR(prefix, 0575); \ + DEF_VAR(prefix, 0576); \ + DEF_VAR(prefix, 0577); \ + DEF_VAR(prefix, 0578); \ + DEF_VAR(prefix, 0579); \ + DEF_VAR(prefix, 0580); \ + DEF_VAR(prefix, 0581); \ + DEF_VAR(prefix, 0582); \ + DEF_VAR(prefix, 0583); \ + DEF_VAR(prefix, 0584); \ + DEF_VAR(prefix, 0585); \ + DEF_VAR(prefix, 0586); \ + DEF_VAR(prefix, 0587); \ + DEF_VAR(prefix, 0588); \ + DEF_VAR(prefix, 0589); \ + DEF_VAR(prefix, 0590); \ + DEF_VAR(prefix, 0591); \ + DEF_VAR(prefix, 0592); \ + DEF_VAR(prefix, 0593); \ + DEF_VAR(prefix, 0594); \ + DEF_VAR(prefix, 0595); \ + DEF_VAR(prefix, 0596); \ + DEF_VAR(prefix, 0597); \ + DEF_VAR(prefix, 0598); \ + DEF_VAR(prefix, 0599); \ + DEF_VAR(prefix, 0600); \ + DEF_VAR(prefix, 0601); \ + DEF_VAR(prefix, 0602); \ + DEF_VAR(prefix, 0603); \ + DEF_VAR(prefix, 0604); \ + DEF_VAR(prefix, 0605); \ + DEF_VAR(prefix, 0606); \ + DEF_VAR(prefix, 0607); \ + DEF_VAR(prefix, 0608); \ + DEF_VAR(prefix, 0609); \ + DEF_VAR(prefix, 0610); \ + DEF_VAR(prefix, 0611); \ + DEF_VAR(prefix, 0612); \ + DEF_VAR(prefix, 0613); \ + DEF_VAR(prefix, 0614); \ + DEF_VAR(prefix, 0615); \ + DEF_VAR(prefix, 0616); \ + DEF_VAR(prefix, 0617); \ + DEF_VAR(prefix, 0618); \ + DEF_VAR(prefix, 0619); \ + DEF_VAR(prefix, 0620); \ + DEF_VAR(prefix, 0621); \ + DEF_VAR(prefix, 0622); \ + DEF_VAR(prefix, 0623); \ + DEF_VAR(prefix, 0624); \ + DEF_VAR(prefix, 0625); \ + DEF_VAR(prefix, 0626); \ + DEF_VAR(prefix, 0627); \ + DEF_VAR(prefix, 0628); \ + DEF_VAR(prefix, 0629); \ + DEF_VAR(prefix, 0630); \ + DEF_VAR(prefix, 0631); \ + DEF_VAR(prefix, 0632); \ + DEF_VAR(prefix, 0633); \ + DEF_VAR(prefix, 0634); \ + DEF_VAR(prefix, 0635); \ + DEF_VAR(prefix, 0636); \ + DEF_VAR(prefix, 0637); \ + DEF_VAR(prefix, 0638); \ + DEF_VAR(prefix, 0639); \ + DEF_VAR(prefix, 0640); \ + DEF_VAR(prefix, 0641); \ + DEF_VAR(prefix, 0642); \ + DEF_VAR(prefix, 0643); \ + DEF_VAR(prefix, 0644); \ + DEF_VAR(prefix, 0645); \ + DEF_VAR(prefix, 0646); \ + DEF_VAR(prefix, 0647); \ + DEF_VAR(prefix, 0648); \ + DEF_VAR(prefix, 0649); \ + DEF_VAR(prefix, 0650); \ + DEF_VAR(prefix, 0651); \ + DEF_VAR(prefix, 0652); \ + DEF_VAR(prefix, 0653); \ + DEF_VAR(prefix, 0654); \ + DEF_VAR(prefix, 0655); \ + DEF_VAR(prefix, 0656); \ + DEF_VAR(prefix, 0657); \ + DEF_VAR(prefix, 0658); \ + DEF_VAR(prefix, 0659); \ + DEF_VAR(prefix, 0660); \ + DEF_VAR(prefix, 0661); \ + DEF_VAR(prefix, 0662); \ + DEF_VAR(prefix, 0663); \ + DEF_VAR(prefix, 0664); \ + DEF_VAR(prefix, 0665); \ + DEF_VAR(prefix, 0666); \ + DEF_VAR(prefix, 0667); \ + DEF_VAR(prefix, 0668); \ + DEF_VAR(prefix, 0669); \ + DEF_VAR(prefix, 0670); \ + DEF_VAR(prefix, 0671); \ + DEF_VAR(prefix, 0672); \ + DEF_VAR(prefix, 0673); \ + DEF_VAR(prefix, 0674); \ + DEF_VAR(prefix, 0675); \ + DEF_VAR(prefix, 0676); \ + DEF_VAR(prefix, 0677); \ + DEF_VAR(prefix, 0678); \ + DEF_VAR(prefix, 0679); \ + DEF_VAR(prefix, 0680); \ + DEF_VAR(prefix, 0681); \ + DEF_VAR(prefix, 0682); \ + DEF_VAR(prefix, 0683); \ + DEF_VAR(prefix, 0684); \ + DEF_VAR(prefix, 0685); \ + DEF_VAR(prefix, 0686); \ + DEF_VAR(prefix, 0687); \ + DEF_VAR(prefix, 0688); \ + DEF_VAR(prefix, 0689); \ + DEF_VAR(prefix, 0690); \ + DEF_VAR(prefix, 0691); \ + DEF_VAR(prefix, 0692); \ + DEF_VAR(prefix, 0693); \ + DEF_VAR(prefix, 0694); \ + DEF_VAR(prefix, 0695); \ + DEF_VAR(prefix, 0696); \ + DEF_VAR(prefix, 0697); \ + DEF_VAR(prefix, 0698); \ + DEF_VAR(prefix, 0699); \ + DEF_VAR(prefix, 0700); \ + DEF_VAR(prefix, 0701); \ + DEF_VAR(prefix, 0702); \ + DEF_VAR(prefix, 0703); \ + DEF_VAR(prefix, 0704); \ + DEF_VAR(prefix, 0705); \ + DEF_VAR(prefix, 0706); \ + DEF_VAR(prefix, 0707); \ + DEF_VAR(prefix, 0708); \ + DEF_VAR(prefix, 0709); \ + DEF_VAR(prefix, 0710); \ + DEF_VAR(prefix, 0711); \ + DEF_VAR(prefix, 0712); \ + DEF_VAR(prefix, 0713); \ + DEF_VAR(prefix, 0714); \ + DEF_VAR(prefix, 0715); \ + DEF_VAR(prefix, 0716); \ + DEF_VAR(prefix, 0717); \ + DEF_VAR(prefix, 0718); \ + DEF_VAR(prefix, 0719); \ + DEF_VAR(prefix, 0720); \ + DEF_VAR(prefix, 0721); \ + DEF_VAR(prefix, 0722); \ + DEF_VAR(prefix, 0723); \ + DEF_VAR(prefix, 0724); \ + DEF_VAR(prefix, 0725); \ + DEF_VAR(prefix, 0726); \ + DEF_VAR(prefix, 0727); \ + DEF_VAR(prefix, 0728); \ + DEF_VAR(prefix, 0729); \ + DEF_VAR(prefix, 0730); \ + DEF_VAR(prefix, 0731); \ + DEF_VAR(prefix, 0732); \ + DEF_VAR(prefix, 0733); \ + DEF_VAR(prefix, 0734); \ + DEF_VAR(prefix, 0735); \ + DEF_VAR(prefix, 0736); \ + DEF_VAR(prefix, 0737); \ + DEF_VAR(prefix, 0738); \ + DEF_VAR(prefix, 0739); \ + DEF_VAR(prefix, 0740); \ + DEF_VAR(prefix, 0741); \ + DEF_VAR(prefix, 0742); \ + DEF_VAR(prefix, 0743); \ + DEF_VAR(prefix, 0744); \ + DEF_VAR(prefix, 0745); \ + DEF_VAR(prefix, 0746); \ + DEF_VAR(prefix, 0747); \ + DEF_VAR(prefix, 0748); \ + DEF_VAR(prefix, 0749); \ + DEF_VAR(prefix, 0750); \ + DEF_VAR(prefix, 0751); \ + DEF_VAR(prefix, 0752); \ + DEF_VAR(prefix, 0753); \ + DEF_VAR(prefix, 0754); \ + DEF_VAR(prefix, 0755); \ + DEF_VAR(prefix, 0756); \ + DEF_VAR(prefix, 0757); \ + DEF_VAR(prefix, 0758); \ + DEF_VAR(prefix, 0759); \ + DEF_VAR(prefix, 0760); \ + DEF_VAR(prefix, 0761); \ + DEF_VAR(prefix, 0762); \ + DEF_VAR(prefix, 0763); \ + DEF_VAR(prefix, 0764); \ + DEF_VAR(prefix, 0765); \ + DEF_VAR(prefix, 0766); \ + DEF_VAR(prefix, 0767); \ + DEF_VAR(prefix, 0768); \ + DEF_VAR(prefix, 0769); \ + DEF_VAR(prefix, 0770); \ + DEF_VAR(prefix, 0771); \ + DEF_VAR(prefix, 0772); \ + DEF_VAR(prefix, 0773); \ + DEF_VAR(prefix, 0774); \ + DEF_VAR(prefix, 0775); \ + DEF_VAR(prefix, 0776); \ + DEF_VAR(prefix, 0777); \ + DEF_VAR(prefix, 0778); \ + DEF_VAR(prefix, 0779); \ + DEF_VAR(prefix, 0780); \ + DEF_VAR(prefix, 0781); \ + DEF_VAR(prefix, 0782); \ + DEF_VAR(prefix, 0783); \ + DEF_VAR(prefix, 0784); \ + DEF_VAR(prefix, 0785); \ + DEF_VAR(prefix, 0786); \ + DEF_VAR(prefix, 0787); \ + DEF_VAR(prefix, 0788); \ + DEF_VAR(prefix, 0789); \ + DEF_VAR(prefix, 0790); \ + DEF_VAR(prefix, 0791); \ + DEF_VAR(prefix, 0792); \ + DEF_VAR(prefix, 0793); \ + DEF_VAR(prefix, 0794); \ + DEF_VAR(prefix, 0795); \ + DEF_VAR(prefix, 0796); \ + DEF_VAR(prefix, 0797); \ + DEF_VAR(prefix, 0798); \ + DEF_VAR(prefix, 0799); \ + DEF_VAR(prefix, 0800); \ + DEF_VAR(prefix, 0801); \ + DEF_VAR(prefix, 0802); \ + DEF_VAR(prefix, 0803); \ + DEF_VAR(prefix, 0804); \ + DEF_VAR(prefix, 0805); \ + DEF_VAR(prefix, 0806); \ + DEF_VAR(prefix, 0807); \ + DEF_VAR(prefix, 0808); \ + DEF_VAR(prefix, 0809); \ + DEF_VAR(prefix, 0810); \ + DEF_VAR(prefix, 0811); \ + DEF_VAR(prefix, 0812); \ + DEF_VAR(prefix, 0813); \ + DEF_VAR(prefix, 0814); \ + DEF_VAR(prefix, 0815); \ + DEF_VAR(prefix, 0816); \ + DEF_VAR(prefix, 0817); \ + DEF_VAR(prefix, 0818); \ + DEF_VAR(prefix, 0819); \ + DEF_VAR(prefix, 0820); \ + DEF_VAR(prefix, 0821); \ + DEF_VAR(prefix, 0822); \ + DEF_VAR(prefix, 0823); \ + DEF_VAR(prefix, 0824); \ + DEF_VAR(prefix, 0825); \ + DEF_VAR(prefix, 0826); \ + DEF_VAR(prefix, 0827); \ + DEF_VAR(prefix, 0828); \ + DEF_VAR(prefix, 0829); \ + DEF_VAR(prefix, 0830); \ + DEF_VAR(prefix, 0831); \ + DEF_VAR(prefix, 0832); \ + DEF_VAR(prefix, 0833); \ + DEF_VAR(prefix, 0834); \ + DEF_VAR(prefix, 0835); \ + DEF_VAR(prefix, 0836); \ + DEF_VAR(prefix, 0837); \ + DEF_VAR(prefix, 0838); \ + DEF_VAR(prefix, 0839); \ + DEF_VAR(prefix, 0840); \ + DEF_VAR(prefix, 0841); \ + DEF_VAR(prefix, 0842); \ + DEF_VAR(prefix, 0843); \ + DEF_VAR(prefix, 0844); \ + DEF_VAR(prefix, 0845); \ + DEF_VAR(prefix, 0846); \ + DEF_VAR(prefix, 0847); \ + DEF_VAR(prefix, 0848); \ + DEF_VAR(prefix, 0849); \ + DEF_VAR(prefix, 0850); \ + DEF_VAR(prefix, 0851); \ + DEF_VAR(prefix, 0852); \ + DEF_VAR(prefix, 0853); \ + DEF_VAR(prefix, 0854); \ + DEF_VAR(prefix, 0855); \ + DEF_VAR(prefix, 0856); \ + DEF_VAR(prefix, 0857); \ + DEF_VAR(prefix, 0858); \ + DEF_VAR(prefix, 0859); \ + DEF_VAR(prefix, 0860); \ + DEF_VAR(prefix, 0861); \ + DEF_VAR(prefix, 0862); \ + DEF_VAR(prefix, 0863); \ + DEF_VAR(prefix, 0864); \ + DEF_VAR(prefix, 0865); \ + DEF_VAR(prefix, 0866); \ + DEF_VAR(prefix, 0867); \ + DEF_VAR(prefix, 0868); \ + DEF_VAR(prefix, 0869); \ + DEF_VAR(prefix, 0870); \ + DEF_VAR(prefix, 0871); \ + DEF_VAR(prefix, 0872); \ + DEF_VAR(prefix, 0873); \ + DEF_VAR(prefix, 0874); \ + DEF_VAR(prefix, 0875); \ + DEF_VAR(prefix, 0876); \ + DEF_VAR(prefix, 0877); \ + DEF_VAR(prefix, 0878); \ + DEF_VAR(prefix, 0879); \ + DEF_VAR(prefix, 0880); \ + DEF_VAR(prefix, 0881); \ + DEF_VAR(prefix, 0882); \ + DEF_VAR(prefix, 0883); \ + DEF_VAR(prefix, 0884); \ + DEF_VAR(prefix, 0885); \ + DEF_VAR(prefix, 0886); \ + DEF_VAR(prefix, 0887); \ + DEF_VAR(prefix, 0888); \ + DEF_VAR(prefix, 0889); \ + DEF_VAR(prefix, 0890); \ + DEF_VAR(prefix, 0891); \ + DEF_VAR(prefix, 0892); \ + DEF_VAR(prefix, 0893); \ + DEF_VAR(prefix, 0894); \ + DEF_VAR(prefix, 0895); \ + DEF_VAR(prefix, 0896); \ + DEF_VAR(prefix, 0897); \ + DEF_VAR(prefix, 0898); \ + DEF_VAR(prefix, 0899); \ + DEF_VAR(prefix, 0900); \ + DEF_VAR(prefix, 0901); \ + DEF_VAR(prefix, 0902); \ + DEF_VAR(prefix, 0903); \ + DEF_VAR(prefix, 0904); \ + DEF_VAR(prefix, 0905); \ + DEF_VAR(prefix, 0906); \ + DEF_VAR(prefix, 0907); \ + DEF_VAR(prefix, 0908); \ + DEF_VAR(prefix, 0909); \ + DEF_VAR(prefix, 0910); \ + DEF_VAR(prefix, 0911); \ + DEF_VAR(prefix, 0912); \ + DEF_VAR(prefix, 0913); \ + DEF_VAR(prefix, 0914); \ + DEF_VAR(prefix, 0915); \ + DEF_VAR(prefix, 0916); \ + DEF_VAR(prefix, 0917); \ + DEF_VAR(prefix, 0918); \ + DEF_VAR(prefix, 0919); \ + DEF_VAR(prefix, 0920); \ + DEF_VAR(prefix, 0921); \ + DEF_VAR(prefix, 0922); \ + DEF_VAR(prefix, 0923); \ + DEF_VAR(prefix, 0924); \ + DEF_VAR(prefix, 0925); \ + DEF_VAR(prefix, 0926); \ + DEF_VAR(prefix, 0927); \ + DEF_VAR(prefix, 0928); \ + DEF_VAR(prefix, 0929); \ + DEF_VAR(prefix, 0930); \ + DEF_VAR(prefix, 0931); \ + DEF_VAR(prefix, 0932); \ + DEF_VAR(prefix, 0933); \ + DEF_VAR(prefix, 0934); \ + DEF_VAR(prefix, 0935); \ + DEF_VAR(prefix, 0936); \ + DEF_VAR(prefix, 0937); \ + DEF_VAR(prefix, 0938); \ + DEF_VAR(prefix, 0939); \ + DEF_VAR(prefix, 0940); \ + DEF_VAR(prefix, 0941); \ + DEF_VAR(prefix, 0942); \ + DEF_VAR(prefix, 0943); \ + DEF_VAR(prefix, 0944); \ + DEF_VAR(prefix, 0945); \ + DEF_VAR(prefix, 0946); \ + DEF_VAR(prefix, 0947); \ + DEF_VAR(prefix, 0948); \ + DEF_VAR(prefix, 0949); \ + DEF_VAR(prefix, 0950); \ + DEF_VAR(prefix, 0951); \ + DEF_VAR(prefix, 0952); \ + DEF_VAR(prefix, 0953); \ + DEF_VAR(prefix, 0954); \ + DEF_VAR(prefix, 0955); \ + DEF_VAR(prefix, 0956); \ + DEF_VAR(prefix, 0957); \ + DEF_VAR(prefix, 0958); \ + DEF_VAR(prefix, 0959); \ + DEF_VAR(prefix, 0960); \ + DEF_VAR(prefix, 0961); \ + DEF_VAR(prefix, 0962); \ + DEF_VAR(prefix, 0963); \ + DEF_VAR(prefix, 0964); \ + DEF_VAR(prefix, 0965); \ + DEF_VAR(prefix, 0966); \ + DEF_VAR(prefix, 0967); \ + DEF_VAR(prefix, 0968); \ + DEF_VAR(prefix, 0969); \ + DEF_VAR(prefix, 0970); \ + DEF_VAR(prefix, 0971); \ + DEF_VAR(prefix, 0972); \ + DEF_VAR(prefix, 0973); \ + DEF_VAR(prefix, 0974); \ + DEF_VAR(prefix, 0975); \ + DEF_VAR(prefix, 0976); \ + DEF_VAR(prefix, 0977); \ + DEF_VAR(prefix, 0978); \ + DEF_VAR(prefix, 0979); \ + DEF_VAR(prefix, 0980); \ + DEF_VAR(prefix, 0981); \ + DEF_VAR(prefix, 0982); \ + DEF_VAR(prefix, 0983); \ + DEF_VAR(prefix, 0984); \ + DEF_VAR(prefix, 0985); \ + DEF_VAR(prefix, 0986); \ + DEF_VAR(prefix, 0987); \ + DEF_VAR(prefix, 0988); \ + DEF_VAR(prefix, 0989); \ + DEF_VAR(prefix, 0990); \ + DEF_VAR(prefix, 0991); \ + DEF_VAR(prefix, 0992); \ + DEF_VAR(prefix, 0993); \ + DEF_VAR(prefix, 0994); \ + DEF_VAR(prefix, 0995); \ + DEF_VAR(prefix, 0996); \ + DEF_VAR(prefix, 0997); \ + DEF_VAR(prefix, 0998); \ + DEF_VAR(prefix, 0999); \ + DEF_VAR(prefix, 1000); \ + DEF_VAR(prefix, 1001); \ + DEF_VAR(prefix, 1002); \ + DEF_VAR(prefix, 1003); \ + DEF_VAR(prefix, 1004); \ + DEF_VAR(prefix, 1005); \ + DEF_VAR(prefix, 1006); \ + DEF_VAR(prefix, 1007); \ + DEF_VAR(prefix, 1008); \ + DEF_VAR(prefix, 1009); \ + DEF_VAR(prefix, 1010); \ + DEF_VAR(prefix, 1011); \ + DEF_VAR(prefix, 1012); \ + DEF_VAR(prefix, 1013); \ + DEF_VAR(prefix, 1014); \ + DEF_VAR(prefix, 1015); \ + DEF_VAR(prefix, 1016); \ + DEF_VAR(prefix, 1017); \ + DEF_VAR(prefix, 1018); \ + DEF_VAR(prefix, 1019); \ + DEF_VAR(prefix, 1020); \ + DEF_VAR(prefix, 1021); \ + DEF_VAR(prefix, 1022); \ + DEF_VAR(prefix, 1023); \ + DEF_VAR(prefix, 1024); diff --git a/ext/opcache/jit/tls/testing/def.c b/ext/opcache/jit/tls/testing/def.c new file mode 100644 index 0000000000000..f18d577e278a5 --- /dev/null +++ b/ext/opcache/jit/tls/testing/def.c @@ -0,0 +1,33 @@ + +/* _tsrm_ls_cache is defined here */ + +#include +#include +#include + +#ifdef NO_SURPLUS +# include "def-vars.h" +DEF_VARS(def); +#endif + +__thread void* _tsrm_ls_cache; + +size_t tsrm_get_ls_cache_tcb_offset(void) { + return 0; +} + +void zend_accel_error(int type, const char *format, ...) { + if (type < 4) { + va_list ap; + va_start(ap, format); + vprintf(format, ap); + va_end(ap); + } +} + +int test(void); + +int decl(void) { + return test(); +} + diff --git a/ext/opcache/jit/tls/testing/main.c b/ext/opcache/jit/tls/testing/main.c new file mode 100644 index 0000000000000..75d40ff7888ea --- /dev/null +++ b/ext/opcache/jit/tls/testing/main.c @@ -0,0 +1,48 @@ +#include +#include + +#ifdef NO_SURPLUS +# include "def-vars.h" +DEF_VARS(main); +#endif + +__thread int some_tls_var; + +#ifndef DL_DECL +int decl(void); +#endif + +int main(void) { + /* Ensure TLS vars are allocated */ + some_tls_var = 1; + + int (*decl_p)(void); +#ifdef DL_DECL + int flags = RTLD_LAZY | RTLD_GLOBAL; +# ifdef RTLD_DEEPBIND + flags |= RTLD_DEEPBIND; +# endif + void *handle = dlopen("./libdef.so", flags); + if (!handle) { + fprintf(stderr, "dlopen: %s\n", dlerror()); + return 1; + } + + decl_p = (int (*)(void)) dlsym(handle, "decl"); + if (!decl_p) { + fprintf(stderr, "dlsym: %s\n", dlerror()); + return 1; + } +#else + decl_p = decl; +#endif + + int ret = decl_p(); + if (!ret) { + fprintf(stderr, "FAIL\n"); + } else { + fprintf(stderr, "OK\n"); + } + + return !ret; +} diff --git a/ext/opcache/jit/tls/testing/test.sh b/ext/opcache/jit/tls/testing/test.sh new file mode 100755 index 0000000000000..c6b003caafddd --- /dev/null +++ b/ext/opcache/jit/tls/testing/test.sh @@ -0,0 +1,232 @@ +#!/bin/sh + +set -e +cd "$(dirname "$0")" + +print_test() { + echo "Testing: $1 (CC=$CC LD=$LD CFLAGS=$CFLAGS)" +} + +exe_def_static_user() { + print_test "TLS var defined in executable, used in same object" + + rm -f main + + $CC $CFLAGS -ggdb3 -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -o user.o -c user.c + $CC $CFLAGS -ggdb3 -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -ggdb3 -o main main.c def.o user.o tls.o + + ./main +} + +exe_def_shared_user() { + print_test "TLS var defined in executable, used in shared library" + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS $LDFLAGS -shared -o libuser.so user.o tls.o + + $CC $CFLAGS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -ggdb3 -fPIC -o main main.c def.o -Wl,-rpath,$(pwd) -L. -luser + + ./main +} + +shared_def_static_user() { + print_test "TLS var defined in shared library, used in same object" + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o user.o tls.o + + $CC $CFLAGS $LDFLAGS -ggdb3 -fPIC -o main main.c -Wl,-rpath,$(pwd) -L. -ldef + + ./main +} + +shared_def_shared_user() { + print_test "TLS var defined in shared object, used in other shared object" + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS $LDFLAGS -shared -o libuser.so user.o tls.o + + $CC $CFLAGS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o -Wl,-rpath,$(pwd) -L. -luser + + $CC $CFLAGS $LDFLAGS -ggdb3 -fPIC -o main main.c -Wl,-rpath,$(pwd) -L. -ldef + + ./main +} + +shared_def_static_user_no_surplus() { + print_test "TLS var defined in shared library, used in same object. Likely no static TLS surplus." + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS -DNO_SURPLUS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o tls.o user.o + + $CC $CFLAGS -DNO_SURPLUS $LDFLAGS -ggdb3 -fPIC -o main main.c -Wl,-rpath,$(pwd) -L. -ldef + + ./main +} + +shared_def_shared_user_no_surplus() { + print_test "TLS var defined in shared object, used in other shared object. Likely no static TLS surplus." + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS $LDFLAGS -shared -o libuser.so user.o tls.o + + $CC $CFLAGS -DNO_SURPLUS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o -Wl,-rpath,$(pwd) -L. -luser + + $CC $CFLAGS -DNO_SURPLUS $LDFLAGS -ggdb3 -fPIC -o main main.c -Wl,-rpath,$(pwd) -L. -ldef + + ./main +} + +dl_def_static_user() { + print_test "TLS var defined in dl()'ed object, used in same object" + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o user.o tls.o + + $CC $CFLAGS $LDFLAGS -DDL_DECL -ggdb3 -fPIC -o main main.c + + ./main +} + +dl_def_shared_user() { + print_test "TLS var defined in dl()'ed object, used in other shared object" + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS $LDFLAGS -shared -o libuser.so user.o tls.o + + $CC $CFLAGS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o -Wl,-rpath,$(pwd) -L. -luser + + $CC $CFLAGS $LDFLAGS -DDL_DECL -ggdb3 -fPIC -o main main.c + + ./main +} + +dl_def_static_user_no_surplus() { + print_test "TLS var defined in dl()'ed object, used in same object. Likely no surplus TLS" + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS -DNO_SURPLUS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o user.o tls.o + + $CC $CFLAGS -DNO_SURPLUS $LDFLAGS -DDL_DECL -ggdb3 -fPIC -o main main.c + + ./main +} + +dl_def_shared_user_no_surplus() { + print_test "TLS var defined in dl()'ed object, used in other shared object. Likely no surplus TLS" + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS $LDFLAGS -shared -o libuser.so user.o tls.o + + $CC $CFLAGS -DNO_SURPLUS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o -Wl,-rpath,$(pwd) -L. -luser + + $CC $CFLAGS -DNO_SURPLUS $LDFLAGS -DDL_DECL -ggdb3 -fPIC -o main main.c + + ./main +} + +if [ -z "$TLSC" ]; then + echo "Variable TLSC is not set" >&2 + exit 1 +fi + +root=$(pwd)/../../../../.. + +# Cheap musl detection +if test -f /etc/alpine-release; then + MUSL="$CFLAGS -D__MUSL__" +else + MUSL= +fi + +if [ "${STATIC_SUPPORT:-yes}" = "yes" ]; then + STATIC=-static +fi + +for CC in clang gcc; do + if [ $CC = gcc ] && [ -f /etc/freebsd-update.conf ]; then + RPATH=-Wl,-rpath,/usr/local/lib/gcc13 + else + RPATH= + fi + case $CC in + gcc) + LDs="" + for l in bdf gold; do + if command -v ld.$l >/dev/null 2>&1; then + LDs="$LDs $l" + fi + done + if [ -z "$LDs" ]; then + LDs=ld + fi + ;; + clang) + LDs="ld" + if command -v ld.lld >/dev/null 2>&1; then + LDs="$LDs lld" + fi + ;; + esac + for LD in $LDs; do + for opt in -O0 -O3; do + CFLAGS="$MACHINE $MUSL $opt -Werror -I$root/ext/opcache -I$root/Zend -I$root" + LDFLAGS="$MACHINE -fuse-ld=$LD $RPATH" + + for pic in "-fPIC" "-fno-PIC $STATIC"; do + CFLAGS="$CFLAGS $pic" exe_def_static_user + done + shared_def_static_user + shared_def_static_user_no_surplus + dl_def_static_user + dl_def_static_user_no_surplus + if [ "$EXTERN_TLS_SUPPORT" = yes ]; then + exe_def_shared_user + shared_def_shared_user + shared_def_shared_user_no_surplus + dl_def_shared_user + dl_def_shared_user_no_surplus + fi + done + done +done + +echo "All OK" >&2 diff --git a/ext/opcache/jit/tls/testing/user.c b/ext/opcache/jit/tls/testing/user.c new file mode 100644 index 0000000000000..c27e608f3f4b1 --- /dev/null +++ b/ext/opcache/jit/tls/testing/user.c @@ -0,0 +1,28 @@ + +/* _tsrm_ls_cache is used / inspected here */ + +#include "../zend_jit_tls.h" + +extern __thread void* _tsrm_ls_cache; + +int test(void) +{ + size_t tcb_offset = 0; + size_t module_index = -1; + size_t module_offset = -1; + + /* Ensure the slot is allocated */ + _tsrm_ls_cache = NULL; + + zend_result result = zend_jit_resolve_tsrm_ls_cache_offsets( + &tcb_offset, &module_index, &module_offset); + + printf("tcb_offset: %zd; module_index: %zd; module_offset: %zd\n", + tcb_offset, module_index, module_offset); + + if (result != SUCCESS) { + return 0; + } + + return zend_jit_tsrm_ls_cache_address(tcb_offset, module_index, module_offset) == &_tsrm_ls_cache; +} diff --git a/ext/opcache/jit/tls/zend_jit_tls.h b/ext/opcache/jit/tls/zend_jit_tls.h new file mode 100644 index 0000000000000..5f90429267256 --- /dev/null +++ b/ext/opcache/jit/tls/zend_jit_tls.h @@ -0,0 +1,40 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | https://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Arnaud Le Blanc | + * +----------------------------------------------------------------------+ + */ + +#ifndef ZEND_JIT_TLS_H +#define ZEND_JIT_TLS_H + +#include "Zend/zend_types.h" + +#include +#include + +zend_result zend_jit_resolve_tsrm_ls_cache_offsets( + size_t *tcb_offset, + size_t *module_index, + size_t *module_offset +); + +/* Used for testing */ +void *zend_jit_tsrm_ls_cache_address( + size_t tcb_offset, + size_t module_index, + size_t module_offset +); + +#endif /* ZEND_JIT_TLS_H */ diff --git a/ext/opcache/jit/tls/zend_jit_tls_aarch64.c b/ext/opcache/jit/tls/zend_jit_tls_aarch64.c new file mode 100644 index 0000000000000..2dc82221e9cb1 --- /dev/null +++ b/ext/opcache/jit/tls/zend_jit_tls_aarch64.c @@ -0,0 +1,255 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | https://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Arnaud Le Blanc | + * +----------------------------------------------------------------------+ + */ + +#include "Zend/zend_portability.h" +#include "Zend/zend_types.h" +#include "TSRM/TSRM.h" +#include "zend_accelerator_debug.h" + +#include +#include + +TSRMLS_CACHE_EXTERN(); + +/* https://developer.arm.com/documentation/ddi0602/2025-03/Base-Instructions/ADRP--Form-PC-relative-address-to-4KB-page- */ +#define AARCH64_ADRP_IMM_MASK 0x60ffffe0 /* bits 30-29, 23-5 */ +#define AARCH64_ADRP_IMMHI_MASK 0x00ffffe0 /* bits 23-5 */ +#define AARCH64_ADRP_IMMLO_MASK 0x60000000 /* bits 30-29 */ +#define AARCH64_ADRP_IMMHI_START 5 +#define AARCH64_ADRP_IMMLO_START 29 +#define AARCH64_ADRP_IMMLO_WIDTH 2 + +#define AARCH64_LDR_UNSIGNED_IMM_MASK 0x003ffc00 /* bits 21-10 */ +#define AARCH64_ADD_IMM_MASK 0x003ffc00 /* bits 21-10 */ +#define AARCH64_MOVZ_IMM_MASK 0x001fffe0 /* bits 20-5 */ +#define AARCH64_MOVZ_HW_MASK 0x00600000 /* bits 22-21 */ +#define AARCH64_MOVK_IMM_MASK 0x001fffe0 /* bits 20-5 */ +#define AARCH64_MOVK_HW_MASK 0x00600000 /* bits 22-21 */ +#define AARCH64_NOP 0xd503201f + +#undef USE_FALLBACK + +#ifdef __MUSL__ + +# define DTV_OFFSET -8 +# define DTV_INDEX_GAP 0 + +typedef struct _dtv_pointer_t { + uintptr_t val; +} dtv_pointer_t; + +typedef struct _tls_descriptor { + size_t index; + size_t offset; +} tls_descriptor; + +#elif defined(__FreeBSD__) + +# define DTV_OFFSET 0 +/* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/22ca6db50f4e6bd75a141f57cf953d8de6531a06/lib/libc/gen/tls.c#L88) */ +# define DTV_INDEX_GAP 1 + +typedef struct _dtv_pointer_t { + uintptr_t val; +} dtv_pointer_t; + +/* https://github.com/freebsd/freebsd-src/blob/c52ca7dd09066648b1cc40f758289404d68ab886/libexec/rtld-elf/aarch64/reloc.c#L180-L184 */ +typedef struct _tls_descriptor { + void* thunk; + int index; + size_t offset; +} tls_descriptor; + +#elif defined(__GLIBC__) + +# define DTV_OFFSET 0 +# define DTV_INDEX_GAP 0 + +typedef struct _dtv_pointer_t { + uintptr_t val; + uintptr_t _; +} dtv_pointer_t; + +typedef struct _tls_descriptor { + size_t index; + size_t offset; +} tls_descriptor; + +#else +# define USE_FALLBACK 1 +#endif + +zend_result zend_jit_resolve_tsrm_ls_cache_offsets( + size_t *tcb_offset, + size_t *module_index, + size_t *module_offset +) { +#ifdef USE_FALLBACK + return FAILURE; +#else + *tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (*tcb_offset != 0) { + return SUCCESS; + } + + void *addr; + uint32_t *insn; + void *thread_pointer; + + __asm__ __volatile__( + /* Load thread pointer address */ + "mrs %0, tpidr_el0\n" + /* Load next instruction address */ + "adr %1, .+4\n\t" + /* General Dynamic code sequence as expected by linkers */ + "adrp x0, :tlsdesc:_tsrm_ls_cache\n" + "ldr x1, [x0, #:tlsdesc_lo12:_tsrm_ls_cache]\n" + "add x0, x0, :tlsdesc_lo12:_tsrm_ls_cache\n" + ".tlsdesccall _tsrm_ls_cache\n" + "blr x1\n" + "mrs x8, tpidr_el0\n" + "add %2, x8, x0\n" + : "=r" (thread_pointer), "=r" (insn), "=r" (addr) + : + : "x0", "x1", "x8"); + + ZEND_ASSERT(addr == &_tsrm_ls_cache); + + /* Check if the general dynamic code was relaxed by the linker */ + + // adrp x0, #any + if ((insn[0] & ~AARCH64_ADRP_IMM_MASK) != 0x90000000) { + zend_accel_error(ACCEL_LOG_DEBUG, "adrp insn does not match: 0x%08" PRIx32 "\n", insn[0]); + goto code_changed; + } + + // ldr x1, [x0, #any] + if ((insn[1] & ~AARCH64_LDR_UNSIGNED_IMM_MASK) != 0xf9400001) { + zend_accel_error(ACCEL_LOG_DEBUG, "ldr insn does not match: 0x%08" PRIx32 "\n", insn[1]); + goto code_changed; + } + + // add x0, x0, any + if ((insn[2] & ~AARCH64_ADD_IMM_MASK) != 0x91000000) { + zend_accel_error(ACCEL_LOG_DEBUG, "add insn does not match: 0x%08" PRIx32 "x\n", insn[2]); + goto code_changed; + } + + /* Code is intact, we can extract immediate values */ + + uint64_t adrp_immhi = (uint64_t)((insn[0] & AARCH64_ADRP_IMMHI_MASK) >> AARCH64_ADRP_IMMHI_START); + uint64_t adrp_immlo = (uint64_t)((insn[0] & AARCH64_ADRP_IMMLO_MASK) >> AARCH64_ADRP_IMMLO_START); + uint64_t adrp_imm = ((adrp_immhi << AARCH64_ADRP_IMMLO_WIDTH) | adrp_immlo) << 12; + uint64_t add_imm = (uint64_t)(insn[2] & AARCH64_ADD_IMM_MASK) >> 10; + uint64_t pc = (uint64_t)insn; + uintptr_t **where = (uintptr_t**)((pc & ~(4096-1)) + adrp_imm + add_imm); + + /* See https://github.com/ARM-software/abi-aa/blob/2a70c42d62e9c3eb5887fa50b71257f20daca6f9/aaelf64/aaelf64.rst + * section "Relocations for thread-local storage". + * The first entry holds a pointer to the variable's TLS descriptor resolver + * function and the second entry holds a platform-specific offset or + * pointer. */ + tls_descriptor *tlsdesc = (tls_descriptor*)(where[1]); + + if ((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer == (uintptr_t)tlsdesc) { + zend_accel_error(ACCEL_LOG_DEBUG, "static tls at offset %p from thread pointer (inferred from tlsdesc)\n", tlsdesc); + *tcb_offset = (uintptr_t)tlsdesc; + return SUCCESS; + } + + *module_index = (tlsdesc->index + DTV_INDEX_GAP) * sizeof(dtv_pointer_t); + *module_offset = tlsdesc->offset; + +# if ZEND_DEBUG + /* We've got the TLS descriptor. Double check: */ + + dtv_pointer_t *dtv = *(dtv_pointer_t**)((uintptr_t)thread_pointer + DTV_OFFSET); + addr = (void*)(((dtv_pointer_t*)((char*)dtv + *module_index))->val + *module_offset); + + ZEND_ASSERT(addr == &_tsrm_ls_cache); +# endif + + zend_accel_error(ACCEL_LOG_DEBUG, "dynamic tls module idx %zu offset %zu (inferred from code)\n", (size_t)tlsdesc->index, tlsdesc->offset); + + return SUCCESS; + +code_changed: + + /* Code was changed by the linker. Check if we recognize the updated code */ + + // movz x0, #0, lsl #16 + if ((insn[0] & ~AARCH64_MOVZ_IMM_MASK) != 0xd2a00000) { + zend_accel_error(ACCEL_LOG_DEBUG, "movz insn does not match: 0x%08" PRIx32 "\n", insn[0]); + return FAILURE; + } + + // movk x0, #0x10 + if ((insn[1] & ~AARCH64_MOVK_IMM_MASK) != 0xf2800000) { + zend_accel_error(ACCEL_LOG_DEBUG, "movk insn does not match: 0x%08" PRIx32 "\n", insn[1]); + return FAILURE; + } + + // nop + for (int i = 0; i < 2; i++) { + if (insn[2+i] != AARCH64_NOP) { + zend_accel_error(ACCEL_LOG_DEBUG, "nop(%d) insn does not match: 0x%08" PRIx32 "\n", i, insn[2+i]); + return FAILURE; + } + } + + /* Extract immediate values */ + + uint64_t movz_imm = (insn[0] & AARCH64_MOVZ_IMM_MASK) >> 5; + uint64_t movz_shift = (((insn[0] & AARCH64_MOVZ_HW_MASK) >> 21) << 4); + uint64_t movk_imm = (insn[1] & AARCH64_MOVK_IMM_MASK) >> 5; + uint64_t offset = (movz_imm << movz_shift) | movk_imm; + + if ((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer == offset) { + zend_accel_error(ACCEL_LOG_DEBUG, "static tls at offset %" PRIxPTR " from thread pointer (inferred from code)\n", offset); + *tcb_offset = offset; + return SUCCESS; + } + + zend_accel_error(ACCEL_LOG_DEBUG, "static tls offset does not match: %" PRIxPTR " (expected %" PRIxPTR ")\n", + offset, ((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer)); + + return FAILURE; +#endif +} + +/* Used for testing */ +void *zend_jit_tsrm_ls_cache_address( + size_t tcb_offset, + size_t module_index, + size_t module_offset +) { + char *thread_pointer; + __asm__ __volatile__( + "mrs %0, tpidr_el0\n" + : "=r" (thread_pointer) + ); + + if (tcb_offset) { + return thread_pointer + tcb_offset; + } + if (module_index != (size_t)-1 && module_offset != (size_t)-1) { + dtv_pointer_t *dtv = *(dtv_pointer_t**)((uintptr_t)thread_pointer + DTV_OFFSET); + return (void*)(((dtv_pointer_t*)((char*)dtv + module_index))->val + module_offset); + } + return NULL; +} diff --git a/ext/opcache/jit/tls/zend_jit_tls_darwin.c b/ext/opcache/jit/tls/zend_jit_tls_darwin.c new file mode 100644 index 0000000000000..47a2f01a5a0ae --- /dev/null +++ b/ext/opcache/jit/tls/zend_jit_tls_darwin.c @@ -0,0 +1,82 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | https://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Dmitry Stogov | + * +----------------------------------------------------------------------+ + */ + +#include "Zend/zend_portability.h" +#include "Zend/zend_types.h" +#include "TSRM/TSRM.h" +#include "zend_accelerator_debug.h" + +#include +#include + +TSRMLS_CACHE_EXTERN(); + +zend_result zend_jit_resolve_tsrm_ls_cache_offsets( + size_t *tcb_offset, + size_t *module_index, + size_t *module_offset +) { + *tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (*tcb_offset != 0) { + return SUCCESS; + } + +#if defined(__x86_64__) + size_t *ti; + __asm__ __volatile__( + "leaq __tsrm_ls_cache(%%rip),%0" + : "=r" (ti)); + *module_offset = ti[2]; + *module_index = ti[1] * 8; + + return SUCCESS; +#endif + + return FAILURE; +} + +/* Used for testing */ +void *zend_jit_tsrm_ls_cache_address( + size_t tcb_offset, + size_t module_index, + size_t module_offset +) { + +#if defined(__x86_64__) + if (tcb_offset) { + char *addr; + __asm__ __volatile__( + "movq %%gs:(%1), %0\n" + : "=r" (addr) + : "r" (tcb_offset) + ); + return addr; + } + if (module_index != (size_t)-1 && module_offset != (size_t)-1) { + char *base; + __asm__ __volatile__( + "movq %%gs:(%1), %0\n" + : "=r" (base) + : "r" (module_index) + ); + return base + module_offset; + } +#endif + + return NULL; +} diff --git a/ext/opcache/jit/tls/zend_jit_tls_win.c b/ext/opcache/jit/tls/zend_jit_tls_win.c new file mode 100644 index 0000000000000..23f0c1e79baaf --- /dev/null +++ b/ext/opcache/jit/tls/zend_jit_tls_win.c @@ -0,0 +1,64 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | https://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Dmitry Stogov | + * +----------------------------------------------------------------------+ + */ + +#include "Zend/zend_portability.h" +#include "Zend/zend_types.h" +#include "TSRM/TSRM.h" +#include "zend_accelerator_debug.h" + +#include +#include + +TSRMLS_CACHE_EXTERN(); + +extern uint32_t _tls_index; +extern char *_tls_start; +extern char *_tls_end; + +zend_result zend_jit_resolve_tsrm_ls_cache_offsets( + size_t *tcb_offset, + size_t *module_index, + size_t *module_offset +) { + /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ + /* Probably, it might be better solution */ +#ifdef _WIN64 + void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index]; +#else + void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index]; +#endif + void *val = _tsrm_ls_cache; + size_t offset = 0; + size_t size = (char*)&_tls_end - (char*)&_tls_start; + + while (offset < size) { + if (*tls_mem == val) { + *module_index = _tls_index * sizeof(void*); + *module_offset = offset; + return SUCCESS; + } + tls_mem++; + offset += sizeof(void*); + } + + if (offset >= size) { + zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size"); + } + + return FAILURE; +} diff --git a/ext/opcache/jit/tls/zend_jit_tls_x86.c b/ext/opcache/jit/tls/zend_jit_tls_x86.c new file mode 100644 index 0000000000000..bca46c8f82664 --- /dev/null +++ b/ext/opcache/jit/tls/zend_jit_tls_x86.c @@ -0,0 +1,239 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | https://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Arnaud Le Blanc | + * +----------------------------------------------------------------------+ + */ + +#include "zend_portability.h" +#include "zend_types.h" +#include "TSRM/TSRM.h" +#include "zend_accelerator_debug.h" +#include "zend_jit_tls.h" + +#include +#include + +TSRMLS_CACHE_EXTERN(); + +#undef USE_FALLBACK + +#ifdef __MUSL__ + +# define DTV_OFFSET 4 +# define DTV_INDEX_GAP 0 + +typedef struct _dtv_pointer_t { + uintptr_t val; +} dtv_pointer_t; + +typedef struct _tls_descriptor { + size_t index; + size_t offset; +} tls_descriptor; + +#elif defined(__FreeBSD__) + +# define DTV_OFFSET 4 +# define DTV_INDEX_GAP 1 + +typedef struct _dtv_pointer_t { + uintptr_t val; +} dtv_pointer_t; + +/* https://github.com/freebsd/freebsd-src/blob/6b94546a7ea2dc593f5765bd5465a8b7bb80c325/libexec/rtld-elf/i386/rtld_machdep.h#L65 */ +typedef struct _tls_descriptor { + unsigned long index; + unsigned long offset; +} tls_descriptor; + +#elif defined(__GLIBC__) + +# define DTV_OFFSET 4 +# define DTV_INDEX_GAP 0 + +typedef struct _dtv_pointer_t { + uintptr_t val; + uintptr_t _; +} dtv_pointer_t; + +typedef struct _tls_descriptor { + size_t index; + size_t offset; +} tls_descriptor; + +#else +# define USE_FALLBACK 1 +#endif + +zend_result zend_jit_resolve_tsrm_ls_cache_offsets( + size_t *tcb_offset, + size_t *module_index, + size_t *module_offset +) { +#ifdef USE_FALLBACK + return FAILURE; +#else + *tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (*tcb_offset != 0) { + return SUCCESS; + } + + void *t_addr; + unsigned char *code; + void *thread_pointer; + + __asm__ __volatile__( + /* Load next instruction address */ + "call 1f\n" + ".subsection 1\n" + "1:\n" + "movl (%%esp), %%ebx\n" + "movl %%ebx, %%esi\n" + "ret\n" + ".previous\n" + /* General Dynamic code sequence as expected by linkers */ + "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n" + "leal _tsrm_ls_cache@TLSGD(,%%ebx,1), %%eax\n" + "call ___tls_get_addr@PLT\n" + /* Load thread pointer address */ + "movl %%gs:0, %%ebx\n" + : "=a" (t_addr), "=S" (code), "=b" (thread_pointer) + ); + + ZEND_ASSERT(t_addr == &_tsrm_ls_cache); + + /* Check if the general dynamic code was relaxed by the linker */ + + // addl any,%ebx + if (memcmp(&code[0], "\x81\xc3", 2) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[0], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "addl insn does not match: 0x%16" PRIx64 "\n", bytes); + goto code_changed; + } + + // leal any(,%ebx,1),%eax + if (memcmp(&code[6], "\x8d\x04\x1d", 3) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[6], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "leal insn does not match: 0x%16" PRIx64 "\n", bytes); + goto code_changed; + } + + // call any + if (memcmp(&code[13], "\xe8", 1) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[13], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "call insn does not match: 0x%16" PRIx64 "\n", bytes); + goto code_changed; + } + + /* Code is intact, we can extract immediate values */ + + uint32_t addl_imm = ((uint32_t)code[5] << 24) + | ((uint32_t)code[4] << 16) + | ((uint32_t)code[3] << 8) + | ((uint32_t)code[2]); + uint32_t leal_imm = ((uint32_t)code[12] << 24) + | ((uint32_t)code[11] << 16) + | ((uint32_t)code[10] << 8) + | ((uint32_t)code[9]); + + tls_descriptor *tlsdesc = (tls_descriptor*)(leal_imm + addl_imm + (uintptr_t)code); + + *module_index = (tlsdesc->index + DTV_INDEX_GAP) * sizeof(dtv_pointer_t); + *module_offset = tlsdesc->offset; + +# if ZEND_DEBUG + /* We've got the TLS descriptor. Double check: */ + + dtv_pointer_t *dtv = *(dtv_pointer_t**)((uintptr_t)thread_pointer + DTV_OFFSET); + void *addr = (void*)(((dtv_pointer_t*)((char*)dtv + *module_index))->val + *module_offset); + + ZEND_ASSERT(addr == &_tsrm_ls_cache); +# endif + + zend_accel_error(ACCEL_LOG_DEBUG, "dynamic tls module idx %zu offset %zu (inferred from code)\n", + (size_t)tlsdesc->index, (size_t)tlsdesc->offset); + + return SUCCESS; + +code_changed: + + /* Code was changed by the linker. Check if we recognize the updated code */ + + /* + * 81 c3 98 2d 00 00 addl $0x2d98,%ebx + * 65 a1 00 00 00 00 movl %gs:0x0,%eax + * 81 e8 04 00 00 00 subl $0x4,%eax + */ + + // movl %gs:0x0,%eax + if (memcmp(&code[6], "\x65\xa1\x00\x00\x00\x00", 6) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[6], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "movl insn does not match: 0x%16" PRIx64 "\n", bytes); + return FAILURE; + } + + // subl $any,%eax + if (memcmp(&code[12], "\x81\xe8", 2) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[6], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "subl insn does not match: 0x%16" PRIx64 "\n", bytes); + return FAILURE; + } + + /* Extract immediate values */ + + uint32_t offset = -(((uint32_t)code[17] << 24) + | ((uint32_t)code[16] << 16) + | ((uint32_t)code[15] << 8) + | ((uint32_t)code[14])); + + if ((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer == offset) { + zend_accel_error(ACCEL_LOG_DEBUG, "static tls at offset %" PRIx32 " from thread pointer (inferred from code)\n", offset); + *tcb_offset = offset; + return SUCCESS; + } + + zend_accel_error(ACCEL_LOG_DEBUG, "static tls offset does not match: 0x%" PRIx32 " (expected 0x%" PRIx32 ")\n", + offset, (uint32_t)((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer)); + + return FAILURE; +#endif +} + +/* Used for testing */ +void *zend_jit_tsrm_ls_cache_address( + size_t tcb_offset, + size_t module_index, + size_t module_offset +) { + char *thread_pointer; + __asm__ __volatile__( + "movl %%gs:0, %0\n" + : "=r" (thread_pointer) + ); + + if (tcb_offset) { + return thread_pointer + tcb_offset; + } + if (module_index != (size_t)-1 && module_offset != (size_t)-1) { + dtv_pointer_t *dtv = *(dtv_pointer_t**)((uintptr_t)thread_pointer + DTV_OFFSET); + return (void*)(((dtv_pointer_t*)((char*)dtv + module_index))->val + module_offset); + } + return NULL; +} diff --git a/ext/opcache/jit/tls/zend_jit_tls_x86_64.c b/ext/opcache/jit/tls/zend_jit_tls_x86_64.c new file mode 100644 index 0000000000000..3adeb1ff8064c --- /dev/null +++ b/ext/opcache/jit/tls/zend_jit_tls_x86_64.c @@ -0,0 +1,222 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | https://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Arnaud Le Blanc | + * +----------------------------------------------------------------------+ + */ + +#include "zend_portability.h" +#include "zend_types.h" +#include "TSRM/TSRM.h" +#include "zend_accelerator_debug.h" +#include "zend_jit_tls.h" + +#include +#include + +TSRMLS_CACHE_EXTERN(); + +#undef USE_FALLBACK + +#ifdef __MUSL__ + +# define DTV_OFFSET 8 +# define DTV_INDEX_GAP 0 + +typedef struct _dtv_pointer_t { + uintptr_t val; +} dtv_pointer_t; + +typedef struct _tls_descriptor { + size_t index; + size_t offset; +} tls_descriptor; + +#elif defined(__FreeBSD__) + +# define DTV_OFFSET 8 +# define DTV_INDEX_GAP 1 + +typedef struct _dtv_pointer_t { + uintptr_t val; +} dtv_pointer_t; + +/* https://github.com/freebsd/freebsd-src/blob/6b94546a7ea2dc593f5765bd5465a8b7bb80c325/libexec/rtld-elf/amd64/rtld_machdep.h#L65 */ +typedef struct _tls_descriptor { + unsigned long index; + unsigned long offset; +} tls_descriptor; + +#elif defined(__GLIBC__) + +# define DTV_OFFSET 8 +# define DTV_INDEX_GAP 0 + +typedef struct _dtv_pointer_t { + uintptr_t val; + uintptr_t _; +} dtv_pointer_t; + +typedef struct _tls_descriptor { + size_t index; + size_t offset; +} tls_descriptor; + +#else +# define USE_FALLBACK 1 +#endif + +zend_result zend_jit_resolve_tsrm_ls_cache_offsets( + size_t *tcb_offset, + size_t *module_index, + size_t *module_offset +) { +#ifdef USE_FALLBACK + return FAILURE; +#else + *tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (*tcb_offset != 0) { + return SUCCESS; + } + + void *addr; + unsigned char *code; + void *thread_pointer; + + __asm__ __volatile__( + /* Load next instruction address */ + "leaq (%%rip), %%rbx\n" + /* General Dynamic code sequence as expected by linkers */ + ".byte 0x66\n" + "leaq _tsrm_ls_cache@tlsgd(%%rip), %%rdi\n" + ".word 0x6666\n" + "rex64\n" + "call __tls_get_addr\n" + /* Load thread pointer address */ + "movq %%fs:0, %%rsi\n" + : "=a" (addr), "=b" (code), "=S" (thread_pointer) + ); + + ZEND_ASSERT(addr == &_tsrm_ls_cache); + + /* Check if the general dynamic code was relaxed by the linker */ + + // data16 leaq any(%rip),%rdi + if (memcmp(&code[0], "\x66\x48\x8d\x3d", 4) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[0], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "leaq insn does not match: 0x%016" PRIx64 "\n", bytes); + goto code_changed; + } + + // data16 data16 rex.W call any + if (memcmp(&code[8], "\x66\x66\x48\xe8", 4) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[8], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "call insn does not match: 0x%016" PRIx64 "\n", bytes); + goto code_changed; + } + + /* Code is intact, we can extract immediate values */ + + uintptr_t leaq_imm = (uintptr_t)(int32_t)((uint32_t)code[7] << 24) + | ((uint32_t)code[6] << 16) + | ((uint32_t)code[5] << 8) + | ((uint32_t)code[4]); + + tls_descriptor *tlsdesc = (tls_descriptor*)(leaq_imm + (uintptr_t)code + 8 /* leaq */); + + *module_index = ((size_t)tlsdesc->index + DTV_INDEX_GAP) * sizeof(dtv_pointer_t); + *module_offset = (size_t)tlsdesc->offset; + +# if ZEND_DEBUG + /* We've got the TLS descriptor. Double check: */ + + dtv_pointer_t *dtv = *(dtv_pointer_t**)((uintptr_t)thread_pointer + DTV_OFFSET); + addr = (void*)(((dtv_pointer_t*)((char*)dtv + *module_index))->val + *module_offset); + + ZEND_ASSERT(addr == &_tsrm_ls_cache); +# endif + + zend_accel_error(ACCEL_LOG_DEBUG, "dynamic tls module idx %zu offset %zu (inferred from code)\n", + (size_t)tlsdesc->index, (size_t)tlsdesc->offset); + + return SUCCESS; + +code_changed: + + /* Code was changed by the linker. Check if we recognize the updated code */ + + /* + * 64 48 8b 04 25 00 00 00 00 movq %fs:0x0,%rax + * 48 8d 80 f8 ff ff ff leaq -0x8(%rax),%rax + */ + + // movq %fs:0x0,%rax + if (memcmp(&code[0], "\x64\x48\x8b\x04\x25\x00\x00\x00\x00", 9) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[0], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "movq insn does not match: 0x%016" PRIx64 "\n", bytes); + return FAILURE; + } + + // leaq any(%rax),$rax + if (memcmp(&code[9], "\x48\x8d\x80", 3) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[10], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "leaq insn does not match: 0x%016" PRIx64 "\n", bytes); + return FAILURE; + } + + /* Extract immediate values */ + + uintptr_t offset = (uintptr_t)(int32_t)(((uint32_t)code[15] << 24) + | ((uint32_t)code[14] << 16) + | ((uint32_t)code[13] << 8) + | ((uint32_t)code[12])); + + if ((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer == offset) { + *tcb_offset = offset; + zend_accel_error(ACCEL_LOG_DEBUG, "static tls at offset %" PRIxPTR " from thread pointer (inferred from code)\n", offset); + return SUCCESS; + } + + zend_accel_error(ACCEL_LOG_DEBUG, "static tls offset does not match: %" PRIxPTR " (expected %" PRIxPTR ")\n", + offset, ((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer)); + + return FAILURE; +#endif +} + +/* Used for testing */ +void *zend_jit_tsrm_ls_cache_address( + size_t tcb_offset, + size_t module_index, + size_t module_offset +) { + char *thread_pointer; + __asm__ __volatile__( + "movq %%fs:0, %0\n" + : "=r" (thread_pointer) + ); + + if (tcb_offset) { + return thread_pointer + tcb_offset; + } + if (module_index != (size_t)-1 && module_offset != (size_t)-1) { + dtv_pointer_t *dtv = *(dtv_pointer_t**)((uintptr_t)thread_pointer + DTV_OFFSET); + return (void*)(((dtv_pointer_t*)((char*)dtv + module_index))->val + module_offset); + } + return NULL; +} diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index c65fe5e3fbf7f..2a2c6c79f17f9 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -18,6 +18,7 @@ #include "jit/ir/ir.h" #include "jit/ir/ir_builder.h" +#include "jit/tls/zend_jit_tls.h" #if defined(IR_TARGET_X86) # define IR_REG_SP 4 /* IR_REG_RSP */ @@ -171,15 +172,9 @@ static uint32_t default_mflags = 0; static bool delayed_call_chain = 0; // TODO: remove this var (use jit->delayed_call_level) ??? #ifdef ZTS -# ifdef _WIN32 -extern uint32_t _tls_index; -extern char *_tls_start; -extern char *_tls_end; -# endif - static size_t tsrm_ls_cache_tcb_offset = 0; -static size_t tsrm_tls_index = 0; -static size_t tsrm_tls_offset = 0; +static size_t tsrm_tls_index = -1; +static size_t tsrm_tls_offset = -1; # define EG_TLS_OFFSET(field) \ (executor_globals_offset + offsetof(zend_executor_globals, field)) @@ -334,6 +329,8 @@ static int zend_jit_assign_to_variable(zend_jit_ctx *jit, zend_jit_addr ref_addr, bool check_exception); +static ir_ref jit_CONST_FUNC(zend_jit_ctx *jit, uintptr_t addr, uint16_t flags); + typedef struct _zend_jit_stub { const char *name; int (*stub)(zend_jit_ctx *jit); @@ -463,6 +460,11 @@ static const char* zend_reg_name(int8_t reg) /* IR helpers */ #ifdef ZTS +static void * ZEND_FASTCALL zend_jit_get_tsrm_ls_cache(void) +{ + return _tsrm_ls_cache; +} + static ir_ref jit_TLS(zend_jit_ctx *jit) { ZEND_ASSERT(jit->ctx.control); @@ -482,9 +484,15 @@ static ir_ref jit_TLS(zend_jit_ctx *jit) ref = insn->op1; } } - jit->tls = ir_TLS( - tsrm_ls_cache_tcb_offset ? tsrm_ls_cache_tcb_offset : tsrm_tls_index, - tsrm_ls_cache_tcb_offset ? IR_NULL : tsrm_tls_offset); + + if (tsrm_ls_cache_tcb_offset == 0 && tsrm_tls_index == -1) { + jit->tls = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_get_tsrm_ls_cache)); + } else { + jit->tls = ir_TLS( + tsrm_ls_cache_tcb_offset ? tsrm_ls_cache_tcb_offset : tsrm_tls_index, + tsrm_ls_cache_tcb_offset ? IR_NULL : tsrm_tls_offset); + } + return jit->tls; } #endif @@ -3125,6 +3133,8 @@ static void zend_jit_setup_disasm(void) REGISTER_DATA(EG(symbol_table)); REGISTER_DATA(CG(map_ptr_base)); +#else /* ZTS */ + REGISTER_HELPER(zend_jit_get_tsrm_ls_cache); #endif #endif } @@ -3291,7 +3301,6 @@ static void zend_jit_setup_unwinder(void) } #endif - static void zend_jit_setup(bool reattached) { #if defined(IR_TARGET_X86) @@ -3310,166 +3319,17 @@ static void zend_jit_setup(bool reattached) } # endif #endif -#ifdef ZTS -#if defined(IR_TARGET_AARCH64) - tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); - -# ifdef __FreeBSD__ - if (tsrm_ls_cache_tcb_offset == 0) { - TLSDescriptor **where; - - __asm__( - "adrp %0, :tlsdesc:_tsrm_ls_cache\n" - "add %0, %0, :tlsdesc_lo12:_tsrm_ls_cache\n" - : "=r" (where)); - /* See https://github.com/ARM-software/abi-aa/blob/2a70c42d62e9c3eb5887fa50b71257f20daca6f9/aaelf64/aaelf64.rst - * section "Relocations for thread-local storage". - * The first entry holds a pointer to the variable's TLS descriptor resolver function and the second entry holds - * a platform-specific offset or pointer. */ - TLSDescriptor *tlsdesc = where[1]; - - tsrm_tls_offset = tlsdesc->offset; - /* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/22ca6db50f4e6bd75a141f57cf953d8de6531a06/lib/libc/gen/tls.c#L88) */ - tsrm_tls_index = (tlsdesc->index + 1) * 8; - } -# elif defined(__MUSL__) - if (tsrm_ls_cache_tcb_offset == 0) { - size_t **where; - - __asm__( - "adrp %0, :tlsdesc:_tsrm_ls_cache\n" - "add %0, %0, :tlsdesc_lo12:_tsrm_ls_cache\n" - : "=r" (where)); - /* See https://github.com/ARM-software/abi-aa/blob/2a70c42d62e9c3eb5887fa50b71257f20daca6f9/aaelf64/aaelf64.rst */ - size_t *tlsdesc = where[1]; - - tsrm_tls_offset = tlsdesc[1]; - tsrm_tls_index = tlsdesc[0] * 8; - } -# else - ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0); -# endif -# elif defined(_WIN64) - tsrm_tls_index = _tls_index * sizeof(void*); - - /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ - /* Probably, it might be better solution */ - do { - void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index]; - void *val = _tsrm_ls_cache; - size_t offset = 0; - size_t size = (char*)&_tls_end - (char*)&_tls_start; - - while (offset < size) { - if (*tls_mem == val) { - tsrm_tls_offset = offset; - break; - } - tls_mem++; - offset += sizeof(void*); - } - if (offset >= size) { - zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size"); - } - } while(0); -# elif defined(ZEND_WIN32) - tsrm_tls_index = _tls_index * sizeof(void*); - - /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ - /* Probably, it might be better solution */ - do { - void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index]; - void *val = _tsrm_ls_cache; - size_t offset = 0; - size_t size = (char*)&_tls_end - (char*)&_tls_start; - - while (offset < size) { - if (*tls_mem == val) { - tsrm_tls_offset = offset; - break; - } - tls_mem++; - offset += sizeof(void*); - } - if (offset >= size) { - zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size"); - } - } while(0); -# elif defined(__APPLE__) && defined(__x86_64__) - tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); - if (tsrm_ls_cache_tcb_offset == 0) { - size_t *ti; - __asm__( - "leaq __tsrm_ls_cache(%%rip),%0" - : "=r" (ti)); - tsrm_tls_offset = ti[2]; - tsrm_tls_index = ti[1] * 8; - } -# elif defined(__GNUC__) && defined(__x86_64__) - tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); - if (tsrm_ls_cache_tcb_offset == 0) { -#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && \ - !defined(__NetBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__) - size_t ret; - - asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0" - : "=r" (ret)); - tsrm_ls_cache_tcb_offset = ret; -#elif defined(__MUSL__) - size_t *ti; - - __asm__( - "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" - : "=D" (ti)); - tsrm_tls_offset = ti[1]; - tsrm_tls_index = ti[0] * 8; -#elif defined(__FreeBSD__) - size_t *ti; - - __asm__( - "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" - : "=D" (ti)); - tsrm_tls_offset = ti[1]; - /* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/bf56e8b9c8639ac4447d223b83cdc128107cc3cd/libexec/rtld-elf/rtld.c#L5260) */ - tsrm_tls_index = (ti[0] + 1) * 8; -#else - size_t *ti; - - __asm__( - "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" - : "=D" (ti)); - tsrm_tls_offset = ti[1]; - tsrm_tls_index = ti[0] * 16; -#endif - } -# elif defined(__GNUC__) && defined(__i386__) - tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); - if (tsrm_ls_cache_tcb_offset == 0) { -#if !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__) - size_t ret; - asm ("leal _tsrm_ls_cache@ntpoff,%0\n" - : "=a" (ret)); - tsrm_ls_cache_tcb_offset = ret; -#else - size_t *ti, _ebx, _ecx, _edx; - - __asm__( - "call 1f\n" - ".subsection 1\n" - "1:\tmovl (%%esp), %%ebx\n\t" - "ret\n" - ".previous\n\t" - "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" - "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t" - "call ___tls_get_addr@plt\n\t" - "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n" - : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx)); - tsrm_tls_offset = ti[1]; - tsrm_tls_index = ti[0] * 8; -#endif +#ifdef ZTS + zend_result result = zend_jit_resolve_tsrm_ls_cache_offsets( + &tsrm_ls_cache_tcb_offset, + &tsrm_tls_index, + &tsrm_tls_offset + ); + if (result == FAILURE) { + zend_accel_error(ACCEL_LOG_INFO, + "Could not get _tsrm_ls_cache offsets, will fallback to runtime resolution"); } -# endif #endif #if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64) diff --git a/ext/opcache/tests/zzz_basic_logging.phpt b/ext/opcache/tests/zzz_basic_logging.phpt index a53c6a8db9de1..d374daa67ba81 100644 --- a/ext/opcache/tests/zzz_basic_logging.phpt +++ b/ext/opcache/tests/zzz_basic_logging.phpt @@ -19,6 +19,7 @@ opcache --FILE-- debug ? 'Yes' : 'No') . "\n"; echo "=====================================================================\n"; - echo "No tests in this branch yet.\n"; + try { + output_group_start($environment, 'Running Opcache TLS tests'); + if (!run_opcache_tls_tests($environment)) { + echo "Failed\n"; + exit(1); + } + } finally { + output_group_end($environment); + } echo "All OK\n"; } +function run_opcache_tls_tests(Environment $environment): bool +{ + if (!$environment->zts) { + echo "Skipping: NTS\n"; + return true; + } + + if (!$environment->debug) { + echo "Skipping: NDEBUG\n"; + return true; + } + + $tlsc = ''; + $machine = ''; + $static_support = 'yes'; + + switch ($environment->cpuArch) { + case 'x86': + $machine = '-m32'; + break; + case 'x86_64': + case 'aarch64': + break; + default: + echo "Skipping: {$environment->cpuArch}\n"; + return true; + } + + switch ($environment->os) { + case 'Linux': + case 'FreeBSD': + $tlsc = __DIR__ . "/ext/opcache/jit/tls/zend_jit_tls_{$environment->cpuArch}.c"; + break; + case 'Darwin': + if ($environment->cpuArch === 'aarch64') { + echo "Skipping: JIT+TLS not supported on MacOS Apple Silicon\n"; + return true; + } + $tlsc = __DIR__ . "/ext/opcache/jit/tls/zend_jit_tls_darwin.c"; + $static_support = 'no'; + break; + default: + echo "Skipping: {$environment->os}\n"; + return true; + } + + echo "TLSC=$tlsc MACHINE=$machine STATIC_SUPPORT=$static_support ext/opcache/jit/tls/testing/test.sh\n"; + + $proc = proc_open( + __DIR__ . '/ext/opcache/jit/tls/testing/test.sh', + [ + 0 => ['pipe', 'r'], + ], + $pipes, + env_vars: array_merge(getenv(), [ + 'TLSC' => $tlsc, + 'MACHINE' => $machine, + 'STATIC_SUPPORT' => $static_support, + ]), + ); + + if (!$proc) { + echo "proc_open() failed\n"; + return false; + } + + fclose($pipes[0]); + + return proc_close($proc) === 0; +} + function output_group_start(Environment $environment, string $name): void { if ($environment->githubAction) {