Avoid hash value recalculation on each counter update

Author: dstogov

ext/opcache/jit/zend_jit.c

@@ -2693,18 +2693,19 @@ void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) {
 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
 {
 	zend_op_array *op_array = &EX(func)->op_array;
-	const void **orig_handlers;
+	zend_jit_op_array_extension *jit_extension;
 	uint32_t i;
 
 	zend_shared_alloc_lock();
-	orig_handlers = (const void**)ZEND_FUNC_INFO(op_array);
+	jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
 
-	if (orig_handlers) {
+	if (jit_extension) {
 		SHM_UNPROTECT();
 		zend_jit_unprotect();
 
+		*(jit_extension->counter) = ZEND_JIT_HOT_COUNTER_INIT;
 		for (i = 0; i < op_array->last; i++) {
-			op_array->opcodes[i].handler = orig_handlers[i];
+			op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
 		}
 		ZEND_SET_FUNC_INFO(op_array, NULL);
 
@@ -2723,19 +2724,20 @@ void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend
 static int zend_jit_setup_hot_counters(zend_op_array *op_array)
 {
 	zend_op *opline = op_array->opcodes;
-	const void **orig_handlers;
+	zend_jit_op_array_extension *jit_extension;
 	zend_cfg cfg;
 	uint32_t i;
 
 	if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
 		return FAILURE;
 	}
 
-	orig_handlers = (const void**)zend_shared_alloc(op_array->last * sizeof(void*));
+	jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension) + (op_array->last - 1) * sizeof(void*));
+	jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
 	for (i = 0; i < op_array->last; i++) {
-		orig_handlers[i] = op_array->opcodes[i].handler;
+		jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
 	}
-	ZEND_SET_FUNC_INFO(op_array, (void*)orig_handlers);
+	ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
 
 	while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
 		opline++;

ext/opcache/jit/zend_jit_internal.h

@@ -28,12 +28,35 @@ extern int zend_jit_profile_counter_rid;
 	ZEND_OP_ARRAY_EXTENSION(op_array, zend_jit_profile_counter_rid)
 
 /* Hot Counters */
+
 #define ZEND_HOT_COUNTERS_COUNT 128
 
 extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
 
 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline);
 
+typedef struct _zend_jit_op_array_extension {
+	int16_t    *counter;
+	const void *orig_handlers[1];
+} zend_jit_op_array_extension;
+
+static zend_always_inline zend_long zend_jit_op_array_hash(const zend_op_array *op_array)
+{
+	uintptr_t x;
+
+	x = (uintptr_t)op_array->opcodes >> 3;
+#if SIZEOF_SIZE_T == 4
+	x = ((x >> 16) ^ x) * 0x45d9f3b;
+	x = ((x >> 16) ^ x) * 0x45d9f3b;
+	x = (x >> 16) ^ x;
+#elif SIZEOF_SIZE_T == 8
+	x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9;
+	x = (x ^ (x >> 27)) * 0x94d049bb133111eb;
+	x = x ^ (x >> 31);
+#endif
+	return x;
+}
+
 extern const zend_op *zend_jit_halt_op;
 
 #ifdef HAVE_GCC_GLOBAL_REGS

ext/opcache/jit/zend_jit_vm_helpers.c

@@ -182,67 +182,40 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLE
 	ZEND_OPCODE_TAIL_CALL(handler);
 }
 
-static zend_always_inline zend_long _op_array_hash(const zend_op_array *op_array)
-{
-	uintptr_t x;
-
-	if (op_array->function_name) {
-		x = (uintptr_t)op_array >> 3;
-	} else {
-		x = (uintptr_t)op_array->filename >> 3;
-	}
-#if SIZEOF_SIZE_T == 4
-	x = ((x >> 16) ^ x) * 0x45d9f3b;
-	x = ((x >> 16) ^ x) * 0x45d9f3b;
-	x = (x >> 16) ^ x;
-#elif SIZEOF_SIZE_T == 8
-	x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9;
-	x = (x ^ (x >> 27)) * 0x94d049bb133111eb;
-	x = x ^ (x >> 31);
-#endif
-	return x;
-}
-
 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
 {
+	zend_jit_op_array_extension *jit_extension =
+		(zend_jit_op_array_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
 #ifndef HAVE_GCC_GLOBAL_REGS
 	const zend_op *opline = EX(opline);
 #endif
-	unsigned int n = _op_array_hash(&EX(func)->op_array)  %
-		(sizeof(zend_jit_hot_counters) / sizeof(zend_jit_hot_counters[0]));
 
-	zend_jit_hot_counters[n] -= ZEND_JIT_HOT_FUNC_COST;
+	*(jit_extension->counter) -= ZEND_JIT_HOT_FUNC_COST;
 
-	if (UNEXPECTED(zend_jit_hot_counters[n] <= 0)) {
-		zend_jit_hot_counters[n] = ZEND_JIT_HOT_COUNTER_INIT;
+	if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
 		zend_jit_hot_func(execute_data, opline);
 		ZEND_OPCODE_RETURN();
 	} else {
-		zend_vm_opcode_handler_t *handlers =
-			(zend_vm_opcode_handler_t*)ZEND_FUNC_INFO(&EX(func)->op_array);
-		zend_vm_opcode_handler_t handler = handlers[opline - EX(func)->op_array.opcodes];
+		zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
 		ZEND_OPCODE_TAIL_CALL(handler);
 	}
 }
 
 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS)
 {
+	zend_jit_op_array_extension *jit_extension =
+		(zend_jit_op_array_extension*)ZEND_FUNC_INFO(&EX(func)->op_array);
 #ifndef HAVE_GCC_GLOBAL_REGS
 	const zend_op *opline = EX(opline);
 #endif
-	unsigned int n = _op_array_hash(&EX(func)->op_array)  %
-		(sizeof(zend_jit_hot_counters) / sizeof(zend_jit_hot_counters[0]));
 
-	zend_jit_hot_counters[n] -= ZEND_JIT_HOT_LOOP_COST;
+	*(jit_extension->counter) -= ZEND_JIT_HOT_LOOP_COST;
 
-	if (UNEXPECTED(zend_jit_hot_counters[n] <= 0)) {
-		zend_jit_hot_counters[n] = ZEND_JIT_HOT_COUNTER_INIT;
+	if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
 		zend_jit_hot_func(execute_data, opline);
 		ZEND_OPCODE_RETURN();
 	} else {
-		zend_vm_opcode_handler_t *handlers =
-			(zend_vm_opcode_handler_t*)ZEND_FUNC_INFO(&EX(func)->op_array);
-		zend_vm_opcode_handler_t handler = handlers[opline - EX(func)->op_array.opcodes];
+		zend_vm_opcode_handler_t handler = (zend_vm_opcode_handler_t)jit_extension->orig_handlers[opline - EX(func)->op_array.opcodes];
 		ZEND_OPCODE_TAIL_CALL(handler);
 	}
 }

ext/opcache/jit/zend_jit_x86.dasc

@@ -2176,68 +2176,27 @@ static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst)
 {
 	|->hybrid_func_counter:
 	|	mov r0, EX->func
-	|	mov r1, aword [r0 + offsetof(zend_op_array, function_name)]
-	|	test r1, r1
-	|	jne	>1
-	|	mov r0, aword [r0 + offsetof(zend_op_array, filename)]
-	|1:
-	|	shr r0, 3
-	|	mov r1, r0
-	|	.if X64
-	|		shr r0, 30
-	|		xor r0, r1
-	|		mov64 r1, 0xbf58476d1ce4e5b9
-	|		imul r0, r1
-	|		mov r1, r0
-	|		shr r0, 27
-	|		xor r0, r1
-	|		mov64 r1, 0x94d049bb133111eb
-	|		imul r0, r1
-	|		mov r1, r0
-	|		shr r0, 31
-	|		xor r1, r0
-	|		and r1, ZEND_HOT_COUNTERS_COUNT-1
-	|		LOAD_ADDR r0, &zend_jit_hot_counters
-	|		sub word [r0+r1*2], ZEND_JIT_HOT_FUNC_COST
-	|	.else
-	|		shr r0, 16
-	|		xor r0, r1
-	|		imul r0, 0x45d9f3b
-	|		mov r1, r0
-	|		shr r0, 16
-	|		xor r0, r1
-	|		imul r0, 0x45d9f3b
-	|		mov r1, r0
-	|		shr r0, 16
-	|		xor r1, r0
-	|		and r1, ZEND_HOT_COUNTERS_COUNT-1
-	|		sub word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_FUNC_COST
-	|	.endif
+	|	mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
+	|	mov r2, aword [r1]
+	|	sub word [r2], ZEND_JIT_HOT_FUNC_COST
 	|	jle >1
-	|	mov r1, EX->func
-	|	GET_IP r0
-	|	sub r0, aword [r1 + offsetof(zend_op_array, opcodes)]
+	|	GET_IP r2
+	|	sub r2, aword [r0 + offsetof(zend_op_array, opcodes)]
 	|	// divide by sizeof(zend_op)
 	|	.if X64
 	||		ZEND_ASSERT(sizeof(zend_op) == 32);
-	|		sar r0, 5
+	|		sar r2, 5
 	|	.else
 	||		ZEND_ASSERT(sizeof(zend_op) == 28);
-	|		sar r0, 2
-	|		imul r0, 0xb6db6db7
+	|		sar r2, 2
+	|		imul r2, 0xb6db6db7
 	|	.endif
-	|	mov r1, aword [r1 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
 	|	.if X64
-	|		jmp aword [r1+r0*8]
+	|		jmp aword [r1+r2*8+8]
 	|	.else
-	|		jmp aword [r1+r0*4]
+	|		jmp aword [r1+r2*4+4]
 	|	.endif
 	|1:
-	|	.if X64
-	|		mov word [r0+r1*2], ZEND_JIT_HOT_COUNTER_INIT
-	|	.else
-	|		mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT
-	|	.endif
 	|	mov FCARG1a, FP
 	|	GET_IP FCARG2a
 	|	EXT_CALL zend_jit_hot_func, r0
@@ -2249,68 +2208,27 @@ static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst)
 {
 	|->hybrid_loop_counter:
 	|	mov r0, EX->func
-	|	mov r1, aword [r0 + offsetof(zend_op_array, function_name)]
-	|	test r1, r1
-	|	jne	>1
-	|	mov r0, aword [r0 + offsetof(zend_op_array, filename)]
-	|1:
-	|	shr r0, 3
-	|	mov r1, r0
-	|	.if X64
-	|		shr r0, 30
-	|		xor r0, r1
-	|		mov64 r1, 0xbf58476d1ce4e5b9
-	|		imul r0, r1
-	|		mov r1, r0
-	|		shr r0, 27
-	|		xor r0, r1
-	|		mov64 r1, 0x94d049bb133111eb
-	|		imul r0, r1
-	|		mov r1, r0
-	|		shr r0, 31
-	|		xor r1, r0
-	|		and r1, ZEND_HOT_COUNTERS_COUNT-1
-	|		LOAD_ADDR r0, &zend_jit_hot_counters
-	|		sub word [r0+r1*2], ZEND_JIT_HOT_LOOP_COST
-	|	.else
-	|		shr r0, 16
-	|		xor r0, r1
-	|		imul r0, 0x45d9f3b
-	|		mov r1, r0
-	|		shr r0, 16
-	|		xor r0, r1
-	|		imul r0, 0x45d9f3b
-	|		mov r1, r0
-	|		shr r0, 16
-	|		xor r1, r0
-	|		and r1, ZEND_HOT_COUNTERS_COUNT-1
-	|		sub word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_LOOP_COST
-	|	.endif
+	|	mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
+	|	mov r2, aword [r1]
+	|	sub word [r2], ZEND_JIT_HOT_LOOP_COST
 	|	jle >1
-	|	mov r1, EX->func
-	|	GET_IP r0
-	|	sub r0, aword [r1 + offsetof(zend_op_array, opcodes)]
+	|	GET_IP r2
+	|	sub r2, aword [r0 + offsetof(zend_op_array, opcodes)]
 	|	// divide by sizeof(zend_op)
 	|	.if X64
 	||		ZEND_ASSERT(sizeof(zend_op) == 32);
-	|		sar r0, 5
+	|		sar r2, 5
 	|	.else
 	||		ZEND_ASSERT(sizeof(zend_op) == 28);
-	|		sar r0, 2
-	|		imul r0, 0xb6db6db7
+	|		sar r2, 2
+	|		imul r2, 0xb6db6db7
 	|	.endif
-	|	mov r1, aword [r1 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
 	|	.if X64
-	|		jmp aword [r1+r0*8]
+	|		jmp aword [r1+r2*8+8]
 	|	.else
-	|		jmp aword [r1+r0*4]
+	|		jmp aword [r1+r2*4+4]
 	|	.endif
 	|1:
-	|	.if X64
-	|		mov word [r0+r1*2], ZEND_JIT_HOT_COUNTER_INIT
-	|	.else
-	|		mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT
-	|	.endif
 	|	mov FCARG1a, FP
 	|	GET_IP FCARG2a
 	|	EXT_CALL zend_jit_hot_func, r0