Optimize observers (#13649)

Inline the lookup whether a function is observed at all.
This strategy is also used for FRAMELESS calls. If the frameless call is observed, we instead allocate a call frame and push the arguments, to call the the function afterwards.
Doing so is still a performance benefit as opposed to executing individual INIT_FCALL+SEND_VAL ops. Thus, even if the frameless call turns out to be observed, the call overhead is slightly lower than before.
If the internal function is not observed at all, the unavoidable overhead is fetching the FLF zend_function pointer and the run-time cache needs to be inspected.

As part of this work, it turned out to be most viable to put the result operand on the ZEND_OP_DATA instead of ZEND_FRAMELESS_ICALL_3, allowing seamless interoperability with the DO_ICALL opcode.
This is a bit unusual in comparison to all other ZEND_OP_DATA usages, but seems to not pose problems overall.

There is also a small issue resolved: trampolines would always use the ZEND_CALL_TRAMPOLINE_SPEC_OBSERVER function due to zend_observer_fcall_op_array_extension being set to -1 too late.

Author: bwoebi

Zend/zend_builtin_functions.c

@@ -1776,7 +1776,7 @@ ZEND_FUNCTION(debug_print_backtrace)
 
 ZEND_API void zend_fetch_debug_backtrace(zval *return_value, int skip_last, int options, int limit) /* {{{ */
 {
-	zend_execute_data *call;
+	zend_execute_data *call, *last_call = NULL;
 	zend_object *object;
 	bool fake_frame = 0;
 	int lineno, frameno = 0;
@@ -1821,6 +1821,7 @@ ZEND_API void zend_fetch_debug_backtrace(zval *return_value, int skip_last, int
 
 	if (skip_last) {
 		/* skip debug_backtrace() */
+		last_call = call;
 		call = call->prev_execute_data;
 	}
 
@@ -1857,9 +1858,15 @@ ZEND_API void zend_fetch_debug_backtrace(zval *return_value, int skip_last, int
 				zval *arg = zend_get_zval_ptr(op_data, op_data->op1_type, &op_data->op1, call);
 				if (Z_TYPE_P(arg) == IS_UNDEF) goto not_frameless_call;
 			}
+			zend_function *func = ZEND_FLF_FUNC(opline);
+			/* Assume frameless functions are not recursive with themselves.
+			 * This condition may be true when observers are enabled:
+			 * Observers will put a call frame on top of the frameless opcode. */
+			if (last_call && last_call->func == func) {
+				goto not_frameless_call;
+			}
 			stack_frame = zend_new_array(8);
 			zend_hash_real_init_mixed(stack_frame);
-			zend_function *func = ZEND_FLF_FUNC(opline);
 			zend_string *name = func->common.function_name;
 			ZVAL_STRINGL(&tmp, ZSTR_VAL(name), ZSTR_LEN(name));
 			_zend_hash_append_ex(stack_frame, ZSTR_KNOWN(ZEND_STR_FUNCTION), &tmp, 1);
@@ -2068,6 +2075,7 @@ ZEND_API void zend_fetch_debug_backtrace(zval *return_value, int skip_last, int
 		} else {
 			fake_frame = 0;
 			include_filename = filename;
+			last_call = call;
 			call = prev;
 		}
 	}

Zend/zend_compile.c

@@ -4575,7 +4575,7 @@ static uint32_t find_frameless_function_offset(uint32_t arity, void *handler)
 
 static const zend_frameless_function_info *find_frameless_function_info(zend_ast_list *args, zend_function *fbc, uint32_t type)
 {
-	if (ZEND_OBSERVER_ENABLED || zend_execute_internal) {
+	if (zend_execute_internal) {
 		return NULL;
 	}
 

Zend/zend_execute.c

@@ -1536,6 +1536,54 @@ static zend_never_inline void zend_assign_to_object_dim(zend_object *obj, zval *
 	}
 }
 
+static void frameless_observed_call_copy(zend_execute_data *call, uint32_t arg, zval *zv)
+{
+	if (Z_ISUNDEF_P(zv)) {
+		ZVAL_NULL(ZEND_CALL_VAR_NUM(call, arg));
+	} else {
+		ZVAL_COPY_DEREF(ZEND_CALL_VAR_NUM(call, arg), zv);
+	}
+}
+
+ZEND_API void zend_frameless_observed_call(zend_execute_data *execute_data)
+{
+	const zend_op *opline = EX(opline);
+	uint8_t num_args = ZEND_FLF_NUM_ARGS(opline->opcode);
+	zend_function *fbc = ZEND_FLF_FUNC(opline);
+	zval *result = EX_VAR(opline->result.var);
+
+	zend_execute_data *call = zend_vm_stack_push_call_frame_ex(zend_vm_calc_used_stack(num_args, fbc), ZEND_CALL_NESTED_FUNCTION, fbc, num_args, NULL);
+	call->prev_execute_data = execute_data;
+
+	switch (num_args) {
+		case 3: frameless_observed_call_copy(call, 2, zend_get_zval_ptr(opline+1, (opline+1)->op1_type, &(opline+1)->op1, execute_data)); ZEND_FALLTHROUGH;
+		case 2: frameless_observed_call_copy(call, 1, zend_get_zval_ptr(opline, opline->op2_type, &opline->op2, execute_data)); ZEND_FALLTHROUGH;
+		case 1: frameless_observed_call_copy(call, 0, zend_get_zval_ptr(opline, opline->op1_type, &opline->op1, execute_data));
+	}
+
+	EG(current_execute_data) = call;
+
+	zend_observer_fcall_begin_prechecked(call, ZEND_OBSERVER_DATA(fbc));
+	fbc->internal_function.handler(call, result);
+	zend_observer_fcall_end(call, result);
+
+	EG(current_execute_data) = execute_data;
+
+	if (UNEXPECTED(EG(exception) != NULL)) {
+		zend_rethrow_exception(execute_data);
+	}
+
+	zend_vm_stack_free_args(call);
+
+	uint32_t call_info = ZEND_CALL_INFO(call);
+	if (UNEXPECTED(call_info & ZEND_CALL_ALLOCATED)) {
+		zend_vm_stack_free_call_frame_ex(call_info, call);
+	} else {
+		EG(vm_stack_top) = (zval*)call;
+	}
+}
+
+
 static zend_always_inline int zend_binary_op(zval *ret, zval *op1, zval *op2 OPLINE_DC)
 {
 	static const binary_op_type zend_binary_ops[] = {

Zend/zend_execute.h

@@ -430,6 +430,8 @@ ZEND_API void zend_cleanup_unfinished_execution(zend_execute_data *execute_data,
 ZEND_API ZEND_ATTRIBUTE_DEPRECATED HashTable *zend_unfinished_execution_gc(zend_execute_data *execute_data, zend_execute_data *call, zend_get_gc_buffer *gc_buffer);
 ZEND_API HashTable *zend_unfinished_execution_gc_ex(zend_execute_data *execute_data, zend_execute_data *call, zend_get_gc_buffer *gc_buffer, bool suspended_by_yield);
 
+ZEND_API void zend_frameless_observed_call(zend_execute_data *execute_data);
+
 zval * ZEND_FASTCALL zend_handle_named_arg(
 		zend_execute_data **call_ptr, zend_string *arg_name,
 		uint32_t *arg_num_ptr, void **cache_slot);

Zend/zend_frameless_function.h

@@ -115,7 +115,7 @@ typedef void (*zend_frameless_function_3)(zval *return_value, zval *op1, zval *o
 extern size_t zend_flf_count;
 extern size_t zend_flf_capacity;
 ZEND_API extern void **zend_flf_handlers;
-extern zend_function **zend_flf_functions;
+ZEND_API extern zend_function **zend_flf_functions;
 
 typedef struct {
 	void *handler;

Zend/zend_globals.h

@@ -194,6 +194,8 @@ struct _zend_executor_globals {
 
 	uint32_t jit_trace_num; /* Used by tracing JIT to reference the currently running trace */
 
+	zend_execute_data *current_observed_frame;
+
 	int ticks_count;
 
 	zend_long precision;

Zend/zend_observer.c

@@ -14,6 +14,7 @@
    +----------------------------------------------------------------------+
    | Authors: Levi Morrison <[email protected]>                               |
    |          Sammy Kaye Powers <[email protected]>                          |
+   |          Bob Weinand <[email protected]>                           |
    +----------------------------------------------------------------------+
 */
 
@@ -23,15 +24,8 @@
 #include "zend_llist.h"
 #include "zend_vm.h"
 
-#define ZEND_OBSERVER_DATA(function) \
-	ZEND_OP_ARRAY_EXTENSION((&(function)->common), ZEND_USER_CODE((function)->type) \
-		? zend_observer_fcall_op_array_extension : zend_observer_fcall_internal_function_extension)
-
 #define ZEND_OBSERVER_NOT_OBSERVED ((void *) 2)
 
-#define ZEND_OBSERVABLE_FN(function) \
-	(ZEND_MAP_PTR(function->common.run_time_cache) && !(function->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))
-
 static zend_llist zend_observers_fcall_list;
 static zend_llist zend_observer_function_declared_callbacks;
 static zend_llist zend_observer_class_linked_callbacks;
@@ -46,8 +40,6 @@ bool zend_observer_errors_observed;
 bool zend_observer_function_declared_observed;
 bool zend_observer_class_linked_observed;
 
-ZEND_TLS zend_execute_data *current_observed_frame;
-
 // Call during minit/startup ONLY
 ZEND_API void zend_observer_fcall_register(zend_observer_fcall_init init)
 {
@@ -107,7 +99,7 @@ ZEND_API void zend_observer_post_startup(void)
 
 ZEND_API void zend_observer_activate(void)
 {
-	current_observed_frame = NULL;
+    EG(current_observed_frame) = NULL;
 }
 
 ZEND_API void zend_observer_shutdown(void)
@@ -127,21 +119,24 @@ static void zend_observer_fcall_install(zend_execute_data *execute_data)
 	zend_function *function = execute_data->func;
 
 	ZEND_ASSERT(RUN_TIME_CACHE(&function->common));
-	zend_observer_fcall_begin_handler *begin_handlers = (zend_observer_fcall_begin_handler *)&ZEND_OBSERVER_DATA(function);
+	zend_observer_fcall_begin_handler *begin_handlers = ZEND_OBSERVER_DATA(function), *begin_handlers_start = begin_handlers;
 	zend_observer_fcall_end_handler *end_handlers = (zend_observer_fcall_end_handler *)begin_handlers + list->count, *end_handlers_start = end_handlers;
 
 	*begin_handlers = ZEND_OBSERVER_NOT_OBSERVED;
 	*end_handlers = ZEND_OBSERVER_NOT_OBSERVED;
+	bool has_handlers = false;
 
 	for (zend_llist_element *element = list->head; element; element = element->next) {
 		zend_observer_fcall_init init;
 		memcpy(&init, element->data, sizeof init);
 		zend_observer_fcall_handlers handlers = init(execute_data);
 		if (handlers.begin) {
 			*(begin_handlers++) = handlers.begin;
+			has_handlers = true;
 		}
 		if (handlers.end) {
 			*(end_handlers++) = handlers.end;
+			has_handlers = true;
 		}
 	}
 
@@ -151,6 +146,10 @@ static void zend_observer_fcall_install(zend_execute_data *execute_data)
 		*end_handlers = *end_handlers_start;
 		*end_handlers_start = tmp;
 	}
+
+	if (!has_handlers) {
+		*begin_handlers_start = ZEND_OBSERVER_NONE_OBSERVED;
+	}
 }
 
 /* We need to provide the ability to retrieve the handler which will move onto the position the current handler was.
@@ -182,8 +181,8 @@ static bool zend_observer_remove_handler(void **first_handler, void *old_handler
 
 ZEND_API void zend_observer_add_begin_handler(zend_function *function, zend_observer_fcall_begin_handler begin) {
 	size_t registered_observers = zend_observers_fcall_list.count;
-	zend_observer_fcall_begin_handler *first_handler = (void *)&ZEND_OBSERVER_DATA(function), *last_handler = first_handler + registered_observers - 1;
-	if (*first_handler == ZEND_OBSERVER_NOT_OBSERVED) {
+	zend_observer_fcall_begin_handler *first_handler = ZEND_OBSERVER_DATA(function), *last_handler = first_handler + registered_observers - 1;
+	if (*first_handler == ZEND_OBSERVER_NOT_OBSERVED || *first_handler == ZEND_OBSERVER_NONE_OBSERVED) {
 		*first_handler = begin;
 	} else {
 		for (zend_observer_fcall_begin_handler *cur_handler = first_handler + 1; cur_handler <= last_handler; ++cur_handler) {
@@ -198,24 +197,47 @@ ZEND_API void zend_observer_add_begin_handler(zend_function *function, zend_obse
 }
 
 ZEND_API bool zend_observer_remove_begin_handler(zend_function *function, zend_observer_fcall_begin_handler begin, zend_observer_fcall_begin_handler *next) {
-	return zend_observer_remove_handler((void **)&ZEND_OBSERVER_DATA(function), begin, (void **)next);
+	void **begin_handlers = (void **)ZEND_OBSERVER_DATA(function);
+	if (zend_observer_remove_handler(begin_handlers, begin, (void**)next)) {
+		// Ensure invariant: ZEND_OBSERVER_NONE_OBSERVED in begin_handlers if both are not observed
+		if (*begin_handlers == ZEND_OBSERVER_NOT_OBSERVED) {
+			size_t registered_observers = zend_observers_fcall_list.count;
+			if (begin_handlers[registered_observers] /* first end handler */ == ZEND_OBSERVER_NOT_OBSERVED) {
+				*begin_handlers = ZEND_OBSERVER_NONE_OBSERVED;
+			}
+		}
+		return true;
+	}
+	return false;
 }
 
 ZEND_API void zend_observer_add_end_handler(zend_function *function, zend_observer_fcall_end_handler end) {
 	size_t registered_observers = zend_observers_fcall_list.count;
-	zend_observer_fcall_end_handler *end_handler = (zend_observer_fcall_end_handler *)&ZEND_OBSERVER_DATA(function) + registered_observers;
+	void **begin_handler = (void **)ZEND_OBSERVER_DATA(function);
+	zend_observer_fcall_end_handler *end_handler = (zend_observer_fcall_end_handler *)begin_handler + registered_observers;
 	// to allow to preserve the invariant that end handlers are in reverse order of begin handlers, push the new end handler in front
 	if (*end_handler != ZEND_OBSERVER_NOT_OBSERVED) {
 		// there's no space for new handlers, then it's forbidden to call this function
 		ZEND_ASSERT(end_handler[registered_observers - 1] == NULL);
 		memmove(end_handler + 1, end_handler, sizeof(end_handler) * (registered_observers - 1));
+	} else if (*begin_handler == ZEND_OBSERVER_NONE_OBSERVED) {
+		*begin_handler = ZEND_OBSERVER_NOT_OBSERVED;
 	}
 	*end_handler = end;
 }
 
 ZEND_API bool zend_observer_remove_end_handler(zend_function *function, zend_observer_fcall_end_handler end, zend_observer_fcall_end_handler *next) {
 	size_t registered_observers = zend_observers_fcall_list.count;
-	return zend_observer_remove_handler((void **)&ZEND_OBSERVER_DATA(function) + registered_observers, end, (void **)next);
+	void **begin_handlers = (void **)ZEND_OBSERVER_DATA(function);
+	void **end_handlers = begin_handlers + registered_observers;
+	if (zend_observer_remove_handler(end_handlers, end, (void**)next)) {
+		// Ensure invariant: ZEND_OBSERVER_NONE_OBSERVED in begin_handlers if both are not observed
+		if (*begin_handlers == ZEND_OBSERVER_NOT_OBSERVED && *end_handlers == ZEND_OBSERVER_NOT_OBSERVED) {
+			*begin_handlers = ZEND_OBSERVER_NONE_OBSERVED;
+		}
+		return true;
+	}
+	return false;
 }
 
 static inline zend_execute_data **prev_observed_frame(zend_execute_data *execute_data) {
@@ -224,33 +246,33 @@ static inline zend_execute_data **prev_observed_frame(zend_execute_data *execute
 	return (zend_execute_data **)&Z_PTR_P(EX_VAR_NUM((ZEND_USER_CODE(func->type) ? func->op_array.last_var : ZEND_CALL_NUM_ARGS(execute_data)) + func->common.T - 1));
 }
 
-static void ZEND_FASTCALL _zend_observe_fcall_begin(zend_execute_data *execute_data)
-{
+static void ZEND_FASTCALL _zend_observe_fcall_begin(zend_execute_data *execute_data) {
 	if (!ZEND_OBSERVER_ENABLED) {
 		return;
 	}
 
-	zend_function *function = execute_data->func;
+	zend_observer_fcall_begin_specialized(execute_data, true);
+}
 
-	if (!ZEND_OBSERVABLE_FN(function)) {
-		return;
-	}
+ZEND_API void ZEND_FASTCALL zend_observer_fcall_begin_prechecked(zend_execute_data *execute_data, zend_observer_fcall_begin_handler *handler)
+{
+	zend_observer_fcall_begin_handler *possible_handlers_end = handler + zend_observers_fcall_list.count;
 
-	zend_observer_fcall_begin_handler *handler = (zend_observer_fcall_begin_handler *)&ZEND_OBSERVER_DATA(function);
 	if (!*handler) {
 		zend_observer_fcall_install(execute_data);
+		if (zend_observer_handler_is_unobserved(handler)) {
+			return;
+		}
 	}
 
-	zend_observer_fcall_begin_handler *possible_handlers_end = handler + zend_observers_fcall_list.count;
-
 	zend_observer_fcall_end_handler *end_handler = (zend_observer_fcall_end_handler *)possible_handlers_end;
 	if (*end_handler != ZEND_OBSERVER_NOT_OBSERVED) {
-		*prev_observed_frame(execute_data) = current_observed_frame;
-		current_observed_frame = execute_data;
-	}
+		*prev_observed_frame(execute_data) = EG(current_observed_frame);
+		EG(current_observed_frame) = execute_data;
 
-	if (*handler == ZEND_OBSERVER_NOT_OBSERVED) {
-		return;
+		if (*handler == ZEND_OBSERVER_NOT_OBSERVED) { // this function must not be called if ZEND_OBSERVER_NONE_OBSERVED, hence sufficient to check
+			return;
+		}
 	}
 
 	do {
@@ -265,17 +287,17 @@ ZEND_API void ZEND_FASTCALL zend_observer_generator_resume(zend_execute_data *ex
 
 ZEND_API void ZEND_FASTCALL zend_observer_fcall_begin(zend_execute_data *execute_data)
 {
-	ZEND_ASSUME(execute_data->func);
-	if (!(execute_data->func->common.fn_flags & ZEND_ACC_GENERATOR)) {
+	ZEND_ASSUME(EX(func));
+	if (!(EX(func)->common.fn_flags & ZEND_ACC_GENERATOR)) {
 		_zend_observe_fcall_begin(execute_data);
 	}
 }
 
 static inline void call_end_observers(zend_execute_data *execute_data, zval *return_value) {
-	zend_function *func = execute_data->func;
+	zend_function *func = EX(func);
 	ZEND_ASSERT(func);
 
-	zend_observer_fcall_end_handler *handler = (zend_observer_fcall_end_handler *)&ZEND_OBSERVER_DATA(func) + zend_observers_fcall_list.count;
+	zend_observer_fcall_end_handler *handler = (zend_observer_fcall_end_handler *)ZEND_OBSERVER_DATA(func) + zend_observers_fcall_list.count;
 	// TODO: Fix exceptions from generators
 	// ZEND_ASSERT(fcall_data);
 	if (!*handler || *handler == ZEND_OBSERVER_NOT_OBSERVED) {
@@ -288,19 +310,16 @@ static inline void call_end_observers(zend_execute_data *execute_data, zval *ret
 	} while (++handler != possible_handlers_end && *handler != NULL);
 }
 
-ZEND_API void ZEND_FASTCALL zend_observer_fcall_end(zend_execute_data *execute_data, zval *return_value)
+ZEND_API void ZEND_FASTCALL zend_observer_fcall_end_prechecked(zend_execute_data *execute_data, zval *return_value)
 {
-	if (execute_data != current_observed_frame) {
-		return;
-	}
 	call_end_observers(execute_data, return_value);
-	current_observed_frame = *prev_observed_frame(execute_data);
+	EG(current_observed_frame) = *prev_observed_frame(execute_data);
 }
 
 ZEND_API void zend_observer_fcall_end_all(void)
 {
-	zend_execute_data *execute_data = current_observed_frame, *original_execute_data = EG(current_execute_data);
-	current_observed_frame = NULL;
+	zend_execute_data *execute_data = EG(current_observed_frame), *original_execute_data = EG(current_execute_data);
+    EG(current_observed_frame) = NULL;
 	while (execute_data) {
 		EG(current_execute_data) = execute_data;
 		call_end_observers(execute_data, NULL);
@@ -401,8 +420,8 @@ ZEND_API void ZEND_FASTCALL zend_observer_fiber_switch_notify(zend_fiber_context
 		callback(from, to);
 	}
 
-	from->top_observed_frame = current_observed_frame;
-	current_observed_frame = to->top_observed_frame;
+	from->top_observed_frame = EG(current_observed_frame);
+    EG(current_observed_frame) = to->top_observed_frame;
 }
 
 ZEND_API void ZEND_FASTCALL zend_observer_fiber_destroy_notify(zend_fiber_context *destroying)