MySQL: Track session-specific system variables changes

include/MySQL_HostGroups_Manager.h

@@ -227,6 +227,15 @@ class MySrvC {	// MySQL Server Container
 	bool shunned_and_kill_all_connections; // if a serious failure is detected, this will cause all connections to die even if the server is just shunned
 	int32_t use_ssl;
 	char *comment;
+
+	// 'server_backoff_time' stores a timestamp that prevents the server from being
+	// considered for random selection ('MyHGC::get_random_MySrvC') until that time passes.
+	//
+	// This is primarily used when `session_track_variables::ENFORCED` mode is active.
+	// If a server lacks the required capabilities in this mode, it is temporarily
+	// excluded from selection for a specified duration.
+	unsigned long long server_backoff_time;
+
 	MySrvConnList *ConnectionsUsed;
 	MySrvConnList *ConnectionsFree;
 	/**

include/MySQL_Session.h

@@ -241,13 +241,65 @@ class MySQL_Session: public Base_Session<MySQL_Session, MySQL_Data_Stream, MySQL
 	bool handler_again___verify_ldap_user_variable();
 	bool handler_again___verify_backend_autocommit();
 	bool handler_again___verify_backend_session_track_gtids();
+	/**
+	 * @brief Verify and configure session variable tracking on backend connections.
+	 *
+	 * === PR 5166: Backend Configuration Entry Point ===
+	 *
+	 * This function is the main orchestrator for setting up session tracking on backend
+	 * connections. It's called during connection initialization to ensure tracking is
+	 * properly configured before query processing begins.
+	 *
+	 * CONFIGURATION LOGIC:
+	 * 1. Check if global mysql-session_track_variables is enabled
+	 * 2. For each tracking flag that's not yet set on the connection:
+	 *    - Mark the flag as sent (prevents duplicate configuration)
+	 *    - Transition session to appropriate configuration state
+	 *    - Return true to indicate state machine needs re-processing
+	 *
+	 * STATE MACHINE INTEGRATION:
+	 * - SETTING_SESSION_TRACK_VARIABLES: Sets session_track_system_variables="*"
+	 * - SETTING_SESSION_TRACK_STATE: Sets session_track_state_change=ON
+	 * - Returns true to continue state machine processing until both are configured
+	 *
+	 * WHY THIS APPROACH:
+	 * - Ensures tracking is configured exactly once per backend connection
+	 * - Integrates cleanly with existing ProxySQL session state machine
+	 * - Handles both tracking capabilities independently for flexibility
+	 * - Prevents redundant SET commands on already configured connections
+	 *
+	 * @return true if session state needs to be re-processed (configuration pending), false otherwise
+	 */
+	bool handler_again___verify_backend_session_track_variables();
 	bool handler_again___verify_backend_multi_statement();
 	bool handler_again___verify_backend_user_schema();
 	bool handler_again___verify_multiple_variables(MySQL_Connection *);
 	bool handler_again___status_SETTING_INIT_CONNECT(int *);
 	bool handler_again___status_SETTING_LDAP_USER_VARIABLE(int *);
 	bool handler_again___status_SETTING_SQL_MODE(int *);
 	bool handler_again___status_SETTING_SESSION_TRACK_GTIDS(int *);
+	/**
+	 * @brief Handle the SETTING_SESSION_TRACK_VARIABLES state.
+	 *
+	 * This method executes the SET command to configure session_track_system_variables="*"
+	 * on the backend connection, enabling the server to track changes to all system
+	 * variables and report them back to ProxySQL.
+	 *
+	 * @param _rc Pointer to return code that will be set with the operation result
+	 * @return true if session state needs to be re-processed, false otherwise
+	 */
+	bool handler_again___status_SETTING_SESSION_TRACK_VARIABLES(int *);
+	/**
+	 * @brief Handle the SETTING_SESSION_TRACK_STATE state.
+	 *
+	 * This method executes the SET command to configure session_track_state_change=ON
+	 * on the backend connection, enabling the server to report when session state
+	 * changes occur (including system variable changes).
+	 *
+	 * @param _rc Pointer to return code that will be set with the operation result
+	 * @return true if session state needs to be re-processed, false otherwise
+	 */
+	bool handler_again___status_SETTING_SESSION_TRACK_STATE(int *);
 	bool handler_again___status_CHANGING_CHARSET(int *_rc);
 	bool handler_again___status_CHANGING_SCHEMA(int *);
 	bool handler_again___status_CONNECTING_SERVER(int *);
@@ -296,6 +348,35 @@ class MySQL_Session: public Base_Session<MySQL_Session, MySQL_Data_Stream, MySQL
 	int RunQuery(MySQL_Data_Stream *myds, MySQL_Connection *myconn);
 	void handler___status_WAITING_CLIENT_DATA();
 	void handler_rc0_Process_GTID(MySQL_Connection *myconn);
+	/**
+	 * @brief Process session variable changes from backend connection response.
+	 *
+	 * === PR 5166: Variable Processing Workflow ===
+	 *
+	 * This function is the core of the variable tracking system and is called after
+	 * every successful query execution when SERVER_SESSION_STATE_CHANGED flag is set.
+	 *
+	 * DETAILED WORKFLOW:
+	 * 1. Extract variable changes from MySQL's session tracking data via get_variables()
+	 * 2. Iterate through all tracked variables in mysql_tracked_variables array
+	 * 3. For each variable that changed in the backend:
+	 *    - Update both client and server variable maps for state consistency
+	 *    - Handle character set variables specially (convert names to internal IDs)
+	 * 4. This ensures ProxySQL's internal state matches the actual backend state
+	 *
+	 * WHY THIS IS NEEDED:
+	 * - SQL statement parsing cannot detect all variable changes (e.g., stored procedures)
+	 * - Some variables are changed implicitly by MySQL server operations
+	 * - Without this tracking, client and backend states can diverge
+	 *
+	 * PERFORMANCE CONSIDERATIONS:
+	 * - Only called when SERVER_SESSION_STATE_CHANGED flag is set
+	 * - Processes all tracked variables but only updates those that actually changed
+	 * - Character set conversions are done only for relevant variables
+	 *
+	 * @param myconn Pointer to the MySQL connection from which to extract variable changes
+	 */
+	void handler_rc0_Process_Variables(MySQL_Connection *myconn);
 	void handler_rc0_RefreshActiveTransactions(MySQL_Connection* myconn);
 	void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_INIT_DB_replace_CLICKHOUSE(PtrSize_t& pkt);
 	void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY___not_mysql(PtrSize_t& pkt);
@@ -489,6 +570,7 @@ class MySQL_Session: public Base_Session<MySQL_Session, MySQL_Data_Stream, MySQL
 	void reset_warning_hostgroup_flag_and_release_connection();
 	void set_previous_status_mode3(bool allow_execute=true);
 	char* get_current_query(int max_length = -1);
+	bool handle_session_track_capabilities();
 	/**
 	 * @brief Attempts to get the server version string from a backend connection in the specified hostgroup.
 	 * @details This function iterates through servers in the hostgroup and checks for any available

include/MySQL_Thread.h

@@ -336,6 +336,49 @@ struct th_metrics_map_idx {
 	};
 };
 
+/**
+ * @brief Configuration structure for session variable tracking functionality.
+ *
+ * === PR 5166: Session Variable Tracking Architecture ===
+ *
+ * This PR introduces comprehensive tracking of session-specific system variables
+ * by leveraging MySQL's native session tracking capabilities. The overall workflow
+ * consists of three main phases:
+ *
+ * 1. CONFIGURATION PHASE:
+ *    - Global enable/disable via mysql-session_track_variables variable
+ *    - Per-connection tracking state managed via flags in MySQL_Connection
+ *    - New session states handle configuration of tracking on backends
+ *
+ * 2. BACKEND SETUP PHASE:
+ *    - When enabled, configure session_track_system_variables="*"
+ *    - Configure session_track_state_change=ON
+ *    - MySQL server then automatically tracks all system variable changes
+ *
+ * 3. PROCESSING PHASE:
+ *    - After each query, check SERVER_SESSION_STATE_CHANGED flag
+ *    - Extract variable changes via MySQL's session tracking protocol
+ *    - Update both client and server variable maps for state consistency
+ *    - Special handling for character set variables (name → ID conversion)
+ *
+ * BENEFITS:
+ * - Captures changes that SQL parsing alone cannot detect (stored procedures, etc.)
+ * - Redundant tracking ensures accurate client/backend state synchronization
+ * - Leverages MySQL server's native capabilities for reliability
+ * - Performance optimized: only processes when session state actually changes
+ */
+struct session_track_variables {
+	enum mode {
+		// Disabled; default mode
+		DISABLED = 0,
+		// Enable session tracking if backend supports it
+		OPTIONAL,
+		// Enforce session tracking; connection fails if backend does
+		// not support CLIENT_DEPRECATE_EOF and CLIENT_SESSION_TRACKING
+		ENFORCED
+	};
+};
+
 /**
  * @brief Structure holding the data for a Client_Host_Cache entry.
  */
@@ -591,6 +634,16 @@ class MySQL_Threads_Handler
 		int processlist_max_query_length;
 
 		bool ignore_min_gtid_annotations;
+		/**
+		 * @brief Configuration flag to enable/disable session variable tracking.
+		 *
+		 * When set to session_track_variables::ENABLED (1), ProxySQL will configure
+		 * backend connections to track system variable changes using MySQL's
+		 * session_track_system_variables and session_track_state_change capabilities.
+		 *
+		 * Default: session_track_variables::DISABLED (0)
+		 */
+		int session_track_variables;
 	} variables;
 	struct {
 		unsigned int mirror_sessions_current;

include/mysql_connection.h

@@ -88,7 +88,9 @@ class MySQL_Connection {
 		char * session_track_gtids;
 		char *ldap_user_variable;
 		char *ldap_user_variable_value;
-		bool session_track_gtids_sent;
+		bool session_track_gtids_sent;		///< Flag indicating if GTID session tracking has been configured on this connection
+		bool session_track_variables_sent;	///< Flag indicating if session_track_system_variables has been configured on this connection
+		bool session_track_state_sent;		///< Flag indicating if session_track_state_change has been configured on this connection
 		bool ldap_user_variable_sent;
 		uint8_t protocol_version;
 		int8_t last_set_autocommit;
@@ -262,6 +264,36 @@ class MySQL_Connection {
 	void reset();
 
 	bool get_gtid(char *buff, uint64_t *trx_id);
+	/**
+	 * @brief Extract session variable changes from MySQL's session tracking system.
+	 *
+	 * === PR 5166: Backend Variable Extraction ===
+	 *
+	 * This method is the interface to MySQL's native session tracking protocol and
+	 * extracts system variable changes that occurred during the last query execution.
+	 *
+	 * TECHNICAL DETAILS:
+	 * - Uses mysql_session_track_get_first/next() to iterate through SESSION_TRACK_SYSTEM_VARIABLES
+	 * - Only processes when SERVER_SESSION_STATE_CHANGED flag is set and no errors occurred
+	 * - Handles the variable/value pairing protocol where variables and values are returned alternately
+	 * - Populates a map with variable names as keys and their new values as values
+	 *
+	 * PROTOCOL FLOW:
+	 * 1. Check if session state changed and no errors occurred
+	 * 2. Get first tracked system variable (name)
+	 * 3. Iterate to get corresponding value, next variable name, etc.
+	 * 4. Build variable_name → value mappings in the provided map
+	 *
+	 * INTEGRATION POINT:
+	 * This is called by handler_rc0_Process_Variables() which then processes the
+	 * extracted changes to update ProxySQL's internal state. The separation allows
+	 * for testing and potential future extensions of the tracking protocol.
+	 *
+	 * @param variables Reference to unordered_map that will be populated with
+	 *                 variable names as keys and their new values as values
+	 * @return true if session variable changes were found and extracted, false otherwise
+	 */
+	bool get_variables(std::unordered_map<std::string, std::string>&);
 	void reduce_auto_increment_delay_token() { if (auto_increment_delay_token) auto_increment_delay_token--; };
 
 	bool match_ff_req_options(const MySQL_Connection *c);

include/proxysql_structs.h

@@ -311,6 +311,8 @@ enum session_status {
 	SETTING_NEXT_TRANSACTION_READ,
 	PROCESSING_EXTENDED_QUERY_SYNC,
 	RESYNCHRONIZING_CONNECTION,
+	SETTING_SESSION_TRACK_VARIABLES,
+	SETTING_SESSION_TRACK_STATE,
 	session_status___NONE // special marker
 };
 
@@ -1305,6 +1307,7 @@ __thread int mysql_thread___client_host_error_counts;
 __thread int mysql_thread___handle_warnings;
 __thread int mysql_thread___evaluate_replication_lag_on_servers_load;
 __thread bool mysql_thread___ignore_min_gtid_annotations;
+__thread int mysql_thread___session_track_variables;
 
 /* variables used for Query Cache */
 __thread int mysql_thread___query_cache_size_MB;
@@ -1609,6 +1612,7 @@ extern __thread int mysql_thread___client_host_error_counts;
 extern __thread int mysql_thread___handle_warnings;
 extern __thread int mysql_thread___evaluate_replication_lag_on_servers_load;
 extern __thread bool mysql_thread___ignore_min_gtid_annotations;
+extern __thread int mysql_thread___session_track_variables;
 
 /* variables used for Query Cache */
 extern __thread int mysql_thread___query_cache_size_MB;

lib/MyHGC.cpp

@@ -38,6 +38,10 @@ MySrvC *MyHGC::get_random_MySrvC(char * gtid_uuid, uint64_t gtid_trxid, int max_
 		for (j=0; j<l; j++) {
 			mysrvc=mysrvs->idx(j);
 			if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE) { // consider this server only if ONLINE
+				// skip servers that are in backoff period
+				if (mysrvc->server_backoff_time > sess->thread->curtime)
+					continue;
+
 				if (mysrvc->myhgc->num_online_servers.load(std::memory_order_relaxed) <= mysrvc->myhgc->attributes.max_num_online_servers) { // number of online servers in HG is within configured range
 					if (mysrvc->ConnectionsUsed->conns_length() < mysrvc->max_connections) { // consider this server only if didn't reach max_connections
 						if (mysrvc->current_latency_us < (mysrvc->max_latency_us ? mysrvc->max_latency_us : mysql_thread___default_max_latency_ms*1000)) { // consider the host only if not too far

lib/MySQL_Protocol.cpp

@@ -1111,9 +1111,13 @@ bool MySQL_Protocol::generate_pkt_initial_handshake(bool send, void **ptr, unsig
 	// variable. This is the first step of ensuring that client connections doesn't
 	// enable 'CLIENT_DEPRECATE_EOF' unless explicitly stated by 'mysql-enable_client_deprecate_eof'.
 	// Second step occurs during client handshake response (process_pkt_handshake_response).
-	if (deprecate_eof_active && mysql_thread___enable_client_deprecate_eof) {
-		extended_capabilities |= CLIENT_DEPRECATE_EOF;
+	if (deprecate_eof_active) {
+		if (mysql_thread___enable_client_deprecate_eof
+			|| mysql_thread___session_track_variables == session_track_variables::ENFORCED) {
+			extended_capabilities |= CLIENT_DEPRECATE_EOF;
+		}
 	}
+
 	// Copy the 'capability_flags_2'
 	uint16_t upper_word = static_cast<uint16_t>(extended_capabilities >> 16);
 	memcpy(_ptr+l, static_cast<void*>(&upper_word), sizeof(upper_word)); l += sizeof(upper_word);