MEDIUM: session: account on server idle conns attached to session

This patch adds a new member <curr_sess_idle_conns> on the server. It
serves as a counter of idle connections attached on a session instead of
regular idle/safe trees. This is used only for private connections.

The objective is to provide a method to detect if there is idle
connections still referencing a server.

This will be particularly useful to ensure that a server is removable.
Currently, this is not yet necessary as idle connections are directly
freed via "del server" handler under thread isolation. However, this
procedure will be replaced by an asynchronous mechanism outside of
thread isolation.

Careful: connections attached to a session but not idle will not be
accounted by this counter. These connections can still be detected via
srv_has_streams() so "del server" will be safe.

This counter is maintain during the whole lifetime of a private
connection. This is mandatory to guarantee "del server" safety and is
conform with other idle server counters. What this means it that
decrement is performed only when the connection transitions from idle to
in use, or just prior to its deletion. For the first case, this is
covered by session_get_conn(). The second case is trickier. It cannot be
done via session_unown_conn() as a private connection may still live a
little longer after its removal from session, most notably when
scheduled for idle purging.

Thus, conn_free() has been adjusted to handle the final decrement. Now,
conn_backend_deinit() is also called for private connections if
CO_FL_SESS_IDLE flag is present. This results in a call to
srv_release_conn() which is responsible to decrement server idle
counters.

Author: a-denoyelle

include/haproxy/server-t.h

@@ -388,6 +388,7 @@ struct server {
 	unsigned int curr_total_conns;          /* Current number of total connections to the server, used or idle, only calculated if strict-maxconn is used */
 	unsigned int max_used_conns;            /* Max number of used connections (the counter is reset at each connection purges */
 	unsigned int est_need_conns;            /* Estimate on the number of needed connections (max of curr and previous max_used) */
+	unsigned int curr_sess_idle_conns;      /* Current number of idle connections attached to a session instead of idle/safe trees. */
 
 	/* Element below are usd by LB algorithms and must be doable in
 	 * parallel to other threads reusing connections above.

src/connection.c

@@ -540,10 +540,9 @@ static void conn_backend_deinit(struct connection *conn)
 	if (obj_type(conn->target) == OBJ_TYPE_SERVER) {
 		struct server *srv = __objt_server(conn->target);
 
-		/* If the connection is not private, it is accounted by the server. */
-		if (!(conn->flags & CO_FL_PRIVATE)) {
+		if (!(conn->flags & CO_FL_PRIVATE) || (conn->flags & CO_FL_SESS_IDLE))
 			srv_release_conn(srv, conn);
-		}
+
 		if (srv->flags & SRV_F_STRICT_MAXCONN)
 			_HA_ATOMIC_DEC(&srv->curr_total_conns);
 	}

src/server.c

@@ -7251,7 +7251,11 @@ static void srv_cleanup_connections(struct server *srv)
 /* removes an idle conn after updating the server idle conns counters */
 void srv_release_conn(struct server *srv, struct connection *conn)
 {
-	if (conn->flags & CO_FL_LIST_MASK) {
+	if (conn->flags & CO_FL_SESS_IDLE) {
+		_HA_ATOMIC_DEC(&srv->curr_sess_idle_conns);
+		conn->flags &= ~CO_FL_SESS_IDLE;
+	}
+	else if (conn->flags & CO_FL_LIST_MASK) {
 		/* The connection is currently in the server's idle list, so tell it
 		 * there's one less connection available in that list.
 		 */

src/session.c

@@ -139,7 +139,6 @@ void session_free(struct session *sess)
 		list_for_each_entry_safe(conn, conn_back, &pconns->conn_list, sess_el) {
 			LIST_DEL_INIT(&conn->sess_el);
 			conn->owner = NULL;
-			conn->flags &= ~CO_FL_SESS_IDLE;
 			LIST_APPEND(&conn_tmp_list, &conn->sess_el);
 		}
 		MT_LIST_DELETE(&pconns->srv_el);
@@ -732,6 +731,9 @@ int session_reinsert_idle_conn(struct session *sess, struct connection *conn)
 /* Check that session <sess> is able to keep idle connection <conn>. This must
  * be called each time a connection stored in a session becomes idle.
  *
+ * If <conn> can be kept as idle in the session, idle sess conn counter of its
+ * target server will be incremented.
+ *
  * Returns 0 if the connection is kept, else non-zero if the connection was
  * explicitely removed from session.
  */
@@ -758,6 +760,8 @@ int session_check_idle_conn(struct session *sess, struct connection *conn)
 	else {
 		conn->flags |= CO_FL_SESS_IDLE;
 		sess->idle_conns++;
+		if (srv)
+			HA_ATOMIC_INC(&srv->curr_sess_idle_conns);
 	}
 
 	return 0;
@@ -771,6 +775,7 @@ struct connection *session_get_conn(struct session *sess, void *target, int64_t
 {
 	struct connection *srv_conn, *res = NULL;
 	struct sess_priv_conns *pconns;
+	struct server *srv;
 
 	HA_SPIN_LOCK(IDLE_CONNS_LOCK, &idle_conns[tid].idle_conns_lock);
 
@@ -787,6 +792,10 @@ struct connection *session_get_conn(struct session *sess, void *target, int64_t
 			if (srv_conn->flags & CO_FL_SESS_IDLE) {
 				srv_conn->flags &= ~CO_FL_SESS_IDLE;
 				sess->idle_conns--;
+
+				srv = objt_server(srv_conn->target);
+				if (srv)
+					HA_ATOMIC_DEC(&srv->curr_sess_idle_conns);
 			}
 
 			res = srv_conn;