tlshd: support setting the record size limit

RFC 8449 [1] Section 4 defines the record_size_limit TLS extension, which
allows peers to negotiate a maximum plaintext record size during the
TLS handshake. The value must be between 64 bytes and 16,384 bytes (2^14).
If a TLS endpoint receives a record larger than its advertised limit, it
must send a fatal record_overflow alert.

This patch fetches maximum support send size as specified by the record
size limit extension or as defined in GnuTLS, this value is then passed to
the kernel through setsockopt() using the new TLS_TX_MAX_PAYLOAD_LEN
option, such that the kernel can ensure outgoing records do not exceed the
size specified.

The respective kernel changes are currently applied to net-next [2].

[1] https://www.rfc-editor.org/rfc/rfc8449#section-4
[2] https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git/commit/?id=82cb5be6ad64198a3a028aeb49dcc7f6224d558a

Signed-off-by: Wilfred Mallawa <[email protected]>

Author: twilfredo

configure.ac

@@ -85,6 +85,9 @@ AC_CHECK_LIB([gnutls], [gnutls_get_system_config_file],
 AC_CHECK_LIB([gnutls], [gnutls_psk_allocate_client_credentials2],
              [AC_DEFINE([HAVE_GNUTLS_PSK_ALLOCATE_CREDENTIALS2], [1],
                         [Define to 1 if you have the gnutls_psk_allocate_client_credentials2 function.])])
+AC_CHECK_LIB([gnutls], [gnutls_record_get_max_send_size],
+             [AC_DEFINE([HAVE_GNUTLS_RECORD_GET_MAX_SEND_SIZE], [1],
+			[Define to 1 if you have the gnutls_record_get_max_send_size function.])])
 
 AC_MSG_CHECKING(for ML-DSA support in gnutls)
 AC_COMPILE_IFELSE(

src/tlshd/handshake.c

@@ -43,10 +43,31 @@
 #include <gnutls/abstract.h>
 
 #include <glib.h>
+#include <linux/tls.h>
 
 #include "tlshd.h"
 #include "netlink.h"
 
+#ifdef HAVE_GNUTLS_RECORD_GET_MAX_SEND_SIZE
+int tlshd_set_record_size(gnutls_session_t session)
+{
+	uint16_t max_send_size;
+	int ret;
+
+	max_send_size = gnutls_record_get_max_send_size(session);
+	/* For TLS 1.3 kernel expects us to account for the ContentType */
+	if (gnutls_protocol_get_version(session) == GNUTLS_TLS1_3)
+		max_send_size -= 1;
+
+	ret = setsockopt(gnutls_transport_get_int(session), SOL_TLS,
+			 TLS_TX_MAX_PAYLOAD_LEN, &max_send_size, sizeof(max_send_size));
+	if (ret < 0)
+		tlshd_log_perror("setsockopt (TLS_TX_MAX_PAYLOAD_LEN)");
+
+	return ret;
+}
+#endif
+
 /**
  * @brief Toggle the use of the Nagle algorithm
  * @param[in]     session  TLS session to modify
@@ -125,6 +146,7 @@ void tlshd_start_tls_handshake(gnutls_session_t session,
 	gnutls_free(desc);
 
 	parms->session_status = tlshd_initialize_ktls(session);
+	tlshd_set_record_size(session);
 }
 
 /**

src/tlshd/ktls.c

@@ -363,6 +363,9 @@ unsigned int tlshd_initialize_ktls(gnutls_session_t session)
 		return EIO;
 	}
 
+	if (tlshd_set_record_size(session) < 0)
+		return EIO;
+
 	gnutls_transport_get_int2(session, &sockin, &sockout);
 
 	switch (gnutls_cipher_get(session)) {

src/tlshd/tlshd.h

@@ -83,6 +83,11 @@ extern void tlshd_start_tls_handshake(gnutls_session_t session,
 				      struct tlshd_handshake_parms *parms);
 extern void tlshd_service_socket(void);
 
+#ifdef HAVE_GNUTLS_RECORD_GET_MAX_SEND_SIZE
+extern int tlshd_set_record_size(gnutls_session_t session);
+#else
+static inline int tlshd_set_record_size(gnutls_session_t session __attribute__((unused))) { return 0; }
+#endif
 /* keyring.c */
 extern bool tlshd_keyring_get_psk_username(key_serial_t serial,
 					   char **username);