tlshd: support setting the record size limit

configure.ac

@@ -85,6 +85,9 @@ AC_CHECK_LIB([gnutls], [gnutls_get_system_config_file],
 AC_CHECK_LIB([gnutls], [gnutls_psk_allocate_client_credentials2],
              [AC_DEFINE([HAVE_GNUTLS_PSK_ALLOCATE_CREDENTIALS2], [1],
                         [Define to 1 if you have the gnutls_psk_allocate_client_credentials2 function.])])
+AC_CHECK_LIB([gnutls], [gnutls_record_get_max_send_size],
+             [AC_DEFINE([HAVE_GNUTLS_RECORD_GET_MAX_SEND_SIZE], [1],
+			[Define to 1 if you have the gnutls_record_get_max_send_size function.])])
 
 AC_MSG_CHECKING(for ML-DSA support in gnutls)
 AC_COMPILE_IFELSE(
@@ -97,6 +100,17 @@ if test "x$have_mldsa" = xyes ; then
 	AC_DEFINE([HAVE_GNUTLS_MLDSA], [1], [Define to 1 if gnutls supports ML-DSA])
 fi
 
+AC_MSG_CHECKING(for TLS_TX_MAX_PAYLOAD_LEN in linux/tls.h)
+AC_COMPILE_IFELSE(
+	[AC_LANG_PROGRAM([[ #include <linux/tls.h> ]],
+		[[ (void) TLS_TX_MAX_PAYLOAD_LEN; ]])],
+	[ have_tls_tx_max_payload_len=yes ],
+	[ have_tls_tx_max_payload_len=no ])
+AC_MSG_RESULT([$have_tls_tx_max_payload_len])
+if test "x$have_tls_tx_max_payload_len" = xyes ; then
+	AC_DEFINE([HAVE_TLS_TX_MAX_PAYLOAD_LEN], [1], [Define to 1 if linux/tls.h defines TLS_TX_MAX_PAYLOAD_LEN])
+fi
+
 AC_SUBST([AM_CPPFLAGS])
 
 AC_CONFIG_FILES([Makefile \

src/tlshd/handshake.c

@@ -43,6 +43,7 @@
 #include <gnutls/abstract.h>
 
 #include <glib.h>
+#include <linux/tls.h>
 
 #include "tlshd.h"
 #include "netlink.h"

src/tlshd/ktls.c

@@ -347,6 +347,26 @@ static bool tlshd_set_chacha20_poly1305_info(gnutls_session_t session, int sock,
 }
 #endif
 
+#if defined(HAVE_GNUTLS_RECORD_GET_MAX_SEND_SIZE) && defined(HAVE_TLS_TX_MAX_PAYLOAD_LEN)
+static int tlshd_set_record_size(gnutls_session_t session)
+{
+	uint16_t max_send_size;
+	int ret;
+
+	max_send_size = gnutls_record_get_max_send_size(session);
+	/* For TLS 1.3 kernel expects us to account for the ContentType */
+	if (gnutls_protocol_get_version(session) == GNUTLS_TLS1_3)
+		max_send_size -= 1;
+
+	ret = setsockopt(gnutls_transport_get_int(session), SOL_TLS,
+			 TLS_TX_MAX_PAYLOAD_LEN, &max_send_size, sizeof(max_send_size));
+	if (ret < 0)
+		tlshd_log_perror("setsockopt (TLS_TX_MAX_PAYLOAD_LEN)");
+
+	return ret;
+}
+#endif
+
 /**
  * @brief Initialize a socket for use by kTLS
  * @param[in]     session  TLS session descriptor
@@ -363,6 +383,11 @@ unsigned int tlshd_initialize_ktls(gnutls_session_t session)
 		return EIO;
 	}
 
+#if defined(HAVE_GNUTLS_RECORD_GET_MAX_SEND_SIZE) && defined(HAVE_TLS_TX_MAX_PAYLOAD_LEN)
+	if (tlshd_set_record_size(session) < 0)
+		return EIO;
+#endif
+
 	gnutls_transport_get_int2(session, &sockin, &sockout);
 
 	switch (gnutls_cipher_get(session)) {