tlshd: support setting the record size limit

RFC 8449 [1] Section 4 defines the record_size_limit TLS extension, which
allows peers to negotiate a maximum plaintext record size during the
TLS handshake. The value must be between 64 bytes and 16,384 bytes (2^14).
If a TLS endpoint receives a record larger than its advertised limit, it
must send a fatal record_overflow alert.

This patch fetches maximum support send size as specified by the record
size limit extension or as defined in GnuTLS, this value is then passed to
the kernel through setsockopt() using the new TLS_TX_MAX_PAYLOAD_LEN
option, such that the kernel can ensure outgoing records do not exceed the
size specified.

The respective kernel changes are currently applied to net-next [2].

[1] https://www.rfc-editor.org/rfc/rfc8449#section-4
[2] https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git/commit/?id=82cb5be6ad64198a3a028aeb49dcc7f6224d558a

Signed-off-by: Wilfred Mallawa <[email protected]>

Author: twilfredo

configure.ac

@@ -85,6 +85,9 @@ AC_CHECK_LIB([gnutls], [gnutls_get_system_config_file],
 AC_CHECK_LIB([gnutls], [gnutls_psk_allocate_client_credentials2],
              [AC_DEFINE([HAVE_GNUTLS_PSK_ALLOCATE_CREDENTIALS2], [1],
                         [Define to 1 if you have the gnutls_psk_allocate_client_credentials2 function.])])
+AC_CHECK_LIB([gnutls], [gnutls_record_get_max_send_size],
+             [AC_DEFINE([HAVE_GNUTLS_RECORD_GET_MAX_SEND_SIZE], [1],
+			[Define to 1 if you have the gnutls_record_get_max_send_size function.])])
 
 AC_MSG_CHECKING(for ML-DSA support in gnutls)
 AC_COMPILE_IFELSE(
@@ -97,6 +100,17 @@ if test "x$have_mldsa" = xyes ; then
 	AC_DEFINE([HAVE_GNUTLS_MLDSA], [1], [Define to 1 if gnutls supports ML-DSA])
 fi
 
+AC_MSG_CHECKING(for TLS_TX_MAX_PAYLOAD_LEN in linux/tls.h)
+AC_COMPILE_IFELSE(
+	[AC_LANG_PROGRAM([[ #include <linux/tls.h> ]],
+		[[ (void) TLS_TX_MAX_PAYLOAD_LEN; ]])],
+	[ have_tls_tx_max_payload_len=yes ],
+	[ have_tls_tx_max_payload_len=no ])
+AC_MSG_RESULT([$have_tls_tx_max_payload_len])
+if test "x$have_tls_tx_max_payload_len" = xyes ; then
+	AC_DEFINE([HAVE_TLS_TX_MAX_PAYLOAD_LEN], [1], [Define to 1 if linux/tls.h defines TLS_TX_MAX_PAYLOAD_LEN])
+fi
+
 AC_SUBST([AM_CPPFLAGS])
 
 AC_CONFIG_FILES([Makefile \

src/tlshd/handshake.c

@@ -43,6 +43,7 @@
 #include <gnutls/abstract.h>
 
 #include <glib.h>
+#include <linux/tls.h>
 
 #include "tlshd.h"
 #include "netlink.h"

src/tlshd/ktls.c

@@ -347,6 +347,26 @@ static bool tlshd_set_chacha20_poly1305_info(gnutls_session_t session, int sock,
 }
 #endif
 
+#if defined(HAVE_GNUTLS_RECORD_GET_MAX_SEND_SIZE) && defined(HAVE_TLS_TX_MAX_PAYLOAD_LEN)
+static int tlshd_set_record_size(gnutls_session_t session)
+{
+	uint16_t max_send_size;
+	int ret;
+
+	max_send_size = gnutls_record_get_max_send_size(session);
+	/* For TLS 1.3 kernel expects us to account for the ContentType */
+	if (gnutls_protocol_get_version(session) == GNUTLS_TLS1_3)
+		max_send_size -= 1;
+
+	ret = setsockopt(gnutls_transport_get_int(session), SOL_TLS,
+			 TLS_TX_MAX_PAYLOAD_LEN, &max_send_size, sizeof(max_send_size));
+	if (ret < 0)
+		tlshd_log_perror("setsockopt (TLS_TX_MAX_PAYLOAD_LEN)");
+
+	return ret;
+}
+#endif
+
 /**
  * @brief Initialize a socket for use by kTLS
  * @param[in]     session  TLS session descriptor
@@ -363,6 +383,11 @@ unsigned int tlshd_initialize_ktls(gnutls_session_t session)
 		return EIO;
 	}
 
+#if defined(HAVE_GNUTLS_RECORD_GET_MAX_SEND_SIZE) && defined(HAVE_TLS_TX_MAX_PAYLOAD_LEN)
+	if (tlshd_set_record_size(session) < 0)
+		return EIO;
+#endif
+
 	gnutls_transport_get_int2(session, &sockin, &sockout);
 
 	switch (gnutls_cipher_get(session)) {