tlshd: support setting the record size limit

RFC 8449 [1] Section 4 defines the record_size_limit TLS extension, which
allows peers to negotiate a maximum plaintext record size during the
TLS handshake. The value must be between 64 bytes and 16,384 bytes (2^14).
If a TLS endpoint receives a record larger than its advertised limit, it
must send a fatal record_overflow alert.

This patch fetches maximum support send size as specified by the record
size limit extension or as defined in GnuTLS, this value is then passed to
the kernel through setsockopt() using the new TLS_TX_MAX_PAYLOAD_LEN
option, such that the kernel can ensure outgoing records do not exceed the
size specified.

The respective kernel changes are currently applied to net-next [2].

[1] https://www.rfc-editor.org/rfc/rfc8449#section-4
[2] https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git/commit/?id=82cb5be6ad64198a3a028aeb49dcc7f6224d558a

Signed-off-by: Wilfred Mallawa <[email protected]>

Author: twilfredo

configure.ac

@@ -85,6 +85,9 @@ AC_CHECK_LIB([gnutls], [gnutls_get_system_config_file],
 AC_CHECK_LIB([gnutls], [gnutls_psk_allocate_client_credentials2],
              [AC_DEFINE([HAVE_GNUTLS_PSK_ALLOCATE_CREDENTIALS2], [1],
                         [Define to 1 if you have the gnutls_psk_allocate_client_credentials2 function.])])
+AC_CHECK_LIB([gnutls], [gnutls_record_get_max_send_size],
+             [AC_DEFINE([HAVE_GNUTLS_MAX_SEND_SIZE], [1],
+			[Define to 1 if you have the gnutls_record_get_max_send_size function.])])
 
 AC_MSG_CHECKING(for ML-DSA support in gnutls)
 AC_COMPILE_IFELSE(

src/tlshd/handshake.c

@@ -43,10 +43,29 @@
 #include <gnutls/abstract.h>
 
 #include <glib.h>
+#include <linux/tls.h>
 
 #include "tlshd.h"
 #include "netlink.h"
 
+#ifdef HAVE_GNUTLS_MAX_SEND_SIZE
+static void tlshd_set_record_size(gnutls_session_t session)
+{
+	uint16_t max_send_size;
+	int ret;
+
+	max_send_size = gnutls_record_get_max_send_size(session);
+	/* For TLS 1.3 kernel expects us to account for the ContentType */
+	if (gnutls_protocol_get_version(session) != GNUTLS_TLS1_3)
+		max_send_size -= 1;
+
+	ret = setsockopt(gnutls_transport_get_int(session), SOL_TLS,
+			 TLS_TX_MAX_PAYLOAD_LEN, &max_send_size, sizeof(max_send_size));
+	if (ret < 0)
+		tlshd_log_perror("setsockopt (TLS_RX_RECORD_SIZE_LIM)");
+}
+#endif
+
 /**
  * @brief Toggle the use of the Nagle algorithm
  * @param[in]     session  TLS session to modify
@@ -125,6 +144,9 @@ void tlshd_start_tls_handshake(gnutls_session_t session,
 	gnutls_free(desc);
 
 	parms->session_status = tlshd_initialize_ktls(session);
+#ifdef HAVE_GNUTLS_MAX_SEND_SIZE
+	tlshd_set_record_size(session);
+#endif
 }
 
 /**