tlshd: support setting the record size limit

RFC 8449 [1] Section 4 defines the record_size_limit TLS extension, which
allows peers to negotiate a maximum plaintext record size during the
TLS handshake. The value must be between 64 bytes and 16,384 bytes (2^14).
If a TLS endpoint receives a record larger than its advertised limit, it
must send a fatal record_overflow alert.

This patch fetches maximum support send size as specified by the record
size limit extension or as defined in GnuTLS, this value is then passed to
the kernel through setsockopt() using the new TLS_TX_MAX_PAYLOAD_LEN
option, such that the kernel can ensure outgoing records do not exceed the
size specified.

The respective kernel changes are currently applied to net-next [2].

[1] https://www.rfc-editor.org/rfc/rfc8449#section-4
[2] https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git/commit/?id=82cb5be6ad64198a3a028aeb49dcc7f6224d558a

Signed-off-by: Wilfred Mallawa <[email protected]>

Author: twilfredo

configure.ac

@@ -85,6 +85,9 @@ AC_CHECK_LIB([gnutls], [gnutls_get_system_config_file],
 AC_CHECK_LIB([gnutls], [gnutls_psk_allocate_client_credentials2],
              [AC_DEFINE([HAVE_GNUTLS_PSK_ALLOCATE_CREDENTIALS2], [1],
                         [Define to 1 if you have the gnutls_psk_allocate_client_credentials2 function.])])
+AC_CHECK_LIB([gnutls], [gnutls_record_get_max_send_size],
+             [AC_DEFINE([HAVE_GNUTLS_MAX_SEND_SIZE], [1],
+			[Define to 1 if you have the gnutls_record_get_max_send_size function.])])
 
 AC_MSG_CHECKING(for ML-DSA support in gnutls)
 AC_COMPILE_IFELSE(

src/tlshd/handshake.c

@@ -43,10 +43,23 @@
 #include <gnutls/abstract.h>
 
 #include <glib.h>
+#include <linux/tls.h>
 
 #include "tlshd.h"
 #include "netlink.h"
 
+#ifdef HAVE_GNUTLS_MAX_SEND_SIZE
+static void tlshd_set_record_size(gnutls_session_t session, uint16_t val)
+{
+	int ret;
+
+	ret = setsockopt(gnutls_transport_get_int(session), SOL_TLS,
+			 TLS_TX_MAX_PAYLOAD_LEN, &val, sizeof(val));
+	if (ret < 0)
+		tlshd_log_perror("setsockopt (TLS_RX_RECORD_SIZE_LIM)");
+}
+#endif
+
 /**
  * @brief Toggle the use of the Nagle algorithm
  * @param[in]     session  TLS session to modify
@@ -93,6 +106,9 @@ static void tlshd_save_nagle(gnutls_session_t session, int *saved)
 void tlshd_start_tls_handshake(gnutls_session_t session,
 			       struct tlshd_handshake_parms *parms)
 {
+#ifdef HAVE_GNUTLS_MAX_SEND_SIZE
+	uint16_t max_send_size;
+#endif
 	int saved, ret;
 	char *desc;
 
@@ -125,6 +141,10 @@ void tlshd_start_tls_handshake(gnutls_session_t session,
 	gnutls_free(desc);
 
 	parms->session_status = tlshd_initialize_ktls(session);
+#ifdef HAVE_GNUTLS_MAX_SEND_SIZE
+	max_send_size = gnutls_record_get_max_send_size(session);
+	tlshd_set_record_size(session, max_send_size);
+#endif
 }
 
 /**