Rework remote plugin status reporting

Redirected the remote plugin's SIGUSR1 status output to a dedicated
/run/audit/remote.state file, logging each metric on its own line and
noting glibc memory statistics when available.

Updated the audisp-remote manual to document the state report and list
its file in the FILES section.

Author: stevegrubb

audisp/plugins/remote/audisp-remote.8

@@ -1,26 +1,34 @@
-.TH AUDISP-REMOTE "8" "August 2018" "Red Hat" "System Administration Utilities"
+.TH AUDISP-REMOTE "8" "May 2024" "Red Hat" "System Administration Utilities"
 .SH NAME
 audisp-remote \- plugin for remote logging 
 .SH SYNOPSIS
 .B audisp-remote
 .SH DESCRIPTION
 \fBaudisp-remote\fP is a plugin for the audit event dispatcher that performs remote logging to an aggregate logging server.
+When the plugin is sent \fBSIGUSR1\fP, it writes a state report to \fBremote.state\fP.
 
 .SH TIPS
 If you are aggregating multiple machines, you should edit auditd.conf to set the name_format to something meaningful and the log_format to enriched. This way you can tell where the event came from and have the user name and groups resolved locally before it is sent off of the machine.
 
 .SH SIGNALS
 .TP
 SIGUSR1
-Causes the audisp-remote program to write the value of some of its internal flags to syslog. The
+Causes the audisp-remote program to write a state report to
+.B remote.state
+in
+.BR /run/audit .
+The
 .IR suspend
 flag tells whether or not logging has been suspended. The
 .IR remote_ended
 flag tells if the connection was broken by the server saying it can't log events. The
 .IR transport_ok
 flag tells whether or not the connection to the remote server is healthy. The
-.IR queue_size
-tells how many records are enqueued to be sent to the remote server.
+.IR queue_length
+tells how many records are enqueued to be sent to the remote server. The
+.IR max_queued_length
+shows the peak queue length since startup. The report also records glibc memory
+consumption when available.
 .TP
 SIGUSR2
 Causes the audisp-remote program to resume logging if it were suspended due to an error.
@@ -29,6 +37,7 @@ Causes the audisp-remote program to resume logging if it were suspended due to a
 /etc/audit/audisp-remote.conf
 /etc/audit/plugins.d/au-remote.conf
 /etc/audit/auditd.conf
+/run/audit/remote.state
 .SH "SEE ALSO"
 .BR auditd.conf (8),
 .BR auditd-plugins (5),

audisp/plugins/remote/audisp-remote.c

@@ -32,6 +32,9 @@
 #include <stdlib.h>
 #include <errno.h>
 #include <time.h>
+#ifdef HAVE_MALLINFO2
+#include <malloc.h>
+#endif
 #include <fcntl.h>
 #include <sys/select.h>
 #include <poll.h>
@@ -74,9 +77,14 @@ static volatile int remote_ended = 1, quiet = 0;
 static int ifd;
 remote_conf_t config;
 static int warned = 0;
+#ifdef HAVE_MALLINFO2
+static struct mallinfo2 last_mi;
+#endif
+static size_t max_queued_length = 0;
 
 /* Constants */
 static const char *SPOOL_FILE = "/var/spool/audit/remote.log";
+#define STATE_FILE AUDIT_RUN_DIR"/remote.state"
 
 /* Local function declarations */
 static int check_message(void);
@@ -147,22 +155,53 @@ static void reload_config(void)
 }
 
 /*
- * SIGSUR1 handler: dump stats
+ * SIGUSR1 handler: write a state report
  */
 static void user1_handler( int sig )
 {
         dump = 1;
 }
 
-static void dump_stats(struct queue *queue)
+#ifdef HAVE_MALLINFO2
+/* Write glibc memory statistics to FILE */
+static void write_memory_state(FILE *f)
+{
+	struct mallinfo2 mi = mallinfo2();
+
+	fprintf(f, "glibc arena (total memory) is: %zu KiB, was: %zu KiB\n",
+		(size_t)mi.arena/1024, (size_t)last_mi.arena/1024);
+	fprintf(f, "glibc uordblks (in use memory) is: %zu KiB, was: %zu KiB\n",
+		(size_t)mi.uordblks/1024,(size_t)last_mi.uordblks/1024);
+	fprintf(f,"glibc fordblks (total free space) is: %zu KiB, was: %zu KiB\n",
+		(size_t)mi.fordblks/1024,(size_t)last_mi.fordblks/1024);
+
+	memcpy(&last_mi, &mi, sizeof(struct mallinfo2));
+}
+#endif
+
+/* Write plugin state to STATE_FILE */
+static void write_state_report(struct queue *queue)
 {
-	syslog(LOG_INFO,
-		"suspend=%s, remote_ended=%s, transport_ok=%s, queued_items=%zu, queue_depth=%u",
-		suspend ? "yes" : "no",
-		remote_ended ? "yes" : "no",
-		transport_ok ? "yes" : "no",
-		q_queue_length(queue),
-		config.queue_depth);
+        char buf[64];
+        mode_t u = umask(0137); // allow 0640
+        FILE *f = fopen(STATE_FILE, "w");
+        umask(u);
+        if (f == NULL)
+                return;
+
+        time_t now = time(NULL);
+        strftime(buf, sizeof(buf), "%x %X", localtime(&now));
+        fprintf(f, "current_time = %s\n", buf);
+        fprintf(f, "suspend = %s\n", suspend ? "yes" : "no");
+        fprintf(f, "remote_ended = %s\n", remote_ended ? "yes" : "no");
+        fprintf(f, "transport_ok = %s\n", transport_ok ? "yes" : "no");
+        fprintf(f, "queue_length = %zu\n", q_queue_length(queue));
+        fprintf(f, "max_queued_length = %zu\n", max_queued_length);
+        fprintf(f, "queue_depth = %u\n", config.queue_depth);
+#ifdef HAVE_MALLINFO2
+	write_memory_state(f);
+#endif
+	fclose(f);
 	dump = 0;
 }
 
@@ -497,6 +536,7 @@ int main(int argc, char *argv[])
 		syslog(LOG_ERR, "Error initializing audit record queue: %m");
 		return 1;
 	}
+	max_queued_length = q_queue_length(queue);
 
 #ifdef HAVE_LIBCAP_NG
 	// Drop capabilities
@@ -521,7 +561,7 @@ int main(int argc, char *argv[])
 			reload_config();
 
 		if (dump)
-			dump_stats(queue);
+			write_state_report(queue);
 
 		/* Setup select flags */
 		FD_ZERO(&rfd);
@@ -607,6 +647,10 @@ int main(int argc, char *argv[])
 							do_overflow_action();
 						else
 							queue_error();
+					} else {
+						size_t len = q_queue_length(queue);
+						if (len > max_queued_length)
+							max_queued_length = len;
 					}
 				} else if (auplugin_fgets_eof())
 					stop = 1;