statd: compress journal snapshots with gzip

Store historical snapshots as compressed .json.gz files to reduce disk
usage. The operational.json file remains uncompressed for easy access.

Signed-off-by: Richard Alpe <[email protected]>

Author: rical

src/statd/Makefile.am

@@ -8,7 +8,7 @@ statd_CFLAGS        = -W -Wall -Wextra
 statd_CFLAGS       += $(jansson_CFLAGS) $(libyang_CFLAGS) $(sysrepo_CFLAGS)
 statd_CFLAGS       += $(libsrx_CFLAGS) $(libite_CFLAGS)
 statd_LDADD         = $(jansson_LIBS) $(libyang_LIBS) $(sysrepo_LIBS)
-statd_LDADD        += $(libsrx_LIBS) $(libite_LIBS) $(EV_LIBS)
+statd_LDADD        += $(libsrx_LIBS) $(libite_LIBS) $(EV_LIBS) -lz
 
 # Test stub for journal retention policy (no dependencies, standalone)
 noinst_PROGRAMS     = journal_retention_stub

src/statd/journal.c

@@ -11,6 +11,7 @@
 #include <sys/stat.h>
 #include <pthread.h>
 #include <dirent.h>
+#include <zlib.h>
 
 #include <srx/common.h>
 
@@ -30,40 +31,73 @@ static void get_timestamp_filename(char *buf, size_t len, time_t ts)
 {
 	struct tm *tm = gmtime(&ts);
 
-	snprintf(buf, len, "%04d%02d%02d-%02d%02d%02d.json",
+	snprintf(buf, len, "%04d%02d%02d-%02d%02d%02d.json.gz",
 		 tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday,
 		 tm->tm_hour, tm->tm_min, tm->tm_sec);
 }
 
-/* Create timestamped snapshot and update operational.json symlink */
+/* Compress a file using gzip */
+static int gzip_file(const char *src, const char *dst)
+{
+	FILE *in;
+	gzFile gz;
+	char buf[4096];
+	size_t n;
+
+	in = fopen(src, "r");
+	if (!in) {
+		ERROR("Error, opening %s: %s", src, strerror(errno));
+		return -1;
+	}
+
+	gz = gzopen(dst, "wb");
+	if (!gz) {
+		ERROR("Error, opening %s: %s", dst, strerror(errno));
+		fclose(in);
+		return -1;
+	}
+
+	while ((n = fread(buf, 1, sizeof(buf), in)) > 0) {
+		if (gzwrite(gz, buf, n) != (int)n) {
+			ERROR("Error, writing to %s", dst);
+			gzclose(gz);
+			fclose(in);
+			unlink(dst);
+			return -1;
+		}
+	}
+
+	gzclose(gz);
+	fclose(in);
+	return 0;
+}
+
+/* Create timestamped snapshot and update operational.json */
 static int create_snapshot(const struct lyd_node *tree)
 {
 	char timestamp_file[300];
 	char timestamp_path[512];
 	time_t now;
 	int ret;
 
+	/* Write latest snapshot as uncompressed operational.json for easy access */
+	ret = lyd_print_path(DUMP_FILE, tree, LYD_JSON, LYD_PRINT_WITHSIBLINGS);
+	if (ret != LY_SUCCESS) {
+		ERROR("Error, writing operational.json: %d", ret);
+		return -1;
+	}
+
+	/* Compress operational.json to timestamped archive */
 	now = time(NULL);
 	get_timestamp_filename(timestamp_file, sizeof(timestamp_file), now);
 	snprintf(timestamp_path, sizeof(timestamp_path), "%s/%s",
 		 JOURNAL_DIR, timestamp_file);
 
-	/* Write timestamped snapshot directly */
-	ret = lyd_print_path(timestamp_path, tree, LYD_JSON,
-			     LYD_PRINT_WITHSIBLINGS);
-	if (ret != LY_SUCCESS) {
-		ERROR("Error, writing snapshot %s: %d", timestamp_file, ret);
-		unlink(timestamp_path);  /* Clean up incomplete file */
+	if (gzip_file(DUMP_FILE, timestamp_path) != 0) {
+		ERROR("Error, compressing snapshot to %s", timestamp_file);
 		return -1;
 	}
 
-	/* Update operational.json symlink to point to latest */
-	unlink(DUMP_FILE);  /* Remove old symlink/file */
-	if (symlink(timestamp_file, DUMP_FILE) != 0) {
-		ERROR("Error, creating symlink to %s: %s",
-		      timestamp_file, strerror(errno));
-	}
-
 	DEBUG("Created snapshot %s", timestamp_file);
 	return 0;
 }

src/statd/journal_retention.c

@@ -30,7 +30,7 @@ static int parse_timestamp_filename(const char *filename, time_t *ts)
 	struct tm tm = {0};
 	int year, mon, day, hour, min, sec;
 
-	if (sscanf(filename, "%4d%2d%2d-%2d%2d%2d.json",
+	if (sscanf(filename, "%4d%2d%2d-%2d%2d%2d.json.gz",
 		   &year, &mon, &day, &hour, &min, &sec) != 6)
 		return -1;
 
@@ -76,7 +76,7 @@ int journal_scan_snapshots(const char *dir, struct snapshot **out_snapshots, int
 
 		if (strcmp(entry->d_name, "operational.json") == 0)
 			continue;
-		if (!strstr(entry->d_name, ".json"))
+		if (!strstr(entry->d_name, ".json.gz"))
 			continue;
 
 		if (parse_timestamp_filename(entry->d_name, &ts) != 0)

test/case/statd/journal-retention/test.py

@@ -23,14 +23,14 @@
 def create_snapshot(test_dir, timestamp):
     """Create an empty snapshot file with the given timestamp"""
     dt = datetime.utcfromtimestamp(timestamp)
-    filename = dt.strftime("%Y%m%d-%H%M%S.json")
+    filename = dt.strftime("%Y%m%d-%H%M%S.json.gz")
     path = os.path.join(test_dir, filename)
     open(path, 'w').close()
     return filename
 
 def count_snapshots(test_dir):
-    """Count JSON snapshot files in directory"""
-    return len([f for f in os.listdir(test_dir) if f.endswith('.json')])
+    """Count compressed snapshot files in directory"""
+    return len([f for f in os.listdir(test_dir) if f.endswith('.json.gz')])
 
 def count_snapshots_by_age(test_dir, now):
     """Count snapshots by age bucket"""
@@ -50,12 +50,12 @@ def count_snapshots_by_age(test_dir, now):
     }
 
     for filename in os.listdir(test_dir):
-        if not filename.endswith('.json'):
+        if not filename.endswith('.json.gz'):
             continue
 
-        # Parse timestamp from filename (YYYYMMDD-HHMMSS.json)
+        # Parse timestamp from filename (YYYYMMDD-HHMMSS.json.gz)
         try:
-            ts_str = filename.replace('.json', '')
+            ts_str = filename.replace('.json.gz', '')
             dt = datetime.strptime(ts_str, "%Y%m%d-%H%M%S")
             ts = int(dt.timestamp())
             age = now - ts