Skip to content

Commit 02460ea

Browse files
SagiROostoSagiROosto
committed
out_logrotate: add hashtable to file paths
Signed-off-by: SagiROosto <[email protected]>
1 parent 92dd67d commit 02460ea

File tree

1 file changed

+82
-19
lines changed

1 file changed

+82
-19
lines changed

‎plugins/out_logrotate/logrotate.c‎

Lines changed: 82 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include <fluent-bit/flb_log_event_decoder.h>
2727
#include <fluent-bit/flb_gzip.h>
2828
#include <fluent-bit/flb_str.h>
29+
#include <fluent-bit/flb_hash_table.h>
2930
#include <msgpack.h>
3031

3132
#include <stdio.h>
@@ -79,7 +80,7 @@ struct flb_logrotate_conf {
7980
size_t max_size; /* Max file size */
8081
int max_files; /* Maximum number of rotated files to keep */
8182
int gzip; /* Whether to gzip rotated files */
82-
size_t current_file_size; /* Current file size in bytes */
83+
struct flb_hash_table *file_sizes; /* Hash table to store file size per filename */
8384
struct flb_output_instance *ins;
8485
};
8586

@@ -123,10 +124,20 @@ static int cb_logrotate_init(struct flb_output_instance *ins,
123124
ctx->delimiter = NULL;
124125
ctx->label_delimiter = NULL;
125126
ctx->template = NULL;
126-
ctx->current_file_size = 0; /* Initialize file size counter */
127+
128+
/* Initialize hash table to store file sizes per filename */
129+
ctx->file_sizes = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 64, 0);
130+
if (!ctx->file_sizes) {
131+
flb_errno();
132+
flb_free(ctx);
133+
return -1;
134+
}
127135

128136
ret = flb_output_config_map_set(ins, (void *) ctx);
129137
if (ret == -1) {
138+
if (ctx->file_sizes) {
139+
flb_hash_table_destroy(ctx->file_sizes);
140+
}
130141
flb_free(ctx);
131142
return -1;
132143
}
@@ -162,6 +173,9 @@ static int cb_logrotate_init(struct flb_output_instance *ins,
162173
}
163174
else {
164175
flb_plg_error(ctx->ins, "unknown format %s. abort.", tmp);
176+
if (ctx->file_sizes) {
177+
flb_hash_table_destroy(ctx->file_sizes);
178+
}
165179
flb_free(ctx);
166180
return -1;
167181
}
@@ -514,25 +528,50 @@ static int mkpath(struct flb_output_instance *ins, const char *dir)
514528
#endif
515529
}
516530

517-
/* Function to check if file size exceeds max size in MB */
518-
static int should_rotate_file(struct flb_logrotate_conf *ctx)
531+
/* Function to check if file size exceeds max size for a specific file */
532+
static int should_rotate_file(struct flb_logrotate_conf *ctx, const char *filename)
519533
{
520-
if (ctx->current_file_size >= ctx->max_size) {
521-
flb_plg_info(ctx->ins, "going to rotate file: current size=%zu max size=%zu", ctx->current_file_size, ctx->max_size);
534+
size_t file_size = 0;
535+
void *out_buf;
536+
size_t out_size;
537+
int ret;
538+
539+
/* Get file size from hash table */
540+
ret = flb_hash_table_get(ctx->file_sizes, filename, strlen(filename),
541+
&out_buf, &out_size);
542+
if (ret == 0 && out_size == sizeof(size_t)) {
543+
file_size = *(size_t *)out_buf;
544+
}
545+
546+
if (file_size >= ctx->max_size) {
547+
flb_plg_info(ctx->ins, "going to rotate file %s: current size=%zu max size=%zu",
548+
filename, file_size, ctx->max_size);
522549
return 1;
523550
}
524551
else {
525-
flb_plg_debug(ctx->ins, "file should not be rotated: current size=%zu max size=%zu", ctx->current_file_size, ctx->max_size);
552+
flb_plg_debug(ctx->ins, "file %s should not be rotated: current size=%zu max size=%zu",
553+
filename, file_size, ctx->max_size);
526554
return 0;
527555
}
528556
}
529557

530-
/* Function to update file size counter using current file position */
531-
static void update_file_size_counter(struct flb_logrotate_conf *ctx, FILE *fp)
558+
/* Function to update file size counter for a specific file */
559+
static void update_file_size_counter(struct flb_logrotate_conf *ctx,
560+
const char *filename, FILE *fp)
532561
{
533562
struct stat st;
563+
size_t file_size;
564+
int ret;
565+
534566
if (fstat(fileno(fp), &st) == 0 && st.st_size >= 0) {
535-
ctx->current_file_size = (size_t) st.st_size;
567+
file_size = (size_t) st.st_size;
568+
569+
/* Store or update file size in hash table */
570+
ret = flb_hash_table_add(ctx->file_sizes, filename, strlen(filename),
571+
&file_size, sizeof(size_t));
572+
if (ret == -1) {
573+
flb_plg_warn(ctx->ins, "failed to update file size for %s", filename);
574+
}
536575
}
537576
}
538577

@@ -711,11 +750,21 @@ static int rotate_file(struct flb_logrotate_conf *ctx, const char *filename)
711750
char timestamp[32];
712751
char rotated_filename[PATH_MAX];
713752
char gzip_filename[PATH_MAX];
753+
size_t file_size = 0;
754+
void *out_buf;
755+
size_t out_size;
714756
int ret = 0;
715757

758+
/* Get file size from hash table for logging */
759+
ret = flb_hash_table_get(ctx->file_sizes, filename, strlen(filename),
760+
&out_buf, &out_size);
761+
if (ret == 0 && out_size == sizeof(size_t)) {
762+
file_size = *(size_t *)out_buf;
763+
}
764+
716765
/* Log rotation event */
717766
flb_plg_info(ctx->ins, "rotating file: %s (current size: %zu bytes)",
718-
filename, ctx->current_file_size);
767+
filename, file_size);
719768

720769
/* Generate timestamp */
721770
generate_timestamp(timestamp, sizeof(timestamp));
@@ -942,7 +991,7 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk,
942991
}
943992

944993
/* Check if file needs rotation based on current size counter */
945-
if (should_rotate_file(ctx)) {
994+
if (should_rotate_file(ctx, out_file)) {
946995
have_directory = false;
947996
directory[0] = '\0';
948997
/* Extract directory and base filename for cleanup */
@@ -971,8 +1020,8 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk,
9711020

9721021
/* Rotate the file */
9731022
if (rotate_file(ctx, out_file) == 0) {
974-
/* Reset file size counter after rotation */
975-
ctx->current_file_size = 0;
1023+
/* Remove file size entry from hash table after rotation */
1024+
flb_hash_table_del(ctx->file_sizes, out_file);
9761025
/* Clean up old rotated files */
9771026
if (have_directory) {
9781027
cleanup_old_files(ctx, directory, base_filename);
@@ -1004,8 +1053,15 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk,
10041053
}
10051054

10061055
/* Initialize file size counter if this is a new file */
1007-
if (ctx->current_file_size == 0) {
1008-
update_file_size_counter(ctx, fp);
1056+
{
1057+
void *out_buf;
1058+
size_t out_size;
1059+
int ret = flb_hash_table_get(ctx->file_sizes, out_file, strlen(out_file),
1060+
&out_buf, &out_size);
1061+
if (ret != 0) {
1062+
/* File not in hash table, initialize it */
1063+
update_file_size_counter(ctx, out_file, fp);
1064+
}
10091065
}
10101066

10111067
/*
@@ -1018,6 +1074,8 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk,
10181074
if (event_chunk->type == FLB_INPUT_METRICS) {
10191075
print_metrics_text(ctx->ins, fp,
10201076
event_chunk->data, event_chunk->size);
1077+
/* Update file size counter */
1078+
update_file_size_counter(ctx, out_file, fp);
10211079
fclose(fp);
10221080
FLB_OUTPUT_RETURN(FLB_OK);
10231081
}
@@ -1042,7 +1100,7 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk,
10421100
} while (total < event_chunk->size);
10431101

10441102
/* Update file size counter */
1045-
update_file_size_counter(ctx, fp);
1103+
update_file_size_counter(ctx, out_file, fp);
10461104
fclose(fp);
10471105
FLB_OUTPUT_RETURN(FLB_OK);
10481106
}
@@ -1104,7 +1162,7 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk,
11041162
log_event.body, ctx);
11051163
break;
11061164
case FLB_OUT_LOGROTATE_FMT_PLAIN:
1107-
plain_output(fp, log_event.body, alloc_size);
1165+
plain_output(fp, log_event.body, alloc_size, config->json_escape_unicode);
11081166
break;
11091167
case FLB_OUT_LOGROTATE_FMT_TEMPLATE:
11101168
template_output(fp,
@@ -1117,7 +1175,7 @@ static void cb_logrotate_flush(struct flb_event_chunk *event_chunk,
11171175
flb_log_event_decoder_destroy(&log_decoder);
11181176

11191177
/* Update file size counter */
1120-
update_file_size_counter(ctx, fp);
1178+
update_file_size_counter(ctx, out_file, fp);
11211179
fclose(fp);
11221180

11231181
FLB_OUTPUT_RETURN(FLB_OK);
@@ -1131,6 +1189,11 @@ static int cb_logrotate_exit(void *data, struct flb_config *config)
11311189
return 0;
11321190
}
11331191

1192+
/* Destroy hash table */
1193+
if (ctx->file_sizes) {
1194+
flb_hash_table_destroy(ctx->file_sizes);
1195+
}
1196+
11341197
flb_free(ctx);
11351198
return 0;
11361199
}

0 commit comments

Comments
 (0)