in_tail: make inotify file watch configurable at runtime (#3283)

Add a "inotify_watcher" option to the tail input to allow disabling inotify at runtime in favor of stat.

Signed-off-by: Zack Wine <[email protected]>

Author: zackwine

plugins/in_tail/CMakeLists.txt

@@ -3,9 +3,15 @@ set(src
   tail_dockermode.c
   tail_scan.c
   tail_config.c
-  tail_fs.c
+  tail_fs_stat.c
   tail.c)
 
+if(FLB_HAVE_INOTIFY)
+set(src
+    ${src}
+    tail_fs_inotify.c)
+endif()
+
 if(FLB_SQLDB)
 set(src
     ${src}

plugins/in_tail/tail.c

@@ -573,6 +573,13 @@ static struct flb_config_map config_map[] = {
      0, FLB_TRUE, offsetof(struct flb_tail_config, exit_on_eof),
      "exit Fluent Bit when reaching EOF on a monitored file."
     },
+#ifdef FLB_HAVE_INOTIFY
+    {
+     FLB_CONFIG_MAP_BOOL, "inotify_watcher", "true",
+     0, FLB_TRUE, offsetof(struct flb_tail_config, inotify_watcher),
+     "set to false to use file stat watcher instead of inotify."
+    },
+#endif
 #ifdef FLB_HAVE_REGEX
     {
      FLB_CONFIG_MAP_STR, "parser", NULL,

plugins/in_tail/tail_config.h

@@ -79,6 +79,9 @@ struct flb_tail_config {
     flb_sds_t key;             /* key for unstructured record  */
     int   skip_long_lines;     /* skip long lines              */
     int   exit_on_eof;         /* exit fluent-bit on EOF, test */
+#ifdef FLB_HAVE_INOTIFY
+    int   inotify_watcher;     /* enable/disable inotify monitor */
+#endif
     flb_sds_t offset_key;      /* key name of file offset      */
 
     /* Database */

plugins/in_tail/tail_file.c

@@ -751,7 +751,7 @@ int flb_tail_file_append(char *path, struct stat *st, int mode,
         mk_list_add(&file->_head, &ctx->files_event);
 
         /* Register this file into the fs_event monitoring */
-        ret = flb_tail_fs_add(file);
+        ret = flb_tail_fs_add(ctx, file);
         if (ret == -1) {
             flb_plg_error(ctx->ins, "could not register file into fs_events");
             goto error;
@@ -817,7 +817,7 @@ void flb_tail_file_remove(struct flb_tail_file *file)
     flb_sds_destroy(file->dmode_buf);
     flb_sds_destroy(file->dmode_lastline);
     mk_list_del(&file->_head);
-    flb_tail_fs_remove(file);
+    flb_tail_fs_remove(ctx, file);
     /* avoid deleting file with -1 fd */
     if (file->fd != -1) {
         close(file->fd);
@@ -1133,7 +1133,7 @@ int flb_tail_file_to_event(struct flb_tail_file *file)
     }
 
     /* Notify the fs-event handler that we will start monitoring this 'file' */
-    ret = flb_tail_fs_add(file);
+    ret = flb_tail_fs_add(ctx, file);
     if (ret == -1) {
         return -1;
     }

plugins/in_tail/tail_fs.h

@@ -27,12 +27,71 @@
 #include "tail_config.h"
 #include "tail_file_internal.h"
 
-int flb_tail_fs_init(struct flb_input_instance *in,
-                     struct flb_tail_config *ctx, struct flb_config *config);
-int flb_tail_fs_add(struct flb_tail_file *file);
-int flb_tail_fs_remove(struct flb_tail_file *file);
-int flb_tail_fs_exit(struct flb_tail_config *ctx);
-void flb_tail_fs_pause(struct flb_tail_config *ctx);
-void flb_tail_fs_resume(struct flb_tail_config *ctx);
+#include "tail_fs_stat.h"
+#ifdef FLB_HAVE_INOTIFY
+#include "tail_fs_inotify.h"
+#endif
+
+static inline int flb_tail_fs_init(struct flb_input_instance *in,
+                     struct flb_tail_config *ctx, struct flb_config *config)
+{
+#ifdef FLB_HAVE_INOTIFY
+    if (ctx->inotify_watcher) {
+        return flb_tail_fs_inotify_init(in, ctx, config);
+    }
+#endif
+    return flb_tail_fs_stat_init(in, ctx, config);
+}
+
+static inline void flb_tail_fs_pause(struct flb_tail_config *ctx)
+{
+#ifdef FLB_HAVE_INOTIFY
+    if (ctx->inotify_watcher) {
+        return flb_tail_fs_inotify_pause(ctx);
+    }
+#endif
+    return flb_tail_fs_stat_pause(ctx);
+}
+
+static inline void flb_tail_fs_resume(struct flb_tail_config *ctx)
+{
+#ifdef FLB_HAVE_INOTIFY
+    if (ctx->inotify_watcher) {
+        return flb_tail_fs_inotify_resume(ctx);
+    }
+#endif
+    return flb_tail_fs_stat_resume(ctx);
+}
+
+static inline int flb_tail_fs_add(struct flb_tail_config *ctx, struct flb_tail_file *file)
+{
+#ifdef FLB_HAVE_INOTIFY
+    if (ctx->inotify_watcher) {
+        return flb_tail_fs_inotify_add(file);
+    }
+#endif
+    return flb_tail_fs_stat_add(file);
+}
+
+static inline int flb_tail_fs_remove(struct flb_tail_config *ctx, struct flb_tail_file *file)
+{
+#ifdef FLB_HAVE_INOTIFY
+    if (ctx->inotify_watcher) {
+        return flb_tail_fs_inotify_remove(file);
+    }
+#endif
+    return flb_tail_fs_stat_remove(file);
+}
+
+static inline int flb_tail_fs_exit(struct flb_tail_config *ctx)
+{
+#ifdef FLB_HAVE_INOTIFY
+    if (ctx->inotify_watcher) {
+        return flb_tail_fs_inotify_exit(ctx);
+    }
+#endif
+    return flb_tail_fs_stat_exit(ctx);
+}
+
 
 #endif

plugins/in_tail/tail_fs_inotify.c

@@ -203,7 +203,7 @@ static int tail_fs_event(struct flb_input_instance *ins,
 
         /* A rotated file must be re-registered */
         flb_tail_file_rotated(file);
-        flb_tail_fs_remove(file);
+        flb_tail_fs_remove(ctx, file);
         flb_tail_fs_add_rotated(file);
     }
 
@@ -291,12 +291,14 @@ static int tail_fs_event(struct flb_input_instance *ins,
 }
 
 /* File System events based on Inotify(2). Linux >= 2.6.32 is suggested */
-int flb_tail_fs_init(struct flb_input_instance *in,
+int flb_tail_fs_inotify_init(struct flb_input_instance *in,
                      struct flb_tail_config *ctx, struct flb_config *config)
 {
     int fd;
     int ret;
 
+    flb_plg_debug(ctx->ins, "flb_tail_fs_inotify_init() initializing inotify tail input");
+
     /* Create inotify instance */
     fd = inotify_init1(IN_NONBLOCK | IN_CLOEXEC);
     if (fd == -1) {
@@ -318,17 +320,17 @@ int flb_tail_fs_init(struct flb_input_instance *in,
     return 0;
 }
 
-void flb_tail_fs_pause(struct flb_tail_config *ctx)
+void flb_tail_fs_inotify_pause(struct flb_tail_config *ctx)
 {
     flb_input_collector_pause(ctx->coll_fd_fs1, ctx->ins);
 }
 
-void flb_tail_fs_resume(struct flb_tail_config *ctx)
+void flb_tail_fs_inotify_resume(struct flb_tail_config *ctx)
 {
     flb_input_collector_resume(ctx->coll_fd_fs1, ctx->ins);
 }
 
-int flb_tail_fs_add(struct flb_tail_file *file)
+int flb_tail_fs_inotify_add(struct flb_tail_file *file)
 {
     int ret;
     struct flb_tail_config *ctx = file->config;
@@ -343,7 +345,7 @@ int flb_tail_fs_add(struct flb_tail_file *file)
     return 0;
 }
 
-int flb_tail_fs_remove(struct flb_tail_file *file)
+int flb_tail_fs_inotify_remove(struct flb_tail_file *file)
 {
     struct flb_tail_config *ctx = file->config;
 
@@ -359,7 +361,7 @@ int flb_tail_fs_remove(struct flb_tail_file *file)
     return 0;
 }
 
-int flb_tail_fs_exit(struct flb_tail_config *ctx)
+int flb_tail_fs_inotify_exit(struct flb_tail_config *ctx)
 {
     (void) ctx;
     return 0;

plugins/in_tail/tail_fs_inotify.h

@@ -0,0 +1,38 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019-2021 The Fluent Bit Authors
+ *  Copyright (C) 2015-2018 Treasure Data Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_TAIL_FS_INOTIFY_H
+#define FLB_TAIL_FS_INOTIFY_H
+
+#include <fluent-bit/flb_info.h>
+#include <fluent-bit/flb_input.h>
+
+#include "tail_config.h"
+#include "tail_file_internal.h"
+
+int flb_tail_fs_inotify_init(struct flb_input_instance *in,
+                          struct flb_tail_config *ctx, struct flb_config *config);
+int flb_tail_fs_inotify_add(struct flb_tail_file *file);
+int flb_tail_fs_inotify_remove(struct flb_tail_file *file);
+int flb_tail_fs_inotify_exit(struct flb_tail_config *ctx);
+void flb_tail_fs_inotify_pause(struct flb_tail_config *ctx);
+void flb_tail_fs_inotify_resume(struct flb_tail_config *ctx);
+
+#endif

plugins/in_tail/tail_fs_stat.c

@@ -178,11 +178,13 @@ static int tail_fs_check(struct flb_input_instance *ins,
 }
 
 /* File System events based on stat(2) */
-int flb_tail_fs_init(struct flb_input_instance *in,
-                     struct flb_tail_config *ctx, struct flb_config *config)
+int flb_tail_fs_stat_init(struct flb_input_instance *in,
+                          struct flb_tail_config *ctx, struct flb_config *config)
 {
     int ret;
 
+    flb_plg_debug(ctx->ins, "flb_tail_fs_stat_init() initializing stat tail input");
+
     /* Set a manual timer to collect events every 0.250 seconds */
     ret = flb_input_set_collector_time(in, tail_fs_event,
                                        0, 250000000, config);
@@ -202,19 +204,19 @@ int flb_tail_fs_init(struct flb_input_instance *in,
     return 0;
 }
 
-void flb_tail_fs_pause(struct flb_tail_config *ctx)
+void flb_tail_fs_stat_pause(struct flb_tail_config *ctx)
 {
     flb_input_collector_pause(ctx->coll_fd_fs1, ctx->ins);
     flb_input_collector_pause(ctx->coll_fd_fs2, ctx->ins);
 }
 
-void flb_tail_fs_resume(struct flb_tail_config *ctx)
+void flb_tail_fs_stat_resume(struct flb_tail_config *ctx)
 {
     flb_input_collector_resume(ctx->coll_fd_fs1, ctx->ins);
     flb_input_collector_resume(ctx->coll_fd_fs2, ctx->ins);
 }
 
-int flb_tail_fs_add(struct flb_tail_file *file)
+int flb_tail_fs_stat_add(struct flb_tail_file *file)
 {
     int ret;
     struct fs_stat *fst;
@@ -237,15 +239,15 @@ int flb_tail_fs_add(struct flb_tail_file *file)
     return 0;
 }
 
-int flb_tail_fs_remove(struct flb_tail_file *file)
+int flb_tail_fs_stat_remove(struct flb_tail_file *file)
 {
     if (file->tail_mode == FLB_TAIL_EVENT) {
         flb_free(file->fs_backend);
     }
     return 0;
 }
 
-int flb_tail_fs_exit(struct flb_tail_config *ctx)
+int flb_tail_fs_stat_exit(struct flb_tail_config *ctx)
 {
     (void) ctx;
     return 0;

plugins/in_tail/tail_fs.c renamed to plugins/in_tail/tail_fs_stat.h

@@ -18,10 +18,21 @@
  *  limitations under the License.
  */
 
+#ifndef FLB_TAIL_FS_STAT_H
+#define FLB_TAIL_FS_STAT_H
+
 #include <fluent-bit/flb_info.h>
+#include <fluent-bit/flb_input.h>
+
+#include "tail_config.h"
+#include "tail_file_internal.h"
+
+int flb_tail_fs_stat_init(struct flb_input_instance *in,
+                          struct flb_tail_config *ctx, struct flb_config *config);
+int flb_tail_fs_stat_add(struct flb_tail_file *file);
+int flb_tail_fs_stat_remove(struct flb_tail_file *file);
+int flb_tail_fs_stat_exit(struct flb_tail_config *ctx);
+void flb_tail_fs_stat_pause(struct flb_tail_config *ctx);
+void flb_tail_fs_stat_resume(struct flb_tail_config *ctx);
 
-#ifdef FLB_HAVE_INOTIFY
-    #include "tail_fs_inotify.c"
-#else
-    #include "tail_fs_stat.c"
 #endif