in_tail: permission error handling

plugins/in_tail/tail.c

@@ -719,11 +719,18 @@ static struct flb_config_map config_map[] = {
      0, FLB_TRUE, offsetof(struct flb_tail_config, skip_empty_lines),
      "Allows to skip empty lines."
     },
-
     {
-      FLB_CONFIG_MAP_BOOL, "truncate_long_lines", "false",
-      0, FLB_TRUE, offsetof(struct flb_tail_config, truncate_long_lines),
-      "Truncate overlong lines after input encoding to UTF-8"
+     FLB_CONFIG_MAP_BOOL, "skip_permission_errors", "false",
+     0, FLB_TRUE, offsetof(struct flb_tail_config, skip_permission_errors),
+     "Skip directories with permission errors instead of failing. When enabled, "
+     "the plugin will continue processing accessible directories even if some directories "
+     "cannot be read due to permission issues. When disabled (default), any permission error will "
+     "cause the plugin to fail entirely."
+    },
+    {
+     FLB_CONFIG_MAP_BOOL, "truncate_long_lines", "false",
+     0, FLB_TRUE, offsetof(struct flb_tail_config, truncate_long_lines),
+     "Truncate overlong lines after input encoding to UTF-8"
     },
 #ifdef __linux__
     {

plugins/in_tail/tail_config.h

@@ -160,6 +160,9 @@ struct flb_tail_config {
     /* List of shell patterns used to exclude certain file names */
     struct mk_list *exclude_list;
 
+    /* Permission handling configuration */
+    int skip_permission_errors; /* skip directories with permission errors (1), or fail (0) */
+
     /* Plugin input instance */
     struct flb_input_instance *ins;
 

plugins/in_tail/tail_scan_glob.c

@@ -21,6 +21,7 @@
 #include <sys/stat.h>
 #include <glob.h>
 #include <fnmatch.h>
+#include <errno.h>
 
 #include <fluent-bit/flb_compat.h>
 #include <fluent-bit/flb_input_plugin.h>
@@ -133,14 +134,27 @@ static int tail_is_excluded(char *path, struct flb_tail_config *ctx)
     return FLB_FALSE;
 }
 
+static int glob_errfunc(const char *epath, int eerrno)
+{
+    (void) epath;
+
+    switch (eerrno) {
+    case EACCES:
+    case ENOENT:
+    case EPERM:
+        return 0;
+    default:
+        return 1;
+    }
+}
+
 static inline int do_glob(const char *pattern, int flags,
-                          void *not_used, glob_t *pglob)
+                          int (*errfunc)(const char *, int), glob_t *pglob)
 {
     int ret;
     int new_flags;
     char *tmp = NULL;
     int tmp_needs_free = FLB_FALSE;
-    (void) not_used;
 
     /* Save current values */
     new_flags = flags;
@@ -171,7 +185,7 @@ static inline int do_glob(const char *pattern, int flags,
     }
 
     /* invoke glob with new parameters */
-    ret = glob(pattern, new_flags, NULL, pglob);
+    ret = glob(pattern, new_flags, errfunc, pglob);
 
     /* remove temporary buffer, if allocated by expand_tilde above.
      * Note that this buffer is only used for libc implementations
@@ -195,6 +209,7 @@ static int tail_scan_path(const char *path, struct flb_tail_config *ctx)
     int64_t mtime;
     struct stat st;
     ssize_t ignored_file_size;
+    int (*errfunc)(const char *, int) = NULL;
 
     ignored_file_size = -1;
 
@@ -203,8 +218,18 @@ static int tail_scan_path(const char *path, struct flb_tail_config *ctx)
     /* Safe reset for globfree() */
     globbuf.gl_pathv = NULL;
 
-    /* Scan the given path */
-    ret = do_glob(path, GLOB_TILDE | GLOB_ERR, NULL, &globbuf);
+    if (ctx->skip_permission_errors) {
+        errfunc = glob_errfunc;
+    }
+
+    /* Scan the given path with error checking enabled. */
+    ret = do_glob(path, GLOB_TILDE | GLOB_ERR, errfunc, &globbuf);
+    if (ret == GLOB_ABORTED && ctx->skip_permission_errors) {
+        flb_plg_warn(ctx->ins, "read error, check permissions: %s", path);
+        globfree(&globbuf);
+        ret = do_glob(path, GLOB_TILDE, NULL, &globbuf);
+    }
+
     if (ret != 0) {
         switch (ret) {
         case GLOB_NOSPACE:

tests/runtime/in_tail.c

@@ -1496,6 +1496,119 @@ void flb_test_path_key()
     test_tail_ctx_destroy(ctx);
 }
 
+void flb_test_in_tail_skip_permission_errors()
+{
+#ifndef _WIN32
+    struct flb_lib_out_cb cb_data;
+    flb_ctx_t *ctx = NULL;
+    char tmpdir_template[] = "flb-tail-glob-XXXXXX";
+    char *tmpdir;
+    char ok_dir[PATH_MAX];
+    char bad_dir[PATH_MAX];
+    char ok_file[PATH_MAX];
+    char pattern[PATH_MAX];
+    int in_ffd;
+    int out_ffd;
+    int ret;
+    int num = 0;
+    int unused;
+    int fd = -1;
+    int started = FLB_FALSE;
+
+    clear_output_num();
+
+    tmpdir = mkdtemp(tmpdir_template);
+    if (!TEST_CHECK(tmpdir != NULL)) {
+        TEST_MSG("mkdtemp failed");
+        return;
+    }
+
+    snprintf(ok_dir, sizeof(ok_dir), "%s/ok", tmpdir);
+    snprintf(bad_dir, sizeof(bad_dir), "%s/noaccess", tmpdir);
+
+    ret = mkdir(ok_dir, 0700);
+    TEST_CHECK(ret == 0);
+
+    ret = mkdir(bad_dir, 0700);
+    TEST_CHECK(ret == 0);
+
+    snprintf(ok_file, sizeof(ok_file), "%s/test.log", ok_dir);
+    fd = open(ok_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
+    TEST_CHECK(fd >= 0);
+    if (fd >= 0) {
+        ret = write(fd, "hello\n", 6);
+        TEST_CHECK(ret == 6);
+        fsync(fd);
+        close(fd);
+        fd = -1;
+    }
+
+    ret = chmod(bad_dir, 0000);
+    TEST_CHECK(ret == 0);
+
+    cb_data.cb = cb_count_msgpack;
+    cb_data.data = &unused;
+
+    ctx = flb_create();
+    TEST_CHECK(ctx != NULL);
+    if (ctx == NULL) {
+        goto cleanup;
+    }
+
+    ret = flb_service_set(ctx,
+                          "Flush", "0.5",
+                          "Grace", "1",
+                          "Log_Level", "info",
+                          NULL);
+    TEST_CHECK(ret == 0);
+
+    in_ffd = flb_input(ctx, (char *) "tail", NULL);
+    TEST_CHECK(in_ffd >= 0);
+
+    snprintf(pattern, sizeof(pattern), "%s/*/*.log", tmpdir);
+    ret = flb_input_set(ctx, in_ffd,
+                        "path", pattern,
+                        "read_from_head", "true",
+                        "skip_permission_errors", "true",
+                        NULL);
+    TEST_CHECK(ret == 0);
+
+    out_ffd = flb_output(ctx, (char *) "lib", &cb_data);
+    TEST_CHECK(out_ffd >= 0);
+    TEST_CHECK(flb_output_set(ctx, out_ffd, NULL) == 0);
+
+    ret = flb_start(ctx);
+    TEST_CHECK(ret == 0);
+    if (ret == 0) {
+        started = FLB_TRUE;
+    }
+
+    wait_num_with_timeout(5000, &num);
+    num = get_output_num();
+    if (!TEST_CHECK(num > 0))  {
+        TEST_MSG("expected output with skip_permission_errors; got=%d", num);
+    }
+
+cleanup:
+    if (ctx != NULL) {
+        if (started == FLB_TRUE) {
+            flb_stop(ctx);
+        }
+        flb_destroy(ctx);
+    }
+    if (fd >= 0) {
+        close(fd);
+    }
+    chmod(bad_dir, 0700);
+    unlink(ok_file);
+    rmdir(ok_dir);
+    rmdir(bad_dir);
+    rmdir(tmpdir);
+#else
+    TEST_MSG("skip on Windows");
+#endif
+}
+
 void flb_test_exclude_path()
 {
     struct flb_lib_out_cb cb_data;
@@ -2660,6 +2773,7 @@ TEST_LIST = {
     {"truncate_long_lines_utf8",     flb_test_in_tail_truncate_long_lines_utf8},
     {"path_comma", flb_test_path_comma},
     {"path_key", flb_test_path_key},
+    {"skip_permission_errors", flb_test_in_tail_skip_permission_errors},
     {"exclude_path", flb_test_exclude_path},
     {"offset_key", flb_test_offset_key},
     {"multiline_offset_key", flb_test_multiline_offset_key},