system_config: support built-in config files

Currently @include directive supports to reuse configuration files,
but no way to apply specific configuration files by default (without
specifying to).

In the previous versions, if you want to manage multiple
configuration files, you must use @include directive explicitly.

In this commit, add an option to specify the directory
(e.g. /etc/fluent/conf.d) which stores additional configuration files.
If there are such files under specified directory, they are loaded by
default without @include directive.

If you want to disable this feature, set empty string
for config_include_dir "".

Signed-off-by: Kentaro Hayashi <[email protected]>

Author: kenhys

lib/fluent/config/element.rb

@@ -71,6 +71,10 @@ def add_element(name, arg = '')
         e
       end
 
+      def merge_elements(elements)
+        @elements.concat(elements)
+      end
+
       def inspect
         attrs = super
         "name:#{@name}, arg:#{@arg}, " + attrs + ", " + @elements.inspect

lib/fluent/env.rb

@@ -24,6 +24,7 @@ module Fluent
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
+  DEFAULT_CONFIG_DIR = ENV['FLUENT_CONF_DIR'] || '/etc/fluent/conf.d'
   DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env
   DEFAULT_DIR_PERMISSION = 0755
   DEFAULT_FILE_PERMISSION = 0644

lib/fluent/supervisor.rb

@@ -17,6 +17,7 @@
 require 'fileutils'
 require 'open3'
 require 'pathname'
+require 'find'
 
 require 'fluent/config'
 require 'fluent/counter'
@@ -806,6 +807,53 @@ def configure(supervisor: false)
 
       $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
 
+      # merge additional configuration
+      config_type = if File.extname(@config_path) == '.yaml' || File.extname(@config_path) == '.yml'
+                      :yaml
+                    else
+                      nil
+                    end
+      if @system_config.config_include_dir&.empty?
+        $log.warn :supervisor, 'default configuration include directory was disabled'
+      else
+        begin
+          Find.find(@system_config.config_include_dir) do |path|
+            next if path == @system_config.config_include_dir
+            if config_type == :yaml
+              if File.extname(path) != '.yaml' && File.extname(path) != '.yml'
+                $log.warn :supervisor, 'skip loading configuration file', path: path
+                next
+              end
+              $log.info :supervisor, 'loading additional configuration file', path: path
+              # wrap additional YAML configuration to make it parsed correctly
+              buffer = "config:\n"
+              buffer << File.readlines(path).collect do |line|
+                "  " + line.chomp
+              end.join("\n")
+              Tempfile.open do |file|
+                file.puts(buffer)
+                file.rewind
+                elements = Fluent::Config::YamlParser.parse(file.path).elements
+                @conf.merge_elements(elements)
+              end
+            else
+              if File.extname(path) == '.yaml' || File.extname(path) == '.yml'
+                $log.warn :supervisor, 'skip loading configuration file', path: path
+                next
+              end
+              $log.info :supervisor, 'loading additional configuration file', path: path
+              elements = Fluent::Config.build(config_path: path,
+                                              encoding: @conf_encoding,
+                                              use_v1_config: @use_v1_config,
+                                              type: config_type).elements
+              @conf.merge_elements(elements)
+            end
+          end
+        rescue Errno::ENOENT
+          $log.warn :supervisor, 'no default configuration include directory was disabled', path: @system_config.config_include_dir
+        end
+      end
+
       @libs.each do |lib|
         require lib
       end

lib/fluent/system_config.rb

@@ -28,7 +28,8 @@ class SystemConfig
       :without_source, :with_source_only, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
       :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
-      :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer
+      :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit, :source_only_buffer,
+      :config_include_dir
     ]
 
     config_param :workers,   :integer, default: 1
@@ -58,6 +59,7 @@ class SystemConfig
     config_param :dir_permission, default: nil do |v|
       v.to_i(8)
     end
+    config_param :config_include_dir, default: ENV["FLUENT_CONF_DIR"] || '/etc/fluent/conf.d'
     config_section :log, required: false, init: true, multi: false do
       config_param :path, :string, default: nil
       config_param :format, :enum, list: [:text, :json], default: :text

test/command/test_fluentd.rb

@@ -1536,4 +1536,93 @@ def send_end(port)
       end_thread&.kill
     end
   end
+
+  sub_test_case "test additional configuration directory" do
+    setup do
+      omit "Not supported on Windows" if Fluent.windows?
+      FileUtils.mkdir_p(File.join(@tmp_dir, "conf.d"))
+    end
+
+    test "disable additional configuration directory" do
+      conf = <<CONF
+      <system>
+        config_include_dir ""
+      </system>
+CONF
+      conf_path = create_conf_file('disabled_config_include_dir.conf', conf)
+      assert_log_matches(create_cmdline(conf_path),
+                         "[warn]: default configuration include directory was disabled")
+    end
+
+    data("test with relative conf.d case" => true,
+         "test with full path conf.d case" => false)
+    test "additional configuration file (conf.d/child.conf) was loaded" do |relative|
+      conf_dir = relative ? "conf.d" : "#{@tmp_dir}/conf.d"
+      conf = <<CONF
+      <system>
+        config_include_dir #{conf_dir}
+      </system>
+CONF
+      conf_path = create_conf_file('parent.conf', conf)
+      create_conf_file('conf.d/child.conf', "")
+      assert_log_matches(create_cmdline(conf_path),
+                         "[info]: loading additional configuration file path=\"#{conf_dir}/child.conf\"")
+    end
+
+    test "skip loading different YAML configuration" do
+      conf = <<CONF
+      <system>
+        config_include_dir #{@tmp_dir}/conf.d
+      </system>
+CONF
+      conf_path = create_conf_file('parent.conf', conf)
+      FileUtils.mkdir_p(File.join(@tmp_dir, "conf.d"))
+      create_conf_file('conf.d/child.yml', "")
+      assert_log_matches(create_cmdline(conf_path),
+                         "[warn]: skip loading configuration file path=\"#{@tmp_dir}/conf.d/child.yml\"")
+    end
+  end
+
+  sub_test_case "test additional configuration directory (YAML)" do
+    setup do
+      omit "Not supported on Windows" if Fluent.windows?
+      FileUtils.mkdir_p(File.join(@tmp_dir, "conf.d"))
+    end
+
+    test "disable additional configuration directory" do
+      conf = <<CONF
+      system:
+        config_include_dir: ""
+CONF
+      conf_path = create_conf_file('disabled_config_include_dir.yml', conf)
+      assert_log_matches(create_cmdline(conf_path),
+                         "[warn]: default configuration include directory was disabled")
+    end
+
+    data("test with relative conf.d case" => true,
+         "test with full path conf.d case" => false)
+    test "additional relative configuration file (conf.d/child.yml) was loaded" do |relative|
+      conf_dir = relative ? "conf.d" : "#{@tmp_dir}/conf.d"
+      conf = <<CONF
+      system:
+        config_include_dir: #{conf_dir}
+CONF
+      conf_path = create_conf_file('parent.yml', conf)
+      create_conf_file('conf.d/child.yml', "")
+      assert_log_matches(create_cmdline(conf_path),
+                         "[info]: loading additional configuration file path=\"#{conf_dir}/child.yml\"")
+    end
+
+    test "skip loading different V1 conf configuration" do
+      conf = <<CONF
+      system:
+        config_include_dir: #{@tmp_dir}/conf.d
+CONF
+      conf_path = create_conf_file('parent.yml', conf)
+      create_conf_file('conf.d/child.conf', "")
+      assert_log_matches(create_cmdline(conf_path),
+                         "[warn]: skip loading configuration file path=\"#{@tmp_dir}/conf.d/child.conf\"")
+    end
+
+  end
 end