fluentd-example-directives.conf

### ########################################################################################
### Visit this website if in double
### https://medium.com/@alluri.prithvi/application-logs-from-kubernetes-to-s3-and-elasticsearch-using-fluentd-2f1b09a9872e
### ########################################################################################

@include systemd.conf
@include kubernetes.conf

## App 1 config 
<source>
  @type tail
  @id XXXX ### Uniq ID
  path XXXX ### Log path
  pos_file XXXX ### pos file
  tag XXXXX ### App tag 
  read_from_head true
  <parse>
    @type regexp
    keep_time_key true
    expression (?<logtime>.*?) - (?<names>[^ ]*) - (?<log_level>[^ ]*) - (?<message>.*)
    time_format %Y/%m/%d %H:%M:%S,%6N%z
  </parse>
</source>

## App 2 config

<source>
  @type tail
  @id XXXXX ## Uniq ID
  path XXXX ### Log path 
  pos_file XXXX ### pos file   
  tag XXXXX ## App tag 
  read_from_head true
  <parse>
    @type regexp
    keep_time_key true
    expression (?<logtime>.*?) - (?<names>[^ ]*) - (?<log_level>[^ ]*) - (?<message>.*)
    time_format %Y/%m/%d %H:%M:%S,%6N%z
  </parse>
</source>

# Store Data in Elasticsearch and S3
<match **>
  @type copy
  deep_copy true
  <store>

@type s3
  aws_key_id XXXXXXXXXXXXXXXXX # Access Key 
  aws_sec_key XXXXXXXXXXXXXXXXXXXXXX # Secret Key 
  s3_bucket XXXXXX # S3 Bucket
  s3_region XXXXX # Bucket Region 
  path "#{ENV['S3_LOGS_BUCKET_PREFIX']}"
  buffer_path /var/log/fluent/s3
  s3_object_key_format %{path}%{time_slice}/cluster-log-%{index}.%{file_extension}
  time_slice_format %Y%m%d-%H
  time_slice_wait 10m
  flush_interval 60s
  buffer_chunk_limit 256m
  </store>
 
 <store>
   @type elasticsearch
   @id out_es
   log_level info
   include_tag_key true
   host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
   port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
   scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'https'}"
   ssl_verify "#{ENV['FLUENT_ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
   reload_connections "#{ENV['FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS'] || 'true'}"
   logstash_prefix "#{ENV['FLUENT_ELASTICSEARCH_LOGSTASH_PREFIX'] || 'applogs'}"
   logstash_format true
   <buffer>
     flush_thread_count 8
     flush_interval 5s
     chunk_limit_size 2M
     queue_limit_length 32
     retry_max_interval 30
     retry_forever true
   </buffer>
  </store>
</match>





<source>
    type forward
    port 24224
    bind 0.0.0.0
</source>

<match *.*>
    @type stdout
</match>

<match **>
    @type copy

    <store>
        @type elasticsearch
        host "#{ENV['ELASTICSEARCH_HOSTNAME'] || 'localhost'}"
        ssl_verify "#{ENV['ELASTICSEARCH_SSL_VERIFY'] || 'true'}"
        port 443
        scheme https
        logstash_format true
        logstash_prefix rabbitmq.lab
        type_name logs.via.fluentd
    </store>

    <store>
        @type s3
        s3_bucket ecosystem.up.bucket
        s3_region eu-central-1
        utc
        path %Y/%m/%d/
        s3_object_key_format %{path}docker_lab_logs.txt
        time_slice_format %Y%m%d%H%M
        buffer_type file
        buffer_path /fluentd/log/s3
    </store>

</match>








<source>
    type forward
    port 24224
    bind 0.0.0.0
</source>

<match *.*>
    type stdout
</match>

<match **>
    @type elasticsearch
    logstash_format true
    host elasticsearch-xxxxxxxxxxxxxxxxx.xxxxx.elb.amazonaws.com
    port 80
    type_name rabbitmq-node-log
</match>





<source>
    type forward
    port 24224
    bind 0.0.0.0
</source>

<match *.*>
    @type stdout
</match>

<match **>
    @type copy

    <store>
        @type elasticsearch
        host vpc-elasticsearch-901260445-ya5j5mubvy6zvqwrxmfsvyptgy.eu-central-1.es.amazonaws.com
        port 443
        scheme https
        logstash_format true
        logstash_prefix rabbitmq.lab
        type_name logs.via.fluentd
    </store>

    <store>
        @type s3
        s3_bucket ecosystem.up.bucket
        s3_region eu-central-1
        utc
        path %Y/%m/%d/
        s3_object_key_format %{path}docker_lab_logs.txt
        time_slice_format %Y%m%d%H%M
        buffer_type file
        buffer_path /fluentd/log/s3
    </store>

</match>