diff --git a/pipeline/filters/aws-metadata.md b/pipeline/filters/aws-metadata.md index f7a2fa54d..9be8488a3 100644 --- a/pipeline/filters/aws-metadata.md +++ b/pipeline/filters/aws-metadata.md @@ -153,7 +153,7 @@ If you run Fluent Bit logs might look like the following: #### `tags_exclude` -Suppose the EC2 instance has three tags: `Name:fluent-bit-docs-example`, `project:fluentbit`, and `department:it`. In this example, the `department` tag is redundant and will be excluded. All of the projects belong to the `it` department, and you don't want to waste storage space on redundant labels. +Suppose the EC2 instance has three tags: `Name:fluent-bit-docs-example`, `project:fluentbit`, and `department:it`. In this example, the `department` tag is redundant and will be excluded. All the projects belong to the `it` department, and you don't want to waste storage space on redundant labels. Here is an example configuration that achieves this: @@ -163,11 +163,11 @@ Here is an example configuration that achieves this: ```yaml pipeline: - filters: - - name: aws - match: '*' - tags_enabled: true - tags_exclude: department + filters: + - name: aws + match: '*' + tags_enabled: true + tags_exclude: department ``` {% endtab %} @@ -175,10 +175,10 @@ pipeline: ```text [FILTER] - Name aws - Match * - tags_enabled true - tags_exclude department + Name aws + Match * + tags_enabled true + tags_exclude department ``` {% endtab %} diff --git a/pipeline/filters/checklist.md b/pipeline/filters/checklist.md index 0422115b3..128142e6c 100644 --- a/pipeline/filters/checklist.md +++ b/pipeline/filters/checklist.md @@ -22,26 +22,26 @@ The plugin supports the following configuration parameters ```yaml pipeline: - inputs: - - name: tail - tag: test1 - path: test1.log - read_from_head: true - parser: json - - filters: - - name: checklist - match: test1 - file: ip_list.txt - lookup_key: $remote_addr - record: - - ioc abc - - badurl null - log_level: debug - - outputs: - - name: stdout - match: test1 + inputs: + - name: tail + tag: test1 + path: test1.log + read_from_head: true + parser: json + + filters: + - name: checklist + match: test1 + file: ip_list.txt + lookup_key: $remote_addr + record: + - ioc abc + - badurl null + log_level: debug + + outputs: + - name: stdout + match: test1 ``` {% endtab %} @@ -49,24 +49,24 @@ pipeline: ```text [INPUT] - name tail - tag test1 - path test1.log - read_from_head true - parser json + name tail + tag test1 + path test1.log + read_from_head true + parser json [FILTER] - name checklist - match test1 - file ip_list.txt - lookup_key $remote_addr - record ioc abc - record badurl null - log_level debug + name checklist + match test1 + file ip_list.txt + lookup_key $remote_addr + record ioc abc + record badurl null + log_level debug [OUTPUT] - name stdout - match test1 + name stdout + match test1 ``` {% endtab %} diff --git a/pipeline/filters/ecs-metadata.md b/pipeline/filters/ecs-metadata.md index 73f666dd9..bfafcb2dc 100644 --- a/pipeline/filters/ecs-metadata.md +++ b/pipeline/filters/ecs-metadata.md @@ -45,34 +45,34 @@ section of the Fluent Bit configuration (not shown). ```yaml pipeline: - inputs: - - name: tail - tag: ecs.* - path: /var/lib/docker/containers/*/*.log - docker_mode: on - docker_mode_flush: 5 - docker_mode_parser: container_firstline - parser: docker - db: /var/fluent-bit/state/flb_container.db - mem_buf_limit: 50MB - skip_long_lines: on - refresh_interval: 10 - rotate_wait: 30 - storage.type: filesystem - read_from_head: off - - filters: - - name: ecs - match: '*' - ecs_tag_prefix: ecs.var.lib.docker.containers. - add: - - ecs_task_id $TaskID - - cluster $ClusterName - - outputs: - - name: stdout - match: '*' - format: json_lines + inputs: + - name: tail + tag: ecs.* + path: /var/lib/docker/containers/*/*.log + docker_mode: on + docker_mode_flush: 5 + docker_mode_parser: container_firstline + parser: docker + db: /var/fluent-bit/state/flb_container.db + mem_buf_limit: 50MB + skip_long_lines: on + refresh_interval: 10 + rotate_wait: 30 + storage.type: filesystem + read_from_head: off + + filters: + - name: ecs + match: '*' + ecs_tag_prefix: ecs.var.lib.docker.containers. + add: + - ecs_task_id $TaskID + - cluster $ClusterName + + outputs: + - name: stdout + match: '*' + format: json_lines ``` {% endtab %} @@ -80,32 +80,32 @@ pipeline: ```text [INPUT] - Name tail - Tag ecs.* - Path /var/lib/docker/containers/*/*.log - Docker_Mode On - Docker_Mode_Flush 5 - Docker_Mode_Parser container_firstline - Parser docker - DB /var/fluent-bit/state/flb_container.db - Mem_Buf_Limit 50MB - Skip_Long_Lines On - Refresh_Interval 10 - Rotate_Wait 30 - storage.type filesystem - Read_From_Head Off + Name tail + Tag ecs.* + Path /var/lib/docker/containers/*/*.log + Docker_Mode On + Docker_Mode_Flush 5 + Docker_Mode_Parser container_firstline + Parser docker + DB /var/fluent-bit/state/flb_container.db + Mem_Buf_Limit 50MB + Skip_Long_Lines On + Refresh_Interval 10 + Rotate_Wait 30 + storage.type filesystem + Read_From_Head Off [FILTER] - Name ecs - Match * - ECS_Tag_Prefix ecs.var.lib.docker.containers. - ADD ecs_task_id $TaskID - ADD cluster $ClusterName + Name ecs + Match * + ECS_Tag_Prefix ecs.var.lib.docker.containers. + ADD ecs_task_id $TaskID + ADD cluster $ClusterName [OUTPUT] - Name stdout - Match * - Format json_lines + Name stdout + Match * + Format json_lines ``` {% endtab %} @@ -115,10 +115,10 @@ The output log should be similar to: ```text { - "date":1665003546.0, - "log":"some message from your container", - "ecs_task_id" "1234567890abcdefghijklmnop", - "cluster": "your_cluster_name", + "date":1665003546.0, + "log":"some message from your container", + "ecs_task_id" "1234567890abcdefghijklmnop", + "cluster": "your_cluster_name", } ``` @@ -129,32 +129,32 @@ The output log should be similar to: ```yaml pipeline: - inputs: - - name: tail - tag: ecs.* - path: /var/lib/docker/containers/*/*.log - docker_mode: on - docker_mode_flush: 5 - docker_mode_parser: container_firstline - parser: docker - db: /var/fluent-bit/state/flb_container.db - mem_buf_limit: 50MB - skip_long_lines: on - refresh_interval: 10 - rotate_wait: 30 - storage.type: filesystem - read_from_head: off - - filters: - - name: ecs - match: '*' - ecs_tag_prefix: ecs.var.lib.docker.containers. - add: resource $ClusterName.$TaskDefinitionFamily.$TaskID.$ECSContainerName - - outputs: - - name: stdout - match: '*' - format: json_lines + inputs: + - name: tail + tag: ecs.* + path: /var/lib/docker/containers/*/*.log + docker_mode: on + docker_mode_flush: 5 + docker_mode_parser: container_firstline + parser: docker + db: /var/fluent-bit/state/flb_container.db + mem_buf_limit: 50MB + skip_long_lines: on + refresh_interval: 10 + rotate_wait: 30 + storage.type: filesystem + read_from_head: off + + filters: + - name: ecs + match: '*' + ecs_tag_prefix: ecs.var.lib.docker.containers. + add: resource $ClusterName.$TaskDefinitionFamily.$TaskID.$ECSContainerName + + outputs: + - name: stdout + match: '*' + format: json_lines ``` {% endtab %} @@ -162,31 +162,31 @@ pipeline: ```text [INPUT] - Name tail - Tag ecs.* - Path /var/lib/docker/containers/*/*.log - Docker_Mode On - Docker_Mode_Flush 5 - Docker_Mode_Parser container_firstline - Parser docker - DB /var/fluent-bit/state/flb_container.db - Mem_Buf_Limit 50MB - Skip_Long_Lines On - Refresh_Interval 10 - Rotate_Wait 30 - storage.type filesystem - Read_From_Head Off + Name tail + Tag ecs.* + Path /var/lib/docker/containers/*/*.log + Docker_Mode On + Docker_Mode_Flush 5 + Docker_Mode_Parser container_firstline + Parser docker + DB /var/fluent-bit/state/flb_container.db + Mem_Buf_Limit 50MB + Skip_Long_Lines On + Refresh_Interval 10 + Rotate_Wait 30 + storage.type filesystem + Read_From_Head Off [FILTER] - Name ecs - Match * - ECS_Tag_Prefix ecs.var.lib.docker.containers. - ADD resource $ClusterName.$TaskDefinitionFamily.$TaskID.$ECSContainerName + Name ecs + Match * + ECS_Tag_Prefix ecs.var.lib.docker.containers. + ADD resource $ClusterName.$TaskDefinitionFamily.$TaskID.$ECSContainerName [OUTPUT] - Name stdout - Match * - Format json_lines + Name stdout + Match * + Format json_lines ``` {% endtab %} @@ -196,9 +196,9 @@ The output log would be similar to: ```text { - "date":1665003546.0, - "log":"some message from your container", - "resource" "cluster.family.1234567890abcdefghijklmnop.app", + "date":1665003546.0, + "log":"some message from your container", + "resource" "cluster.family.1234567890abcdefghijklmnop.app", } ``` @@ -214,29 +214,29 @@ This example shows a use case for the `Cluster_Metadata_Only` option attaching c ```yaml pipeline: - inputs: - - name: tail - tag: ecsagent.* - path: /var/log/ecs/* - db: /var/fluent-bit/state/flb_ecs.db - mem_buf_limit: 50MB - skip_long_lines: on - refresh_interval: 10 - rotate_wait: 30 - storage.type: filesystem - # Collect all logs on instance - read_from_head: on - - filters: - - name: ecs - match: '*' - cluster_metadata_only: on - add: cluster $ClusterName - - outputs: - - name: stdout - match: '*' - format: json_lines + inputs: + - name: tail + tag: ecsagent.* + path: /var/log/ecs/* + db: /var/fluent-bit/state/flb_ecs.db + mem_buf_limit: 50MB + skip_long_lines: on + refresh_interval: 10 + rotate_wait: 30 + storage.type: filesystem + # Collect all logs on instance + read_from_head: on + + filters: + - name: ecs + match: '*' + cluster_metadata_only: on + add: cluster $ClusterName + + outputs: + - name: stdout + match: '*' + format: json_lines ``` {% endtab %} @@ -244,29 +244,29 @@ pipeline: ```text [INPUT] - Name tail - Tag ecsagent.* - Path /var/log/ecs/* - DB /var/fluent-bit/state/flb_ecs.db - Mem_Buf_Limit 50MB - Skip_Long_Lines On - Refresh_Interval 10 - Rotate_Wait 30 - storage.type filesystem - # Collect all logs on instance - Read_From_Head On + Name tail + Tag ecsagent.* + Path /var/log/ecs/* + DB /var/fluent-bit/state/flb_ecs.db + Mem_Buf_Limit 50MB + Skip_Long_Lines On + Refresh_Interval 10 + Rotate_Wait 30 + storage.type filesystem + # Collect all logs on instance + Read_From_Head On [FILTER] - Name ecs - Match * - Cluster_Metadata_Only On - ADD cluster $ClusterName + Name ecs + Match * + Cluster_Metadata_Only On + ADD cluster $ClusterName [OUTPUT] - Name stdout - Match * - Format json_lines + Name stdout + Match * + Format json_lines ``` {% endtab %} -{% endtabs %} +{% endtabs %} \ No newline at end of file diff --git a/pipeline/filters/geoip2-filter.md b/pipeline/filters/geoip2-filter.md index c7d8c0270..7ad7cca78 100644 --- a/pipeline/filters/geoip2-filter.md +++ b/pipeline/filters/geoip2-filter.md @@ -27,22 +27,22 @@ The following configuration processes the incoming `remote_addr` and appends cou ```yaml pipeline: - inputs: - - name: dummy - dummy: {"remote_addr": "8.8.8.8"} - - filters: - - name: gioip2 - match: '*' - database: GioLite2-City.mmdb - lookup_key: remote_addr - record: - - country remote_addr %{country.names.en} - - isocode remote_addr %{country.iso_code} - - outputs: - - name: stdout - match: '*' + inputs: + - name: dummy + dummy: {"remote_addr": "8.8.8.8"} + + filters: + - name: gioip2 + match: '*' + database: GioLite2-City.mmdb + lookup_key: remote_addr + record: + - country remote_addr %{country.names.en} + - isocode remote_addr %{country.iso_code} + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -50,20 +50,20 @@ pipeline: ```text [INPUT] - Name dummy - Dummy {"remote_addr": "8.8.8.8"} + Name dummy + Dummy {"remote_addr": "8.8.8.8"} [FILTER] - Name geoip2 - Match * - Database GeoLite2-City.mmdb - Lookup_key remote_addr - Record country remote_addr %{country.names.en} - Record isocode remote_addr %{country.iso_code} + Name geoip2 + Match * + Database GeoLite2-City.mmdb + Lookup_key remote_addr + Record country remote_addr %{country.names.en} + Record isocode remote_addr %{country.iso_code} [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} diff --git a/pipeline/filters/grep.md b/pipeline/filters/grep.md index 6db4b106f..38a9820d3 100644 --- a/pipeline/filters/grep.md +++ b/pipeline/filters/grep.md @@ -42,7 +42,7 @@ When using the command line, pay close attention to quote the regular expression The following command loads the [tail](../../pipeline/inputs/tail) plugin and reads the content of `lines.txt`. Then the `grep` filter applies a regular expression rule over the `log` field created by the `tail` plugin and only passes records with a field value starting with `aa`: ```shell -./fluent-bit -i tail -p 'path=lines.txt' -F grep -p 'regex=log aa' -m '*' -o stdout +fluent-bit -i tail -p 'path=lines.txt' -F grep -p 'regex=log aa' -m '*' -o stdout ``` ### Configuration file @@ -52,22 +52,22 @@ The following command loads the [tail](../../pipeline/inputs/tail) plugin and re ```yaml service: - parsers_file: /path/to/parsers.conf - + parsers_file: /path/to/parsers.conf + pipeline: - inputs: - - name: tail - path: lines.txt - parser: json - - filters: - - name: grep - match: '*' - regex: log aa - - outputs: - - name: stdout - match: '*' + inputs: + - name: tail + path: lines.txt + parser: json + + filters: + - name: grep + match: '*' + regex: log aa + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -75,27 +75,28 @@ pipeline: ```text [SERVICE] - parsers_file /path/to/parsers.conf + parsers_file /path/to/parsers.conf [INPUT] - name tail - path lines.txt - parser json + name tail + path lines.txt + parser json [FILTER] - name grep - match * - regex log aa + name grep + match * + regex log aa [OUTPUT] - name stdout - match * + name stdout + match * ``` {% endtab %} {% endtabs %} -The filter lets you use multiple rules which are applied in order, you can have many `Regex` and `Exclude` entries as required ([more information](#multiple-conditions). + +The filter allows to use multiple rules which are applied in order, you can have many `Regex` and `Exclude` entries as required ([more information](#multiple-conditions)). ### Nested fields example @@ -103,20 +104,20 @@ To match or exclude records based on nested values, you can use [Record Accessor Consider the following record example: -```text +```json { - "log": "something", - "kubernetes": { - "pod_name": "myapp-0", - "namespace_name": "default", - "pod_id": "216cd7ae-1c7e-11e8-bb40-000c298df552", - "labels": { - "app": "myapp" - }, - "host": "minikube", - "container_name": "myapp", - "docker_id": "370face382c7603fdd309d8c6aaaf434fd98b92421ce" - } + "log": "something", + "kubernetes": { + "pod_name": "myapp-0", + "namespace_name": "default", + "pod_id": "216cd7ae-1c7e-11e8-bb40-000c298df552", + "labels": { + "app": "myapp" + }, + "host": "minikube", + "container_name": "myapp", + "docker_id": "370face382c7603fdd309d8c6aaaf434fd98b92421ce" + } } ``` @@ -126,12 +127,12 @@ For example, to exclude records that match the nested field `kubernetes.labels.a {% tab title="fluent-bit.yaml" %} ```yaml -pipeline: - - filters: - - name: grep - match: '*' - exclude: $kubernetes['labels']['app'] myapp +pipeline: + + filters: + - name: grep + match: '*' + exclude: $kubernetes['labels']['app'] myapp ``` {% endtab %} @@ -139,9 +140,9 @@ pipeline: ```text [FILTER] - Name grep - Match * - Exclude $kubernetes['labels']['app'] myapp + Name grep + Match * + Exclude $kubernetes['labels']['app'] myapp ``` {% endtab %} @@ -161,15 +162,15 @@ The following example checks for a specific valid value for the key: ```yaml pipeline: - - filters: - # Use Grep to verify the contents of the iot_timestamp value. - # If the iot_timestamp key does not exist, this will fail - # and exclude the row. - - name: grep - alias: filter-iots-grep - match: iots_thread.* - regex: iot_timestamp ^\d{4}-\d{2}-\d{2} + + filters: + # Use Grep to verify the contents of the iot_timestamp value. + # If the iot_timestamp key does not exist, this will fail + # and exclude the row. + - name: grep + alias: filter-iots-grep + match: iots_thread.* + regex: iot_timestamp ^\d{4}-\d{2}-\d{2} ``` {% endtab %} @@ -180,10 +181,10 @@ pipeline: # If the iot_timestamp key does not exist, this will fail # and exclude the row. [FILTER] - Name grep - Alias filter-iots-grep - Match iots_thread.* - Regex iot_timestamp ^\d{4}-\d{2}-\d{2} + Name grep + Alias filter-iots-grep + Match iots_thread.* + Regex iot_timestamp ^\d{4}-\d{2}-\d{2} ``` {% endtab %} @@ -197,7 +198,8 @@ or is missing or empty, then it will be excluded. If you want to set multiple `Regex` or `Exclude`, you must use the `legacy` mode. In this case, the `Exclude` must be first and you can have only one `Regex`. If `Exclude` match, the string is blocked. You can have multiple `Exclude` entry. After, if there is no `Regex`, the line is sent to the output. -if there is a `Regex` and it match, the line is sent to the output, else, it's blocked. + +If there is a `Regex` and it matches, the line is sent to the output, else, it's blocked. If you want to set multiple `Regex` or `Exclude`, you can use `Logical_Op` property to use logical conjunction or disjunction. @@ -208,22 +210,22 @@ If `Logical_Op` is set, setting both `Regex` and `Exclude` results in an error. ```yaml pipeline: - inputs: - - name: dummy - dummy: '{"endpoint":"localhost", "value":"something"}' - tag: dummy - - filters: - - name: grep - match: '*' - logical_op: or - regex: - - value something - - value error - - outputs: - - name: stdout - match: '*' + inputs: + - name: dummy + dummy: '{"endpoint":"localhost", "value":"something"}' + tag: dummy + + filters: + - name: grep + match: '*' + logical_op: or + regex: + - value something + - value error + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -231,20 +233,20 @@ pipeline: ```text [INPUT] - Name dummy - Dummy {"endpoint":"localhost", "value":"something"} - Tag dummy + Name dummy + Dummy {"endpoint":"localhost", "value":"something"} + Tag dummy [FILTER] - Name grep - Match * - Logical_Op or - Regex value something - Regex value error + Name grep + Match * + Logical_Op or + Regex value something + Regex value error [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} @@ -253,28 +255,6 @@ pipeline: The output looks similar to: ```text -Fluent Bit v4.0.3 -* Copyright (C) 2015-2025 The Fluent Bit Authors -* Fluent Bit is a CNCF sub-project under the umbrella of Fluentd -* https://fluentbit.io - -______ _ _ ______ _ _ ___ _____ -| ___| | | | | ___ (_) | / || _ | -| |_ | |_ _ ___ _ __ | |_ | |_/ /_| |_ __ __/ /| || |/' | -| _| | | | | |/ _ \ '_ \| __| | ___ \ | __| \ \ / / /_| || /| | -| | | | |_| | __/ | | | |_ | |_/ / | |_ \ V /\___ |\ |_/ / -\_| |_|\__,_|\___|_| |_|\__| \____/|_|\__| \_/ |_(_)___/ - - -[2025/07/03 16:15:34] [ info] [fluent bit] version=4.0.3, commit=3a91b155d6, pid=23196 -[2025/07/03 16:15:34] [ info] [storage] ver=1.5.3, type=memory, sync=normal, checksum=off, max_chunks_up=128 -[2025/07/03 16:15:34] [ info] [simd ] disabled -[2025/07/03 16:15:34] [ info] [cmetrics] version=1.0.3 -[2025/07/03 16:15:34] [ info] [ctraces ] version=0.6.6 -[2025/07/03 16:15:34] [ info] [input:dummy:dummy.0] initializing -[2025/07/03 16:15:34] [ info] [input:dummy:dummy.0] storage_strategy='memory' (memory only) -[2025/07/03 16:15:34] [ info] [output:stdout:stdout.0] worker #0 started -[2025/07/03 16:15:34] [ info] [sp] stream processor started [0] dummy: [1674348410.558341857, {"endpoint"=>"localhost", "value"=>"something"}] [0] dummy: [1674348411.546425499, {"endpoint"=>"localhost", "value"=>"something"}] -``` +``` \ No newline at end of file diff --git a/pipeline/filters/kubernetes.md b/pipeline/filters/kubernetes.md index e7b3a209e..0e849cbec 100644 --- a/pipeline/filters/kubernetes.md +++ b/pipeline/filters/kubernetes.md @@ -72,11 +72,11 @@ Kubernetes filter provides several ways to process the data contained in the `lo ```yaml parsers: - - name: docker - format: json - time_key: time - time_format: '%Y-%m-%dT%H:%M:%S.%L' - time_keep: on + - name: docker + format: json + time_key: time + time_format: '%Y-%m-%dT%H:%M:%S.%L' + time_keep: on ``` {% endtab %} @@ -84,11 +84,11 @@ parsers: ```text [PARSER] - Name docker - Format json - Time_Key time - Time_Format %Y-%m-%dT%H:%M:%S.%L - Time_Keep On + Name docker + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep On ``` {% endtab %} @@ -196,21 +196,21 @@ Kubernetes Filter depends on either [Tail](../inputs/tail.md) or [Systemd](../in ```yaml pipeline: - inputs: - - name: tail - tag: kube.* - path: /var/log/containers/*.log - multiline.parser: docker,cri - - filters: - - name: kubernetes - match: 'kube.*' - kube_url: https://kubernetes.default.svc:443 - kube_ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - kube_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - kube_tag_prefix: kube.var.log.containers. - merge_log: on - merge_log_key: log_processed + inputs: + - name: tail + tag: kube.* + path: /var/log/containers/*.log + multiline.parser: docker,cri + + filters: + - name: kubernetes + match: 'kube.*' + kube_url: https://kubernetes.default.svc:443 + kube_ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + kube_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kube_tag_prefix: kube.var.log.containers. + merge_log: on + merge_log_key: log_processed ``` {% endtab %} @@ -218,20 +218,20 @@ pipeline: ```text [INPUT] - Name tail - Tag kube.* - Path /var/log/containers/*.log - multiline.parser docker, cri + Name tail + Tag kube.* + Path /var/log/containers/*.log + multiline.parser docker, cri [FILTER] - Name kubernetes - Match kube.* - Kube_URL https://kubernetes.default.svc:443 - Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token - Kube_Tag_Prefix kube.var.log.containers. - Merge_Log On - Merge_Log_Key log_processed + Name kubernetes + Match kube.* + Kube_URL https://kubernetes.default.svc:443 + Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token + Kube_Tag_Prefix kube.var.log.containers. + Merge_Log On + Merge_Log_Key log_processed ``` {% endtab %} @@ -299,24 +299,24 @@ One such use case involves splitting logs by namespace, pods, containers or cont ```yaml parsers: - - name: custom-tag - format: regex - regex: '^(?[^_]+)\.(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)\.(?.+)\.(?[a-z0-9]{64})' - + - name: custom-tag + format: regex + regex: '^(?[^_]+)\.(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)\.(?.+)\.(?[a-z0-9]{64})' + pipeline: - inputs: - - name: tail - tag: kube.... - path: /var/log/containers/*.log - tag_regex: '(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$' - parser: cri - - filters: - - name: kubernetes - match: 'kube.*' - kube_tag_prefix: kube. - regex_parser: custom-tag - merge_log: on + inputs: + - name: tail + tag: kube.... + path: /var/log/containers/*.log + tag_regex: '(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$' + parser: cri + + filters: + - name: kubernetes + match: 'kube.*' + kube_tag_prefix: kube. + regex_parser: custom-tag + merge_log: on ``` {% endtab %} @@ -324,23 +324,23 @@ pipeline: ```text [PARSER] - Name custom-tag - Format regex - Regex ^(?[^_]+)\.(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)\.(?.+)\.(?[a-z0-9]{64}) + Name custom-tag + Format regex + Regex ^(?[^_]+)\.(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)\.(?.+)\.(?[a-z0-9]{64}) [INPUT] - Name tail - Tag kube.... - Path /var/log/containers/*.log - Tag_Regex (?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$ - Parser cri + Name tail + Tag kube.... + Path /var/log/containers/*.log + Tag_Regex (?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$ + Parser cri [FILTER] - Name kubernetes - Match kube.* - Kube_Tag_Prefix kube. - Regex_Parser custom-tag - Merge_Log On + Name kubernetes + Match kube.* + Kube_Tag_Prefix kube. + Regex_Parser custom-tag + Merge_Log On ``` {% endtab %} @@ -407,27 +407,27 @@ Fluent Bit configuration example: ```yaml pipeline: - inputs: - - name: tail - tag: kube.* - path: /var/log/containers/*.log - db: /var/log/flb_kube.db - parser: docker - docker_mode: on - mem_buf_limit: 50MB - skip_login_lines: on - refresh_interval: 10 - - filters: - - name: kubernetes - match: 'kube.*' - kube_url: https://kubernetes.default.svc.cluster.local:443 - kube_ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - kube_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - merge_log: on - buffer_size: 0 - use_kubelet: ture - kubelet_port: 10250 + inputs: + - name: tail + tag: kube.* + path: /var/log/containers/*.log + db: /var/log/flb_kube.db + parser: docker + docker_mode: on + mem_buf_limit: 50MB + skip_login_lines: on + refresh_interval: 10 + + filters: + - name: kubernetes + match: 'kube.*' + kube_url: https://kubernetes.default.svc.cluster.local:443 + kube_ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + kube_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + merge_log: on + buffer_size: 0 + use_kubelet: ture + kubelet_port: 10250 ``` {% endtab %} @@ -435,26 +435,26 @@ pipeline: ```yaml [INPUT] - Name tail - Tag kube.* - Path /var/log/containers/*.log - DB /var/log/flb_kube.db - Parser docker - Docker_Mode On - Mem_Buf_Limit 50MB - Skip_Long_Lines On - Refresh_Interval 10 + Name tail + Tag kube.* + Path /var/log/containers/*.log + DB /var/log/flb_kube.db + Parser docker + Docker_Mode On + Mem_Buf_Limit 50MB + Skip_Long_Lines On + Refresh_Interval 10 [FILTER] - Name kubernetes - Match kube.* - Kube_URL https://kubernetes.default.svc.cluster.local:443 - Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token - Merge_Log On - Buffer_Size 0 - Use_Kubelet true - Kubelet_Port 10250 + Name kubernetes + Match kube.* + Kube_URL https://kubernetes.default.svc.cluster.local:443 + Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token + Merge_Log On + Buffer_Size 0 + Use_Kubelet true + Kubelet_Port 10250 ``` {% endtab %} diff --git a/pipeline/filters/log_to_metrics.md b/pipeline/filters/log_to_metrics.md index 236c1c464..b0a1f0e96 100644 --- a/pipeline/filters/log_to_metrics.md +++ b/pipeline/filters/log_to_metrics.md @@ -4,9 +4,7 @@ description: Generate metrics from logs # Logs to metrics -![](https://static.scarf.sh/a.png?x-pxid=768830f6-8d2d-4231-9e5e-259ce6797ba5) - -The _Log to metrics_ filter lets you generate log-derived metrics. It supports modes to count records, provide a gauge for field values, or create a histogram. You can also match or exclude specific records based on regular expression patterns for values or nested values. +The _log to metrics_ filter lets you generate log-derived metrics. It supports modes to count records, provide a guage for field values, or create a histogram. You can also match or exclude specific records based on regular expression patterns for values or nested values. This filter doesn't actually act as a record filter and therefore doesn't change or drop records. All records will pass through this filter untouched, and any generated metrics will be emitted into a separate metric pipeline. @@ -53,32 +51,32 @@ The following example takes records from two `dummy` inputs and counts all messa ```yaml service: - flush: 1 - log_level: info - + flush: 1 + log_level: info + pipeline: - inputs: - - name: dummy - dummy: '{"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"}' - tag: dummy.log - - - name: dummy - dummy: '{"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"}' - tag: dummy.log2 - - filters: - - name: log_to_metrics - match: 'dummy.log*' - tag: test_metric - metric_mode: counter - metric_name: count_all_dummy_messages - metric_description: 'This metric counts dummy messages' - - outputs: - - name: prometheus_exporter - match: '*' - host: 0.0.0.0 - port: 9999 + inputs: + - name: dummy + dummy: '{"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"}' + tag: dummy.log + + - name: dummy + dummy: '{"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"}' + tag: dummy.log2 + + filters: + - name: log_to_metrics + match: 'dummy.log*' + tag: test_metric + metric_mode: counter + metric_name: count_all_dummy_messages + metric_description: 'This metric counts dummy messages' + + outputs: + - name: prometheus_exporter + match: '*' + host: 0.0.0.0 + port: 9999 ``` {% endtab %} @@ -86,32 +84,32 @@ pipeline: ```text [SERVICE] - flush 1 - log_level info + flush 1 + log_level info [INPUT] - Name dummy - Dummy {"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"} - Tag dummy.log + Name dummy + Dummy {"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"} + Tag dummy.log [INPUT] - Name dummy - Dummy {"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"} - Tag dummy.log2 + Name dummy + Dummy {"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"} + Tag dummy.log2 [FILTER] - name log_to_metrics - match dummy.log* - tag test_metric - metric_mode counter - metric_name count_all_dummy_messages - metric_description This metric counts dummy messages + name log_to_metrics + match dummy.log* + tag test_metric + metric_mode counter + metric_name count_all_dummy_messages + metric_description This metric counts dummy messages [OUTPUT] - name prometheus_exporter - match * - host 0.0.0.0 - port 9999 + name prometheus_exporter + match * + host 0.0.0.0 + port 9999 ``` {% endtab %} @@ -122,14 +120,13 @@ Run this configuration file with Prometheus to collect the metrics from the Flue ```yaml # config global: - scrape_interval: 5s + scrape_interval: 5s scrape_configs: - - # Scraping Fluent Bit example. - - job_name: "fluentbit" - static_configs: - - targets: ["localhost:9999"] + # Scraping Fluent Bit example. + - job_name: "fluentbit" + static_configs: + - targets: ["localhost:9999"] ``` {% endtab %} @@ -138,7 +135,7 @@ scrape_configs: You can then use a tool like curl to retrieve the generated metric: ```shell -$ ./curl -s http://127.0.0.1:9999/metrics +$ curl -s http://127.0.0.1:9999/metrics # HELP log_metric_counter_count_all_dummy_messages This metric counts dummy messages @@ -155,39 +152,39 @@ The `gauge` mode needs a `value_field` to specify where to generate the metric v ```yaml service: - flush: 1 - log_level: info - + flush: 1 + log_level: info + pipeline: - inputs: - - name: dummy - dummy: '{"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"}' - tag: dummy.log - - - name: dummy - dummy: '{"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"}' - tag: dummy.log2 - - filters: - - name: log_to_metrics - match: 'dummy.log*' - tag: test_metric - metric_mode: gauge - metric_name: current_duration - metric_description: 'This metric shows the current duration' - value_field: duration - kubernetes_mode: on - regex: 'message .*el.*' - add_label: app $kubernetes['labels']['app'] - label_field: - - color - - shape - - outputs: - - name: prometheus_exporter - match: '*' - host: 0.0.0.0 - port: 9999 + inputs: + - name: dummy + dummy: '{"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"}' + tag: dummy.log + + - name: dummy + dummy: '{"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"}' + tag: dummy.log2 + + filters: + - name: log_to_metrics + match: 'dummy.log*' + tag: test_metric + metric_mode: gauge + metric_name: current_duration + metric_description: 'This metric shows the current duration' + value_field: duration + kubernetes_mode: on + regex: 'message .*el.*' + add_label: app $kubernetes['labels']['app'] + label_field: + - color + - shape + + outputs: + - name: prometheus_exporter + match: '*' + host: 0.0.0.0 + port: 9999 ``` {% endtab %} @@ -195,38 +192,38 @@ pipeline: ```text [SERVICE] - flush 1 - log_level info + flush 1 + log_level info [INPUT] - Name dummy - Dummy {"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"} - Tag dummy.log + Name dummy + Dummy {"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"} + Tag dummy.log [INPUT] - Name dummy - Dummy {"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"} - Tag dummy.log2 + Name dummy + Dummy {"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"} + Tag dummy.log2 [FILTER] - name log_to_metrics - match dummy.log* - tag test_metric - metric_mode gauge - metric_name current_duration - metric_description This metric shows the current duration - value_field duration - kubernetes_mode on - regex message .*el.* - add_label app $kubernetes['labels']['app'] - label_field color - label_field shape - + name log_to_metrics + match dummy.log* + tag test_metric + metric_mode gauge + metric_name current_duration + metric_description This metric shows the current duration + value_field duration + kubernetes_mode on + regex message .*el.* + add_label app $kubernetes['labels']['app'] + label_field color + label_field shape + [OUTPUT] - name prometheus_exporter - match * - host 0.0.0.0 - port 9999 + name prometheus_exporter + match * + host 0.0.0.0 + port 9999 ``` {% endtab %} @@ -237,14 +234,13 @@ Run this configuration file with Prometheus to collect the metrics from the Flue ```yaml # config global: - scrape_interval: 5s + scrape_interval: 5s scrape_configs: - - # Scraping Fluent Bit example. - - job_name: "fluentbit" - static_configs: - - targets: ["localhost:9999"] + # Scraping Fluent Bit example. + - job_name: "fluentbit" + static_configs: + - targets: ["localhost:9999"] ``` {% endtab %} @@ -253,7 +249,7 @@ scrape_configs: You can then use a tool like curl to retrieve the generated metric: ```shell -$ ./curl -s http://127.0.0.1:9999/metrics +$ curl -s http://127.0.0.1:9999/metrics # HELP log_metric_gauge_current_duration This metric shows the current duration @@ -280,39 +276,39 @@ Similar to the `gauge` mode, the `histogram` mode needs a `value_field` to speci ```yaml service: - flush: 1 - log_level: info - + flush: 1 + log_level: info + pipeline: - inputs: - - name: dummy - dummy: '{"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"}' - tag: dummy.log - - - name: dummy - dummy: '{"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"}' - tag: dummy.log2 - - filters: - - name: log_to_metrics - match: 'dummy.log*' - tag: test_metric - metric_mode: histogram - metric_name: current_duration - metric_description: 'This metric shows the request duration' - value_field: duration - kubernetes_mode: on - regex: 'message .*el.*' - add_label: app $kubernetes['labels']['app'] - label_field: - - color - - shape - - outputs: - - name: prometheus_exporter - match: '*' - host: 0.0.0.0 - port: 9999 + inputs: + - name: dummy + dummy: '{"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"}' + tag: dummy.log + + - name: dummy + dummy: '{"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"}' + tag: dummy.log2 + + filters: + - name: log_to_metrics + match: 'dummy.log*' + tag: test_metric + metric_mode: histogram + metric_name: current_duration + metric_description: 'This metric shows the request duration' + value_field: duration + kubernetes_mode: on + regex: 'message .*el.*' + add_label: app $kubernetes['labels']['app'] + label_field: + - color + - shape + + outputs: + - name: prometheus_exporter + match: '*' + host: 0.0.0.0 + port: 9999 ``` {% endtab %} @@ -320,38 +316,38 @@ pipeline: ```text [SERVICE] - flush 1 - log_level info + flush 1 + log_level info [INPUT] - Name dummy - Dummy {"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"} - Tag dummy.log + Name dummy + Dummy {"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"} + Tag dummy.log [INPUT] - Name dummy - Dummy {"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"} - Tag dummy.log2 + Name dummy + Dummy {"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"} + Tag dummy.log2 [FILTER] - name log_to_metrics - match dummy.log* - tag test_metric - metric_mode histogram - metric_name current_duration - metric_description This metric shows the request duration - value_field duration - kubernetes_mode on - regex message .*el.* - add_label app $kubernetes['labels']['app'] - label_field color - label_field shape - + name log_to_metrics + match dummy.log* + tag test_metric + metric_mode histogram + metric_name current_duration + metric_description This metric shows the request duration + value_field duration + kubernetes_mode on + regex message .*el.* + add_label app $kubernetes['labels']['app'] + label_field color + label_field shape + [OUTPUT] - name prometheus_exporter - match * - host 0.0.0.0 - port 9999 + name prometheus_exporter + match * + host 0.0.0.0 + port 9999 ``` {% endtab %} @@ -362,14 +358,13 @@ Run this configuration file with Prometheus to collect the metrics from the Flue ```yaml # config global: - scrape_interval: 5s + scrape_interval: 5s scrape_configs: - - # Scraping Fluent Bit example. - - job_name: "fluentbit" - static_configs: - - targets: ["localhost:9999"] + # Scraping Fluent Bit example. + - job_name: "fluentbit" + static_configs: + - targets: ["localhost:9999"] ``` {% endtab %} @@ -378,7 +373,7 @@ scrape_configs: You can then use a tool like curl to retrieve the generated metric: ```shell -$ ./curl -s http://127.0.0.1:2021/metrics +$ curl -s http://127.0.0.1:2021/metrics # HELP log_metric_histogram_current_duration This metric shows the request duration @@ -420,47 +415,47 @@ In the resulting output, there are several buckets by default: `0.005, 0.01, 0.0 ```yaml service: - flush: 1 - log_level: info - + flush: 1 + log_level: info + pipeline: - inputs: - - name: dummy - dummy: '{"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"}' - tag: dummy.log - - - name: dummy - dummy: '{"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"}' - tag: dummy.log2 - - filters: - - name: log_to_metrics - match: 'dummy.log*' - tag: test_metric - metric_mode: histogram - metric_name: current_duration - metric_description: 'This metric shows the HTTP request duration as histogram in milliseconds' - value_field: duration - kubernetes_mode: on - bucket: - - 1 - - 5 - - 10 - - 50 - - 1000 - - 250 - - 500 - - 1000 - regex: 'message .*el.*' - label_field: - - color - - shape - - outputs: - - name: prometheus_exporter - match: '*' - host: 0.0.0.0 - port: 9999 + inputs: + - name: dummy + dummy: '{"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"}' + tag: dummy.log + + - name: dummy + dummy: '{"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"}' + tag: dummy.log2 + + filters: + - name: log_to_metrics + match: 'dummy.log*' + tag: test_metric + metric_mode: histogram + metric_name: current_duration + metric_description: 'This metric shows the HTTP request duration as histogram in milliseconds' + value_field: duration + kubernetes_mode: on + bucket: + - 1 + - 5 + - 10 + - 50 + - 1000 + - 250 + - 500 + - 1000 + regex: 'message .*el.*' + label_field: + - color + - shape + + outputs: + - name: prometheus_exporter + match: '*' + host: 0.0.0.0 + port: 9999 ``` {% endtab %} @@ -468,45 +463,45 @@ pipeline: ```text [SERVICE] - flush 1 - log_level info + flush 1 + log_level info [INPUT] - Name dummy - Dummy {"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"} - Tag dummy.log + Name dummy + Dummy {"message":"dummy", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 20, "color": "red", "shape": "circle"} + Tag dummy.log [INPUT] - Name dummy - Dummy {"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"} - Tag dummy.log2 + Name dummy + Dummy {"message":"hello", "kubernetes":{"namespace_name": "default", "docker_id": "abc123", "pod_name": "pod1", "container_name": "mycontainer", "pod_id": "def456", "labels":{"app": "app1"}}, "duration": 60, "color": "blue", "shape": "square"} + Tag dummy.log2 [FILTER] - name log_to_metrics - match dummy.log* - tag test_metric - metric_mode histogram - metric_name current_duration - metric_description This metric shows the HTTP request duration as histogram in milliseconds - value_field duration - kubernetes_mode on - bucket 1 - bucket 5 - bucket 10 - bucket 50 - bucket 100 - bucket 250 - bucket 500 - bucket 1000 - regex message .*el.* - label_field color - label_field shape - + name log_to_metrics + match dummy.log* + tag test_metric + metric_mode histogram + metric_name current_duration + metric_description This metric shows the HTTP request duration as histogram in milliseconds + value_field duration + kubernetes_mode on + bucket 1 + bucket 5 + bucket 10 + bucket 50 + bucket 100 + bucket 250 + bucket 500 + bucket 1000 + regex message .*el.* + label_field color + label_field shape + [OUTPUT] - name prometheus_exporter - match * - host 0.0.0.0 - port 9999 + name prometheus_exporter + match * + host 0.0.0.0 + port 9999 ``` {% endtab %} @@ -517,14 +512,13 @@ Run this configuration file with Prometheus to collect the metrics from the Flue ```yaml # config global: - scrape_interval: 5s + scrape_interval: 5s scrape_configs: - - # Scraping Fluent Bit example. - - job_name: "fluentbit" - static_configs: - - targets: ["localhost:9999"] + # Scraping Fluent Bit example. + - job_name: "fluentbit" + static_configs: + - targets: ["localhost:9999"] ``` {% endtab %} diff --git a/pipeline/filters/lua.md b/pipeline/filters/lua.md index 429a3d97f..a63f0bd4b 100644 --- a/pipeline/filters/lua.md +++ b/pipeline/filters/lua.md @@ -31,7 +31,7 @@ To test the Lua filter, you can run the plugin from the command line or through From the command line you can use the following options: ```shell -./fluent-bit -i dummy -F lua -p script=test.lua -p call=cb_print -m '*' -o null +fluent-bit -i dummy -F lua -p script=test.lua -p call=cb_print -m '*' -o null ``` ### Configuration file @@ -94,8 +94,8 @@ The Lua script can have one or multiple callbacks that can be used by this filte ```lua function cb_print(tag, timestamp, record) - ... - return code, timestamp, record + ... + return code, timestamp, record end ``` @@ -129,8 +129,8 @@ For more advanced use cases, especially when working with structured formats lik ```lua function cb_metadata(tag, timestamp, group, metadata, record) - ... - return code, timestamp, metadata, record + ... + return code, timestamp, metadata, record end ``` @@ -170,15 +170,15 @@ When using the extended prototype, you can return multiple records with their re ```lua function cb_metadata(tag, ts, group, metadata, record) - -- first record with its metadata - m1 = {foo = "meta1"} - r1 = {msg = "first log", old_record = record} + -- first record with its metadata + m1 = {foo = "meta1"} + r1 = {msg = "first log", old_record = record} - -- second record with its metadata - m2 = {foo = "meta2"} - r2 = {msg = "second log", old_record = record} + -- second record with its metadata + m2 = {foo = "meta2"} + r2 = {msg = "second log", old_record = record} - return 1, ts, {m1, m2}, {r1, r2} + return 1, ts, {m1, m2}, {r1, r2} end ``` @@ -298,11 +298,11 @@ pipeline: match: '*' call: append_tag code: | - function append_tag(tag, timestamp, record) - new_record = record - new_record["my_env"] = FLB_ENV - return 1, timestamp, new_record - end + function append_tag(tag, timestamp, record) + new_record = record + new_record["my_env"] = FLB_ENV + return 1, timestamp, new_record + end outputs: - name: stdout @@ -314,24 +314,24 @@ pipeline: ```text [SERVICE] - flush 1 - daemon off - log_level debug + flush 1 + daemon off + log_level debug [INPUT] - Name random - Tag test - Samples 10 + Name random + Tag test + Samples 10 [FILTER] - Name Lua - Match * - call append_tag - code function append_tag(tag, timestamp, record) new_record = record new_record["tag"] = tag return 1, timestamp, new_record end + Name Lua + Match * + call append_tag + code function append_tag(tag, timestamp, record) new_record = record new_record["tag"] = tag return 1, timestamp, new_record end [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} @@ -379,11 +379,11 @@ pipeline: ```text [FILTER] -Name lua -Alias filter-iots-lua -Match iots_thread.* -Script filters.lua -Call set_landscape_deployment + Name lua + Alias filter-iots-lua + Match iots_thread.* + Script filters.lua + Call set_landscape_deployment ``` {% endtab %} @@ -394,24 +394,24 @@ filters.lua: -- Use a Lua function to create some additional entries based -- on substrings from the kubernetes properties. function set_landscape_deployment(tag, timestamp, record) - local landscape = os.getenv("KUBERNETES_SERVICE_HOST") - if landscape then - -- Strip the landscape name from this field, KUBERNETES_SERVICE_HOST - -- Should be of this format - -- api.sandboxbsh-a.project.domain.com - -- Take off the leading "api." - -- sandboxbsh-a.project.domain.com - --print("landscape1:" .. landscape) - landscape = landscape:gsub("^[^.]+.", "") - --print("landscape2:" .. landscape) - -- Take off everything including and after the - in the cluster name - -- sandboxbsh - landscape = landscape:gsub("-.*$", "") - -- print("landscape3:" .. landscape) - record["iot_landscape"] = landscape - end - -- 2 - replace existing record with this update - return 2, timestamp, record + local landscape = os.getenv("KUBERNETES_SERVICE_HOST") + if landscape then + -- Strip the landscape name from this field, KUBERNETES_SERVICE_HOST + -- Should be of this format + -- api.sandboxbsh-a.project.domain.com + -- Take off the leading "api." + -- sandboxbsh-a.project.domain.com + --print("landscape1:" .. landscape) + landscape = landscape:gsub("^[^.]+.", "") + --print("landscape2:" .. landscape) + -- Take off everything including and after the - in the cluster name + -- sandboxbsh + landscape = landscape:gsub("-.*$", "") + -- print("landscape3:" .. landscape) + record["iot_landscape"] = landscape + end + -- 2 - replace existing record with this update + return 2, timestamp, record end ``` @@ -425,11 +425,11 @@ For example: ```lua function cb_split(tag, timestamp, record) - if record["x"] ~= nil then - return 2, timestamp, record["x"] - else - return 2, timestamp, record - end + if record["x"] ~= nil then + return 2, timestamp, record["x"] + else + return 2, timestamp, record + end end ``` @@ -458,17 +458,17 @@ pipeline: ```text [Input] - Name stdin + Name stdin [Filter] - Name lua - Match * - script test.lua - call cb_split + Name lua + Match * + script test.lua + call cb_split [Output] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} @@ -550,22 +550,22 @@ pipeline: ```text [INPUT] - Name tail - Path /var/log/containers/*_istio-proxy-*.log - multiline.parser docker, cri - Tag istio.* - Mem_Buf_Limit 64MB - Skip_Long_Lines Off + Name tail + Path /var/log/containers/*_istio-proxy-*.log + multiline.parser docker, cri + Tag istio.* + Mem_Buf_Limit 64MB + Skip_Long_Lines Off [FILTER] - Name lua - Match istio.* - Script response_code_filter.lua - call cb_response_code_filter + Name lua + Match istio.* + Script response_code_filter.lua + call cb_response_code_filter [Output] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} @@ -575,34 +575,34 @@ pipeline: ```json { - "log": { - "response_code": 200, - "bytes_sent": 111328341, - "authority": "randomservice.randomservice", - "duration": 14493, - "request_id": "2e9d38f8-36a9-40a6-bdb2-47c8eb7d399d", - "upstream_local_address": "10.11.82.178:42738", - "downstream_local_address": "10.10.21.17:80", - "upstream_cluster": "outbound|80||randomservice.svc.cluster.local", - "x_forwarded_for": null, - "route_name": "default", - "upstream_host": "10.11.6.90:80", - "user_agent": "RandomUserAgent", - "response_code_details": "via_upstream", - "downstream_remote_address": "10.11.82.178:51096", - "bytes_received": 1148, - "path": "/?parameter=random", - "response_flags": "-", - "start_time": "2022-07-28T11:16:51.663Z", - "upstream_transport_failure_reason": null, - "method": "POST", - "connection_termination_details": null, - "protocol": "HTTP/1.1", - "requested_server_name": null, - "upstream_service_time": "6161" - }, - "stream": "stdout", - "time": "2022-07-28T11:17:06.704109897Z" + "log": { + "response_code": 200, + "bytes_sent": 111328341, + "authority": "randomservice.randomservice", + "duration": 14493, + "request_id": "2e9d38f8-36a9-40a6-bdb2-47c8eb7d399d", + "upstream_local_address": "10.11.82.178:42738", + "downstream_local_address": "10.10.21.17:80", + "upstream_cluster": "outbound|80||randomservice.svc.cluster.local", + "x_forwarded_for": null, + "route_name": "default", + "upstream_host": "10.11.6.90:80", + "user_agent": "RandomUserAgent", + "response_code_details": "via_upstream", + "downstream_remote_address": "10.11.82.178:51096", + "bytes_received": 1148, + "path": "/?parameter=random", + "response_flags": "-", + "start_time": "2022-07-28T11:16:51.663Z", + "upstream_transport_failure_reason": null, + "method": "POST", + "connection_termination_details": null, + "protocol": "HTTP/1.1", + "requested_server_name": null, + "upstream_service_time": "6161" + }, + "stream": "stdout", + "time": "2022-07-28T11:17:06.704109897Z" } ``` @@ -620,23 +620,23 @@ Script `custom_datetime_format.lua`: ```lua function convert_to_utc(tag, timestamp, record) - local date_time = record["pub_date"] - local new_record = record - if date_time then - if string.find(date_time, ",") then - local pattern = "(%a+, %d+ %a+ %d+ %d+:%d+:%d+) ([+-]%d%d%d%d)" - local date_part, zone_part = date_time:match(pattern) - - if date_part and zone_part then - local command = string.format("date -u -d '%s %s' +%%Y-%%m-%%dT%%H:%%M:%%SZ", date_part, zone_part) - local handle = io.popen(command) - local result = handle:read("*a") - handle:close() - new_record["pub_date"] = result:match("%S+") - end - end + local date_time = record["pub_date"] + local new_record = record + if date_time then + if string.find(date_time, ",") then + local pattern = "(%a+, %d+ %a+ %d+ %d+:%d+:%d+) ([+-]%d%d%d%d)" + local date_part, zone_part = date_time:match(pattern) + + if date_part and zone_part then + local command = string.format("date -u -d '%s %s' +%%Y-%%m-%%dT%%H:%%M:%%SZ", date_part, zone_part) + local handle = io.popen(command) + local result = handle:read("*a") + handle:close() + new_record["pub_date"] = result:match("%S+") + end end - return 1, timestamp, new_record + end + return 1, timestamp, new_record end ``` @@ -662,24 +662,24 @@ pipeline: - name: lua match: '*' code: | - function convert_to_utc(tag, timestamp, record) - local date_time = record["pub_date"] - local new_record = record - if date_time then - if string.find(date_time, ",") then - local pattern = "(%a+, %d+ %a+ %d+ %d+:%d+:%d+) ([+-]%d%d%d%d)" - local date_part, zone_part = date_time:match(pattern) - if date_part and zone_part then - local command = string.format("date -u -d '%s %s' +%%Y-%%m-%%dT%%H:%%M:%%SZ", date_part, zone_part) - local handle = io.popen(command) - local result = handle:read("*a") - handle:close() - new_record["pub_date"] = result:match("%S+") - end - end + function convert_to_utc(tag, timestamp, record) + local date_time = record["pub_date"] + local new_record = record + if date_time then + if string.find(date_time, ",") then + local pattern = "(%a+, %d+ %a+ %d+ %d+:%d+:%d+) ([+-]%d%d%d%d)" + local date_part, zone_part = date_time:match(pattern) + if date_part and zone_part then + local command = string.format("date -u -d '%s %s' +%%Y-%%m-%%dT%%H:%%M:%%SZ", date_part, zone_part) + local handle = io.popen(command) + local result = handle:read("*a") + handle:close() + new_record["pub_date"] = result:match("%S+") end - return 1, timestamp, new_record - end + end + end + return 1, timestamp, new_record + end call: convert_to_utc outputs: @@ -691,24 +691,24 @@ pipeline: ```text [INPUT] - Name dummy - Dummy {"event": "Restock", "pub_date": "Tue, 30 Jul 2024 18:01:06 +0000"} - Tag event_category_a + Name dummy + Dummy {"event": "Restock", "pub_date": "Tue, 30 Jul 2024 18:01:06 +0000"} + Tag event_category_a [INPUT] - Name dummy - Dummy {"event": "Soldout", "pub_date": "Mon, 29 Jul 2024 10:15:00 +0600"} - Tag event_category_b + Name dummy + Dummy {"event": "Soldout", "pub_date": "Mon, 29 Jul 2024 10:15:00 +0600"} + Tag event_category_b [FILTER] - Name lua - Match * - Script custom_datetime_format.lua - call convert_to_utc + Name lua + Match * + Script custom_datetime_format.lua + call convert_to_utc [Output] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} @@ -781,11 +781,11 @@ pipeline: match: '*' call: append_tag code: | - function append_tag(tag, timestamp, record) - new_record = record - new_record["my_env"] = FLB_ENV - return 1, timestamp, new_record - end + function append_tag(tag, timestamp, record) + new_record = record + new_record["my_env"] = FLB_ENV + return 1, timestamp, new_record + end outputs: - name: stdout @@ -799,4 +799,4 @@ pipeline: ```text test: [[1731990257.781970977, {}], {"my_env"=>{"A"=>"aaa", "C"=>"ccc", "HOSTNAME"=>"monox-2.lan", "B"=>"bbb"}, "rand_value"=>4805047635809401856}] -``` +``` \ No newline at end of file diff --git a/pipeline/filters/modify.md b/pipeline/filters/modify.md index 115e9bc94..cc3b4a3af 100644 --- a/pipeline/filters/modify.md +++ b/pipeline/filters/modify.md @@ -50,7 +50,7 @@ The plugin supports the following rules: | `Move_to_start` | `WILDCARD:KEY` | _none_ | Move key/value pairs with keys matching `KEY` to the start of the message. | | `Move_to_end` | `WILDCARD:KEY` | _none_ | Move key/value pairs with keys matching `KEY` to the end of the message. | -- Rules are case insensitive, but parameters aren't. +- Rules are case-insensitive, but parameters aren't. - Any number of rules can be set in a filter instance. - Rules are applied in the order they appear, with each rule operating on the result of the previous rule. @@ -71,7 +71,7 @@ The plugin supports the following conditions: | `Matching_keys_have_matching_values` | `REGEXP:KEY` | `REGEXP:VALUE` | Is `true` if all keys matching `KEY` have values that match `VALUE`. | | `Matching_keys_do_not_have_matching_values` | `REGEXP:KEY` | `REGEXP:VALUE` | Is `true` if all keys matching `KEY` have values that don't match `VALUE`. | -- Conditions are case insensitive, but parameters aren't. +- Conditions are case-insensitive, but parameters aren't. - Any number of conditions can be set. - Conditions apply to the whole filter instance and all its rules. _Not_ to individual rules. - All conditions must be `true` for the rules to be applied. @@ -95,18 +95,18 @@ which outputs data similar to the following: Using the command line mode requires quotes parse the wildcard properly. The use of a configuration file is recommended. ```shell -./fluent-bit -i mem \ - -p 'tag=mem.local' \ - -F modify \ - -p 'Add=Service1 SOMEVALUE' \ - -p 'Add=Service2 SOMEVALUE3' \ - -p 'Add=Mem.total2 TOTALMEM2' \ - -p 'Rename=Mem.free MEMFREE' \ - -p 'Rename=Mem.used MEMUSED' \ - -p 'Rename=Swap.total SWAPTOTAL' \ - -p 'Add=Mem.total TOTALMEM' \ - -m '*' \ - -o stdout +fluent-bit -i mem \ + -p 'tag=mem.local' \ + -F modify \ + -p 'Add=Service1 SOMEVALUE' \ + -p 'Add=Service2 SOMEVALUE3' \ + -p 'Add=Mem.total2 TOTALMEM2' \ + -p 'Rename=Mem.free MEMFREE' \ + -p 'Rename=Mem.used MEMUSED' \ + -p 'Rename=Swap.total SWAPTOTAL' \ + -p 'Add=Mem.total TOTALMEM' \ + -m '*' \ + -o stdout ``` ### Configuration file @@ -116,26 +116,26 @@ Using the command line mode requires quotes parse the wildcard properly. The use ```yaml pipeline: - inputs: - - name: mem - tag: mem.local - - filters: - - name: modify - match: '*' - Add: - - Service1 SOMEVALUE - - Service3 SOMEVALUE3 - - Mem.total2 TOTALMEM2 - - Mem.total TOTALMEM - Rename: - - Mem.free MEMFREE - - Mem.used MEMUSED - - Swap.total SWAPTOTAL - - outputs: - - name: stdout - match: '*' + inputs: + - name: mem + tag: mem.local + + filters: + - name: modify + match: '*' + Add: + - Service1 SOMEVALUE + - Service3 SOMEVALUE3 + - Mem.total2 TOTALMEM2 + - Mem.total TOTALMEM + Rename: + - Mem.free MEMFREE + - Mem.used MEMUSED + - Swap.total SWAPTOTAL + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -143,23 +143,23 @@ pipeline: ```text [INPUT] - Name mem - Tag mem.local + Name mem + Tag mem.local [FILTER] - Name modify - Match * - Add Service1 SOMEVALUE - Add Service3 SOMEVALUE3 - Add Mem.total2 TOTALMEM2 - Rename Mem.free MEMFREE - Rename Mem.used MEMUSED - Rename Swap.total SWAPTOTAL - Add Mem.total TOTALMEM + Name modify + Match * + Add Service1 SOMEVALUE + Add Service3 SOMEVALUE3 + Add Mem.total2 TOTALMEM2 + Rename Mem.free MEMFREE + Rename Mem.used MEMUSED + Rename Swap.total SWAPTOTAL + Add Mem.total TOTALMEM [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} @@ -186,35 +186,35 @@ The output of both the command line and configuration invocations should be iden ```yaml pipeline: - inputs: - - name: mem - tag: mem.local - interval_sec: 1 - - filters: - - name: modify - match: mem.* - Condition: - - Key_Does_Not_Exist cpustats - - Key_Exists Mem.used - Set: cpustats UNKNOWN - - - name: modify - match: mem.* - Condition: Key_Value_Does_Not_Equal cpustats KNOWN - Add: sourcetype memstats - - - name: modify - match: mem.* - Condition: Key_Value_Equals cpustats UNKNOWN - Remove_wildcard: - - Mem - - Swap - Add: cpustats_more STILL_UNKNOWN - - outputs: - - name: stdout - match: '*' + inputs: + - name: mem + tag: mem.local + interval_sec: 1 + + filters: + - name: modify + match: mem.* + Condition: + - Key_Does_Not_Exist cpustats + - Key_Exists Mem.used + Set: cpustats UNKNOWN + + - name: modify + match: mem.* + Condition: Key_Value_Does_Not_Equal cpustats KNOWN + Add: sourcetype memstats + + - name: modify + match: mem.* + Condition: Key_Value_Equals cpustats UNKNOWN + Remove_wildcard: + - Mem + - Swap + Add: cpustats_more STILL_UNKNOWN + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -222,34 +222,34 @@ pipeline: ```text [INPUT] - Name mem - Tag mem.local - Interval_Sec 1 + Name mem + Tag mem.local + Interval_Sec 1 [FILTER] - Name modify - Match mem.* - Condition Key_Does_Not_Exist cpustats - Condition Key_Exists Mem.used - Set cpustats UNKNOWN + Name modify + Match mem.* + Condition Key_Does_Not_Exist cpustats + Condition Key_Exists Mem.used + Set cpustats UNKNOWN [FILTER] - Name modify - Match mem.* - Condition Key_Value_Does_Not_Equal cpustats KNOWN - Add sourcetype memstats + Name modify + Match mem.* + Condition Key_Value_Does_Not_Equal cpustats KNOWN + Add sourcetype memstats [FILTER] - Name modify - Match mem.* - Condition Key_Value_Equals cpustats UNKNOWN - Remove_wildcard Mem - Remove_wildcard Swap - Add cpustats_more STILL_UNKNOWN + Name modify + Match mem.* + Condition Key_Value_Equals cpustats UNKNOWN + Remove_wildcard Mem + Remove_wildcard Swap + Add cpustats_more STILL_UNKNOWN [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} @@ -274,28 +274,28 @@ pipeline: ```yaml pipeline: - inputs: - - name: mem - tag: mem.local - interval_sec: 1 - - filters: - - name: modify - match: mem.* - Remove_wildcard: - - Mem - - Swap - set: - - This_plugin_is_on 🔥 - - 🔥 is_hot - - ❄️ is_cold - - 💦 is_wet - copy: 🔥 💦 - rename: 💦 ❄️ - - outputs: - - name: stdout - match: '*' + inputs: + - name: mem + tag: mem.local + interval_sec: 1 + + filters: + - name: modify + match: mem.* + Remove_wildcard: + - Mem + - Swap + set: + - This_plugin_is_on 🔥 + - 🔥 is_hot + - ❄️ is_cold + - 💦 is_wet + copy: 🔥 💦 + rename: 💦 ❄️ + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -303,24 +303,24 @@ pipeline: ```text [INPUT] - Name mem - Tag mem.local + Name mem + Tag mem.local [FILTER] - Name modify - Match * - Remove_Wildcard Mem - Remove_Wildcard Swap - Set This_plugin_is_on 🔥 - Set 🔥 is_hot - Copy 🔥 💦 - Rename 💦 ❄️ - Set ❄️ is_cold - Set 💦 is_wet + Name modify + Match * + Remove_Wildcard Mem + Remove_Wildcard Swap + Set This_plugin_is_on 🔥 + Set 🔥 is_hot + Copy 🔥 💦 + Rename 💦 ❄️ + Set ❄️ is_cold + Set 💦 is_wet [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} diff --git a/pipeline/filters/multiline-stacktrace.md b/pipeline/filters/multiline-stacktrace.md index a3017fdbc..4a904595b 100644 --- a/pipeline/filters/multiline-stacktrace.md +++ b/pipeline/filters/multiline-stacktrace.md @@ -6,7 +6,7 @@ description: Concatenate multiline or stack trace log messages. The Multiline filter helps concatenate messages that originally belonged to one context but were split across multiple records or log lines. Common examples are stack traces or applications that print logs in multiple lines. -Along with multiline filters, you can enable one of the following built-in Fluent Bit parsers with auto detection and multi-format support: +Along with multiline filters, you can enable one of the following built-in Fluent Bit parsers with auto-detection and multi-format support: - Go - Python @@ -22,7 +22,7 @@ When using this filter: This filter only performs buffering that persists across different Chunks when `Buffer` is enabled. Otherwise, the filter processes one chunk at a time and isn't suitable for most inputs which might send multiline messages in separate chunks. -When buffering is enabled, the filter doesn't immediately emit messages it receives. It uses the `in_emitter` plugin, similar to the [Rewrite Tag filter](./rewrite-tag.md), and emits messages once they're fully concatenated, or a timeout is reached. +When buffering is enabled, the filter doesn't immediately emit messages it receives. It uses the `in_emitter` plugin, similar to the [Rewrite Tag filter](rewrite-tag.md), and emits messages once they're fully concatenated, or a timeout is reached. {% endhint %} @@ -30,7 +30,7 @@ When buffering is enabled, the filter doesn't immediately emit messages it recei Since concatenated records are re-emitted to the head of the Fluent Bit log pipeline, you can not configure multiple multiline filter definitions that match the same tags. This will cause an infinite loop in the Fluent Bit pipeline; to use multiple parsers on the same logs, configure a single filter definitions with a comma separated list of parsers for `multiline.parser`. For more, see issue [#5235](https://github.com/fluent/fluent-bit/issues/5235). -Secondly, for the same reason, the multiline filter should be the first filter. Logs will be re-emitted by the multiline filter to the head of the pipeline- the filter will ignore its own re-emitted records, but other filters won't. If there are filters before the multiline filter, they will be applied twice. +Secondly, for the same reason, the multiline filter should be the first filter. Logs will be re-emitted by the multiline filter to the head of the pipeline - the filter will ignore its own re-emitted records, but other filters won't. If there are filters before the multiline filter, they will be applied twice. {% endhint %} @@ -65,25 +65,25 @@ This is the primary Fluent Bit YAML configuration file. It includes the `parsers ```yaml service: - flush: 1 - log_level: info - parsers_file: parsers_multiline.yaml - + flush: 1 + log_level: info + parsers_file: parsers_multiline.yaml + pipeline: - inputs: - - name: tail - path: test.log - read_from_head: true - - filters: - - name: multiline - match: '*' - multiline.key_content: log - multiline.parser: go,multiline-regex-test - - outputs: - - name: stdout - match: '*' + inputs: + - name: tail + path: test.log + read_from_head: true + + filters: + - name: multiline + match: '*' + multiline.key_content: log + multiline.parser: go,multiline-regex-test + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -93,25 +93,24 @@ This is the primary Fluent Bit classic configuration file. It includes the `pars ```text [SERVICE] - flush 1 - log_level info - parsers_file parsers_multiline.conf + flush 1 + log_level info + parsers_file parsers_multiline.conf [INPUT] - name tail - path test.log - read_from_head true + name tail + path test.log + read_from_head true [FILTER] - name multiline - match * - multiline.key_content log - multiline.parser go, multiline-regex-test + name multiline + match * + multiline.key_content log + multiline.parser go, multiline-regex-test [OUTPUT] - name stdout - match * - + name stdout + match * ``` {% endtab %} @@ -121,39 +120,9 @@ This file defines a multiline parser for the example. A second multiline parser ```yaml multiline_parsers: - - name: multiline-regex-test - type: regex - flush_timeout: 1000 - # - # Regex rules for multiline parsing - # --------------------------------- - # - # configuration hints: - # - # - first state always has the name: start_state - # - every field in the rule must be inside double quotes - # - # rules | state name | regex pattern | next state - # ------|---------------|-------------------------------------------- - rules: - - state: start_state - regex: '/([a-zA-Z]+ \d+ \d+\:\d+\:\d+)(.*)/' - next_state: cont - - state: cont - regex: '/^\s+at.*/' - next_state: cont -``` - -{% endtab %} -{% tab title="parsers_multiline.conf" %} - -This file defines a multiline parser for the example. A second multiline parser called `go` is used in `fluent-bit.conf`, but this one is a built-in parser. - -```text -[MULTILINE_PARSER] - name multiline-regex-test - type regex - flush_timeout 1000 + - name: multiline-regex-test + type: regex + flush_timeout: 1000 # # Regex rules for multiline parsing # --------------------------------- @@ -165,9 +134,38 @@ This file defines a multiline parser for the example. A second multiline parser # # rules | state name | regex pattern | next state # ------|---------------|-------------------------------------------- - rule "start_state" "/([A-Za-z]+ \d+ \d+\:\d+\:\d+)(.*)/" "cont" - rule "cont" "/^\s+at.*/" "cont" + rules: + - state: start_state + regex: '/([a-zA-Z]+ \d+ \d+\:\d+\:\d+)(.*)/' + next_state: cont + - state: cont + regex: '/^\s+at.*/' + next_state: cont +``` + +{% endtab %} +{% tab title="parsers_multiline.conf" %} + +This file defines a multiline parser for the example. A second multiline parser called `go` is used in `fluent-bit.conf`, but this one is a built-in parser. +```text +[MULTILINE_PARSER] + name multiline-regex-test + type regex + flush_timeout 1000 + # + # Regex rules for multiline parsing + # --------------------------------- + # + # configuration hints: + # + # - first state always has the name: start_state + # - every field in the rule must be inside double quotes + # + # rules | state name | regex pattern | next state + # ------|---------------|-------------------------------------------- + rule "start_state" "/([A-Za-z]+ \d+ \d+\:\d+\:\d+)(.*)/" "cont" + rule "cont" "/^\s+at.*/" "cont" ``` {% endtab %} @@ -236,7 +234,6 @@ runtime.goexit() created by runtime.gcenable /usr/local/go/src/runtime/mgc.go:216 +0x58 one more line, no multiline - ``` {% endtab %} @@ -245,7 +242,7 @@ one more line, no multiline Running Fluent Bit with the given configuration file: ```shell -./fluent-bit -c fluent-bit.conf +fluent-bit -c fluent-bit.conf ``` Should return something like the following: @@ -311,7 +308,6 @@ runtime.goexit() created by runtime.gcenable /usr/local/go/src/runtime/mgc.go:216 +0x58"}] [4] tail.0: [1626736433.143585473, {"log"=>"one more line, no multiline"}] - ``` Lines that don't match a pattern aren't considered as part of the multiline message, while the ones that matched the rules were concatenated properly. @@ -332,11 +328,11 @@ Fluent Bit can re-combine these logs that were split by the runtime and remove t ```yaml pipeline: - filters: - - name: multiline - match: '*' - multiline.key_content: log - mode: partial_message + filters: + - name: multiline + match: '*' + multiline.key_content: log + mode: partial_message ``` {% endtab %} @@ -344,10 +340,10 @@ pipeline: ```text [FILTER] - name multiline - match * - multiline.key_content log - mode partial_message + name multiline + match * + multiline.key_content log + mode partial_message ``` {% endtab %} diff --git a/pipeline/filters/nest.md b/pipeline/filters/nest.md index 84466ba07..2f680ddd6 100644 --- a/pipeline/filters/nest.md +++ b/pipeline/filters/nest.md @@ -87,7 +87,7 @@ Using the command line mode requires quotes to parse the wildcard properly. The The following command loads the _mem_ plugin. Then the _nest_ filter matches the wildcard rule to the keys and nests the keys matching `Mem.*` under the new key `NEST`. ```shell -./fluent-bit -i mem -p 'tag=mem.local' -F nest -p 'Operation=nest' -p 'Wildcard=Mem.*' -p 'Nest_under=Memstats' -p 'Remove_prefix=Mem.' -m '*' -o stdout +fluent-bit -i mem -p 'tag=mem.local' -F nest -p 'Operation=nest' -p 'Wildcard=Mem.*' -p 'Nest_under=Memstats' -p 'Remove_prefix=Mem.' -m '*' -o stdout ``` ### Nest configuration file @@ -97,21 +97,21 @@ The following command loads the _mem_ plugin. Then the _nest_ filter matches the ```yaml pipeline: - inputs: - - name: mem - tag: mem.local - - filters: - - name: nest - match: '*' - operation: nest - wildcard: Mem.* - nest_under: Memstats - remove_prefix: Mem. - - outputs: - - name: stdout - match: '*' + inputs: + - name: mem + tag: mem.local + + filters: + - name: nest + match: '*' + operation: nest + wildcard: Mem.* + nest_under: Memstats + remove_prefix: Mem. + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -119,20 +119,20 @@ pipeline: ```text [INPUT] - Name mem - Tag mem.local + Name mem + Tag mem.local [FILTER] - Name nest - Match * - Operation nest - Wildcard Mem.* - Nest_under Memstats - Remove_prefix Mem. - + Name nest + Match * + Operation nest + Wildcard Mem.* + Nest_under Memstats + Remove_prefix Mem. + [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} @@ -158,29 +158,29 @@ This example nests all `Mem.*` and `Swap.*` items under the `Stats` key and then ```yaml pipeline: - inputs: - - name: mem - tag: mem.local - - filters: - - name: nest - match: '*' - Operation: nest - Wildcard: - - Mem.* - - Swap.* - Nest_under: Stats - Add_prefix: NESTED - - - name: nest - match: '*' - Operation: lift - Nested_under: Stats - Remove_prefix: NESTED - - outputs: - - name: stdout - match: '*' + inputs: + - name: mem + tag: mem.local + + filters: + - name: nest + match: '*' + Operation: nest + Wildcard: + - Mem.* + - Swap.* + Nest_under: Stats + Add_prefix: NESTED + + - name: nest + match: '*' + Operation: lift + Nested_under: Stats + Remove_prefix: NESTED + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -188,28 +188,28 @@ pipeline: ```text [INPUT] - Name mem - Tag mem.local + Name mem + Tag mem.local [FILTER] - Name nest - Match * - Operation nest - Wildcard Mem.* - Wildcard Swap.* - Nest_under Stats - Add_prefix NESTED + Name nest + Match * + Operation nest + Wildcard Mem.* + Wildcard Swap.* + Nest_under Stats + Add_prefix NESTED [FILTER] - Name nest - Match * - Operation lift - Nested_under Stats - Remove_prefix NESTED - + Name nest + Match * + Operation lift + Nested_under Stats + Remove_prefix NESTED + [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} @@ -233,32 +233,32 @@ This example takes the keys starting with `Mem.*` and nests them under `LAYER1`, ```yaml pipeline: - inputs: - - name: mem - tag: mem.local - - filters: - - name: nest - match: '*' - Operation: nest - Wildcard: Mem.* - Nest_under: LAYER1 - - - name: nest - match: '*' - Operation: nest - Wildcard: LAYER1* - Nest_under: LAYER2 - - - name: nest - match: '*' - Operation: nest - Wildcard: LAYER2* - Nest_under: LAYER3 - - outputs: - - name: stdout - match: '*' + inputs: + - name: mem + tag: mem.local + + filters: + - name: nest + match: '*' + Operation: nest + Wildcard: Mem.* + Nest_under: LAYER1 + + - name: nest + match: '*' + Operation: nest + Wildcard: LAYER1* + Nest_under: LAYER2 + + - name: nest + match: '*' + Operation: nest + Wildcard: LAYER2* + Nest_under: LAYER3 + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -266,33 +266,33 @@ pipeline: ```text [INPUT] - Name mem - Tag mem.local + Name mem + Tag mem.local [FILTER] - Name nest - Match * - Operation nest - Wildcard Mem.* - Nest_under LAYER1 + Name nest + Match * + Operation nest + Wildcard Mem.* + Nest_under LAYER1 [FILTER] - Name nest - Match * - Operation nest - Wildcard LAYER1* - Nest_under LAYER2 + Name nest + Match * + Operation nest + Wildcard LAYER1* + Nest_under LAYER2 [FILTER] - Name nest - Match * - Operation nest - Wildcard LAYER2* - Nest_under LAYER3 + Name nest + Match * + Operation nest + Wildcard LAYER2* + Nest_under LAYER3 [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} @@ -331,50 +331,50 @@ This example uses the 3-level deep nesting of Example 2 and applies the `lift` f ```yaml pipeline: - inputs: - - name: mem - tag: mem.local - - filters: - - name: nest - match: '*' - Operation: nest - Wildcard: Mem.* - Nest_under: LAYER1 - - - name: nest - match: '*' - Operation: nest - Wildcard: LAYER1* - Nest_under: LAYER2 - - - name: nest - match: '*' - Operation: nest - Wildcard: LAYER2* - Nest_under: LAYER3 - - - name: nest - match: '*' - Operation: lift - Nested_under: LAYER3 - Add_prefix: Lifted3_ - - - name: nest - match: '*' - Operation: lift - Nested_under: Lifted3_LAYER2 - Add_prefix: Lifted3_Lifted2_ - - - name: nest - match: '*' - Operation: lift - Nested_under: Lifted3_Lifted2_LAYER1 - Add_prefix: Lifted3_Lifted2_Lifted1_ - - outputs: - - name: stdout - match: '*' + inputs: + - name: mem + tag: mem.local + + filters: + - name: nest + match: '*' + Operation: nest + Wildcard: Mem.* + Nest_under: LAYER1 + + - name: nest + match: '*' + Operation: nest + Wildcard: LAYER1* + Nest_under: LAYER2 + + - name: nest + match: '*' + Operation: nest + Wildcard: LAYER2* + Nest_under: LAYER3 + + - name: nest + match: '*' + Operation: lift + Nested_under: LAYER3 + Add_prefix: Lifted3_ + + - name: nest + match: '*' + Operation: lift + Nested_under: Lifted3_LAYER2 + Add_prefix: Lifted3_Lifted2_ + + - name: nest + match: '*' + Operation: lift + Nested_under: Lifted3_Lifted2_LAYER1 + Add_prefix: Lifted3_Lifted2_Lifted1_ + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -382,54 +382,54 @@ pipeline: ```text [INPUT] - Name mem - Tag mem.local + Name mem + Tag mem.local [FILTER] - Name nest - Match * - Operation nest - Wildcard Mem.* - Nest_under LAYER1 + Name nest + Match * + Operation nest + Wildcard Mem.* + Nest_under LAYER1 [FILTER] - Name nest - Match * - Operation nest - Wildcard LAYER1* - Nest_under LAYER2 + Name nest + Match * + Operation nest + Wildcard LAYER1* + Nest_under LAYER2 [FILTER] - Name nest - Match * - Operation nest - Wildcard LAYER2* - Nest_under LAYER3 + Name nest + Match * + Operation nest + Wildcard LAYER2* + Nest_under LAYER3 [FILTER] - Name nest - Match * - Operation lift - Nested_under LAYER3 - Add_prefix Lifted3_ + Name nest + Match * + Operation lift + Nested_under LAYER3 + Add_prefix Lifted3_ [FILTER] - Name nest - Match * - Operation lift - Nested_under Lifted3_LAYER2 - Add_prefix Lifted3_Lifted2_ + Name nest + Match * + Operation lift + Nested_under Lifted3_LAYER2 + Add_prefix Lifted3_Lifted2_ [FILTER] - Name nest - Match * - Operation lift - Nested_under Lifted3_Lifted2_LAYER1 - Add_prefix Lifted3_Lifted2_Lifted1_ + Name nest + Match * + Operation lift + Nested_under Lifted3_Lifted2_LAYER1 + Add_prefix Lifted3_Lifted2_Lifted1_ [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} diff --git a/pipeline/filters/nightfall.md b/pipeline/filters/nightfall.md index 39a35773d..73cfd9836 100644 --- a/pipeline/filters/nightfall.md +++ b/pipeline/filters/nightfall.md @@ -26,22 +26,22 @@ The following is an example of a configuration file for the Nightfall filter: ```yaml pipeline: - inputs: - - name: http - host: 0.0.0.0 - port: 8000 - - filters: - - name: nightfall - match: '*' - nightfall_api_key: - policy_id: 5991946b-1cc8-4c38-9240-72677029a3f7 - sampling_rate: 1 - tls.ca_path: /etc/ssl/certs - - outputs: - - name: stdout - match: '*' + inputs: + - name: http + host: 0.0.0.0 + port: 8000 + + filters: + - name: nightfall + match: '*' + nightfall_api_key: + policy_id: 5991946b-1cc8-4c38-9240-72677029a3f7 + sampling_rate: 1 + tls.ca_path: /etc/ssl/certs + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -49,21 +49,21 @@ pipeline: ```text [INPUT] - name http - host 0.0.0.0 - port 8000 + name http + host 0.0.0.0 + port 8000 [FILTER] - Name nightfall - Match * - nightfall_api_key - policy_id 5991946b-1cc8-4c38-9240-72677029a3f7 - sampling_rate 1 - tls.ca_path /etc/ssl/certs + Name nightfall + Match * + nightfall_api_key + policy_id 5991946b-1cc8-4c38-9240-72677029a3f7 + sampling_rate 1 + tls.ca_path /etc/ssl/certs [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} @@ -75,10 +75,10 @@ After you configure the filter, you can use it from the command line by running ```shell # For YAML configuration. -./fluent-bit -c /PATH_TO_CONF_FILE/fluent-bit.yaml +fluent-bit -c /PATH_TO_CONF_FILE/fluent-bit.yaml # For classic configuration. -./fluent-bit -c /PATH_TO_CONF_FILE/fluent-bit.conf +fluent-bit -c /PATH_TO_CONF_FILE/fluent-bit.conf ``` Replace _`PATH_TO_CONF_FILE`_ with the path for where your filter configuration file diff --git a/pipeline/filters/parser.md b/pipeline/filters/parser.md index d492cc76f..18c208ad2 100644 --- a/pipeline/filters/parser.md +++ b/pipeline/filters/parser.md @@ -26,9 +26,9 @@ This is an example of parsing a record `{"data":"100 0.5 true This is example"}` ```yaml parsers: - - name: dummy_test - format: regex - regex: '^(?[^ ]+) (?[^ ]+) (?[^ ]+) (?.+)$' + - name: dummy_test + format: regex + regex: '^(?[^ ]+) (?[^ ]+) (?[^ ]+) (?.+)$' ``` {% endtab %} @@ -36,9 +36,9 @@ parsers: ```text [PARSER] - Name dummy_test - Format regex - Regex ^(?[^ ]+) (?[^ ]+) (?[^ ]+) (?.+)$ + Name dummy_test + Format regex + Regex ^(?[^ ]+) (?[^ ]+) (?[^ ]+) (?.+)$ ``` {% endtab %} @@ -51,23 +51,23 @@ The path of the parser file should be written in configuration file under the `[ ```yaml service: - parsers_file: /path/to/parsers.yaml + parsers_file: /path/to/parsers.yaml pipeline: - inputs: - - name: dummy - tag: dummy.data - dummy: '{"data":"100 0.5 true This is example"}' - - filters: - - name: parser - match: 'dummy.*' - key_name: data - parser: dummy_test - - outputs: - - name: stdout - match: '*' + inputs: + - name: dummy + tag: dummy.data + dummy: '{"data":"100 0.5 true This is example"}' + + filters: + - name: parser + match: 'dummy.*' + key_name: data + parser: dummy_test + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -75,22 +75,22 @@ pipeline: ```text [SERVICE] - Parsers_File /path/to/parsers.conf + Parsers_File /path/to/parsers.conf [INPUT] - Name dummy - Tag dummy.data - Dummy {"data":"100 0.5 true This is example"} + Name dummy + Tag dummy.data + Dummy {"data":"100 0.5 true This is example"} [FILTER] - Name parser - Match dummy.* - Key_Name data - Parser dummy_test + Name parser + Match dummy.* + Key_Name data + Parser dummy_test [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} @@ -100,37 +100,16 @@ The output when running the corresponding configuration is as follows: ```text # For YAML configuration. -$ ./fluent-bit --config fluent-bit.yaml +$ fluent-bit --config fluent-bit.yaml # For classic configuration. -$ ./fluent-bit --config fluent-bit.conf - -Fluent Bit v4.0.3 -* Copyright (C) 2015-2025 The Fluent Bit Authors -* Fluent Bit is a CNCF sub-project under the umbrella of Fluentd -* https://fluentbit.io - -______ _ _ ______ _ _ ___ _____ -| ___| | | | | ___ (_) | / || _ | -| |_ | |_ _ ___ _ __ | |_ | |_/ /_| |_ __ __/ /| || |/' | -| _| | | | | |/ _ \ '_ \| __| | ___ \ | __| \ \ / / /_| || /| | -| | | | |_| | __/ | | | |_ | |_/ / | |_ \ V /\___ |\ |_/ / -\_| |_|\__,_|\___|_| |_|\__| \____/|_|\__| \_/ |_(_)___/ - - -[2025/07/03 16:15:34] [ info] [fluent bit] version=4.0.3, commit=3a91b155d6, pid=23196 -[2025/07/03 16:15:34] [ info] [storage] ver=1.5.3, type=memory, sync=normal, checksum=off, max_chunks_up=128 -[2025/07/03 16:15:34] [ info] [simd ] disabled -[2025/07/03 16:15:34] [ info] [cmetrics] version=1.0.3 -[2025/07/03 16:15:34] [ info] [ctraces ] version=0.6.6 -[2025/07/03 16:15:34] [ info] [input:dummy:dummy.0] initializing -[2025/07/03 16:15:34] [ info] [input:dummy:dummy.0] storage_strategy='memory' (memory only) -[2025/07/03 16:15:34] [ info] [output:stdout:stdout.0] worker #0 started -[2025/07/03 16:15:34] [ info] [sp] stream processor started +$ fluent-bit --config fluent-bit.conf + +... [0] dummy.data: [[1750323528.603308000, {}], {"INT"=>"100", "FLOAT"=>"0.5", "BOOL"=>"true", "STRING"=>"This is example"}] [0] dummy.data: [[1750323529.603788000, {}], {"INT"=>"100", "FLOAT"=>"0.5", "BOOL"=>"true", "STRING"=>"This is example"}] [0] dummy.data: [[1750323530.604204000, {}], {"INT"=>"100", "FLOAT"=>"0.5", "BOOL"=>"true", "STRING"=>"This is example"}] -[0] dummy.data: [[1750323531.603961000, {}], {"INT"=>"100", "FLOAT"=>"0.5", "BOOL"=>"true", "STRING"=>"This is example"}] +[0] dummy.data: [[1750323531.603961000, {}], {"INT"=>"100", "FLOAT"=>"0.5", "BOOL"=>"true", "STRING"=>"This is example"}]e ``` You can see the records `{"data":"100 0.5 true This is example"}` are parsed. @@ -146,9 +125,9 @@ If you enable `Reserve_Data`, all other fields are preserved. First the contents ```yaml parsers: - - name: dummy_test - format: regex - regex: '^(?[^ ]+) (?[^ ]+) (?[^ ]+) (?.+)$' + - name: dummy_test + format: regex + regex: '^(?[^ ]+) (?[^ ]+) (?[^ ]+) (?.+)$' ``` {% endtab %} @@ -156,9 +135,9 @@ parsers: ```text [PARSER] - Name dummy_test - Format regex - Regex ^(?[^ ]+) (?[^ ]+) (?[^ ]+) (?.+)$ + Name dummy_test + Format regex + Regex ^(?[^ ]+) (?[^ ]+) (?[^ ]+) (?.+)$ ``` {% endtab %} @@ -171,24 +150,24 @@ Now add `Reserve_Data` to the filter section of the corresponding configuration ```yaml service: - parsers_file: /path/to/parsers.yaml + parsers_file: /path/to/parsers.yaml pipeline: - inputs: - - name: dummy - tag: dummy.data - dummy: '{"data":"100 0.5 true This is example", "key1":"value1", "key2":"value2"}' - - filters: - - name: parser - match: 'dummy.*' - key_name: data - parser: dummy_test - reserve_data: on - - outputs: - - name: stdout - match: '*' + inputs: + - name: dummy + tag: dummy.data + dummy: '{"data":"100 0.5 true This is example", "key1":"value1", "key2":"value2"}' + + filters: + - name: parser + match: 'dummy.*' + key_name: data + parser: dummy_test + reserve_data: on + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -196,23 +175,23 @@ pipeline: ```text [SERVICE] - Parsers_File /path/to/parsers.conf + Parsers_File /path/to/parsers.conf [INPUT] - Name dummy - Tag dummy.data - Dummy {"data":"100 0.5 true This is example", "key1":"value1", "key2":"value2"} + Name dummy + Tag dummy.data + Dummy {"data":"100 0.5 true This is example", "key1":"value1", "key2":"value2"} [FILTER] - Name parser - Match dummy.* - Key_Name data - Parser dummy_test - Reserve_Data On - - [OUTPUT] - Name stdout - Match * + Name parser + Match dummy.* + Key_Name data + Parser dummy_test + Reserve_Data On + +[OUTPUT] + Name stdout + Match * ``` {% endtab %} @@ -222,33 +201,12 @@ The output when running the corresponding configuration is as follows: ```text # For YAML configuration. -$ ./fluent-bit --config fluent-bit.yaml +$ fluent-bit --config fluent-bit.yaml # For classic configuration. -$ ./fluent-bit --config fluent-bit.conf - -Fluent Bit v4.0.3 -* Copyright (C) 2015-2025 The Fluent Bit Authors -* Fluent Bit is a CNCF sub-project under the umbrella of Fluentd -* https://fluentbit.io - -______ _ _ ______ _ _ ___ _____ -| ___| | | | | ___ (_) | / || _ | -| |_ | |_ _ ___ _ __ | |_ | |_/ /_| |_ __ __/ /| || |/' | -| _| | | | | |/ _ \ '_ \| __| | ___ \ | __| \ \ / / /_| || /| | -| | | | |_| | __/ | | | |_ | |_/ / | |_ \ V /\___ |\ |_/ / -\_| |_|\__,_|\___|_| |_|\__| \____/|_|\__| \_/ |_(_)___/ - - -[2025/07/03 16:15:34] [ info] [fluent bit] version=4.0.3, commit=3a91b155d6, pid=23196 -[2025/07/03 16:15:34] [ info] [storage] ver=1.5.3, type=memory, sync=normal, checksum=off, max_chunks_up=128 -[2025/07/03 16:15:34] [ info] [simd ] disabled -[2025/07/03 16:15:34] [ info] [cmetrics] version=1.0.3 -[2025/07/03 16:15:34] [ info] [ctraces ] version=0.6.6 -[2025/07/03 16:15:34] [ info] [input:dummy:dummy.0] initializing -[2025/07/03 16:15:34] [ info] [input:dummy:dummy.0] storage_strategy='memory' (memory only) -[2025/07/03 16:15:34] [ info] [output:stdout:stdout.0] worker #0 started -[2025/07/03 16:15:34] [ info] [sp] stream processor started +$ fluent-bit --config fluent-bit.conf + +... [0] dummy.data: [[1750325238.681398000, {}], {"INT"=>"100", "FLOAT"=>"0.5", "BOOL"=>"true", "STRING"=>"This is example", "key1"=>"value1", "key2"=>"value2"}] [0] dummy.data: [[1750325239.682090000, {}], {"INT"=>"100", "FLOAT"=>"0.5", "BOOL"=>"true", "STRING"=>"This is example", "key1"=>"value1", "key2"=>"value2"}] [0] dummy.data: [[1750325240.682903000, {}], {"INT"=>"100", "FLOAT"=>"0.5", "BOOL"=>"true", "STRING"=>"This is example", "key1"=>"value1", "key2"=>"value2"}] @@ -261,9 +219,9 @@ If you enable `Reserve_Data` and `Preserve_Key`, the original key field will als ```yaml parsers: - - name: dummy_test - format: regex - regex: '^(?[^ ]+) (?[^ ]+) (?[^ ]+) (?.+)$' + - name: dummy_test + format: regex + regex: '^(?[^ ]+) (?[^ ]+) (?[^ ]+) (?.+)$' ``` {% endtab %} @@ -271,9 +229,9 @@ parsers: ```text [PARSER] - Name dummy_test - Format regex - Regex ^(?[^ ]+) (?[^ ]+) (?[^ ]+) (?.+)$ + Name dummy_test + Format regex + Regex ^(?[^ ]+) (?[^ ]+) (?[^ ]+) (?.+)$ ``` {% endtab %} @@ -286,25 +244,25 @@ Now add `Reserve_Data` and `Preserve_Key`to the filter section of the correspond ```yaml service: - parsers_file: /path/to/parsers.yaml + parsers_file: /path/to/parsers.yaml pipeline: - inputs: - - name: dummy - tag: dummy.data - dummy: '{"data":"100 0.5 true This is example", "key1":"value1", "key2":"value2"}' - - filters: - - name: parser - match: 'dummy.*' - key_name: data - parser: dummy_test - reserve_data: on - preserve_key: on - - outputs: - - name: stdout - match: '*' + inputs: + - name: dummy + tag: dummy.data + dummy: '{"data":"100 0.5 true This is example", "key1":"value1", "key2":"value2"}' + + filters: + - name: parser + match: 'dummy.*' + key_name: data + parser: dummy_test + reserve_data: on + preserve_key: on + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -312,24 +270,24 @@ pipeline: ```text [SERVICE] - Parsers_File /path/to/parsers.conf + Parsers_File /path/to/parsers.conf [INPUT] - Name dummy - Tag dummy.data - Dummy {"data":"100 0.5 true This is example", "key1":"value1", "key2":"value2"} + Name dummy + Tag dummy.data + Dummy {"data":"100 0.5 true This is example", "key1":"value1", "key2":"value2"} [FILTER] - Name parser - Match dummy.* - Key_Name data - Parser dummy_test - Reserve_Data On - Preserve_Key On - - [OUTPUT] - Name stdout - Match * + Name parser + Match dummy.* + Key_Name data + Parser dummy_test + Reserve_Data On + Preserve_Key On + +[OUTPUT] + Name stdout + Match * ``` {% endtab %} @@ -339,33 +297,12 @@ The output when running the corresponding configuration is as follows: ```text # For YAML configuration. -$ ./fluent-bit --config fluent-bit.yaml +$ fluent-bit --config fluent-bit.yaml # For classic configuration. -$ ./fluent-bit --config fluent-bit.conf - -Fluent Bit v4.0.3 -* Copyright (C) 2015-2025 The Fluent Bit Authors -* Fluent Bit is a CNCF sub-project under the umbrella of Fluentd -* https://fluentbit.io - -______ _ _ ______ _ _ ___ _____ -| ___| | | | | ___ (_) | / || _ | -| |_ | |_ _ ___ _ __ | |_ | |_/ /_| |_ __ __/ /| || |/' | -| _| | | | | |/ _ \ '_ \| __| | ___ \ | __| \ \ / / /_| || /| | -| | | | |_| | __/ | | | |_ | |_/ / | |_ \ V /\___ |\ |_/ / -\_| |_|\__,_|\___|_| |_|\__| \____/|_|\__| \_/ |_(_)___/ - - -[2025/07/03 16:15:34] [ info] [fluent bit] version=4.0.3, commit=3a91b155d6, pid=23196 -[2025/07/03 16:15:34] [ info] [storage] ver=1.5.3, type=memory, sync=normal, checksum=off, max_chunks_up=128 -[2025/07/03 16:15:34] [ info] [simd ] disabled -[2025/07/03 16:15:34] [ info] [cmetrics] version=1.0.3 -[2025/07/03 16:15:34] [ info] [ctraces ] version=0.6.6 -[2025/07/03 16:15:34] [ info] [input:dummy:dummy.0] initializing -[2025/07/03 16:15:34] [ info] [input:dummy:dummy.0] storage_strategy='memory' (memory only) -[2025/07/03 16:15:34] [ info] [output:stdout:stdout.0] worker #0 started -[2025/07/03 16:15:34] [ info] [sp] stream processor started +$ fluent-bit --config fluent-bit.conf + +... [0] dummy.data: [[1750325678.572817000, {}], {"INT"=>"100", "FLOAT"=>"0.5", "BOOL"=>"true", "STRING"=>"This is example", "data"=>"100 0.5 true This is example", "key1"=>"value1", "key2"=>"value2"}] [0] dummy.data: [[1750325679.574538000, {}], {"INT"=>"100", "FLOAT"=>"0.5", "BOOL"=>"true", "STRING"=>"This is example", "data"=>"100 0.5 true This is example", "key1"=>"value1", "key2"=>"value2"}] [0] dummy.data: [[1750325680.569750000, {}], {"INT"=>"100", "FLOAT"=>"0.5", "BOOL"=>"true", "STRING"=>"This is example", "data"=>"100 0.5 true This is example", "key1"=>"value1", "key2"=>"value2"}] diff --git a/pipeline/filters/record-modifier.md b/pipeline/filters/record-modifier.md index ec5629db7..47a4d2d70 100644 --- a/pipeline/filters/record-modifier.md +++ b/pipeline/filters/record-modifier.md @@ -1,6 +1,6 @@ # Record modifier -The _Record modifier_ filter lets you append fields to a record, or exclude specific fields. +The _Record Modifier_ [filter](record-modifier.md) lets you append fields to a record, or exclude specific fields. ## Configuration parameters @@ -33,20 +33,20 @@ The following configuration file appends a product name and hostname to a record ```yaml pipeline: - inputs: - - name: mem - tag: mem.local - - filters: - - name: record_modifier - match: '*' - record: - - hostname ${HOSTNAME} - - product Awesome_Tool - - outputs: - - name: stdout - match: '*' + inputs: + - name: mem + tag: mem.local + + filters: + - name: record_modifier + match: '*' + record: + - hostname ${HOSTNAME} + - product Awesome_Tool + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -54,18 +54,18 @@ pipeline: ```text [INPUT] - Name mem - Tag mem.local + Name mem + Tag mem.local [OUTPUT] - Name stdout - Match * + Name stdout + Match * [FILTER] - Name record_modifier - Match * - Record hostname ${HOSTNAME} - Record product Awesome_Tool + Name record_modifier + Match * + Record hostname ${HOSTNAME} + Record product Awesome_Tool ``` {% endtab %} @@ -74,7 +74,7 @@ pipeline: You can run the filter from command line: ```shell -./fluent-bit -i mem -o stdout -F record_modifier -p 'Record=hostname ${HOSTNAME}' -p 'Record=product Awesome_Tool' -m '*' +fluent-bit -i mem -o stdout -F record_modifier -p 'Record=hostname ${HOSTNAME}' -p 'Record=product Awesome_Tool' -m '*' ``` The output looks something like: @@ -92,21 +92,21 @@ The following configuration file removes `Swap.*` fields: ```yaml pipeline: - inputs: - - name: mem - tag: mem.local - - filters: - - name: record_modifier - match: '*' - remove_key: - - Swap.total - - Swap.used - - Swap.free - - outputs: - - name: stdout - match: '*' + inputs: + - name: mem + tag: mem.local + + filters: + - name: record_modifier + match: '*' + remove_key: + - Swap.total + - Swap.used + - Swap.free + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -114,19 +114,19 @@ pipeline: ```text [INPUT] - Name mem - Tag mem.local + Name mem + Tag mem.local [OUTPUT] - Name stdout - Match * + Name stdout + Match * [FILTER] - Name record_modifier - Match * - Remove_key Swap.total - Remove_key Swap.used - Remove_key Swap.free + Name record_modifier + Match * + Remove_key Swap.total + Remove_key Swap.used + Remove_key Swap.free ``` {% endtab %} @@ -135,7 +135,7 @@ pipeline: You can also run the filter from command line. ```shell -./fluent-bit -i mem -o stdout -F record_modifier -p 'Remove_key=Swap.total' -p 'Remove_key=Swap.free' -p 'Remove_key=Swap.used' -m '*' +fluent-bit -i mem -o stdout -F record_modifier -p 'Remove_key=Swap.total' -p 'Remove_key=Swap.free' -p 'Remove_key=Swap.used' -m '*' ``` The output looks something like: @@ -153,21 +153,21 @@ The following configuration file retains `Mem.*` fields. ```yaml pipeline: - inputs: - - name: mem - tag: mem.local - - filters: - - name: record_modifier - match: '*' - Allowlist_key: - - Mem.total - - Mem.used - - Mem.free - - outputs: - - name: stdout - match: '*' + inputs: + - name: mem + tag: mem.local + + filters: + - name: record_modifier + match: '*' + Allowlist_key: + - Mem.total + - Mem.used + - Mem.free + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -175,19 +175,19 @@ pipeline: ```text [INPUT] - Name mem - Tag mem.local + Name mem + Tag mem.local [FILTER] - Name record_modifier - Match * - Allowlist_key Mem.total - Allowlist_key Mem.used - Allowlist_key Mem.free - - [OUTPUT] - Name stdout - Match * + Name record_modifier + Match * + Allowlist_key Mem.total + Allowlist_key Mem.used + Allowlist_key Mem.free + +[OUTPUT] + Name stdout + Match * ``` {% endtab %} @@ -196,7 +196,7 @@ pipeline: You can also run the filter from command line: ```shell -./fluent-bit -i mem -o stdout -F record_modifier -p 'Allowlist_key=Mem.total' -p 'Allowlist_key=Mem.free' -p 'Allowlist_key=Mem.used' -m '*' +fluent-bit -i mem -o stdout -F record_modifier -p 'Allowlist_key=Mem.total' -p 'Allowlist_key=Mem.free' -p 'Allowlist_key=Mem.used' -m '*' ``` The output looks something like: diff --git a/pipeline/filters/rewrite-tag.md b/pipeline/filters/rewrite-tag.md index 9e3944e4e..4d4aa2290 100644 --- a/pipeline/filters/rewrite-tag.md +++ b/pipeline/filters/rewrite-tag.md @@ -84,7 +84,7 @@ If the regular expression doesn't match an incoming record, the rule will be ski If a regular expression has matched the value of the defined key in the rule, you can compose a new tag for that specific record. The tag is a concatenated string that can contain any of the following characters: `a-z`,`A-Z`, `0-9` and `.-,`. -A tag can take any string value from the matching record, the original tag it self, environment variables, or general placeholders. +A tag can take any string value from the matching record, the original tag itself, environment variables, or general placeholders. Consider the following incoming data on the rule: @@ -121,24 +121,24 @@ The following configuration example will emit a dummy record. The filter will re ```yaml service: - flush: 1 - log_level: info + flush: 1 + log_level: info pipeline: - inputs: - - name: dummy - tag: test_tag - dummy: '{"tool": "fluent", "sub": {"s1": {"s2": "bit"}}}' - - filters: - - name: rewrite_tag - match: test_tag - rule: $tool ^(fluent)$ from.$TAG.new.$tool.$sub['s1']['s2'].out false - emitter_name: re_emitted - - outputs: - - name: stdout - match: from.* + inputs: + - name: dummy + tag: test_tag + dummy: '{"tool": "fluent", "sub": {"s1": {"s2": "bit"}}}' + + filters: + - name: rewrite_tag + match: test_tag + rule: $tool ^(fluent)$ from.$TAG.new.$tool.$sub['s1']['s2'].out false + emitter_name: re_emitted + + outputs: + - name: stdout + match: from.* ``` {% endtab %} @@ -146,23 +146,23 @@ pipeline: ```text [SERVICE] - Flush 1 - Log_Level info + Flush 1 + Log_Level info [INPUT] - NAME dummy - Dummy {"tool": "fluent", "sub": {"s1": {"s2": "bit"}}} - Tag test_tag + NAME dummy + Dummy {"tool": "fluent", "sub": {"s1": {"s2": "bit"}}} + Tag test_tag [FILTER] - Name rewrite_tag - Match test_tag - Rule $tool ^(fluent)$ from.$TAG.new.$tool.$sub['s1']['s2'].out false - Emitter_Name re_emitted + Name rewrite_tag + Match test_tag + Rule $tool ^(fluent)$ from.$TAG.new.$tool.$sub['s1']['s2'].out false + Emitter_Name re_emitted [OUTPUT] - Name stdout - Match from.* + Name stdout + Match from.* ``` {% endtab %} @@ -171,30 +171,9 @@ pipeline: The original tag `test_tag` will be rewritten as `from.test_tag.new.fluent.bit.out`: ```shell -$ ./fluent-bit -c example.conf - -Fluent Bit v4.0.3 -* Copyright (C) 2015-2025 The Fluent Bit Authors -* Fluent Bit is a CNCF sub-project under the umbrella of Fluentd -* https://fluentbit.io - -______ _ _ ______ _ _ ___ _____ -| ___| | | | | ___ (_) | / || _ | -| |_ | |_ _ ___ _ __ | |_ | |_/ /_| |_ __ __/ /| || |/' | -| _| | | | | |/ _ \ '_ \| __| | ___ \ | __| \ \ / / /_| || /| | -| | | | |_| | __/ | | | |_ | |_/ / | |_ \ V /\___ |\ |_/ / -\_| |_|\__,_|\___|_| |_|\__| \____/|_|\__| \_/ |_(_)___/ - - -[2025/07/03 16:15:34] [ info] [fluent bit] version=4.0.3, commit=3a91b155d6, pid=23196 -[2025/07/03 16:15:34] [ info] [storage] ver=1.5.3, type=memory, sync=normal, checksum=off, max_chunks_up=128 -[2025/07/03 16:15:34] [ info] [simd ] disabled -[2025/07/03 16:15:34] [ info] [cmetrics] version=1.0.3 -[2025/07/03 16:15:34] [ info] [ctraces ] version=0.6.6 -[2025/07/03 16:15:34] [ info] [input:dummy:dummy.0] initializing -[2025/07/03 16:15:34] [ info] [input:dummy:dummy.0] storage_strategy='memory' (memory only) -[2025/07/03 16:15:34] [ info] [output:stdout:stdout.0] worker #0 started -[2025/07/03 16:15:34] [ info] [sp] stream processor started +$ fluent-bit -c example.conf + +... [0] from.test_tag.new.fluent.bit.out: [1580436933.000050569, {"tool"=>"fluent", "sub"=>{"s1"=>{"s2"=>"bit"}}}] ``` @@ -209,7 +188,7 @@ The `rewrite_tag` filter emits new records that go through the beginning of the Using the previously provided configuration, when you query the metrics exposed in the HTTP interface: ```shell -./curl http://127.0.0.1:2020/api/v1/metrics/ | jq +curl http://127.0.0.1:2020/api/v1/metrics/ | jq ``` You will see metrics output similar to the following: diff --git a/pipeline/filters/standard-output.md b/pipeline/filters/standard-output.md index a7df5cc9f..1b7471373 100644 --- a/pipeline/filters/standard-output.md +++ b/pipeline/filters/standard-output.md @@ -9,40 +9,15 @@ The plugin has no configuration parameters. Use the following command from the command line: ```shell -$ ./fluent-bit -i cpu -F stdout -m '*' -o null +fluent-bit -i cpu -F stdout -m '*' -o null ``` Fluent Bit specifies gathering [CPU](../inputs/cpu-metrics.md) usage metrics and prints them out in a human-readable way when they flow through the stdout plugin. ```text -Fluent Bit v4.0.3 -* Copyright (C) 2015-2025 The Fluent Bit Authors -* Fluent Bit is a CNCF sub-project under the umbrella of Fluentd -* https://fluentbit.io - -______ _ _ ______ _ _ ___ _____ -| ___| | | | | ___ (_) | / || _ | -| |_ | |_ _ ___ _ __ | |_ | |_/ /_| |_ __ __/ /| || |/' | -| _| | | | | |/ _ \ '_ \| __| | ___ \ | __| \ \ / / /_| || /| | -| | | | |_| | __/ | | | |_ | |_/ / | |_ \ V /\___ |\ |_/ / -\_| |_|\__,_|\___|_| |_|\__| \____/|_|\__| \_/ |_(_)___/ - - -[2025/07/03 16:15:34] [ info] [fluent bit] version=4.0.3, commit=3a91b155d6, pid=23196 -[2025/07/03 16:15:34] [ info] [storage] ver=1.5.3, type=memory, sync=normal, checksum=off, max_chunks_up=128 -[2025/07/03 16:15:34] [ info] [simd ] disabled -[2025/07/03 16:15:34] [ info] [cmetrics] version=1.0.3 -[2025/07/03 16:15:34] [ info] [ctraces ] version=0.6.6 -[2025/07/03 16:15:34] [ info] [input:dummy:dummy.0] initializing -[2025/07/03 16:15:34] [ info] [input:dummy:dummy.0] storage_strategy='memory' (memory only) -[2025/07/03 16:15:34] [ info] [output:stdout:stdout.0] worker #0 started -[2025/07/03 16:15:34] [ info] [sp] stream processor started +... [0] cpu.0: [1622789640.379532062, {"cpu_p"=>9.000000, "user_p"=>6.500000, "system_p"=>2.500000, "cpu0.p_cpu"=>8.000000, "cpu0.p_user"=>6.000000, "cpu0.p_system"=>2.000000, "cpu1.p_cpu"=>9.000000, "cpu1.p_user"=>6.000000, "cpu1.p_system"=>3.000000}] [0] cpu.0: [1622789641.379529426, {"cpu_p"=>22.500000, "user_p"=>18.000000, "system_p"=>4.500000, "cpu0.p_cpu"=>34.000000, "cpu0.p_user"=>30.000000, "cpu0.p_system"=>4.000000, "cpu1.p_cpu"=>11.000000, "cpu1.p_user"=>6.000000, "cpu1.p_system"=>5.000000}] [0] cpu.0: [1622789642.379544020, {"cpu_p"=>26.500000, "user_p"=>16.000000, "system_p"=>10.500000, "cpu0.p_cpu"=>30.000000, "cpu0.p_user"=>24.000000, "cpu0.p_system"=>6.000000, "cpu1.p_cpu"=>22.000000, "cpu1.p_user"=>8.000000, "cpu1.p_system"=>14.000000}] [0] cpu.0: [1622789643.379507371, {"cpu_p"=>39.500000, "user_p"=>34.500000, "system_p"=>5.000000, "cpu0.p_cpu"=>52.000000, "cpu0.p_user"=>48.000000, "cpu0.p_system"=>4.000000, "cpu1.p_cpu"=>28.000000, "cpu1.p_user"=>21.000000, "cpu1.p_system"=>7.000000}] -^C[2021/06/04 14:54:04] [engine] caught signal (SIGINT) -[2021/06/04 14:54:04] [ info] [input] pausing cpu.0 -[2021/06/04 14:54:04] [ warn] [engine] service will stop in 5 seconds -[2021/06/04 14:54:08] [ info] [engine] service stopped ``` \ No newline at end of file diff --git a/pipeline/filters/sysinfo.md b/pipeline/filters/sysinfo.md index b5aab964f..439a65a83 100644 --- a/pipeline/filters/sysinfo.md +++ b/pipeline/filters/sysinfo.md @@ -27,19 +27,19 @@ The following configuration file is to append the Fluent Bit version and operati ```yaml pipeline: - inputs: - - name: dummy - tag: test - - filters: - - name: sysinfo - match: '*' - Fluentbit_version_key: flb_ver - Os_name_key: os_name - - outputs: - - name: stdout - match: '*' + inputs: + - name: dummy + tag: test + + filters: + - name: sysinfo + match: '*' + Fluentbit_version_key: flb_ver + Os_name_key: os_name + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -47,18 +47,18 @@ pipeline: ```text [INPUT] - Name dummy - Tag test + Name dummy + Tag test [FILTER] - Name sysinfo - Match * - Fluentbit_version_key flb_ver - Os_name_key os_name + Name sysinfo + Match * + Fluentbit_version_key flb_ver + Os_name_key os_name [OUTPUT] - name stdout - match * + name stdout + match * ``` {% endtab %} @@ -67,7 +67,7 @@ pipeline: You can also run the filter from command line. ```shell -$ ./fluent-bit -i dummy -o stdout -F sysinfo -m '*' -p fluentbit_version_key=flb_ver -p os_name_key=os_name +fluent-bit -i dummy -o stdout -F sysinfo -m '*' -p fluentbit_version_key=flb_ver -p os_name_key=os_name ``` The output will be something like the following: diff --git a/pipeline/filters/tensorflow.md b/pipeline/filters/tensorflow.md index 40ba83a1f..b59f16995 100644 --- a/pipeline/filters/tensorflow.md +++ b/pipeline/filters/tensorflow.md @@ -6,7 +6,7 @@ Tensorflow Lite is a lightweight open source deep learning framework used for mo The Tensorflow plugin for Fluent Bit has the following limitations: -- Currently supports single-input models +- Currently, supports single-input models - Uses Tensorflow 2.3 header files ## Configuration parameters @@ -29,7 +29,7 @@ To create a Tensorflow Lite shared library: 1. Run the following command to create the shared library: ```shell - ./bazel build -c opt //tensorflow/lite/c:tensorflowlite_c # see https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/c + bazel build -c opt //tensorflow/lite/c:tensorflowlite_c # see https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/c ``` The script creates the shared library @@ -41,7 +41,7 @@ To create a Tensorflow Lite shared library: The Tensorflow filter plugin is disabled by default. You must build Fluent Bit with the Tensorflow plugin enabled. In addition, it requires access to Tensorflow Lite header files to compile. Therefore, you must pass the address of the Tensorflow source code on your machine to the [build script](https://github.com/fluent/fluent-bit#build-from-scratch): ```shell -./cmake -DFLB_FILTER_TENSORFLOW=On -DTensorflow_DIR= ... +cmake -DFLB_FILTER_TENSORFLOW=On -DTensorflow_DIR= ... ``` ### Command line @@ -51,7 +51,7 @@ If the Tensorflow plugin initializes correctly, it reports successful creation o The command: ```shell -./fluent-bit -i mqtt -p 'tag=mqtt.data' -F tensorflow -m '*' -p 'input_field=image' -p 'model_file=/home/user/model.tflite' -p +fluent-bit -i mqtt -p 'tag=mqtt.data' -F tensorflow -m '*' -p 'input_field=image' -p 'model_file=/home/user/model.tflite' -p ``` produces an output like: @@ -72,26 +72,26 @@ produces an output like: ```yaml service: - flush: 1 - daemon: off - log_level: info + flush: 1 + daemon: off + log_level: info pipeline: - inputs: - - name: mqtt - tag: mqtt.data - - filters: - - name: tensorflow - match: mqtt.data - input_field: image - model_file: /home/m/model.tflite - include_input_fields: false - normalization_value: 255 - - outputs: - - name: stdout - match: '*' + inputs: + - name: mqtt + tag: mqtt.data + + filters: + - name: tensorflow + match: mqtt.data + input_field: image + model_file: /home/m/model.tflite + include_input_fields: false + normalization_value: 255 + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -99,25 +99,25 @@ pipeline: ```text [SERVICE] - Flush 1 - Daemon Off - Log_Level info + Flush 1 + Daemon Off + Log_Level info [INPUT] - Name mqtt - Tag mqtt.data + Name mqtt + Tag mqtt.data [FILTER] - Name tensorflow - Match mqtt.data - input_field image - model_file /home/m/model.tflite - include_input_fields false - normalization_value 255 + Name tensorflow + Match mqtt.data + input_field image + model_file /home/m/model.tflite + include_input_fields false + normalization_value 255 [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} diff --git a/pipeline/filters/throttle.md b/pipeline/filters/throttle.md index 259bb89cd..2af148301 100644 --- a/pipeline/filters/throttle.md +++ b/pipeline/filters/throttle.md @@ -112,7 +112,7 @@ It's suggested to use a configuration file. The following command will load the Tail plugin and read the content of the `lines.txt` file. Then, the Throttle filter will apply a rate limit and only pass the records which are read below the `rate`: ```shell -./fluent-bit -i tail -p 'path=lines.txt' -F throttle -p 'rate=1' -m '*' -o stdout +fluent-bit -i tail -p 'path=lines.txt' -F throttle -p 'rate=1' -m '*' -o stdout ``` ### Configuration File @@ -122,20 +122,20 @@ The following command will load the Tail plugin and read the content of the `lin ```yaml pipeline: - inputs: - - name: tail - path: lines.txt - - filters: - - name: throttle - match: '*' - rate: 1000 - window: 300 - interval: 1s - - outputs: - - name: stdout - match: '*' + inputs: + - name: tail + path: lines.txt + + filters: + - name: throttle + match: '*' + rate: 1000 + window: 300 + interval: 1s + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -143,19 +143,19 @@ pipeline: ```text [INPUT] - Name tail - Path lines.txt + Name tail + Path lines.txt [FILTER] - Name throttle - Match * - Rate 1000 - Window 300 - Interval 1s + Name throttle + Match * + Rate 1000 + Window 300 + Interval 1s [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} diff --git a/pipeline/filters/type-converter.md b/pipeline/filters/type-converter.md index 09e206b83..93c2e7c65 100644 --- a/pipeline/filters/type-converter.md +++ b/pipeline/filters/type-converter.md @@ -2,8 +2,6 @@ The _Type Converter_ filter plugin converts data types and appends new key-value pairs. - - You can use this filter in combination with plugins which expect incoming string value. For example, [Grep](grep.md) and [Modify](modify.md). ## Configuration parameters @@ -42,20 +40,20 @@ The plugin outputs `uint` values and `filter_type_converter` converts them into ```yaml pipeline: - inputs: - - name: mem - - filters: - - name: type_converter - match: '*' - uint_key: - - Mem.total Mem.total_str string - - Mem.used Mem.used_str string - - Mem.free Mem.free_str string - - outputs: - - name: stdout - match: '*' + inputs: + - name: mem + + filters: + - name: type_converter + match: '*' + uint_key: + - Mem.total Mem.total_str string + - Mem.used Mem.used_str string + - Mem.free Mem.free_str string + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -63,18 +61,18 @@ pipeline: ```text [INPUT] - Name mem + Name mem [FILTER] - Name type_converter - Match * - uint_key Mem.total Mem.total_str string - uint_key Mem.used Mem.used_str string - uint_key Mem.free Mem.free_str string + Name type_converter + Match * + uint_key Mem.total Mem.total_str string + uint_key Mem.used Mem.used_str string + uint_key Mem.free Mem.free_str string [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %} @@ -83,7 +81,7 @@ pipeline: You can also run the filter from command line. ```shell -./fluent-bit -i mem -o stdout -F type_converter -p 'uint_key=Mem.total Mem.total_str string' -p 'uint_key=Mem.used Mem.used_str string' -p 'uint_key=Mem.free Mem.free_str string' -m '*' +fluent-bit -i mem -o stdout -F type_converter -p 'uint_key=Mem.total Mem.total_str string' -p 'uint_key=Mem.used Mem.used_str string' -p 'uint_key=Mem.free Mem.free_str string' -m '*' ``` The output will be diff --git a/pipeline/filters/wasm.md b/pipeline/filters/wasm.md index 89d2367b0..12740ef0b 100644 --- a/pipeline/filters/wasm.md +++ b/pipeline/filters/wasm.md @@ -34,22 +34,22 @@ Here is a configuration example. ```yaml pipeline: - inputs: - - name: dummy - tag: dummy.local - - filters: - - name: wasm - match: 'dummy.*' - event_format: json # or msgpack - wasm_path: /path/to/wasm_program.wasm - function_name: filter_function_name - # Note: run Fluent Bit from the 'wasm_path' location. - accessible_paths: /path/to/accessible - - outputs: - - name: stdout - match: '*' + inputs: + - name: dummy + tag: dummy.local + + filters: + - name: wasm + match: 'dummy.*' + event_format: json # or msgpack + wasm_path: /path/to/wasm_program.wasm + function_name: filter_function_name + # Note: run Fluent Bit from the 'wasm_path' location. + accessible_paths: /path/to/accessible + + outputs: + - name: stdout + match: '*' ``` {% endtab %} @@ -57,20 +57,20 @@ pipeline: ```text [INPUT] - Name dummy - Tag dummy.local + Name dummy + Tag dummy.local [FILTER] - Name wasm - Match dummy.* - Event_Format json # or msgpack - WASM_Path /path/to/wasm_program.wasm - Function_Name filter_function_name - Accessible_Paths .,/path/to/accessible + Name wasm + Match dummy.* + Event_Format json # or msgpack + WASM_Path /path/to/wasm_program.wasm + Function_Name filter_function_name + Accessible_Paths .,/path/to/accessible [OUTPUT] - Name stdout - Match * + Name stdout + Match * ``` {% endtab %}