diff --git a/apps/couchbase.go b/apps/couchbase.go index 95f9721198..7e720fcba0 100644 --- a/apps/couchbase.go +++ b/apps/couchbase.go @@ -20,7 +20,6 @@ import ( "sort" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" - "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" "github.com/GoogleCloudPlatform/ops-agent/internal/secret" ) @@ -241,56 +240,40 @@ func init() { confgenerator.MetricsReceiverTypes.RegisterType(func() confgenerator.MetricsReceiver { return &MetricsReceiverCouchbase{} }) } -// LoggingReceiverCouchbase is a struct used for generating the fluentbit component for couchbase logs -type LoggingReceiverCouchbase struct { - confgenerator.ConfigComponent `yaml:",inline"` - ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"` -} +type LoggingProcessorMacroCouchbase struct{} // Type returns the string identifier for the general couchbase logs -func (lr LoggingReceiverCouchbase) Type() string { +func (p LoggingProcessorMacroCouchbase) Type() string { return "couchbase_general" } -// Components returns the logging components of the couchbase access logs -func (lr LoggingReceiverCouchbase) Components(ctx context.Context, tag string) []fluentbit.Component { - if len(lr.ReceiverMixin.IncludePaths) == 0 { - lr.ReceiverMixin.IncludePaths = []string{ - "/opt/couchbase/var/lib/couchbase/logs/couchdb.log", - "/opt/couchbase/var/lib/couchbase/logs/info.log", - "/opt/couchbase/var/lib/couchbase/logs/debug.log", - "/opt/couchbase/var/lib/couchbase/logs/error.log", - "/opt/couchbase/var/lib/couchbase/logs/babysitter.log", - } - } - components := lr.ReceiverMixin.Components(ctx, tag) - components = append(components, confgenerator.LoggingProcessorParseMultilineRegex{ - LoggingProcessorParseRegexComplex: confgenerator.LoggingProcessorParseRegexComplex{ - Parsers: []confgenerator.RegexParser{ - { - Regex: `^\[(?[^:]*):(?[^,]*),(?\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),(?[^:]*):([^:]+):(?[^\]]+)\](?.*)$`, - Parser: confgenerator.ParserShared{ - TimeKey: "timestamp", - TimeFormat: "%Y-%m-%dT%H:%M:%S.%L", +func (p LoggingProcessorMacroCouchbase) Expand(ctx context.Context) []confgenerator.InternalLoggingProcessor { + return []confgenerator.InternalLoggingProcessor{ + confgenerator.LoggingProcessorParseMultilineRegex{ + LoggingProcessorParseRegexComplex: confgenerator.LoggingProcessorParseRegexComplex{ + Parsers: []confgenerator.RegexParser{ + { + Regex: `^\[(?[^:]*):(?[^,]*),(?\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),(?[^:]*):([^:]+):(?[^\]]+)\](?[\s|\S]*)?`, + Parser: confgenerator.ParserShared{ + TimeKey: "timestamp", + TimeFormat: "%Y-%m-%dT%H:%M:%S.%L", + }, }, }, }, - }, - Rules: []confgenerator.MultilineRule{ - { - StateName: "start_state", - NextState: "cont", - Regex: `^\[([^\s+:]*):`, - }, - { - StateName: "cont", - NextState: "cont", - Regex: `^(?!\[([^\s+:]*):).*$`, + Rules: []confgenerator.MultilineRule{ + { + StateName: "start_state", + NextState: "cont", + Regex: `^\[([^\s+:]*):`, + }, + { + StateName: "cont", + NextState: "cont", + Regex: `^(?!\[([^\s+:]*):).*$`, + }, }, }, - }.Components(ctx, tag, lr.Type())...) - - components = append(components, confgenerator.LoggingProcessorModifyFields{ Fields: map[string]*confgenerator.ModifyField{ "severity": { @@ -303,53 +286,40 @@ func (lr LoggingReceiverCouchbase) Components(ctx context.Context, tag string) [ }, MapValuesExclusive: true, }, - InstrumentationSourceLabel: instrumentationSourceValue(lr.Type()), + InstrumentationSourceLabel: instrumentationSourceValue(p.Type()), }, - }.Components(ctx, tag, lr.Type())...) - return components + }, + } } -// LoggingProcessorCouchbaseHTTPAccess is a struct that will generate the fluentbit components for the http access logs -type LoggingProcessorCouchbaseHTTPAccess struct { - confgenerator.ConfigComponent `yaml:",inline"` - ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"` +// Components returns the logging components of the couchbase access logs +func loggingReceiverFilesMixinCouchbase() confgenerator.LoggingReceiverFilesMixin { + return confgenerator.LoggingReceiverFilesMixin{ + IncludePaths: []string{ + "/opt/couchbase/var/lib/couchbase/logs/couchdb.log", + "/opt/couchbase/var/lib/couchbase/logs/info.log", + "/opt/couchbase/var/lib/couchbase/logs/debug.log", + "/opt/couchbase/var/lib/couchbase/logs/error.log", + "/opt/couchbase/var/lib/couchbase/logs/babysitter.log", + }, + } } +// LoggingProcessorCouchbaseHTTPAccess is a struct that will generate the fluentbit components for the http access logs +type LoggingProcessorMacroCouchbaseHTTPAccess struct{} + // Type returns the string for the couchbase http access logs -func (lp LoggingProcessorCouchbaseHTTPAccess) Type() string { +func (p LoggingProcessorMacroCouchbaseHTTPAccess) Type() string { return "couchbase_http_access" } // Components returns the fluentbit components for the http access logs of couchbase -func (lp LoggingProcessorCouchbaseHTTPAccess) Components(ctx context.Context, tag string) []fluentbit.Component { - if len(lp.ReceiverMixin.IncludePaths) == 0 { - lp.ReceiverMixin.IncludePaths = []string{ - "/opt/couchbase/var/lib/couchbase/logs/http_access.log", - "/opt/couchbase/var/lib/couchbase/logs/http_access_internal.log", - } - } - c := lp.ReceiverMixin.Components(ctx, tag) - // TODO: Harden the genericAccessLogParser so it can be used. It didn't work here since there are some minor differences with the - // referer fields and there are additional fields after the user agent here but not in the other apps. - c = append(c, - confgenerator.LoggingProcessorParseRegex{ - Regex: `^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(?[^\]]*)\] "(?\S+) (?\S+) (?\S+)" (?[^ ]*) (?[^ ]*\S+) (?[^ ]*) "(?[^\"]*)" (?.*)$`, - ParserShared: confgenerator.ParserShared{ - TimeKey: "timestamp", - TimeFormat: `%d/%b/%Y:%H:%M:%S %z`, - Types: map[string]string{ - "size": "integer", - "code": "integer", - }, - }, - }.Components(ctx, tag, lp.Type())..., - ) +func (p LoggingProcessorMacroCouchbaseHTTPAccess) Expand(ctx context.Context) []confgenerator.InternalLoggingProcessor { mf := confgenerator.LoggingProcessorModifyFields{ Fields: map[string]*confgenerator.ModifyField{ - InstrumentationSourceLabel: instrumentationSourceValue(lp.Type()), + InstrumentationSourceLabel: instrumentationSourceValue(p.Type()), }, } - // Generate the httpRequest structure. for _, field := range []string{ "remoteIp", "requestMethod", @@ -369,56 +339,75 @@ func (lp LoggingProcessorCouchbaseHTTPAccess) Components(ctx context.Context, ta mf.Fields[dest].OmitIf = fmt.Sprintf(`%s = "-"`, src) } } - c = append(c, mf.Components(ctx, tag, lp.Type())...) - return c + + return []confgenerator.InternalLoggingProcessor{ + // TODO: Harden the genericAccessLogParser so it can be used. It didn't work here since there are some minor differences with the + // referer fields and there are additional fields after the user agent here but not in the other apps. + confgenerator.LoggingProcessorParseRegex{ + Regex: `^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(?[^\]]*)\] "(?\S+) (?\S+) (?\S+)" (?[^ ]*) (?[^ ]*\S+) (?[^ ]*) "(?[^\"]*)" (?.*)$`, + ParserShared: confgenerator.ParserShared{ + TimeKey: "timestamp", + TimeFormat: `%d/%b/%Y:%H:%M:%S %z`, + Types: map[string]string{ + "size": "integer", + "code": "integer", + }, + }, + }, + confgenerator.LoggingProcessorModifyFields{ + Fields: map[string]*confgenerator.ModifyField{ + InstrumentationSourceLabel: instrumentationSourceValue(p.Type()), + }, + }, + mf, + } } -// LoggingProcessorCouchbaseGOXDCR is a struct that iwll generate the fluentbit components for the goxdcr logs -type LoggingProcessorCouchbaseGOXDCR struct { - confgenerator.ConfigComponent `yaml:",inline"` - ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"` +func loggingReceiverFilesMixinCouchbaseHTTPAccess() confgenerator.LoggingReceiverFilesMixin { + return confgenerator.LoggingReceiverFilesMixin{ + IncludePaths: []string{ + "/opt/couchbase/var/lib/couchbase/logs/http_access.log", + "/opt/couchbase/var/lib/couchbase/logs/http_access_internal.log", + }, + } } +// LoggingProcessorCouchbaseGOXDCR is a struct that iwll generate the fluentbit components for the goxdcr logs +type LoggingProcessorMacroCouchbaseGOXDCR struct{} + // Type returns the type string for the cross datacenter logs of couchbase -func (lg LoggingProcessorCouchbaseGOXDCR) Type() string { +func (p LoggingProcessorMacroCouchbaseGOXDCR) Type() string { return "couchbase_goxdcr" } // Components returns the fluentbit components for the couchbase goxdcr logs -func (lg LoggingProcessorCouchbaseGOXDCR) Components(ctx context.Context, tag string) []fluentbit.Component { - if len(lg.ReceiverMixin.IncludePaths) == 0 { - lg.ReceiverMixin.IncludePaths = []string{ - "/opt/couchbase/var/lib/couchbase/logs/goxdcr.log", - } - } - - c := lg.ReceiverMixin.Components(ctx, tag) - c = append(c, confgenerator.LoggingProcessorParseMultilineRegex{ - LoggingProcessorParseRegexComplex: confgenerator.LoggingProcessorParseRegexComplex{ - Parsers: []confgenerator.RegexParser{ - { - Regex: `^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d*Z) (?\w+) (?\w+.\w+): (?.*)$`, - Parser: confgenerator.ParserShared{ - TimeKey: "timestamp", - TimeFormat: "%Y-%m-%dT%H:%M:%S.%L", +func (p LoggingProcessorMacroCouchbaseGOXDCR) Expand(ctx context.Context) []confgenerator.InternalLoggingProcessor { + return []confgenerator.InternalLoggingProcessor{ + confgenerator.LoggingProcessorParseMultilineRegex{ + LoggingProcessorParseRegexComplex: confgenerator.LoggingProcessorParseRegexComplex{ + Parsers: []confgenerator.RegexParser{ + { + Regex: `^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d*Z) (?\w+) (?\w+.\w+): (?[\s|\S]*)?`, + Parser: confgenerator.ParserShared{ + TimeKey: "timestamp", + TimeFormat: "%Y-%m-%dT%H:%M:%S.%L", + }, }, }, }, - }, - Rules: []confgenerator.MultilineRule{ - { - StateName: "start_state", - NextState: "cont", - Regex: `^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}`, - }, - { - StateName: "cont", - NextState: "cont", - Regex: `^(?!\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})`, + Rules: []confgenerator.MultilineRule{ + { + StateName: "start_state", + NextState: "cont", + Regex: `^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}`, + }, + { + StateName: "cont", + NextState: "cont", + Regex: `^(?!\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})`, + }, }, }, - }.Components(ctx, tag, lg.Type())...) - c = append(c, confgenerator.LoggingProcessorModifyFields{ Fields: map[string]*confgenerator.ModifyField{ "severity": { @@ -431,15 +420,22 @@ func (lg LoggingProcessorCouchbaseGOXDCR) Components(ctx context.Context, tag st }, MapValuesExclusive: true, }, - InstrumentationSourceLabel: instrumentationSourceValue(lg.Type()), + InstrumentationSourceLabel: instrumentationSourceValue(p.Type()), }, - }.Components(ctx, tag, lg.Type())..., - ) - return c + }, + } +} + +func loggingReceiverFilesMixinCouchbaseGOXDCR() confgenerator.LoggingReceiverFilesMixin { + return confgenerator.LoggingReceiverFilesMixin{ + IncludePaths: []string{ + "/opt/couchbase/var/lib/couchbase/logs/goxdcr.log", + }, + } } func init() { - confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverCouchbase{} }) - confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingProcessorCouchbaseHTTPAccess{} }) - confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingProcessorCouchbaseGOXDCR{} }) + confgenerator.RegisterLoggingFilesProcessorMacro[LoggingProcessorMacroCouchbase](loggingReceiverFilesMixinCouchbase) + confgenerator.RegisterLoggingFilesProcessorMacro[LoggingProcessorMacroCouchbaseHTTPAccess](loggingReceiverFilesMixinCouchbaseHTTPAccess) + confgenerator.RegisterLoggingFilesProcessorMacro[LoggingProcessorMacroCouchbaseGOXDCR](loggingReceiverFilesMixinCouchbaseGOXDCR) } diff --git a/confgenerator/testdata/feature/golden.csv b/confgenerator/testdata/feature/golden.csv index b0b8416221..23d5b640d1 100644 --- a/confgenerator/testdata/feature/golden.csv +++ b/confgenerator/testdata/feature/golden.csv @@ -1,20 +1,8 @@ App,Field,Override, -*apps.LoggingProcessorCouchbaseGOXDCR,confgenerator.ConfigComponent.Type, -*apps.LoggingProcessorCouchbaseGOXDCR,confgenerator.LoggingReceiverFilesMixin.BufferInMemory, -*apps.LoggingProcessorCouchbaseGOXDCR,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, -*apps.LoggingProcessorCouchbaseGOXDCR,confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, -*apps.LoggingProcessorCouchbaseHTTPAccess,confgenerator.ConfigComponent.Type, -*apps.LoggingProcessorCouchbaseHTTPAccess,confgenerator.LoggingReceiverFilesMixin.BufferInMemory, -*apps.LoggingProcessorCouchbaseHTTPAccess,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, -*apps.LoggingProcessorCouchbaseHTTPAccess,confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, *apps.LoggingProcessorIisAccess,confgenerator.ConfigComponent.Type, *apps.LoggingProcessorOracleDBAlert,confgenerator.ConfigComponent.Type, *apps.LoggingProcessorOracleDBAudit,confgenerator.ConfigComponent.Type, *apps.LoggingReceiverActiveDirectoryDS,confgenerator.ConfigComponent.Type, -*apps.LoggingReceiverCouchbase,confgenerator.ConfigComponent.Type, -*apps.LoggingReceiverCouchbase,confgenerator.LoggingReceiverFilesMixin.BufferInMemory, -*apps.LoggingReceiverCouchbase,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, -*apps.LoggingReceiverCouchbase,confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, *apps.LoggingReceiverIisAccess,apps.LoggingProcessorIisAccess.confgenerator.ConfigComponent.Type, *apps.LoggingReceiverIisAccess,confgenerator.LoggingReceiverFilesMixin.BufferInMemory, *apps.LoggingReceiverIisAccess,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, @@ -139,6 +127,9 @@ App,Field,Override, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCassandraDebug],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCassandraGC],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCassandraSystem],confgenerator.ConfigComponent.Type, +*confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCouchbaseGOXDCR],confgenerator.ConfigComponent.Type, +*confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCouchbaseHTTPAccess],confgenerator.ConfigComponent.Type, +*confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCouchbase],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCouchdb],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroElasticsearchGC],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroElasticsearchJson],confgenerator.ConfigComponent.Type, @@ -173,6 +164,12 @@ App,Field,Override, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCassandraGC]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCassandraSystem]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCassandraSystem]],confgenerator.ConfigComponent.Type, +*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCouchbaseGOXDCR]],ReceiverMacro, +*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCouchbaseGOXDCR]],confgenerator.ConfigComponent.Type, +*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCouchbaseHTTPAccess]],ReceiverMacro, +*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCouchbaseHTTPAccess]],confgenerator.ConfigComponent.Type, +*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCouchbase]],ReceiverMacro, +*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCouchbase]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCouchdb]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroCouchdb]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroElasticsearchGC]],ReceiverMacro, diff --git a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux-gpu/error b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux-gpu/error index 4234c581d9..be5b342e8a 100644 --- a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux-gpu/error +++ b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux-gpu/error @@ -1 +1 @@ -logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchbase_general, couchbase_goxdcr, couchbase_http_access, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux/error b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux/error index 4234c581d9..be5b342e8a 100644 --- a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux/error +++ b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux/error @@ -1 +1 @@ -logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchbase_general, couchbase_goxdcr, couchbase_http_access, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows-2012/error b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows-2012/error index 2b0b761c84..71dc213de7 100644 --- a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows-2012/error +++ b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows-2012/error @@ -1 +1 @@ -logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchbase_general, couchbase_goxdcr, couchbase_http_access, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows/error b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows/error index 2b0b761c84..71dc213de7 100644 --- a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows/error +++ b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows/error @@ -1 +1 @@ -logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchbase_general, couchbase_goxdcr, couchbase_http_access, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error index 41783f233b..c06e595037 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchbase_general, couchbase_goxdcr, couchbase_http_access, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error index 41783f233b..c06e595037 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchbase_general, couchbase_goxdcr, couchbase_http_access, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error index 7c76f2b1ee..a4cc570ab9 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchbase_general, couchbase_goxdcr, couchbase_http_access, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error index 7c76f2b1ee..a4cc570ab9 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchbase_general, couchbase_goxdcr, couchbase_http_access, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hadoop, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system, zookeeper_general]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/3bca332f661c48e682cdb7a2bdf766e8.lua b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/3bca332f661c48e682cdb7a2bdf766e8.lua new file mode 100644 index 0000000000..fb1b4cee6f --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/3bca332f661c48e682cdb7a2bdf766e8.lua @@ -0,0 +1,16 @@ + +local function trim_newline(s) + -- Check for a Windows-style carriage return and newline (\r\n) + if string.sub(s, -2) == "\r\n" then + return string.sub(s, 1, -3) + -- Check for a Unix/Linux-style newline (\n) + elseif string.sub(s, -1) == "\n" then + return string.sub(s, 1, -2) + end + -- If no trailing newline is found, return the original string + return s +end +function strip_newline(tag, timestamp, record) + record["message"] = trim_newline(record["message"]) + return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/70b18de0556f28eb634ccb944b2b33d0.lua b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/70b18de0556f28eb634ccb944b2b33d0.lua new file mode 100644 index 0000000000..787c643396 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/70b18de0556f28eb634ccb944b2b33d0.lua @@ -0,0 +1,12 @@ + +function process(tag, timestamp, record) +local v = "agent.googleapis.com/couchbase_http_access"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["logging.googleapis.com/instrumentation_source"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/feature_tracking_otlp.json index 7ba3c2f031..3f348e3634 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_general"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_goxdcr"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_http_access"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_general"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_general"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_goxdcr"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_goxdcr"}},{"key":"key","value":{"stringValue":"[1].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_http_access"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_http_access"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/features.yaml b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/features.yaml index 3bc86ae96d..fa4b32883f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/features.yaml @@ -13,16 +13,28 @@ - module: logging feature: service:otel_logging key: otel_logging_supported_config - value: "false" + value: "true" - module: logging feature: receivers:couchbase_general key: "[0].enabled" value: "true" +- module: logging + feature: receivers:couchbase_general + key: "[0].include_paths.__length" + value: "5" - module: logging feature: receivers:couchbase_goxdcr key: "[1].enabled" value: "true" +- module: logging + feature: receivers:couchbase_goxdcr + key: "[1].include_paths.__length" + value: "1" - module: logging feature: receivers:couchbase_http_access key: "[2].enabled" value: "true" +- module: logging + feature: receivers:couchbase_http_access + key: "[2].include_paths.__length" + value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_main.conf index 4c856c9ace..d99aed7c1d 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_main.conf @@ -30,6 +30,7 @@ Rotate_Wait 30 Skip_Long_Lines On Tag couchbase.couchbase_general + multiline.parser multiline.couchbase.couchbase_general storage.type filesystem [INPUT] @@ -45,6 +46,7 @@ Rotate_Wait 30 Skip_Long_Lines On Tag couchbase.couchbase_goxdcr + multiline.parser multiline.couchbase.couchbase_goxdcr storage.type filesystem [INPUT] @@ -115,10 +117,15 @@ storage.type memory [FILTER] - Match couchbase.couchbase_general - Multiline.Key_Content message - Multiline.Parser couchbase.couchbase_general.couchbase_general.multiline - Name multiline + Match couchbase.couchbase_general + Name modify + Rename log message + +[FILTER] + Match couchbase.couchbase_general + Name lua + call strip_newline + script 3bca332f661c48e682cdb7a2bdf766e8.lua [FILTER] Match couchbase.couchbase_general @@ -131,7 +138,7 @@ Match couchbase.couchbase_general Name parser Reserve_Data True - Parser couchbase.couchbase_general.couchbase_general.0 + Parser couchbase.couchbase_general.0.0 [FILTER] Match couchbase.couchbase_general @@ -152,10 +159,15 @@ script e26740abdc1963695135a3e2d6c55410.lua [FILTER] - Match couchbase.couchbase_goxdcr - Multiline.Key_Content message - Multiline.Parser couchbase.couchbase_goxdcr.couchbase_goxdcr.multiline - Name multiline + Match couchbase.couchbase_goxdcr + Name modify + Rename log message + +[FILTER] + Match couchbase.couchbase_goxdcr + Name lua + call strip_newline + script 3bca332f661c48e682cdb7a2bdf766e8.lua [FILTER] Match couchbase.couchbase_goxdcr @@ -168,7 +180,7 @@ Match couchbase.couchbase_goxdcr Name parser Reserve_Data True - Parser couchbase.couchbase_goxdcr.couchbase_goxdcr.0 + Parser couchbase.couchbase_goxdcr.0.0 [FILTER] Match couchbase.couchbase_goxdcr @@ -199,7 +211,7 @@ Match couchbase.couchbase_http_access Name parser Reserve_Data True - Parser couchbase.couchbase_http_access.couchbase_http_access + Parser couchbase.couchbase_http_access.0 [FILTER] Match couchbase.couchbase_http_access @@ -207,6 +219,12 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua +[FILTER] + Match couchbase.couchbase_http_access + Name lua + call process + script 70b18de0556f28eb634ccb944b2b33d0.lua + [FILTER] Match couchbase.couchbase_http_access Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_parser.conf index c692b0f7f6..4543667908 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux-gpu/fluent_bit_parser.conf @@ -1,20 +1,20 @@ [PARSER] Format regex - Name couchbase.couchbase_general.couchbase_general.0 - Regex ^\[(?[^:]*):(?[^,]*),(?\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),(?[^:]*):([^:]+):(?[^\]]+)\](?.*)$ + Name couchbase.couchbase_general.0.0 + Regex ^\[(?[^:]*):(?[^,]*),(?\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),(?[^:]*):([^:]+):(?[^\]]+)\](?[\s|\S]*)? Time_Format %Y-%m-%dT%H:%M:%S.%L Time_Key timestamp [PARSER] Format regex - Name couchbase.couchbase_goxdcr.couchbase_goxdcr.0 - Regex ^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d*Z) (?\w+) (?\w+.\w+): (?.*)$ + Name couchbase.couchbase_goxdcr.0.0 + Regex ^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d*Z) (?\w+) (?\w+.\w+): (?[\s|\S]*)? Time_Format %Y-%m-%dT%H:%M:%S.%L Time_Key timestamp [PARSER] Format regex - Name couchbase.couchbase_http_access.couchbase_http_access + Name couchbase.couchbase_http_access.0 Regex ^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(?[^\]]*)\] "(?\S+) (?\S+) (?\S+)" (?[^ ]*) (?[^ ]*\S+) (?[^ ]*) "(?[^\"]*)" (?.*)$ Time_Format %d/%b/%Y:%H:%M:%S %z Time_Key timestamp @@ -36,14 +36,14 @@ [MULTILINE_PARSER] flush_timeout 1000 - name couchbase.couchbase_general.couchbase_general.multiline + name multiline.couchbase.couchbase_general type regex rule "start_state" "^\[([^\s+:]*):" "cont" rule "cont" "^(?!\[([^\s+:]*):).*$" "cont" [MULTILINE_PARSER] flush_timeout 1000 - name couchbase.couchbase_goxdcr.couchbase_goxdcr.multiline + name multiline.couchbase.couchbase_goxdcr type regex rule "start_state" "^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}" "cont" rule "cont" "^(?!\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/3bca332f661c48e682cdb7a2bdf766e8.lua b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/3bca332f661c48e682cdb7a2bdf766e8.lua new file mode 100644 index 0000000000..fb1b4cee6f --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/3bca332f661c48e682cdb7a2bdf766e8.lua @@ -0,0 +1,16 @@ + +local function trim_newline(s) + -- Check for a Windows-style carriage return and newline (\r\n) + if string.sub(s, -2) == "\r\n" then + return string.sub(s, 1, -3) + -- Check for a Unix/Linux-style newline (\n) + elseif string.sub(s, -1) == "\n" then + return string.sub(s, 1, -2) + end + -- If no trailing newline is found, return the original string + return s +end +function strip_newline(tag, timestamp, record) + record["message"] = trim_newline(record["message"]) + return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/70b18de0556f28eb634ccb944b2b33d0.lua b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/70b18de0556f28eb634ccb944b2b33d0.lua new file mode 100644 index 0000000000..787c643396 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/70b18de0556f28eb634ccb944b2b33d0.lua @@ -0,0 +1,12 @@ + +function process(tag, timestamp, record) +local v = "agent.googleapis.com/couchbase_http_access"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["logging.googleapis.com/instrumentation_source"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/feature_tracking_otlp.json index 7ba3c2f031..3f348e3634 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_general"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_goxdcr"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_http_access"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_general"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_general"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_goxdcr"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_goxdcr"}},{"key":"key","value":{"stringValue":"[1].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_http_access"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_http_access"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/features.yaml b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/features.yaml index 3bc86ae96d..fa4b32883f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/features.yaml @@ -13,16 +13,28 @@ - module: logging feature: service:otel_logging key: otel_logging_supported_config - value: "false" + value: "true" - module: logging feature: receivers:couchbase_general key: "[0].enabled" value: "true" +- module: logging + feature: receivers:couchbase_general + key: "[0].include_paths.__length" + value: "5" - module: logging feature: receivers:couchbase_goxdcr key: "[1].enabled" value: "true" +- module: logging + feature: receivers:couchbase_goxdcr + key: "[1].include_paths.__length" + value: "1" - module: logging feature: receivers:couchbase_http_access key: "[2].enabled" value: "true" +- module: logging + feature: receivers:couchbase_http_access + key: "[2].include_paths.__length" + value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_main.conf index 4c856c9ace..d99aed7c1d 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_main.conf @@ -30,6 +30,7 @@ Rotate_Wait 30 Skip_Long_Lines On Tag couchbase.couchbase_general + multiline.parser multiline.couchbase.couchbase_general storage.type filesystem [INPUT] @@ -45,6 +46,7 @@ Rotate_Wait 30 Skip_Long_Lines On Tag couchbase.couchbase_goxdcr + multiline.parser multiline.couchbase.couchbase_goxdcr storage.type filesystem [INPUT] @@ -115,10 +117,15 @@ storage.type memory [FILTER] - Match couchbase.couchbase_general - Multiline.Key_Content message - Multiline.Parser couchbase.couchbase_general.couchbase_general.multiline - Name multiline + Match couchbase.couchbase_general + Name modify + Rename log message + +[FILTER] + Match couchbase.couchbase_general + Name lua + call strip_newline + script 3bca332f661c48e682cdb7a2bdf766e8.lua [FILTER] Match couchbase.couchbase_general @@ -131,7 +138,7 @@ Match couchbase.couchbase_general Name parser Reserve_Data True - Parser couchbase.couchbase_general.couchbase_general.0 + Parser couchbase.couchbase_general.0.0 [FILTER] Match couchbase.couchbase_general @@ -152,10 +159,15 @@ script e26740abdc1963695135a3e2d6c55410.lua [FILTER] - Match couchbase.couchbase_goxdcr - Multiline.Key_Content message - Multiline.Parser couchbase.couchbase_goxdcr.couchbase_goxdcr.multiline - Name multiline + Match couchbase.couchbase_goxdcr + Name modify + Rename log message + +[FILTER] + Match couchbase.couchbase_goxdcr + Name lua + call strip_newline + script 3bca332f661c48e682cdb7a2bdf766e8.lua [FILTER] Match couchbase.couchbase_goxdcr @@ -168,7 +180,7 @@ Match couchbase.couchbase_goxdcr Name parser Reserve_Data True - Parser couchbase.couchbase_goxdcr.couchbase_goxdcr.0 + Parser couchbase.couchbase_goxdcr.0.0 [FILTER] Match couchbase.couchbase_goxdcr @@ -199,7 +211,7 @@ Match couchbase.couchbase_http_access Name parser Reserve_Data True - Parser couchbase.couchbase_http_access.couchbase_http_access + Parser couchbase.couchbase_http_access.0 [FILTER] Match couchbase.couchbase_http_access @@ -207,6 +219,12 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua +[FILTER] + Match couchbase.couchbase_http_access + Name lua + call process + script 70b18de0556f28eb634ccb944b2b33d0.lua + [FILTER] Match couchbase.couchbase_http_access Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_parser.conf index c692b0f7f6..4543667908 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/linux/fluent_bit_parser.conf @@ -1,20 +1,20 @@ [PARSER] Format regex - Name couchbase.couchbase_general.couchbase_general.0 - Regex ^\[(?[^:]*):(?[^,]*),(?\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),(?[^:]*):([^:]+):(?[^\]]+)\](?.*)$ + Name couchbase.couchbase_general.0.0 + Regex ^\[(?[^:]*):(?[^,]*),(?\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),(?[^:]*):([^:]+):(?[^\]]+)\](?[\s|\S]*)? Time_Format %Y-%m-%dT%H:%M:%S.%L Time_Key timestamp [PARSER] Format regex - Name couchbase.couchbase_goxdcr.couchbase_goxdcr.0 - Regex ^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d*Z) (?\w+) (?\w+.\w+): (?.*)$ + Name couchbase.couchbase_goxdcr.0.0 + Regex ^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d*Z) (?\w+) (?\w+.\w+): (?[\s|\S]*)? Time_Format %Y-%m-%dT%H:%M:%S.%L Time_Key timestamp [PARSER] Format regex - Name couchbase.couchbase_http_access.couchbase_http_access + Name couchbase.couchbase_http_access.0 Regex ^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(?[^\]]*)\] "(?\S+) (?\S+) (?\S+)" (?[^ ]*) (?[^ ]*\S+) (?[^ ]*) "(?[^\"]*)" (?.*)$ Time_Format %d/%b/%Y:%H:%M:%S %z Time_Key timestamp @@ -36,14 +36,14 @@ [MULTILINE_PARSER] flush_timeout 1000 - name couchbase.couchbase_general.couchbase_general.multiline + name multiline.couchbase.couchbase_general type regex rule "start_state" "^\[([^\s+:]*):" "cont" rule "cont" "^(?!\[([^\s+:]*):).*$" "cont" [MULTILINE_PARSER] flush_timeout 1000 - name couchbase.couchbase_goxdcr.couchbase_goxdcr.multiline + name multiline.couchbase.couchbase_goxdcr type regex rule "start_state" "^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}" "cont" rule "cont" "^(?!\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/3bca332f661c48e682cdb7a2bdf766e8.lua b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/3bca332f661c48e682cdb7a2bdf766e8.lua new file mode 100644 index 0000000000..fb1b4cee6f --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/3bca332f661c48e682cdb7a2bdf766e8.lua @@ -0,0 +1,16 @@ + +local function trim_newline(s) + -- Check for a Windows-style carriage return and newline (\r\n) + if string.sub(s, -2) == "\r\n" then + return string.sub(s, 1, -3) + -- Check for a Unix/Linux-style newline (\n) + elseif string.sub(s, -1) == "\n" then + return string.sub(s, 1, -2) + end + -- If no trailing newline is found, return the original string + return s +end +function strip_newline(tag, timestamp, record) + record["message"] = trim_newline(record["message"]) + return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/70b18de0556f28eb634ccb944b2b33d0.lua b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/70b18de0556f28eb634ccb944b2b33d0.lua new file mode 100644 index 0000000000..787c643396 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/70b18de0556f28eb634ccb944b2b33d0.lua @@ -0,0 +1,12 @@ + +function process(tag, timestamp, record) +local v = "agent.googleapis.com/couchbase_http_access"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["logging.googleapis.com/instrumentation_source"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/feature_tracking_otlp.json index 7ba3c2f031..3f348e3634 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_general"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_goxdcr"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_http_access"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_general"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_general"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_goxdcr"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_goxdcr"}},{"key":"key","value":{"stringValue":"[1].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_http_access"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_http_access"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/features.yaml b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/features.yaml index 3bc86ae96d..fa4b32883f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/features.yaml @@ -13,16 +13,28 @@ - module: logging feature: service:otel_logging key: otel_logging_supported_config - value: "false" + value: "true" - module: logging feature: receivers:couchbase_general key: "[0].enabled" value: "true" +- module: logging + feature: receivers:couchbase_general + key: "[0].include_paths.__length" + value: "5" - module: logging feature: receivers:couchbase_goxdcr key: "[1].enabled" value: "true" +- module: logging + feature: receivers:couchbase_goxdcr + key: "[1].include_paths.__length" + value: "1" - module: logging feature: receivers:couchbase_http_access key: "[2].enabled" value: "true" +- module: logging + feature: receivers:couchbase_http_access + key: "[2].include_paths.__length" + value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_main.conf index 02f5db79ef..12a985b3ee 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_main.conf @@ -30,6 +30,7 @@ Rotate_Wait 30 Skip_Long_Lines On Tag couchbase.couchbase_general + multiline.parser multiline.couchbase.couchbase_general storage.type filesystem [INPUT] @@ -45,6 +46,7 @@ Rotate_Wait 30 Skip_Long_Lines On Tag couchbase.couchbase_goxdcr + multiline.parser multiline.couchbase.couchbase_goxdcr storage.type filesystem [INPUT] @@ -108,10 +110,15 @@ storage.type memory [FILTER] - Match couchbase.couchbase_general - Multiline.Key_Content message - Multiline.Parser couchbase.couchbase_general.couchbase_general.multiline - Name multiline + Match couchbase.couchbase_general + Name modify + Rename log message + +[FILTER] + Match couchbase.couchbase_general + Name lua + call strip_newline + script 3bca332f661c48e682cdb7a2bdf766e8.lua [FILTER] Match couchbase.couchbase_general @@ -124,7 +131,7 @@ Match couchbase.couchbase_general Name parser Reserve_Data True - Parser couchbase.couchbase_general.couchbase_general.0 + Parser couchbase.couchbase_general.0.0 [FILTER] Match couchbase.couchbase_general @@ -145,10 +152,15 @@ script e26740abdc1963695135a3e2d6c55410.lua [FILTER] - Match couchbase.couchbase_goxdcr - Multiline.Key_Content message - Multiline.Parser couchbase.couchbase_goxdcr.couchbase_goxdcr.multiline - Name multiline + Match couchbase.couchbase_goxdcr + Name modify + Rename log message + +[FILTER] + Match couchbase.couchbase_goxdcr + Name lua + call strip_newline + script 3bca332f661c48e682cdb7a2bdf766e8.lua [FILTER] Match couchbase.couchbase_goxdcr @@ -161,7 +173,7 @@ Match couchbase.couchbase_goxdcr Name parser Reserve_Data True - Parser couchbase.couchbase_goxdcr.couchbase_goxdcr.0 + Parser couchbase.couchbase_goxdcr.0.0 [FILTER] Match couchbase.couchbase_goxdcr @@ -192,7 +204,7 @@ Match couchbase.couchbase_http_access Name parser Reserve_Data True - Parser couchbase.couchbase_http_access.couchbase_http_access + Parser couchbase.couchbase_http_access.0 [FILTER] Match couchbase.couchbase_http_access @@ -200,6 +212,12 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua +[FILTER] + Match couchbase.couchbase_http_access + Name lua + call process + script 70b18de0556f28eb634ccb944b2b33d0.lua + [FILTER] Match couchbase.couchbase_http_access Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_parser.conf index 01ef8df562..96298653de 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows-2012/fluent_bit_parser.conf @@ -1,20 +1,20 @@ [PARSER] Format regex - Name couchbase.couchbase_general.couchbase_general.0 - Regex ^\[(?[^:]*):(?[^,]*),(?\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),(?[^:]*):([^:]+):(?[^\]]+)\](?.*)$ + Name couchbase.couchbase_general.0.0 + Regex ^\[(?[^:]*):(?[^,]*),(?\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),(?[^:]*):([^:]+):(?[^\]]+)\](?[\s|\S]*)? Time_Format %Y-%m-%dT%H:%M:%S.%L Time_Key timestamp [PARSER] Format regex - Name couchbase.couchbase_goxdcr.couchbase_goxdcr.0 - Regex ^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d*Z) (?\w+) (?\w+.\w+): (?.*)$ + Name couchbase.couchbase_goxdcr.0.0 + Regex ^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d*Z) (?\w+) (?\w+.\w+): (?[\s|\S]*)? Time_Format %Y-%m-%dT%H:%M:%S.%L Time_Key timestamp [PARSER] Format regex - Name couchbase.couchbase_http_access.couchbase_http_access + Name couchbase.couchbase_http_access.0 Regex ^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(?[^\]]*)\] "(?\S+) (?\S+) (?\S+)" (?[^ ]*) (?[^ ]*\S+) (?[^ ]*) "(?[^\"]*)" (?.*)$ Time_Format %d/%b/%Y:%H:%M:%S %z Time_Key timestamp @@ -43,14 +43,14 @@ [MULTILINE_PARSER] flush_timeout 1000 - name couchbase.couchbase_general.couchbase_general.multiline + name multiline.couchbase.couchbase_general type regex rule "start_state" "^\[([^\s+:]*):" "cont" rule "cont" "^(?!\[([^\s+:]*):).*$" "cont" [MULTILINE_PARSER] flush_timeout 1000 - name couchbase.couchbase_goxdcr.couchbase_goxdcr.multiline + name multiline.couchbase.couchbase_goxdcr type regex rule "start_state" "^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}" "cont" rule "cont" "^(?!\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})" "cont" diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/3bca332f661c48e682cdb7a2bdf766e8.lua b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/3bca332f661c48e682cdb7a2bdf766e8.lua new file mode 100644 index 0000000000..fb1b4cee6f --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/3bca332f661c48e682cdb7a2bdf766e8.lua @@ -0,0 +1,16 @@ + +local function trim_newline(s) + -- Check for a Windows-style carriage return and newline (\r\n) + if string.sub(s, -2) == "\r\n" then + return string.sub(s, 1, -3) + -- Check for a Unix/Linux-style newline (\n) + elseif string.sub(s, -1) == "\n" then + return string.sub(s, 1, -2) + end + -- If no trailing newline is found, return the original string + return s +end +function strip_newline(tag, timestamp, record) + record["message"] = trim_newline(record["message"]) + return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/70b18de0556f28eb634ccb944b2b33d0.lua b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/70b18de0556f28eb634ccb944b2b33d0.lua new file mode 100644 index 0000000000..787c643396 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/70b18de0556f28eb634ccb944b2b33d0.lua @@ -0,0 +1,12 @@ + +function process(tag, timestamp, record) +local v = "agent.googleapis.com/couchbase_http_access"; +(function(value) +if record["logging.googleapis.com/labels"] == nil +then +record["logging.googleapis.com/labels"] = {} +end +record["logging.googleapis.com/labels"]["logging.googleapis.com/instrumentation_source"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/feature_tracking_otlp.json index 7ba3c2f031..3f348e3634 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_general"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_goxdcr"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_http_access"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_general"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_general"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"5"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_goxdcr"}},{"key":"key","value":{"stringValue":"[1].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_goxdcr"}},{"key":"key","value":{"stringValue":"[1].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_http_access"}},{"key":"key","value":{"stringValue":"[2].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:couchbase_http_access"}},{"key":"key","value":{"stringValue":"[2].include_paths.__length"}},{"key":"value","value":{"stringValue":"2"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/features.yaml b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/features.yaml index 3bc86ae96d..fa4b32883f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/features.yaml @@ -13,16 +13,28 @@ - module: logging feature: service:otel_logging key: otel_logging_supported_config - value: "false" + value: "true" - module: logging feature: receivers:couchbase_general key: "[0].enabled" value: "true" +- module: logging + feature: receivers:couchbase_general + key: "[0].include_paths.__length" + value: "5" - module: logging feature: receivers:couchbase_goxdcr key: "[1].enabled" value: "true" +- module: logging + feature: receivers:couchbase_goxdcr + key: "[1].include_paths.__length" + value: "1" - module: logging feature: receivers:couchbase_http_access key: "[2].enabled" value: "true" +- module: logging + feature: receivers:couchbase_http_access + key: "[2].include_paths.__length" + value: "2" diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_main.conf index 02f5db79ef..12a985b3ee 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_main.conf @@ -30,6 +30,7 @@ Rotate_Wait 30 Skip_Long_Lines On Tag couchbase.couchbase_general + multiline.parser multiline.couchbase.couchbase_general storage.type filesystem [INPUT] @@ -45,6 +46,7 @@ Rotate_Wait 30 Skip_Long_Lines On Tag couchbase.couchbase_goxdcr + multiline.parser multiline.couchbase.couchbase_goxdcr storage.type filesystem [INPUT] @@ -108,10 +110,15 @@ storage.type memory [FILTER] - Match couchbase.couchbase_general - Multiline.Key_Content message - Multiline.Parser couchbase.couchbase_general.couchbase_general.multiline - Name multiline + Match couchbase.couchbase_general + Name modify + Rename log message + +[FILTER] + Match couchbase.couchbase_general + Name lua + call strip_newline + script 3bca332f661c48e682cdb7a2bdf766e8.lua [FILTER] Match couchbase.couchbase_general @@ -124,7 +131,7 @@ Match couchbase.couchbase_general Name parser Reserve_Data True - Parser couchbase.couchbase_general.couchbase_general.0 + Parser couchbase.couchbase_general.0.0 [FILTER] Match couchbase.couchbase_general @@ -145,10 +152,15 @@ script e26740abdc1963695135a3e2d6c55410.lua [FILTER] - Match couchbase.couchbase_goxdcr - Multiline.Key_Content message - Multiline.Parser couchbase.couchbase_goxdcr.couchbase_goxdcr.multiline - Name multiline + Match couchbase.couchbase_goxdcr + Name modify + Rename log message + +[FILTER] + Match couchbase.couchbase_goxdcr + Name lua + call strip_newline + script 3bca332f661c48e682cdb7a2bdf766e8.lua [FILTER] Match couchbase.couchbase_goxdcr @@ -161,7 +173,7 @@ Match couchbase.couchbase_goxdcr Name parser Reserve_Data True - Parser couchbase.couchbase_goxdcr.couchbase_goxdcr.0 + Parser couchbase.couchbase_goxdcr.0.0 [FILTER] Match couchbase.couchbase_goxdcr @@ -192,7 +204,7 @@ Match couchbase.couchbase_http_access Name parser Reserve_Data True - Parser couchbase.couchbase_http_access.couchbase_http_access + Parser couchbase.couchbase_http_access.0 [FILTER] Match couchbase.couchbase_http_access @@ -200,6 +212,12 @@ call parser_merge_record script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua +[FILTER] + Match couchbase.couchbase_http_access + Name lua + call process + script 70b18de0556f28eb634ccb944b2b33d0.lua + [FILTER] Match couchbase.couchbase_http_access Name lua diff --git a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_parser.conf index 01ef8df562..96298653de 100644 --- a/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_couchbase/golden/windows/fluent_bit_parser.conf @@ -1,20 +1,20 @@ [PARSER] Format regex - Name couchbase.couchbase_general.couchbase_general.0 - Regex ^\[(?[^:]*):(?[^,]*),(?\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),(?[^:]*):([^:]+):(?[^\]]+)\](?.*)$ + Name couchbase.couchbase_general.0.0 + Regex ^\[(?[^:]*):(?[^,]*),(?\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),(?[^:]*):([^:]+):(?[^\]]+)\](?[\s|\S]*)? Time_Format %Y-%m-%dT%H:%M:%S.%L Time_Key timestamp [PARSER] Format regex - Name couchbase.couchbase_goxdcr.couchbase_goxdcr.0 - Regex ^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d*Z) (?\w+) (?\w+.\w+): (?.*)$ + Name couchbase.couchbase_goxdcr.0.0 + Regex ^(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d*Z) (?\w+) (?\w+.\w+): (?[\s|\S]*)? Time_Format %Y-%m-%dT%H:%M:%S.%L Time_Key timestamp [PARSER] Format regex - Name couchbase.couchbase_http_access.couchbase_http_access + Name couchbase.couchbase_http_access.0 Regex ^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(?[^\]]*)\] "(?\S+) (?\S+) (?\S+)" (?[^ ]*) (?[^ ]*\S+) (?[^ ]*) "(?[^\"]*)" (?.*)$ Time_Format %d/%b/%Y:%H:%M:%S %z Time_Key timestamp @@ -43,14 +43,14 @@ [MULTILINE_PARSER] flush_timeout 1000 - name couchbase.couchbase_general.couchbase_general.multiline + name multiline.couchbase.couchbase_general type regex rule "start_state" "^\[([^\s+:]*):" "cont" rule "cont" "^(?!\[([^\s+:]*):).*$" "cont" [MULTILINE_PARSER] flush_timeout 1000 - name couchbase.couchbase_goxdcr.couchbase_goxdcr.multiline + name multiline.couchbase.couchbase_goxdcr type regex rule "start_state" "^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}" "cont" rule "cont" "^(?!\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})" "cont" diff --git a/transformation_test/testdata/logging_processor-couchbase-general/config.yaml b/transformation_test/testdata/logging_processor-couchbase-general/config.yaml new file mode 100644 index 0000000000..3692c2992c --- /dev/null +++ b/transformation_test/testdata/logging_processor-couchbase-general/config.yaml @@ -0,0 +1 @@ +- type: couchbase_general diff --git a/transformation_test/testdata/logging_processor-couchbase-general/input.log b/transformation_test/testdata/logging_processor-couchbase-general/input.log new file mode 100644 index 0000000000..2a238ce912 --- /dev/null +++ b/transformation_test/testdata/logging_processor-couchbase-general/input.log @@ -0,0 +1,8 @@ +[ns_server:info,2025-08-18T09:15:12.123Z,ns_1@127.0.0.1:0000:menelaus_web]Bucket 'travel-data' created with 3 replicas +[ns_server:warn,2025-08-18T09:16:05.452Z,ns_1@127.0.0.1:0000:index_updater]Index build taking longer than expected on bucket 'travel-data' +[ns_server:error,2025-08-18T09:16:33.764Z,ns_1@127.0.0.1:0000:bucket_engine]Failed to allocate memory for vbuckets: insufficient quota +[ns_server:debug,2025-08-18T09:16:55.987Z,ns_1@127.0.0.1:0000:rebalance]Rebalance progress: 45% completed +Continuation line giving rebalance details +Still continuing additional rebalance logs with memory usage stats... +[ns_server:info,2025-08-18T09:17:30.333Z,ns_1@127.0.0.1:0000:cluster_manager]Node 'ns_1@127.0.0.1' added to cluster +[ns_server:error,2025-08-18T09:18:00.741Z,ns_1@127.0.0.1:0000:ssl_certificate]SSL handshake failed: certificate expired diff --git a/transformation_test/testdata/logging_processor-couchbase-general/output_fluentbit.yaml b/transformation_test/testdata/logging_processor-couchbase-general/output_fluentbit.yaml new file mode 100644 index 0000000000..6323704f0c --- /dev/null +++ b/transformation_test/testdata/logging_processor-couchbase-general/output_fluentbit.yaml @@ -0,0 +1,68 @@ +- entries: + - jsonPayload: + level: info + message: Bucket 'travel-data' created with 3 replicas + node_name: ns_1@127.0.0.1 + source: menelaus_web + type: ns_server + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_general + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: 2025-08-18T09:15:12.123000000Z + - jsonPayload: + level: warn + message: Index build taking longer than expected on bucket 'travel-data' + node_name: ns_1@127.0.0.1 + source: index_updater + type: ns_server + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_general + logName: projects/my-project/logs/transformation_test + severity: 400.0 + timestamp: 2025-08-18T09:16:05.452000000Z + - jsonPayload: + level: error + message: "Failed to allocate memory for vbuckets: insufficient quota" + node_name: ns_1@127.0.0.1 + source: bucket_engine + type: ns_server + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_general + logName: projects/my-project/logs/transformation_test + severity: 500.0 + timestamp: 2025-08-18T09:16:33.764000000Z + - jsonPayload: + level: debug + message: |- + Rebalance progress: 45% completed + Continuation line giving rebalance details + Still continuing additional rebalance logs with memory usage stats... + node_name: ns_1@127.0.0.1 + source: rebalance + type: ns_server + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_general + logName: projects/my-project/logs/transformation_test + severity: 100.0 + timestamp: 2025-08-18T09:16:55.987000000Z + - jsonPayload: + level: info + message: Node 'ns_1@127.0.0.1' added to cluster + node_name: ns_1@127.0.0.1 + source: cluster_manager + type: ns_server + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_general + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: 2025-08-18T09:17:30.333000000Z + partialSuccess: true + resource: + labels: {} + type: gce_instance diff --git a/transformation_test/testdata/logging_processor-couchbase-general/output_otel.yaml b/transformation_test/testdata/logging_processor-couchbase-general/output_otel.yaml new file mode 100644 index 0000000000..fc48a90a3b --- /dev/null +++ b/transformation_test/testdata/logging_processor-couchbase-general/output_otel.yaml @@ -0,0 +1,194 @@ +- entries: + - jsonPayload: + level: info + message: Bucket 'travel-data' created with 3 replicas + node_name: ns_1@127.0.0.1 + source: menelaus_web + timestamp: 2025-08-18T09:15:12.123Z + type: ns_server + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_general + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + severity: INFO + timestamp: now + - jsonPayload: + level: warn + message: Index build taking longer than expected on bucket 'travel-data' + node_name: ns_1@127.0.0.1 + source: index_updater + timestamp: 2025-08-18T09:16:05.452Z + type: ns_server + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_general + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + severity: WARNING + timestamp: now + - jsonPayload: + level: error + message: "Failed to allocate memory for vbuckets: insufficient quota" + node_name: ns_1@127.0.0.1 + source: bucket_engine + timestamp: 2025-08-18T09:16:33.764Z + type: ns_server + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_general + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + severity: ERROR + timestamp: now + - jsonPayload: + level: debug + message: |- + Rebalance progress: 45% completed + Continuation line giving rebalance details + Still continuing additional rebalance logs with memory usage stats... + node_name: ns_1@127.0.0.1 + source: rebalance + timestamp: 2025-08-18T09:16:55.987Z + type: ns_server + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_general + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + severity: DEBUG + timestamp: now + - jsonPayload: + level: info + message: Node 'ns_1@127.0.0.1' added to cluster + node_name: ns_1@127.0.0.1 + source: cluster_manager + timestamp: 2025-08-18T09:17:30.333Z + type: ns_server + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_general + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + severity: INFO + timestamp: now + partialSuccess: true +- entries: + - jsonPayload: + level: error + message: "SSL handshake failed: certificate expired" + node_name: ns_1@127.0.0.1 + source: ssl_certificate + timestamp: 2025-08-18T09:18:00.741Z + type: ns_server + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_general + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + severity: ERROR + timestamp: now + partialSuccess: true +- collector_errors: + - caller: ottl@v0.136.0/parser.go:410 + error: "parsing time \"2025-08-18T09:15:12.123Z\": extra text: \"Z\"" + level: warn + msg: failed to execute statement + otelcol.component.id: transform/logs_transformation__test_my-log-name_1 + otelcol.component.kind: processor + otelcol.pipeline.id: logs/logs_transformation__test_my-log-name + otelcol.signal: logs + resource: + service.instance.id: test-service-instance-id + service.name: otelopscol + service.version: v0.136.0 + statement: set(log.cache["__time_valid"], true) where ((log.body != nil and log.body["timestamp"] != nil) and Time(log.body["timestamp"], "%Y-%m-%dT%H:%M:%S.%L") != nil) + - caller: ottl@v0.136.0/parser.go:410 + error: "parsing time \"2025-08-18T09:16:05.452Z\": extra text: \"Z\"" + level: warn + msg: failed to execute statement + otelcol.component.id: transform/logs_transformation__test_my-log-name_1 + otelcol.component.kind: processor + otelcol.pipeline.id: logs/logs_transformation__test_my-log-name + otelcol.signal: logs + resource: + service.instance.id: test-service-instance-id + service.name: otelopscol + service.version: v0.136.0 + statement: set(log.cache["__time_valid"], true) where ((log.body != nil and log.body["timestamp"] != nil) and Time(log.body["timestamp"], "%Y-%m-%dT%H:%M:%S.%L") != nil) + - caller: ottl@v0.136.0/parser.go:410 + error: "parsing time \"2025-08-18T09:16:33.764Z\": extra text: \"Z\"" + level: warn + msg: failed to execute statement + otelcol.component.id: transform/logs_transformation__test_my-log-name_1 + otelcol.component.kind: processor + otelcol.pipeline.id: logs/logs_transformation__test_my-log-name + otelcol.signal: logs + resource: + service.instance.id: test-service-instance-id + service.name: otelopscol + service.version: v0.136.0 + statement: set(log.cache["__time_valid"], true) where ((log.body != nil and log.body["timestamp"] != nil) and Time(log.body["timestamp"], "%Y-%m-%dT%H:%M:%S.%L") != nil) + - caller: ottl@v0.136.0/parser.go:410 + error: "parsing time \"2025-08-18T09:16:55.987Z\": extra text: \"Z\"" + level: warn + msg: failed to execute statement + otelcol.component.id: transform/logs_transformation__test_my-log-name_1 + otelcol.component.kind: processor + otelcol.pipeline.id: logs/logs_transformation__test_my-log-name + otelcol.signal: logs + resource: + service.instance.id: test-service-instance-id + service.name: otelopscol + service.version: v0.136.0 + statement: set(log.cache["__time_valid"], true) where ((log.body != nil and log.body["timestamp"] != nil) and Time(log.body["timestamp"], "%Y-%m-%dT%H:%M:%S.%L") != nil) + - caller: ottl@v0.136.0/parser.go:410 + error: "parsing time \"2025-08-18T09:17:30.333Z\": extra text: \"Z\"" + level: warn + msg: failed to execute statement + otelcol.component.id: transform/logs_transformation__test_my-log-name_1 + otelcol.component.kind: processor + otelcol.pipeline.id: logs/logs_transformation__test_my-log-name + otelcol.signal: logs + resource: + service.instance.id: test-service-instance-id + service.name: otelopscol + service.version: v0.136.0 + statement: set(log.cache["__time_valid"], true) where ((log.body != nil and log.body["timestamp"] != nil) and Time(log.body["timestamp"], "%Y-%m-%dT%H:%M:%S.%L") != nil) + - caller: ottl@v0.136.0/parser.go:410 + error: "parsing time \"2025-08-18T09:18:00.741Z\": extra text: \"Z\"" + level: warn + msg: failed to execute statement + otelcol.component.id: transform/logs_transformation__test_my-log-name_1 + otelcol.component.kind: processor + otelcol.pipeline.id: logs/logs_transformation__test_my-log-name + otelcol.signal: logs + resource: + service.instance.id: test-service-instance-id + service.name: otelopscol + service.version: v0.136.0 + statement: set(log.cache["__time_valid"], true) where ((log.body != nil and log.body["timestamp"] != nil) and Time(log.body["timestamp"], "%Y-%m-%dT%H:%M:%S.%L") != nil) diff --git a/transformation_test/testdata/logging_processor-couchbase-goxcdr/config.yaml b/transformation_test/testdata/logging_processor-couchbase-goxcdr/config.yaml new file mode 100644 index 0000000000..868ced49b1 --- /dev/null +++ b/transformation_test/testdata/logging_processor-couchbase-goxcdr/config.yaml @@ -0,0 +1 @@ +- type: couchbase_goxdcr diff --git a/transformation_test/testdata/logging_processor-couchbase-goxcdr/input.log b/transformation_test/testdata/logging_processor-couchbase-goxcdr/input.log new file mode 100644 index 0000000000..5f722187da --- /dev/null +++ b/transformation_test/testdata/logging_processor-couchbase-goxcdr/input.log @@ -0,0 +1,6 @@ +2025-08-18T14:27:02.452Z INFO PipelineManager: Starting replication pipeline for bucket travel-data +2025-08-18T14:27:10.874Z WARN XDCR.CheckpointService: Failed to save checkpoint due to timeout +2025-08-18T14:27:22.345Z ERROR Router: Failed to route mutation for key "user:12345", retrying... +Additional detail: document size exceeded configured threshold +2025-08-18T14:27:45.657Z DEBUG BackfillManager: Backfill task created for vbucket 102 +2025-08-18T14:28:00.998Z INFO PipelineManager: Replication pipeline stopped successfully diff --git a/transformation_test/testdata/logging_processor-couchbase-goxcdr/output_fluentbit.yaml b/transformation_test/testdata/logging_processor-couchbase-goxcdr/output_fluentbit.yaml new file mode 100644 index 0000000000..b4cc7f21d7 --- /dev/null +++ b/transformation_test/testdata/logging_processor-couchbase-goxcdr/output_fluentbit.yaml @@ -0,0 +1,47 @@ +- entries: + - jsonPayload: + level: INFO + log_type: PipelineManager + message: Starting replication pipeline for bucket travel-data + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_goxdcr + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: 2025-08-18T14:27:02.452000000Z + - jsonPayload: + level: WARN + log_type: XDCR.CheckpointService + message: Failed to save checkpoint due to timeout + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_goxdcr + logName: projects/my-project/logs/transformation_test + severity: 400.0 + timestamp: 2025-08-18T14:27:10.874000000Z + - jsonPayload: + level: ERROR + log_type: Router + message: |- + Failed to route mutation for key "user:12345", retrying... + Additional detail: document size exceeded configured threshold + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_goxdcr + logName: projects/my-project/logs/transformation_test + severity: 500.0 + timestamp: 2025-08-18T14:27:22.345000000Z + - jsonPayload: + level: DEBUG + log_type: BackfillManager + message: Backfill task created for vbucket 102 + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_goxdcr + logName: projects/my-project/logs/transformation_test + severity: 100.0 + timestamp: 2025-08-18T14:27:45.657000000Z + partialSuccess: true + resource: + labels: {} + type: gce_instance diff --git a/transformation_test/testdata/logging_processor-couchbase-goxcdr/output_otel.yaml b/transformation_test/testdata/logging_processor-couchbase-goxcdr/output_otel.yaml new file mode 100644 index 0000000000..09bd074381 --- /dev/null +++ b/transformation_test/testdata/logging_processor-couchbase-goxcdr/output_otel.yaml @@ -0,0 +1,152 @@ +- entries: + - jsonPayload: + level: INFO + log_type: PipelineManager + message: Starting replication pipeline for bucket travel-data + timestamp: 2025-08-18T14:27:02.452Z + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_goxdcr + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + severity: INFO + timestamp: now + - jsonPayload: + level: WARN + log_type: XDCR.CheckpointService + message: Failed to save checkpoint due to timeout + timestamp: 2025-08-18T14:27:10.874Z + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_goxdcr + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + severity: WARNING + timestamp: now + - jsonPayload: + level: ERROR + log_type: Router + message: |- + Failed to route mutation for key "user:12345", retrying... + Additional detail: document size exceeded configured threshold + timestamp: 2025-08-18T14:27:22.345Z + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_goxdcr + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + severity: ERROR + timestamp: now + - jsonPayload: + level: DEBUG + log_type: BackfillManager + message: Backfill task created for vbucket 102 + timestamp: 2025-08-18T14:27:45.657Z + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_goxdcr + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + severity: DEBUG + timestamp: now + partialSuccess: true +- entries: + - jsonPayload: + level: INFO + log_type: PipelineManager + message: Replication pipeline stopped successfully + timestamp: 2025-08-18T14:28:00.998Z + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_goxdcr + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + severity: INFO + timestamp: now + partialSuccess: true +- collector_errors: + - caller: ottl@v0.136.0/parser.go:410 + error: "parsing time \"2025-08-18T14:27:02.452Z\": extra text: \"Z\"" + level: warn + msg: failed to execute statement + otelcol.component.id: transform/logs_transformation__test_my-log-name_1 + otelcol.component.kind: processor + otelcol.pipeline.id: logs/logs_transformation__test_my-log-name + otelcol.signal: logs + resource: + service.instance.id: test-service-instance-id + service.name: otelopscol + service.version: v0.136.0 + statement: set(log.cache["__time_valid"], true) where ((log.body != nil and log.body["timestamp"] != nil) and Time(log.body["timestamp"], "%Y-%m-%dT%H:%M:%S.%L") != nil) + - caller: ottl@v0.136.0/parser.go:410 + error: "parsing time \"2025-08-18T14:27:10.874Z\": extra text: \"Z\"" + level: warn + msg: failed to execute statement + otelcol.component.id: transform/logs_transformation__test_my-log-name_1 + otelcol.component.kind: processor + otelcol.pipeline.id: logs/logs_transformation__test_my-log-name + otelcol.signal: logs + resource: + service.instance.id: test-service-instance-id + service.name: otelopscol + service.version: v0.136.0 + statement: set(log.cache["__time_valid"], true) where ((log.body != nil and log.body["timestamp"] != nil) and Time(log.body["timestamp"], "%Y-%m-%dT%H:%M:%S.%L") != nil) + - caller: ottl@v0.136.0/parser.go:410 + error: "parsing time \"2025-08-18T14:27:22.345Z\": extra text: \"Z\"" + level: warn + msg: failed to execute statement + otelcol.component.id: transform/logs_transformation__test_my-log-name_1 + otelcol.component.kind: processor + otelcol.pipeline.id: logs/logs_transformation__test_my-log-name + otelcol.signal: logs + resource: + service.instance.id: test-service-instance-id + service.name: otelopscol + service.version: v0.136.0 + statement: set(log.cache["__time_valid"], true) where ((log.body != nil and log.body["timestamp"] != nil) and Time(log.body["timestamp"], "%Y-%m-%dT%H:%M:%S.%L") != nil) + - caller: ottl@v0.136.0/parser.go:410 + error: "parsing time \"2025-08-18T14:27:45.657Z\": extra text: \"Z\"" + level: warn + msg: failed to execute statement + otelcol.component.id: transform/logs_transformation__test_my-log-name_1 + otelcol.component.kind: processor + otelcol.pipeline.id: logs/logs_transformation__test_my-log-name + otelcol.signal: logs + resource: + service.instance.id: test-service-instance-id + service.name: otelopscol + service.version: v0.136.0 + statement: set(log.cache["__time_valid"], true) where ((log.body != nil and log.body["timestamp"] != nil) and Time(log.body["timestamp"], "%Y-%m-%dT%H:%M:%S.%L") != nil) + - caller: ottl@v0.136.0/parser.go:410 + error: "parsing time \"2025-08-18T14:28:00.998Z\": extra text: \"Z\"" + level: warn + msg: failed to execute statement + otelcol.component.id: transform/logs_transformation__test_my-log-name_1 + otelcol.component.kind: processor + otelcol.pipeline.id: logs/logs_transformation__test_my-log-name + otelcol.signal: logs + resource: + service.instance.id: test-service-instance-id + service.name: otelopscol + service.version: v0.136.0 + statement: set(log.cache["__time_valid"], true) where ((log.body != nil and log.body["timestamp"] != nil) and Time(log.body["timestamp"], "%Y-%m-%dT%H:%M:%S.%L") != nil) diff --git a/transformation_test/testdata/logging_processor-couchbase-http-access/config.yaml b/transformation_test/testdata/logging_processor-couchbase-http-access/config.yaml new file mode 100644 index 0000000000..713c80708c --- /dev/null +++ b/transformation_test/testdata/logging_processor-couchbase-http-access/config.yaml @@ -0,0 +1 @@ +- type: couchbase_http_access diff --git a/transformation_test/testdata/logging_processor-couchbase-http-access/input.log b/transformation_test/testdata/logging_processor-couchbase-http-access/input.log new file mode 100644 index 0000000000..be5756c2bd --- /dev/null +++ b/transformation_test/testdata/logging_processor-couchbase-http-access/input.log @@ -0,0 +1,10 @@ +192.168.1.10 couchbase.local - [18/Aug/2025:09:20:11 +0000] "GET /pools/default/buckets/travel-data HTTP/1.1" 200 512 - "curl/7.68.0" Request took 5ms +192.168.1.12 couchbase.local - [18/Aug/2025:09:20:42 +0000] "POST /pools/default/buckets/travel-data/docs HTTP/1.1" 201 123 - "python-requests/2.26.0" Request took 12ms +192.168.1.13 couchbase.local - [18/Aug/2025:09:21:10 +0000] "GET /pools/default/buckets/missing-data HTTP/1.1" 404 0 - "Mozilla/5.0 (Windows NT 10.0; Win64; x64)" Request took 2ms +192.168.1.14 couchbase.local - [18/Aug/2025:09:21:25 +0000] "PUT /pools/default/buckets/travel-data/docs/user:123 HTTP/1.1" 500 64 - "node-fetch/1.0" Internal Server Error +192.168.1.15 couchbase.local - [18/Aug/2025:09:22:00 +0000] "DELETE /pools/default/buckets/old-data HTTP/1.1" 403 64 - "curl/7.88.1" Unauthorized delete attempt +192.168.1.16 couchbase.local - [18/Aug/2025:09:22:35 +0000] "GET /pools/default/buckets/travel-data/docs?include_docs=true HTTP/1.1" 302 0 http://referrer.example.com "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36" Redirected to login +192.168.1.17 couchbase.local - [18/Aug/2025:09:23:01 +0000] "HEAD /pools/default/buckets/travel-data/docs/user:999 HTTP/1.1" 304 0 - "Go-http-client/1.1" Not modified +192.168.1.18 couchbase.local - [18/Aug/2025:09:23:40 +0000] "OPTIONS /pools/default/buckets/travel-data HTTP/1.1" 204 0 - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" Preflight check passed +192.168.1.19 couchbase.local - [18/Aug/2025:09:24:15 +0000] "GET /pools/default/buckets/travel-data/docs?limit=1000 HTTP/1.1" 413 0 - "curl/8.0.1" Payload too large +192.168.1.20 couchbase.local - [18/Aug/2025:09:25:00 +0000] "PATCH /pools/default/buckets/travel-data/docs/user:321 HTTP/1.1" 409 128 - "python-requests/2.31.0" Document conflict diff --git a/transformation_test/testdata/logging_processor-couchbase-http-access/output_fluentbit.yaml b/transformation_test/testdata/logging_processor-couchbase-http-access/output_fluentbit.yaml new file mode 100644 index 0000000000..e82ac5492d --- /dev/null +++ b/transformation_test/testdata/logging_processor-couchbase-http-access/output_fluentbit.yaml @@ -0,0 +1,245 @@ +- entries: + - httpRequest: + cacheFillBytes: 0.0 + cacheHit: false + cacheLookup: false + cacheValidatedWithOriginServer: false + protocol: HTTP/1.1 + referer: "" + remoteIp: 192.168.1.10 + requestMethod: GET + requestSize: 0.0 + requestUrl: /pools/default/buckets/travel-data + responseSize: 512.0 + serverIp: "" + status: 200.0 + userAgent: curl/7.68.0 + jsonPayload: + host: couchbase.local + message: Request took 5ms + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T09:20:11.000000000Z + - httpRequest: + cacheFillBytes: 0.0 + cacheHit: false + cacheLookup: false + cacheValidatedWithOriginServer: false + protocol: HTTP/1.1 + referer: "" + remoteIp: 192.168.1.12 + requestMethod: POST + requestSize: 0.0 + requestUrl: /pools/default/buckets/travel-data/docs + responseSize: 123.0 + serverIp: "" + status: 201.0 + userAgent: python-requests/2.26.0 + jsonPayload: + host: couchbase.local + message: Request took 12ms + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T09:20:42.000000000Z + - httpRequest: + cacheFillBytes: 0.0 + cacheHit: false + cacheLookup: false + cacheValidatedWithOriginServer: false + protocol: HTTP/1.1 + referer: "" + remoteIp: 192.168.1.13 + requestMethod: GET + requestSize: 0.0 + requestUrl: /pools/default/buckets/missing-data + responseSize: 0.0 + serverIp: "" + status: 404.0 + userAgent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) + jsonPayload: + host: couchbase.local + message: Request took 2ms + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T09:21:10.000000000Z + - httpRequest: + cacheFillBytes: 0.0 + cacheHit: false + cacheLookup: false + cacheValidatedWithOriginServer: false + protocol: HTTP/1.1 + referer: "" + remoteIp: 192.168.1.14 + requestMethod: PUT + requestSize: 0.0 + requestUrl: /pools/default/buckets/travel-data/docs/user:123 + responseSize: 64.0 + serverIp: "" + status: 500.0 + userAgent: node-fetch/1.0 + jsonPayload: + host: couchbase.local + message: Internal Server Error + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T09:21:25.000000000Z + - httpRequest: + cacheFillBytes: 0.0 + cacheHit: false + cacheLookup: false + cacheValidatedWithOriginServer: false + protocol: HTTP/1.1 + referer: "" + remoteIp: 192.168.1.15 + requestMethod: DELETE + requestSize: 0.0 + requestUrl: /pools/default/buckets/old-data + responseSize: 64.0 + serverIp: "" + status: 403.0 + userAgent: curl/7.88.1 + jsonPayload: + host: couchbase.local + message: Unauthorized delete attempt + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T09:22:00.000000000Z + - httpRequest: + cacheFillBytes: 0.0 + cacheHit: false + cacheLookup: false + cacheValidatedWithOriginServer: false + protocol: HTTP/1.1 + referer: http://referrer.example.com + remoteIp: 192.168.1.16 + requestMethod: GET + requestSize: 0.0 + requestUrl: /pools/default/buckets/travel-data/docs?include_docs=true + responseSize: 0.0 + serverIp: "" + status: 302.0 + userAgent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36 + jsonPayload: + host: couchbase.local + message: Redirected to login + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T09:22:35.000000000Z + - httpRequest: + cacheFillBytes: 0.0 + cacheHit: false + cacheLookup: false + cacheValidatedWithOriginServer: false + protocol: HTTP/1.1 + referer: "" + remoteIp: 192.168.1.17 + requestMethod: HEAD + requestSize: 0.0 + requestUrl: /pools/default/buckets/travel-data/docs/user:999 + responseSize: 0.0 + serverIp: "" + status: 304.0 + userAgent: Go-http-client/1.1 + jsonPayload: + host: couchbase.local + message: Not modified + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T09:23:01.000000000Z + - httpRequest: + cacheFillBytes: 0.0 + cacheHit: false + cacheLookup: false + cacheValidatedWithOriginServer: false + protocol: HTTP/1.1 + referer: "" + remoteIp: 192.168.1.18 + requestMethod: OPTIONS + requestSize: 0.0 + requestUrl: /pools/default/buckets/travel-data + responseSize: 0.0 + serverIp: "" + status: 204.0 + userAgent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) + jsonPayload: + host: couchbase.local + message: Preflight check passed + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T09:23:40.000000000Z + - httpRequest: + cacheFillBytes: 0.0 + cacheHit: false + cacheLookup: false + cacheValidatedWithOriginServer: false + protocol: HTTP/1.1 + referer: "" + remoteIp: 192.168.1.19 + requestMethod: GET + requestSize: 0.0 + requestUrl: /pools/default/buckets/travel-data/docs?limit=1000 + responseSize: 0.0 + serverIp: "" + status: 413.0 + userAgent: curl/8.0.1 + jsonPayload: + host: couchbase.local + message: Payload too large + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T09:24:15.000000000Z + - httpRequest: + cacheFillBytes: 0.0 + cacheHit: false + cacheLookup: false + cacheValidatedWithOriginServer: false + protocol: HTTP/1.1 + referer: "" + remoteIp: 192.168.1.20 + requestMethod: PATCH + requestSize: 0.0 + requestUrl: /pools/default/buckets/travel-data/docs/user:321 + responseSize: 128.0 + serverIp: "" + status: 409.0 + userAgent: python-requests/2.31.0 + jsonPayload: + host: couchbase.local + message: Document conflict + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/transformation_test + timestamp: 2025-08-18T09:25:00.000000000Z + partialSuccess: true + resource: + labels: {} + type: gce_instance diff --git a/transformation_test/testdata/logging_processor-couchbase-http-access/output_otel.yaml b/transformation_test/testdata/logging_processor-couchbase-http-access/output_otel.yaml new file mode 100644 index 0000000000..ceca30f07c --- /dev/null +++ b/transformation_test/testdata/logging_processor-couchbase-http-access/output_otel.yaml @@ -0,0 +1,218 @@ +- entries: + - httpRequest: + protocol: HTTP/1.1 + remoteIp: 192.168.1.10 + requestMethod: GET + requestUrl: /pools/default/buckets/travel-data + responseSize: "512" + status: 200 + userAgent: curl/7.68.0 + jsonPayload: + host: couchbase.local + message: Request took 5ms + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + timestamp: 2025-08-18T09:20:11Z + - httpRequest: + protocol: HTTP/1.1 + remoteIp: 192.168.1.12 + requestMethod: POST + requestUrl: /pools/default/buckets/travel-data/docs + responseSize: "123" + status: 201 + userAgent: python-requests/2.26.0 + jsonPayload: + host: couchbase.local + message: Request took 12ms + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + timestamp: 2025-08-18T09:20:42Z + - httpRequest: + protocol: HTTP/1.1 + remoteIp: 192.168.1.13 + requestMethod: GET + requestUrl: /pools/default/buckets/missing-data + status: 404 + userAgent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) + jsonPayload: + host: couchbase.local + message: Request took 2ms + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + timestamp: 2025-08-18T09:21:10Z + - httpRequest: + protocol: HTTP/1.1 + remoteIp: 192.168.1.14 + requestMethod: PUT + requestUrl: /pools/default/buckets/travel-data/docs/user:123 + responseSize: "64" + status: 500 + userAgent: node-fetch/1.0 + jsonPayload: + host: couchbase.local + message: Internal Server Error + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + timestamp: 2025-08-18T09:21:25Z + - httpRequest: + protocol: HTTP/1.1 + remoteIp: 192.168.1.15 + requestMethod: DELETE + requestUrl: /pools/default/buckets/old-data + responseSize: "64" + status: 403 + userAgent: curl/7.88.1 + jsonPayload: + host: couchbase.local + message: Unauthorized delete attempt + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + timestamp: 2025-08-18T09:22:00Z + - httpRequest: + protocol: HTTP/1.1 + referer: http://referrer.example.com + remoteIp: 192.168.1.16 + requestMethod: GET + requestUrl: /pools/default/buckets/travel-data/docs?include_docs=true + status: 302 + userAgent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36 + jsonPayload: + host: couchbase.local + message: Redirected to login + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + timestamp: 2025-08-18T09:22:35Z + - httpRequest: + protocol: HTTP/1.1 + remoteIp: 192.168.1.17 + requestMethod: HEAD + requestUrl: /pools/default/buckets/travel-data/docs/user:999 + status: 304 + userAgent: Go-http-client/1.1 + jsonPayload: + host: couchbase.local + message: Not modified + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + timestamp: 2025-08-18T09:23:01Z + - httpRequest: + protocol: HTTP/1.1 + remoteIp: 192.168.1.18 + requestMethod: OPTIONS + requestUrl: /pools/default/buckets/travel-data + status: 204 + userAgent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) + jsonPayload: + host: couchbase.local + message: Preflight check passed + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + timestamp: 2025-08-18T09:23:40Z + - httpRequest: + protocol: HTTP/1.1 + remoteIp: 192.168.1.19 + requestMethod: GET + requestUrl: /pools/default/buckets/travel-data/docs?limit=1000 + status: 413 + userAgent: curl/8.0.1 + jsonPayload: + host: couchbase.local + message: Payload too large + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + timestamp: 2025-08-18T09:24:15Z + - httpRequest: + protocol: HTTP/1.1 + remoteIp: 192.168.1.20 + requestMethod: PATCH + requestUrl: /pools/default/buckets/travel-data/docs/user:321 + responseSize: "128" + status: 409 + userAgent: python-requests/2.31.0 + jsonPayload: + host: couchbase.local + message: Document conflict + user: - + labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/couchbase_http_access + logName: projects/my-project/logs/my-log-name + resource: + labels: + instance_id: test-instance-id + zone: test-zone + type: gce_instance + timestamp: 2025-08-18T09:25:00Z + partialSuccess: true