Skip to content

Commit 308261b

Browse files
feat: Update Elasticsearch json parser to use LoggingReceiverMacro (#1987)
Co-authored-by: Francisco Valente Castro <1435136+franciscovalentecastro@users.noreply.github.com>
1 parent e8bf4e7 commit 308261b

7 files changed

Lines changed: 251 additions & 34 deletions

File tree

apps/elasticsearch.go

Lines changed: 26 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -149,38 +149,35 @@ func init() {
149149
confgenerator.MetricsReceiverTypes.RegisterType(func() confgenerator.MetricsReceiver { return &MetricsReceiverElasticsearch{} })
150150
}
151151

152-
type LoggingProcessorElasticsearchJson struct {
153-
confgenerator.ConfigComponent `yaml:",inline"`
152+
type LoggingProcessorMacroElasticsearchJson struct {
154153
}
155154

156-
func (LoggingProcessorElasticsearchJson) Type() string {
155+
func (LoggingProcessorMacroElasticsearchJson) Type() string {
157156
return "elasticsearch_json"
158157
}
159158

160-
func (p LoggingProcessorElasticsearchJson) Components(ctx context.Context, tag, uid string) []fluentbit.Component {
161-
c := []fluentbit.Component{}
162-
159+
func (p LoggingProcessorMacroElasticsearchJson) Expand(ctx context.Context) []confgenerator.InternalLoggingProcessor {
163160
// sample log line:
164161
// {"type": "server", "timestamp": "2022-01-17T18:31:47,365Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "ubuntu-jammy", "message": "initialized" }
165162
// Logs are formatted based on configuration (log4j);
166163
// See https://artifacts.elastic.co/javadoc/org/elasticsearch/elasticsearch/7.16.2/org/elasticsearch/common/logging/ESJsonLayout.html
167164
// for general layout, and https://www.elastic.co/guide/en/elasticsearch/reference/current/logging.html for general configuration of logging
168-
169-
jsonParser := &confgenerator.LoggingProcessorParseJson{
170-
ParserShared: confgenerator.ParserShared{
171-
TimeKey: "timestamp",
172-
TimeFormat: "%Y-%m-%dT%H:%M:%S,%L%z",
165+
processors := []confgenerator.InternalLoggingProcessor{
166+
confgenerator.LoggingProcessorParseJson{
167+
ParserShared: confgenerator.ParserShared{
168+
TimeKey: "timestamp",
169+
TimeFormat: "%Y-%m-%dT%H:%M:%S,%L%z",
170+
},
173171
},
172+
p.severityParser(),
174173
}
175174

176-
c = append(c, jsonParser.Components(ctx, tag, uid)...)
177-
c = append(c, p.severityParser(ctx, tag, uid)...)
178-
c = append(c, p.nestingProcessors(ctx, tag, uid)...)
175+
processors = append(processors, p.nestingProcessors()...)
179176

180-
return c
177+
return processors
181178
}
182179

183-
func (p LoggingProcessorElasticsearchJson) severityParser(ctx context.Context, tag, uid string) []fluentbit.Component {
180+
func (p LoggingProcessorMacroElasticsearchJson) severityParser() confgenerator.InternalLoggingProcessor {
184181
return confgenerator.LoggingProcessorModifyFields{
185182
Fields: map[string]*confgenerator.ModifyField{
186183
"severity": {
@@ -199,10 +196,10 @@ func (p LoggingProcessorElasticsearchJson) severityParser(ctx context.Context, t
199196
},
200197
InstrumentationSourceLabel: instrumentationSourceValue(p.Type()),
201198
},
202-
}.Components(ctx, tag, uid)
199+
}
203200
}
204201

205-
func (p LoggingProcessorElasticsearchJson) nestingProcessors(ctx context.Context, tag, uid string) []fluentbit.Component {
202+
func (p LoggingProcessorMacroElasticsearchJson) nestingProcessors() []confgenerator.InternalLoggingProcessor {
206203
// The majority of these prefixes come from here:
207204
// https://www.elastic.co/guide/en/elasticsearch/reference/7.16/audit-event-types.html#audit-event-attributes
208205
// Non-audit logs are formatted using the layout documented here, giving the "cluster" prefix:
@@ -223,17 +220,17 @@ func (p LoggingProcessorElasticsearchJson) nestingProcessors(ctx context.Context
223220
"cluster",
224221
}
225222

226-
c := make([]fluentbit.Component, 0, len(prefixes))
223+
processors := make([]confgenerator.InternalLoggingProcessor, 0, len(prefixes))
227224
for _, prefix := range prefixes {
228225
nestProcessor := confgenerator.LoggingProcessorNestWildcard{
229226
Wildcard: fmt.Sprintf("%s.*", prefix),
230227
NestUnder: prefix,
231228
RemovePrefix: fmt.Sprintf("%s.", prefix),
232229
}
233-
c = append(c, nestProcessor.Components(ctx, tag, uid)...)
230+
processors = append(processors, nestProcessor)
234231
}
235232

236-
return c
233+
return processors
237234
}
238235

239236
type LoggingProcessorElasticsearchGC struct {
@@ -271,12 +268,12 @@ func (p LoggingProcessorElasticsearchGC) Components(ctx context.Context, tag, ui
271268
return c
272269
}
273270

274-
type LoggingReceiverElasticsearchJson struct {
275-
LoggingProcessorElasticsearchJson `yaml:",inline"`
276-
ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline"`
271+
type LoggingReceiverMacroElasticsearchJson struct {
272+
LoggingProcessorMacroElasticsearchJson `yaml:",inline"`
273+
ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"`
277274
}
278275

279-
func (r LoggingReceiverElasticsearchJson) Components(ctx context.Context, tag string) []fluentbit.Component {
276+
func (r LoggingReceiverMacroElasticsearchJson) Expand(ctx context.Context) (confgenerator.InternalLoggingReceiver, []confgenerator.InternalLoggingProcessor) {
280277
if len(r.ReceiverMixin.IncludePaths) == 0 {
281278
// Default JSON logs for Elasticsearch
282279
r.ReceiverMixin.IncludePaths = []string{
@@ -296,7 +293,6 @@ func (r LoggingReceiverElasticsearchJson) Components(ctx context.Context, tag st
296293
// -- snip --
297294
// "at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:166) ~[elasticsearch-7.16.2.jar:7.16.2]",
298295
// "... 6 more"] }
299-
300296
r.ReceiverMixin.MultilineRules = []confgenerator.MultilineRule{
301297
{
302298
StateName: "start_state",
@@ -310,8 +306,7 @@ func (r LoggingReceiverElasticsearchJson) Components(ctx context.Context, tag st
310306
},
311307
}
312308

313-
c := r.ReceiverMixin.Components(ctx, tag)
314-
return append(c, r.LoggingProcessorElasticsearchJson.Components(ctx, tag, "elasticsearch_json")...)
309+
return &r.ReceiverMixin, r.LoggingProcessorMacroElasticsearchJson.Expand(ctx)
315310
}
316311

317312
type LoggingReceiverElasticsearchGC struct {
@@ -332,6 +327,8 @@ func (r LoggingReceiverElasticsearchGC) Components(ctx context.Context, tag stri
332327
}
333328

334329
func init() {
335-
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverElasticsearchJson{} })
330+
confgenerator.RegisterLoggingReceiverMacro(func() LoggingReceiverMacroElasticsearchJson {
331+
return LoggingReceiverMacroElasticsearchJson{}
332+
})
336333
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverElasticsearchGC{} })
337334
}

confgenerator/testdata/feature/golden.csv

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,10 +65,6 @@ App,Field,Override,
6565
*apps.LoggingReceiverElasticsearchGC,confgenerator.LoggingReceiverFilesMixin.BufferInMemory,
6666
*apps.LoggingReceiverElasticsearchGC,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath,
6767
*apps.LoggingReceiverElasticsearchGC,confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval,
68-
*apps.LoggingReceiverElasticsearchJson,apps.LoggingProcessorElasticsearchJson.confgenerator.ConfigComponent.Type,
69-
*apps.LoggingReceiverElasticsearchJson,confgenerator.LoggingReceiverFilesMixin.BufferInMemory,
70-
*apps.LoggingReceiverElasticsearchJson,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath,
71-
*apps.LoggingReceiverElasticsearchJson,confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval,
7268
*apps.LoggingReceiverHadoop,apps.LoggingProcessorHadoop.confgenerator.ConfigComponent.Type,
7369
*apps.LoggingReceiverHadoop,confgenerator.LoggingReceiverFilesMixin.BufferInMemory,
7470
*apps.LoggingReceiverHadoop,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath,
@@ -259,3 +255,7 @@ App,Field,Override,
259255
*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroRedis]],confgenerator.ConfigComponent.Type,
260256
*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroVarnish]],ReceiverMacro,
261257
*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroVarnish]],confgenerator.ConfigComponent.Type,
258+
*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroElasticsearchJson],apps.LoggingReceiverMacroElasticsearchJson.confgenerator.LoggingReceiverFilesMixin.BufferInMemory,
259+
*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroElasticsearchJson],apps.LoggingReceiverMacroElasticsearchJson.confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath,
260+
*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroElasticsearchJson],apps.LoggingReceiverMacroElasticsearchJson.confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval,
261+
*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroElasticsearchJson],confgenerator.ConfigComponent.Type,
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
- type: elasticsearch_json
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{"type": "server", "timestamp": "2024-06-10T12:34:56,789Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "node-1", "message": "starting ..."}
2+
{"type": "server", "timestamp": "2024-06-10T12:35:00,321Z", "level": "WARN", "component": "o.e.d.z.ZenDiscovery", "node.name": "node-1", "message": "failed to connect to master", "user.run_by.name": "admin", "event.action": "connect"}
3+
{"type": "server", "timestamp": "2024-06-10T12:35:10,123Z", "level": "ERROR", "component": "o.e.b.Bootstrap", "node.name": "node-1", "message": "Exception", "error.type": "java.lang.IllegalStateException", "error.message": "failed to obtain node locks"}
4+
{"type": "server", "timestamp": "2024-06-10T12:35:15,456Z", "level": "DEBUG", "component": "o.e.n.Node", "node.name": "node-1", "message": "stopping ..."}
5+
{"type": "server", "timestamp": "2024-06-10T12:35:16,789Z", "level": "TRACE", "component": "o.e.n.Node", "node.name": "node-1", "message": "stopped"}
6+
{"type": "server", "timestamp": "2024-06-10T12:35:17,123Z", "level": "DEPRECATION", "component": "o.e.n.Node", "node.name": "node-1", "message": "deprecated setting used"}
7+
{"type": "server", "timestamp": "2024-06-10T12:35:18,456Z", "level": "CRITICAL", "component": "o.e.n.Node", "node.name": "node-1", "message": "critical error occurred"}
8+
{"type": "server", "timestamp": "2024-06-10T12:35:19,456Z", "level": "FATAL", "component": "o.e.n.Node", "node.name": "node-1", "message": "fatal error occurred"}
9+
{"type": "server", "timestamp": "2024-06-10T12:35:20,456Z", "level": "UNKNOWN", "component": "o.e.n.Node", "node.name": "node-1", "message": "unknown log level"}
10+
{"type": "server", "timestamp": "2024-06-10T12:35:21,456Z", "level": "INFO", "message": "missing component and node name"}
11+
{"type": "server", "timestamp": "2024-06-10T12:35:22,456Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "message": "missing node name"}
12+
{"type": "server", "timestamp": "2024-06-10T12:35:23,456Z", "level": "INFO", "component": "o.e.n.Node", "node.name": "node-1", "message": "missing cluster name"}
13+
{"type": "server", "timestamp": "2024-06-10T12:35:24,456Z", "level": "INFO", "component": "o.e.n.Node", "node.name": "node-1", "message": "nested fields", "user.run_by.name": "testuser", "authentication.token.id": "abc123"}
14+
{"type": "server", "timestamp": "2024-06-10T12:35:25,456Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "node.name": "node-1", "message": "uncaught exception in thread [main]", "stacktrace": ["org.elasticsearch.bootstrap.StartupException: java.lang.IllegalArgumentException: unknown setting [invalid.key]", "at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:166)", "... 6 more"]}
15+
{"type": "server", "timestamp": "2024-06-10T12:35:26,456Z", "level": "INFO", "component": "o.e.n.Node", "node.name": "node-1", "message": "invalid timestamp", "timestamp": "not-a-timestamp"}
Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
- entries:
2+
- jsonPayload:
3+
cluster:
4+
name: elasticsearch
5+
component: o.e.n.Node
6+
level: INFO
7+
message: starting ...
8+
node:
9+
name: node-1
10+
type: server
11+
labels:
12+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
13+
logName: projects/my-project/logs/transformation_test
14+
severity: 200.0
15+
timestamp: 2024-06-10T12:34:56.789000000Z
16+
- jsonPayload:
17+
component: o.e.d.z.ZenDiscovery
18+
event:
19+
action: connect
20+
level: WARN
21+
message: failed to connect to master
22+
node:
23+
name: node-1
24+
type: server
25+
user:
26+
run_by:
27+
name: admin
28+
labels:
29+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
30+
logName: projects/my-project/logs/transformation_test
31+
severity: 400.0
32+
timestamp: 2024-06-10T12:35:00.321000000Z
33+
- jsonPayload:
34+
component: o.e.b.Bootstrap
35+
error.message: failed to obtain node locks
36+
error.type: java.lang.IllegalStateException
37+
level: ERROR
38+
message: Exception
39+
node:
40+
name: node-1
41+
type: server
42+
labels:
43+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
44+
logName: projects/my-project/logs/transformation_test
45+
severity: 500.0
46+
timestamp: 2024-06-10T12:35:10.123000000Z
47+
- jsonPayload:
48+
component: o.e.n.Node
49+
level: DEBUG
50+
message: stopping ...
51+
node:
52+
name: node-1
53+
type: server
54+
labels:
55+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
56+
logName: projects/my-project/logs/transformation_test
57+
severity: 100.0
58+
timestamp: 2024-06-10T12:35:15.456000000Z
59+
- jsonPayload:
60+
component: o.e.n.Node
61+
level: TRACE
62+
message: stopped
63+
node:
64+
name: node-1
65+
type: server
66+
labels:
67+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
68+
logName: projects/my-project/logs/transformation_test
69+
severity: 100.0
70+
timestamp: 2024-06-10T12:35:16.789000000Z
71+
- jsonPayload:
72+
component: o.e.n.Node
73+
level: DEPRECATION
74+
message: deprecated setting used
75+
node:
76+
name: node-1
77+
type: server
78+
labels:
79+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
80+
logName: projects/my-project/logs/transformation_test
81+
severity: 400.0
82+
timestamp: 2024-06-10T12:35:17.123000000Z
83+
- jsonPayload:
84+
component: o.e.n.Node
85+
level: CRITICAL
86+
message: critical error occurred
87+
node:
88+
name: node-1
89+
type: server
90+
labels:
91+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
92+
logName: projects/my-project/logs/transformation_test
93+
severity: 500.0
94+
timestamp: 2024-06-10T12:35:18.456000000Z
95+
- jsonPayload:
96+
component: o.e.n.Node
97+
level: FATAL
98+
message: fatal error occurred
99+
node:
100+
name: node-1
101+
type: server
102+
labels:
103+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
104+
logName: projects/my-project/logs/transformation_test
105+
severity: 600.0
106+
timestamp: 2024-06-10T12:35:19.456000000Z
107+
- jsonPayload:
108+
component: o.e.n.Node
109+
level: UNKNOWN
110+
message: unknown log level
111+
node:
112+
name: node-1
113+
type: server
114+
labels:
115+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
116+
logName: projects/my-project/logs/transformation_test
117+
timestamp: 2024-06-10T12:35:20.456000000Z
118+
- jsonPayload:
119+
level: INFO
120+
message: missing component and node name
121+
type: server
122+
labels:
123+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
124+
logName: projects/my-project/logs/transformation_test
125+
severity: 200.0
126+
timestamp: 2024-06-10T12:35:21.456000000Z
127+
- jsonPayload:
128+
cluster:
129+
name: elasticsearch
130+
component: o.e.n.Node
131+
level: INFO
132+
message: missing node name
133+
type: server
134+
labels:
135+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
136+
logName: projects/my-project/logs/transformation_test
137+
severity: 200.0
138+
timestamp: 2024-06-10T12:35:22.456000000Z
139+
- jsonPayload:
140+
component: o.e.n.Node
141+
level: INFO
142+
message: missing cluster name
143+
node:
144+
name: node-1
145+
type: server
146+
labels:
147+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
148+
logName: projects/my-project/logs/transformation_test
149+
severity: 200.0
150+
timestamp: 2024-06-10T12:35:23.456000000Z
151+
- jsonPayload:
152+
authentication:
153+
token:
154+
id: abc123
155+
component: o.e.n.Node
156+
level: INFO
157+
message: nested fields
158+
node:
159+
name: node-1
160+
type: server
161+
user:
162+
run_by:
163+
name: testuser
164+
labels:
165+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
166+
logName: projects/my-project/logs/transformation_test
167+
severity: 200.0
168+
timestamp: 2024-06-10T12:35:24.456000000Z
169+
- jsonPayload:
170+
component: o.e.b.ElasticsearchUncaughtExceptionHandler
171+
level: ERROR
172+
message: uncaught exception in thread [main]
173+
node:
174+
name: node-1
175+
stacktrace:
176+
- "org.elasticsearch.bootstrap.StartupException: java.lang.IllegalArgumentException: unknown setting [invalid.key]"
177+
- at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:166)
178+
- ... 6 more
179+
type: server
180+
labels:
181+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
182+
logName: projects/my-project/logs/transformation_test
183+
severity: 500.0
184+
timestamp: 2024-06-10T12:35:25.456000000Z
185+
- jsonPayload:
186+
component: o.e.n.Node
187+
level: INFO
188+
message: invalid timestamp
189+
node:
190+
name: node-1
191+
timestamp: not-a-timestamp
192+
type: server
193+
labels:
194+
logging.googleapis.com/instrumentation_source: agent.googleapis.com/elasticsearch_json
195+
logName: projects/my-project/logs/transformation_test
196+
severity: 200.0
197+
timestamp: 2024-06-10T12:35:26.456000000Z
198+
partialSuccess: true
199+
resource:
200+
labels: {}
201+
type: gce_instance
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
- config_error: "failed generating OTel processor: &confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroElasticsearchJson]{ConfigComponent:confgenerator.ConfigComponent{Type:\"elasticsearch_json\"}, ProcessorMacro:apps.LoggingProcessorMacroElasticsearchJson{}}, err: unimplemented"

transformation_test/transformation_test.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import (
3333
"time"
3434

3535
logpb "cloud.google.com/go/logging/apiv2/loggingpb"
36-
_ "github.com/GoogleCloudPlatform/ops-agent/apps"
36+
"github.com/GoogleCloudPlatform/ops-agent/apps"
3737
"github.com/GoogleCloudPlatform/ops-agent/confgenerator"
3838
"github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit"
3939
"github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel"
@@ -593,4 +593,6 @@ func sanitizeStacktrace(t *testing.T, input string) string {
593593
func init() {
594594
// The processors registered here are only meant to be used in transformation tests.
595595
confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &confgenerator.LoggingProcessorWindowsEventLogV1{} })
596+
confgenerator.RegisterLoggingProcessorMacro[apps.LoggingProcessorMacroElasticsearchJson]()
597+
596598
}

0 commit comments

Comments
 (0)