Skip to content

Commit e1fd1ea

Browse files
efd6efd6
authored
f5_bigip: url decode user agent strings (#11222)
Care is taken not to interpret the + mark for SURT prefix as a URL-encoded space[1]. [1]§6.1.1. http://crawler.archive.org/articles/user_manual/config.html
1 parent 6244b5b commit e1fd1ea

File tree

9 files changed

+129
-1
lines changed

9 files changed

+129
-1
lines changed

‎packages/f5_bigip/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "1.20.0"
3+
changes:
4+
- description: URL decode user agent strings.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/11222
27
- version: "1.19.1"
38
changes:
49
- description: Fix ASM script processor when `event.original` is absent.

‎packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-apm.log‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
{"hostname":"example.com","errdefs_msgno":"01490102:5:","partition_name":"Common","session_id":"acb12bdc","Access_Profile":"/Common/test/abc","Partition":"Common","Session_Id":"ab32cda23","Access_Policy_Result":"Logon_Deny","tenant":"Common","application":"Test application","telemetryEventCategory":"APM","f5telemetry_timestamp":"2020-12-03T22:10:07.783Z"}
22
{"hostname":"example.com","errdefs_msgno":"01490511:5:","partition_name":"/Common","session_id":"ab32cdba231","Access_Profile":"/Common/abc","Partition":"/Common","Session_ID":"ab231bcda","Max_concurrent_Users":"2","telemetryEventCategory":"APM","f5telemetry_timestamp":"2021-10-01T08:00:03.319Z","tenant":"Common"}
33
{"Bytes_In":"12","Bytes_Out":"0","User_Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1","Client_IP":"81.2.69.144","Continent":"Asia","Country":"India","Listener":"test_listener","Reputation":"test_reputation","State":"test","Virtual_IP":"81.2.69.192","telemetryEventCategory":"APM"}
4+
{"Bytes_In":"12","Bytes_Out":"0","User_Agent":"SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1","Client_IP":"81.2.69.144","Continent":"Asia","Country":"India","Listener":"test_listener","Reputation":"test_reputation","State":"test","Virtual_IP":"81.2.69.192","telemetryEventCategory":"APM"}

‎packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-apm.log-expected.json‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,83 @@
209209
},
210210
"version": "12.0"
211211
}
212+
},
213+
{
214+
"client": {
215+
"ip": "81.2.69.144"
216+
},
217+
"ecs": {
218+
"version": "8.11.0"
219+
},
220+
"event": {
221+
"category": [
222+
"network"
223+
],
224+
"kind": "event",
225+
"original": "{\"Bytes_In\":\"12\",\"Bytes_Out\":\"0\",\"User_Agent\":\"SonyEricssonK750i%2FR1CA%20Browser%2FSEMC-Browser%2F4.2%20Profile%2FMIDP-2.0%20Configuration%2FCLDC-1.1\",\"Client_IP\":\"81.2.69.144\",\"Continent\":\"Asia\",\"Country\":\"India\",\"Listener\":\"test_listener\",\"Reputation\":\"test_reputation\",\"State\":\"test\",\"Virtual_IP\":\"81.2.69.192\",\"telemetryEventCategory\":\"APM\"}",
226+
"type": [
227+
"info"
228+
]
229+
},
230+
"f5_bigip": {
231+
"log": {
232+
"bytes": {
233+
"in": 12,
234+
"out": 0
235+
},
236+
"client": {
237+
"ip": "81.2.69.144"
238+
},
239+
"continent": "Asia",
240+
"country": "India",
241+
"listener": "test_listener",
242+
"reputation": "test_reputation",
243+
"state": "test",
244+
"telemetry": {
245+
"event": {
246+
"category": "APM"
247+
}
248+
},
249+
"user": {
250+
"agent": "SonyEricssonK750i/R1CA Browser/SEMC-Browser/4.2 Profile/MIDP-2.0 Configuration/CLDC-1.1"
251+
},
252+
"virtual": {
253+
"ip": "81.2.69.192"
254+
}
255+
}
256+
},
257+
"host": {
258+
"geo": {
259+
"continent_name": "Asia",
260+
"country_name": "India"
261+
}
262+
},
263+
"network": {
264+
"bytes": 12,
265+
"direction": "ingress"
266+
},
267+
"observer": {
268+
"product": "Application Performance Monitoring",
269+
"vendor": "F5"
270+
},
271+
"related": {
272+
"ip": [
273+
"81.2.69.144",
274+
"81.2.69.192"
275+
]
276+
},
277+
"tags": [
278+
"preserve_original_event",
279+
"preserve_duplicate_custom_fields"
280+
],
281+
"user_agent": {
282+
"device": {
283+
"name": "Ericsson K750i"
284+
},
285+
"name": "SEMC-Browser",
286+
"original": "SonyEricssonK750i/R1CA Browser/SEMC-Browser/4.2 Profile/MIDP-2.0 Configuration/CLDC-1.1",
287+
"version": "4.2"
288+
}
212289
}
213290
]
214291
}

‎packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bigip_bot_and_dos.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -643,6 +643,15 @@ processors:
643643
field: f5_bigip.log.http.request
644644
pattern: '%{f5_bigip.log.http.method} %{f5_bigip.log.http.path} %{f5_bigip.log.http.version}\r\nHost: %{f5_bigip.log.http.host}\r\nConnection: %{f5_bigip.log.http.connection}\r\nPragma: %{f5_bigip.log.http.pragma}\r\nCache-Control: %{f5_bigip.log.http.cache_control}\r\nUser-Agent: %{f5_bigip.log.http.user_agent}\r\n%{f5_bigip.log.http.other_headers}\r\n'
645645
ignore_failure: true
646+
- gsub:
647+
field: f5_bigip.log.http.user_agent
648+
pattern: '(\([^)]*)\+(https?://)'
649+
replacement: '$1%2b$2'
650+
ignore_missing: true
651+
- urldecode:
652+
field: f5_bigip.log.http.user_agent
653+
ignore_missing: true
654+
ignore_failure: true
646655
- user_agent:
647656
field: f5_bigip.log.http.user_agent
648657
ignore_missing: true

‎packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bigipapm.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,15 @@ processors:
7575
tag: rename_user_agent
7676
target_field: f5_bigip.log.user.agent
7777
ignore_missing: true
78+
- gsub:
79+
field: f5_bigip.log.user.agent
80+
pattern: '(\([^)]*)\+(https?://)'
81+
replacement: '$1%2b$2'
82+
ignore_missing: true
83+
- urldecode:
84+
field: f5_bigip.log.user.agent
85+
ignore_missing: true
86+
ignore_failure: true
7887
- user_agent:
7988
field: f5_bigip.log.user.agent
8089
tag: user_agent_processor

‎packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bigipasm.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,15 @@ processors:
364364
field: related.hosts
365365
tag: lowercase_related_hosts
366366
ignore_missing: true
367+
- gsub:
368+
field: f5_bigip.log.request.user_agent
369+
pattern: '(\([^)]*)\+(https?://)'
370+
replacement: '$1%2b$2'
371+
ignore_missing: true
372+
- urldecode:
373+
field: f5_bigip.log.request.user_agent
374+
ignore_missing: true
375+
ignore_failure: true
367376
- user_agent:
368377
field: f5_bigip.log.request.user_agent
369378
ignore_missing: true

‎packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bigipavr.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,15 @@ processors:
273273
tag: rename_user_agent
274274
target_field: f5_bigip.log.user.agent
275275
ignore_missing: true
276+
- gsub:
277+
field: f5_bigip.log.user.agent
278+
pattern: '(\([^)]*)\+(https?://)'
279+
replacement: '$1%2b$2'
280+
ignore_missing: true
281+
- urldecode:
282+
field: f5_bigip.log.user.agent
283+
ignore_missing: true
284+
ignore_failure: true
276285
- user_agent:
277286
field: f5_bigip.log.user.agent
278287
tag: user_agent_log_user_agent

‎packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bigipltm.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,15 @@ processors:
137137
tag: rename_http_user_agent
138138
target_field: f5_bigip.log.http.user_agent
139139
ignore_missing: true
140+
- gsub:
141+
field: f5_bigip.log.http.user_agent
142+
pattern: '(\([^)]*)\+(https?://)'
143+
replacement: '$1%2b$2'
144+
ignore_missing: true
145+
- urldecode:
146+
field: f5_bigip.log.http.user_agent
147+
ignore_missing: true
148+
ignore_failure: true
140149
- user_agent:
141150
field: f5_bigip.log.http.user_agent
142151
tag: user_agent_on_http_user_agent

‎packages/f5_bigip/manifest.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
format_version: "3.0.2"
22
name: f5_bigip
33
title: F5 BIG-IP
4-
version: "1.19.1"
4+
version: "1.20.0"
55
description: Collect logs from F5 BIG-IP with Elastic Agent.
66
type: integration
77
categories:

0 commit comments

Comments
 (0)