Skip to content

Commit a773b73

Browse files
kcreddykcreddy
authored
zscaler_zia.firewall: Fix source/destination ip mapping (#11613)
[zscaler_zia] Fix source/destination ip mapping in `firewall` logs. Currently the ECS `source.ip` and `destination.ip` are copied not only from ZScaler client's source and destination ips, but also from the proxy server's source and destination ips. From security detection point of view, the flow is clearly defined if `source.ip` and `destination.ip` are mapped only from client's perspective instead of mixing them with proxy's source and destination ips. Also, the current array representation of `source.ip` and `destination.ip` makes it unable to be used with `geoip` processor. This PR: - Removes the mapping from proxy server's source and destination to ECS `source.ip` and `destination.ip` respectively. - Adds `source.nat.ip` from zscaler's `tsip` field. - Adds `geoip` processor to `source.ip` and `destination.ip`. - Updates `related.ip` to reflect from custom fields of proxy's source and destination ips.
1 parent 90580c4 commit a773b73

File tree

8 files changed

+156
-204
lines changed

8 files changed

+156
-204
lines changed

‎packages/zscaler_zia/changelog.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,12 @@
11
# newer versions go on top
2+
- version: "3.3.0"
3+
changes:
4+
- description: Fix source/destination ip mapping in firewall logs.
5+
type: bugfix
6+
link: https://github.com/elastic/integrations/pull/11613
7+
- description: Add geoip processor to source and destination ip.
8+
type: enhancement
9+
link: https://github.com/elastic/integrations/pull/11613
210
- version: "3.2.4"
311
changes:
412
- description: Improve data processing in the web pipeline.

‎packages/zscaler_zia/data_stream/firewall/_dev/test/pipeline/test-firewall-http-endpoint.log-expected.json‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,7 @@
88
"geo": {
99
"country_iso_code": "USA"
1010
},
11-
"ip": [
12-
"1.128.0.0"
13-
],
11+
"ip": "1.128.0.0",
1412
"port": [
1513
22,
1614
443
@@ -87,10 +85,10 @@
8785
"geo": {
8886
"country_name": "United States"
8987
},
90-
"ip": [
91-
"0.0.0.0",
92-
"1.128.0.0"
93-
],
88+
"ip": "0.0.0.0",
89+
"nat": {
90+
"ip": "89.160.20.128"
91+
},
9492
"port": [
9593
22
9694
]
@@ -238,9 +236,7 @@
238236
"@timestamp": "2022-12-31T02:22:22.000Z",
239237
"destination": {
240238
"bytes": 0,
241-
"ip": [
242-
"0.0.0.0"
243-
],
239+
"ip": "0.0.0.0",
244240
"port": [
245241
120,
246242
456
@@ -279,9 +275,10 @@
279275
},
280276
"source": {
281277
"bytes": 0,
282-
"ip": [
283-
"0.0.0.0"
284-
],
278+
"ip": "0.0.0.0",
279+
"nat": {
280+
"ip": "0.0.0.0"
281+
},
285282
"port": [
286283
123,
287284
0
@@ -348,12 +345,15 @@
348345
"bytes": 10000,
349346
"domain": "www.example.com",
350347
"geo": {
351-
"country_iso_code": "USA"
348+
"continent_name": "Europe",
349+
"country_iso_code": "NO",
350+
"country_name": "Norway",
351+
"location": {
352+
"lat": 62.0,
353+
"lon": 10.0
354+
}
352355
},
353-
"ip": [
354-
"2a02:cf40::",
355-
"67.43.156.0"
356-
],
356+
"ip": "2a02:cf40::",
357357
"port": [
358358
22,
359359
443
@@ -407,8 +407,8 @@
407407
],
408408
"ip": [
409409
"2a02:cf40::",
410-
"67.43.156.0",
411410
"0.0.0.0",
411+
"67.43.156.0",
412412
"1.128.0.0",
413413
"89.160.20.128"
414414
],
@@ -432,10 +432,10 @@
432432
"geo": {
433433
"country_name": "United States"
434434
},
435-
"ip": [
436-
"0.0.0.0",
437-
"1.128.0.0"
438-
],
435+
"ip": "0.0.0.0",
436+
"nat": {
437+
"ip": "89.160.20.128"
438+
},
439439
"port": [
440440
25,
441441
22

‎packages/zscaler_zia/data_stream/firewall/_dev/test/pipeline/test-firewall.log-expected.json‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,7 @@
88
"geo": {
99
"country_iso_code": "USA"
1010
},
11-
"ip": [
12-
"1.128.0.0"
13-
],
11+
"ip": "1.128.0.0",
1412
"port": [
1513
22,
1614
443
@@ -86,9 +84,10 @@
8684
"geo": {
8785
"country_name": "United States"
8886
},
89-
"ip": [
90-
"1.128.0.0"
91-
],
87+
"ip": "1.128.0.0",
88+
"nat": {
89+
"ip": "89.160.20.128"
90+
},
9291
"port": [
9392
22
9493
]
@@ -238,12 +237,15 @@
238237
"bytes": 10000,
239238
"domain": "www.example.com",
240239
"geo": {
241-
"country_iso_code": "USA"
240+
"continent_name": "Europe",
241+
"country_iso_code": "NO",
242+
"country_name": "Norway",
243+
"location": {
244+
"lat": 62.0,
245+
"lon": 10.0
246+
}
242247
},
243-
"ip": [
244-
"2a02:cf40::",
245-
"67.43.156.0"
246-
],
248+
"ip": "2a02:cf40::",
247249
"port": [
248250
22,
249251
443
@@ -297,8 +299,8 @@
297299
],
298300
"ip": [
299301
"2a02:cf40::",
300-
"67.43.156.0",
301302
"0.0.0.0",
303+
"67.43.156.0",
302304
"1.128.0.0",
303305
"89.160.20.128"
304306
],
@@ -322,10 +324,10 @@
322324
"geo": {
323325
"country_name": "United States"
324326
},
325-
"ip": [
326-
"0.0.0.0",
327-
"1.128.0.0"
328-
],
327+
"ip": "0.0.0.0",
328+
"nat": {
329+
"ip": "89.160.20.128"
330+
},
329331
"port": [
330332
25,
331333
22

‎packages/zscaler_zia/data_stream/firewall/_dev/test/pipeline/test-unicode.json-expected.json‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,15 @@
66
"bytes": 10000,
77
"domain": "www.example.com",
88
"geo": {
9-
"country_iso_code": "USA"
9+
"continent_name": "Europe",
10+
"country_iso_code": "NO",
11+
"country_name": "Norway",
12+
"location": {
13+
"lat": 62.0,
14+
"lon": 10.0
15+
}
1016
},
11-
"ip": [
12-
"2a02:cf40::",
13-
"67.43.156.0"
14-
],
17+
"ip": "2a02:cf40::",
1518
"port": [
1619
22,
1720
443
@@ -65,8 +68,8 @@
6568
],
6669
"ip": [
6770
"2a02:cf40::",
68-
"67.43.156.0",
6971
"0.0.0.0",
72+
"67.43.156.0",
7073
"1.128.0.0",
7174
"89.160.20.128"
7275
],
@@ -90,10 +93,10 @@
9093
"geo": {
9194
"country_name": "United States"
9295
},
93-
"ip": [
94-
"0.0.0.0",
95-
"1.128.0.0"
96-
],
96+
"ip": "0.0.0.0",
97+
"nat": {
98+
"ip": "89.160.20.128"
99+
},
97100
"port": [
98101
25,
99102
22

‎packages/zscaler_zia/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 41 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -164,12 +164,22 @@ processors:
164164
- append:
165165
field: error.message
166166
value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
167-
- append:
167+
- set:
168+
field: destination.ip
169+
tag: set_destination_ip_from_zscaler_zia_firewall_client_destination_ip
170+
copy_from: zscaler_zia.firewall.client.destination.ip
171+
ignore_empty_value: true
172+
- geoip:
168173
field: destination.ip
169-
tag: append_zscaler_zia_firewall_client_destination_ip_into_destination_ip
170-
value: '{{{zscaler_zia.firewall.client.destination.ip}}}'
174+
target_field: destination.geo
175+
tag: geoip_destination_ip
176+
ignore_missing: true
177+
- append:
178+
field: related.ip
179+
value: '{{{destination.ip}}}'
180+
tag: append_related_ip_from_destination_ip
181+
if: ctx.destination?.ip != null
171182
allow_duplicates: false
172-
if: ctx.zscaler_zia?.firewall?.client?.destination?.ip != null
173183
- convert:
174184
field: json.cdport
175185
tag: convert_cdport_to_long
@@ -208,12 +218,22 @@ processors:
208218
- append:
209219
field: error.message
210220
value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
211-
- append:
221+
- set:
222+
field: source.ip
223+
tag: set_source_ip_from_zscaler_zia_firewall_client_source_ip
224+
copy_from: zscaler_zia.firewall.client.source.ip
225+
ignore_empty_value: true
226+
- geoip:
212227
field: source.ip
213-
tag: append_zscaler_zia_firewall_client_source_ip_into_source_ip
214-
value: '{{{zscaler_zia.firewall.client.source.ip}}}'
228+
target_field: source.geo
229+
tag: geoip_source_ip
230+
ignore_missing: true
231+
- append:
232+
field: related.ip
233+
value: '{{{source.ip}}}'
234+
tag: append_related_ip_from_source_ip
235+
if: ctx.source?.ip != null
215236
allow_duplicates: false
216-
if: ctx.zscaler_zia?.firewall?.client?.source?.ip != null
217237
- convert:
218238
field: json.csport
219239
tag: convert_csport_to_long
@@ -283,6 +303,7 @@ processors:
283303
field: destination.geo.country_iso_code
284304
tag: set_destination_geo_country_iso_code_from_firewall_destination_country
285305
copy_from: zscaler_zia.firewall.destination.country
306+
if: ctx.destination?.geo?.country_iso_code == null
286307
ignore_empty_value: true
287308
- rename:
288309
field: json.deviceappversion
@@ -763,21 +784,11 @@ processors:
763784
field: error.message
764785
value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
765786
- append:
766-
field: destination.ip
767-
tag: append_zscaler_zia_firewall_server_destination_ip_into_destination_ip
787+
field: related.ip
768788
value: '{{{zscaler_zia.firewall.server.destination.ip}}}'
769-
allow_duplicates: false
789+
tag: append_related_ip_from_zscaler_zia_firewall_server_destination_ip
770790
if: ctx.zscaler_zia?.firewall?.server?.destination?.ip != null
771-
- foreach:
772-
field: destination.ip
773-
tag: foreach_destination_ip_to_append_related_ip_from_destination_ip
774-
if: ctx.destination?.ip instanceof List
775-
processor:
776-
append:
777-
field: related.ip
778-
tag: append_related_ip_from_destination_ip
779-
value: '{{{_ingest._value}}}'
780-
allow_duplicates: false
791+
allow_duplicates: false
781792
- convert:
782793
field: json.sdport
783794
tag: convert_sdport_to_long
@@ -821,21 +832,11 @@ processors:
821832
field: error.message
822833
value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
823834
- append:
824-
field: source.ip
825-
tag: append_zscaler_zia_firewall_server_source_ip_into_source_ip
835+
field: related.ip
826836
value: '{{{zscaler_zia.firewall.server.source.ip}}}'
827-
allow_duplicates: false
837+
tag: append_related_ip_from_zscaler_zia_firewall_server_source_ip
828838
if: ctx.zscaler_zia?.firewall?.server?.source?.ip != null
829-
- foreach:
830-
field: source.ip
831-
tag: foreach_source_ip_to_append_related_ip_from_source_ip
832-
if: ctx.source?.ip instanceof List
833-
processor:
834-
append:
835-
field: related.ip
836-
tag: append_related_ip_from_source_ip
837-
value: '{{{_ingest._value}}}'
838-
allow_duplicates: false
839+
allow_duplicates: false
839840
- convert:
840841
field: json.ssport
841842
tag: convert_ssport_to_long
@@ -887,6 +888,7 @@ processors:
887888
field: source.geo.country_name
888889
tag: set_source_geo_country_name_from_firewall_source_ip_country
889890
copy_from: zscaler_zia.firewall.source_ip_country
891+
if: ctx.source?.geo?.country_name == null
890892
ignore_empty_value: true
891893
- rename:
892894
field: json.stateful
@@ -969,6 +971,11 @@ processors:
969971
- append:
970972
field: error.message
971973
value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
974+
- set:
975+
field: source.nat.ip
976+
tag: set_source_nat_ip_from_zscaler_zia_firewall_tunnel_ip
977+
copy_from: zscaler_zia.firewall.tunnel.ip
978+
ignore_empty_value: true
972979
- append:
973980
field: related.ip
974981
value: '{{{zscaler_zia.firewall.tunnel.ip}}}'

0 commit comments

Comments
 (0)