crowdstrike: improvements and fixes to ingest pipelines for all data streams

packages/crowdstrike/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.2.0"
+  changes:
+    - description: Improvements and fixes to ingest pipelines for all data streams.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/16730
 - version: "3.1.0"
   changes:
     - description: Improve ingest pipeline maintainability.

packages/crowdstrike/data_stream/alert/_dev/test/pipeline/test-alert.log-expected.json

@@ -299,6 +299,7 @@
             "related": {
                 "hash": [
                     "b07f77fd3f9828b2c9d61f8a36609741",
+                    "caef4ae19056eeb122a0540508fa8984cea960173ada0dc648cb846d6ef5dd33",
                     "cdf9cfebb400ce89d5b6032bfcdc693b",
                     "b26a6791b72753d2317efd5e1363d93fdd33e611c8b9e08a3b24ea4d755b81fd",
                     "8cc3fcdd7d52d2d5221303c213e044ae",
@@ -1035,6 +1036,7 @@
             "related": {
                 "hash": [
                     "b07f77fd3f9828b2c9d61f8a36609741",
+                    "caef4ae19056eeb122a0540508fa8984cea960173ada0dc648cb846d6ef5dd33",
                     "cdf9cfebb400ce89d5b6032bfcdc693b",
                     "b26a6791b72753d2317efd5e1363d93fdd33e611c8b9e08a3b24ea4d755b81fd",
                     "8cc3fcdd7d52d2d5221303c213e044ae",
@@ -1374,6 +1376,7 @@
             "related": {
                 "hash": [
                     "b07f77fd3f9828b2c9d61f8a36609741",
+                    "caef4ae19056eeb122a0540508fa8984cea960173ada0dc648cb846d6ef5dd33",
                     "cdf9cfebb400ce89d5b6032bfcdc693b",
                     "b26a6791b72753d2317efd5e1363d93fdd33e611c8b9e08a3b24ea4d755b81fd",
                     "8cc3fcdd7d52d2d5221303c213e044ae",

packages/crowdstrike/data_stream/alert/elasticsearch/ingest_pipeline/default.yml

@@ -33,6 +33,10 @@ processors:
       ignore_missing: true
       description: The `message` field is no longer required if the document has an `event.original` field.
       if: ctx.event?.original != null
+  - terminate:
+      tag: data_collection_error
+      if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null
+      description: error message set and no data to process.
   - json:
       field: event.original
       tag: json_event_original
@@ -158,11 +162,6 @@ processors:
       tag: rename_child_process_ids
       target_field: crowdstrike.alert.child_process_ids
       ignore_missing: true
-  - rename:
-      field: json.aggregate_id
-      tag: rename_aggregate_id
-      target_field: crowdstrike.alert.aggregate_id
-      ignore_missing: true
   - rename:
       field: json.cid
       tag: rename_cid
@@ -844,9 +843,9 @@ processors:
       ignore_missing: true
   - append:
       field: related.hash
-      value: '{{{crowdstrike.alert.parent_details.sha256}}}'
+      value: '{{{crowdstrike.alert.grandparent_details.sha256}}}'
       allow_duplicates: false
-      if: ctx.crowdstrike?.alert?.parent_details?.sha256 != null
+      if: ctx.crowdstrike?.alert?.grandparent_details?.sha256 != null
   - date:
       field: json.grandparent_details.timestamp
       tag: date_grandparent_details_timestamp
@@ -1147,7 +1146,7 @@ processors:
       ignore_missing: true
   - append:
       field: crowdstrike.alert.ioc_values
-      tag: append_ioc_context_ioc_value_to_ioc_values
+      tag: append_ioc_value_to_ioc_values
       value: '{{{crowdstrike.alert.ioc_value}}}'
       if: ctx.crowdstrike?.alert?.ioc_value != null
       allow_duplicates: false
@@ -2117,19 +2116,19 @@ processors:
   - script:
       lang: painless
       description: Script to set event.severity.
-      tag: set_event_severity
+      tag: set_event_severity_from_severity
       if: ctx.crowdstrike?.alert?.severity instanceof long && ctx.crowdstrike.alert.severity_name == null
       source: |-
         long severity = ctx.crowdstrike.alert.severity;
         if (0 <= severity && severity < 20) {
           ctx.crowdstrike.alert.severity_name = "info";
-        } if (20 <= severity && severity < 40) {
+        } else if (20 <= severity && severity < 40) {
           ctx.crowdstrike.alert.severity_name = "low";
-        } if (40 <= severity && severity < 60) {
+        } else if (40 <= severity && severity < 60) {
           ctx.crowdstrike.alert.severity_name = "medium";
-        } if (60 <= severity && severity < 80) {
+        } else if (60 <= severity && severity < 80) {
           ctx.crowdstrike.alert.severity_name = "high";
-        } if (80 <= severity && severity <= 100) {
+        } else if (80 <= severity && severity <= 100) {
           ctx.crowdstrike.alert.severity_name = "critical";
         }
       on_failure:
@@ -2139,7 +2138,7 @@ processors:
   - script:
       lang: painless
       description: Script to set event.severity.
-      tag: set_event_severity
+      tag: set_event_severity_from_severity_name
       if: ctx.crowdstrike?.alert?.severity_name instanceof String
       source: |-
         ctx.event = ctx.event ?: [:];
@@ -2808,7 +2807,7 @@ processors:
         drop(ctx);
   - set:
       field: event.kind
-      tag: set_pipeline_error_to_event_kind
+      tag: set_pipeline_error_to_event_kind_processor
       value: pipeline_error
       if: ctx.error?.message != null
   - append:

packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-event-stream.log-expected.json

@@ -28,6 +28,18 @@
                 }
             },
             "destination": {
+                "geo": {
+                    "city_name": "London",
+                    "continent_name": "Europe",
+                    "country_iso_code": "GB",
+                    "country_name": "United Kingdom",
+                    "location": {
+                        "lat": 51.5142,
+                        "lon": -0.0931
+                    },
+                    "region_iso_code": "GB-ENG",
+                    "region_name": "England"
+                },
                 "ip": "81.2.69.142",
                 "port": 445
             },
@@ -228,6 +240,7 @@
                     "iam"
                 ],
                 "created": "2023-11-02T11:12:03.065Z",
+                "end": "2023-11-02T11:09:00.000Z",
                 "id": "7a79b2de-4e10-41fb-818f-2bdf53c1625a",
                 "kind": "event",
                 "original": "{\"metadata\":{\"customerIDString\":\"abcabcabc22222\",\"offset\":8694126,\"eventType\":\"IdentityProtectionEvent\",\"eventCreationTime\":1698923523065,\"version\":\"1.0\"},\"event\":{\"IncidentType\":\"Unusuallogintoanendpoint\",\"IncidentDescription\":\"Unusuallogintoanendpoint\",\"Severity\":4,\"SeverityName\":\"LOW\",\"StartTime\":1698923340000,\"EndTime\":1698923340000,\"IdentityProtectionIncidentId\":\"7a79b2de-4e10-41fb-818f-2bdf53c1625a\",\"UserName\":\"NA.NET.ABC.com\\\\abc.service\",\"EndpointName\":\"itreg1d3wp8vh3.eu.net.ABC.com\",\"EndpointIp\":\"\",\"Category\":\"Detections\",\"NumbersOfAlerts\":1,\"NumberOfCompromisedEntities\":2,\"State\":\"NEW\",\"FalconHostLink\":\"https://falcon.crowdstrike.com/identity-protection/detections/abcabcabc22222:ind:abcabcabc22222:7A79B2DE-4E10-41FB-818F-2BDF53C1625A\"}}",
@@ -652,6 +665,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "d64d27be4c907b0a2a0c32e1b1a4a44a1333a936fe63127d69df41c859da8c9e"
                 },
+                "name": "rundll32.exe",
                 "path": "\\Device\\HarddiskVolume4\\Windows\\System32"
             },
             "host": {

packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-data-protection-detection-summary.log-expected.json

@@ -167,7 +167,7 @@
                 ]
             },
             "file": {
-                "extension": ".txt",
+                "extension": "txt",
                 "hash": {
                     "sha256": "21cdfbea0d48077b93b4c4cccb23b7aadfea6f4e7fbb1237fdfd0caebf6efd08"
                 },

packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-detection-summary.log-expected.json

@@ -96,6 +96,7 @@
                     "sha1": "7e54a24d4df5b7fe4a75d7ce1c027705c368666b",
                     "sha256": "0b2cde5b355bda69aeb15159fa98b5554053f0936259a53c6dfb0934389238a2"
                 },
+                "name": "TESTFILE.exe",
                 "path": "\\Device\\HarddiskVolume3\\Users\\ELASTICUSER\\Software\\TESTSOFTWARE"
             },
             "host": {

packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-epp-detection-summary.log-expected.json

@@ -85,6 +85,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "faf0c5e4ef7dbcfd863377c55a4d1d87a3f6a58c13a8a9882e11012066f31976"
                 },
+                "name": "rfusclient.exe",
                 "path": "\\Device\\HarddiskVolume2\\Program Files (x86)\\Remote Utilities - Host\\rfusclient.exe"
             },
             "host": {
@@ -314,6 +315,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "29aa689f38158d2e8941fa54e436f0260890af31cecad1e8799e5c2df7bc1ecc"
                 },
+                "name": "git",
                 "path": "/usr/bin/git"
             },
             "host": {
@@ -431,6 +433,7 @@
                 "hash": {
                     "sha256": "774f50830a645392a94338815913e281096f1594ce5f4d992cf3f167fde509a1"
                 },
+                "name": "testfile.vmx",
                 "path": "D:\\RECYCLER\\testpath\\testfile.vmx"
             },
             "host": {
@@ -623,6 +626,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "29aa689f38158d2e8941fa54e436f0260890af31cecad1e8799e5c2df7bc1ecc"
                 },
+                "name": "git",
                 "path": "/usr/bin/git"
             },
             "host": {
@@ -777,6 +781,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "0000000000000000000000000000000000000000000000000000000000000000"
                 },
+                "name": "choice.exe",
                 "path": "\\Device\\HarddiskVolume4\\Windows\\System32\\choice.exe"
             },
             "host": {
@@ -941,6 +946,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "0000000000000000000000000000000000000000000000000000000000000000"
                 },
+                "name": "atera.exe.exe",
                 "path": "\\Device\\HarddiskVolume4\\Users\\Administrator\\Desktop\\atera.exe.exe"
             },
             "host": {
@@ -1095,6 +1101,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "0cb73775cb5eff70ee9189030eb00bfeb03b7245f2e12edc7d59119e86fd7490"
                 },
+                "name": "whoami.rtf",
                 "path": "/home/devuser/whoami.rtf"
             },
             "host": {
@@ -1249,6 +1256,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "b2088f2a4a4648dd659ad60169369184a98528aeb7c0766273870b96dd4fa682"
                 },
+                "name": "cmd.exe",
                 "path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe"
             },
             "host": {
@@ -1406,6 +1414,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "27c6ca0e877b6e7a9c7fb2c2e2919df8eaf7db84d82b2652e9a90c31a271db06"
                 },
+                "name": "cmd.exe",
                 "path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe"
             },
             "host": {
@@ -1563,6 +1572,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "27c6ca0e877b6e7a9c7fb2c2e2919df8eaf7db84d82b2652e9a90c31a271db06"
                 },
+                "name": "cmd.exe",
                 "path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe"
             },
             "host": {
@@ -1720,6 +1730,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "27c6ca0e877b6e7a9c7fb2c2e2919df8eaf7db84d82b2652e9a90c31a271db06"
                 },
+                "name": "cmd.exe",
                 "path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe"
             },
             "host": {
@@ -1876,6 +1887,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "27c6ca0e877b6e7a9c7fb2c2e2919df8eaf7db84d82b2652e9a90c31a271db06"
                 },
+                "name": "cmd.exe",
                 "path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe"
             },
             "host": {
@@ -2033,6 +2045,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "a1eb4e497f40111f44a1dca4674ead06dce7ea37d6b8df5a84b5022bf4072930"
                 },
+                "name": "choice.exe",
                 "path": "\\Device\\HarddiskVolume4\\Windows\\System32\\choice.exe"
             },
             "host": {
@@ -2190,6 +2203,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "c43e9716c26e88801364e80e997f986760f5d82b5649f9e149b164396dd08c3d"
                 },
+                "name": "bash",
                 "path": "/usr/bin/bash"
             },
             "host": {
@@ -2745,6 +2759,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "a9cc4e666b1a0cb0b6349fbe076e2bd7d2ac5b10f40d6466eb369e9e596978f7"
                 },
+                "name": "explorer.exe",
                 "path": "\\Device\\HarddiskVolume4\\Windows\\explorer.exe"
             },
             "host": {

packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-events.log-expected.json

@@ -55,6 +55,7 @@
                     "md5": "ac4c51eb24aa95b77f705ab159189e24",
                     "sha256": "6a671b92a69755de6fd063fcbe4ba926d83b49f78c42dbaeed8cdb6bbc57576a"
                 },
+                "name": "explorer.exe",
                 "path": "\\Device\\HarddiskVolume1\\Windows"
             },
             "host": {

packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-identity-protection-incident.log-expected.json

@@ -1,7 +1,7 @@
 {
     "expected": [
         {
-            "@timestamp": "2023-06-16T05:03:56.553Z",
+            "@timestamp": "2023-06-16T05:03:56.383Z",
             "crowdstrike": {
                 "event": {
                     "Category": "Incidents",
@@ -27,12 +27,13 @@
                     "iam"
                 ],
                 "created": "2023-06-16T05:03:56.581Z",
+                "end": "2023-06-16T05:03:56.553Z",
                 "id": "INC-123",
                 "kind": "event",
                 "original": "{\n    \"metadata\": {\n        \"customerIDString\": \"3426316453426234\",\n        \"offset\": 2664264,\n        \"eventType\": \"IdentityProtectionEvent\",\n        \"eventCreationTime\": 1686891836581,\n        \"version\": \"1.0\"\n    },\n    \"event\": {\n        \"IncidentType\": \"UNUSUAL_ENDPOINT_USE\",\n        \"IncidentDescription\": \"User seen coming from a location that deviates from their baseline.\",\n        \"Severity\": 1,\n        \"SeverityName\": \"INFO\",\n        \"StartTime\": 1686891836383,\n        \"EndTime\": 1686891836553,\n        \"IdentityProtectionIncidentId\": \"INC-123\",\n        \"UserName\": \"TESTUSER\",\n        \"EndpointName\": \"TESTHOSTNAME\",\n        \"EndpointIp\": \"89.160.20.112\",\n        \"Category\": \"Incidents\",\n        \"NumbersOfAlerts\": 1,\n        \"NumberOfCompromisedEntities\": 1,\n        \"State\": \"NEW\",\n        \"FalconHostLink\": \"https://falcon.crowdstrike.com/identity-protection/incidents/INC-123\"\n    }\n}",
                 "reference": "https://falcon.crowdstrike.com/identity-protection/incidents/INC-123",
                 "severity": 21,
-                "start": "2023-06-16T05:03:56.553Z",
+                "start": "2023-06-16T05:03:56.383Z",
                 "type": [
                     "info"
                 ]

packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-recon-notification.log-expected.json

@@ -23,7 +23,7 @@
                 "version": "8.17.0"
             },
             "event": {
-                "action": "recon-notification-",
+                "action": "recon-notification-post",
                 "category": [
                     "threat"
                 ],
@@ -40,6 +40,7 @@
                 "vendor": "Crowdstrike"
             },
             "rule": {
+                "description": "TestRuleTopic",
                 "id": "123543-0cb1-4806-9ccd-a543123",
                 "name": "TestRuleName",
                 "ruleset": "TestRuleTopic"

packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-sample.log-expected.json

@@ -494,6 +494,7 @@
                     "sha1": "0000000000000000000000000000000000000000",
                     "sha256": "0a123b185f9a32fde1df59897089014c92e3d08a0533b54baa72ba2a93d64deb"
                 },
+                "name": "filename.exe",
                 "path": "\\Device\\HarddiskVolume2\\ProgramData\\file\\path"
             },
             "host": {

packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/cspm_events.yml

@@ -97,23 +97,23 @@ processors:
       timezone: UTC
       formats:
         - UNIX_MS
-      tag: date_event_created_timestamp
+      tag: date_timestamp_ms
       if: "ctx.crowdstrike?.event?.Timestamp != null && String.valueOf(ctx.crowdstrike.event.Timestamp).length() >= 12"
   - date:
       field: crowdstrike.event.Timestamp
       target_field: "@timestamp"
       timezone: UTC
       formats:
         - UNIX
-      tag: date_event_created_timestamp
+      tag: date_timestamp
       if: 'ctx.crowdstrike?.event?.Timestamp != null && String.valueOf(ctx.crowdstrike.event.Timestamp).length() <= 11'
   - date:
       field: crowdstrike.event.EventCreatedTimestamp
       target_field: "@timestamp"
       timezone: UTC
       formats:
         - UNIX_MS
-      tag: date_event_created_timestamp
+      tag: date_event_created_timestamp_ms
       if: "ctx.crowdstrike?.event?.EventCreatedTimestamp != null && String.valueOf(ctx.crowdstrike.event.EventCreatedTimestamp).length() >= 12"
   - date:
       field: crowdstrike.event.EventCreatedTimestamp
@@ -134,7 +134,7 @@ processors:
       timezone: UTC
       formats:
         - UNIX_MS
-      tag: date_resource_create_time
+      tag: date_resource_create_time_ms
       if: "ctx.crowdstrike?.event?.ResourceCreateTime != null && ctx.crowdstrike.event.ResourceCreateTime != 0 && String.valueOf(ctx.crowdstrike.event.ResourceCreateTime).length() >= 12"
   - date:
       field: crowdstrike.event.ResourceCreateTime