diff --git a/packages/crowdstrike/changelog.yml b/packages/crowdstrike/changelog.yml index a390514b5f2..baf73ce1b17 100644 --- a/packages/crowdstrike/changelog.yml +++ b/packages/crowdstrike/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.2.0" + changes: + - description: Improvements and fixes to ingest pipelines for all data streams. + type: enhancement + link: https://github.com/elastic/integrations/pull/16730 - version: "3.1.0" changes: - description: Improve ingest pipeline maintainability. diff --git a/packages/crowdstrike/data_stream/alert/_dev/test/pipeline/test-alert.log-expected.json b/packages/crowdstrike/data_stream/alert/_dev/test/pipeline/test-alert.log-expected.json index 77330c051f4..911cd2426c3 100644 --- a/packages/crowdstrike/data_stream/alert/_dev/test/pipeline/test-alert.log-expected.json +++ b/packages/crowdstrike/data_stream/alert/_dev/test/pipeline/test-alert.log-expected.json @@ -299,6 +299,7 @@ "related": { "hash": [ "b07f77fd3f9828b2c9d61f8a36609741", + "caef4ae19056eeb122a0540508fa8984cea960173ada0dc648cb846d6ef5dd33", "cdf9cfebb400ce89d5b6032bfcdc693b", "b26a6791b72753d2317efd5e1363d93fdd33e611c8b9e08a3b24ea4d755b81fd", "8cc3fcdd7d52d2d5221303c213e044ae", @@ -1035,6 +1036,7 @@ "related": { "hash": [ "b07f77fd3f9828b2c9d61f8a36609741", + "caef4ae19056eeb122a0540508fa8984cea960173ada0dc648cb846d6ef5dd33", "cdf9cfebb400ce89d5b6032bfcdc693b", "b26a6791b72753d2317efd5e1363d93fdd33e611c8b9e08a3b24ea4d755b81fd", "8cc3fcdd7d52d2d5221303c213e044ae", @@ -1374,6 +1376,7 @@ "related": { "hash": [ "b07f77fd3f9828b2c9d61f8a36609741", + "caef4ae19056eeb122a0540508fa8984cea960173ada0dc648cb846d6ef5dd33", "cdf9cfebb400ce89d5b6032bfcdc693b", "b26a6791b72753d2317efd5e1363d93fdd33e611c8b9e08a3b24ea4d755b81fd", "8cc3fcdd7d52d2d5221303c213e044ae", diff --git a/packages/crowdstrike/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/crowdstrike/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index 52b60d2eed9..ca22a3da8f8 100644 --- a/packages/crowdstrike/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/crowdstrike/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -33,6 +33,10 @@ processors: ignore_missing: true description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null + - terminate: + tag: data_collection_error + if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null + description: error message set and no data to process. - json: field: event.original tag: json_event_original @@ -158,11 +162,6 @@ processors: tag: rename_child_process_ids target_field: crowdstrike.alert.child_process_ids ignore_missing: true - - rename: - field: json.aggregate_id - tag: rename_aggregate_id - target_field: crowdstrike.alert.aggregate_id - ignore_missing: true - rename: field: json.cid tag: rename_cid @@ -844,9 +843,9 @@ processors: ignore_missing: true - append: field: related.hash - value: '{{{crowdstrike.alert.parent_details.sha256}}}' + value: '{{{crowdstrike.alert.grandparent_details.sha256}}}' allow_duplicates: false - if: ctx.crowdstrike?.alert?.parent_details?.sha256 != null + if: ctx.crowdstrike?.alert?.grandparent_details?.sha256 != null - date: field: json.grandparent_details.timestamp tag: date_grandparent_details_timestamp @@ -1147,7 +1146,7 @@ processors: ignore_missing: true - append: field: crowdstrike.alert.ioc_values - tag: append_ioc_context_ioc_value_to_ioc_values + tag: append_ioc_value_to_ioc_values value: '{{{crowdstrike.alert.ioc_value}}}' if: ctx.crowdstrike?.alert?.ioc_value != null allow_duplicates: false @@ -2117,19 +2116,19 @@ processors: - script: lang: painless description: Script to set event.severity. - tag: set_event_severity + tag: set_event_severity_from_severity if: ctx.crowdstrike?.alert?.severity instanceof long && ctx.crowdstrike.alert.severity_name == null source: |- long severity = ctx.crowdstrike.alert.severity; if (0 <= severity && severity < 20) { ctx.crowdstrike.alert.severity_name = "info"; - } if (20 <= severity && severity < 40) { + } else if (20 <= severity && severity < 40) { ctx.crowdstrike.alert.severity_name = "low"; - } if (40 <= severity && severity < 60) { + } else if (40 <= severity && severity < 60) { ctx.crowdstrike.alert.severity_name = "medium"; - } if (60 <= severity && severity < 80) { + } else if (60 <= severity && severity < 80) { ctx.crowdstrike.alert.severity_name = "high"; - } if (80 <= severity && severity <= 100) { + } else if (80 <= severity && severity <= 100) { ctx.crowdstrike.alert.severity_name = "critical"; } on_failure: @@ -2139,7 +2138,7 @@ processors: - script: lang: painless description: Script to set event.severity. - tag: set_event_severity + tag: set_event_severity_from_severity_name if: ctx.crowdstrike?.alert?.severity_name instanceof String source: |- ctx.event = ctx.event ?: [:]; @@ -2808,7 +2807,7 @@ processors: drop(ctx); - set: field: event.kind - tag: set_pipeline_error_to_event_kind + tag: set_pipeline_error_to_event_kind_processor value: pipeline_error if: ctx.error?.message != null - append: diff --git a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-event-stream.log-expected.json b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-event-stream.log-expected.json index fef38de39f3..27ae1683dfc 100644 --- a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-event-stream.log-expected.json +++ b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-event-stream.log-expected.json @@ -28,6 +28,18 @@ } }, "destination": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, "ip": "81.2.69.142", "port": 445 }, @@ -228,6 +240,7 @@ "iam" ], "created": "2023-11-02T11:12:03.065Z", + "end": "2023-11-02T11:09:00.000Z", "id": "7a79b2de-4e10-41fb-818f-2bdf53c1625a", "kind": "event", "original": "{\"metadata\":{\"customerIDString\":\"abcabcabc22222\",\"offset\":8694126,\"eventType\":\"IdentityProtectionEvent\",\"eventCreationTime\":1698923523065,\"version\":\"1.0\"},\"event\":{\"IncidentType\":\"Unusuallogintoanendpoint\",\"IncidentDescription\":\"Unusuallogintoanendpoint\",\"Severity\":4,\"SeverityName\":\"LOW\",\"StartTime\":1698923340000,\"EndTime\":1698923340000,\"IdentityProtectionIncidentId\":\"7a79b2de-4e10-41fb-818f-2bdf53c1625a\",\"UserName\":\"NA.NET.ABC.com\\\\abc.service\",\"EndpointName\":\"itreg1d3wp8vh3.eu.net.ABC.com\",\"EndpointIp\":\"\",\"Category\":\"Detections\",\"NumbersOfAlerts\":1,\"NumberOfCompromisedEntities\":2,\"State\":\"NEW\",\"FalconHostLink\":\"https://falcon.crowdstrike.com/identity-protection/detections/abcabcabc22222:ind:abcabcabc22222:7A79B2DE-4E10-41FB-818F-2BDF53C1625A\"}}", @@ -652,6 +665,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "d64d27be4c907b0a2a0c32e1b1a4a44a1333a936fe63127d69df41c859da8c9e" }, + "name": "rundll32.exe", "path": "\\Device\\HarddiskVolume4\\Windows\\System32" }, "host": { diff --git a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-data-protection-detection-summary.log-expected.json b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-data-protection-detection-summary.log-expected.json index 687d44db08c..9135d23df01 100644 --- a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-data-protection-detection-summary.log-expected.json +++ b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-data-protection-detection-summary.log-expected.json @@ -167,7 +167,7 @@ ] }, "file": { - "extension": ".txt", + "extension": "txt", "hash": { "sha256": "21cdfbea0d48077b93b4c4cccb23b7aadfea6f4e7fbb1237fdfd0caebf6efd08" }, diff --git a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-detection-summary.log-expected.json b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-detection-summary.log-expected.json index 4de5211de39..a0b207a7627 100644 --- a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-detection-summary.log-expected.json +++ b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-detection-summary.log-expected.json @@ -96,6 +96,7 @@ "sha1": "7e54a24d4df5b7fe4a75d7ce1c027705c368666b", "sha256": "0b2cde5b355bda69aeb15159fa98b5554053f0936259a53c6dfb0934389238a2" }, + "name": "TESTFILE.exe", "path": "\\Device\\HarddiskVolume3\\Users\\ELASTICUSER\\Software\\TESTSOFTWARE" }, "host": { diff --git a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-epp-detection-summary.log-expected.json b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-epp-detection-summary.log-expected.json index 5e13d40501f..ab8182ab293 100644 --- a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-epp-detection-summary.log-expected.json +++ b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-epp-detection-summary.log-expected.json @@ -85,6 +85,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "faf0c5e4ef7dbcfd863377c55a4d1d87a3f6a58c13a8a9882e11012066f31976" }, + "name": "rfusclient.exe", "path": "\\Device\\HarddiskVolume2\\Program Files (x86)\\Remote Utilities - Host\\rfusclient.exe" }, "host": { @@ -314,6 +315,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "29aa689f38158d2e8941fa54e436f0260890af31cecad1e8799e5c2df7bc1ecc" }, + "name": "git", "path": "/usr/bin/git" }, "host": { @@ -431,6 +433,7 @@ "hash": { "sha256": "774f50830a645392a94338815913e281096f1594ce5f4d992cf3f167fde509a1" }, + "name": "testfile.vmx", "path": "D:\\RECYCLER\\testpath\\testfile.vmx" }, "host": { @@ -623,6 +626,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "29aa689f38158d2e8941fa54e436f0260890af31cecad1e8799e5c2df7bc1ecc" }, + "name": "git", "path": "/usr/bin/git" }, "host": { @@ -777,6 +781,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, + "name": "choice.exe", "path": "\\Device\\HarddiskVolume4\\Windows\\System32\\choice.exe" }, "host": { @@ -941,6 +946,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "0000000000000000000000000000000000000000000000000000000000000000" }, + "name": "atera.exe.exe", "path": "\\Device\\HarddiskVolume4\\Users\\Administrator\\Desktop\\atera.exe.exe" }, "host": { @@ -1095,6 +1101,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "0cb73775cb5eff70ee9189030eb00bfeb03b7245f2e12edc7d59119e86fd7490" }, + "name": "whoami.rtf", "path": "/home/devuser/whoami.rtf" }, "host": { @@ -1249,6 +1256,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "b2088f2a4a4648dd659ad60169369184a98528aeb7c0766273870b96dd4fa682" }, + "name": "cmd.exe", "path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe" }, "host": { @@ -1406,6 +1414,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "27c6ca0e877b6e7a9c7fb2c2e2919df8eaf7db84d82b2652e9a90c31a271db06" }, + "name": "cmd.exe", "path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe" }, "host": { @@ -1563,6 +1572,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "27c6ca0e877b6e7a9c7fb2c2e2919df8eaf7db84d82b2652e9a90c31a271db06" }, + "name": "cmd.exe", "path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe" }, "host": { @@ -1720,6 +1730,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "27c6ca0e877b6e7a9c7fb2c2e2919df8eaf7db84d82b2652e9a90c31a271db06" }, + "name": "cmd.exe", "path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe" }, "host": { @@ -1876,6 +1887,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "27c6ca0e877b6e7a9c7fb2c2e2919df8eaf7db84d82b2652e9a90c31a271db06" }, + "name": "cmd.exe", "path": "\\Device\\HarddiskVolume4\\Windows\\System32\\cmd.exe" }, "host": { @@ -2033,6 +2045,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "a1eb4e497f40111f44a1dca4674ead06dce7ea37d6b8df5a84b5022bf4072930" }, + "name": "choice.exe", "path": "\\Device\\HarddiskVolume4\\Windows\\System32\\choice.exe" }, "host": { @@ -2190,6 +2203,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "c43e9716c26e88801364e80e997f986760f5d82b5649f9e149b164396dd08c3d" }, + "name": "bash", "path": "/usr/bin/bash" }, "host": { @@ -2745,6 +2759,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "a9cc4e666b1a0cb0b6349fbe076e2bd7d2ac5b10f40d6466eb369e9e596978f7" }, + "name": "explorer.exe", "path": "\\Device\\HarddiskVolume4\\Windows\\explorer.exe" }, "host": { diff --git a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-events.log-expected.json b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-events.log-expected.json index 80ff18fd0bd..d47f6c729a2 100644 --- a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-events.log-expected.json +++ b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-events.log-expected.json @@ -55,6 +55,7 @@ "md5": "ac4c51eb24aa95b77f705ab159189e24", "sha256": "6a671b92a69755de6fd063fcbe4ba926d83b49f78c42dbaeed8cdb6bbc57576a" }, + "name": "explorer.exe", "path": "\\Device\\HarddiskVolume1\\Windows" }, "host": { diff --git a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-identity-protection-incident.log-expected.json b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-identity-protection-incident.log-expected.json index 734498e2d43..286f1e441ef 100644 --- a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-identity-protection-incident.log-expected.json +++ b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-identity-protection-incident.log-expected.json @@ -1,7 +1,7 @@ { "expected": [ { - "@timestamp": "2023-06-16T05:03:56.553Z", + "@timestamp": "2023-06-16T05:03:56.383Z", "crowdstrike": { "event": { "Category": "Incidents", @@ -27,12 +27,13 @@ "iam" ], "created": "2023-06-16T05:03:56.581Z", + "end": "2023-06-16T05:03:56.553Z", "id": "INC-123", "kind": "event", "original": "{\n \"metadata\": {\n \"customerIDString\": \"3426316453426234\",\n \"offset\": 2664264,\n \"eventType\": \"IdentityProtectionEvent\",\n \"eventCreationTime\": 1686891836581,\n \"version\": \"1.0\"\n },\n \"event\": {\n \"IncidentType\": \"UNUSUAL_ENDPOINT_USE\",\n \"IncidentDescription\": \"User seen coming from a location that deviates from their baseline.\",\n \"Severity\": 1,\n \"SeverityName\": \"INFO\",\n \"StartTime\": 1686891836383,\n \"EndTime\": 1686891836553,\n \"IdentityProtectionIncidentId\": \"INC-123\",\n \"UserName\": \"TESTUSER\",\n \"EndpointName\": \"TESTHOSTNAME\",\n \"EndpointIp\": \"89.160.20.112\",\n \"Category\": \"Incidents\",\n \"NumbersOfAlerts\": 1,\n \"NumberOfCompromisedEntities\": 1,\n \"State\": \"NEW\",\n \"FalconHostLink\": \"https://falcon.crowdstrike.com/identity-protection/incidents/INC-123\"\n }\n}", "reference": "https://falcon.crowdstrike.com/identity-protection/incidents/INC-123", "severity": 21, - "start": "2023-06-16T05:03:56.553Z", + "start": "2023-06-16T05:03:56.383Z", "type": [ "info" ] diff --git a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-recon-notification.log-expected.json b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-recon-notification.log-expected.json index 748c57e95b4..5d97d87ebb0 100644 --- a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-recon-notification.log-expected.json +++ b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-recon-notification.log-expected.json @@ -23,7 +23,7 @@ "version": "8.17.0" }, "event": { - "action": "recon-notification-", + "action": "recon-notification-post", "category": [ "threat" ], @@ -40,6 +40,7 @@ "vendor": "Crowdstrike" }, "rule": { + "description": "TestRuleTopic", "id": "123543-0cb1-4806-9ccd-a543123", "name": "TestRuleName", "ruleset": "TestRuleTopic" diff --git a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-sample.log-expected.json b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-sample.log-expected.json index f88705558b5..d4d2cf604b8 100644 --- a/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-sample.log-expected.json +++ b/packages/crowdstrike/data_stream/falcon/_dev/test/pipeline/test-falcon-sample.log-expected.json @@ -494,6 +494,7 @@ "sha1": "0000000000000000000000000000000000000000", "sha256": "0a123b185f9a32fde1df59897089014c92e3d08a0533b54baa72ba2a93d64deb" }, + "name": "filename.exe", "path": "\\Device\\HarddiskVolume2\\ProgramData\\file\\path" }, "host": { diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/cspm_events.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/cspm_events.yml index 5f047ac7679..5690ade0725 100644 --- a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/cspm_events.yml +++ b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/cspm_events.yml @@ -97,7 +97,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_event_created_timestamp + tag: date_timestamp_ms if: "ctx.crowdstrike?.event?.Timestamp != null && String.valueOf(ctx.crowdstrike.event.Timestamp).length() >= 12" - date: field: crowdstrike.event.Timestamp @@ -105,7 +105,7 @@ processors: timezone: UTC formats: - UNIX - tag: date_event_created_timestamp + tag: date_timestamp if: 'ctx.crowdstrike?.event?.Timestamp != null && String.valueOf(ctx.crowdstrike.event.Timestamp).length() <= 11' - date: field: crowdstrike.event.EventCreatedTimestamp @@ -113,7 +113,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_event_created_timestamp + tag: date_event_created_timestamp_ms if: "ctx.crowdstrike?.event?.EventCreatedTimestamp != null && String.valueOf(ctx.crowdstrike.event.EventCreatedTimestamp).length() >= 12" - date: field: crowdstrike.event.EventCreatedTimestamp @@ -134,7 +134,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_resource_create_time + tag: date_resource_create_time_ms if: "ctx.crowdstrike?.event?.ResourceCreateTime != null && ctx.crowdstrike.event.ResourceCreateTime != 0 && String.valueOf(ctx.crowdstrike.event.ResourceCreateTime).length() >= 12" - date: field: crowdstrike.event.ResourceCreateTime diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/data_protection_detection_summary.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/data_protection_detection_summary.yml index 5ba34df5405..b7b4f130b2c 100644 --- a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/data_protection_detection_summary.yml +++ b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/data_protection_detection_summary.yml @@ -202,7 +202,7 @@ processors: def idx = ctx.crowdstrike.event.Filename.lastIndexOf('.'); if (idx != -1) { ctx.file = ctx.file ?: [:]; - ctx.file.extension = ctx.crowdstrike.event.Filename.substring(idx).toLowerCase(); + ctx.file.extension = ctx.crowdstrike.event.Filename.substring(idx + 1).toLowerCase(); } on_failure: - append: @@ -235,7 +235,7 @@ processors: ignore_empty_value: true - foreach: field: crowdstrike.event.MitreAttack - tag: foreach_event_MitreAttack + tag: foreach_event_MitreAttack_tactic_name if: ctx.crowdstrike?.event?.MitreAttack instanceof List processor: append: @@ -245,7 +245,7 @@ processors: allow_duplicates: false - foreach: field: crowdstrike.event.MitreAttack - tag: foreach_event_MitreAttack + tag: foreach_event_MitreAttack_tactic_id if: ctx.crowdstrike?.event?.MitreAttack instanceof List processor: append: @@ -255,7 +255,7 @@ processors: allow_duplicates: false - foreach: field: crowdstrike.event.MitreAttack - tag: foreach_event_MitreAttack + tag: foreach_event_MitreAttack_technique_name if: ctx.crowdstrike?.event?.MitreAttack instanceof List processor: append: @@ -265,7 +265,7 @@ processors: allow_duplicates: false - foreach: field: crowdstrike.event.MitreAttack - tag: foreach_event_MitreAttack + tag: foreach_event_MitreAttack_technique_id if: ctx.crowdstrike?.event?.MitreAttack instanceof List processor: append: diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/default.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/default.yml index 089874ffa1a..b09f800bf4e 100644 --- a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/default.yml +++ b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/default.yml @@ -126,7 +126,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_utc_timestamp + tag: date_utc_timestamp_ms if: 'ctx.crowdstrike?.event?.UTCTimestamp != null && String.valueOf(ctx.crowdstrike.event.UTCTimestamp).length() >= 12' - date: field: crowdstrike.event.UTCTimestamp @@ -149,7 +149,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_event_creation_time + tag: date_event_creation_time_ms if: 'ctx.crowdstrike?.metadata?.eventCreationTime != null && String.valueOf(ctx.crowdstrike.metadata.eventCreationTime).length() >= 12' - set: field: '@timestamp' @@ -390,7 +390,7 @@ processors: ignore_missing: true - geoip: field: destination.ip - target_field: source.geo + target_field: destination.geo ignore_missing: true - geoip: database_file: GeoLite2-ASN.mmdb diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/detection_summary.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/detection_summary.yml index 623f28229c8..33d318bd4cb 100644 --- a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/detection_summary.yml +++ b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/detection_summary.yml @@ -23,7 +23,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_process_start_time + tag: date_process_start_time_ms if: 'ctx.crowdstrike?.event?.ProcessStartTime != null && String.valueOf(ctx.crowdstrike.event.ProcessStartTime).length() >= 12' - date: field: crowdstrike.event.ProcessStartTime @@ -39,7 +39,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_process_end_time + tag: date_process_end_time_ms if: 'ctx.crowdstrike?.event?.ProcessEndTime != null && String.valueOf(ctx.crowdstrike.event.ProcessEndTime).length() >= 12' - date: field: crowdstrike.event.ProcessEndTime @@ -96,10 +96,10 @@ processors: copy_from: message tag: set_rule_description if: ctx.message != null - - rename: - field: crowdstrike.event.FileName - target_field: process.name - ignore_missing: true + - set: + field: process.name + copy_from: crowdstrike.event.FileName + ignore_empty_value: true - rename: field: crowdstrike.event.MachineDomain target_field: host.domain @@ -168,7 +168,7 @@ processors: target_field: host.mac ignore_missing: true tag: rename_mac_address - if: ctx.cropwdstrike?.event?.MacAddress != null + if: ctx.crowdstrike?.event?.MacAddress != null - uppercase: field: host.mac ignore_missing: true diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/firewall_match.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/firewall_match.yml index d5d4d33b778..bb78003e2ba 100644 --- a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/firewall_match.yml +++ b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/firewall_match.yml @@ -26,7 +26,7 @@ processors: field: _tmp_.action value: "Allowed" tag: set_allowed_action - if: ctx.crowdstrike?.event?.RuleAction != null && ctx.crowdstrike?.event?.RuleAction == "2" + if: ctx.crowdstrike?.event?.RuleAction != null && ctx.crowdstrike?.event?.RuleAction == "1" - append: field: event.type value: denied @@ -121,7 +121,6 @@ processors: } if (result.size() > 0) { ctx.network = ctx.network ?: [:]; - ctx.network.direction = ctx.network.direction ?: [:]; } if (result.size() == 1) { ctx.network.direction = result[0]; @@ -132,53 +131,53 @@ processors: field: crowdstrike.event.RemoteAddress target_field: source.ip ignore_missing: true - tag: rename_remote_address + tag: rename_remote_address_to_source_ip if: ctx.crowdstrike?.event?.RemoteAddress != null && ctx.network?.direction == "ingress" - rename: field: crowdstrike.event.LocalAddress target_field: destination.ip ignore_missing: true - tag: rename_local_address + tag: rename_local_address_to_destination_ip if: ctx.crowdstrike?.event?.LocalAddress != null && ctx.network?.direction == "ingress" - convert: field: crowdstrike.event.LocalPort target_field: destination.port type: long ignore_missing: true - tag: convert_local_port + tag: convert_local_port_to_destination_port if: ctx.crowdstrike?.event?.LocalPort != null && ctx.network?.direction == "ingress" - convert: field: crowdstrike.event.RemotePort target_field: source.port type: long ignore_missing: true - tag: convert_remote_port + tag: convert_remote_port_to_source_port if: ctx.crowdstrike?.event?.RemotePort != null && ctx.network?.direction == "ingress" - rename: field: crowdstrike.event.RemoteAddress target_field: destination.ip ignore_missing: true - tag: rename_remote_address + tag: rename_remote_address_to_destination_ip if: ctx.crowdstrike?.event?.RemoteAddress != null && ctx.network?.direction == "egress" - rename: field: crowdstrike.event.LocalAddress target_field: source.ip ignore_missing: true - tag: rename_local_address + tag: rename_local_address_to_source_ip if: ctx.crowdstrike?.event?.LocalAddress != null && ctx.network?.direction == "egress" - convert: field: crowdstrike.event.LocalPort target_field: source.port type: long ignore_missing: true - tag: convert_local_port + tag: convert_local_port_to_source_port if: ctx.crowdstrike?.event?.LocalPort != null && ctx.network?.direction == "egress" - convert: field: crowdstrike.event.RemotePort target_field: destination.port type: long ignore_missing: true - tag: convert_remote_port + tag: convert_remote_port_to_destination_port if: ctx.crowdstrike?.event?.RemotePort != null && ctx.network?.direction == "egress" - rename: field: crowdstrike.event.Platform diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/identity_protection_incident.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/identity_protection_incident.yml index e3832f28053..5097208c2bc 100644 --- a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/identity_protection_incident.yml +++ b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/identity_protection_incident.yml @@ -46,7 +46,7 @@ processors: field: crowdstrike.event.EndpointName target_field: host.hostname ignore_missing: true - tag: rename_user_name + tag: rename_endpoint_name - append: field: host.ip value: '{{{crowdstrike.event.EndpointIp}}}' @@ -74,7 +74,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_event_start_time_epoch + tag: date_event_start_time_epoch_ms if: "ctx.crowdstrike?.event?.StartTime != null && ctx.crowdstrike.event.StartTime.length() >= 12" - date: field: crowdstrike.event.StartTime @@ -87,30 +87,30 @@ processors: - convert: field: crowdstrike.event.EndTime type: string - tag: convert_start_time_epoch + tag: convert_end_time_epoch ignore_missing: true if: ctx.crowdstrike?.event?.EndTime != null - gsub: field: crowdstrike.event.EndTime pattern: "\\d{6}$" replacement: "" - tag: gsub_start_time_epoch + tag: gsub_end_time_epoch if: "ctx.crowdstrike?.event?.EndTime != null && ctx.crowdstrike.event.EndTime.length() > 18" - date: field: crowdstrike.event.EndTime - target_field: event.start + target_field: event.end timezone: UTC formats: - UNIX_MS - tag: date_event_start_time_epoch + tag: date_event_end_time_epoch_ms if: "ctx.crowdstrike?.event?.EndTime != null && ctx.crowdstrike.event.EndTime.length() >= 12" - date: field: crowdstrike.event.EndTime - target_field: event.start + target_field: event.end timezone: UTC formats: - UNIX - tag: date_event_start_time_epoch + tag: date_event_end_time_epoch if: 'ctx.crowdstrike?.event?.EndTime != null && ctx.crowdstrike.event.EndTime.length() <= 11' - set: field: '@timestamp' diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/incident_summary.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/incident_summary.yml index 50f0e4188b0..bb97e11d14c 100644 --- a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/incident_summary.yml +++ b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/incident_summary.yml @@ -33,7 +33,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_incident_start_time + tag: date_incident_start_time_ms if: 'ctx.crowdstrike?.event?.IncidentStartTime != null && String.valueOf(ctx.crowdstrike.event.IncidentStartTime).length() >= 12' - date: field: crowdstrike.event.IncidentStartTime @@ -49,7 +49,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_incident_end_time + tag: date_incident_end_time_ms if: 'ctx.crowdstrike?.event?.IncidentEndTime != null && String.valueOf(ctx.crowdstrike.event.IncidentEndTime).length() >= 12' - date: field: crowdstrike.event.IncidentEndTime diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/ipd_detection_summary.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/ipd_detection_summary.yml index 9d2c40fccab..770a50f12ff 100644 --- a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/ipd_detection_summary.yml +++ b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/ipd_detection_summary.yml @@ -130,7 +130,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_context_timestamp + tag: date_context_timestamp_ms if: "ctx.crowdstrike?.event?.ContextTimeStamp != null && ctx.crowdstrike.event.ContextTimeStamp.length() >= 12" - date: field: crowdstrike.event.ContextTimeStamp @@ -158,7 +158,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_account_creation_timestamp + tag: date_account_creation_timestamp_ms if: "ctx.crowdstrike?.event?.AccountCreationTimeStamp != null && ctx.crowdstrike.event.AccountCreationTimeStamp.length() >= 12" - date: field: crowdstrike.event.AccountCreationTimeStamp @@ -186,7 +186,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_event_start_time_epoch + tag: date_event_start_time_epoch_ms if: "ctx.crowdstrike?.event?.StartTime != null && ctx.crowdstrike.event.StartTime.length() >= 12" - date: field: crowdstrike.event.StartTime @@ -199,14 +199,14 @@ processors: - convert: field: crowdstrike.event.EndTime type: string - tag: convert_start_time_epoch + tag: convert_end_time_epoch ignore_missing: true if: ctx.crowdstrike?.event?.EndTime != null - gsub: field: crowdstrike.event.EndTime pattern: "\\d{6}$" replacement: "" - tag: gsub_start_time_epoch + tag: gsub_end_time_epoch if: "ctx.crowdstrike?.event?.EndTime != null && ctx.crowdstrike.event.EndTime.length() > 18" - date: field: crowdstrike.event.EndTime @@ -214,7 +214,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_event_end_time_epoch + tag: date_event_end_time_epoch_ms if: "ctx.crowdstrike?.event?.EndTime != null && ctx.crowdstrike.event.EndTime.length() >= 12" - date: field: crowdstrike.event.EndTime @@ -252,7 +252,7 @@ processors: field: related.hosts value: "{{{crowdstrike.event.AdditionalAccountName}}}" allow_duplicates: false - tag: append_additional_account_domain + tag: append_additional_account_name if: ctx.crowdstrike?.event?.AdditionalAccountName != null - append: field: related.hosts @@ -264,7 +264,7 @@ processors: field: related.ip value: "{{{crowdstrike.event.AdditionalEndpointIpAddress}}}" allow_duplicates: false - tag: append_additional_endpoint_hostname + tag: append_additional_endpoint_ip if: ctx.crowdstrike?.event?.AdditionalEndpointIpAddress != null on_failure: diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/mobile_detection_summary.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/mobile_detection_summary.yml index f4b93464489..e53df8ca7d2 100644 --- a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/mobile_detection_summary.yml +++ b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/mobile_detection_summary.yml @@ -34,7 +34,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_event_creation_time + tag: date_event_creation_time_ms if: 'ctx.crowdstrike?.event?.ContextTimeStamp != null && String.valueOf(ctx.crowdstrike.event.ContextTimeStamp).length() >= 12' - rename: field: crowdstrike.event.MobileDetectionId diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/recon_notification_summary.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/recon_notification_summary.yml index d178fb034e9..2d949e5de32 100644 --- a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/recon_notification_summary.yml +++ b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/recon_notification_summary.yml @@ -17,7 +17,7 @@ processors: if: ctx.crowdstrike?.event?.ItemType == null - set: field: event.action - value: "recon-notification-{{{ctx.crowdstrike.event.ItemType}}}" + value: "recon-notification-{{{crowdstrike.event.ItemType}}}" if: ctx.crowdstrike?.event?.ItemType != null - rename: field: crowdstrike.event.ItemId @@ -34,11 +34,11 @@ processors: target_field: rule.name ignore_missing: true tag: rename_rule_name - - rename: - field: crowdstrike.event.RuleTopic - target_field: rule.ruleset - ignore_missing: true - tag: rename_rule_topic + - set: + field: rule.ruleset + copy_from: crowdstrike.event.RuleTopic + ignore_empty_value: true + tag: set_rule_ruleset - rename: field: crowdstrike.event.RuleTopic target_field: rule.description @@ -50,7 +50,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_event_matched_timestamp + tag: date_event_matched_timestamp_ms if: "ctx.crowdstrike?.event?.MatchedTimestamp != null && String.valueOf(ctx.crowdstrike.event.MatchedTimestamp).length() >= 12" - date: field: crowdstrike.event.MatchedTimestamp @@ -66,7 +66,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_event_matched_timestamp + tag: date_item_posted_timestamp_ms if: "ctx.crowdstrike?.event?.ItemPostedTimestamp != null && String.valueOf(ctx.crowdstrike.event.ItemPostedTimestamp).length() >= 12" - date: field: crowdstrike.event.ItemPostedTimestamp @@ -74,7 +74,7 @@ processors: timezone: UTC formats: - UNIX - tag: date_event_matched_timestamp + tag: date_item_posted_timestamp if: 'ctx.crowdstrike?.event?.ItemPostedTimestamp != null && String.valueOf(ctx.crowdstrike.event.ItemPostedTimestamp).length() <= 11' diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/remote_response_session_end.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/remote_response_session_end.yml index 5faf311ac2f..ae1bfc61525 100644 --- a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/remote_response_session_end.yml +++ b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/remote_response_session_end.yml @@ -35,7 +35,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_end_timestamp + tag: date_end_timestamp_ms if: 'ctx.crowdstrike?.event?.EndTimestamp != null && String.valueOf(ctx.crowdstrike.event.EndTimestamp).length() >= 12' - date: field: crowdstrike.event.EndTimestamp diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/remote_response_session_start.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/remote_response_session_start.yml index c6d9675c236..23c1da8011f 100644 --- a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/remote_response_session_start.yml +++ b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/remote_response_session_start.yml @@ -35,7 +35,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_start_timestamp + tag: date_start_timestamp_ms if: 'ctx.crowdstrike?.event?.StartTimestamp != null && String.valueOf(ctx.crowdstrike.event.StartTimestamp).length() >= 12' - date: field: crowdstrike.event.StartTimestamp diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/scheduled_report_notification_event.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/scheduled_report_notification_event.yml index 09c034b506e..aa1c8862436 100644 --- a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/scheduled_report_notification_event.yml +++ b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/scheduled_report_notification_event.yml @@ -9,21 +9,21 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_utc_timestamp + tag: date_execution_start_ms if: 'ctx.crowdstrike?.event?.ExecutionMetadata?.ExecutionStart != null && String.valueOf(ctx.crowdstrike.event.ExecutionMetadata.ExecutionStart).length() >= 12' - date: field: crowdstrike.event.ExecutionMetadata.SearchWindowStart timezone: UTC formats: - UNIX_MS - tag: date_utc_timestamp + tag: date_search_window_start_ms if: 'ctx.crowdstrike?.event?.ExecutionMetadata?.SearchWindowStart != null && String.valueOf(ctx.crowdstrike.event.ExecutionMetadata.SearchWindowStart).length() >= 12' - date: field: crowdstrike.event.ExecutionMetadata.SearchWindowEnd timezone: UTC formats: - UNIX_MS - tag: date_utc_timestamp + tag: date_search_window_end_ms if: 'ctx.crowdstrike?.event?.ExecutionMetadata?.SearchWindowEnd != null && String.valueOf(ctx.crowdstrike.event.ExecutionMetadata.SearchWindowEnd).length() >= 12' - convert: field: crowdstrike.event.ExecutionMetadata.ExecutionDuration diff --git a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/xdr_detection_summary.yml b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/xdr_detection_summary.yml index ef17b721f15..f6c690f45e7 100644 --- a/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/xdr_detection_summary.yml +++ b/packages/crowdstrike/data_stream/falcon/elasticsearch/ingest_pipeline/xdr_detection_summary.yml @@ -102,37 +102,37 @@ processors: field: crowdstrike.event.SHA256Hashes separator: "," target_field: related.hash - tag: split_domain_names + tag: split_sha256_hashes if: ctx.crowdstrike?.event?.SHA256Hashes != null && ctx.crowdstrike?.event?.SHA256Hashes.contains(",") - append: field: related.hash value: "{{{crowdstrike.event.SHA256Hashes}}}" allow_duplicates: false - tag: append_domain_names + tag: append_sha256_hashes if: ctx.crowdstrike?.event?.SHA256Hashes != null && !ctx.crowdstrike?.event?.SHA256Hashes.contains(",") - split: field: crowdstrike.event.MD5Hashes separator: "," target_field: related.hash - tag: split_domain_names + tag: split_md5_hashes if: ctx.crowdstrike?.event?.MD5Hashes != null && ctx.crowdstrike?.event?.MD5Hashes.contains(",") - append: field: related.hash value: "{{{crowdstrike.event.MD5Hashes}}}" allow_duplicates: false - tag: append_domain_names + tag: append_md5_hashes if: ctx.crowdstrike?.event?.MD5Hashes != null && !ctx.crowdstrike?.event?.MD5Hashes.contains(",") - split: field: crowdstrike.event.Users separator: "," target_field: related.user - tag: split_domain_names + tag: split_users if: ctx.crowdstrike?.event?.Users != null && ctx.crowdstrike?.event?.Users.contains(",") - append: field: related.user value: "{{{crowdstrike.event.Users}}}" allow_duplicates: false - tag: append_domain_names + tag: append_users if: ctx.crowdstrike?.event?.Users != null && !ctx.crowdstrike?.event?.Users.contains(",") - set: field: rule.description @@ -157,7 +157,7 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_event_start_time_epoch + tag: date_event_start_time_epoch_ms if: "ctx.crowdstrike?.event?.StartTimeEpoch != null && String.valueOf(ctx.crowdstrike.event.StartTimeEpoch).length() >= 12" - date: field: crowdstrike.event.StartTimeEpoch @@ -190,11 +190,11 @@ processors: timezone: UTC formats: - UNIX_MS - tag: date_event_end_time_epoch + tag: date_event_end_time_epoch_ms if: "ctx.crowdstrike?.event?.EndTimeEpoch != null && String.valueOf(ctx.crowdstrike.event.EndTimeEpoch).length() >= 12" - date: field: crowdstrike.event.EndTimeEpoch - target_field: process.end + target_field: event.end timezone: UTC formats: - UNIX diff --git a/packages/crowdstrike/data_stream/fdr/_dev/test/pipeline/test-fdr-data-protection-detection-summary.log-expected.json b/packages/crowdstrike/data_stream/fdr/_dev/test/pipeline/test-fdr-data-protection-detection-summary.log-expected.json index c46fff2bbdf..cdb340a5bd2 100644 --- a/packages/crowdstrike/data_stream/fdr/_dev/test/pipeline/test-fdr-data-protection-detection-summary.log-expected.json +++ b/packages/crowdstrike/data_stream/fdr/_dev/test/pipeline/test-fdr-data-protection-detection-summary.log-expected.json @@ -72,7 +72,7 @@ ] }, "file": { - "extension": ".txt", + "extension": "txt", "hash": { "sha256": "21cdfbea0d48077b93b4c4cccb23b7aadfea6f4e7fbb1237fdfd0caebf6efd08" }, diff --git a/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/cspm_ioa.yml b/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/cspm_ioa.yml index de57fe193fe..4bf46f00efc 100644 --- a/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/cspm_ioa.yml +++ b/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/cspm_ioa.yml @@ -1,5 +1,5 @@ --- -description: Pipeline for processing Could Security Posture's IOA events. +description: Pipeline for processing Cloud Security Posture's IOA events. processors: # Event categorization - set: @@ -252,9 +252,9 @@ processors: field: event.kind value: pipeline_error - append: - tag: append_preserve_original_event_into_event_kind_a0b8d607 + tag: append_preserve_original_event_into_tags_9fe66b2c if: ctx.error?.message != null - field: event.kind + field: tags value: preserve_original_event allow_duplicates: false on_failure: diff --git a/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/cspm_iom.yml b/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/cspm_iom.yml index a977b4ef9db..a5d97ab1f0c 100644 --- a/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/cspm_iom.yml +++ b/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/cspm_iom.yml @@ -1,5 +1,5 @@ --- -description: Pipeline for processing Could Security Posture's IOM and CloudSecurityIOMEvaluation events. +description: Pipeline for processing Cloud Security Posture's IOM and CloudSecurityIOMEvaluation events. processors: # Event categorization - set: @@ -482,9 +482,9 @@ processors: field: event.kind value: pipeline_error - append: - tag: append_preserve_original_event_into_event_kind_a0b8d607 + tag: append_preserve_original_event_into_tags_9fe66b2c if: ctx.error?.message != null - field: event.kind + field: tags value: preserve_original_event allow_duplicates: false on_failure: diff --git a/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/data_protection_detection_summary.yml b/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/data_protection_detection_summary.yml index 8bc7e87d9fb..f52301bec7b 100644 --- a/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/data_protection_detection_summary.yml +++ b/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/data_protection_detection_summary.yml @@ -248,17 +248,17 @@ processors: field: related.hash value: '{{{file.hash.sha256}}}' - script: - tag: extract_file_extension_from_filename_966bee27 + tag: extract_file_extension_from_filename_adc57e41 if: ctx.crowdstrike?.Filename != null source: |- def idx = ctx.crowdstrike.Filename.lastIndexOf('.'); if (idx != -1) { ctx.file = ctx.file ?: [:]; - ctx.file.extension = ctx.crowdstrike.Filename.substring(idx).toLowerCase(); + ctx.file.extension = ctx.crowdstrike.Filename.substring(idx + 1).toLowerCase(); } on_failure: - append: - tag: append_error_message_443e1395 + tag: append_error_message_1a00ddc7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - lowercase: @@ -339,9 +339,9 @@ processors: field: event.kind value: pipeline_error - append: - tag: append_preserve_original_event_into_event_kind_a0b8d607 + tag: append_preserve_original_event_into_tags_9fe66b2c if: ctx.error?.message != null - field: event.kind + field: tags value: preserve_original_event allow_duplicates: false on_failure: @@ -354,7 +354,7 @@ on_failure: field: event.kind value: pipeline_error - append: - tag: append_preserve_original_event_into_event_kind_c274f7a2 - field: event.kind + tag: append_preserve_original_event_into_tags_d762b9c5 + field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/default.yml b/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/default.yml index 4e9d5b2f049..7b899a1c932 100644 --- a/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/default.yml +++ b/packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/default.yml @@ -822,18 +822,18 @@ processors: copy_from: host.hostname ignore_empty_value: true ignore_failure: true + - append: + tag: append_related_hosts_369b21b5 + if: ctx.crowdstrike?.info?.host?.ComputerName != null + field: related.hosts + value: '{{{crowdstrike.info.host.ComputerName}}}' + allow_duplicates: false - rename: tag: rename_crowdstrike_info_host_ComputerName_to_host_name_a1ee7f6f if: ctx.host?.name == null field: crowdstrike.info.host.ComputerName target_field: host.name ignore_missing: true - - append: - tag: append_related_hosts_2fa72197 - if: ctx.host?.name != null - field: related.hosts - value: '{{{crowdstrike.info.host.ComputerName}}}' - allow_duplicates: false - append: tag: append_related_hosts_452ef445 if: ctx.host?.name != null @@ -1088,7 +1088,7 @@ processors: ignore_missing: true - script: description: Calculate process.uptime - tag: process_uptime_528bb619 + tag: process_uptime_d1b24ab9 if: |- ctx.crowdstrike?.ProcessStartTime != null && ctx.crowdstrike?.ProcessStartTime != "" && ctx.crowdstrike?.ProcessEndTime != null && ctx.crowdstrike?.ProcessEndTime != "" @@ -1097,7 +1097,7 @@ processors: float e = Float.parseFloat(ctx.crowdstrike?.ProcessEndTime); if (e >= s) { if (ctx.process == null) { - ctx.process = []; + ctx.process = [:]; } ctx.process.uptime = (long) ((e-s)/1000L); } @@ -1615,8 +1615,8 @@ processors: ignore_empty_value: true ignore_failure: true - append: - tag: append_user_domain_76fba8f0 - if: ctx.user?.domain != null && ctx._temp?.user_parts != null && ctx._temp.user_parts.size() == 2 && ctx.user.domain != ctx._temp.user_parts[0] + tag: append_user_domain_536a37ed + if: ctx._temp?.user_parts != null && ctx._temp.user_parts.size() == 2 field: user.domain value: '{{{_temp.user_parts.1}}}' allow_duplicates: false @@ -2255,12 +2255,6 @@ processors: field: process.executable copy_from: crowdstrike.ContextImageFileName ignore_empty_value: true - - set: - tag: set_process_entity_id_d4090d0f - if: ctx.event?.action != null && ctx.event.action.endsWith('Written') && ctx.host?.os?.type == 'linux' - field: process.entity_id - copy_from: crowdstrike.ContextProcessId - ignore_empty_value: true - set: tag: set_file_hash_sha256_1abd2cde if: ctx.event?.action != null && ctx.event.action.endsWith('Written') && ctx.host?.os?.type == 'linux' @@ -2837,9 +2831,9 @@ processors: field: event.kind value: pipeline_error - append: - tag: append_preserve_original_event_into_event_kind_a0b8d607 + tag: append_preserve_original_event_into_tags_9fe66b2c if: ctx.error?.message != null - field: event.kind + field: tags value: preserve_original_event allow_duplicates: false on_failure: diff --git a/packages/crowdstrike/data_stream/host/elasticsearch/ingest_pipeline/default.yml b/packages/crowdstrike/data_stream/host/elasticsearch/ingest_pipeline/default.yml index 3e7ae345182..9baa24c43cb 100644 --- a/packages/crowdstrike/data_stream/host/elasticsearch/ingest_pipeline/default.yml +++ b/packages/crowdstrike/data_stream/host/elasticsearch/ingest_pipeline/default.yml @@ -43,6 +43,10 @@ processors: ignore_missing: true description: The `message` field is no longer required if the document has an `event.original` field. if: ctx.event?.original != null + - terminate: + tag: data_collection_error + if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null + description: error message set and no data to process. - json: field: event.original tag: json_event_original @@ -846,10 +850,10 @@ processors: } drop(ctx); description: Drops null/empty values recursively. - - append: + - set: field: event.kind + tag: set_pipeline_error_into_event_kind value: pipeline_error - allow_duplicates: false if: ctx.error?.message != null - append: field: tags diff --git a/packages/crowdstrike/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/crowdstrike/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index 2d46b71eb7f..dcbdcb22528 100644 --- a/packages/crowdstrike/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/crowdstrike/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -139,7 +139,7 @@ processors: processor: date: field: _ingest._value.evaluation_logic.created_timestamp - tag: date_apps_evaluation_logic_created_timestamp + tag: date_apps_evaluation_logic_created_timestamp_1 target_field: _ingest._value.evaluation_logic.created_timestamp formats: - ISO8601 @@ -222,7 +222,7 @@ processors: - foreach: field: json.apps if: ctx.json?.apps instanceof List - tag: foreach_apps_evaluation_logic_id + tag: foreach_apps_evaluation_logic_id_simplified processor: foreach: field: _ingest._value.evaluation_logic.simplified_logic @@ -237,7 +237,7 @@ processors: - foreach: field: json.apps if: ctx.json?.apps instanceof List - tag: foreach_apps_evaluation_logic_negate + tag: foreach_apps_evaluation_logic_negate_simplified processor: foreach: field: _ingest._value.evaluation_logic.simplified_logic @@ -728,7 +728,7 @@ processors: processor: date: field: _ingest._value.patch_publication_date - tag: date_apps_evaluation_logic_created_timestamp + tag: date_patch_publication_date target_field: _ingest._value.patch_publication_date formats: - ISO8601 diff --git a/packages/crowdstrike/manifest.yml b/packages/crowdstrike/manifest.yml index a505a95e7a4..f90782538eb 100644 --- a/packages/crowdstrike/manifest.yml +++ b/packages/crowdstrike/manifest.yml @@ -1,6 +1,6 @@ name: crowdstrike title: CrowdStrike -version: "3.1.0" +version: "3.2.0" description: Collect logs from Crowdstrike with Elastic Agent. type: integration format_version: "3.4.0"