Add related.entity to cloudtrail integration (#11115)

* Add  to cloudtrail integration

* Add another found key

* Update docs

* Fix tests

* Append entities based on event name

* Change structure of entity pipeline and add tests

* Remove keys

* Replace 2 brackets by 3 entity.yml

* Change implementation to painless

* Move processor to inline inside deafault.yml

* Add description to

* Update packages/aws/changelog.yml

Co-authored-by: Andrew Kroh <[email protected]>

* Add helper function to add multiple entities at the same time

* Use standard double quotes in strings

* Update docs

* Fix bracket spacing

---------

Co-authored-by: Andrew Kroh <[email protected]>

Author: romulets

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.25.0-preview01"
+  changes:
+    - description: Add related.entity field.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11115
 - version: "2.25.0"
   changes:
     - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json

@@ -51,6 +51,12 @@
                 "name": "admin"
             },
             "related": {
+                "entity": [
+                    "EX_PRINCIPAL_ID",
+                    "arn:aws:iam::123456789012:user/Alice",
+                    "Bob",
+                    "Alice"
+                ],
                 "user": [
                     "Alice",
                     "Bob"

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json

@@ -93,6 +93,18 @@
                     "info"
                 ]
             },
+            "related": {
+                "entity": [
+                    "AROAIN5ATK5U7KEXAMPLE:JohnRole1",
+                    "arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1",
+                    "Role2WithTags",
+                    "JohnDoe",
+                    "arn:aws:iam::111122223333:role/JohnRole2",
+                    "arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags",
+                    "arn:aws:iam::111111111111:role/JohnRole1",
+                    "arn:aws:iam::111111111111:role/JohnRole2"
+                ]
+            },
             "source": {
                 "address": "81.2.69.144",
                 "geo": {
@@ -228,6 +240,17 @@
                     "info"
                 ]
             },
+            "related": {
+                "entity": [
+                    "AROAIN5ATK5U7KEXAMPLE:JohnRole1",
+                    "arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1",
+                    "Role2WithTags",
+                    "JohnDoe",
+                    "arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags",
+                    "arn:aws:iam::111111111111:role/JohnRole1",
+                    "arn:aws:iam::111111111111:role/JohnRole2"
+                ]
+            },
             "source": {
                 "address": "81.2.69.144",
                 "geo": {

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json

@@ -44,6 +44,11 @@
                 ]
             },
             "related": {
+                "entity": [
+                    "0123456789012",
+                    "Alice",
+                    "arn:aws:iam::0123456789012:user/Alice"
+                ],
                 "user": [
                     "Alice"
                 ]
@@ -110,6 +115,11 @@
                 ]
             },
             "related": {
+                "entity": [
+                    "0123456789012",
+                    "Alice",
+                    "arn:aws:iam::0123456789012:user/Alice"
+                ],
                 "user": [
                     "Alice"
                 ]

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json

@@ -140,6 +140,7 @@
                 "path": "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz"
             },
             "related": {
+                "entity": [],
                 "hash": [
                     "10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987"
                 ]

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json

@@ -55,6 +55,11 @@
                 ]
             },
             "related": {
+                "entity": [
+                    "AIDACKCEVSQ6C2EXAMPLE",
+                    "JohnDoe",
+                    "arn:aws:iam::111122223333:user/JohnDoe"
+                ],
                 "user": [
                     "JohnDoe"
                 ]
@@ -158,6 +163,11 @@
                 ]
             },
             "related": {
+                "entity": [
+                    "AIDACKCEVSQ6C2EXAMPLE",
+                    "JaneDoe",
+                    "arn:aws:iam::111122223333:user/JaneDoe"
+                ],
                 "user": [
                     "JaneDoe"
                 ]
@@ -270,6 +280,14 @@
                     "info"
                 ]
             },
+            "related": {
+                "entity": [
+                    "arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName",
+                    "arn:aws:iam::123456789012:role/RoleToBeAssumed",
+                    "AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName",
+                    "RoleToBeAssumed"
+                ]
+            },
             "source": {
                 "address": "89.160.20.156",
                 "as": {

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json

@@ -61,6 +61,13 @@
                 ]
             },
             "related": {
+                "entity": [
+                    "EXAMPLE_ID",
+                    "Bob",
+                    "EXAMPLE_KEY_ID",
+                    "Alice",
+                    "arn:aws:iam::0123456789012:user/Alice"
+                ],
                 "user": [
                     "Alice",
                     "Bob"

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-db-instance-json.log

@@ -0,0 +1 @@
+{"awsRegion":"us-east-1","eventCategory":"Management","eventID":"d82a0cd1-6987-459a-b7bc-557a06bf16f2","eventName":"CreateDBInstance","eventSource":"rds.amazonaws.com","eventTime":"2024-09-11T09:29:51Z","eventType":"AwsApiCall","eventVersion":"1.08","readOnly":false,"recipientAccountId":"1010101010101","requestID":"b51e7190-610a-40c7-bb1c-a0895e3518f8","requestParameters":{"allocatedStorage":20,"dBInstanceClass":"db.t3.micro","dBInstanceIdentifier":"test-cloudtrail-event-instance-14340","engine":"mysql","masterUserPassword":"HIDDEN_DUE_TO_SECURITY_REASONS","masterUsername":"admin"},"responseElements":{"allocatedStorage":20,"associatedRoles":[],"autoMinorVersionUpgrade":true,"backupRetentionPeriod":1,"backupTarget":"region","cACertificateIdentifier":"rds-ca-rsa2048-g1","certificateDetails":{"cAIdentifier":"rds-ca-rsa2048-g1"},"copyTagsToSnapshot":false,"customerOwnedIpEnabled":false,"dBInstanceArn":"arn:aws:rds:us-east-1:1010101010101:db:test-cloudtrail-event-instance-14340","dBInstanceClass":"db.t3.micro","dBInstanceIdentifier":"test-cloudtrail-event-instance-14340","dBInstanceStatus":"creating","dBParameterGroups":[{"dBParameterGroupName":"default.mysql8.0","parameterApplyStatus":"in-sync"}],"dBSecurityGroups":[],"dBSubnetGroup":{"dBSubnetGroupDescription":"default","dBSubnetGroupName":"default","subnetGroupStatus":"Complete","subnets":[{"subnetAvailabilityZone":{"name":"us-east-1d"},"subnetIdentifier":"subnet-c4bf5e9b","subnetOutpost":{},"subnetStatus":"Active"},{"subnetAvailabilityZone":{"name":"us-east-1a"},"subnetIdentifier":"subnet-0a0bee6c","subnetOutpost":{},"subnetStatus":"Active"},{"subnetAvailabilityZone":{"name":"us-east-1e"},"subnetIdentifier":"subnet-37391109","subnetOutpost":{},"subnetStatus":"Active"},{"subnetAvailabilityZone":{"name":"us-east-1b"},"subnetIdentifier":"subnet-fee506df","subnetOutpost":{},"subnetStatus":"Active"},{"subnetAvailabilityZone":{"name":"us-east-1f"},"subnetIdentifier":"subnet-bf6ab5b1","subnetOutpost":{},"subnetStatus":"Active"},{"subnetAvailabilityZone":{"name":"us-east-1c"},"subnetIdentifier":"subnet-8bdf6bc6","subnetOutpost":{},"subnetStatus":"Active"}],"vpcId":"vpc-73d2e309"},"dbInstancePort":0,"dbiResourceId":"db-ANY6I3FNUJC7WQKYS5RFPU7ORM","dedicatedLogVolume":false,"deletionProtection":false,"domainMemberships":[],"engine":"mysql","engineLifecycleSupport":"open-source-rds-extended-support","engineVersion":"8.0.35","httpEndpointEnabled":false,"iAMDatabaseAuthenticationEnabled":false,"licenseModel":"general-public-license","masterUsername":"admin","monitoringInterval":0,"multiAZ":false,"networkType":"IPV4","optionGroupMemberships":[{"optionGroupName":"default:mysql-8-0","status":"in-sync"}],"pendingModifiedValues":{"masterUserPassword":"HIDDEN_DUE_TO_SECURITY_REASONS"},"performanceInsightsEnabled":false,"preferredBackupWindow":"09:23-09:53","preferredMaintenanceWindow":"sun:06:55-sun:07:25","publiclyAccessible":true,"readReplicaDBInstanceIdentifiers":[],"storageEncrypted":false,"storageThroughput":0,"storageType":"gp2","tagList":[],"vpcSecurityGroups":[{"status":"active","vpcSecurityGroupId":"sg-4e483165"}]},"sourceIPAddress":"216.160.83.56","tlsDetails":{"cipherSuite":"TLS_AES_128_GCM_SHA256","clientProvidedHostHeader":"rds.us-east-1.amazonaws.com","tlsVersion":"TLSv1.3"},"userAgent":"aws-cli/2.14.5 Python/3.11.6 Darwin/23.6.0 exec-env/grimoire_7ea17849-045c-4e81-8a81-d0822d5c4aaf exe/x86_64 prompt/off command/rds.create-db-instance","userIdentity":{"accessKeyId":"ACCESS_KEY_EXAMPLE","accountId":"1010101010101","arn":"arn:aws:iam::1010101010101:user/[email protected]","principalId":"AIDA2IBR2EZTJMPOR52WV","type":"IAMUser","userName":"[email protected]"}}