azure: add related.entity field to activitylogs default ingest pipeline (#11233)

Author: orouz

packages/azure/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.19.0"
+  changes:
+    - description: Add entity identifiers to `related.entity` in activitylogs.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11233
 - version: "1.18.0"
   changes:
     - description: Add entity identifiers to `related.entity`.

packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-edgecases.log-expected.json

@@ -35,6 +35,11 @@
             "log": {
                 "level": "Information"
             },
+            "related": {
+                "entity": [
+                    "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.domainRegistration"
+                ]
+            },
             "tags": [
                 "preserve_original_event"
             ]
@@ -279,6 +284,9 @@
                 }
             },
             "related": {
+                "entity": [
+                    "/tenants/c7f1e3ce-ba66-40a7-91bd-9594b36223fc/providers/Microsoft.aadiam"
+                ],
                 "ip": [
                     "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6"
                 ]
@@ -529,6 +537,11 @@
             "geo": {
                 "name": "GB"
             },
+            "related": {
+                "entity": [
+                    "/tenants/c7f1e3ce-ba66-40a7-91bd-9594b36223fc/providers/Microsoft.aadiam"
+                ]
+            },
             "source": {
                 "address": "127.0.0.0/8"
             },

packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-identity.log-expected.json

@@ -243,6 +243,9 @@
                 "region_name": "England"
             },
             "related": {
+                "entity": [
+                    "/tenants/c7f1e3ce-ba66-40a7-91bd-9594b36223fc/providers/Microsoft.aadiam"
+                ],
                 "ip": [
                     "81.2.69.143"
                 ]

packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json

@@ -84,6 +84,10 @@
                 "level": "Information"
             },
             "related": {
+                "entity": [
+                    "/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY",
+                    "8a4de8b5-095c-47d0-a96f-a75130c61d53"
+                ],
                 "ip": [
                     "81.2.69.144"
                 ]

packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml

@@ -313,10 +313,23 @@ processors:
   - set:
       field: event.kind
       value: event
+  - append:
+      field: related.entity
+      value: '{{{ azure.resource_id }}}'
+      allow_duplicates: false
+      if: ctx.azure?.resource_id != null && ctx.azure.resource_id != ''
+  - append:
+      field: related.entity
+      value: '{{{ azure.activitylogs.identity.authorization.evidence.principal_id }}}'
+      allow_duplicates: false
+      if: >
+        ctx.azure?.activitylogs?.identity?.authorization?.evidence?.principal_id != null &&
+        ctx.azure.activitylogs.identity.authorization.evidence.principal_id != ''
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
 on_failure:
   - set:
       field: error.message
       value: |-
         Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
+

packages/azure/data_stream/activitylogs/fields/fields.yml

@@ -147,3 +147,10 @@
 
         Not typically used in automated geolocation.'
       level: extended
+- name: related.entity
+  description: |
+    All the entity identifiers related to the document. If the document
+    contains multiple entities, identifiers belonging to different entities
+    will be present. Example identifiers include cloud resource IDs, ARNs,
+    email addresses, or hostnames.
+  type: keyword

packages/azure/docs/activitylogs.md

@@ -213,4 +213,5 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | host.containerized | If the host is a container. | boolean |
 | host.os.build | OS build information. | keyword |
 | host.os.codename | OS codename, if any. | keyword |
+| related.entity | All the entity identifiers related to the document. If the document contains multiple entities, identifiers belonging to different entities will be present. Example identifiers include cloud resource IDs, ARNs, email addresses, or hostnames. | keyword |
 

packages/azure/manifest.yml

@@ -1,6 +1,6 @@
 name: azure
 title: Azure Logs
-version: 1.18.0
+version: 1.19.0
 description: This Elastic integration collects logs from Azure
 type: integration
 icons: