Event Integrations Structurizr POC

docs/admin_console/event_integrations/models.dsl

@@ -0,0 +1,216 @@
+!element server {
+    azure_service_bus = container "Azure Service Bus" {
+        description "AMQP service used for pub/sub architecture for Events and Integrations"
+        tags "Events", "Azure", "ASB", "Cloud-Only"
+
+        event_topic = component "Event Topic" {
+            description "The main entry point for all events in the system. When an event occurs, it is published to this topic."
+            tags "Events", "ASB", "Event Tier"
+        }
+
+        integration_topic = component "Integration Topic" {
+            description "Events that have integrations configured are processed and put on the integration topic with a routing key for their specific integration handler to process."
+            tags "Events", "ASB", "Integrations", "Integration Tier"
+        }
+
+        eventsWriteSub = component "events-write-subscription" {
+            description "Subscription for EventRepositoryHandler to write all events into azure table storage."
+            tags "ASB", "Subscription", "Event Tier"
+        }
+
+        eventsSlackSub = component "events-slack-subscription" {
+            description "Subscription for slack-specific EventIntegrationHandler which publishes processed events to the integration tier if there is a Slack integration configured."
+            tags "ASB", "Subscription", "Event Tier", "Slack"
+        }
+
+        eventsWebhookSub = component "events-webhook-subscription" {
+            description "Subscription for webhook-specific EventIntegrationHandler which publishes processed events to the integration tier if there is a webhook integration configured."
+            tags "ASB", "Subscription", "Event Tier", "Webhook"
+        }
+
+        eventsHecSub = component "events-hec-subscription" {
+            description "Subscription for HEC-specific EventIntegrationHandler which publishes processed events to the integration tier if there is a HEC integration configured."
+            tags "ASB", "Subscription", "Event Tier", "HEC"
+        }
+
+        integrationSlackSub = component "integration-slack-subscription" {
+            description "Integration-level subscription for Slack IntegrationMessages. Correlation filter: Label = 'slack'."
+            tags "ASB", "Subscription", "Integration Tier", "Slack"
+        }
+
+        integrationWebhookSub = component "integration-webhook-subscription" {
+            description "Integration-level subscription for Webhook IntegrationMessages. Correlation filter: Label = 'webhook'."
+            tags "ASB", "Subscription", "Integration Tier", "Webhook"
+        }
+
+        integrationHecSub = component "integration-hec-subscription" {
+            description "Integration-level subscription for HEC IntegrationMessages. Correlation filter: Label = 'hec'."
+            tags "ASB", "Subscription", "Integration Tier", "HEC"
+        }
+    }
+
+    rabbit_mq = container "RabbitMQ" {
+        tags "Events"
+        tags "RabbitMQ"
+        tags "Self-Hosted-Only"
+
+        event_exchange = component "Event Exchange" {
+            tags "Events", "Event Tier"
+        }
+
+        integration_exchange = component "Integration Exchange" {
+            tags "Events", "Integrations", "Integration Tier"
+        }
+
+        eventsWriteQueue = component "events-write-queue" {
+            description "Queue for EventRepositoryHandler to write all events into the database."
+            tags "RabbitMQ", "Queue", "Event Tier"
+        }
+
+        eventsSlackQueue = component "events-slack-queue" {
+            description "Queue for slack-specific EventIntegrationHandler which publishes processed events to the integration tier if there is a Slack integration configured."
+            tags "RabbitMQ", "Queue", "Event Tier", "Slack"
+        }
+
+        eventsWebhookQueue = component "events-webhook-queue" {
+            description "Queue for webhook-specific EventIntegrationHandler which publishes processed events to the integration tier if there is a webhook integration configured."
+            tags "RabbitMQ", "Queue", "Event Tier", "Webhook"
+        }
+
+        eventsHecQueue = component "events-hec-queue" {
+            description "Queue for HEC-specific EventIntegrationHandler which publishes processed events to the integration tier if there is a HEC integration configured."
+            tags "RabbitMQ", "Queue", "Event Tier", "HEC"
+        }
+
+        integrationSlackQueue = component "integration-slack-queue" {
+            description "Integration-level queue for Slack IntegrationMessages. Routing key = 'slack'."
+            tags "RabbitMQ", "Queue", "Integration Tier", "Slack"
+        }
+
+        integrationWebhookQueue = component "integration-webhook-queue" {
+            description "Integration-level queue for Webhook IntegrationMessages. Routing key = 'webhook'."
+            tags "RabbitMQ", "Queue", "Integration Tier", "Webhook"
+        }
+
+        integrationHecQueue = component "integration-hec-queue" {
+            description "Integration-level queue for HEC IntegrationMessages. Routing key = 'hec'."
+            tags "RabbitMQ", "Queue", "Integration Tier", "HEC"
+        }
+
+        integrationSlackRetryQueue = component "integration-slack-retry-queue" {
+            description "Integration-level retry queue for Slack IntegrationMessages. Routing key Label = 'slack-retry'."
+            tags "RabbitMQ", "Queue", "Integration Tier", "Slack"
+        }
+
+        integrationWebhookRetryQueue = component "integration-webhook-retry-queue" {
+            description "Integration-level retry queue for Webhook IntegrationMessages. Routing key = 'webhook-retry'."
+            tags "RabbitMQ", "Queue", "Integration Tier", "Webhook"
+        }
+
+        integrationHecRetryQueue = component "integration-hec-retry-queue" {
+            description "Integration-level retry queue for HEC IntegrationMessages. Routing key = 'hec-retry'."
+            tags "RabbitMQ", "Queue", "Integration Tier", "HEC"
+        }
+    }
+}
+
+!element server.events_processor {
+    tags "Cloud-Only"
+
+    event_repository_handler = component "EventRepositoryHandler" {
+        description "Handles all events, passing them off to the IEventWriteService with the `persistent` key for long term storage."
+    }
+    event_listener = component "AzureServiceBusEventListenerService" {
+        description "Listens to a specific subscription and passes off to a handler to handle events"
+    }
+    integration_listener = component "AzureServiceBusIntegrationListenerService" {
+        description "Listens to a specific subscription and passes off to a handler to handle IntegrationMessages"
+    }
+    event_integration_handler = component "EventIntegrationHandler" {
+        description "Fetches the relevent configurations when an event comes in and hands the event to its paired integration handler for processing."
+    }
+    slack_integration_handler = component "SlackIntegrationHandler" {
+        description "Processes Slack IntegrationMessages, posting them to the configured channels."
+    }
+    webhook_integration_handler = component "WebhookIntegrationHandler" {
+        description "Processes Webhook and HEC IntegrationMessages, posting them to the configured URI."
+    }
+    integration_configuration_details_cache_service = component "IntegrationConfigurationDetailsCacheService" {
+        description "Caches all configurations for integrations in memory so that events can be handled without adding database load."
+    }
+    slack_service = component "SlackService" {
+        description "Handles all API interaction with Slack."
+    }
+    http_client = component "HttpClient" {
+        description "Performs any Http functions for Webhooks / HEC."
+    }
+    integration_filter_service = component "IntegrationFilterService" {
+        description "Processes filters from configurations to determine if an event should be processed out to the integration."
+    }
+}
+
+!element server.events {
+    event_listener = component "RabbitMqEventListenerService" {
+        description "Listens to a specific queue and passes off to a handler to handle events"
+    }
+    integration_listener = component "RabbitMqIntegrationListenerService" {
+        description "Listens to a specific queue and passes off to a handler to handle IntegrationMessages"
+    }
+    event_repository_handler = component "EventRepositoryHandler" {
+        tags "Self-Hosted-Only"
+        description "Handles all events, passing them off to the IEventWriteService with the `persistent` key for long term storage."
+    }
+    event_integration_handler = component "EventIntegrationHandler" {
+        tags "Self-Hosted-Only"
+        description "Fetches the relevent configurations when an event comes in and hands the event to its paired integration handler for processing."
+    }
+    slack_integration_handler = component "SlackIntegrationHandler" {
+        tags "Self-Hosted-Only"
+        description "Processes Slack IntegrationMessages, posting them to the configured channels."
+    }
+    webhook_integration_handler = component "WebhookIntegrationHandler" {
+        tags "Self-Hosted-Only"
+        description "Processes Webhook and HEC IntegrationMessages, posting them to the configured URI."
+    }
+    integration_configuration_details_cache_service = component "IntegrationConfigurationDetailsCacheService" {
+        tags "Self-Hosted-Only"
+        description "Caches all configurations for integrations in memory so that events can be handled without adding database load."
+    }
+    slack_service = component "SlackService" {
+        tags "Self-Hosted-Only"
+        description "Handles all API interaction with Slack."
+    }
+    http_client = component "HttpClient" {
+        tags "Self-Hosted-Only"
+        description "Performs any Http functions for Webhooks / HEC."
+    }
+    integration_filter_service = component "IntegrationFilterService" {
+        tags "Self-Hosted-Only"
+        description "Processes filters from configurations to determine if an event should be processed out to the integration."
+    }
+}
+
+external_services = softwareSystem "External Services" {
+    tags "External", "Events", "Integrations"
+    description "External services (e.g. SIEM, Slack, et al) that consume events via integrations"
+
+    slack = container "Slack" {
+        tags "External", "Events", "Integrations", "Slack"
+        description "Slack messaging service. Receives messages via configured event integrations."
+    }
+
+    splunk = container "Splunk" {
+        tags "External", "Events", "Integrations", "Splunk"
+        description "Splunk SIEM service. Receives events via configured event integrations."
+    }
+
+    datadog = container "Datadog" {
+        tags "External", "Events", "Integrations", "Datadog"
+        description "Datadog SIEM service. Receives events via configured event integrations."
+    }
+
+    crowdstrike = container "Crowdstrike Falcon" {
+        tags "External", "Events", "Integrations", "Crowdstrike Falcon", "Crowdstrike"
+        description "Crowdstrike Falcon SIEM service. Receives events via configured event integrations."
+    }
+}

docs/admin_console/event_integrations/relationships.dsl

@@ -0,0 +1,79 @@
+# Top Level event publishing
+server.api -> server.azure_service_bus.event_topic "Sends events to"
+server.events -> server.azure_service_bus.event_topic "Sends events to"
+server.api -> server.rabbit_mq.event_exchange "Sends events to"
+server.events -> server.rabbit_mq.event_exchange "Sends events to"
+
+# Azure Service Bus topics, subscriptions, and routing
+eventsWriteSub = server.azure_service_bus.event_topic -> server.azure_service_bus.eventsWriteSub  "Subscribes via fan-out"
+eventsHecSub = server.azure_service_bus.event_topic -> server.azure_service_bus.eventsHecSub "Subscribes via fan-out"
+eventsWebhookSub = server.azure_service_bus.event_topic -> server.azure_service_bus.eventsWebhookSub  "Subscribes via fan-out"
+eventsSlackSub = server.azure_service_bus.event_topic -> server.azure_service_bus.eventsSlackSub "Subscribes via fan-out"
+integrationSlackSub = server.azure_service_bus.integration_topic -> server.azure_service_bus.integrationSlackSub "Subscribes via filter on Slack key"
+integrationWebhookSub = server.azure_service_bus.integration_topic -> server.azure_service_bus.integrationWebhookSub "Subscribes via filter on Webhook key"
+integrationHecSub = server.azure_service_bus.integration_topic -> server.azure_service_bus.integrationHecSub "Subscribes via filter on HEC key"
+
+eventsWriteListener = server.events_processor.event_listener -> server.azure_service_bus.eventsWriteSub "Listens to"
+eventsWriteDelegate = server.events_processor.event_listener -> server.events_processor.event_repository_handler "Delegates to"
+eventsHecListener = server.events_processor.event_listener -> server.azure_service_bus.eventsHecSub "Listens to"
+eventsWebhookListener = server.events_processor.event_listener -> server.azure_service_bus.eventsWebhookSub "Listens to"
+eventsSlackListener = server.events_processor.event_listener -> server.azure_service_bus.eventsSlackSub "Listens to"
+eventsIntegrationHandlerDelegate = server.events_processor.event_listener -> server.events_processor.event_integration_handler "Delegates to"
+
+eventIntegrationHandlerPublish = server.events_processor.event_integration_handler -> server.azure_service_bus.integration_topic "Publishes To"
+eventIntegrationHandlerCache = server.events_processor.event_integration_handler -> server.events_processor.integration_configuration_details_cache_service "Fetches configurations from"
+cacheDatabaseFetch = server.events_processor.integration_configuration_details_cache_service -> server.database "Fetches configurations from"
+eventIntegrationHandlerFilter = server.events_processor.event_integration_handler -> server.events_processor.integration_filter_service "Runs filters"
+eventRepositoryDatabase = server.events_processor.event_repository_handler -> server.database  "Writes events to"
+
+integrationSlackListener = server.events_processor.integration_listener -> server.azure_service_bus.integrationSlackSub "Listens to"
+integrationSlackDelegate = server.events_processor.integration_listener -> server.events_processor.slack_integration_handler "Delegates to"
+integrationWebhookListener = server.events_processor.integration_listener -> server.azure_service_bus.integrationWebhookSub "Listens to"
+integrationHecListener = server.events_processor.integration_listener -> server.azure_service_bus.integrationHecSub "Listens to"
+integrationWebhookDelegate = server.events_processor.integration_listener -> server.events_processor.webhook_integration_handler "Delegates to"
+
+# RabbitMQ exchanges, queues, and routing
+eventsWriteQueue = server.rabbit_mq.event_exchange -> server.rabbit_mq.eventsWriteQueue  "Subscribes via fan-out"
+eventsHecQueue = server.rabbit_mq.event_exchange -> server.rabbit_mq.eventsHecQueue "Subscribes via fan-out"
+eventsWebhookQueue = server.rabbit_mq.event_exchange -> server.rabbit_mq.eventsWebhookQueue "Subscribes via fan-out"
+eventsSlackQueue = server.rabbit_mq.event_exchange -> server.rabbit_mq.eventsSlackQueue "Subscribes via fan-out"
+integrationSlackQueue = server.rabbit_mq.integration_exchange -> server.rabbit_mq.integrationSlackQueue "Subscribes via filter on Slack key"
+integrationWebhookQueue = server.rabbit_mq.integration_exchange -> server.rabbit_mq.integrationWebhookQueue "Subscribes via filter on Webhook key"
+integrationHecQueue = server.rabbit_mq.integration_exchange -> server.rabbit_mq.integrationHecQueue "Subscribes via filter on HEC key"
+integrationSlackRetryQueue = server.rabbit_mq.integrationSlackRetryQueue -> server.rabbit_mq.integrationSlackQueue "DLQ after configured retry timing"
+integrationWebhookRetryQueue = server.rabbit_mq.integrationWebhookRetryQueue -> server.rabbit_mq.integrationWebhookQueue "DLQ after configured retry timing"
+integrationHecRetryQueue = server.rabbit_mq.integrationHecRetryQueue -> server.rabbit_mq.integrationHecQueue "DLQ after configured retry timing"
+
+eventsWriteListener_events = server.events.event_listener -> server.rabbit_mq.eventsWriteQueue "Listens to"
+eventsHecListener_events = server.events.event_listener -> server.rabbit_mq.eventsHecQueue "Listens to"
+eventsWebhookListener_events = server.events.event_listener -> server.rabbit_mq.eventsWebhookQueue "Listens to"
+eventsSlackListener_events = server.events.event_listener -> server.rabbit_mq.eventsSlackQueue "Listens to"
+eventsWriteDelegate_events = server.events.event_listener -> server.events.event_repository_handler "Delegates to"
+
+eventRepositoryDatabase_events = server.events.event_repository_handler -> server.database "Writes events to" tags "Self-Hosted-Only"
+eventsIntegrationHandlerDelegate_events = server.events.event_listener -> server.events.event_integration_handler "Delegates to"
+cacheDatabaseFetch_events = server.events.integration_configuration_details_cache_service -> server.database "Fetches configurations from" tags "Self-Hosted-Only"
+eventIntegrationHandlerCache_events = server.events.event_integration_handler -> server.events.integration_configuration_details_cache_service "Fetches configurations from"
+eventIntegrationHandlerFilter_events = server.events.event_integration_handler -> server.events.integration_filter_service "Runs filters"
+eventIntegrationHandlerPublish_events = server.events.event_integration_handler -> server.rabbit_mq.integration_exchange "Publishes To"
+
+integrationSlackListener_events = server.events.integration_listener -> server.rabbit_mq.integrationSlackQueue "Listens to"
+integrationWebhookListener_events = server.events.integration_listener -> server.rabbit_mq.integrationWebhookQueue "Listens to"
+integrationHecListener_events = server.events.integration_listener -> server.rabbit_mq.integrationHecQueue "Listens to"
+integrationSlackDelegate_events = server.events.integration_listener -> server.events.slack_integration_handler "Delegates to"
+integrationWebhookDelegate_events = server.events.integration_listener -> server.events.webhook_integration_handler "Delegates to"
+
+# External Services
+slackToSlackService = server.events_processor.slack_integration_handler -> server.events_processor.slack_service "Uses"
+slackServiceToSlack = server.events_processor.slack_service  -> external_services.slack "Publishes configured events to"
+handlerHttpClient = server.events_processor.webhook_integration_handler -> server.events_processor.http_client "Uses"
+httpToCrowdstrike = server.events_processor.http_client -> external_services.crowdstrike "Publishes configured events to"
+httpToDatadog = server.events_processor.http_client -> external_services.datadog "Publishes configured events to"
+httpToSplunk = server.events_processor.http_client -> external_services.splunk "Publishes configured events to"
+
+slackToSlackService_events = server.events.slack_integration_handler -> server.events.slack_service "Uses" tags "Self-Hosted-Only"
+slackServiceToSlack_events = server.events.slack_service  -> external_services.slack "Publishes configured events to" tags "Self-Hosted-Only"
+handlerHttpClient_events = server.events.webhook_integration_handler -> server.events.http_client "Uses" tags "Self-Hosted-Only"
+httpToCrowdstrike_events = server.events.http_client -> external_services.crowdstrike "Publishes configured events to" tags "Self-Hosted-Only"
+httpToDatadog_events = server.events.http_client -> external_services.datadog "Publishes configured events to" tags "Self-Hosted-Only"
+httpToSplunk_events = server.events.http_client -> external_services.splunk "Publishes configured events to" tags "Self-Hosted-Only"