From 275699164f8fce4e909ac81fe41f9d36a24e83eb Mon Sep 17 00:00:00 2001 From: Tomas Kubla Date: Wed, 21 Dec 2022 14:46:12 +0000 Subject: [PATCH 01/63] Add go-httpbin --- docker-compose.override.dev.yml | 2 ++ docker-compose.override.unit_tests.yml | 2 ++ docker-compose.override.unit_tests_cicd.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml index f3a281af061..59c3ebdc857 100644 --- a/docker-compose.override.dev.yml +++ b/docker-compose.override.dev.yml @@ -53,3 +53,5 @@ services: published: 8025 protocol: tcp mode: host + go-httpbin: + image: mccutchen/go-httpbin:v2.5.3@sha256:866e254ec9fc44cdba12e8e9f1730e666958ceedfa32d09787bc95f9d4d01679 diff --git a/docker-compose.override.unit_tests.yml b/docker-compose.override.unit_tests.yml index 164d7a87084..beeff5f2174 100644 --- a/docker-compose.override.unit_tests.yml +++ b/docker-compose.override.unit_tests.yml @@ -51,6 +51,8 @@ services: redis: image: busybox:1.36.1-musl entrypoint: ['echo', 'skipping', 'redis'] + go-httpbin: + image: mccutchen/go-httpbin:v2.5.3@sha256:866e254ec9fc44cdba12e8e9f1730e666958ceedfa32d09787bc95f9d4d01679 volumes: defectdojo_postgres_unit_tests: {} defectdojo_media_unit_tests: {} diff --git a/docker-compose.override.unit_tests_cicd.yml b/docker-compose.override.unit_tests_cicd.yml index b39f4cf034d..03bd9c9f92b 100644 --- a/docker-compose.override.unit_tests_cicd.yml +++ b/docker-compose.override.unit_tests_cicd.yml @@ -50,6 +50,8 @@ services: redis: image: busybox:1.36.1-musl entrypoint: ['echo', 'skipping', 'redis'] + go-httpbin: + image: mccutchen/go-httpbin:v2.5.3@sha256:866e254ec9fc44cdba12e8e9f1730e666958ceedfa32d09787bc95f9d4d01679 volumes: defectdojo_postgres_unit_tests: {} defectdojo_media_unit_tests: {} From 77e29cc7f32c3dd056f46c2d28929a2f7268713c Mon Sep 17 00:00:00 2001 From: Tomas Kubla Date: Thu, 22 Dec 2022 19:37:10 +0000 Subject: [PATCH 02/63] First round of changes --- docs/content/en/integrations/notifications.md | 7 +- .../0169_add_webhooks_notifications.py | 109 ++++++++++++++++++ dojo/models.py | 11 ++ dojo/notifications/helper.py | 77 +++++++++++++ dojo/templates/dojo/notifications.html | 3 + dojo/templates/dojo/system_settings.html | 2 +- dojo/templates/dojo/view_product_details.html | 3 + tests/notifications_test.py | 5 + 8 files changed, 215 insertions(+), 2 deletions(-) create mode 100644 dojo/db_migrations/0169_add_webhooks_notifications.py diff --git a/docs/content/en/integrations/notifications.md b/docs/content/en/integrations/notifications.md index d5af295f0eb..3d979dbbd64 100644 --- a/docs/content/en/integrations/notifications.md +++ b/docs/content/en/integrations/notifications.md @@ -18,6 +18,7 @@ The following notification methods currently exist: - Email - Slack - Microsoft Teams + - Webhooks - Alerts within DefectDojo (default) You can set these notifications on a global scope (if you have @@ -124,4 +125,8 @@ However, there is a specific use-case when the user decides to disable notificat The scope of this setting is customizable (see environmental variable `DD_NOTIFICATIONS_SYSTEM_LEVEL_TRUMP`). -For more information about this behavior see the [related pull request #9699](https://github.com/DefectDojo/django-DefectDojo/pull/9699/) \ No newline at end of file +For more information about this behavior see the [related pull request #9699](https://github.com/DefectDojo/django-DefectDojo/pull/9699/) + +### Webhooks + +# TODO \ No newline at end of file diff --git a/dojo/db_migrations/0169_add_webhooks_notifications.py b/dojo/db_migrations/0169_add_webhooks_notifications.py new file mode 100644 index 00000000000..725545c117f --- /dev/null +++ b/dojo/db_migrations/0169_add_webhooks_notifications.py @@ -0,0 +1,109 @@ +# Generated by Django 3.2.15 on 2022-12-22 19:19 + +from django.db import migrations, models +import multiselectfield.db.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0168_alter_system_settings_time_zone'), + ] + + operations = [ + migrations.AlterField( + model_name='notifications', + name='auto_close_engagement', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), max_length=33), + ), + migrations.AlterField( + model_name='notifications', + name='close_engagement', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), max_length=33), + ), + migrations.AlterField( + model_name='notifications', + name='code_review', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), max_length=33), + ), + migrations.AlterField( + model_name='notifications', + name='engagement_added', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), max_length=33), + ), + migrations.AlterField( + model_name='notifications', + name='jira_update', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), help_text='JIRA sync happens in the background, errors will be shown as notifications/alerts so make sure to subscribe', max_length=33, verbose_name='JIRA problems'), + ), + migrations.AlterField( + model_name='notifications', + name='other', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), max_length=33), + ), + migrations.AlterField( + model_name='notifications', + name='product_added', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), max_length=33), + ), + migrations.AlterField( + model_name='notifications', + name='product_type_added', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), max_length=33), + ), + migrations.AlterField( + model_name='notifications', + name='review_requested', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), max_length=33), + ), + migrations.AlterField( + model_name='notifications', + name='risk_acceptance_expiration', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default='alert', help_text='Get notified of (upcoming) Risk Acceptance expiries', max_length=33, verbose_name='Risk Acceptance Expiration'), + ), + migrations.AlterField( + model_name='notifications', + name='scan_added', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), help_text='Triggered whenever an (re-)import has been done that created/updated/closed findings.', max_length=33), + ), + migrations.AlterField( + model_name='notifications', + name='sla_breach', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), help_text='Get notified of (upcoming) SLA breaches', max_length=33, verbose_name='SLA breach'), + ), + migrations.AlterField( + model_name='notifications', + name='stale_engagement', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), max_length=33), + ), + migrations.AlterField( + model_name='notifications', + name='test_added', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), max_length=33), + ), + migrations.AlterField( + model_name='notifications', + name='upcoming_engagement', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), max_length=33), + ), + migrations.AlterField( + model_name='notifications', + name='user_mentioned', + field=multiselectfield.db.fields.MultiSelectField(blank=True, choices=[('slack', 'slack'), ('msteams', 'msteams'), ('mail', 'mail'), ('webhooks', 'webhooks'), ('alert', 'alert')], default=('alert', 'alert'), max_length=33), + ), + migrations.AddField( + model_name='system_settings', + name='enable_webhooks_notifications', + field=models.BooleanField(default=False, verbose_name='Enable Webhook notifications'), + ), + migrations.AddField( + model_name='system_settings', + name='webhooks_url', + field=models.CharField(blank=True, default='', help_text='The full URL of the incoming webhook', max_length=400), + ), + migrations.AddField( + model_name='system_settings', + name='webhooks_token', + field=models.CharField(blank=True, default='', help_text='Token required for interacting with Webhook endpoint', max_length=100), + ), + ] diff --git a/dojo/models.py b/dojo/models.py index 5048f30427f..9ccdb9a0b0f 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -353,6 +353,15 @@ class System_Settings(models.Model): mail_notifications_to = models.CharField(max_length=200, default="", blank=True) + enable_webhooks_notifications = \ + models.BooleanField(default=False, + verbose_name=_('Enable Webhook notifications'), + blank=False) + webhooks_url = models.CharField(max_length=400, default='', blank=True, + help_text=_('The full URL of the incoming webhook')) + webhooks_token = models.CharField(max_length=100, default='', blank=True, + help_text=_('Token required for interacting with Webhook endpoint')) + false_positive_history = models.BooleanField( default=False, help_text=_( "(EXPERIMENTAL) DefectDojo will automatically mark the finding as a " @@ -4015,12 +4024,14 @@ def set_obj(self, obj): NOTIFICATION_CHOICE_SLACK = ("slack", "slack") NOTIFICATION_CHOICE_MSTEAMS = ("msteams", "msteams") NOTIFICATION_CHOICE_MAIL = ("mail", "mail") +NOTIFICATION_CHOICE_WEBHOOKS = ("webhooks", "webhooks") NOTIFICATION_CHOICE_ALERT = ("alert", "alert") NOTIFICATION_CHOICES = ( NOTIFICATION_CHOICE_SLACK, NOTIFICATION_CHOICE_MSTEAMS, NOTIFICATION_CHOICE_MAIL, + NOTIFICATION_CHOICE_WEBHOOKS, NOTIFICATION_CHOICE_ALERT, ) diff --git a/dojo/notifications/helper.py b/dojo/notifications/helper.py index 5a7ccf0dc60..7acf7591c73 100644 --- a/dojo/notifications/helper.py +++ b/dojo/notifications/helper.py @@ -4,12 +4,14 @@ from django.conf import settings from django.core.exceptions import FieldDoesNotExist from django.core.mail import EmailMessage +from django.conf import settings from django.db.models import Count, Prefetch, Q from django.template import TemplateDoesNotExist from django.template.loader import render_to_string from django.urls import reverse from django.utils.translation import gettext as _ +from dojo import __version__ as dd_version from dojo.authorization.roles_permissions import Permissions from dojo.celery import app from dojo.decorators import dojo_async_task, we_want_async @@ -170,6 +172,7 @@ def process_notifications(event, notifications=None, **kwargs): slack_enabled = get_system_setting("enable_slack_notifications") msteams_enabled = get_system_setting("enable_msteams_notifications") mail_enabled = get_system_setting("enable_mail_notifications") + webhooks_enabled = get_system_setting("enable_webhooks_notifications") if slack_enabled and "slack" in getattr(notifications, event, getattr(notifications, "other")): logger.debug("Sending Slack Notification") @@ -183,6 +186,10 @@ def process_notifications(event, notifications=None, **kwargs): logger.debug("Sending Mail Notification") send_mail_notification(event, notifications.user, **kwargs) + if webhooks_enabled and 'webhooks' in getattr(notifications, event, getattr(notifications, 'other')): + logger.debug('Sending Webhooks Notification') + send_webhooks_notification(event, notifications.user, **kwargs) + if "alert" in getattr(notifications, event, getattr(notifications, "other")): logger.debug(f"Sending Alert to {notifications.user}") send_alert_notification(event, notifications.user, **kwargs) @@ -309,6 +316,76 @@ def send_mail_notification(event, user=None, *args, **kwargs): log_alert(e, "Email Notification", title=kwargs["title"], description=str(e), url=kwargs["url"]) +@dojo_async_task +@app.task +def send_webhooks_notification(event, user=None, *args, **kwargs): + from dojo.utils import get_system_setting + + try: + if get_system_setting('webhooks_url') is not None: + logger.debug('sending webhook message') + headers={ + "User-Agent": f"DefectDojo-{dd_version}", + "X-DefectDojo-Event": event, + "X-DefectDojo-Instance": settings.SITE_URL, + } + if get_system_setting('webhooks_token') is not None: + headers["X-DefectDojo-Token"] = get_system_setting('webhooks_token') + if user: + headers["X-DefectDojo-User"] = user + res = requests.request( + method='POST', + url=get_system_setting('webhooks_url'), + headers=headers, + data=create_notification_message(event, user, 'webhooks', *args, **kwargs)) + if res.status_code != 200: + logger.error("Error when sending message to Webhooks") + logger.error(res.status_code) + logger.error(res.text) + raise RuntimeError('Error posting message to Webhooks: ' + res.text) + else: + logger.info('URL for Webhooks not configured: skipping system notification') + except Exception as e: + logger.exception(e) + log_alert(e, "Webhooks Notification", title=kwargs['title'], description=str(e), url=kwargs['url']) + pass + + +@dojo_async_task +@app.task +def send_webhooks_notification(event, user=None, *args, **kwargs): + from dojo.utils import get_system_setting + + try: + if get_system_setting('webhooks_url') is not None: + logger.debug('sending webhook message') + headers={ + "User-Agent": f"DefectDojo-{dd_version}", + "X-DefectDojo-Event": event, + "X-DefectDojo-Instance": settings.SITE_URL, + } + if get_system_setting('webhooks_token') is not None: + headers["X-DefectDojo-Token"] = get_system_setting('webhooks_token') + if user: + headers["X-DefectDojo-User"] = user + res = requests.request( + method='POST', + url=get_system_setting('webhooks_url'), + headers=headers, + data=create_notification_message(event, user, 'webhooks', *args, **kwargs)) + if res.status_code != 200: + logger.error("Error when sending message to Webhooks") + logger.error(res.status_code) + logger.error(res.text) + raise RuntimeError('Error posting message to Webhooks: ' + res.text) + else: + logger.info('URL for Webhooks not configured: skipping system notification') + except Exception as e: + logger.exception(e) + log_alert(e, "Webhooks Notification", title=kwargs['title'], description=str(e), url=kwargs['url']) + pass + + def send_alert_notification(event, user=None, *args, **kwargs): logger.debug("sending alert notification to %s", user) try: diff --git a/dojo/templates/dojo/notifications.html b/dojo/templates/dojo/notifications.html index 52d87393c45..81fac49d5cc 100644 --- a/dojo/templates/dojo/notifications.html +++ b/dojo/templates/dojo/notifications.html @@ -89,6 +89,9 @@

{% if 'mail' in enabled_notifications %} {% trans "Mail" %} {% endif %} + {% if 'webhooks' in enabled_notifications %} + {% trans "Webhooks" %} + {% endif %} {% trans "Alert" %} diff --git a/dojo/templates/dojo/system_settings.html b/dojo/templates/dojo/system_settings.html index 693abe712f0..02510452e16 100644 --- a/dojo/templates/dojo/system_settings.html +++ b/dojo/templates/dojo/system_settings.html @@ -62,7 +62,7 @@

System Settings

} $(function () { - $.each(['slack','msteams','mail', 'grade'], function (index, value) { + $.each(['slack','msteams','mail','webhooks','grade'], function (index, value) { updatenotificationsgroup(value); $('#id_enable_' + value + '_notifications').change(function() { updatenotificationsgroup(value)}); }); diff --git a/dojo/templates/dojo/view_product_details.html b/dojo/templates/dojo/view_product_details.html index 30dd863fc3c..301264524f4 100644 --- a/dojo/templates/dojo/view_product_details.html +++ b/dojo/templates/dojo/view_product_details.html @@ -658,6 +658,9 @@