Add parser for AWS Security Finding Format (ASFF) (#7838)

* Add parser for AWS Security Finding Format (ASFF)

* Add doc

* Add missing file

* Add de-duplcation configuration

* Fix flake8

docs/content/en/integrations/parsers/file/asff.md

@@ -0,0 +1,10 @@
+---
+title: "AWS Security Finding Format (ASFF)"
+toc_hide: true
+---
+
+AWS Security Hub consumes, aggregates, organizes, and prioritizes findings from AWS security services and from the third-party product integrations. Security Hub processes these findings using a standard findings format called the AWS Security Finding Format (ASFF), which eliminates the need for time-consuming data conversion efforts. Then it correlates ingested findings across products to prioritize the most important ones.
+
+Reference: https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-findings-format.html
+
+Prowler tool can generate this format with option `-M json-asff`.

dojo/settings/settings.dist.py

@@ -1340,6 +1340,7 @@ def saml2_attrib_map_format(dict):
     'Aqua Scan': DEDUPE_ALGO_HASH_CODE,
     'AuditJS Scan': DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL,
     'AWS Prowler Scan': DEDUPE_ALGO_HASH_CODE,
+    "AWS Security Finding Format (ASFF) Scan": DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL,
     'Burp REST API': DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL,
     'Bandit Scan': DEDUPE_ALGO_HASH_CODE,
     'CargoAudit Scan': DEDUPE_ALGO_HASH_CODE,

dojo/tools/asff/parser.py

@@ -0,0 +1,62 @@
+import json
+
+import dateutil
+
+from dojo.models import Finding
+
+# https://docs.aws.amazon.com/securityhub/1.0/APIReference/API_Severity.html
+SEVERITY_MAPPING = {
+    "INFORMATIONAL": "Info",  # No issue was found.
+    "LOW": "Low",  # The issue does not require action on its own.
+    "MEDIUM": "Medium",  # The issue must be addressed but not urgently.
+    "HIGH": "High",  # The issue must be addressed as a priority.
+    "CRITICAL": "Critical",  # The issue must be remediated immediately to avoid it escalating.
+}
+
+
+class AsffParser(object):
+    def get_scan_types(self):
+        return ["AWS Security Finding Format (ASFF) Scan"]
+
+    def get_label_for_scan_types(self, scan_type):
+        return "AWS Security Finding Format (ASFF)"
+
+    def get_description_for_scan_types(self, scan_type):
+        return """AWS Security Finding Format (ASFF). https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-findings-format-syntax.html"""
+
+    def get_findings(self, file, test):
+        data = json.load(file)
+        result = list()
+        for item in data:
+            result.append(
+                Finding(
+                    title=item.get("Title"),
+                    description=item.get("Description"),
+                    date=dateutil.parser.parse(item.get("CreatedAt")),
+                    severity=self.get_severity(item.get("Severity")),
+                    active=True,  # TODO manage attribute 'RecordState'
+                    unique_id_from_tool=item.get("Id"),
+                )
+            )
+        return result
+
+    def get_severity(self, data):
+        if data.get("Label"):
+            return SEVERITY_MAPPING[data.get("Label")]
+        elif isinstance(data.get("Normalized"), int):
+            # 0 - INFORMATIONAL
+            # 1–39 - LOW
+            # 40–69 - MEDIUM
+            # 70–89 - HIGH
+            # 90–100 - CRITICAL
+            if data.get("Normalized") > 89:
+                return "Critical"
+            elif data.get("Normalized") > 69:
+                return "High"
+            elif data.get("Normalized") > 39:
+                return "Medium"
+            elif data.get("Normalized") > 0:
+                return "Low"
+            else:
+                return "Info"
+        return None

unittests/dojo_test_case.py

@@ -1,21 +1,24 @@
+import copy
+import json
+import logging
 import os
-from django.utils import timezone
-from vcr_unittest import VCRTestCase
-from dojo.models import DojoMeta, Product_Type, Test_Type, User, Endpoint, Notes, Finding, Endpoint_Status, Test, JIRA_Issue, JIRA_Project, \
-                        Product
-from dojo.models import System_Settings, Engagement
+import pprint
+from itertools import chain
+
+from django.test import TestCase
 from django.urls import reverse
-from rest_framework.test import APITestCase, APIClient
+from django.utils import timezone
+from django.utils.http import urlencode
 from rest_framework.authtoken.models import Token
-import json
-from django.test import TestCase
-from itertools import chain
+from rest_framework.test import APIClient, APITestCase
+from vcr_unittest import VCRTestCase
+
 from dojo.jira_link import helper as jira_helper
 from dojo.jira_link.views import get_custom_field
-import logging
-import pprint
-import copy
-from django.utils.http import urlencode
+from dojo.models import (SEVERITIES, DojoMeta, Endpoint, Endpoint_Status,
+                         Engagement, Finding, JIRA_Issue, JIRA_Project, Notes,
+                         Product, Product_Type, System_Settings, Test,
+                         Test_Type, User)
 
 logger = logging.getLogger(__name__)
 
@@ -401,6 +404,10 @@ class DojoTestCase(TestCase, DojoTestUtilsMixin):
     def __init__(self, *args, **kwargs):
         TestCase.__init__(self, *args, **kwargs)
 
+    def common_check_finding(self, finding):
+        self.assertIn(finding.severity, SEVERITIES)
+        finding.clean()
+
 
 class DojoAPITestCase(APITestCase, DojoTestUtilsMixin):
 

unittests/scans/asff/guardduty/Unusual Behaviors-User-Persistence IAMUser-NetworkPermissions.json

@@ -0,0 +1,107 @@
+[
+    {
+        "AwsAccountId": "123456789012",
+        "CreatedAt": "2020-11-11T11:47:26.511Z",
+        "Description": "APIs commonly used to change the network access permissions for security groups, routes and ACLs, was invoked by IAM principal GeneratedFindingUserName. Such activity is not typically seen from this principal.",
+        "FirstObservedAt": "2020-11-11T11:47:26.511Z",
+        "GeneratorId": "arn:aws:guardduty:eu-west-1:123456789012:detector/cab6a714deb3b739eaddacbdfd5ef2f2",
+        "Id": "arn:aws:guardduty:eu-west-1:123456789012:detector/cab6a714deb3b739eaddacbdfd5ef2f2/finding/d6badb90e557d4bd811488a53ca89895",
+        "LastObservedAt": "2020-11-28T23:11:30.811Z",
+        "ProductArn": "arn:aws:securityhub:eu-west-1::product/aws/guardduty",
+        "ProductFields": {
+            "aws/guardduty/service/action/actionType": "AWS_API_CALL",
+            "aws/guardduty/service/action/awsApiCallAction/affectedResources": "THIS_WAS_PREVIOUSLY_EMPTY_AND_WAS_ADDED_BY_NE",
+            "aws/guardduty/service/action/awsApiCallAction/api": "GeneratedFindingAPIName",
+            "aws/guardduty/service/action/awsApiCallAction/callerType": "Remote IP",
+            "aws/guardduty/service/action/awsApiCallAction/remoteIpDetails/city/cityName": "GeneratedFindingCityName",
+            "aws/guardduty/service/action/awsApiCallAction/remoteIpDetails/country/countryName": "GeneratedFindingCountryName",
+            "aws/guardduty/service/action/awsApiCallAction/remoteIpDetails/geoLocation/lat": "0",
+            "aws/guardduty/service/action/awsApiCallAction/remoteIpDetails/geoLocation/lon": "0",
+            "aws/guardduty/service/action/awsApiCallAction/remoteIpDetails/ipAddressV4": "198.51.100.0",
+            "aws/guardduty/service/action/awsApiCallAction/remoteIpDetails/organization/asn": "-1",
+            "aws/guardduty/service/action/awsApiCallAction/remoteIpDetails/organization/asnOrg": "GeneratedFindingASNOrg",
+            "aws/guardduty/service/action/awsApiCallAction/remoteIpDetails/organization/isp": "GeneratedFindingISP",
+            "aws/guardduty/service/action/awsApiCallAction/remoteIpDetails/organization/org": "GeneratedFindingORG",
+            "aws/guardduty/service/action/awsApiCallAction/serviceName": "GeneratedFindingAPIServiceName",
+            "aws/guardduty/service/additionalInfo/recentApiCalls.0_/api": "GeneratedFindingAPIName1",
+            "aws/guardduty/service/additionalInfo/recentApiCalls.0_/count": "2",
+            "aws/guardduty/service/additionalInfo/recentApiCalls.1_/api": "GeneratedFindingAPIName2",
+            "aws/guardduty/service/additionalInfo/recentApiCalls.1_/count": "2",
+            "aws/guardduty/service/additionalInfo/sample": "true",
+            "aws/guardduty/service/archived": "false",
+            "aws/guardduty/service/count": "2",
+            "aws/guardduty/service/detectorId": "cab6a714deb3b739eaddacbdfd5ef2f2",
+            "aws/guardduty/service/eventFirstSeen": "2020-11-11T11:47:26.511Z",
+            "aws/guardduty/service/eventLastSeen": "2020-11-28T23:11:30.811Z",
+            "aws/guardduty/service/resourceRole": "TARGET",
+            "aws/guardduty/service/serviceName": "guardduty",
+            "aws/securityhub/CompanyName": "Amazon",
+            "aws/securityhub/FindingId": "arn:aws:securityhub:eu-west-1::product/aws/guardduty/arn:aws:guardduty:eu-west-1:123456789012:detector/cab6a714deb3b739eaddacbdfd5ef2f2/finding/d6badb90e557d4bd811488a53ca89895",
+            "aws/securityhub/ProductName": "GuardDuty"
+        },
+        "RecordState": "ACTIVE",
+        "Resources": [
+            {
+                "Details": {
+                    "AwsEc2Instance": {
+                        "IamInstanceProfileArn": "arn:aws:iam::123456789012:example/instance/profile",
+                        "ImageId": "ami-99999999",
+                        "IpV4Addresses": [
+                            "10.0.0.1",
+                            "198.51.100.0"
+                        ],
+                        "LaunchedAt": "2016-08-02T02:05:06Z",
+                        "SubnetId": "GeneratedFindingSubnetId",
+                        "Type": "m3.xlarge",
+                        "VpcId": "GeneratedFindingVPCId"
+                    }
+                },
+                "Id": "arn:aws:ec2:eu-west-1:123456789012:instance/i-99999999",
+                "Partition": "aws",
+                "Region": "eu-west-1",
+                "Tags": {
+                    "GeneratedFindingInstaceTag1": "GeneratedFindingInstaceValue1",
+                    "GeneratedFindingInstaceTag2": "GeneratedFindingInstaceTagValue2",
+                    "GeneratedFindingInstaceTag3": "GeneratedFindingInstaceTagValue3",
+                    "GeneratedFindingInstaceTag4": "GeneratedFindingInstaceTagValue4",
+                    "GeneratedFindingInstaceTag5": "GeneratedFindingInstaceTagValue5",
+                    "GeneratedFindingInstaceTag6": "GeneratedFindingInstaceTagValue6",
+                    "GeneratedFindingInstaceTag7": "GeneratedFindingInstaceTagValue7",
+                    "GeneratedFindingInstaceTag8": "GeneratedFindingInstaceTagValue8",
+                    "GeneratedFindingInstaceTag9": "GeneratedFindingInstaceTagValue9"
+                },
+                "Type": "AwsEc2Instance"
+            },
+            {
+                "Details": {
+                    "AwsIamAccessKey": {
+                        "PrincipalId": "GeneratedFindingPrincipalId",
+                        "PrincipalName": "GeneratedFindingUserName",
+                        "PrincipalType": "IAMUser"
+                    }
+                },
+                "Id": "AWS::IAM::AccessKey:GeneratedFindingAccessKeyId",
+                "Partition": "aws",
+                "Region": "eu-west-1",
+                "Type": "AwsIamAccessKey"
+            }
+        ],
+        "SchemaVersion": "2018-10-08",
+        "Severity": {
+            "Label": "MEDIUM",
+            "Normalized": 50,
+            "Product": 5
+        },
+        "SourceUrl": "https://eu-west-1.console.aws.amazon.com/guardduty/home?region=eu-west-1#/findings?macros=current&fId=d6badb90e557d4bd811488a53ca89895",
+        "Title": "Unusual changes to network permissions by GeneratedFindingUserName.",
+        "Types": [
+            "TTPs/Persistence/Persistence:IAMUser-NetworkPermissions",
+            "Unusual Behaviors/User/Persistence:IAMUser-NetworkPermissions"
+        ],
+        "UpdatedAt": "2020-11-28T23:11:30.811Z",
+        "Workflow": {
+            "Status": "NEW"
+        },
+        "WorkflowState": "NEW"
+    }
+]