Merge branch 'master' into auditor

.gitignore

@@ -15,6 +15,7 @@ resource_stats.png
 data/
 private_commands/
 output/
+.vscode/
 Pipfile.lock
 auditor/node_modules
 auditor/.cdk.staging

cloudmapper.py

@@ -30,7 +30,7 @@
 import pkgutil
 import importlib
 
-__version__ = "2.6.5"
+__version__ = "2.6.6"
 
 
 def show_help(commands):

commands/prepare.py

@@ -101,33 +101,33 @@ def get_subnets(az):
 
 def get_ec2s(region):
     instances = query_aws(region.account, "ec2-describe-instances", region.region)
-    resource_filter = '.Reservations[].Instances[] | select(.State.Name == "running")'
+    resource_filter = '.Reservations[]?.Instances[] | select(.State.Name == "running")'
     return pyjq.all(resource_filter, instances)
 
 
 def get_elbs(region):
     load_balancers = query_aws(
         region.account, "elb-describe-load-balancers", region.region
     )
-    return pyjq.all(".LoadBalancerDescriptions[]", load_balancers)
+    return pyjq.all(".LoadBalancerDescriptions[]?", load_balancers)
 
 
 def get_elbv2s(region):
     # ALBs and NLBs
     load_balancers = query_aws(
         region.account, "elbv2-describe-load-balancers", region.region
     )
-    return pyjq.all(".LoadBalancers[]", load_balancers)
+    return pyjq.all(".LoadBalancers[]?", load_balancers)
 
 
 def get_vpc_endpoints(region):
     endpoints = query_aws(region.account, "ec2-describe-vpc-endpoints", region.region)
-    return pyjq.all(".VpcEndpoints[]", endpoints)
+    return pyjq.all(".VpcEndpoints[]?", endpoints)
 
 
 def get_rds_instances(region):
     instances = query_aws(region.account, "rds-describe-db-instances", region.region)
-    return pyjq.all(".DBInstances[]", instances)
+    return pyjq.all(".DBInstances[]?", instances)
 
 
 def get_ecs_tasks(region):
@@ -150,18 +150,18 @@ def get_ecs_tasks(region):
 
 def get_lambda_functions(region):
     functions = query_aws(region.account, "lambda-list-functions", region.region)
-    return pyjq.all(".Functions[]|select(.VpcConfig!=null)", functions)
+    return pyjq.all(".Functions[]?|select(.VpcConfig!=null)", functions)
 
 
 def get_redshift(region):
     clusters = query_aws(region.account, "redshift-describe-clusters", region.region)
-    return pyjq.all(".Clusters[]", clusters)
+    return pyjq.all(".Clusters[]?", clusters)
 
 
 def get_elasticsearch(region):
     es_domains = []
     domain_json = query_aws(region.account, "es-list-domain-names", region.region)
-    domains = pyjq.all(".DomainNames[]", domain_json)
+    domains = pyjq.all(".DomainNames[]?", domain_json)
     for domain in domains:
         es = get_parameter_file(
             region, "es", "describe-elasticsearch-domain", domain["DomainName"]

config/audit_config_override.yaml.example

@@ -8,3 +8,10 @@
 # S3_PUBLIC_POLICY_GETOBJECT_ONLY:
 #   # This is an array of regexes, and must match the entire string
 #   ignore_resources: [".*demo"]
+
+# Example 3: Use a custom auditor to add your own auditing functions
+# CUSTOM_GUARDDUTY_OFF:
+#   title: "[Custom] GuardDuty is not enabled"
+#   description: GuardDuty is an AWS threat detection service that detects compromised access keys, EC2 instances, and more. It should be enabled in all regions.
+#   severity: Medium
+#   group: GuardDuty

config/custom_auditor.py

@@ -0,0 +1,69 @@
+from __future__ import print_function
+from shared.query import query_aws, get_parameter_file
+from shared.common import Finding
+
+# To use custom auditing, you must copy this file to ./private_commands/custom_auditory.py
+# and uncomment and modify the functions below.
+
+__description__ = "Custom auditing functions"
+
+# def custom_audit_guardduty(findings, region):
+#     """
+#     A custom auditor must be named a method that starts with "custom_audit_"
+#     You can create your own private auditors without needing to fork the project.
+#     You can mute the real ones, such as GUARDDUTY_OFF, and use your own.
+#     Note that you must add your own ID's and other info to audit_config_override.yaml
+#     """
+
+#     #
+#     # Custom logic
+#     #
+#     if region.name not in ['ap-southeast-1']:
+#         return
+
+#     #
+#     # The rest of this function is the same as real one
+#     #
+#     detector_list_json = query_aws(
+#         region.account, "guardduty-list-detectors", region
+#     )
+#     if not detector_list_json:
+#         # GuardDuty must not exist in this region (or the collect data is old)
+#         return
+#     is_enabled = False
+#     for detector in detector_list_json["DetectorIds"]:
+#         detector_json = get_parameter_file(
+#             region, "guardduty", "get-detector", detector
+#         )
+#         if detector_json["Status"] == "ENABLED":
+#             is_enabled = True
+#     if not is_enabled:
+#         findings.add(Finding(region, "CUSTOM_GUARDDUTY_OFF", None, None))
+
+
+# def custom_filter(finding, conf):
+#     """
+#     Return True if the finding should be filtered, or false to allow it.
+#     """
+
+#     # This filters the GUARDDUTY_OFF rule, so that it only alerts if GuardDuty 
+#     # is off in ap-southeast-1.  This is useful if you have SCPs or similar to
+#     # disable other regions and therefore only care about issues in ap-southeast-1.
+#     if finding.issue_id == 'GUARDDUTY_OFF' and finding.region.name not in ['ap-southeast-1']:
+#         return True
+
+#     # Accessible data includes:
+#     # - finding.issue_id: Ex. "GUARDDUTY_OFF"
+#     # - finding.region.name: Ex. "ap-southeast-1"
+#     # - finding.region.account.name: The account name: Ex. "prod"
+#     # - finding.region.account.local_id: The Accoutn ID: Ex. "000000000000"
+#     # - finding.resource_id: Ex. the S3 bucket name
+#     # - finding.resource_details: The dictionary associated with the finding
+#     #     The resource_details are finding type specific.
+#     # - conf["severity"]
+#     # - conf["title"]
+#     # - conf["description"]
+
+#     # Note that you could also modify the title, severity, etc.
+
+#     return False

shared/audit.py

@@ -4,6 +4,9 @@
 import pyjq
 import traceback
 import re
+import pkgutil
+import importlib
+import inspect
 
 from policyuniverse.policy import Policy
 
@@ -18,9 +21,12 @@
     days_between,
 )
 from shared.query import query_aws, get_parameter_file
-from shared.nodes import Account, Region
+from shared.nodes import Account, Region, get_name
 from shared.iam_audit import find_admins_in_account
 
+# Global
+custom_filter = None
+
 
 class Findings(object):
     findings = None
@@ -49,6 +55,9 @@ def finding_is_filtered(finding, conf):
         if re.search(ignore_regex, finding.resource_id):
             return True
 
+    if custom_filter and custom_filter(finding, conf):
+        return True
+
     return False
 
 
@@ -64,6 +73,13 @@ def load_audit_config():
             # Over-write the values from audit_config
             if audit_override:
                 for finding_id in audit_override:
+                    if finding_id not in audit_config:
+                        audit_config[finding_id] = {
+                            "title": "Unknown",
+                            "description": "Unknown",
+                            "severity": "High",
+                            "group": "unknown",
+                        }
                     for k in audit_override[finding_id]:
                         audit_config[finding_id][k] = audit_override[finding_id][k]
     return audit_config
@@ -169,23 +185,19 @@ def audit_s3_block_policy(findings, region):
 
 
 def audit_guardduty(findings, region):
-    for region_json in get_regions(region.account):
-        region = Region(region.account, region_json)
-        detector_list_json = query_aws(
-            region.account, "guardduty-list-detectors", region
+    detector_list_json = query_aws(region.account, "guardduty-list-detectors", region)
+    if not detector_list_json:
+        # GuardDuty must not exist in this region (or the collect data is old)
+        return
+    is_enabled = False
+    for detector in detector_list_json["DetectorIds"]:
+        detector_json = get_parameter_file(
+            region, "guardduty", "get-detector", detector
         )
-        if not detector_list_json:
-            # GuardDuty must not exist in this region (or the collect data is old)
-            continue
-        is_enabled = False
-        for detector in detector_list_json["DetectorIds"]:
-            detector_json = get_parameter_file(
-                region, "guardduty", "get-detector", detector
-            )
-            if detector_json["Status"] == "ENABLED":
-                is_enabled = True
-        if not is_enabled:
-            findings.add(Finding(region, "GUARDDUTY_OFF", None, None))
+        if detector_json["Status"] == "ENABLED":
+            is_enabled = True
+    if not is_enabled:
+        findings.add(Finding(region, "GUARDDUTY_OFF", None, None))
 
 
 def audit_iam(findings, region):
@@ -672,7 +684,11 @@ def audit_ec2(findings, region):
                             region,
                             "EC2_OLD",
                             instance["InstanceId"],
-                            resource_details={"Age in days": age_in_days},
+                            resource_details={
+                                "Age in days": age_in_days,
+                                "Name": get_name(instance, "InstanceId"),
+                                "Tags": instance.get("Tags", {}),
+                            },
                         )
                     )
 
@@ -695,7 +711,11 @@ def audit_ec2(findings, region):
                         region,
                         "EC2_SOURCE_DEST_CHECK_OFF",
                         instance["InstanceId"],
-                        resource_details={"routes": route_to_instance},
+                        resource_details={
+                            "routes": route_to_instance,
+                            "Name": get_name(instance, "InstanceId"),
+                            "Tags": instance.get("Tags", {}),
+                        },
                     )
                 )
 
@@ -968,11 +988,26 @@ def audit_lightsail(findings, region):
 def audit(accounts):
     findings = Findings()
 
+    custom_auditor = None
+    commands_path = "private_commands"
+    for importer, command_name, _ in pkgutil.iter_modules([commands_path]):
+        if "custom_auditor" != command_name:
+            continue
+
+        full_package_name = "%s.%s" % (commands_path, command_name)
+        custom_auditor = importlib.import_module(full_package_name)
+
+        for name, method in inspect.getmembers(custom_auditor, inspect.isfunction):
+            if name.startswith("custom_filter"):
+                global custom_filter
+                custom_filter = method
+
     for account in accounts:
         account = Account(None, account)
 
         for region_json in get_regions(account):
             region = Region(account, region_json)
+
             try:
                 if region.name == "us-east-1":
                     audit_s3_buckets(findings, region)
@@ -984,7 +1019,7 @@ def audit(accounts):
                     audit_route53(findings, region)
                     audit_cloudfront(findings, region)
                     audit_s3_block_policy(findings, region)
-                    audit_guardduty(findings, region)
+                audit_guardduty(findings, region)
                 audit_ebs_snapshots(findings, region)
                 audit_rds_snapshots(findings, region)
                 audit_rds(findings, region)
@@ -1012,4 +1047,26 @@ def audit(accounts):
                         },
                     )
                 )
+
+            # Run custom auditor if it exists
+            try:
+                if custom_auditor is not None:
+                    for name, method in inspect.getmembers(
+                        custom_auditor, inspect.isfunction
+                    ):
+                        if name.startswith("custom_audit_"):
+                            method(findings, region)
+            except Exception as e:
+                findings.add(
+                    Finding(
+                        region,
+                        "EXCEPTION",
+                        str(e),
+                        resource_details={
+                            "exception": str(e),
+                            "traceback": str(traceback.format_exc()),
+                        },
+                    )
+                )
+
     return findings

shared/find_unused.py

@@ -86,6 +86,14 @@ def find_unused_network_interfaces(region):
 
     return unused_network_interfaces
 
+def find_unused_elastic_load_balancers(region):
+    unused_elastic_load_balancers = []
+    elastic_load_balancers = query_aws(region.account, "elb-describe-load-balancers", region)
+    for elastic_load_balancer in pyjq.all(".LoadBalancerDescriptions[] | select(.Instances == [])", elastic_load_balancers):
+        unused_elastic_load_balancers.append({"LoadBalancerName": elastic_load_balancer["LoadBalancerName"]})
+
+    return unused_elastic_load_balancers
+
 
 def add_if_exists(dictionary, key, value):
     if value:
@@ -119,6 +127,11 @@ def find_unused_resources(accounts):
                 "network_interfaces",
                 find_unused_network_interfaces(region),
             )
+            add_if_exists(
+                unused_resources_for_region,
+                "elastic_load_balancers",
+                find_unused_elastic_load_balancers(region),
+            )
 
             unused_resources_for_account.append(
                 {

shared/iam_audit.py

@@ -324,7 +324,7 @@ def find_admins_in_account(
                                 },
                             )
                         )
-            elif stmt["Action"] == "sts:AssumeRoleWithSAML":
+            elif stmt["Action"] in ["sts:AssumeRoleWithSAML", "sts:AssumeRoleWithWebIdentity"]:
                 continue
             else:
                 findings.add(