diff --git a/.gitignore b/.gitignore index a59fa299d..40bb3ce50 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ venv/ web/data.json .coverage htmlcov/ +account-data/ diff --git a/cloudmapper.py b/cloudmapper.py index 57d7f22cd..d6bb7bcda 100755 --- a/cloudmapper.py +++ b/cloudmapper.py @@ -23,184 +23,49 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------- -This script manages CloudMapper, a tool for creating network diagrams of AWS environments. +This script manages CloudMapper, a tool for analyzing AWS environments. """ from __future__ import (absolute_import, division, print_function) import json import argparse import sys -from cloudmapper.webserver import run_webserver +import pkgutil +import commands -__version__ = "1.0.0" +__version__ = "2.0.0" -def get_account(account_name, config, config_filename): - for account in config["accounts"]: - if account["name"] == account_name: - return account - if account_name is None and account.get("default", False): - return account - - # Else could not find account - if account_name is None: - exit("ERROR: Must specify an account, or set one in {} as a default".format(config_filename)) - exit("ERROR: Account named \"{}\" not found in {}".format(account_name, config_filename)) - - -def run_gathering(arguments): - from cloudmapper.gatherer import gather - parser = argparse.ArgumentParser() - parser.add_argument("--config", help="Config file name", - default="config.json", type=str) - parser.add_argument("--account-name", help="Account to collect from", - required=False, type=str) - parser.add_argument("--profile-name", help="AWS profile name", - required=False, type=str) - parser.add_argument('--clean', help='Remove any existing data for the account before gathering', action='store_true') - - args = parser.parse_args(arguments) - - if not args.account_name: - try: - config = json.load(open(args.config)) - except IOError: - exit("ERROR: Unable to load config file \"{}\"".format(args.config)) - except ValueError as e: - exit("ERROR: Config file \"{}\" could not be loaded ({}), see config.json.demo for an example".format(args.config, e)) - args.account_name = get_account(args.account_name, config, args.config)['name'] - - gather(args) - - -def run_configure(arguments): - from cloudmapper.configure import configure - if len(arguments) == 0: - exit("ERROR: Missing action for configure. Should be in {add-cidr|add-account|remove-cidr|remove-account}") - return - action = arguments[0] - arguments = arguments[1:] - parser = argparse.ArgumentParser() - parser.add_argument("--config-file", help="Path to the config file", - default="config.json", type=str) - if action == 'add-account' or action == 'remove-account': - required = True if action.startswith('add') else False - parser.add_argument("--name", help="Account name", - required=required, type=str) - parser.add_argument("--id", help="Account ID", - required=required, type=str) - parser.add_argument("--default", help="Default account", - required=False, default="False", type=str) - elif action == 'add-cidr' or action == 'remove-cidr': - required = True if action.startswith('add') else False - parser.add_argument("--cidr", help="CIDR IP", - required=required, type=str) - parser.add_argument("--name", help="CIDR Name", - required=required, type=str) - args = parser.parse_args(arguments) - configure(action, args) - - -def run_prepare(arguments): - from cloudmapper.prepare import prepare - - # Parse arguments - parser = argparse.ArgumentParser() - parser.add_argument("--config", help="Config file name", - default="config.json", type=str) - parser.add_argument("--account-name", help="Account to collect from", - required=False, type=str) - parser.add_argument("--regions", help="Regions to restrict to (ex. us-east-1,us-west-2)", - default=None, type=str) - parser.add_argument("--vpc-ids", help="VPC ids to restrict to (ex. vpc-1234,vpc-abcd)", - default=None, type=str) - parser.add_argument("--vpc-names", help="VPC names to restrict to (ex. prod,dev)", - default=None, type=str) - parser.add_argument("--internal-edges", help="Show all connections (default)", - dest='internal_edges', action='store_true') - parser.add_argument("--no-internal-edges", help="Only show connections to external CIDRs", - dest='internal_edges', action='store_false') - parser.add_argument("--inter-rds-edges", help="Show connections between RDS instances", - dest='inter_rds_edges', action='store_true') - parser.add_argument("--no-inter-rds-edges", help="Do not show connections between RDS instances (default)", - dest='inter_rds_edges', action='store_false') - parser.add_argument("--read-replicas", help="Show RDS read replicas (default)", - dest='read_replicas', action='store_true') - parser.add_argument("--no-read-replicas", help="Do not show RDS read replicas", - dest='read_replicas', action='store_false') - parser.add_argument("--azs", help="Show availability zones (default)", - dest='azs', action='store_true') - parser.add_argument("--no-azs", help="Do not show availability zones", - dest='azs', action='store_false') - parser.add_argument("--collapse-by-tag", help="Collapse nodes with the same tag to a single node", - dest='collapse_by_tag', default=None, type=str) - - parser.set_defaults(internal_edges=True) - parser.set_defaults(inter_rds_edges=False) - parser.set_defaults(read_replicas=True) - parser.set_defaults(azs=True) - - args = parser.parse_args(arguments) - - outputfilter = {} - if args.regions: - # Regions are given as 'us-east-1,us-west-2'. Split this by the comma, - # wrap each with quotes, and add the comma back. This is needed for how we do filtering. - outputfilter["regions"] = ','.join(['"' + r + '"' for r in args.regions.split(',')]) - if args.vpc_ids: - outputfilter["vpc-ids"] = ','.join(['"' + r + '"' for r in args.vpc_ids.split(',')]) - if args.vpc_names: - outputfilter["vpc-names"] = ','.join(['"' + r + '"' for r in args.vpc_names.split(',')]) - - outputfilter["internal_edges"] = args.internal_edges - outputfilter["read_replicas"] = args.read_replicas - outputfilter["inter_rds_edges"] = args.inter_rds_edges - outputfilter["azs"] = args.azs - outputfilter["collapse_by_tag"] = args.collapse_by_tag - - # Read accounts file - try: - config = json.load(open(args.config)) - except IOError: - exit("ERROR: Unable to load config file \"{}\"".format(args.config)) - except ValueError as e: - exit("ERROR: Config file \"{}\" could not be loaded ({}), see config.json.demo for an example".format(args.config, e)) - account = get_account(args.account_name, config, args.config) - - prepare(account, config, outputfilter) - - -def show_help(): +def show_help(commands): print("CloudMapper {}".format(__version__)) - print("usage: {} [gather|prepare|serve] [...]".format(sys.argv[0])) - print(" configure: Configure and create your config file") - print(" gather: Queries AWS for account data and caches it locally") - print(" prepare: Prepares the data for viewing") - print(" serve: Runs a local webserver for viewing the data") + print("usage: {} [{}] [...]".format(sys.argv[0], "|".join(sorted(commands.keys())))) + for command, module in sorted(commands.items()): + print(" {}: {}".format(command, module.__description__)) exit(-1) def main(): """Entry point for the CLI.""" - + + # Load commands + # TODO: This adds half a second to the start time. Is there a smarter way to do this? + commands = {} + commands_path = 'commands' + for importer, command_name, _ in pkgutil.iter_modules([commands_path]): + full_package_name = '%s.%s' % (commands_path, command_name) + module = importer.find_module(command_name + ).load_module(full_package_name) + commands[command_name] = module + # Parse command if len(sys.argv) <= 1: - show_help() - + show_help(commands) + command = sys.argv[1] arguments = sys.argv[2:] - if command == "prepare": - run_prepare(arguments) - elif command == "serve": - run_webserver(arguments) - elif command == "gather": - run_gathering(arguments) - elif command == "configure": - run_configure(arguments) + if command in commands: + commands[command].run(arguments) else: - show_help() - - print("Complete") - + show_help(commands) if __name__ == "__main__": main() diff --git a/cloudmapper/gatherer.py b/cloudmapper/gatherer.py deleted file mode 100644 index 47fd2e444..000000000 --- a/cloudmapper/gatherer.py +++ /dev/null @@ -1,118 +0,0 @@ -import datetime -import json -from os import mkdir, path -from shutil import rmtree -import boto3 - - -def datetime_handler(x): - if isinstance(x, datetime.datetime): - return x.isoformat() - raise TypeError("Unknown type") - - -def gather(arguments): - account_dir = './{}'.format(arguments.account_name) - - if arguments.clean and path.exists(account_dir): - rmtree(account_dir) - - try: - mkdir(account_dir) - except OSError: - # Already exists - pass - - print("* Getting region names") - session_data = {} - - if arguments.profile_name: - session_data['profile_name'] = arguments.profile_name - - session = boto3.Session(**session_data) - ec2 = session.client('ec2') - - region_list = ec2.describe_regions() - with open("{}/describe-regions.json".format(account_dir), 'w+') as f: - f.write(json.dumps(region_list, indent=4, sort_keys=True)) - - print("* Creating directory for each region name") - - for region in region_list['Regions']: - try: - mkdir('{}/{}'.format(account_dir, region.get('RegionName', 'Unknown'))) - except OSError: - # Already exists - pass - runners_list = [ - { - "Name": "VPC", - "Function": "describe_vpcs", - "Handler": "ec2", - "FileName": "describe-vpcs.json", - }, - { - "Name": "AZ", - "Function": "describe_availability_zones", - "Handler": "ec2", - "FileName": "describe-availability-zones.json", - }, - { - "Name": "Subnet", - "Function": "describe_subnets", - "Handler": "ec2", - "FileName": "describe-subnets.json", - }, - { - "Name": "EC2", - "Function": "describe_instances", - "Handler": "ec2", - "FileName": "describe-instances.json" - }, - { - "Name": "RDS", - "Function": "describe_db_instances", - "Handler": "rds", - "FileName": "describe-db-instances.json", - }, - { - "Name": "ELB", - "Function": "describe_load_balancers", - "Handler": "elb", - "FileName": "describe-load-balancers.json", - }, - { - "Name": "ALBs and NLBs", - "Function": "describe_load_balancers", - "Handler": "elbv2", - "FileName": "describe-load-balancers-v2.json" - }, - { - "Name": "Security Groups", - "Function": "describe_security_groups", - "Handler": "ec2", - "FileName": "describe-security-groups.json", - }, - { - "Name": "Network interface", - "Function": "describe_network_interfaces", - "Handler": "ec2", - "FileName": "describe-network-interfaces.json", - }, - { - "Name": "VPC Peering", - "Function": "describe_vpc_peering_connections", - "Handler": "ec2", - "FileName": "describe-vpc-peering-connections.json" - } - ] - - for runner in runners_list: - print("* Getting {} info".format(runner['Name'])) - for region in region_list['Regions']: - handler = boto3.client(runner['Handler'], region_name=region['RegionName']) - method_to_call = getattr(handler, runner["Function"]) - data = method_to_call() - data.pop('ResponseMetadata', None) - with open("{}/{}/{}".format(account_dir, region.get('RegionName', 'Unknown'), runner['FileName']), 'w+') as f: - f.write(json.dumps(data, indent=4, sort_keys=True, default=datetime_handler)) diff --git a/cloudmapper/__init__.py b/commands/__init__.py similarity index 100% rename from cloudmapper/__init__.py rename to commands/__init__.py diff --git a/commands/collect.py b/commands/collect.py new file mode 100644 index 000000000..5341f05df --- /dev/null +++ b/commands/collect.py @@ -0,0 +1,562 @@ +import datetime +import json +import os.path +import os +import argparse +from shared.common import get_account, datetime_handler +from shutil import rmtree +import boto3 +import pyjq +from botocore.exceptions import ClientError, EndpointConnectionError + +__description__ = "Run AWS API calls to collect data from the account" + + +def snakecase(s): + return s.replace('-', '_') + + +def get_identifier_from_parameter(parameter): + if isinstance(parameter, list): + identifier = parameter[0] + else: + identifier = parameter + + return identifier + +def get_filename_from_parameter(parameter): + if isinstance(parameter, list): + filename = parameter[1] + else: + filename = parameter + + return filename.replace('/', '-') + +def make_directory(path): + try: + os.mkdir(path) + except OSError: + # Already exists + pass + +def call_function(outputfile, handler, method_to_call, parameters): + """Calls the AWS API function and downloads the data""" + # TODO: Decorate this with rate limiters from + # https://github.com/Netflix-Skunkworks/cloudaux/blob/master/cloudaux/aws/decorators.py + + data = None + if os.path.isfile(outputfile): + # Data already collected, so skip + return + + print "Making call for {}".format(outputfile) + try: + if handler.can_paginate(method_to_call): + paginator = handler.get_paginator(method_to_call) + page_iterator = paginator.paginate(**parameters) + + for response in page_iterator: + if not data: + data = response + else: + print " ...paginating" + for k in data: + if isinstance(data[k], list): + data[k].extend(response[k]) + + else: + function = getattr(handler, method_to_call) + data = function(**parameters) + + except ClientError as e: + if "NoSuchBucketPolicy" in str(e): + pass + else: + print "ClientError: {}".format(e) + except EndpointConnectionError as e: + pass + + # Remove unused values + if data is not None: + data.pop('ResponseMetadata', None) + data.pop('Marker', None) + data.pop('IsTruncated', None) + + with open(outputfile, 'w+') as f: + f.write(json.dumps(data, indent=4, sort_keys=True, default=datetime_handler)) + + +def collect(arguments): + account_dir = './{}'.format(arguments.account_name) + + if arguments.clean and os.path.exists(account_dir): + rmtree(account_dir) + + make_directory("account-data") + make_directory("account-data/{}".format(account_dir)) + + print("* Getting region names") + session_data = {} + + if arguments.profile_name: + session_data['profile_name'] = arguments.profile_name + + session = boto3.Session(**session_data) + ec2 = session.client('ec2') + + region_list = ec2.describe_regions() + with open("account-data/{}/describe-regions.json".format(account_dir), 'w+') as f: + f.write(json.dumps(region_list, indent=4, sort_keys=True)) + + print("* Creating directory for each region name") + for region in region_list['Regions']: + make_directory('account-data/{}/{}'.format(account_dir, region.get('RegionName', 'Unknown'))) + + # Services that will only be queried in us-east-1 + universal_services = ['sts', 'iam', 'route53', 'route53domains', 's3'] + + runners_list = [ + { + # Put this first so the report can be downloaded later + "Service": "iam", + "Request": "generate-credential-report", + }, + { + "Service": "sts", + "Request": "get-caller-identity", + }, + { + "Service": "iam", + "Request": "get-account-authorization-details", + }, + { + "Service": "iam", + "Request": "get-account-password-policy", + }, + { + "Service": "iam", + "Request": "get-account-summary", + }, + { + "Service": "iam", + "Request": "list-account-aliases", + }, + { + "Service": "iam", + "Request": "get-credential-report", + }, + { + "Service": "iam", + "Request": "list-saml-providers", + }, + { + "Service": "iam", + "Request": "get-saml-provider", + "ParameterName": "SamlProviderArn", + "ParameterValue": "iam-list-saml-providers.json|.OpenIDConnectProviderList[]|.Arn", + }, + { + "Service": "iam", + "Request": "list-open-id-connect-providers", + }, + { + "Service": "iam", + "Request": "get-open-id-connect-providers", + "ParameterName": "SamlProviderArn", + "ParameterValue": "iam-list-open-id-connect-providers.json|.SAMLProviderList[]|.Arn", + }, + { + "Service": "s3", + "Request": "list-buckets" + }, + { + "Service": "s3", + "Request": "get-bucket-acl", + "ParameterName": "Bucket", + "ParameterValue": "s3-list-buckets.json|.Buckets[].Name", + }, + { + "Service": "s3", + "Request": "get-bucket-policy", + "ParameterName": "Bucket", + "ParameterValue": "s3-list-buckets.json|.Buckets[].Name", + }, + { + "Service": "route53", + "Request": "list-hosted-zones", + }, + { + "Service": "route53", + "Request": "list-resource-record-sets", + "ParameterName": "HostedZoneId", + "ParameterValue": "route53-list-hosted-zones.json|.HostedZones[]|[.Id,.Name]", + }, + { + "Service": "route53domains", + "Request": "list-domains", + }, + { + "Service": "ec2", + "Request": "describe-vpcs" + }, + { + "Service": "ec2", + "Request": "describe-availability-zones" + }, + { + "Service": "ec2", + "Request": "describe-subnets" + }, + { + "Service": "ec2", + "Request": "describe-instances" + }, + { + "Service": "ec2", + "Request": "describe-addresses" + }, + { + "Service": "cloudtrail", + "Request": "describe-trails" + }, + { + "Service": "rds", + "Request": "describe-db-instances" + }, + { + "Service": "rds", + "Request": "describe-db-snapshots" + }, + { + "Service": "rds", + "Request": "describe-db-snapshot-attributes", + "ParameterName": "DBSnapshotIdentifier", + "ParameterValue": "rds-describe-db-snapshots.json|.DBSnapshots[]|.DBSnapshotIdentifier", + }, + { + "Service": "elb", + "Request": "describe-load-balancers" + }, + { + "Service": "elb", + "Request": "describe-load-balancer-policies" + }, + { + "Service": "elbv2", + "Request": "describe-load-balancers" + }, + { + "Service": "redshift", + "Request": "describe-clusters" + }, + { + "Service": "sqs", + "Request": "list-queues" + }, + { + "Service": "sqs", + "Request": "get-queue-attributes", + "ParameterName": "QueueUrl", + "ParameterValue": "sqs-list-queues.json|.QueueUrls[]", + }, + { + "Service": "sns", + "Request": "list-topics" + }, + { + "Service": "ec2", + "Request": "describe-security-groups" + }, + { + "Service": "ec2", + "Request": "describe-network-interfaces" + }, + { + "Service": "ec2", + "Request": "describe-vpc-peering-connections" + }, + { + "Service": "directconnect", + "Request": "describe-connections" + }, + { + "Service": "autoscaling", + "Request": "describe-policies" + }, + { + "Service": "autoscaling", + "Request": "describe-auto-scaling-groups" + }, + { + "Service": "cloudformation", + "Request": "describe-stacks" + }, + { + "Service": "cloudformation", + "Request": "get-template", + "ParameterName": "StackName", + "ParameterValue": "cloudformation-describe-stacks.json|.Stacks[]|.StackName", + }, + { + "Service": "cloudformation", + "Request": "describe-stack-resources", + "ParameterName": "StackName", + "ParameterValue": "cloudformation-describe-stacks.json|.Stacks[]|.StackName", + }, + { + "Service": "cloudfront", + "Request": "list-distributions" + }, + { + "Service": "cloudsearch", + "Request": "describe-domains" + }, + { + "Service": "cloudsearch", + "Request": "describe-service-access-policies", + "ParameterName": "DomainName", + "ParameterValue": "cloudsearch-describe-domains.json|.DomainStatusList[]|.DomainName", + }, + { + "Service": "cloudwatch", + "Request": "describe-alarms" + }, + { + "Service": "config", + "Request": "describe-config-rules" + }, + { + "Service": "config", + "Request": "describe-configuration-recorders" + }, + { + "Service": "config", + "Request": "describe-delivery-channels" + }, + { + "Service": "ec2", + "Request": "describe-images", + "Parameters": [ + {"Name": "Owners", "Value": ["self"]} + ] + }, + { + "Service": "ec2", + "Request": "describe-network-acls" + }, + { + "Service": "ec2", + "Request": "describe-route-tables" + }, + { + "Service": "ec2", + "Request": "describe-flow-logs" + }, + { + "Service": "ec2", + "Request": "describe-snapshots", + "Parameters": [ + {"Name": "OwnerIds", "Value": ["self"]}, + {"Name": "RestorableByUserIds", "Value": ["all"]} + ] + }, + { + "Service": "ec2", + "Request": "describe-vpc-endpoint-connections" + }, + { + "Service": "ec2", + "Request": "describe-vpn-connections" + }, + { + "Service": "ec2", + "Request": "describe-vpn-gateways" + }, + { + "Service": "ecr", + "Request": "describe-repositories" + }, + { + "Service": "ecr", + "Request": "get-repository-policy", + "ParameterName": "repositoryName", + "ParameterValue": "ecr-describe-repositories.json|.repositories[]|.repositoryName" + }, + { + "Service": "elasticache", + "Request": "describe-cache-clusters" + }, + { + "Service": "elasticbeanstalk", + "Request": "describe-applications" + }, + { + "Service": "efs", + "Request": "describe-file-systems" + }, + { + "Service": "es", + "Request": "list-domain-names" + }, + { + "Service": "es", + "Request": "describe-elasticsearch-domain", + "ParameterName": "DomainName", + "ParameterValue": "es-list-domain-names.json|.DomainNames[]|.DomainName" + }, + { + "Service": "events", + "Request": "describe-event-bus" + }, + { + "Service": "events", + "Request": "list-rules" + }, + { + "Service": "firehose", + "Request": "list-delivery-streams" + }, + { + "Service": "firehose", + "Request": "describe-delivery-stream", + "ParameterName": "DeliveryStreamName", + "ParameterValue": "firehose-list-delivery-streams.json|.DeliveryStreamNames[]" + }, + { + "Service": "glacier", + "Request": "list-vaults", + "Parameters": [ + {"Name": "accountId", "Value": "-"} + ] + }, + { + "Service": "glacier", + "Request": "get-vault-access-policy", + "Parameters": [ + {"Name": "accountId", "Value": "-"} + ], + "ParameterName": "vaultName", + "ParameterValue": "glacier-list-vaults.json|.VaultList[]|.VaultName" + }, + { + "Service": "kms", + "Request": "list-keys" + }, + { + "Service": "kms", + "Request": "list-grants", + "ParameterName": "KeyId", + "ParameterValue": "kms-list-keys.json|.Keys[]|.KeyId" + }, + { + "Service": "kms", + "Request": "list-key-policies", + "ParameterName": "KeyId", + "ParameterValue": "kms-list-keys.json|.Keys[]|.KeyId" + }, + { + "Service": "kms", + "Request": "get-key-rotation-status", + "ParameterName": "KeyId", + "ParameterValue": "kms-list-keys.json|.Keys[]|.KeyId" + }, + { + "Service": "lambda", + "Request": "list-functions" + }, + { + "Service": "lambda", + "Request": "get-policy", + "ParameterName": "FunctionName", + "ParameterValue": "lambda-list-functins.json|.Functions[]|.FunctionName" + }, + { + "Service": "logs", + "Request": "describe-destinations" + }, + { + "Service": "logs", + "Request": "describe-log-groups" + }, + { + "Service": "logs", + "Request": "describe-resource-policies" + }, + ] + + for runner in runners_list: + print("* Getting {}:{} info".format(runner['Service'], runner['Request'])) + + parameters = {} + if runner.get('Parameters', False): + # TODO: I need to consolidate the Parameters and ParameterName/ParameterValue + # variables + for parameter in runner['Parameters']: + parameters[parameter['Name']] = parameter['Value'] + + for region in region_list['Regions']: + # Only call universal services in us-east-1 + if runner['Service'] in universal_services and region['RegionName'] != 'us-east-1': + continue + handler = boto3.client(runner['Service'], region_name=region['RegionName']) + + filepath = "account-data/{}/{}/{}-{}".format( + account_dir, + region['RegionName'], + runner['Service'], + runner['Request']) + + + method_to_call = snakecase(runner["Request"]) + if runner.get('ParameterName', False): + make_directory(filepath) + + parameter_file = runner['ParameterValue'].split('|')[0] + parameter_file = "account-data/{}/{}/{}".format(account_dir, region['RegionName'], parameter_file) + + if not os.path.isfile(parameter_file): + # The file where parameters are obtained from does not exist + continue + + with open(parameter_file, 'r') as f: + parameter_values = json.load(f) + pyjq_parse_string = '|'.join(runner['ParameterValue'].split('|')[1:]) + for parameter in pyjq.all(pyjq_parse_string, parameter_values): + data = "" + + filename = get_filename_from_parameter(parameter) + identifier = get_identifier_from_parameter(parameter) + parameters[runner['ParameterName']] = identifier + + outputfile = "{}/{}".format( + filepath, + filename) + + call_function(outputfile, handler, method_to_call, parameters) + else: + filepath = filepath+".json" + call_function(filepath, handler, method_to_call, parameters) + + +def run(arguments): + parser = argparse.ArgumentParser() + parser.add_argument("--config", help="Config file name", + default="config.json", type=str) + parser.add_argument("--account-name", help="Account to collect from", + required=False, type=str) + parser.add_argument("--profile-name", help="AWS profile name", + required=False, type=str) + parser.add_argument('--clean', help='Remove any existing data for the account before gathering', action='store_true') + + args = parser.parse_args(arguments) + + if not args.account_name: + try: + config = json.load(open(args.config)) + except IOError: + exit("ERROR: Unable to load config file \"{}\"".format(args.config)) + except ValueError as e: + exit("ERROR: Config file \"{}\" could not be loaded ({}), see config.json.demo for an example".format(args.config, e)) + args.account_name = get_account(args.account_name, config, args.config)['name'] + + collect(args) diff --git a/cloudmapper/configure.py b/commands/configure.py similarity index 57% rename from cloudmapper/configure.py rename to commands/configure.py index 5846b2fe3..b62688f43 100644 --- a/cloudmapper/configure.py +++ b/commands/configure.py @@ -1,6 +1,10 @@ import json -import netaddr import os.path +import netaddr +import argparse +from shared.common import get_account + +__description__ = "Add and remove items from the config file" def configure(action, arguments): @@ -47,3 +51,31 @@ def configure(action, arguments): with open(arguments.config_file, 'w+') as f: f.write(json.dumps(config, indent=4, sort_keys=True)) + + +def run(arguments): + if len(arguments) == 0: + exit("ERROR: Missing action for configure.\n" + "Usage: [add-cidr|add-account|remove-cidr|remove-account]") + return + action = arguments[0] + arguments = arguments[1:] + parser = argparse.ArgumentParser() + parser.add_argument("--config-file", help="Path to the config file", + default="config.json", type=str) + if action == 'add-account' or action == 'remove-account': + required = True if action.startswith('add') else False + parser.add_argument("--name", help="Account name", + required=required, type=str) + parser.add_argument("--id", help="Account ID", + required=required, type=str) + parser.add_argument("--default", help="Default account", + required=False, default="False", type=str) + elif action == 'add-cidr' or action == 'remove-cidr': + required = True if action.startswith('add') else False + parser.add_argument("--cidr", help="CIDR IP", + required=required, type=str) + parser.add_argument("--name", help="CIDR Name", + required=required, type=str) + args = parser.parse_args(arguments) + configure(action, args) diff --git a/cloudmapper/prepare.py b/commands/prepare.py similarity index 80% rename from cloudmapper/prepare.py rename to commands/prepare.py index 4c0c39c74..c28cd7b8c 100644 --- a/cloudmapper/prepare.py +++ b/commands/prepare.py @@ -23,24 +23,16 @@ --------------------------------------------------------------------------- """ +import os.path import json import itertools +import argparse +from shared.common import get_account, query_aws import pyjq -import os.path from netaddr import IPNetwork, IPAddress -from cloudmapper.nodes import Account, Region, Vpc, Az, Subnet, Ec2, Elb, Rds, Cidr, Connection - - -def query_aws(account, query, region=None): - if not region: - file_name = "{}/{}.json".format(account.name, query) - else: - file_name = "{}/{}/{}.json".format(account.name, region.name, query) - if os.path.isfile(file_name): - return json.load(open(file_name)) - else: - return {} +from shared.nodes import Account, Region, Vpc, Az, Subnet, Ec2, Elb, Rds, Cidr, Connection +__description__ = "Generate network connection information file" def get_regions(account, outputfilter): # aws ec2 describe-regions @@ -375,3 +367,68 @@ def prepare(account, config, outputfilter): with open('web/data.json', 'w') as outfile: json.dump(cytoscape_json, outfile, indent=4) +def run(arguments): + # Parse arguments + parser = argparse.ArgumentParser() + parser.add_argument("--config", help="Config file name", + default="config.json", type=str) + parser.add_argument("--account-name", help="Account to collect from", + required=False, type=str) + parser.add_argument("--regions", help="Regions to restrict to (ex. us-east-1,us-west-2)", + default=None, type=str) + parser.add_argument("--vpc-ids", help="VPC ids to restrict to (ex. vpc-1234,vpc-abcd)", + default=None, type=str) + parser.add_argument("--vpc-names", help="VPC names to restrict to (ex. prod,dev)", + default=None, type=str) + parser.add_argument("--internal-edges", help="Show all connections (default)", + dest='internal_edges', action='store_true') + parser.add_argument("--no-internal-edges", help="Only show connections to external CIDRs", + dest='internal_edges', action='store_false') + parser.add_argument("--inter-rds-edges", help="Show connections between RDS instances", + dest='inter_rds_edges', action='store_true') + parser.add_argument("--no-inter-rds-edges", help="Do not show connections between RDS instances (default)", + dest='inter_rds_edges', action='store_false') + parser.add_argument("--read-replicas", help="Show RDS read replicas (default)", + dest='read_replicas', action='store_true') + parser.add_argument("--no-read-replicas", help="Do not show RDS read replicas", + dest='read_replicas', action='store_false') + parser.add_argument("--azs", help="Show availability zones (default)", + dest='azs', action='store_true') + parser.add_argument("--no-azs", help="Do not show availability zones", + dest='azs', action='store_false') + parser.add_argument("--collapse-by-tag", help="Collapse nodes with the same tag to a single node", + dest='collapse_by_tag', default=None, type=str) + + parser.set_defaults(internal_edges=True) + parser.set_defaults(inter_rds_edges=False) + parser.set_defaults(read_replicas=True) + parser.set_defaults(azs=True) + + args = parser.parse_args(arguments) + + outputfilter = {} + if args.regions: + # Regions are given as 'us-east-1,us-west-2'. Split this by the comma, + # wrap each with quotes, and add the comma back. This is needed for how we do filtering. + outputfilter["regions"] = ','.join(['"' + r + '"' for r in args.regions.split(',')]) + if args.vpc_ids: + outputfilter["vpc-ids"] = ','.join(['"' + r + '"' for r in args.vpc_ids.split(',')]) + if args.vpc_names: + outputfilter["vpc-names"] = ','.join(['"' + r + '"' for r in args.vpc_names.split(',')]) + + outputfilter["internal_edges"] = args.internal_edges + outputfilter["read_replicas"] = args.read_replicas + outputfilter["inter_rds_edges"] = args.inter_rds_edges + outputfilter["azs"] = args.azs + outputfilter["collapse_by_tag"] = args.collapse_by_tag + + # Read accounts file + try: + config = json.load(open(args.config)) + except IOError: + exit("ERROR: Unable to load config file \"{}\"".format(args.config)) + except ValueError as e: + exit("ERROR: Config file \"{}\" could not be loaded ({}), see config.json.demo for an example".format(args.config, e)) + account = get_account(args.account_name, config, args.config) + + prepare(account, config, outputfilter) \ No newline at end of file diff --git a/cloudmapper/webserver.py b/commands/webserver.py similarity index 96% rename from cloudmapper/webserver.py rename to commands/webserver.py index 09272b30d..66f1f4197 100644 --- a/cloudmapper/webserver.py +++ b/commands/webserver.py @@ -29,6 +29,7 @@ from six.moves.BaseHTTPServer import HTTPServer from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler +__description__ = "Run a webserver to display network or wot map" class RootedHTTPServer(HTTPServer): def __init__(self, base_path, *args, **kwargs): @@ -47,8 +48,8 @@ def translate_path(self, path): words = [_f for _f in words if _f] path = self.base_path for word in words: - drive, word = os.path.splitdrive(word) - head, word = os.path.split(word) + _, word = os.path.splitdrive(word) + _, word = os.path.split(word) if word in (os.curdir, os.pardir): continue if '?' in word: diff --git a/requirements.txt b/requirements.txt index 661429ecd..1df70194b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,7 @@ pyjq==2.1.0 netaddr==0.7.19 six==1.10.0 boto3==1.5.32 +pandas==0.22.0 +matplotlib=2.2.2 +geoip2=2.8.0 +policyuniverse=1.1.0.1 diff --git a/shared/__init__.py b/shared/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/shared/common.py b/shared/common.py new file mode 100644 index 000000000..e22b0ad3a --- /dev/null +++ b/shared/common.py @@ -0,0 +1,81 @@ +from __future__ import print_function +import sys +import argparse +import json +import os +import pyjq + + +def datetime_handler(x): + if isinstance(x, datetime.datetime): + return x.isoformat() + raise TypeError("Unknown type") + + +def make_list(v): + if not isinstance(v, list): + return [v] + return v + + +def query_aws(account, query, region=None): + if not region: + file_name = 'account-data/{}/{}.json'.format(account.name, query) + else: + file_name = 'account-data/{}/{}/{}.json'.format(account.name, region.name, query) + if os.path.isfile(file_name): + return json.load(open(file_name)) + else: + return {} + + +def get_account(account_name, config, config_filename): + for account in config["accounts"]: + if account["name"] == account_name: + return account + if account_name is None and account.get("default", False): + return account + + # Else could not find account + if account_name is None: + exit("ERROR: Must specify an account, or set one in {} as a default".format(config_filename)) + exit("ERROR: Account named \"{}\" not found in {}".format(account_name, config_filename)) + + +def parse_arguments(arguments, parser=None): + """Returns (args, accounts, config)""" + if parser is None: + parser = argparse.ArgumentParser() + parser.add_argument("--config", help="Config file name", + default="config.json", type=str) + parser.add_argument("--accounts", help="Accounts to collect from", + required=True, type=str) + parser.add_argument("--log_level", help="Log level to record (DEBUG, INFO, WARN, ERROR)", + default="INFO", required=False, type=str) + args = parser.parse_args(arguments) + + global LOG_LEVEL + LOG_LEVEL = Severity.str_to_int(args.log_level) + + # Read accounts file + try: + config = json.load(open(args.config)) + except IOError: + exit("ERROR: Unable to load config file \"{}\"".format(args.config)) + except ValueError as e: + exit("ERROR: Config file \"{}\" could not be loaded ({}), see config.json.demo for an example".format(args.config, e)) + + # Get accounts + account_names = args.accounts.split(',') + accounts = [] + # TODO Need to be able to tag accounts into sets (ex. Prod, or by business unit) so the tag can be referenced + # as opposed to the individual account names. + for account_name in account_names: + if account_name == 'all': + for account in config["accounts"]: + accounts.append(account) + break + accounts.append(get_account(account_name, config, args.config)) + + + return (args, accounts, config) \ No newline at end of file diff --git a/cloudmapper/nodes.py b/shared/nodes.py similarity index 100% rename from cloudmapper/nodes.py rename to shared/nodes.py