From a5a0cb1177531fc4824dfaa18c0c99ff684c2344 Mon Sep 17 00:00:00 2001 From: dolevf Date: Tue, 6 Oct 2020 21:58:37 -0400 Subject: [PATCH] clean up, standardize HTTP requests --- bin/scanner.py | 15 ++-- core/reports.py | 1 - core/triage.py | 83 ++++++++++++-------- core/utils.py | 1 - rules/bruteforce/rule_ftp-bf.py | 2 - rules/bruteforce/rule_mongodb-bf.py | 6 +- rules/bruteforce/rule_mysql-bf.py | 2 - rules/bruteforce/rule_psql-bf.py | 4 +- rules/bruteforce/rule_redis-bf.py | 5 +- rules/configuration/rule_cors-null-origin.py | 2 +- rules/configuration/rule_cors-wildcard.py | 2 +- rules/configuration/rule_ssh-auth-check.py | 3 +- 12 files changed, 69 insertions(+), 57 deletions(-) diff --git a/bin/scanner.py b/bin/scanner.py index bae5c1f..ff4a4c4 100644 --- a/bin/scanner.py +++ b/bin/scanner.py @@ -1,12 +1,11 @@ import time from core.redis import rds -from core.utils import Utils from core.logging import logger from core.port_scanner import Scanner +from core.parser import ConfParser def scanner(): - utils = Utils() scanner = Scanner() logger.info('Scanner process started') @@ -16,20 +15,22 @@ def scanner(): time.sleep(10) continue - conf = rds.get_scan_config() + conf = rds.get_scan_config() if not conf: time.sleep(10) continue - hosts = rds.get_ips_to_scan(limit = conf['config']['scan_opts']['parallel_scan']) + c = ConfParser(conf) + + hosts = rds.get_ips_to_scan(limit = c.get_cfg_scan_threads()) if hosts: conf = rds.get_scan_config() scan_data = scanner.scan(hosts, - max_ports = conf['config']['scan_opts']['max_ports'], - custom_ports = conf['config']['scan_opts']['custom_ports'], - interface = conf['config']['scan_opts']['interface']) + max_ports = c.get_cfg_max_ports(), + custom_ports = c.get_cfg_custom_ports(), + interface = c.get_cfg_netinterface()) if scan_data: for host, values in scan_data.items(): diff --git a/core/reports.py b/core/reports.py index 01c4d14..edb563d 100644 --- a/core/reports.py +++ b/core/reports.py @@ -91,7 +91,6 @@ def generate_xml(vulns): description = xml.SubElement(vuln_element, "description") description.text = value['rule_desc'] - confirm = xml.SubElement(vuln_element, "confirm") confirm.text = value['rule_confirm'] diff --git a/core/triage.py b/core/triage.py index 2b5a528..bb02332 100644 --- a/core/triage.py +++ b/core/triage.py @@ -16,22 +16,45 @@ class Triage: def __init__(self): + self.global_timeout = 10 self.headers = { 'User-Agent':USER_AGENT } - def http_request(self, ip, port, headers=None, follow_redirects=True, uri='/'): + def http_request(self, ip, port, method="GET", params=None, data=None, json=None, headers=None, follow_redirects=True, timeout=None, uri='/'): resp = None if headers: self.headers = {**headers, **self.headers} + if method not in ('GET', 'POST', 'OPTIONS', 'PUT', 'DELETE', 'HEAD'): + logger.error('HTTP Method is not supported.') + return + + if not timeout: + timeout = self.global_timeout + url = 'http://{}:{}{}'.format(ip, port, uri) if port == 443 or port == 8443 or '443' in str(port): url = 'https://{}:{}{}'.format(ip, port, uri) try: - resp = requests.get(url, verify=False, timeout=8, allow_redirects=follow_redirects, headers=self.headers) + if method == 'GET': + resp = requests.get(url, verify=False, timeout=timeout, params=params, allow_redirects=follow_redirects, headers=self.headers) + elif method == 'PUT': + resp = requests.put(url, verify=False, timeout=timeout, params=params, data=data, json=json, allow_redirects=follow_redirects, headers=self.headers) + elif method == 'POST': + resp = requests.post(url, verify=False, timeout=timeout, params=params, data=data, json=json, allow_redirects=follow_redirects, headers=self.headers) + elif method == 'OPTIONS': + resp = requests.options(url, verify=False, timeout=timeout, params=params, allow_redirects=follow_redirects, headers=self.headers) + elif method == 'DELETE': + resp = requests.delete(url, verify=False, timeout=timeout, params=params, data=data, json=json, allow_redirects=follow_redirects, headers=self.headers) + elif method == 'HEAD': + resp = requests.head(url, verify=False, timeout=timeout, params=params, allow_redirects=follow_redirects, headers=self.headers) + else: + # Default to GET. + resp = requests.get(url, verify=False, timeout=timeout, params=params, data=data, json=json, allow_redirects=follow_redirects, headers=self.headers) + except requests.exceptions.ConnectTimeout: logger.debug('http_request {} {} (Timeout)'.format(ip, port)) @@ -60,31 +83,38 @@ def string_in_headers(self, resp, string): return resp return False - def socket_banner(self, ip, port): + def get_tcp_socket_banner(self, ip, port, timeout=None): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) socket_banner = None - sock.settimeout(6) + if not timeout: + timeout = self.global_timeout + sock.settimeout(timeout) try: result = sock.connect_ex((ip, port)) if result == 0: socket_banner = str(sock.recv(1024)) - except Exception as e: - logger.debug('socket_open banner {} {} {}'.format(ip, port, e)) + except: + pass finally: sock.close() return socket_banner - def socket_open(self, ip, port): + def is_socket_open(self, ip, port, timeout=None): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(6) + + if not timeout: + timeout = self.global_timeout + + sock.settimeout(timeout) + try: result = sock.connect_ex((ip, port)) if result == 0: return True - except Exception as e: + except: pass finally: @@ -92,14 +122,6 @@ def socket_open(self, ip, port): return False - def is_ssh(self, ip, port): - is_ssh = False - banner = self.socket_banner(ip, port) - if banner and 'SSH' in str(banner): - is_ssh = True - - return is_ssh - def run_cmd(self, command): result = None p = Popen(shlex.split(command), stdin=PIPE, stdout=PIPE, stderr=PIPE) @@ -115,18 +137,17 @@ def has_cves(self, cpe): if not any(char.isdigit() for char in cpe): return False - try: - req = requests.get('https://nvd.nist.gov/vuln/search/results?form_type=Advanced&cves=on&cpe_version={}'.format(cpe), verify=False, timeout=5) - if req: - soup = BeautifulSoup(req.text, 'html.parser') - for a in soup.find_all('a', href=True): - if a.has_attr('data-testid') and a.contents: - sevs = ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL'] - if any(word in a.contents[0] for word in sevs): - score, sev = a.contents[0].split() - if float(score) >= 8.9: - return True - except: - pass - + req = self.http_request('nvd.nist.gov', 443, method="GET", uri='/vuln/search/results?form_type=Advanced&cves=on&cpe_version=' + cpe) + if not req: + return + + soup = BeautifulSoup(req.text, 'html.parser') + for a in soup.find_all('a', href=True): + if a.has_attr('data-testid') and a.contents: + sevs = ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL'] + if any(word in a.contents[0] for word in sevs): + score, sev = a.contents[0].split() + if float(score) >= 8.9: + return True + return False diff --git a/core/utils.py b/core/utils.py index 7904877..2eab7dc 100644 --- a/core/utils.py +++ b/core/utils.py @@ -99,7 +99,6 @@ def is_valid_port(self, port): except TypeError: return False - def get_primary_ip(self): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: diff --git a/rules/bruteforce/rule_ftp-bf.py b/rules/bruteforce/rule_ftp-bf.py index 978e761..5a3849b 100644 --- a/rules/bruteforce/rule_ftp-bf.py +++ b/rules/bruteforce/rule_ftp-bf.py @@ -1,7 +1,6 @@ import ftplib from core.redis import rds -from core.triage import Triage from core.parser import ScanParser, ConfParser from db.db_passwds import known_weak from db.db_users import known_users @@ -33,7 +32,6 @@ def ftp_attack(self, ip, username, password): def check_rule(self, ip, port, values, conf): c = ConfParser(conf) - t = Triage() p = ScanParser(port, values) domain = p.get_domain() diff --git a/rules/bruteforce/rule_mongodb-bf.py b/rules/bruteforce/rule_mongodb-bf.py index 8d93254..4aff3d9 100644 --- a/rules/bruteforce/rule_mongodb-bf.py +++ b/rules/bruteforce/rule_mongodb-bf.py @@ -1,6 +1,5 @@ from pymongo import MongoClient from core.redis import rds -from core.triage import Triage from core.parser import ScanParser, ConfParser from db.db_passwds import known_weak from db.db_users import known_users @@ -27,7 +26,7 @@ def mongodb_attack(self, ip, port, username, password): MongoClient(ip, port).list_database_names() return True - except Exception as e: + except: return else: @@ -37,12 +36,11 @@ def mongodb_attack(self, ip, port, username, password): MongoClient('mongodb://{username}:{password}@{ip}/admin'.format(username=username, password=password, ip=ip)) return True - except Exception as e: + except: return def check_rule(self, ip, port, values, conf): c = ConfParser(conf) - t = Triage() p = ScanParser(port, values) domain = p.get_domain() diff --git a/rules/bruteforce/rule_mysql-bf.py b/rules/bruteforce/rule_mysql-bf.py index 5de54af..c784cbb 100644 --- a/rules/bruteforce/rule_mysql-bf.py +++ b/rules/bruteforce/rule_mysql-bf.py @@ -1,7 +1,6 @@ import mysql.connector from core.redis import rds -from core.triage import Triage from core.parser import ScanParser, ConfParser from db.db_passwds import known_weak from db.db_users import known_users @@ -31,7 +30,6 @@ def mysql_attack(self, ip, username, password): def check_rule(self, ip, port, values, conf): c = ConfParser(conf) - t = Triage() p = ScanParser(port, values) domain = p.get_domain() diff --git a/rules/bruteforce/rule_psql-bf.py b/rules/bruteforce/rule_psql-bf.py index d17e37c..e8b8c92 100644 --- a/rules/bruteforce/rule_psql-bf.py +++ b/rules/bruteforce/rule_psql-bf.py @@ -1,6 +1,5 @@ import psycopg2 from core.redis import rds -from core.triage import Triage from core.parser import ScanParser, ConfParser from db.db_passwds import known_weak from db.db_users import known_users @@ -24,14 +23,13 @@ def psql_attack(self, ip, username, password): connection.close() return True - except Exception as e: + except: return return False def check_rule(self, ip, port, values, conf): c = ConfParser(conf) - t = Triage() p = ScanParser(port, values) domain = p.get_domain() diff --git a/rules/bruteforce/rule_redis-bf.py b/rules/bruteforce/rule_redis-bf.py index 76f5371..e647a2c 100644 --- a/rules/bruteforce/rule_redis-bf.py +++ b/rules/bruteforce/rule_redis-bf.py @@ -1,10 +1,10 @@ import socket from core.redis import rds -from core.triage import Triage from core.parser import ScanParser, ConfParser from db.db_passwds import known_weak from core.logging import logger + class Rule: def __init__(self): self.rule = 'BRF_DD00' @@ -36,7 +36,6 @@ def redis_attack(self, ip, port, password): def check_rule(self, ip, port, values, conf): c = ConfParser(conf) - t = Triage() p = ScanParser(port, values) domain = p.get_domain() @@ -72,7 +71,7 @@ def check_rule(self, ip, port, values, conf): }) return except: - return + pass return diff --git a/rules/configuration/rule_cors-null-origin.py b/rules/configuration/rule_cors-null-origin.py index db3c420..7f38cbc 100644 --- a/rules/configuration/rule_cors-null-origin.py +++ b/rules/configuration/rule_cors-null-origin.py @@ -7,7 +7,7 @@ def __init__(self): self.rule = 'CFG_32A0' self.rule_severity = 1 self.rule_description = 'This rule checks if Cross Origin Resource Sharing policy trusts null origins' - self.rule_confirm = 'CORS Allows Null Origins' + self.rule_confirm = 'CORS Policy Allows Null Origins' self.rule_details = '' self.rule_mitigation = '''Consider hardening your Cross Origin Resource Sharing Policy to define specific Origins \ https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS''' diff --git a/rules/configuration/rule_cors-wildcard.py b/rules/configuration/rule_cors-wildcard.py index 23f9538..2558b2a 100644 --- a/rules/configuration/rule_cors-wildcard.py +++ b/rules/configuration/rule_cors-wildcard.py @@ -7,7 +7,7 @@ def __init__(self): self.rule = 'CFG_DFFF' self.rule_severity = 1 self.rule_description = 'This rule checks if Cross Origin Resource Sharing Headers support Wildcard Origins' - self.rule_confirm = 'Webserver is allowing all domains in CORS' + self.rule_confirm = 'CORS Policy allows any domain' self.rule_details = '' self.rule_mitigation = '''Consider hardening your Cross Origin Resource Sharing Policy to define specific Origins \ https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS''' diff --git a/rules/configuration/rule_ssh-auth-check.py b/rules/configuration/rule_ssh-auth-check.py index 503308d..43528da 100644 --- a/rules/configuration/rule_ssh-auth-check.py +++ b/rules/configuration/rule_ssh-auth-check.py @@ -19,8 +19,9 @@ def check_rule(self, ip, port, values, conf): p = ScanParser(port, values) domain = p.get_domain() + module = p.get_module() - if port in ssh_ports and t.is_ssh(ip, port): + if port in ssh_ports and 'ssh' in module.lower(): output = t.run_cmd('ssh -o PreferredAuthentications=none -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o NoHostAuthenticationForLocalhost=yes user@"{}" -p "{}"'.format(ip, port)) if output and 'password' in str(output): self.rule_details = 'Server accepts passwords as an authentication option'