Skip to content

Commit

Permalink
clean up, standardize HTTP requests
Browse files Browse the repository at this point in the history
  • Loading branch information
dolevf committed Oct 7, 2020
1 parent 2aa0e40 commit a5a0cb1
Show file tree
Hide file tree
Showing 12 changed files with 69 additions and 57 deletions.
15 changes: 8 additions & 7 deletions bin/scanner.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
import time

from core.redis import rds
from core.utils import Utils
from core.logging import logger
from core.port_scanner import Scanner
from core.parser import ConfParser

def scanner():
utils = Utils()
scanner = Scanner()

logger.info('Scanner process started')
Expand All @@ -16,20 +15,22 @@ def scanner():
time.sleep(10)
continue

conf = rds.get_scan_config()
conf = rds.get_scan_config()

if not conf:
time.sleep(10)
continue

hosts = rds.get_ips_to_scan(limit = conf['config']['scan_opts']['parallel_scan'])
c = ConfParser(conf)

hosts = rds.get_ips_to_scan(limit = c.get_cfg_scan_threads())

if hosts:
conf = rds.get_scan_config()
scan_data = scanner.scan(hosts,
max_ports = conf['config']['scan_opts']['max_ports'],
custom_ports = conf['config']['scan_opts']['custom_ports'],
interface = conf['config']['scan_opts']['interface'])
max_ports = c.get_cfg_max_ports(),
custom_ports = c.get_cfg_custom_ports(),
interface = c.get_cfg_netinterface())

if scan_data:
for host, values in scan_data.items():
Expand Down
1 change: 0 additions & 1 deletion core/reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,6 @@ def generate_xml(vulns):

description = xml.SubElement(vuln_element, "description")
description.text = value['rule_desc']


confirm = xml.SubElement(vuln_element, "confirm")
confirm.text = value['rule_confirm']
Expand Down
83 changes: 52 additions & 31 deletions core/triage.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,45 @@

class Triage:
def __init__(self):
self.global_timeout = 10
self.headers = {
'User-Agent':USER_AGENT
}

def http_request(self, ip, port, headers=None, follow_redirects=True, uri='/'):
def http_request(self, ip, port, method="GET", params=None, data=None, json=None, headers=None, follow_redirects=True, timeout=None, uri='/'):
resp = None

if headers:
self.headers = {**headers, **self.headers}

if method not in ('GET', 'POST', 'OPTIONS', 'PUT', 'DELETE', 'HEAD'):
logger.error('HTTP Method is not supported.')
return

if not timeout:
timeout = self.global_timeout

url = 'http://{}:{}{}'.format(ip, port, uri)

if port == 443 or port == 8443 or '443' in str(port):
url = 'https://{}:{}{}'.format(ip, port, uri)
try:
resp = requests.get(url, verify=False, timeout=8, allow_redirects=follow_redirects, headers=self.headers)
if method == 'GET':
resp = requests.get(url, verify=False, timeout=timeout, params=params, allow_redirects=follow_redirects, headers=self.headers)
elif method == 'PUT':
resp = requests.put(url, verify=False, timeout=timeout, params=params, data=data, json=json, allow_redirects=follow_redirects, headers=self.headers)
elif method == 'POST':
resp = requests.post(url, verify=False, timeout=timeout, params=params, data=data, json=json, allow_redirects=follow_redirects, headers=self.headers)
elif method == 'OPTIONS':
resp = requests.options(url, verify=False, timeout=timeout, params=params, allow_redirects=follow_redirects, headers=self.headers)
elif method == 'DELETE':
resp = requests.delete(url, verify=False, timeout=timeout, params=params, data=data, json=json, allow_redirects=follow_redirects, headers=self.headers)
elif method == 'HEAD':
resp = requests.head(url, verify=False, timeout=timeout, params=params, allow_redirects=follow_redirects, headers=self.headers)
else:
# Default to GET.
resp = requests.get(url, verify=False, timeout=timeout, params=params, data=data, json=json, allow_redirects=follow_redirects, headers=self.headers)


except requests.exceptions.ConnectTimeout:
logger.debug('http_request {} {} (Timeout)'.format(ip, port))
Expand Down Expand Up @@ -60,46 +83,45 @@ def string_in_headers(self, resp, string):
return resp
return False

def socket_banner(self, ip, port):
def get_tcp_socket_banner(self, ip, port, timeout=None):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket_banner = None

sock.settimeout(6)
if not timeout:
timeout = self.global_timeout
sock.settimeout(timeout)
try:
result = sock.connect_ex((ip, port))
if result == 0:
socket_banner = str(sock.recv(1024))
except Exception as e:
logger.debug('socket_open banner {} {} {}'.format(ip, port, e))
except:
pass

finally:
sock.close()

return socket_banner

def socket_open(self, ip, port):
def is_socket_open(self, ip, port, timeout=None):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(6)

if not timeout:
timeout = self.global_timeout

sock.settimeout(timeout)

try:
result = sock.connect_ex((ip, port))
if result == 0:
return True
except Exception as e:
except:
pass

finally:
sock.close()

return False

def is_ssh(self, ip, port):
is_ssh = False
banner = self.socket_banner(ip, port)
if banner and 'SSH' in str(banner):
is_ssh = True

return is_ssh

def run_cmd(self, command):
result = None
p = Popen(shlex.split(command), stdin=PIPE, stdout=PIPE, stderr=PIPE)
Expand All @@ -115,18 +137,17 @@ def has_cves(self, cpe):
if not any(char.isdigit() for char in cpe):
return False

try:
req = requests.get('https://nvd.nist.gov/vuln/search/results?form_type=Advanced&cves=on&cpe_version={}'.format(cpe), verify=False, timeout=5)
if req:
soup = BeautifulSoup(req.text, 'html.parser')
for a in soup.find_all('a', href=True):
if a.has_attr('data-testid') and a.contents:
sevs = ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL']
if any(word in a.contents[0] for word in sevs):
score, sev = a.contents[0].split()
if float(score) >= 8.9:
return True
except:
pass

req = self.http_request('nvd.nist.gov', 443, method="GET", uri='/vuln/search/results?form_type=Advanced&cves=on&cpe_version=' + cpe)
if not req:
return

soup = BeautifulSoup(req.text, 'html.parser')
for a in soup.find_all('a', href=True):
if a.has_attr('data-testid') and a.contents:
sevs = ['LOW', 'MEDIUM', 'HIGH', 'CRITICAL']
if any(word in a.contents[0] for word in sevs):
score, sev = a.contents[0].split()
if float(score) >= 8.9:
return True

return False
1 change: 0 additions & 1 deletion core/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,6 @@ def is_valid_port(self, port):
except TypeError:
return False


def get_primary_ip(self):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
Expand Down
2 changes: 0 additions & 2 deletions rules/bruteforce/rule_ftp-bf.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import ftplib

from core.redis import rds
from core.triage import Triage
from core.parser import ScanParser, ConfParser
from db.db_passwds import known_weak
from db.db_users import known_users
Expand Down Expand Up @@ -33,7 +32,6 @@ def ftp_attack(self, ip, username, password):

def check_rule(self, ip, port, values, conf):
c = ConfParser(conf)
t = Triage()
p = ScanParser(port, values)

domain = p.get_domain()
Expand Down
6 changes: 2 additions & 4 deletions rules/bruteforce/rule_mongodb-bf.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from pymongo import MongoClient
from core.redis import rds
from core.triage import Triage
from core.parser import ScanParser, ConfParser
from db.db_passwds import known_weak
from db.db_users import known_users
Expand All @@ -27,7 +26,7 @@ def mongodb_attack(self, ip, port, username, password):
MongoClient(ip, port).list_database_names()
return True

except Exception as e:
except:
return

else:
Expand All @@ -37,12 +36,11 @@ def mongodb_attack(self, ip, port, username, password):
MongoClient('mongodb://{username}:{password}@{ip}/admin'.format(username=username, password=password, ip=ip))
return True

except Exception as e:
except:
return

def check_rule(self, ip, port, values, conf):
c = ConfParser(conf)
t = Triage()
p = ScanParser(port, values)

domain = p.get_domain()
Expand Down
2 changes: 0 additions & 2 deletions rules/bruteforce/rule_mysql-bf.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import mysql.connector

from core.redis import rds
from core.triage import Triage
from core.parser import ScanParser, ConfParser
from db.db_passwds import known_weak
from db.db_users import known_users
Expand Down Expand Up @@ -31,7 +30,6 @@ def mysql_attack(self, ip, username, password):

def check_rule(self, ip, port, values, conf):
c = ConfParser(conf)
t = Triage()
p = ScanParser(port, values)

domain = p.get_domain()
Expand Down
4 changes: 1 addition & 3 deletions rules/bruteforce/rule_psql-bf.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import psycopg2
from core.redis import rds
from core.triage import Triage
from core.parser import ScanParser, ConfParser
from db.db_passwds import known_weak
from db.db_users import known_users
Expand All @@ -24,14 +23,13 @@ def psql_attack(self, ip, username, password):
connection.close()
return True

except Exception as e:
except:
return

return False

def check_rule(self, ip, port, values, conf):
c = ConfParser(conf)
t = Triage()
p = ScanParser(port, values)

domain = p.get_domain()
Expand Down
5 changes: 2 additions & 3 deletions rules/bruteforce/rule_redis-bf.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import socket

from core.redis import rds
from core.triage import Triage
from core.parser import ScanParser, ConfParser
from db.db_passwds import known_weak
from core.logging import logger

class Rule:
def __init__(self):
self.rule = 'BRF_DD00'
Expand Down Expand Up @@ -36,7 +36,6 @@ def redis_attack(self, ip, port, password):

def check_rule(self, ip, port, values, conf):
c = ConfParser(conf)
t = Triage()
p = ScanParser(port, values)

domain = p.get_domain()
Expand Down Expand Up @@ -72,7 +71,7 @@ def check_rule(self, ip, port, values, conf):
})
return
except:
return
pass

return

Expand Down
2 changes: 1 addition & 1 deletion rules/configuration/rule_cors-null-origin.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ def __init__(self):
self.rule = 'CFG_32A0'
self.rule_severity = 1
self.rule_description = 'This rule checks if Cross Origin Resource Sharing policy trusts null origins'
self.rule_confirm = 'CORS Allows Null Origins'
self.rule_confirm = 'CORS Policy Allows Null Origins'
self.rule_details = ''
self.rule_mitigation = '''Consider hardening your Cross Origin Resource Sharing Policy to define specific Origins \
https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS'''
Expand Down
2 changes: 1 addition & 1 deletion rules/configuration/rule_cors-wildcard.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ def __init__(self):
self.rule = 'CFG_DFFF'
self.rule_severity = 1
self.rule_description = 'This rule checks if Cross Origin Resource Sharing Headers support Wildcard Origins'
self.rule_confirm = 'Webserver is allowing all domains in CORS'
self.rule_confirm = 'CORS Policy allows any domain'
self.rule_details = ''
self.rule_mitigation = '''Consider hardening your Cross Origin Resource Sharing Policy to define specific Origins \
https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS'''
Expand Down
3 changes: 2 additions & 1 deletion rules/configuration/rule_ssh-auth-check.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ def check_rule(self, ip, port, values, conf):
p = ScanParser(port, values)

domain = p.get_domain()
module = p.get_module()

if port in ssh_ports and t.is_ssh(ip, port):
if port in ssh_ports and 'ssh' in module.lower():
output = t.run_cmd('ssh -o PreferredAuthentications=none -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o NoHostAuthenticationForLocalhost=yes user@"{}" -p "{}"'.format(ip, port))
if output and 'password' in str(output):
self.rule_details = 'Server accepts passwords as an authentication option'
Expand Down

0 comments on commit a5a0cb1

Please sign in to comment.