diff --git a/backend/config.py b/backend/config.py index 991adfa..d4be448 100644 --- a/backend/config.py +++ b/backend/config.py @@ -144,9 +144,11 @@ # A string of "debug", "info", "warning", "error", "critical" # -errorlog = '-' +errorlog = '/var/raptor/log/error.log' loglevel = 'info' -accesslog = '-' +accesslog = '/var/raptor/log/access.log' +#access_format = "request-date pid remote-address username status-line response-code referer user-agent request-header" +access_log_format = '%(t)s %(p)s %(h)s %(u)s %(r)s %(s)s %(b)s %(f)s %(a)s %({Header}i)s' # # Process naming diff --git a/backend/raptor/android.py b/backend/raptor/android.py index 7c63991..a81d1bd 100644 --- a/backend/raptor/android.py +++ b/backend/raptor/android.py @@ -7,6 +7,7 @@ import os, sys, re, linecache, json from lxml import etree +import log plugin_name = "android_manifest" diff --git a/backend/raptor/codescan.py b/backend/raptor/codescan.py index 4df889a..18b7454 100644 --- a/backend/raptor/codescan.py +++ b/backend/raptor/codescan.py @@ -2,6 +2,7 @@ import os, sys, re, fnmatch, json, base64, time from datetime import datetime from android import * +import log def version(): return 'beta' @@ -123,7 +124,7 @@ def scan_line(self, line, fpath): issue["warning_type"] = str(rule["title"]) issue["warning_code"] = str(rule["id"]) issue["message"] = str(rule["description"]) - issue["file"] = re.sub('/(clones|uploads)/[a-zA-Z0-9]{56}/', '', fpath.replace(os.getcwd(),'')) + issue["file"] = re.sub('/var/raptor/(clones|uploads)/[a-zA-Z0-9]{56}/', '', fpath.replace(os.getcwd(),'')) issue["line"] = int(line_num) + 1 issue["link"] = str(rule["link"]) issue["code"] = line_content.strip("\n").strip("\r").strip("\t").strip(" ") diff --git a/backend/raptor/externalscan.py b/backend/raptor/externalscan.py index 1a92e35..681698c 100644 --- a/backend/raptor/externalscan.py +++ b/backend/raptor/externalscan.py @@ -1,6 +1,7 @@ #!usr/bin/python import os, sys, subprocess, shutil, json, linecache, base64, fnmatch, traceback import BeautifulSoup as bs +import log bin_paths = {} bin_paths['nodejs'] = '/usr/bin/nodejs' if os.path.isfile('/usr/bin/nodejs') else '/usr/local/bin/node' diff --git a/backend/raptor/fsb.py b/backend/raptor/fsb.py index 7509f14..891a88c 100644 --- a/backend/raptor/fsb.py +++ b/backend/raptor/fsb.py @@ -9,6 +9,7 @@ ''' import os, sys, re, json, base64 +import log def get_localImports(fpath): imports = [] @@ -105,7 +106,7 @@ def scan_line(delim_line, fpath, root_path): fsb_issue['warning_type'] = str(rule['title']) fsb_issue['warning_code'] = str(rule['id']) fsb_issue['message'] = str(rule['description']) - fsb_issue['file'] = re.sub('/(clones|uploads)/[a-zA-Z0-9]{56}/', '', fpath.replace(os.getcwd(), '').replace(root_path, '')) + fsb_issue['file'] = re.sub('/var/raptor/(clones|uploads)/[a-zA-Z0-9]{56}/', '', fpath.replace(os.getcwd(), '').replace(root_path, '')) fsb_issue['line'] = int(line_num) + 1 fsb_issue['link'] = str(rule['link']) fsb_issue['code'] = line_content.strip('\n').strip('\r').strip('\t').strip(' ') diff --git a/backend/raptor/gitrob.py b/backend/raptor/gitrob.py index 55bea1e..16a0381 100644 --- a/backend/raptor/gitrob.py +++ b/backend/raptor/gitrob.py @@ -7,6 +7,7 @@ https://github.com/michenriksen/gitrob created by Michael Henriksen. ''' import os, sys, re, json, base64 +import log def load_gitrob_rules(fname): file = open(fname, 'r') @@ -35,7 +36,7 @@ def gitrob_scan(root_path, rules_path): gitrob_issue['warning_type'] = 'Sensitive Information Disclosure' gitrob_issue['warning_code'] = 'SID' gitrob_issue['message'] = str(gitrob_rule['caption']) - gitrob_issue['file'] = re.sub('/(clones|uploads)/[a-zA-Z0-9]{56}/', '', file_path.replace(os.getcwd(), '').replace(root_path, '')).lstrip('/') + gitrob_issue['file'] = re.sub('/var/raptor/(clones|uploads)/[a-zA-Z0-9]{56}/', '', file_path.replace(os.getcwd(), '').replace(root_path, '')).lstrip('/') gitrob_issue['line'] = '' gitrob_issue['link'] = 'https://www.owasp.org/index.php/Top_10_2013-A6-Sensitive_Data_Exposure' gitrob_issue['code'] = 'n/a' diff --git a/backend/raptor/init.py b/backend/raptor/init.py index a395373..4e0acbf 100644 --- a/backend/raptor/init.py +++ b/backend/raptor/init.py @@ -6,6 +6,7 @@ from externalscan import * from fsb import * from gitrob import * +import log rulepacks = ['common', 'android', 'php', 'actionscript'] @@ -40,6 +41,7 @@ def scan_all(scan_path, repo_path): for rulepack in rulepacks: rule_path = 'rules/%s.rulepack' % rulepack report_path = scan_path + '/%s_report.json' % rulepack + log.logger.debug('scanning with [%s] rulepack' % (rulepack)) result = Scanner(scan_path, rule_path, report_path) if len(result.issues) > 0: @@ -47,7 +49,7 @@ def scan_all(scan_path, repo_path): results.append(issue) total_issues += 1 - print "[INFO] Started gitrob plugin" + log.logger.debug("scanning with [gitrob] plugin") for rulepack in plugin_rulepacks: if rulepack.startswith('gitrob'): rule_path = 'rules/%s.rulepack' % rulepack @@ -58,7 +60,7 @@ def scan_all(scan_path, repo_path): results.append(issue) total_issues += 1 - print "[INFO] Started fsb plugin" + log.logger.debug("scanning with [fsb] plugin") for rulepack in plugin_rulepacks: if rulepack.startswith('fsb_'): rule_path = 'rules/%s.rulepack' % rulepack @@ -69,21 +71,21 @@ def scan_all(scan_path, repo_path): results.append(issue) total_issues += 1 - print "[INFO] Started scanjs plugin" + log.logger.debug("scanning with [scanjs] plugin") js_results = scanjs(scan_path) if len(js_results) > 0 and js_results != 'error': for js_issue in js_results: results.append(js_issue) total_issues += 1 - print "[INFO] Started brakeman plugin" + log.logger.debug("scanning with [brakeman] plugin") ror_results = scan_brakeman(scan_path) if len(ror_results) > 0 and ror_results != 'error': for ror_result in ror_results: results.append(ror_result) total_issues += 1 - print "[INFO] Started rips plugin" + log.logger.debug("scanning with [rips] plugin") php_results = scan_phprips(scan_path) if len(php_results) > 0 and php_results != 'error': for php_result in php_results: @@ -107,11 +109,11 @@ def scan_all(scan_path, repo_path): return json def clone(repo_name, internal): + clone_directory = os.environ['git_clone_dir'] uniq_path = hashlib.sha224(repo_name).hexdigest() - - uniq_path = hashlib.sha224(repo_name).hexdigest() - if os.path.isdir(os.getcwd() + '/clones/' + uniq_path): - shutil.rmtree(os.getcwd() + '/clones/' + uniq_path) + + if os.path.isdir(os.path.join(clone_directory, uniq_path)): + shutil.rmtree(os.path.join(clone_directory, uniq_path)) if internal: repo_url = '%s/%s.git' % (os.environ['int_git_url'], repo_name) @@ -119,10 +121,10 @@ def clone(repo_name, internal): repo_url = '%s/%s.git' % (os.environ['ext_git_url'], repo_name) try: - clone_dir = os.getcwd() + '/clones/' + clone_dir = clone_directory if not os.path.isdir(clone_dir): os.makedirs(clone_dir) - repo_path = clone_dir + uniq_path + repo_path = os.path.join(clone_dir, uniq_path) if internal==True: username = os.environ['int_git_user'] @@ -135,32 +137,31 @@ def clone(repo_name, internal): git_obj = git.clone_repository(repo_url, repo_path, credentials=login_info) return repo_path except Exception, e: - print e if str(e).find('Unexpected HTTP status code: 404'): - print "Repo doesn't exists" + log.logger.error("repo doesn't exists") return "Repo doesn't exists" - #return str(e) + log.logger.error(e) def delete_residue(path, report_files): shutil.rmtree(path) def start(repo_path, report_dir, internal): - print "==============New Scan: [github] ===================" - print "[INFO] Now cloning: %s" % (repo_path) + log.logger.debug("==============New Scan: [github] ===================") + log.logger.debug("Now cloning: %s" % (repo_path)) cloned_path = clone(repo_path, internal) if os.path.isdir(cloned_path): - print "[INFO] Now scanning: %s" % repo_path + log.logger.debug("[INFO] Now scanning: %s" % repo_path) results = scan_all(cloned_path, repo_path) - print "[INFO] Scan complete! Deleting ..." + log.logger.debug("[INFO] Scan complete! Deleting ...") delete_residue(cloned_path, rulepacks) return results def scan_zip(upload_id, zip_name, report_dir): - print "==============New Scan: [zip] ===================" - extracted_path = os.path.join(os.path.abspath('./uploads'), upload_id) + log.logger.debug("==============New Scan: [zip] ===================") + extracted_path = os.path.join(os.path.abspath(os.environ['zip_upload_dir']), upload_id) if os.path.exists(extracted_path): - print "[INFO] Now scanning: %s" % zip_name + log.logger.debug("Now scanning: %s" % zip_name) results = scan_all(extracted_path, zip_name) - print "[INFO] Scan complete! Deleting ..." + log.logger.debug("Scan complete! Deleting ...") delete_residue(extracted_path, zip_name) return results \ No newline at end of file diff --git a/backend/raptor/log.py b/backend/raptor/log.py new file mode 100644 index 0000000..af87342 --- /dev/null +++ b/backend/raptor/log.py @@ -0,0 +1,9 @@ +import logging, logging.handlers + +LOG_FILENAME = '/var/raptor/log/debug.log' + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + +handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=1024, backupCount=8) +logger.addHandler(handler) \ No newline at end of file diff --git a/backend/server.py b/backend/server.py index 7300c3c..ae3bc55 100644 --- a/backend/server.py +++ b/backend/server.py @@ -4,14 +4,14 @@ from werkzeug.contrib.fixers import ProxyFix from werkzeug import secure_filename import sys, os, json, threading, hashlib, shutil, zipfile, requests, time - +from raptor import log app = Flask(__name__) app.debug=True @app.errorhandler(500) def server_error(e): - return 'Internal Server Error', 500 + return 'Internal Server Error', 500 @app.route('/', methods=['GET']) def help(): @@ -21,6 +21,7 @@ def help(): def heartbeat(): return '{"status":"true", "time":%s}' % (str(int(time.time()))) +#server-side call; nginx route not required @app.route('/internal/scan/', methods=['GET']) def internal_repo_scan(): repo = request.args.get('r') @@ -35,10 +36,10 @@ def internal_repo_scan(): fhandle = open(report_directory, "w") content = fhandle.write(results) fhandle.close() - print "[INFO] Report created at %s" % (report_directory) + log.logger.debug("Report created at %s" % (report_directory)) return jsonify(json_results) - +#server-side call; nginx route not required @app.route('/external/scan/', methods=['GET']) def external_repo_scan(): repo = request.args.get('r') @@ -53,7 +54,7 @@ def external_repo_scan(): fhandle = open(report_directory, "w") content = fhandle.write(results) fhandle.close() - print "[INFO] Report created at %s" % (report_directory) + log.logger.debug("Report created at %s" % (report_directory)) return jsonify(json_results) @app.route('/purge/', methods=['GET']) @@ -69,20 +70,20 @@ def delete_report(): os.remove(report_path) resp_content = "Success" except Exception as e: - print "[ERROR] %s: %s" % (report_path, str(e)) + log.logger.error("%s: %s" % (report_path, str(e))) resp_content = "Failure" else: resp_content = "Failure" return resp_content -UPLOAD_FOLDER = os.path.abspath('./uploads') +UPLOAD_FOLDER = os.path.abspath(os.environ['zip_upload_dir']) ALLOWED_EXTENSIONS = set(['zip']) try: os.makedirs(UPLOAD_FOLDER) except Exception as e: if ' File exists: ' in str(e): - print "[INFO] %s" % str(e) + log.logger.debug("%s" % str(e)) else: raise e @@ -103,7 +104,7 @@ def unzip(fname, path='.'): else: return False except Exception as e: - print e + log.logger.error(e) return False t = threading.Thread(target=unzip, args=(fname, path)) t.start() @@ -127,7 +128,7 @@ def index(): unzip_thread(new_path, os.path.join(UPLOAD_FOLDER, new_fname.rstrip('.zip'))) return redirect('/raptor/scan.php?scan_name=%s&upload_id=%s&zip_name=%s' % (scan_name, new_fname.rstrip('.zip'), upld_file.filename), code=302) except Exception as e: - print e + log.logger.error(e) #server-side call; nginx route not required @app.route('/zip/scan/', methods=['GET']) @@ -145,9 +146,10 @@ def zip_scan(): fhandle = open(report_directory, "w") content = fhandle.write(results) fhandle.close() - print "[INFO] Report created at %s" % (report_directory) + log.logger.debug("Report created at %s" % (report_directory)) return jsonify(json_results) +#exposed via nginx route @app.route('/raptor/githook', methods=['POST']) def gitHook(): try: @@ -177,10 +179,10 @@ def gitHook(): content = fhandle.write(results) fhandle.close() - print "[INFO] Report created at %s" % (report_directory) + log.logger.debug("Report created at %s" % (report_directory)) return jsonify(json_results) except Exception as e: - print str(e) + log.logger.error(str(e)) return "" app.wsgi_app = ProxyFix(app.wsgi_app) diff --git a/frontend/delete_report.php b/frontend/delete_report.php index 1789cf3..04a1248 100755 --- a/frontend/delete_report.php +++ b/frontend/delete_report.php @@ -13,6 +13,9 @@ curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); $response = curl_exec($ch); curl_close($ch); + if ($response === 'Success') { + $_SESSION['delete_id'][$report_id] = ''; + } header('Location: history.php'); } diff --git a/frontend/issues.php b/frontend/issues.php index 4246e57..638f7b7 100755 --- a/frontend/issues.php +++ b/frontend/issues.php @@ -5,9 +5,11 @@ if (!empty($_SESSION['current_scan_report'])) { -$data = json_decode(file_get_contents($_SESSION['current_scan_report']), true); - +if (file_exists($_SESSION['current_scan_report'])) { + $data = json_decode(file_get_contents($_SESSION['current_scan_report']), true); } else { + $_SESSION['current_scan_report'] = ''; +}} else { error_log("[ERROR] session: current_scan_report is null."); } diff --git a/frontend/proxy.php b/frontend/proxy.php index b2d3e11..720d5ef 100755 --- a/frontend/proxy.php +++ b/frontend/proxy.php @@ -43,7 +43,7 @@ function write_to_file($data, $username, $scan_name) { if( !empty($_SESSION['git_repo']) && !empty($_SESSION['scan_active']) ) { $path = PATH . '/' . $_SESSION['user_name'] . '/' . $_SESSION['scan_name'] . '/' . $_SESSION['git_repo'] . '/' . time() . '.json'; - error_log($path); + //error_log('DEBUG: '.$_SESSION['git_repo']); $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, ENDPOINT . '/' . $_SESSION['git_type'] . '/scan/?r=' . $_SESSION['git_repo'] . '&p=' . $path); diff --git a/frontend/scan.php b/frontend/scan.php index a2d4edb..130b569 100644 --- a/frontend/scan.php +++ b/frontend/scan.php @@ -8,6 +8,7 @@ @$upload_id = $_REQUEST['upload_id']; @$zip_name = $_REQUEST['zip_name']; + function normalize_git_path($git_repo) { #point your github server endpoints here, and do NOT forget the trailing slash @@ -27,7 +28,8 @@ function normalize_git_path($git_repo) { if ($git_repo[strlen($git_repo)-1] === '/') $git_repo[strlen($git_repo)-1] = ''; - + + $git_repo = preg_replace( '/[^[:print:]]/', '',$git_repo); return $git_repo; } diff --git a/install.sh b/install.sh index 05f22eb..9296824 100644 --- a/install.sh +++ b/install.sh @@ -33,6 +33,12 @@ rm -rf libgit2-0.22.0.tar.gz rm -rf libgit2-0.22.0/ mkdir -p /var/raptor/scan_results chmod -R 777 /var/raptor/scan_results #development purpose only +mkdir -p /var/raptor/log +chmod -R 777 /var/raptor/log #development purpose only +mkdir -p /var/raptor/uploads +chmod -R 777 /var/raptor/uploads #development purpose only +mkdir -p /var/raptor/clones +chmod -R 777 /var/raptor/clones #development purpose only mkdir -p /var/www/html/raptor cp -r frontend/* /var/www/html/raptor rm -rf /etc/apache2/sites-available/raptor.conf diff --git a/start.sh b/start.sh index 1ef15e0..9cccc9e 100644 --- a/start.sh +++ b/start.sh @@ -1,7 +1,8 @@ #!/bin/sh export reportpath="/var/raptor/scan_results" - +export zip_upload_dir="/var/raptor/uploads" +export git_clone_dir="/var/raptor/clones" #IMPORTANT: Do NOT add the trailing slash after the URLs. ############PUBLIC###############