logging and modular restructuring

backend/config.py

@@ -144,9 +144,11 @@
 #       A string of "debug", "info", "warning", "error", "critical"
 #
 
-errorlog = '-'
+errorlog = '/var/raptor/log/error.log'
 loglevel = 'info'
-accesslog = '-'
+accesslog = '/var/raptor/log/access.log'
+#access_format = "request-date pid remote-address username status-line response-code referer user-agent request-header"
+access_log_format = '%(t)s %(p)s %(h)s %(u)s %(r)s %(s)s %(b)s %(f)s %(a)s %({Header}i)s'
 
 #
 # Process naming

backend/raptor/android.py

@@ -7,6 +7,7 @@
 
 import os, sys, re, linecache, json
 from lxml import etree
+import log
 
 plugin_name = "android_manifest"
 

backend/raptor/codescan.py

@@ -2,6 +2,7 @@
 import os, sys, re, fnmatch, json, base64, time
 from datetime import datetime
 from android import *
+import log
 
 def version():
     return 'beta'
@@ -123,7 +124,7 @@ def scan_line(self, line, fpath):
                     issue["warning_type"] =  str(rule["title"])
                     issue["warning_code"] = str(rule["id"])
                     issue["message"] = str(rule["description"])
-                    issue["file"] = re.sub('/(clones|uploads)/[a-zA-Z0-9]{56}/', '', fpath.replace(os.getcwd(),''))
+                    issue["file"] = re.sub('/var/raptor/(clones|uploads)/[a-zA-Z0-9]{56}/', '', fpath.replace(os.getcwd(),''))
                     issue["line"] = int(line_num) + 1
                     issue["link"] = str(rule["link"])
                     issue["code"] = line_content.strip("\n").strip("\r").strip("\t").strip(" ")

backend/raptor/externalscan.py

@@ -1,6 +1,7 @@
 #!usr/bin/python
 import os, sys, subprocess, shutil, json, linecache, base64, fnmatch, traceback
 import BeautifulSoup as bs
+import log
 
 bin_paths = {}
 bin_paths['nodejs'] = '/usr/bin/nodejs' if os.path.isfile('/usr/bin/nodejs') else '/usr/local/bin/node'

backend/raptor/fsb.py

@@ -9,6 +9,7 @@
 
 '''
 import os, sys, re, json, base64
+import log
 
 def get_localImports(fpath):
     imports = []
@@ -105,7 +106,7 @@ def scan_line(delim_line, fpath, root_path):
                     fsb_issue['warning_type'] =  str(rule['title'])
                     fsb_issue['warning_code'] = str(rule['id'])
                     fsb_issue['message'] = str(rule['description'])
-                    fsb_issue['file'] = re.sub('/(clones|uploads)/[a-zA-Z0-9]{56}/', '', fpath.replace(os.getcwd(), '').replace(root_path, ''))
+                    fsb_issue['file'] = re.sub('/var/raptor/(clones|uploads)/[a-zA-Z0-9]{56}/', '', fpath.replace(os.getcwd(), '').replace(root_path, ''))
                     fsb_issue['line'] = int(line_num) + 1
                     fsb_issue['link'] = str(rule['link'])
                     fsb_issue['code'] = line_content.strip('\n').strip('\r').strip('\t').strip(' ')

backend/raptor/gitrob.py

@@ -7,6 +7,7 @@
 https://github.com/michenriksen/gitrob created by Michael Henriksen.
 '''
 import os, sys, re, json, base64
+import log
 
 def load_gitrob_rules(fname):
     file = open(fname, 'r')
@@ -35,7 +36,7 @@ def gitrob_scan(root_path, rules_path):
                     gitrob_issue['warning_type'] =  'Sensitive Information Disclosure'
                     gitrob_issue['warning_code'] = 'SID'
                     gitrob_issue['message'] = str(gitrob_rule['caption'])
-                    gitrob_issue['file'] = re.sub('/(clones|uploads)/[a-zA-Z0-9]{56}/', '', file_path.replace(os.getcwd(), '').replace(root_path, '')).lstrip('/')
+                    gitrob_issue['file'] = re.sub('/var/raptor/(clones|uploads)/[a-zA-Z0-9]{56}/', '', file_path.replace(os.getcwd(), '').replace(root_path, '')).lstrip('/')
                     gitrob_issue['line'] = ''
                     gitrob_issue['link'] = 'https://www.owasp.org/index.php/Top_10_2013-A6-Sensitive_Data_Exposure'
                     gitrob_issue['code'] = 'n/a'

backend/raptor/init.py

@@ -6,6 +6,7 @@
 from externalscan import *
 from fsb import *
 from gitrob import *
+import log
 
 
 rulepacks = ['common', 'android', 'php', 'actionscript']
@@ -40,14 +41,15 @@ def scan_all(scan_path, repo_path):
     for rulepack in rulepacks:
         rule_path = 'rules/%s.rulepack' % rulepack
         report_path = scan_path + '/%s_report.json' % rulepack
+        log.logger.debug('scanning with [%s] rulepack' % (rulepack))
         result = Scanner(scan_path, rule_path, report_path)
 
         if len(result.issues) > 0:
             for issue in result.issues:
                 results.append(issue)
                 total_issues += 1
 
-    print "[INFO] Started gitrob plugin"
+    log.logger.debug("scanning with [gitrob] plugin")
     for rulepack in plugin_rulepacks:
         if rulepack.startswith('gitrob'):
             rule_path = 'rules/%s.rulepack' % rulepack
@@ -58,7 +60,7 @@ def scan_all(scan_path, repo_path):
                     results.append(issue)
                     total_issues += 1
 
-    print "[INFO] Started fsb plugin"
+    log.logger.debug("scanning with [fsb] plugin")
     for rulepack in plugin_rulepacks:
         if rulepack.startswith('fsb_'):
             rule_path = 'rules/%s.rulepack' % rulepack
@@ -69,21 +71,21 @@ def scan_all(scan_path, repo_path):
                     results.append(issue)
                     total_issues += 1
 
-    print "[INFO] Started scanjs plugin"
+    log.logger.debug("scanning with [scanjs] plugin")
     js_results = scanjs(scan_path)
     if len(js_results) > 0 and js_results != 'error':
         for js_issue in js_results:
             results.append(js_issue)
             total_issues += 1
 
-    print "[INFO] Started brakeman plugin"
+    log.logger.debug("scanning with [brakeman] plugin")
     ror_results = scan_brakeman(scan_path)
     if len(ror_results) > 0 and ror_results != 'error':
         for ror_result in ror_results:
             results.append(ror_result)
             total_issues += 1
 
-    print "[INFO] Started rips plugin"
+    log.logger.debug("scanning with [rips] plugin")
     php_results = scan_phprips(scan_path)
     if len(php_results) > 0 and php_results != 'error':
         for php_result in php_results:
@@ -107,22 +109,22 @@ def scan_all(scan_path, repo_path):
     return json
 
 def clone(repo_name, internal):
+    clone_directory = os.environ['git_clone_dir']
     uniq_path = hashlib.sha224(repo_name).hexdigest()
-
-    uniq_path = hashlib.sha224(repo_name).hexdigest()
-    if os.path.isdir(os.getcwd() + '/clones/' + uniq_path):
-        shutil.rmtree(os.getcwd() + '/clones/' + uniq_path)
+
+    if os.path.isdir(os.path.join(clone_directory, uniq_path)):
+        shutil.rmtree(os.path.join(clone_directory, uniq_path))
 
     if internal:
         repo_url = '%s/%s.git' % (os.environ['int_git_url'], repo_name)
     else:
         repo_url = '%s/%s.git' % (os.environ['ext_git_url'], repo_name)
 
     try:
-        clone_dir = os.getcwd() + '/clones/'
+        clone_dir = clone_directory
         if not os.path.isdir(clone_dir):
             os.makedirs(clone_dir)
-        repo_path = clone_dir + uniq_path
+        repo_path = os.path.join(clone_dir, uniq_path)
 
         if internal==True:
             username = os.environ['int_git_user']
@@ -135,32 +137,31 @@ def clone(repo_name, internal):
         git_obj = git.clone_repository(repo_url, repo_path, credentials=login_info)            
         return repo_path
     except Exception, e:
-        print e
         if str(e).find('Unexpected HTTP status code: 404'):
-            print "Repo doesn't exists"
+            log.logger.error("repo doesn't exists")
             return "Repo doesn't exists"
-        #return str(e)
+        log.logger.error(e)
 
 def delete_residue(path, report_files):
     shutil.rmtree(path)
 
 def start(repo_path, report_dir, internal):
-    print "==============New Scan: [github] ==================="
-    print "[INFO] Now cloning: %s" % (repo_path)
+    log.logger.debug("==============New Scan: [github] ===================")
+    log.logger.debug("Now cloning: %s" % (repo_path))
     cloned_path = clone(repo_path, internal)
     if os.path.isdir(cloned_path):
-        print "[INFO] Now scanning: %s" % repo_path
+        log.logger.debug("[INFO] Now scanning: %s" % repo_path)
         results = scan_all(cloned_path, repo_path)
-        print "[INFO] Scan complete! Deleting ..."
+        log.logger.debug("[INFO] Scan complete! Deleting ...")
         delete_residue(cloned_path, rulepacks)
         return results
 
 def scan_zip(upload_id, zip_name, report_dir):
-    print "==============New Scan: [zip] ==================="
-    extracted_path = os.path.join(os.path.abspath('./uploads'), upload_id)
+    log.logger.debug("==============New Scan: [zip] ===================")
+    extracted_path = os.path.join(os.path.abspath(os.environ['zip_upload_dir']), upload_id)
     if os.path.exists(extracted_path):
-        print "[INFO] Now scanning: %s" % zip_name
+        log.logger.debug("Now scanning: %s" % zip_name)
         results = scan_all(extracted_path, zip_name)
-        print "[INFO] Scan complete! Deleting ..."
+        log.logger.debug("Scan complete! Deleting ...")
         delete_residue(extracted_path, zip_name)
         return results

backend/raptor/log.py

@@ -0,0 +1,9 @@
+import logging, logging.handlers
+
+LOG_FILENAME = '/var/raptor/log/debug.log'
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=1024, backupCount=8)
+logger.addHandler(handler)

backend/server.py

@@ -4,14 +4,14 @@
 from werkzeug.contrib.fixers import ProxyFix
 from werkzeug import secure_filename
 import sys, os, json, threading, hashlib, shutil, zipfile, requests, time
-
+from raptor import log
 
 app = Flask(__name__)
 app.debug=True
 
 @app.errorhandler(500)
 def server_error(e):
-    return 'Internal Server Error', 500    
+    return 'Internal Server Error', 500
 
 @app.route('/', methods=['GET'])
 def help():
@@ -21,6 +21,7 @@ def help():
 def heartbeat():
     return '{"status":"true", "time":%s}' % (str(int(time.time())))
 
+#server-side call; nginx route not required
 @app.route('/internal/scan/', methods=['GET'])
 def internal_repo_scan():
     repo = request.args.get('r')
@@ -35,10 +36,10 @@ def internal_repo_scan():
     fhandle = open(report_directory, "w")
     content = fhandle.write(results)
     fhandle.close()
-    print "[INFO] Report created at %s" % (report_directory)
+    log.logger.debug("Report created at %s" % (report_directory))
     return jsonify(json_results)
 
-
+#server-side call; nginx route not required
 @app.route('/external/scan/', methods=['GET'])
 def external_repo_scan():
     repo = request.args.get('r')
@@ -53,7 +54,7 @@ def external_repo_scan():
     fhandle = open(report_directory, "w")
     content = fhandle.write(results)
     fhandle.close()
-    print "[INFO] Report created at %s" % (report_directory)
+    log.logger.debug("Report created at %s" % (report_directory))
     return jsonify(json_results)
 
 @app.route('/purge/', methods=['GET'])
@@ -69,20 +70,20 @@ def delete_report():
                 os.remove(report_path)
                 resp_content = "Success"
         except Exception as e:
-            print "[ERROR] %s: %s" % (report_path, str(e))
+            log.logger.error("%s: %s" % (report_path, str(e)))
             resp_content = "Failure"
     else:
         resp_content = "Failure"
     return resp_content
 
-UPLOAD_FOLDER = os.path.abspath('./uploads')
+UPLOAD_FOLDER = os.path.abspath(os.environ['zip_upload_dir'])
 ALLOWED_EXTENSIONS = set(['zip'])
 
 try:
     os.makedirs(UPLOAD_FOLDER)
 except Exception as e:
     if ' File exists: ' in str(e):
-        print "[INFO] %s" % str(e)
+        log.logger.debug("%s" % str(e))
     else:
         raise e
 
@@ -103,7 +104,7 @@ def unzip(fname, path='.'):
             else:
                 return False
         except Exception as e:
-            print e
+            log.logger.error(e)
             return False
     t = threading.Thread(target=unzip, args=(fname, path))
     t.start()
@@ -127,7 +128,7 @@ def index():
                 unzip_thread(new_path, os.path.join(UPLOAD_FOLDER, new_fname.rstrip('.zip')))
                 return redirect('/raptor/scan.php?scan_name=%s&upload_id=%s&zip_name=%s' % (scan_name, new_fname.rstrip('.zip'), upld_file.filename), code=302)
             except Exception as e:
-                print e
+                log.logger.error(e)
 
 #server-side call; nginx route not required
 @app.route('/zip/scan/', methods=['GET'])
@@ -145,9 +146,10 @@ def zip_scan():
     fhandle = open(report_directory, "w")
     content = fhandle.write(results)
     fhandle.close()
-    print "[INFO] Report created at %s" % (report_directory)
+    log.logger.debug("Report created at %s" % (report_directory))
     return jsonify(json_results)
 
+#exposed via nginx route
 @app.route('/raptor/githook', methods=['POST'])
 def gitHook():
     try:
@@ -177,10 +179,10 @@ def gitHook():
             content = fhandle.write(results)
             fhandle.close()
 
-            print "[INFO] Report created at %s" % (report_directory)
+            log.logger.debug("Report created at %s" % (report_directory))
             return jsonify(json_results)
     except Exception as e:
-        print str(e)
+        log.logger.error(str(e))
     return ""
 
 app.wsgi_app = ProxyFix(app.wsgi_app)

frontend/delete_report.php

@@ -13,6 +13,9 @@
     curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
     $response = curl_exec($ch);
     curl_close($ch);
+    if ($response === 'Success') {
+    	$_SESSION['delete_id'][$report_id] = '';
+    }
 	header('Location: history.php');
 }
 

frontend/issues.php

@@ -5,9 +5,11 @@
 
 if (!empty($_SESSION['current_scan_report'])) {
 
-$data = json_decode(file_get_contents($_SESSION['current_scan_report']), true);
-
+if (file_exists($_SESSION['current_scan_report'])) {
+  $data = json_decode(file_get_contents($_SESSION['current_scan_report']), true);
 } else {
+  $_SESSION['current_scan_report'] = '';
+}} else {
 error_log("[ERROR] session: current_scan_report is null.");
 }
 

frontend/proxy.php

@@ -43,7 +43,7 @@ function write_to_file($data, $username, $scan_name) {
 if( !empty($_SESSION['git_repo']) && !empty($_SESSION['scan_active']) ) {
 
 	$path = PATH . '/' . $_SESSION['user_name'] . '/' . $_SESSION['scan_name'] . '/' . $_SESSION['git_repo'] . '/' . 	time() . '.json';
-	error_log($path);
+	//error_log('DEBUG: <proxy.php>'.$_SESSION['git_repo']);
 
 	$ch = curl_init();
 	curl_setopt($ch, CURLOPT_URL, ENDPOINT . '/' . $_SESSION['git_type'] . '/scan/?r=' . $_SESSION['git_repo'] . '&p=' . $path);

frontend/scan.php

@@ -8,6 +8,7 @@
 @$upload_id = $_REQUEST['upload_id'];
 @$zip_name = $_REQUEST['zip_name'];
 
+
 function normalize_git_path($git_repo) {  
 
 #point your github server endpoints here, and do NOT forget the trailing slash
@@ -27,7 +28,8 @@ function normalize_git_path($git_repo) {
 
     if ($git_repo[strlen($git_repo)-1] === '/')
       $git_repo[strlen($git_repo)-1] = '';
-
+
+    $git_repo = preg_replace( '/[^[:print:]]/', '',$git_repo);
     return $git_repo;
 }