Add Suricata socket support, fix default UA, small things here and there

Dockerfile

@@ -6,6 +6,14 @@ RUN apt-get -y update
 RUN apt-get install --force-yes -y -q python python-setuptools libyaml-dev libpq-dev python-dev libpcap-dev git net-tools openjdk-7-jre firefox xvfb graphviz snort
 
 
-RUN git clone https://github.com/espenfjo/FjoSpidie.git --single-branch -b v2 /opt/fjospidie
+RUN mkdir -p /opt/fjospidie
+ADD fjospidie/ /opt/fjospidie/fjospidie
+ADD lib/ /opt/fjospidie/lib
+ADD ez_setup.py /opt/fjospidie/
+ADD fjospidie.conf /opt/fjospidie/
+ADD run.sh /opt/fjospidie/
+
+#RUN git clone https://github.com/espenfjo/FjoSpidie.git --single-branch -b v2 /opt/fjospidie
+
 RUN python /opt/fjospidie/ez_setup.py install
-ENV RUNNABLE_USER_DIR /opt/fjospidie
+ENV RUNNABLE_USER_DIR /opt/fjospidie

README.md

@@ -32,7 +32,8 @@ Install requirements with `apt-get install --force-yes -y -q python python-setup
 
 Build
 =====
-python ez_setup.py install
+`python ez_setup.py install`
+Or! If you want to run this in Docker. `docker build .`
 
 
 Configuration
@@ -45,6 +46,11 @@ Run `python fjospidie --url http://www.google.com` to analyse google.com.
 or run `xvfb-run -a python fjospidie --url http://www.google.com` to run the spider in a xvfb server isntead
 of the default X11 server.
 
+To run with suricata you need access to suricatas socket.
+If run through docker you can eg. run:
+`docker run -v /mnt/fjospidie:/mnt/fjospidie -i -t espenfjo/fjospidie:last bash -c "cd /opt/fjospidie; xvfb-run -a python fjospidie --url http://google.no/"`
+This will mount `/mnt/fjospidie` from the host inside your container. `/mnt/fjospidie` needs to contain your Suricata socket.
+
 Web Interface
 =============
 The FjoSpidie Web interface can be found here:

ez_setup.py

@@ -5,7 +5,7 @@
     version="2.0",
     packages=find_packages(),
     dependency_links = ['https://github.com/espenfjo/browsermob-proxy-py/archive/0.5.1.tar.gz#egg=browsermob-proxy-0.5.1',
-'https://github.com/espenfjo/pylibpcap/archive/0.6.4.1.tar.gz#egg=pylibpcap-0.6.4.1'],
+                        'https://github.com/espenfjo/pylibpcap/archive/0.6.4.1.tar.gz#egg=pylibpcap-0.6.4.1', 'lib/suricatasc'],
     install_requires=[
-        'selenium', 'pyaml', 'psycopg2', 'browsermob-proxy', 'harpy', 'pydot', 'pylibpcap','configobj'],
+        'selenium', 'pyaml', 'psycopg2', 'browsermob-proxy', 'harpy', 'pydot', 'pylibpcap','configobj','simplejson'],
 )

fjospidie.conf.dist

@@ -1,6 +1,7 @@
-databaseHost=<server>
-databasePort=5432
-databaseUser=fjospidie
+database_host=<server>
+database_port=5432
+database_user=fjospidie
+database_password=pw
 database=fjospidie
-databasePassword=pw
+suricata=False
 snort_config=/etc/snort/snort.conf

fjospidie/Report.py

@@ -14,9 +14,9 @@ def __init__(self, timestamp, config):
         else:
             self.uuid = config.uuid
 
-        self.db = psycopg2.connect(host=config.databaseHost,
-                                   user=config.databaseUser,
-                                   password=config.databasePassword,
+        self.db = psycopg2.connect(host=config.database_host,
+                                   user=config.database_user,
+                                   password=config.database_password,
                                    database=config.database)
         self.cur = self.db.cursor()
         self.db.autocommit = False

fjospidie/WebRunner.py

@@ -7,8 +7,8 @@
 from selenium.webdriver.firefox.webdriver import WebDriver
 from harpy.har import Har
 
-referer = 'http://www.google.com/search?q=hei+&oq=SUP&sourceid=firefox&ie=UTF-8'
-useragent = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2b5) Gecko/20091204 Firefox/3.6b5 Java/1.7.0_11'
+referer   = 'http://www.google.com/search?q=hei+&oq=SUP&sourceid=firefox&ie=UTF-8'
+useragent = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:24.0) Gecko/20100101 Firefox/24.0'
 URLs = []
 
 def run_webdriver(start_url, port, config):
@@ -29,7 +29,6 @@ def run_webdriver(start_url, port, config):
     server = Server("lib/browsermob/bin/browsermob-proxy", {'port': port})
     server.start()
     proxy = server.create_proxy()
-#    proxy = Client("localhost:8080")
     proxy.headers({'User-Agent': useragent, 'Accept-Encoding': ""})
 
     request_js=(

fjospidie/engine/PcapEngine.py

@@ -10,17 +10,18 @@
 import psycopg2
 
 class PcapEngine(threading.Thread):
-    def __init__(self, report):
+    def __init__(self, report, pcap_folder):
         threading.Thread.__init__(self)
         self.pcap_path = None
         self.report = report
+        self.pcap_folder = pcap_folder
 
     def run(self):
         logging.info("Starting PCAP engine")
         self.p = pcap.pcapObject()
         snaplen = 64 * 1024
         timeout = 1
-        pcap_file = tempfile.NamedTemporaryFile(prefix="snort", suffix="pcap", delete=False)
+        pcap_file = tempfile.NamedTemporaryFile(prefix="snort", suffix="pcap", delete=False, dir=self.pcap_folder)
         self.pcap_path = pcap_file.name
         logging.debug("PCAPing to " + self.pcap_path)
         dev = self.find_default_adapter()

fjospidie/engine/SuricataEngine.py

@@ -0,0 +1,74 @@
+import logging
+import threading
+import socket
+import json
+import time
+import os
+from suricata.suricatasc import *
+from snort.SnortAlert import SnortAlert
+
+class SuricataEngine(threading.Thread):
+    def __init__(self, config, report, connections, suricata_dir, pcap_path, socket):
+        threading.Thread.__init__(self)                
+        self.socket = socket
+        self.connections = connections
+        self.report = report
+        self.suricata_dir = suricata_dir
+        self.pcap_path = pcap_path
+        self.alerts = []
+        self.config = config
+
+    def run(self):
+        logging.info("Starting SuricataEngine")
+        os.makedirs(self.suricata_dir)
+        debug = False
+        if self.config.debug:
+            debug = True
+
+        sc = SuricataSC(self.socket, verbose=debug)
+        try:
+            sc.connect()            
+        except SuricataNetException, err:
+            logging.error("Unable to connect to socket %s: %s" % (self.socket, err))
+            return
+        except SuricataReturnException, err:
+            logging.error( "Unable to negotiate version with server: %s" % (err))
+            return            
+
+        arguments = {}
+        arguments["filename"]   = self.pcap_path
+        arguments["output-dir"] = self.suricata_dir
+
+        cmdret = sc.send_command("pcap-file", arguments)
+        if cmdret["return"] == "NOK":
+            logging.error(json.dumps(cmdret["message"], sort_keys=True, indent=4, separators=(',', ': ')))
+        else:
+            logging.debug(json.dumps(cmdret["message"], sort_keys=True, indent=4, separators=(',', ': ')))
+
+        self.check_ok(sc, 0)
+
+        alert_file = self.suricata_dir + "/fast.log"
+        with open(alert_file) as f:
+             for line in f:
+                 alert = SnortAlert(line)
+                 self.alerts.append(alert)
+
+        self.report.add_alerts( self.alerts)
+        logging.info("Stopping SnortEngine")
+
+
+    def check_ok(self, sc, count):
+        cmdret = sc.send_command("pcap-current")
+        if cmdret["return"] == "NOK":
+            logging.error(json.dumps(cmdret["message"], sort_keys=True, indent=4, separators=(',', ': ')))
+        else:
+            if cmdret["message"] == "None":
+                logging.debug(json.dumps(cmdret["message"], sort_keys=True, indent=4, separators=(',', ': ')))
+            else:
+                if count < 60:
+                    time.sleep(0.5)
+                    count +=1
+                    self.check_ok(sc, count)
+                else:
+                    logging.error("No result from suricata...")
+