Make scanners use kibble.ini instead of config.yaml (#122)

This PR moved watson, azure, picoapi and git configuration to
kibble.ini config file. It fixes also reading ES configuration.
In this way all Kibble configuration is in single place.

Author: turbaszek

kibble.ini

@@ -25,6 +25,26 @@ scratchdir = /tmp
 # each node will gat 1/4th of all jobs to work on.
 balance =
 
+[git]
+# Comma-separated branch names
+wanted_branches =
+
+# Watson/BlueMix configuration for sentiment analysis, if applicable
+[watson]
+username =
+password =
+api = https://gateway-location.watsonplatform.net/tone-analyzer/api
+
+# Azure Text Analysis API configuration, if applicable
+[azure]
+apikey =
+location = west-us
+
+# picoAPI Text Analysis configuration
+[picoapi]
+key =
+
+
 [elasticsearch]
 # Elasticsearch database name
 dbname = kibble

kibble/scanners/README.md

@@ -3,7 +3,7 @@ The Kibble Scanners collect information for the Kibble Suite.
 
 ## Setup instructions:
 
- - Edit conf/config.yaml to match your Kibble service
+ - Edit kibble.ini to match your Kibble service
 
 ## How to run:
 

kibble/scanners/brokers/kibbleES.py

@@ -119,14 +119,13 @@ class KibbleBit:
     """ KibbleBit class with direct ElasticSearch access """
 
     def __init__(self, broker, organisation, tid):
-        self.config = broker.config
         self.organisation = organisation
         self.broker = broker
         self.json_queue = []
         self.queueMax = 1000  # Entries to keep before bulk pushing
         self.pluginname = ""
         self.tid = tid
-        self.dbname = self.broker.config["elasticsearch"]["database"]
+        self.dbname = conf.get("elasticsearch", "database")
 
     def __del__(self):
         """ On unload/delete, push the last chunks of data to ES """
@@ -144,7 +143,7 @@ def pprint(self, string, err=False):
     def update_source(self, source):
         """ Updates a source document, usually with a status update """
         self.broker.DB.index(
-            index=self.broker.config["elasticsearch"]["database"],
+            index=self.dbname,
             doc_type="source",
             id=source["sourceID"],
             body=source,
@@ -153,7 +152,7 @@ def update_source(self, source):
     def get(self, doctype, docid):
         """ Fetches a document from the DB """
         doc = self.broker.DB.get(
-            index=self.broker.config["elasticsearch"]["database"],
+            index=self.dbname,
             doc_type=doctype,
             id=docid,
         )
@@ -164,14 +163,14 @@ def get(self, doctype, docid):
     def exists(self, doctype, docid):
         """ Checks whether a document already exists or not """
         return self.broker.DB.exists(
-            index=self.broker.config["elasticsearch"]["database"],
+            index=self.dbname,
             doc_type=doctype,
             id=docid,
         )
 
     def index(self, doctype, docid, document):
         """ Adds a new document to the index """
-        dbname = self.broker.config["elasticsearch"]["database"]
+        dbname = self.dbname
         self.broker.DB.index(index=dbname, doc_type=doctype, id=docid, body=document)
 
     def append(self, t, doc):
@@ -195,7 +194,7 @@ def bulk(self):
             js = entry
             doc = js
             js["@version"] = 1
-            dbname = self.broker.config["elasticsearch"]["database"]
+            dbname = self.dbname
             if self.broker.noTypes:
                 dbname += "_%s" % js["doctype"]
                 js_arr.append(
@@ -233,6 +232,7 @@ def __init__(self, broker, org):
 
         self.broker = broker
         self.id = org
+        self.dbname = conf.get("elasticsearch", "database")
 
     def sources(self, sourceType=None, view=None):
         """ Get all sources or sources of a specific type for an org """
@@ -241,7 +241,7 @@ def sources(self, sourceType=None, view=None):
         mustArray = [{"term": {"organisation": self.id}}]
         if view:
             res = self.broker.DB.get(
-                index=self.broker.config["elasticsearch"]["database"],
+                index=self.dbname,
                 doc_type="view",
                 id=view,
             )
@@ -252,7 +252,7 @@ def sources(self, sourceType=None, view=None):
             mustArray.append({"term": {"type": sourceType}})
         # Run the search, fetch all results, 9999 max. TODO: Scroll???
         res = self.broker.DB.search(
-            index=self.broker.config["elasticsearch"]["database"],
+            index=self.dbname,
             doc_type="source",
             size=9999,
             body={"query": {"bool": {"must": mustArray}}, "sort": {"sourceURL": "asc"}},

kibble/scanners/config.yaml

@@ -24,6 +24,8 @@
 import tempfile
 import time
 
+from kibble.configuration import conf
+
 title = "Census Scanner for Git"
 version = "0.1.0"
 
@@ -50,7 +52,7 @@ def scan(kibble_bit, source):
     rid = source["sourceID"]
     url = source["sourceURL"]
     rootpath = "%s/%s/git" % (
-        kibble_bit.config["scanner"]["scratchdir"],
+        conf.get("scanner", "scratchdir"),
         source["organisation"],
     )
     gpath = os.path.join(rootpath, rid)

kibble/scanners/scanners/git-census.py

@@ -23,6 +23,7 @@
 import subprocess
 import time
 
+from kibble.configuration import conf
 from kibble.scanners.utils import sloc
 
 title = "Git Evolution Scanner"
@@ -138,7 +139,7 @@ def scan(kibble_bit, source):
 
     rid = source["sourceID"]
     rootpath = "%s/%s/git" % (
-        kibble_bit.config["scanner"]["scratchdir"],
+        conf.get("scanner", "scratchdir"),
         source["organisation"],
     )
     gpath = os.path.join(rootpath, rid)
@@ -158,7 +159,7 @@ def scan(kibble_bit, source):
         rid = source["sourceID"]
         url = source["sourceURL"]
         rootpath = "%s/%s/git" % (
-            kibble_bit.config["scanner"]["scratchdir"],
+            conf.get("scanner", "scratchdir"),
             source["organisation"],
         )
         gpath = os.path.join(rootpath, rid)

kibble/scanners/scanners/git-evolution.py

@@ -19,6 +19,7 @@
 import subprocess
 import time
 
+from kibble.configuration import conf
 from kibble.scanners.utils import git, sloc
 
 """ Source Lines of Code counter for Git """
@@ -42,7 +43,7 @@ def scan(kibble_bit, source):
     rid = source["sourceID"]
     url = source["sourceURL"]
     rootpath = "%s/%s/git" % (
-        kibble_bit.config["scanner"]["scratchdir"],
+        conf.get("scanner", "scratchdir"),
         source["organisation"],
     )
     gpath = os.path.join(rootpath, rid)
@@ -58,7 +59,7 @@ def scan(kibble_bit, source):
         kibble_bit.update_source(source)
 
         try:
-            branch = git.defaultBranch(source, gpath)
+            branch = git.default_branch(source, gpath)
             subprocess.call("cd %s && git checkout %s" % (gpath, branch), shell=True)
         except:  # pylint: disable=bare-except
             kibble_bit.pprint("SLoC counter failed to find main branch for %s!!" % url)

kibble/scanners/scanners/git-sloc.py

@@ -19,6 +19,7 @@
 import subprocess
 import time
 
+from kibble.configuration import conf
 from kibble.scanners.utils import git
 
 title = "Sync plugin for Git repositories"
@@ -41,7 +42,7 @@ def scan(kibble_bit, source):
     path = source["sourceID"]
     url = source["sourceURL"]
     rootpath = "%s/%s/git" % (
-        kibble_bit.config["scanner"]["scratchdir"],
+        conf.get("scanner", "scratchdir"),
         source["organisation"],
     )
 
@@ -79,7 +80,7 @@ def scan(kibble_bit, source):
             kibble_bit.pprint("Repo %s exists, fetching changes..." % datapath)
 
             # Do we have a default branch here?
-            branch = git.defaultBranch(source, datapath, kibble_bit)
+            branch = git.default_branch(source, datapath)
             if len(branch) == 0:
                 source["default_branch"] = branch
                 source["steps"]["sync"] = {

kibble/scanners/scanners/git-sync.py

@@ -19,6 +19,7 @@
 import time
 
 from kibble.scanners.utils import jsonapi, kpe
+from kibble.settings import AZURE_ENABLED, PICOAPI_ENABLED, WATSON_ENABLED
 
 """
 This is a Kibble scanner plugin for Apache Pony Mail sources.
@@ -64,7 +65,7 @@ def scan(kibble_bit, source):
         kibble_bit.update_source(source)
         return
 
-    if not "azure" in kibble_bit.config and not "picoapi" in kibble_bit.config:
+    if not AZURE_ENABLED and not PICOAPI_ENABLED:
         kibble_bit.pprint(
             "No Azure/picoAPI creds configured, skipping key phrase extraction"
         )
@@ -110,12 +111,12 @@ def scan(kibble_bit, source):
         bodies.append(body)
     if bodies:
         KPEs = None
-        if "watson" in kibble_bit.config:
+        if WATSON_ENABLED:
             pass  # Haven't written this yet
-        elif "azure" in kibble_bit.config:
-            KPEs = kpe.azureKPE(kibble_bit, bodies)
-        elif "picoapi" in kibble_bit.config:
-            KPEs = kpe.picoKPE(kibble_bit, bodies)
+        elif AZURE_ENABLED:
+            KPEs = kpe.azure_kpe(kibble_bit, bodies)
+        elif PICOAPI_ENABLED:
+            KPEs = kpe.pico_kpe(kibble_bit, bodies)
         if not KPEs:
             kibble_bit.pprint("Hit rate limit, not trying further emails for now.")
 

kibble/scanners/scanners/ponymail-kpe.py

@@ -21,7 +21,9 @@
 import re
 import time
 
+from kibble.configuration import conf
 from kibble.scanners.utils import jsonapi, tone
+from kibble.settings import AZURE_ENABLED, PICOAPI_ENABLED, WATSON_ENABLED
 
 title = "Tone/Mood Scanner plugin for Apache Pony Mail"
 version = "0.1.0"
@@ -61,11 +63,7 @@ def scan(kibble_bit, source):
         kibble_bit.update_source(source)
         return
 
-    if (
-        not "watson" in kibble_bit.config
-        and not "azure" in kibble_bit.config
-        and not "picoapi" in kibble_bit.config
-    ):
+    if not WATSON_ENABLED and not AZURE_ENABLED and not PICOAPI_ENABLED:
         kibble_bit.pprint(
             "No Watson/Azure/picoAPI creds configured, skipping tone analyzer"
         )
@@ -110,12 +108,12 @@ def scan(kibble_bit, source):
         bodies.append(body)
     if bodies:
         moods = None
-        if "watson" in kibble_bit.config:
-            moods = tone.watsonTone(kibble_bit, bodies)
-        elif "azure" in kibble_bit.config:
-            moods = tone.azureTone(kibble_bit, bodies)
-        elif "picoapi" in kibble_bit.config:
-            moods = tone.picoTone(kibble_bit, bodies)
+        if WATSON_ENABLED:
+            moods = tone.watson_tone(kibble_bit, bodies)
+        elif AZURE_ENABLED:
+            moods = tone.azure_tone(kibble_bit, bodies)
+        elif PICOAPI_ENABLED:
+            moods = tone.pico_tone(kibble_bit, bodies)
         if not moods:
             kibble_bit.pprint("Hit rate limit, not trying further emails for now.")
 

kibble/scanners/scanners/ponymail-tone.py

@@ -17,21 +17,20 @@
 
 """ This is the Kibble git utility plugin """
 
-import os
 import re
 import subprocess
-import sys
 
+from kibble.configuration import conf
 
-def defaultBranch(source, datapath, KibbleBit=None):
+
+def default_branch(source, datapath):
     """ Tries to figure out what the main branch of a repo is """
-    wanted_branches = ["master", "main", "trunk"]
-    branch = ""
     # If we have an override of branches we like, use 'em
-    if KibbleBit and KibbleBit.config.get("git"):
-        wanted_branches = KibbleBit.config["git"].get(
-            "wanted_branches", wanted_branches
-        )
+    wanted_branches = conf.get("git", "wanted_branches", fallback=None)
+    if wanted_branches:
+        wanted_branches = wanted_branches.split(",")
+    else:
+        wanted_branches = ["master", "main", "trunk"]
 
     # For each wanted branch, in order, look for it in our clone,
     # and return the name if found.