diff --git a/kibble/scanners/brokers/kibbleES.py b/kibble/scanners/brokers/kibbleES.py index 4d372f63..5e6eb0db 100644 --- a/kibble/scanners/brokers/kibbleES.py +++ b/kibble/scanners/brokers/kibbleES.py @@ -141,7 +141,7 @@ def pprint(self, string, err=False): else: print(line) - def updateSource(self, source): + def update_source(self, source): """ Updates a source document, usually with a status update """ self.broker.DB.index( index=self.broker.config["elasticsearch"]["database"], diff --git a/kibble/scanners/scanners/bugzilla.py b/kibble/scanners/scanners/bugzilla.py index e1f1bfb1..fa1833ce 100644 --- a/kibble/scanners/scanners/bugzilla.py +++ b/kibble/scanners/scanners/bugzilla.py @@ -43,7 +43,7 @@ def accepts(source): return False -def getTime(string): +def get_time(string): return time.mktime( time.strptime(re.sub(r"[zZ]", "", str(string)), "%Y-%m-%dT%H:%M:%S") ) @@ -81,7 +81,7 @@ def moved(js): return False -def wasclosed(js): +def was_closed(js): if "changelog" in js: cjs = js["changelog"]["histories"] for citem in cjs: @@ -118,7 +118,7 @@ def pchange(js): return False -def scanTicket(bug, KibbleBit, source, openTickets, u, dom): +def scan_ticket(bug, kibble_bit, source, open_tickets, u, dom): try: key = bug["id"] dhash = hashlib.sha224( @@ -126,24 +126,24 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom): "ascii", errors="replace" ) ).hexdigest() - found = KibbleBit.exists("issue", dhash) + found = kibble_bit.exists("issue", dhash) parseIt = False if not found: parseIt = True else: - ticket = KibbleBit.get("issue", dhash) - if ticket["status"] == "closed" and key in openTickets: - KibbleBit.pprint("Ticket was reopened, reparsing") + ticket = kibble_bit.get("issue", dhash) + if ticket["status"] == "closed" and key in open_tickets: + kibble_bit.pprint("Ticket was reopened, reparsing") parseIt = True - elif ticket["status"] == "open" and not key in openTickets: - KibbleBit.pprint("Ticket was recently closed, parsing it") + elif ticket["status"] == "open" and not key in open_tickets: + kibble_bit.pprint("Ticket was recently closed, parsing it") parseIt = True else: pass # print("Ticket hasn't changed, ignoring...") if parseIt: - KibbleBit.pprint("Parsing data from BugZilla for #%s" % key) + kibble_bit.pprint("Parsing data from BugZilla for #%s" % key) params = {"ids": [int(key)], "limit": 0} if ( @@ -163,12 +163,12 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom): js = js["result"]["bugs"][0] creator = {"name": bug["creator"], "email": js["creator"]} closer = {} - cd = getTime(js["creation_time"]) + cd = get_time(js["creation_time"]) rd = None status = "open" if js["status"] in ["CLOSED", "RESOLVED"]: status = "closed" - KibbleBit.pprint("%s was closed, finding out who did that" % key) + kibble_bit.pprint("%s was closed, finding out who did that" % key) ticketsURL = "%s?method=Bug.history¶ms=[%s]" % ( u, urllib.parse.quote(json.dumps(params)), @@ -182,10 +182,10 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom): and "added" in change and change["added"] in ["CLOSED", "RESOLVED"] ): - rd = getTime(item["when"]) + rd = get_time(item["when"]) closer = {"name": item["who"], "email": item["who"]} break - KibbleBit.pprint("Counting comments for %s..." % key) + kibble_bit.pprint("Counting comments for %s..." % key) ticketsURL = "%s?method=Bug.comments¶ms=[%s]" % ( u, urllib.parse.quote(json.dumps(params)), @@ -202,7 +202,7 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom): "ascii", errors="replace" ) ).hexdigest() - found = KibbleBit.exists("person", pid) + found = kibble_bit.exists("person", pid) if not found: params["names"] = [closer["email"]] ticketsURL = "%s?method=User.get¶ms=[%s]" % ( @@ -213,7 +213,7 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom): try: ujs = jsonapi.get(ticketsURL) displayName = ujs["result"]["users"][0]["real_name"] - except: + except: # pylint: disable=bare-except displayName = closer["email"] if displayName and len(displayName) > 0: # Add to people db @@ -225,7 +225,7 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom): "id": pid, } # print("Updating person DB for closer: %s (%s)" % (displayName, closerEmail)) - KibbleBit.index("person", pid, jsp) + kibble_bit.index("person", pid, jsp) if creator: pid = hashlib.sha1( @@ -233,7 +233,7 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom): "ascii", errors="replace" ) ).hexdigest() - found = KibbleBit.exists("person", pid) + found = kibble_bit.exists("person", pid) if not found: if not creator["name"]: params["names"] = [creator["email"]] @@ -244,7 +244,7 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom): try: ujs = jsonapi.get(ticketsURL) creator["name"] = ujs["result"]["users"][0]["real_name"] - except: + except: # pylint: disable=bare-except creator["name"] = creator["email"] if creator["name"] and len(creator["name"]) > 0: # Add to people db @@ -255,7 +255,7 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom): "organisation": source["organisation"], "id": pid, } - KibbleBit.index("person", pid, jsp) + kibble_bit.index("person", pid, jsp) jso = { "id": dhash, @@ -280,17 +280,17 @@ def scanTicket(bug, KibbleBit, source, openTickets, u, dom): "comments": comments, "title": title, } - KibbleBit.append("issue", jso) + kibble_bit.append("issue", jso) time.sleep(0.5) # BugZilla is notoriously slow. Maybe remove this later return True except Exception as err: - KibbleBit.pprint(err) + kibble_bit.pprint(err) return False -class bzThread(Thread): +class BzThread(Thread): def __init__(self, KibbleBit, source, block, pt, ot, u, dom): - super(bzThread, self).__init__() + super(BzThread, self).__init__() self.KibbleBit = KibbleBit self.source = source self.block = block @@ -300,9 +300,9 @@ def __init__(self, KibbleBit, source, block, pt, ot, u, dom): self.dom = dom def run(self): - badOnes = 0 + bad_ones = 0 - while len(self.pendingTickets) > 0 and badOnes <= 50: + while len(self.pendingTickets) > 0 and bad_ones <= 50: if len(self.pendingTickets) % 10 == 0: self.KibbleBit.pprint( "%u elements left to count" % len(self.pendingTickets) @@ -317,12 +317,12 @@ def run(self): self.block.release() return self.block.release() - if not scanTicket( + if not scan_ticket( rl, self.KibbleBit, self.source, self.openTickets, self.u, self.dom ): self.KibbleBit.pprint("Ticket %s seems broken, skipping" % rl["id"]) - badOnes += 1 - if badOnes > 50: + bad_ones += 1 + if bad_ones > 50: self.KibbleBit.pprint("Too many errors, bailing!") self.source["steps"]["issues"] = { "time": time.time(), @@ -331,13 +331,13 @@ def run(self): "running": False, "good": False, } - self.KibbleBit.updateSource(self.source) + self.KibbleBit.update_source(self.source) return else: - badOnes = 0 + bad_ones = 0 -def scan(KibbleBit, source): +def scan(kibble_bit, source): url = source["sourceURL"] source["steps"]["issues"] = { @@ -346,7 +346,7 @@ def scan(KibbleBit, source): "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) bz = re.match(r"(https?://\S+?)(/jsonrpc\.cgi)?[\s:?]+(.+)", url) if bz: @@ -357,8 +357,8 @@ def scan(KibbleBit, source): and len(source["creds"]["username"]) > 0 ): creds = "%s:%s" % (source["creds"]["username"], source["creds"]["password"]) - pendingTickets = [] - openTickets = [] + pending_tickets = [] + open_tickets = [] # Get base URL, list and domain to parse dom = bz.group(1) @@ -404,20 +404,20 @@ def scan(KibbleBit, source): "offset": 1, } - ticketsURL = "%s?method=Bug.search¶ms=[%s]" % ( + tickets_url = "%s?method=Bug.search¶ms=[%s]" % ( u, urllib.parse.quote(json.dumps(params)), ) while True: try: - js = jsonapi.get(ticketsURL, auth=creds) - except: - KibbleBit.pprint("Couldn't fetch more tickets, bailing") + js = jsonapi.get(tickets_url, auth=creds) + except: # pylint: disable=bare-except + kibble_bit.pprint("Couldn't fetch more tickets, bailing") break if len(js["result"]["bugs"]) > 0: - KibbleBit.pprint( + kibble_bit.pprint( "%s: Found %u tickets..." % ( source["sourceURL"], @@ -425,28 +425,30 @@ def scan(KibbleBit, source): ) ) for bug in js["result"]["bugs"]: - pendingTickets.append(bug) + pending_tickets.append(bug) if not bug["status"] in ["RESOLVED", "CLOSED"]: - openTickets.append(bug["id"]) + open_tickets.append(bug["id"]) params["offset"] += 10000 - ticketsURL = "%s?method=Bug.search¶ms=[%s]" % ( + tickets_url = "%s?method=Bug.search¶ms=[%s]" % ( u, urllib.parse.quote(json.dumps(params)), ) else: - KibbleBit.pprint("No more tickets left to scan") + kibble_bit.pprint("No more tickets left to scan") break - KibbleBit.pprint( + kibble_bit.pprint( "Found %u open tickets, %u closed." - % (len(openTickets), len(pendingTickets) - len(openTickets)) + % (len(open_tickets), len(pending_tickets) - len(open_tickets)) ) block = Lock() threads = [] # TODO: Fix this loop for i in range(0, 4): - t = bzThread(KibbleBit, source, block, pendingTickets, openTickets, u, dom) + t = BzThread( + kibble_bit, source, block, pending_tickets, open_tickets, u, dom + ) threads.append(t) t.start() @@ -460,4 +462,4 @@ def scan(KibbleBit, source): "running": False, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) diff --git a/kibble/scanners/scanners/buildbot.py b/kibble/scanners/scanners/buildbot.py index 190fb5e3..6d2e18d2 100644 --- a/kibble/scanners/scanners/buildbot.py +++ b/kibble/scanners/scanners/buildbot.py @@ -38,7 +38,7 @@ def accepts(source): return False -def scanJob(KibbleBit, source, job, creds): +def scan_job(KibbleBit, source, job, creds): """ Scans a single job for activity """ dhash = hashlib.sha224( ("%s-%s-%s" % (source["organisation"], source["sourceID"], job)).encode( @@ -64,7 +64,7 @@ def scanJob(KibbleBit, source, job, creds): builddoc = None try: builddoc = KibbleBit.get("ci_build", buildhash) - except: + except: # pylint: disable=bare-except pass # If this build already completed, no need to parse it again @@ -120,11 +120,11 @@ def scanJob(KibbleBit, source, job, creds): return False -class buildbotThread(threading.Thread): +class BuildbotThread(threading.Thread): """ Generic thread class for scheduling multiple scans at once """ def __init__(self, block, KibbleBit, source, creds, jobs): - super(buildbotThread, self).__init__() + super(BuildbotThread, self).__init__() self.block = block self.KibbleBit = KibbleBit self.creds = creds @@ -132,8 +132,8 @@ def __init__(self, block, KibbleBit, source, creds, jobs): self.jobs = jobs def run(self): - badOnes = 0 - while len(self.jobs) > 0 and badOnes <= 50: + bad_ones = 0 + while len(self.jobs) > 0 and bad_ones <= 50: self.block.acquire() try: job = self.jobs.pop(0) @@ -144,10 +144,10 @@ def run(self): self.block.release() return self.block.release() - if not scanJob(self.KibbleBit, self.source, job, self.creds): + if not scan_job(self.KibbleBit, self.source, job, self.creds): self.KibbleBit.pprint("[%s] This borked, trying another one" % job) - badOnes += 1 - if badOnes > 100: + bad_ones += 1 + if bad_ones > 100: self.KibbleBit.pprint("Too many errors, bailing!") self.source["steps"]["ci"] = { "time": time.time(), @@ -156,13 +156,13 @@ def run(self): "running": False, "good": False, } - self.KibbleBit.updateSource(self.source) + self.KibbleBit.update_source(self.source) return else: - badOnes = 0 + bad_ones = 0 -def scan(KibbleBit, source): +def scan(kibble_bit, source): # Simple URL check buildbot = re.match(r"(https?://.+)", source["sourceURL"]) if buildbot: @@ -173,16 +173,16 @@ def scan(KibbleBit, source): "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) - KibbleBit.pprint("Parsing Buildbot activity at %s" % source["sourceURL"]) + kibble_bit.pprint("Parsing Buildbot activity at %s" % source["sourceURL"]) source["steps"]["ci"] = { "time": time.time(), "status": "Downloading changeset", "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) # Buildbot may neeed credentials creds = None @@ -195,9 +195,9 @@ def scan(KibbleBit, source): creds = "%s:%s" % (source["creds"]["username"], source["creds"]["password"]) # Get the job list - sURL = source["sourceURL"] - KibbleBit.pprint("Getting job list...") - builders = jsonapi.get("%s/json/builders" % sURL, auth=creds) + s_url = source["sourceURL"] + kibble_bit.pprint("Getting job list...") + builders = jsonapi.get("%s/json/builders" % s_url, auth=creds) # Save queue snapshot NOW = int(datetime.datetime.utcnow().timestamp()) @@ -211,8 +211,8 @@ def scan(KibbleBit, source): # Scan queue items blocked = 0 stuck = 0 - queueSize = 0 - actualQueueSize = 0 + queue_size = 0 + actual_queue_size = 0 building = 0 jobs = [] @@ -222,10 +222,10 @@ def scan(KibbleBit, source): building += 1 if data.get("pendingBuilds", 0) > 0: # All queued items, even offline builders - actualQueueSize += data.get("pendingBuilds", 0) + actual_queue_size += data.get("pendingBuilds", 0) # Only queues with an online builder (actually waiting stuff) if data["state"] == "building": - queueSize += data.get("pendingBuilds", 0) + queue_size += data.get("pendingBuilds", 0) blocked += data.get("pendingBuilds", 0) # Blocked by running builds # Stuck builds (iow no builder available) if data["state"] == "offline": @@ -236,7 +236,7 @@ def scan(KibbleBit, source): "id": queuehash, "date": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(NOW)), "time": NOW, - "size": queueSize, + "size": queue_size, "blocked": blocked, "stuck": stuck, "building": building, @@ -246,15 +246,15 @@ def scan(KibbleBit, source): "organisation": source["organisation"], "upsert": True, } - KibbleBit.append("ci_queue", queuedoc) + kibble_bit.append("ci_queue", queuedoc) - KibbleBit.pprint("Found %u builders in Buildbot" % len(jobs)) + kibble_bit.pprint("Found %u builders in Buildbot" % len(jobs)) threads = [] block = threading.Lock() - KibbleBit.pprint("Scanning jobs using 4 sub-threads") + kibble_bit.pprint("Scanning jobs using 4 sub-threads") for i in range(0, 4): - t = buildbotThread(block, KibbleBit, source, creds, jobs) + t = BuildbotThread(block, kibble_bit, source, creds, jobs) threads.append(t) t.start() @@ -262,7 +262,7 @@ def scan(KibbleBit, source): t.join() # We're all done, yaay - KibbleBit.pprint("Done scanning %s" % source["sourceURL"]) + kibble_bit.pprint("Done scanning %s" % source["sourceURL"]) source["steps"]["ci"] = { "time": time.time(), @@ -271,4 +271,4 @@ def scan(KibbleBit, source): "running": False, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) diff --git a/kibble/scanners/scanners/discourse.py b/kibble/scanners/scanners/discourse.py index ee99dde6..4e1d92f0 100644 --- a/kibble/scanners/scanners/discourse.py +++ b/kibble/scanners/scanners/discourse.py @@ -39,11 +39,11 @@ def accepts(source): return False -def scanJob(KibbleBit, source, cat, creds): +def scan_job(kibble_bit, source, cat, creds): """ Scans a single discourse category for activity """ # Get $discourseURL/c/$catID catURL = os.path.join(source["sourceURL"], "c/%s" % cat["id"]) - KibbleBit.pprint("Scanning Discourse category '%s' at %s" % (cat["slug"], catURL)) + kibble_bit.pprint("Scanning Discourse category '%s' at %s" % (cat["slug"], catURL)) page = 0 allUsers = {} @@ -91,8 +91,8 @@ def scanJob(KibbleBit, source, cat, creds): # Store it (or, queue storage) unless it exists. # We don't wanna override better data, so we check if # it's there first. - if not KibbleBit.exists("person", dhash): - KibbleBit.append("person", userDoc) + if not kibble_bit.exists("person", dhash): + kibble_bit.append("person", userDoc) # Now, for each topic, we'll store a topic document for topic in catjson["topic_list"]["topics"]: @@ -118,8 +118,8 @@ def scanJob(KibbleBit, source, cat, creds): # Determine whether we should scan this topic or continue to the next one. # We'll do this by seeing if the topic already exists and has no changes or not. - if KibbleBit.exists("forum_topic", dhash): - fdoc = KibbleBit.get("forum_topic", dhash) + if kibble_bit.exists("forum_topic", dhash): + fdoc = kibble_bit.get("forum_topic", dhash) # If update in the old doc was >= current update timestamp, skip the topic if fdoc["updated"] >= UpdatedDate: continue @@ -146,8 +146,8 @@ def scanJob(KibbleBit, source, cat, creds): + "/t/%s/%s" % (topic["slug"], topic["id"]), } - KibbleBit.append("forum_topic", topicdoc) - KibbleBit.pprint("%s is new or changed, scanning" % topicdoc["url"]) + kibble_bit.append("forum_topic", topicdoc) + kibble_bit.pprint("%s is new or changed, scanning" % topicdoc["url"]) # Now grab all the individual replies/posts # Remember to not have it count as a visit! @@ -157,7 +157,7 @@ def scanJob(KibbleBit, source, cat, creds): posts = pjson["post_stream"]["posts"] # For each post/reply, construct a forum_entry document - KibbleBit.pprint("%s has %u posts" % (pURL, len(posts))) + kibble_bit.pprint("%s has %u posts" % (pURL, len(posts))) for post in posts: phash = hashlib.sha224( ( @@ -199,7 +199,7 @@ def scanJob(KibbleBit, source, cat, creds): allUsers[user["id"]] = userDoc # Store it (or, queue storage) - KibbleBit.append("person", userDoc) + kibble_bit.append("person", userDoc) # Get post date CreatedDate = datetime.datetime.strptime( @@ -222,18 +222,18 @@ def scanJob(KibbleBit, source, cat, creds): "text": post["cooked"], "url": topicdoc["url"], } - KibbleBit.append("forum_post", pdoc) + kibble_bit.append("forum_post", pdoc) else: - KibbleBit.pprint("Fetching discourse data failed!") + kibble_bit.pprint("Fetching discourse data failed!") return False return True -class discourseThread(threading.Thread): +class DiscourseThread(threading.Thread): """ Generic thread class for scheduling multiple scans at once """ def __init__(self, block, KibbleBit, source, creds, jobs): - super(discourseThread, self).__init__() + super(DiscourseThread, self).__init__() self.block = block self.KibbleBit = KibbleBit self.creds = creds @@ -241,8 +241,8 @@ def __init__(self, block, KibbleBit, source, creds, jobs): self.jobs = jobs def run(self): - badOnes = 0 - while len(self.jobs) > 0 and badOnes <= 50: + bad_ones = 0 + while len(self.jobs) > 0 and bad_ones <= 50: self.block.acquire() try: job = self.jobs.pop(0) @@ -253,12 +253,12 @@ def run(self): self.block.release() return self.block.release() - if not scanJob(self.KibbleBit, self.source, job, self.creds): + if not scan_job(self.KibbleBit, self.source, job, self.creds): self.KibbleBit.pprint( "[%s] This borked, trying another one" % job["name"] ) - badOnes += 1 - if badOnes > 10: + bad_ones += 1 + if bad_ones > 10: self.KibbleBit.pprint("Too many errors, bailing!") self.source["steps"]["forum"] = { "time": time.time(), @@ -267,13 +267,13 @@ def run(self): "running": False, "good": False, } - self.KibbleBit.updateSource(self.source) + self.KibbleBit.update_source(self.source) return else: - badOnes = 0 + bad_ones = 0 -def scan(KibbleBit, source): +def scan(kibble_bit, source): # Simple URL check discourse = re.match(r"(https?://.+)", source["sourceURL"]) if discourse: @@ -284,17 +284,17 @@ def scan(KibbleBit, source): "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) pendingJobs = [] - KibbleBit.pprint("Parsing Discourse activity at %s" % source["sourceURL"]) + kibble_bit.pprint("Parsing Discourse activity at %s" % source["sourceURL"]) source["steps"]["forum"] = { "time": time.time(), "status": "Downloading changeset", "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) # Discourse may neeed credentials (if basic auth) creds = None @@ -308,20 +308,20 @@ def scan(KibbleBit, source): # Get the list of categories sURL = source["sourceURL"] - KibbleBit.pprint("Getting categories...") + kibble_bit.pprint("Getting categories...") catjs = jsonapi.get("%s/categories_and_latest" % sURL, auth=creds) # Directly assign the category list as pending jobs queue, ezpz. pendingJobs = catjs["category_list"]["categories"] - KibbleBit.pprint("Found %u categories" % len(pendingJobs)) + kibble_bit.pprint("Found %u categories" % len(pendingJobs)) # Now fire off 4 threads to parse the categories threads = [] block = threading.Lock() - KibbleBit.pprint("Scanning jobs using 4 sub-threads") + kibble_bit.pprint("Scanning jobs using 4 sub-threads") for i in range(0, 4): - t = discourseThread(block, KibbleBit, source, creds, pendingJobs) + t = DiscourseThread(block, kibble_bit, source, creds, pendingJobs) threads.append(t) t.start() @@ -329,7 +329,7 @@ def scan(KibbleBit, source): t.join() # We're all done, yaay - KibbleBit.pprint("Done scanning %s" % source["sourceURL"]) + kibble_bit.pprint("Done scanning %s" % source["sourceURL"]) source["steps"]["forum"] = { "time": time.time(), @@ -338,4 +338,4 @@ def scan(KibbleBit, source): "running": False, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) diff --git a/kibble/scanners/scanners/gerrit.py b/kibble/scanners/scanners/gerrit.py index b82dab6a..fa60dffd 100644 --- a/kibble/scanners/scanners/gerrit.py +++ b/kibble/scanners/scanners/gerrit.py @@ -160,15 +160,15 @@ def make_person(repo, raw_person): } -def update_issue(KibbleBit, issue): +def update_issue(kibble_bit, issue): id = issue["id"] - KibbleBit.pprint("Updating issue: " + id) - KibbleBit.index("issue", id, issue) + kibble_bit.pprint("Updating issue: " + id) + kibble_bit.index("issue", id, issue) -def update_person(KibbleBit, person): - KibbleBit.pprint("Updating person: " + person["name"] + " - " + person["email"]) - KibbleBit.index("person", person["id"], {"doc": person, "doc_as_upsert": True}) +def update_person(kibble_bit, person): + kibble_bit.pprint("Updating person: " + person["name"] + " - " + person["email"]) + kibble_bit.index("person", person["id"], {"doc": person, "doc_as_upsert": True}) def status_changed(stored_change, change): @@ -177,14 +177,14 @@ def status_changed(stored_change, change): return stored_change["status"] != change["status"] -def scan(KibbleBit, source): +def scan(kibble_bit, source): source["steps"]["issues"] = { "time": time.time(), "status": "Analyzing Gerrit tickets...", "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) url = source["sourceURL"] # Try matching foo.bar/r/project/subfoo @@ -217,8 +217,8 @@ def scan(KibbleBit, source): dhash = make_hash(source, change) stored_change = None - if KibbleBit.exists("issue", dhash): - stored_change = KibbleBit.get("issue", dhash) + if kibble_bit.exists("issue", dhash): + stored_change = kibble_bit.get("issue", dhash) if not status_changed(stored_change, change): # print("change %s seen already and status unchanged. Skipping." % @@ -228,7 +228,7 @@ def scan(KibbleBit, source): details = change_details(base_url, change) issue_doc = make_issue(source, base_url, details) - update_issue(KibbleBit, issue_doc) + update_issue(kibble_bit, issue_doc) labels = details["labels"] change_people = [] @@ -247,7 +247,7 @@ def scan(KibbleBit, source): for person in change_people: if "email" in person and person["email"] not in people: people[person["email"]] = person - update_person(KibbleBit, make_person(source, person)) + update_person(kibble_bit, make_person(source, person)) except requests.HTTPError as e: print(e) @@ -258,4 +258,4 @@ def scan(KibbleBit, source): "running": False, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) diff --git a/kibble/scanners/scanners/git-census.py b/kibble/scanners/scanners/git-census.py index aa19a037..790ae492 100644 --- a/kibble/scanners/scanners/git-census.py +++ b/kibble/scanners/scanners/git-census.py @@ -38,7 +38,7 @@ def accepts(source): return False -def scan(KibbleBit, source): +def scan(kibble_bit, source): """ Conduct a census scan """ people = {} idseries = {} @@ -50,7 +50,7 @@ def scan(KibbleBit, source): rid = source["sourceID"] url = source["sourceURL"] rootpath = "%s/%s/git" % ( - KibbleBit.config["scanner"]["scratchdir"], + kibble_bit.config["scanner"]["scratchdir"], source["organisation"], ) gpath = os.path.join(rootpath, rid) @@ -63,9 +63,8 @@ def scan(KibbleBit, source): "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) gname = rid - inp = "" modificationDates = {} # Did we do a census before? if "census" in source and source["census"] > 0: @@ -92,7 +91,7 @@ def scan(KibbleBit, source): inp = f.read() f.close() os.unlink(tmp.name) - KibbleBit.pprint("Parsing log for %s (%s)..." % (rid, url)) + kibble_bit.pprint("Parsing log for %s (%s)..." % (rid, url)) for m in re.finditer( u":([a-f0-9]+)\|([^\r\n|]+)\|([^\r\n|]+)\|([^\r\n|]+)\|([^\r\n|]+)\|([\d+]+)\r?\n([^:]+?):", inp, @@ -145,7 +144,7 @@ def scan(KibbleBit, source): if delete > 100000000: delete = 0 if delete > 1000000 or insert > 1000000: - KibbleBit.pprint( + kibble_bit.pprint( "gigantic diff for %s (%s), ignoring" % (gpath, source["sourceURL"]) ) @@ -250,7 +249,7 @@ def scan(KibbleBit, source): "vcs": "git", "files_changed": filelist, } - KibbleBit.append( + kibble_bit.append( "person", { "upsert": True, @@ -265,7 +264,7 @@ def scan(KibbleBit, source): ).hexdigest(), }, ) - KibbleBit.append( + kibble_bit.append( "person", { "upsert": True, @@ -280,11 +279,11 @@ def scan(KibbleBit, source): ).hexdigest(), }, ) - KibbleBit.append("code_commit", js) - KibbleBit.append("code_commit_unique", jsx) + kibble_bit.append("code_commit", js) + kibble_bit.append("code_commit_unique", jsx) if True: # Do file changes?? Might wanna make this optional - KibbleBit.pprint("Scanning file changes for %s" % source["sourceURL"]) + kibble_bit.pprint("Scanning file changes for %s" % source["sourceURL"]) for filename in modificationDates: fid = hashlib.sha1( ("%s/%s" % (source["sourceID"], filename)).encode( @@ -310,11 +309,11 @@ def scan(KibbleBit, source): time.gmtime(modificationDates[filename]["created"]), ), } - found = KibbleBit.exists("file_history", fid) + found = kibble_bit.exists("file_history", fid) if found: del jsfe["created"] del jsfe["createdDate"] - KibbleBit.append("file_history", jsfe) + kibble_bit.append("file_history", jsfe) source["steps"]["census"] = { "time": time.time(), @@ -324,4 +323,4 @@ def scan(KibbleBit, source): "good": True, } source["census"] = time.time() - KibbleBit.updateSource(source) + kibble_bit.update_source(source) diff --git a/kibble/scanners/scanners/git-evolution.py b/kibble/scanners/scanners/git-evolution.py index 07a7ada2..b533b4bb 100644 --- a/kibble/scanners/scanners/git-evolution.py +++ b/kibble/scanners/scanners/git-evolution.py @@ -46,12 +46,12 @@ def get_first_ref(gpath): % gpath, shell=True, ) - except: + except: # pylint: disable=bare-except print("Could not get first ref, exiting!") return None -def acquire(KibbleBit, source): +def acquire(kibble_bit, source): source["steps"]["evolution"] = { "time": time.time(), "status": "Evolution scan started at " @@ -59,10 +59,10 @@ def acquire(KibbleBit, source): "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) -def release(KibbleBit, source, status, exception=None, good=False): +def release(kibble_bit, source, status, exception=None, good=False): source["steps"]["evolution"] = { "time": time.time(), "status": status, @@ -72,7 +72,7 @@ def release(KibbleBit, source, status, exception=None, good=False): if exception: source["steps"]["evolution"].update({"exception": exception}) - KibbleBit.updateSource(source) + kibble_bit.update_source(source) def check_branch(gpath, date, branch): @@ -82,7 +82,7 @@ def check_branch(gpath, date, branch): shell=True, ) return True - except: + except: # pylint: disable=bare-except return False @@ -116,7 +116,7 @@ def find_branch(date, gpath): stderr=subprocess.DEVNULL, ) return "master" - except: + except: # pylint: disable=bare-except os.chdir(gpath) try: return ( @@ -129,22 +129,22 @@ def find_branch(date, gpath): .strip() .strip("* ") ) - except: + except: # pylint: disable=bare-except # print("meh! no branch") return None -def scan(KibbleBit, source): +def scan(kibble_bit, source): rid = source["sourceID"] rootpath = "%s/%s/git" % ( - KibbleBit.config["scanner"]["scratchdir"], + kibble_bit.config["scanner"]["scratchdir"], source["organisation"], ) gpath = os.path.join(rootpath, rid) gname = source["sourceID"] - KibbleBit.pprint("Doing evolution scan of %s" % gname) + kibble_bit.pprint("Doing evolution scan of %s" % gname) inp = get_first_ref(gpath) if inp: @@ -158,13 +158,13 @@ def scan(KibbleBit, source): rid = source["sourceID"] url = source["sourceURL"] rootpath = "%s/%s/git" % ( - KibbleBit.config["scanner"]["scratchdir"], + kibble_bit.config["scanner"]["scratchdir"], source["organisation"], ) gpath = os.path.join(rootpath, rid) if source["steps"]["sync"]["good"] and os.path.exists(gpath): - acquire(KibbleBit, source) + acquire(kibble_bit, source) branch = find_branch(date, gpath) if not branch: @@ -178,7 +178,7 @@ def scan(KibbleBit, source): branch_exists = check_branch(gpath, date, branch) if not branch_exists: - KibbleBit.pprint("Not trunk either (bad repo?), skipping") + kibble_bit.pprint("Not trunk either (bad repo?), skipping") release( source, "Could not do evolutionary scan of code", @@ -207,10 +207,10 @@ def scan(KibbleBit, source): dhash = hashlib.sha224( (source["sourceID"] + date).encode("ascii", "replace") ).hexdigest() - found = KibbleBit.exists("evolution", dhash) + found = kibble_bit.exists("evolution", dhash) if not found: checkout(gpath, date, branch) - KibbleBit.pprint( + kibble_bit.pprint( "Running cloc on %s (%s) at %s" % (gname, source["sourceURL"], date) ) @@ -229,7 +229,7 @@ def scan(KibbleBit, source): "cost": cost, "languages": languages, } - KibbleBit.index("evolution", dhash, js) + kibble_bit.index("evolution", dhash, js) quarter -= 3 if quarter <= 0: quarter += 12 @@ -238,9 +238,9 @@ def scan(KibbleBit, source): # decrease month by 3 now = time.mktime(datetime.date(year, quarter, 1).timetuple()) except Exception as e: - KibbleBit.pprint(e) + kibble_bit.pprint(e) release( - KibbleBit, + kibble_bit, source, "Evolution scan failed at " + time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()), @@ -249,7 +249,7 @@ def scan(KibbleBit, source): return release( - KibbleBit, + kibble_bit, source, "Evolution scan completed at " + time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()), diff --git a/kibble/scanners/scanners/git-sloc.py b/kibble/scanners/scanners/git-sloc.py index d9633577..8b44c543 100644 --- a/kibble/scanners/scanners/git-sloc.py +++ b/kibble/scanners/scanners/git-sloc.py @@ -37,12 +37,12 @@ def accepts(source): return False -def scan(KibbleBit, source): +def scan(kibble_bit, source): rid = source["sourceID"] url = source["sourceURL"] rootpath = "%s/%s/git" % ( - KibbleBit.config["scanner"]["scratchdir"], + kibble_bit.config["scanner"]["scratchdir"], source["organisation"], ) gpath = os.path.join(rootpath, rid) @@ -55,16 +55,16 @@ def scan(KibbleBit, source): "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) try: branch = git.defaultBranch(source, gpath) subprocess.call("cd %s && git checkout %s" % (gpath, branch), shell=True) - except: - KibbleBit.pprint("SLoC counter failed to find main branch for %s!!" % url) + except: # pylint: disable=bare-except + kibble_bit.pprint("SLoC counter failed to find main branch for %s!!" % url) return False - KibbleBit.pprint("Running SLoC count for %s" % url) + kibble_bit.pprint("Running SLoC count for %s" % url) languages, codecount, comment, blank, years, cost = sloc.count(gpath) sloc_ = { @@ -84,4 +84,4 @@ def scan(KibbleBit, source): "running": False, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) diff --git a/kibble/scanners/scanners/git-sync.py b/kibble/scanners/scanners/git-sync.py index e2e9422b..c1c40f6d 100644 --- a/kibble/scanners/scanners/git-sync.py +++ b/kibble/scanners/scanners/git-sync.py @@ -30,18 +30,18 @@ def accepts(source): if source["type"] == "git": return True # There are cases where we have a github repo, but don't wanna analyze the code, just issues - if source["type"] == "github" and source.get("issuesonly", False) == False: + if source["type"] == "github" and source.get("issuesonly", False) is False: return True return False -def scan(KibbleBit, source): +def scan(kibble_bit, source): # Get some vars, construct a data path for the repo path = source["sourceID"] url = source["sourceURL"] rootpath = "%s/%s/git" % ( - KibbleBit.config["scanner"]["scratchdir"], + kibble_bit.config["scanner"]["scratchdir"], source["organisation"], ) @@ -50,20 +50,20 @@ def scan(KibbleBit, source): try: os.makedirs(rootpath, exist_ok=True) print("Created root path %s" % rootpath) - except Exception as err: + except: # pylint: disable=bare-except source["steps"]["sync"] = { "time": time.time(), "status": "Could not create root scratch dir - permision denied?", "running": False, "good": False, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) return # This is were the repo should be cloned datapath = os.path.join(rootpath, path) - KibbleBit.pprint("Checking out %s as %s" % (url, path)) + kibble_bit.pprint("Checking out %s as %s" % (url, path)) try: source["steps"]["sync"] = { @@ -72,14 +72,14 @@ def scan(KibbleBit, source): "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) # If we already checked this out earlier, just sync it. if os.path.exists(datapath): - KibbleBit.pprint("Repo %s exists, fetching changes..." % datapath) + kibble_bit.pprint("Repo %s exists, fetching changes..." % datapath) # Do we have a default branch here? - branch = git.defaultBranch(source, datapath, KibbleBit) + branch = git.defaultBranch(source, datapath, kibble_bit) if len(branch) == 0: source["default_branch"] = branch source["steps"]["sync"] = { @@ -89,14 +89,14 @@ def scan(KibbleBit, source): "running": False, "good": False, } - KibbleBit.updateSource(source) - KibbleBit.pprint( + kibble_bit.update_source(source) + kibble_bit.pprint( "No default branch found for %s (%s)" % (source["sourceID"], source["sourceURL"]) ) return - KibbleBit.pprint("Using branch %s" % branch) + kibble_bit.pprint("Using branch %s" % branch) # Try twice checking out the main branch and fetching changes. # Sometimes we need to clean up after older scanners, which is # why we try twice. If first attempt fails, clean up and try again. @@ -137,11 +137,11 @@ def scan(KibbleBit, source): shell=True, stderr=subprocess.STDOUT, ) - except: + except: # pylint: disable=bare-except pass # This is a new repo, clone it! else: - KibbleBit.pprint("%s is new, cloning...!" % datapath) + kibble_bit.pprint("%s is new, cloning...!" % datapath) subprocess.check_output( "GIT_TERMINAL_PROMPT=0 cd %s && git clone %s %s" % (rootpath, url, path), @@ -150,8 +150,8 @@ def scan(KibbleBit, source): ) except subprocess.CalledProcessError as err: - KibbleBit.pprint("Repository sync failed (no master?)") - KibbleBit.pprint(str(err.output)) + kibble_bit.pprint("Repository sync failed (no master?)") + kibble_bit.pprint(str(err.output)) source["steps"]["sync"] = { "time": time.time(), "status": "Sync failed at " @@ -160,7 +160,7 @@ def scan(KibbleBit, source): "good": False, "exception": str(err.output), } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) return # All good, yay! @@ -171,4 +171,4 @@ def scan(KibbleBit, source): "running": False, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) diff --git a/kibble/scanners/scanners/github-issues.py b/kibble/scanners/scanners/github-issues.py index 06bbde67..cdfcde1e 100644 --- a/kibble/scanners/scanners/github-issues.py +++ b/kibble/scanners/scanners/github-issues.py @@ -124,29 +124,29 @@ def status_changed(stored_issue, issue): return stored_issue["status"] != issue["status"] -def update_issue(KibbleBit, issue): - KibbleBit.append("issue", issue) +def update_issue(kibble_bit, issue): + kibble_bit.append("issue", issue) -def update_person(KibbleBit, person): +def update_person(kibble_bit, person): person["upsert"] = True - KibbleBit.append("person", person) + kibble_bit.append("person", person) -def scan(KibbleBit, source, firstAttempt=True): +def scan(kibble_bit, source, first_attempt=True): auth = None people = {} if "creds" in source: - KibbleBit.pprint("Using auth for repo %s" % source["sourceURL"]) + kibble_bit.pprint("Using auth for repo %s" % source["sourceURL"]) creds = source["creds"] if creds and "username" in creds: auth = (creds["username"], creds["password"]) TL = github.get_tokens_left(auth=auth) - KibbleBit.pprint("Scanning for GitHub issues (%u tokens left on GitHub)" % TL) + kibble_bit.pprint("Scanning for GitHub issues (%u tokens left on GitHub)" % TL) # Have we scanned before? If so, only do a 3 month scan here. - doneBefore = False + done_before = False if source.get("steps") and source["steps"].get("issues"): - doneBefore = True + done_before = True source["steps"]["issues"] = { "time": time.time(), "status": "Issue scan started at " @@ -154,13 +154,13 @@ def scan(KibbleBit, source, firstAttempt=True): "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) try: - if doneBefore: + if done_before: since = time.strftime( "%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - (3 * 30 * 86400)) ) - KibbleBit.pprint("Fetching changes since %s" % since) + kibble_bit.pprint("Fetching changes since %s" % since) issues = github.get_all( source, github.issues, @@ -174,7 +174,7 @@ def scan(KibbleBit, source, firstAttempt=True): params={"filter": "all", "state": "all"}, auth=auth, ) - KibbleBit.pprint( + kibble_bit.pprint( "Fetched %s issues for %s" % (str(len(issues)), source["sourceURL"]) ) @@ -185,24 +185,24 @@ def scan(KibbleBit, source, firstAttempt=True): source, issue, github.user(issue["user"]["url"], auth=auth) ) people[issue["user"]["login"]] = person - update_person(KibbleBit, person) + update_person(kibble_bit, person) if "closed_by" in issue and not issue["closed_by"]["login"] in people: closer = make_person( source, issue, github.user(issue["closed_by"]["url"], auth=auth) ) people[issue["closed_by"]["login"]] = closer - update_person(KibbleBit, closer) + update_person(kibble_bit, closer) doc = make_issue(source, issue, people) dhash = doc["id"] - if KibbleBit.exists("issue", dhash): - es_doc = KibbleBit.get("issue", dhash) + if kibble_bit.exists("issue", dhash): + es_doc = kibble_bit.get("issue", dhash) if not status_changed(es_doc, doc): # KibbleBit.pprint("change %s seen already and status unchanged. Skipping." % issue['id']) continue - update_issue(KibbleBit, doc) + update_issue(kibble_bit, doc) source["steps"]["issues"] = { "time": time.time(), @@ -211,28 +211,30 @@ def scan(KibbleBit, source, firstAttempt=True): "running": False, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) except requests.HTTPError as e: # If we errored out because of rate limiting, retry later, otherwise bail - if firstAttempt: + if first_attempt: sleeps = 0 if github.get_tokens_left(auth=auth) < 10: - KibbleBit.pprint("Hit rate limits, trying to sleep it off!") + kibble_bit.pprint("Hit rate limits, trying to sleep it off!") while github.get_tokens_left(auth=auth) < 10: sleeps += 1 if sleeps > 24: - KibbleBit.pprint( + kibble_bit.pprint( "Slept for too long without finding a reset rate limit, giving up!" ) break time.sleep(300) # Sleep 5 min, then check again.. # If we have tokens, try one more time... if github.get_tokens_left(auth=auth) > 10: - scan(KibbleBit, source, False) # If this one fails, bail completely + scan( + kibble_bit, source, False + ) # If this one fails, bail completely return - KibbleBit.pprint("HTTP Error, rate limit exceeded?") + kibble_bit.pprint("HTTP Error, rate limit exceeded?") source["steps"]["issues"] = { "time": time.time(), "status": "Issue scan failed at " @@ -242,4 +244,4 @@ def scan(KibbleBit, source, firstAttempt=True): "running": False, "good": False, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) diff --git a/kibble/scanners/scanners/github-stats.py b/kibble/scanners/scanners/github-stats.py index 504d1516..ed4eda14 100644 --- a/kibble/scanners/scanners/github-stats.py +++ b/kibble/scanners/scanners/github-stats.py @@ -32,26 +32,26 @@ def accepts(source): return False -def getTime(string): +def get_time(string): """ Convert GitHub timestamp to epoch """ return time.mktime( time.strptime(re.sub(r"Z", "", str(string)), "%Y-%m-%dT%H:%M:%S") ) -def scan(KibbleBit, source): +def scan(kibble_bit, source): # Get some vars, construct a data path for the repo url = source["sourceURL"] auth = None if "creds" in source: - KibbleBit.pprint("Using auth for repo %s" % source["sourceURL"]) + kibble_bit.pprint("Using auth for repo %s" % source["sourceURL"]) creds = source["creds"] if creds and "username" in creds: auth = (creds["username"], creds["password"]) else: - KibbleBit.pprint( + kibble_bit.pprint( "GitHub stats requires auth, none provided. Ignoring this repo." ) return @@ -62,13 +62,13 @@ def scan(KibbleBit, source): "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) # Get views views = github.views(url, auth) if "views" in views: for el in views["views"]: - ts = getTime(el["timestamp"]) + ts = get_time(el["timestamp"]) shash = hashlib.sha224( ( "%s-%s-%s-clones" @@ -85,13 +85,13 @@ def scan(KibbleBit, source): "ghtype": "views", "id": shash, } - KibbleBit.append("ghstats", bit) + kibble_bit.append("ghstats", bit) # Get clones clones = github.clones(url, auth) if "clones" in clones: for el in clones["clones"]: - ts = getTime(el["timestamp"]) + ts = get_time(el["timestamp"]) shash = hashlib.sha224( ( "%s-%s-%s-clones" @@ -108,14 +108,14 @@ def scan(KibbleBit, source): "ghtype": "clones", "id": shash, } - KibbleBit.append("ghstats", bit) + kibble_bit.append("ghstats", bit) # Get referrers refs = github.referrers(url, auth) if refs: for el in refs: el["timestamp"] = time.strftime("%Y-%m-%dT%H:%M:%S", time.time()) - ts = getTime(el["timestamp"]) + ts = get_time(el["timestamp"]) shash = hashlib.sha224( ( "%s-%s-%s-refs" % (source["organisation"], url, el["timestamp"]) @@ -131,7 +131,7 @@ def scan(KibbleBit, source): "ghtype": "referrers", "id": shash, } - KibbleBit.append("ghstats", bit) - except: + kibble_bit.append("ghstats", bit) + except: # pylint: disable=bare-except pass # All done! diff --git a/kibble/scanners/scanners/jenkins.py b/kibble/scanners/scanners/jenkins.py index f7ec495e..3db31ac2 100644 --- a/kibble/scanners/scanners/jenkins.py +++ b/kibble/scanners/scanners/jenkins.py @@ -39,7 +39,7 @@ def accepts(source): return False -def scanJob(KibbleBit, source, job, creds): +def scan_job(kibble_bit, source, job, creds): """ Scans a single job for activity """ NOW = int(datetime.datetime.utcnow().timestamp()) jname = job["name"] @@ -50,16 +50,14 @@ def scanJob(KibbleBit, source, job, creds): "ascii", errors="replace" ) ).hexdigest() - doc = None - found = KibbleBit.exists("cijob", dhash) # Get $jenkins/job/$job-name/json... - jobURL = ( + job_url = ( "%s/api/json?depth=2&tree=builds[number,status,timestamp,id,result,duration]" % job["fullURL"] ) - KibbleBit.pprint(jobURL) - jobjson = jsonapi.get(jobURL, auth=creds) + kibble_bit.pprint(job_url) + jobjson = jsonapi.get(job_url, auth=creds) # If valid JSON, ... if jobjson: @@ -72,15 +70,15 @@ def scanJob(KibbleBit, source, job, creds): ).hexdigest() builddoc = None try: - builddoc = KibbleBit.get("ci_build", buildhash) - except: + builddoc = kibble_bit.get("ci_build", buildhash) + except: # pylint: disable=bare-except pass # If this build already completed, no need to parse it again if builddoc and builddoc.get("completed", False): continue - KibbleBit.pprint( + kibble_bit.pprint( "[%s-%s] This is new or pending, analyzing..." % (jname, build["id"]) ) @@ -117,7 +115,7 @@ def scanJob(KibbleBit, source, job, creds): "completed": completed, "duration": build["duration"], "job": jname, - "jobURL": jobURL, + "job_url": job_url, "status": status, "started": int(build["timestamp"] / 1000), "ci": "jenkins", @@ -127,20 +125,20 @@ def scanJob(KibbleBit, source, job, creds): "organisation": source["organisation"], "upsert": True, } - KibbleBit.append("ci_build", doc) + kibble_bit.append("ci_build", doc) # Yay, it worked! return True # Boo, it failed! - KibbleBit.pprint("Fetching job data failed!") + kibble_bit.pprint("Fetching job data failed!") return False -class jenkinsThread(threading.Thread): +class Jenkinsthread(threading.Thread): """ Generic thread class for scheduling multiple scans at once """ def __init__(self, block, KibbleBit, source, creds, jobs): - super(jenkinsThread, self).__init__() + super(Jenkinsthread, self).__init__() self.block = block self.KibbleBit = KibbleBit self.creds = creds @@ -148,8 +146,8 @@ def __init__(self, block, KibbleBit, source, creds, jobs): self.jobs = jobs def run(self): - badOnes = 0 - while len(self.jobs) > 0 and badOnes <= 50: + bad_ones = 0 + while len(self.jobs) > 0 and bad_ones <= 50: self.block.acquire() try: job = self.jobs.pop(0) @@ -165,12 +163,12 @@ def run(self): ssource = dict(self.source) if jfolder: ssource["sourceURL"] += "/job/" + jfolder - if not scanJob(self.KibbleBit, ssource, job, self.creds): + if not scan_job(self.KibbleBit, ssource, job, self.creds): self.KibbleBit.pprint( "[%s] This borked, trying another one" % job["name"] ) - badOnes += 1 - if badOnes > 100: + bad_ones += 1 + if bad_ones > 100: self.KibbleBit.pprint("Too many errors, bailing!") self.source["steps"]["issues"] = { "time": time.time(), @@ -179,13 +177,13 @@ def run(self): "running": False, "good": False, } - self.KibbleBit.updateSource(self.source) + self.KibbleBit.update_source(self.source) return else: - badOnes = 0 + bad_ones = 0 -def scan(KibbleBit, source): +def scan(kibble_bit, source): # Simple URL check jenkins = re.match(r"(https?://.+)", source["sourceURL"]) if jenkins: @@ -196,17 +194,16 @@ def scan(KibbleBit, source): "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) - pendingJobs = [] - KibbleBit.pprint("Parsing Jenkins activity at %s" % source["sourceURL"]) + kibble_bit.pprint("Parsing Jenkins activity at %s" % source["sourceURL"]) source["steps"]["issues"] = { "time": time.time(), "status": "Downloading changeset", "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) # Jenkins may neeed credentials creds = None @@ -219,15 +216,15 @@ def scan(KibbleBit, source): creds = "%s:%s" % (source["creds"]["username"], source["creds"]["password"]) # Get the job list - sURL = source["sourceURL"] - KibbleBit.pprint("Getting job list...") + s_url = source["sourceURL"] + kibble_bit.pprint("Getting job list...") jobsjs = jsonapi.get( - "%s/api/json?tree=jobs[name,color]&depth=1" % sURL, auth=creds + "%s/api/json?tree=jobs[name,color]&depth=1" % s_url, auth=creds ) # Get the current queue - KibbleBit.pprint("Getting job queue...") - queuejs = jsonapi.get("%s/queue/api/json?depth=1" % sURL, auth=creds) + kibble_bit.pprint("Getting job queue...") + queuejs = jsonapi.get("%s/queue/api/json?depth=1" % s_url, auth=creds) # Save queue snapshot NOW = int(datetime.datetime.utcnow().timestamp()) @@ -256,7 +253,7 @@ def scan(KibbleBit, source): # Count how many jobs are building, find any folders... actual_jobs, building = get_all_jobs( - KibbleBit, source, jobsjs.get("jobs", []), creds + kibble_bit, source, jobsjs.get("jobs", []), creds ) # Write up a queue doc @@ -275,16 +272,16 @@ def scan(KibbleBit, source): "organisation": source["organisation"], "upsert": True, } - KibbleBit.append("ci_queue", queuedoc) + kibble_bit.append("ci_queue", queuedoc) - pendingJobs = actual_jobs - KibbleBit.pprint("Found %u jobs in Jenkins" % len(pendingJobs)) + pending_jobs = actual_jobs + kibble_bit.pprint("Found %u jobs in Jenkins" % len(pending_jobs)) threads = [] block = threading.Lock() - KibbleBit.pprint("Scanning jobs using 4 sub-threads") + kibble_bit.pprint("Scanning jobs using 4 sub-threads") for i in range(0, 4): - t = jenkinsThread(block, KibbleBit, source, creds, pendingJobs) + t = Jenkinsthread(block, kibble_bit, source, creds, pending_jobs) threads.append(t) t.start() @@ -292,7 +289,7 @@ def scan(KibbleBit, source): t.join() # We're all done, yaay - KibbleBit.pprint("Done scanning %s" % source["sourceURL"]) + kibble_bit.pprint("Done scanning %s" % source["sourceURL"]) source["steps"]["issues"] = { "time": time.time(), @@ -301,10 +298,10 @@ def scan(KibbleBit, source): "running": False, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) -def get_all_jobs(KibbleBit, source, joblist, creds): +def get_all_jobs(kibble_bit, source, joblist, creds): real_jobs = [] building = 0 for job in joblist: @@ -314,30 +311,30 @@ def get_all_jobs(KibbleBit, source, joblist, creds): "jenkins.branch.OrganizationFolder", "org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject", ]: - KibbleBit.pprint("%s is a jobs folder, expanding..." % job["name"]) - csURL = "%s/job/%s" % ( + kibble_bit.pprint("%s is a jobs folder, expanding..." % job["name"]) + cs_url = "%s/job/%s" % ( source["sourceURL"], urllib.parse.quote(job["name"].replace("/", "%2F")), ) try: child_jobs = jsonapi.get( - "%s/api/json?tree=jobs[name,color]&depth=1" % csURL, auth=creds + "%s/api/json?tree=jobs[name,color]&depth=1" % cs_url, auth=creds ) csource = dict(source) - csource["sourceURL"] = csURL + csource["sourceURL"] = cs_url if not csource.get("folder"): csource["folder"] = job["name"] else: csource["folder"] += "-" + job["name"] cjobs, cbuilding = get_all_jobs( - KibbleBit, csource, child_jobs.get("jobs", []), creds + kibble_bit, csource, child_jobs.get("jobs", []), creds ) building += cbuilding for cjob in cjobs: real_jobs.append(cjob) - except: - KibbleBit.pprint("Couldn't get child jobs, bailing") - print("%s/api/json?tree=jobs[name,color]&depth=1" % csURL) + except: # pylint: disable=bare-except + kibble_bit.pprint("Couldn't get child jobs, bailing") + print("%s/api/json?tree=jobs[name,color]&depth=1" % cs_url) # Or standard job? else: # Is it building? diff --git a/kibble/scanners/scanners/jira.py b/kibble/scanners/scanners/jira.py index 41dafa6b..14789c9d 100644 --- a/kibble/scanners/scanners/jira.py +++ b/kibble/scanners/scanners/jira.py @@ -43,7 +43,7 @@ def accepts(source): return False -def getTime(string): +def get_time(string): return time.mktime( time.strptime(re.sub(r"\..*", "", str(string)), "%Y-%m-%dT%H:%M:%S") ) @@ -121,7 +121,7 @@ def pchange(js): return False -def scanTicket(KibbleBit, key, u, source, creds, openTickets): +def scan_ticket(kibble_bit, key, u, source, creds, open_tickets): """ Scans a single ticket for activity and people """ dhash = hashlib.sha224( @@ -129,8 +129,7 @@ def scanTicket(KibbleBit, key, u, source, creds, openTickets): "ascii", errors="replace" ) ).hexdigest() - found = True - parseIt = False + parse_it = False # the 'domain' var we try to figure out here is used # for faking email addresses and keep them unique, @@ -140,62 +139,62 @@ def scanTicket(KibbleBit, key, u, source, creds, openTickets): if m: domain = m.group(1) - found = KibbleBit.exists("issue", dhash) + found = kibble_bit.exists("issue", dhash) if not found: - KibbleBit.pprint("[%s] We've never seen this ticket before, parsing..." % key) - parseIt = True + kibble_bit.pprint("[%s] We've never seen this ticket before, parsing..." % key) + parse_it = True else: - ticket = KibbleBit.get("issue", dhash) - if ticket["status"] == "closed" and key in openTickets: - KibbleBit.pprint("[%s] Ticket was reopened, reparsing" % key) - parseIt = True - elif ticket["status"] == "open" and not key in openTickets: - KibbleBit.pprint("[%s] Ticket was recently closed, parsing it" % key) - parseIt = True + ticket = kibble_bit.get("issue", dhash) + if ticket["status"] == "closed" and key in open_tickets: + kibble_bit.pprint("[%s] Ticket was reopened, reparsing" % key) + parse_it = True + elif ticket["status"] == "open" and not key in open_tickets: + kibble_bit.pprint("[%s] Ticket was recently closed, parsing it" % key) + parse_it = True else: if ( ticket["issueCreator"] == "unknown@kibble" or ticket["issueCloser"] == "unknown@kibble" ): # Gotta redo these! - parseIt = True - KibbleBit.pprint( + parse_it = True + kibble_bit.pprint( "[%s] Ticket contains erroneous data from a previous scan, reparsing" % key ) # This is just noise! # KibbleBit.pprint("[%s] Ticket hasn't changed, ignoring..." % key) - if parseIt: - KibbleBit.pprint("[%s] Parsing data from JIRA at %s..." % (key, domain)) - queryURL = ( + if parse_it: + kibble_bit.pprint("[%s] Parsing data from JIRA at %s..." % (key, domain)) + query_url = ( "%s/rest/api/2/issue/%s?fields=creator,reporter,status,issuetype,summary,assignee,resolutiondate,created,priority,changelog,comment,resolution,votes&expand=changelog" % (u, key) ) - jiraURL = "%s/browse/%s" % (u, key) + jira_url = "%s/browse/%s" % (u, key) try: - tjson = jsonapi.get(queryURL, auth=creds) + tjson = jsonapi.get(query_url, auth=creds) if not tjson: - KibbleBit.pprint("%s does not exist (404'ed)" % key) + kibble_bit.pprint("%s does not exist (404'ed)" % key) return False except requests.exceptions.ConnectionError as err: - KibbleBit.pprint(f"Connection error: {err}, skipping this ticket for now!") + kibble_bit.pprint(f"Connection error: {err}, skipping this ticket for now!") return False st, closer = wasclosed(tjson) if st and not closer: - KibbleBit.pprint("Closed but no closer??") - closerEmail = None + kibble_bit.pprint("Closed but no closer??") + closer_email = None status = "closed" if st else "open" # Make sure we actually have field data to work with if not tjson.get("fields") or not tjson["fields"].get("created"): - KibbleBit.pprint( + kibble_bit.pprint( "[%s] JIRA response is missing field data, ignoring ticket." % key ) return False - cd = getTime(tjson["fields"]["created"]) + cd = get_time(tjson["fields"]["created"]) rd = ( - getTime(tjson["fields"]["resolutiondate"]) + get_time(tjson["fields"]["resolutiondate"]) if "resolutiondate" in tjson["fields"] and tjson["fields"]["resolutiondate"] else None ) @@ -221,40 +220,40 @@ def scanTicket(KibbleBit, key, u, source, creds, openTickets): title = tjson["fields"]["summary"] if closer: # print("Parsing closer") - closerEmail = ( + closer_email = ( closer.get("emailAddress", closer.get("name")) .replace(" dot ", ".", 10) .replace(" at ", "@", 1) ) - if not "@" in closerEmail: - closerEmail = "%s@%s" % (closerEmail, domain) - displayName = closer.get("displayName", "Unkown") - if displayName and len(displayName) > 0: + if not "@" in closer_email: + closer_email = "%s@%s" % (closer_email, domain) + display_name = closer.get("displayName", "Unkown") + if display_name and len(display_name) > 0: # Add to people db pid = hashlib.sha1( - ("%s%s" % (source["organisation"], closerEmail)).encode( + ("%s%s" % (source["organisation"], closer_email)).encode( "ascii", errors="replace" ) ).hexdigest() jsp = { - "name": displayName, - "email": closerEmail, + "name": display_name, + "email": closer_email, "organisation": source["organisation"], "id": pid, "upsert": True, } - KibbleBit.append("person", jsp) + kibble_bit.append("person", jsp) if creator: creator = creator.replace(" dot ", ".", 10).replace(" at ", "@", 1) if not "@" in creator: creator = "%s@%s" % (creator, domain) - displayName = ( + display_name = ( tjson["fields"]["reporter"]["displayName"] if tjson["fields"]["reporter"] else None ) - if displayName and len(displayName) > 0: + if display_name and len(display_name) > 0: # Add to people db pid = hashlib.sha1( ("%s%s" % (source["organisation"], creator)).encode( @@ -262,13 +261,13 @@ def scanTicket(KibbleBit, key, u, source, creds, openTickets): ) ).hexdigest() jsp = { - "name": displayName, + "name": display_name, "email": creator, "organisation": source["organisation"], "id": pid, "upsert": True, } - KibbleBit.append("person", jsp) + kibble_bit.append("person", jsp) if assignee and not "@" in assignee: assignee = "%s@%s" % (assignee, domain) jso = { @@ -276,12 +275,12 @@ def scanTicket(KibbleBit, key, u, source, creds, openTickets): "key": key, "organisation": source["organisation"], "sourceID": source["sourceID"], - "url": jiraURL, + "url": jira_url, "status": status, "created": cd, "closed": rd, "issuetype": "issue", - "issueCloser": closerEmail, + "issueCloser": closer_email, "createdDate": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(cd)), "closedDate": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(rd)) if rd @@ -294,7 +293,7 @@ def scanTicket(KibbleBit, key, u, source, creds, openTickets): "comments": comments, "title": title, } - KibbleBit.append("issue", jso) + kibble_bit.append("issue", jso) return True @@ -304,19 +303,19 @@ def scanTicket(KibbleBit, key, u, source, creds, openTickets): # return False -class jiraThread(threading.Thread): - def __init__(self, block, KibbleBit, source, creds, pt, ot): - super(jiraThread, self).__init__() +class JiraThread(threading.Thread): + def __init__(self, block, kibble_bit, source, creds, pt, ot): + super(JiraThread, self).__init__() self.block = block - self.KibbleBit = KibbleBit + self.KibbleBit = kibble_bit self.creds = creds self.source = source self.pendingTickets = pt self.openTickets = ot def run(self): - badOnes = 0 - while len(self.pendingTickets) > 0 and badOnes <= 50: + bad_ones = 0 + while len(self.pendingTickets) > 0 and bad_ones <= 50: # print("%u elements left to count" % len(pendingTickets)) self.block.acquire() try: @@ -329,12 +328,12 @@ def run(self): self.block.release() return self.block.release() - if not scanTicket( + if not scan_ticket( self.KibbleBit, rl[0], rl[1], rl[2], self.creds, self.openTickets ): self.KibbleBit.pprint("[%s] This borked, trying another one" % rl[0]) - badOnes += 1 - if badOnes > 100: + bad_ones += 1 + if bad_ones > 100: self.KibbleBit.pprint("Too many errors, bailing!") self.source["steps"]["issues"] = { "time": time.time(), @@ -343,13 +342,13 @@ def run(self): "running": False, "good": False, } - self.KibbleBit.updateSource(self.source) + self.KibbleBit.update_source(self.source) return else: - badOnes = 0 + bad_ones = 0 -def scan(KibbleBit, source): +def scan(kibble_bit, source): jira = re.match(r"(https?://.+)/browse/([A-Z0-9]+)", source["sourceURL"]) if jira: @@ -363,7 +362,7 @@ def scan(KibbleBit, source): ): creds = "%s:%s" % (source["creds"]["username"], source["creds"]["password"]) if not creds: - KibbleBit.pprint( + kibble_bit.pprint( "JIRA at %s requires authentication, but none was found! Bailing." % source["sourceURL"] ) @@ -373,7 +372,7 @@ def scan(KibbleBit, source): "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) return source["steps"]["issues"] = { @@ -382,74 +381,75 @@ def scan(KibbleBit, source): "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) - pendingTickets = [] - KibbleBit.pprint("Parsing JIRA activity at %s" % source["sourceURL"]) + pending_tickets = [] + kibble_bit.pprint("Parsing JIRA activity at %s" % source["sourceURL"]) source["steps"]["issues"] = { "time": time.time(), "status": "Downloading changeset", "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) # Get base URL, list and domain to parse u = jira.group(1) instance = jira.group(2) - lastTicket = 0 - latestURL = ( + last_ticket = 0 + latest_url = ( "%s/rest/api/2/search?jql=project=%s+order+by+createdDate+DESC&fields=id,key&maxResults=1" % (u, instance) ) js = None - js = jsonapi.get(latestURL, auth=creds) + js = jsonapi.get(latest_url, auth=creds) if "issues" in js and len(js["issues"]) == 1: key = js["issues"][0]["key"] m = re.search(r"-(\d+)$", key) if m: - lastTicket = int(m.group(1)) + last_ticket = int(m.group(1)) - openTickets = [] - startAt = 0 - badTries = 0 - while True and badTries < 10: - openURL = ( + open_tickets = [] + start_at = 0 + bad_tries = 0 + while bad_tries < 10: + open_url = ( "%s/rest/api/2/search?jql=project=%s+and+status=open+order+by+createdDate+ASC&fields=id,key&maxResults=100&startAt=%u" - % (u, instance, startAt) + % (u, instance, start_at) ) # print(openURL) try: - ojs = jsonapi.get(openURL, auth=creds) + ojs = jsonapi.get(open_url, auth=creds) if not "issues" in ojs or len(ojs["issues"]) == 0: break for item in ojs["issues"]: - openTickets.append(item["key"]) - KibbleBit.pprint("Found %u open tickets" % len(openTickets)) - startAt += 100 - except: - KibbleBit.pprint("JIRA borked, retrying") - badTries += 1 - KibbleBit.pprint("Found %u open tickets" % len(openTickets)) - - badOnes = 0 - for i in reversed(range(1, lastTicket + 1)): + open_tickets.append(item["key"]) + kibble_bit.pprint("Found %u open tickets" % len(open_tickets)) + start_at += 100 + except: # pylint: disable=bare-except + kibble_bit.pprint("JIRA borked, retrying") + bad_tries += 1 + kibble_bit.pprint("Found %u open tickets" % len(open_tickets)) + + for i in reversed(range(1, last_ticket + 1)): key = "%s-%u" % (instance, i) - pendingTickets.append([key, u, source]) + pending_tickets.append([key, u, source]) threads = [] block = threading.Lock() - KibbleBit.pprint("Scanning tickets using 4 sub-threads") + kibble_bit.pprint("Scanning tickets using 4 sub-threads") for i in range(0, 4): - t = jiraThread(block, KibbleBit, source, creds, pendingTickets, openTickets) + t = JiraThread( + block, kibble_bit, source, creds, pending_tickets, open_tickets + ) threads.append(t) t.start() for t in threads: t.join() - KibbleBit.pprint("Done scanning %s" % source["sourceURL"]) + kibble_bit.pprint("Done scanning %s" % source["sourceURL"]) source["steps"]["issues"] = { "time": time.time(), @@ -458,4 +458,4 @@ def scan(KibbleBit, source): "running": False, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) diff --git a/kibble/scanners/scanners/pipermail.py b/kibble/scanners/scanners/pipermail.py index 606a97eb..9a0c598c 100644 --- a/kibble/scanners/scanners/pipermail.py +++ b/kibble/scanners/scanners/pipermail.py @@ -43,11 +43,11 @@ def accepts(source): return False -def scan(KibbleBit, source): +def scan(kibble_bit, source): url = source["sourceURL"] pipermail = re.match(r"(https?://.+/(archives|pipermail)/.+?)/?$", url) if pipermail: - KibbleBit.pprint("Scanning Pipermail source %s" % url) + kibble_bit.pprint("Scanning Pipermail source %s" % url) skipped = 0 source["steps"]["mail"] = { @@ -56,10 +56,10 @@ def scan(KibbleBit, source): "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) dt = time.gmtime(time.time()) - firstYear = 1970 + first_year = 1970 year = dt[0] month = dt[1] if month <= 0: @@ -70,7 +70,7 @@ def scan(KibbleBit, source): knowns = {} # While we have older archives, continue to parse - monthNames = [ + month_names = [ "December", "January", "February", @@ -85,16 +85,15 @@ def scan(KibbleBit, source): "November", "December", ] - while firstYear <= year: - gzurl = "%s/%04u-%s.txt.gz" % (url, year, monthNames[month]) + while first_year <= year: + gzurl = "%s/%04u-%s.txt.gz" % (url, year, month_names[month]) pd = datetime.date(year, month, 1).timetuple() dhash = hashlib.sha224( ("%s %s" % (source["organisation"], gzurl)).encode( "ascii", errors="replace" ) ).hexdigest() - found = False - found = KibbleBit.exists("mailstats", dhash) + found = kibble_bit.exists("mailstats", dhash) if ( months <= 1 or not found ): # Always parse this month's stats and the previous month :) @@ -153,7 +152,7 @@ def scan(KibbleBit, source): if m: name = m.group(1).replace('"', "").strip() sender = m.group(2) - if not sender in posters: + if sender not in posters: posters[sender] = {"name": name, "email": sender} senders[message.get("message-id", "??")] = sender mdate = email.utils.parsedate_tz(message["date"]) @@ -161,15 +160,15 @@ def scan(KibbleBit, source): "%Y/%m/%d %H:%M:%S", time.gmtime(email.utils.mktime_tz(mdate)), ) - if not sender in knowns: + if sender not in knowns: sid = hashlib.sha1( ("%s%s" % (source["organisation"], sender)).encode( "ascii", errors="replace" ) ).hexdigest() - knowns[sender] = KibbleBit.exists("person", sid) - if not sender in knowns: - KibbleBit.append( + knowns[sender] = kibble_bit.exists("person", sid) + if sender not in knowns: + kibble_bit.append( "person", { "name": name, @@ -196,20 +195,17 @@ def scan(KibbleBit, source): "ts": email.utils.mktime_tz(mdate), "id": message["message-id"], } - KibbleBit.append("email", jse) + kibble_bit.append("email", jse) - for sender in posters: - no_posters += 1 + no_posters = len(posters) + topics = len(rawtopics) i = 0 - topics = 0 - for key in rawtopics: - topics += 1 for key in reversed(sorted(rawtopics, key=lambda x: x)): val = rawtopics[key] i += 1 if i > 10: break - KibbleBit.pprint( + kibble_bit.pprint( "Found top 10: %s (%s emails)" % (key, val) ) shash = hashlib.sha224( @@ -238,7 +234,7 @@ def scan(KibbleBit, source): "ts": time.mktime(pd), "id": mlhash, } - KibbleBit.index("mailtop", mlhash, jst) + kibble_bit.index("mailtop", mlhash, jst) jso = { "organisation": source["organisation"], @@ -249,24 +245,24 @@ def scan(KibbleBit, source): "emails": emails, "topics": topics, } - KibbleBit.index("mailstats", dhash, jso) + kibble_bit.index("mailstats", dhash, jso) os.unlink(mailFile) except Exception as err: - KibbleBit.pprint( + kibble_bit.pprint( "Couldn't parse %s, skipping: %s" % (gzurl, err) ) skipped += 1 if skipped > 12: - KibbleBit.pprint( + kibble_bit.pprint( "12 skips in a row, breaking off (no more data?)" ) break else: - KibbleBit.pprint("Couldn't find %s, skipping." % gzurl) + kibble_bit.pprint("Couldn't find %s, skipping." % gzurl) skipped += 1 if skipped > 12: - KibbleBit.pprint( + kibble_bit.pprint( "12 skips in a row, breaking off (no more data?)" ) break @@ -282,13 +278,13 @@ def scan(KibbleBit, source): "running": False, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) else: - KibbleBit.pprint("Invalid Pipermail URL detected: %s" % url, True) + kibble_bit.pprint("Invalid Pipermail URL detected: %s" % url, True) source["steps"]["mail"] = { "time": time.time(), "status": "Invalid or malformed URL detected!", "running": False, "good": False, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) diff --git a/kibble/scanners/scanners/ponymail-kpe.py b/kibble/scanners/scanners/ponymail-kpe.py index 50edac75..8db47c25 100644 --- a/kibble/scanners/scanners/ponymail-kpe.py +++ b/kibble/scanners/scanners/ponymail-kpe.py @@ -47,11 +47,11 @@ def accepts(source): return False -def scan(KibbleBit, source): +def scan(kibble_bit, source): # Validate URL first url = re.match(r"(https?://.+)/list\.html\?(.+)@(.+)", source["sourceURL"]) if not url: - KibbleBit.pprint( + kibble_bit.pprint( "Malformed or invalid Pony Mail URL passed to scanner: %s" % source["sourceURL"] ) @@ -61,11 +61,11 @@ def scan(KibbleBit, source): "running": False, "good": False, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) return - if not "azure" in KibbleBit.config and not "picoapi" in KibbleBit.config: - KibbleBit.pprint( + if not "azure" in kibble_bit.config and not "picoapi" in kibble_bit.config: + kibble_bit.pprint( "No Azure/picoAPI creds configured, skipping key phrase extraction" ) return @@ -74,15 +74,15 @@ def scan(KibbleBit, source): if "creds" in source and source["creds"]: cookie = source["creds"].get("cookie", None) - rootURL = re.sub(r"list.html.+", "", source["sourceURL"]) + root_url = re.sub(r"list.html.+", "", source["sourceURL"]) query = { "query": {"bool": {"must": [{"term": {"sourceID": source["sourceID"]}}]}}, "sort": [{"ts": "desc"}], } # Get an initial count of commits - res = KibbleBit.broker.DB.search( - index=KibbleBit.dbname, doc_type="email", body=query, size=MAX_COUNT * 4 + res = kibble_bit.broker.DB.search( + index=kibble_bit.dbname, doc_type="email", body=query, size=MAX_COUNT * 4 ) ec = 0 hits = [] @@ -93,30 +93,31 @@ def scan(KibbleBit, source): if ec > MAX_COUNT: break if "kpe" not in eml: - emlurl = "%s/api/email.lua?id=%s" % (rootURL, eml["id"]) - KibbleBit.pprint("Fetching %s" % emlurl) + emlurl = "%s/api/email.lua?id=%s" % (root_url, eml["id"]) + kibble_bit.pprint("Fetching %s" % emlurl) rv = None try: rv = jsonapi.get(emlurl, cookie=cookie) if rv and "body" in rv: hits.append([hit["_id"], rv["body"], eml]) except Exception as err: - KibbleBit.pprint(f"Server error: {err}, skipping this email") + kibble_bit.pprint(f"Server error: {err}, skipping this email") bodies = [] for hit in hits: body = hit[1] - bid = hit[0] + # bid = hit[0] bodies.append(body) if bodies: - if "watson" in KibbleBit.config: + KPEs = None + if "watson" in kibble_bit.config: pass # Haven't written this yet - elif "azure" in KibbleBit.config: - KPEs = kpe.azureKPE(KibbleBit, bodies) - elif "picoapi" in KibbleBit.config: - KPEs = kpe.picoKPE(KibbleBit, bodies) + elif "azure" in kibble_bit.config: + KPEs = kpe.azureKPE(kibble_bit, bodies) + elif "picoapi" in kibble_bit.config: + KPEs = kpe.picoKPE(kibble_bit, bodies) if not KPEs: - KibbleBit.pprint("Hit rate limit, not trying further emails for now.") + kibble_bit.pprint("Hit rate limit, not trying further emails for now.") a = 0 for hit in hits: @@ -128,7 +129,7 @@ def scan(KibbleBit, source): kpe_ = ["_NULL_"] eml["kpe"] = kpe_ print("Key phrases for %s: %s" % (bid, ", ".join(kpe_))) - KibbleBit.index("email", bid, eml) + kibble_bit.index("email", bid, eml) else: - KibbleBit.pprint("No emails to analyze") - KibbleBit.pprint("Done with key phrase extraction") + kibble_bit.pprint("No emails to analyze") + kibble_bit.pprint("Done with key phrase extraction") diff --git a/kibble/scanners/scanners/ponymail-tone.py b/kibble/scanners/scanners/ponymail-tone.py index e0dea32e..4ae9330c 100644 --- a/kibble/scanners/scanners/ponymail-tone.py +++ b/kibble/scanners/scanners/ponymail-tone.py @@ -44,11 +44,11 @@ def accepts(source): return False -def scan(KibbleBit, source): +def scan(kibble_bit, source): # Validate URL first url = re.match(r"(https?://.+)/list\.html\?(.+)@(.+)", source["sourceURL"]) if not url: - KibbleBit.pprint( + kibble_bit.pprint( "Malformed or invalid Pony Mail URL passed to scanner: %s" % source["sourceURL"] ) @@ -58,15 +58,15 @@ def scan(KibbleBit, source): "running": False, "good": False, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) return if ( - not "watson" in KibbleBit.config - and not "azure" in KibbleBit.config - and not "picoapi" in KibbleBit.config + not "watson" in kibble_bit.config + and not "azure" in kibble_bit.config + and not "picoapi" in kibble_bit.config ): - KibbleBit.pprint( + kibble_bit.pprint( "No Watson/Azure/picoAPI creds configured, skipping tone analyzer" ) return @@ -75,15 +75,15 @@ def scan(KibbleBit, source): if "creds" in source and source["creds"]: cookie = source["creds"].get("cookie", None) - rootURL = re.sub(r"list.html.+", "", source["sourceURL"]) + root_url = re.sub(r"list.html.+", "", source["sourceURL"]) query = { "query": {"bool": {"must": [{"term": {"sourceID": source["sourceID"]}}]}}, "sort": [{"ts": "desc"}], } # Get an initial count of commits - res = KibbleBit.broker.DB.search( - index=KibbleBit.dbname, doc_type="email", body=query, size=MAX_COUNT * 4 + res = kibble_bit.broker.DB.search( + index=kibble_bit.dbname, doc_type="email", body=query, size=MAX_COUNT * 4 ) ec = 0 hits = [] @@ -94,30 +94,30 @@ def scan(KibbleBit, source): if ec > MAX_COUNT: break if "mood" not in eml: - emlurl = "%s/api/email.lua?id=%s" % (rootURL, eml["id"]) - KibbleBit.pprint("Fetching %s" % emlurl) - rv = None + emlurl = "%s/api/email.lua?id=%s" % (root_url, eml["id"]) + kibble_bit.pprint("Fetching %s" % emlurl) try: rv = jsonapi.get(emlurl, cookie=cookie) if rv and "body" in rv: hits.append([hit["_id"], rv["body"], eml]) except Exception as err: - KibbleBit.pprint(f"Server error: {err}, skipping this email") + kibble_bit.pprint(f"Server error: {err}, skipping this email") bodies = [] for hit in hits: body = hit[1] - bid = hit[0] + # bid = hit[0] bodies.append(body) if bodies: - if "watson" in KibbleBit.config: - moods = tone.watsonTone(KibbleBit, bodies) - elif "azure" in KibbleBit.config: - moods = tone.azureTone(KibbleBit, bodies) - elif "picoapi" in KibbleBit.config: - moods = tone.picoTone(KibbleBit, bodies) + moods = None + if "watson" in kibble_bit.config: + moods = tone.watsonTone(kibble_bit, bodies) + elif "azure" in kibble_bit.config: + moods = tone.azureTone(kibble_bit, bodies) + elif "picoapi" in kibble_bit.config: + moods = tone.picoTone(kibble_bit, bodies) if not moods: - KibbleBit.pprint("Hit rate limit, not trying further emails for now.") + kibble_bit.pprint("Hit rate limit, not trying further emails for now.") a = 0 for hit in hits: @@ -131,7 +131,7 @@ def scan(KibbleBit, source): if s > hm[0]: hm = [s, m] print("Likeliest overall mood for %s: %s" % (bid, hm[1])) - KibbleBit.index("email", bid, eml) + kibble_bit.index("email", bid, eml) else: - KibbleBit.pprint("No emails to analyze") - KibbleBit.pprint("Done with tone analysis") + kibble_bit.pprint("No emails to analyze") + kibble_bit.pprint("Done with tone analysis") diff --git a/kibble/scanners/scanners/ponymail.py b/kibble/scanners/scanners/ponymail.py index 92a5b0bc..69a49ca8 100644 --- a/kibble/scanners/scanners/ponymail.py +++ b/kibble/scanners/scanners/ponymail.py @@ -45,42 +45,42 @@ def accepts(source): return False -def countSubs(struct, kids=0): +def count_subs(struct, kids=0): """ Counts replies in a thread """ if "children" in struct and len(struct["children"]) > 0: for child in struct["children"]: kids += 1 - kids += countSubs(child) + kids += count_subs(child) return kids -def repliedTo(emails, struct): - myList = {} +def replied_to(emails, struct): + my_list = {} for eml in struct: - myID = eml["tid"] + my_id = eml["tid"] if "children" in eml: for child in eml["children"]: - myList[child["tid"]] = myID + my_list[child["tid"]] = my_id if len(child["children"]) > 0: - cList = repliedTo(emails, child["children"]) - myList.update(cList) - return myList + c_list = replied_to(emails, child["children"]) + my_list.update(c_list) + return my_list -def getSender(email): +def get_sender(email): sender = email["from"] m = re.match(r"(.+)\s*<(.+)>", email["from"], flags=re.UNICODE) if m: - name = m.group(1).replace('"', "").strip() + # name = m.group(1).replace('"', "").strip() sender = m.group(2) return sender -def scan(KibbleBit, source): +def scan(kibble_bit, source): # Validate URL first url = re.match(r"(https?://.+)/list\.html\?(.+)@(.+)", source["sourceURL"]) if not url: - KibbleBit.pprint( + kibble_bit.pprint( "Malformed or invalid Pony Mail URL passed to scanner: %s" % source["sourceURL"] ) @@ -90,7 +90,7 @@ def scan(KibbleBit, source): "running": False, "good": False, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) return # Pony Mail requires a UI cookie in order to work. Maked sure we have one! @@ -98,7 +98,7 @@ def scan(KibbleBit, source): if "creds" in source and source["creds"]: cookie = source["creds"].get("cookie", None) if not cookie: - KibbleBit.pprint( + kibble_bit.pprint( "Pony Mail instance at %s requires an authorized cookie, none found! Bailing." % source["sourceURL"] ) @@ -108,18 +108,18 @@ def scan(KibbleBit, source): "running": False, "good": False, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) return # Notify scanner and DB that this is valid and we've begun parsing - KibbleBit.pprint("%s is a valid Pony Mail address, parsing" % source["sourceURL"]) + kibble_bit.pprint("%s is a valid Pony Mail address, parsing" % source["sourceURL"]) source["steps"]["mail"] = { "time": time.time(), "status": "Downloading Pony Mail statistics", "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) # Get base URL, list and domain to parse u = url.group(1) @@ -128,7 +128,7 @@ def scan(KibbleBit, source): # Get this month dt = time.gmtime(time.time()) - firstYear = 1970 + first_year = 1970 year = dt[0] month = dt[1] if month <= 0: @@ -140,7 +140,7 @@ def scan(KibbleBit, source): knowns = {} # While we have older archives, continue to parse - while firstYear <= year: + while first_year <= year: statsurl = "%s/api/stats.lua?list=%s&domain=%s&d=%s" % ( u, l, @@ -153,36 +153,36 @@ def scan(KibbleBit, source): ) ).hexdigest() found = False - if KibbleBit.exists("mailstats", dhash): + if kibble_bit.exists("mailstats", dhash): found = True if months <= 1 or not found: # Always parse this month's stats :) months += 1 - KibbleBit.pprint("Parsing %04u-%02u" % (year, month)) - KibbleBit.pprint(statsurl) + kibble_bit.pprint("Parsing %04u-%02u" % (year, month)) + kibble_bit.pprint(statsurl) pd = datetime.date(year, month, 1).timetuple() try: js = jsonapi.get(statsurl, cookie=cookie) except Exception as err: - KibbleBit.pprint(f"Server error: {err}, skipping this month") + kibble_bit.pprint(f"Server error: {err}, skipping this month") month -= 1 if month <= 0: month += 12 year -= 1 continue if "firstYear" in js: - firstYear = js["firstYear"] + first_year = js["firstYear"] # print("First Year is %u" % firstYear) else: - KibbleBit.pprint("JSON was missing fields, aborting!") + kibble_bit.pprint("JSON was missing fields, aborting!") break - replyList = repliedTo(js["emails"], js["thread_struct"]) + reply_list = replied_to(js["emails"], js["thread_struct"]) topics = js["no_threads"] posters = {} no_posters = 0 emails = len(js["emails"]) top10 = [] for eml in js["thread_struct"]: - count = countSubs(eml, 0) + count = count_subs(eml, 0) subject = "" for reml in js["emails"]: if reml["id"] == eml["tid"]: @@ -202,7 +202,7 @@ def scan(KibbleBit, source): i += 1 if i > 10: break - KibbleBit.pprint("Found top 10: %s (%s emails)" % (top[1], top[2])) + kibble_bit.pprint("Found top 10: %s (%s emails)" % (top[1], top[2])) md = time.strftime("%Y/%m/%d %H:%M:%S", pd) mlhash = hashlib.sha224( ( @@ -221,7 +221,7 @@ def scan(KibbleBit, source): "ts": time.mktime(pd), "id": mlhash, } - KibbleBit.index("mailtop", mlhash, jst) + kibble_bit.index("mailtop", mlhash, jst) for email in js["emails"]: sender = email["from"] @@ -238,10 +238,10 @@ def scan(KibbleBit, source): "ascii", errors="replace" ) ).hexdigest() - if KibbleBit.exists("person", sid): + if kibble_bit.exists("person", sid): knowns[sender] = True if not sender in knowns or name != sender: - KibbleBit.append( + kibble_bit.append( "person", { "upsert": True, @@ -256,12 +256,12 @@ def scan(KibbleBit, source): }, ) knowns[sender] = True - replyTo = None - if email["id"] in replyList: - rt = replyList[email["id"]] + reply_to = None + if email["id"] in reply_list: + rt = reply_list[email["id"]] for eml in js["emails"]: if eml["id"] == rt: - replyTo = getSender(eml) + reply_to = get_sender(eml) print("Email was reply to %s" % sender) jse = { "organisation": source["organisation"], @@ -273,14 +273,13 @@ def scan(KibbleBit, source): "sender": sender, "address": sender, "subject": email["subject"], - "replyto": replyTo, + "replyto": reply_to, "ts": email["epoch"], "id": email["id"], "upsert": True, } - KibbleBit.append("email", jse) - for sender in posters: - no_posters += 1 + kibble_bit.append("email", jse) + no_posters = len(posters) jso = { "organisation": source["organisation"], @@ -292,7 +291,7 @@ def scan(KibbleBit, source): "topics": topics, } # print("Indexing as %s" % dhash) - KibbleBit.index("mailstats", dhash, jso) + kibble_bit.index("mailstats", dhash, jso) month -= 1 if month <= 0: month += 12 @@ -305,4 +304,4 @@ def scan(KibbleBit, source): "running": False, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) diff --git a/kibble/scanners/scanners/travis.py b/kibble/scanners/scanners/travis.py index 22b4975a..2c7852f6 100644 --- a/kibble/scanners/scanners/travis.py +++ b/kibble/scanners/scanners/travis.py @@ -39,44 +39,37 @@ def accepts(source): return False -def scanJob(KibbleBit, source, bid, token, TLD): +def scan_job(kibble_bit, source, bid, token, TLD): """ Scans a single job for activity """ - # NOW = int(datetime.datetime.utcnow().timestamp()) - dhash = hashlib.sha224( - ("%s-%s-%s" % (source["organisation"], source["sourceURL"], bid)).encode( - "ascii", errors="replace" - ) - ).hexdigest() - # Get the job data pages = 0 offset = 0 last_page = False - oURL = "https://api.travis-ci.%s/repo/%s/builds" % (TLD, bid) + o_url = "https://api.travis-ci.%s/repo/%s/builds" % (TLD, bid) # For as long as pagination makes sense... while not last_page: - bURL = "https://api.travis-ci.%s/repo/%s/builds?limit=100&offset=%u" % ( + b_url = "https://api.travis-ci.%s/repo/%s/builds?limit=100&offset=%u" % ( TLD, bid, offset, ) - KibbleBit.pprint("Scanning %s" % bURL) + kibble_bit.pprint("Scanning %s" % b_url) rv = requests.get( - bURL, + b_url, headers={"Travis-API-Version": "3", "Authorization": "token %s" % token}, ) if rv.status_code == 200: repojs = rv.json() # If travis tells us it's the last page, trust it. if repojs["@pagination"]["is_last"]: - KibbleBit.pprint( + kibble_bit.pprint( "Assuming this is the last page we need (travis says so)" ) last_page = True - KibbleBit.pprint( - "%s has %u builds done" % (bURL, repojs["@pagination"]["count"]) + kibble_bit.pprint( + "%s has %u builds done" % (b_url, repojs["@pagination"]["count"]) ) # BREAKER: If we go past count somehow, and travis doesn't say so, bork anyway @@ -85,10 +78,10 @@ def scanJob(KibbleBit, source, bid, token, TLD): offset += 100 for build in repojs.get("builds", []): - buildID = build["id"] - buildProject = build["repository"]["slug"] - startedAt = build["started_at"] - finishedAt = build["finished_at"] + build_id = build["id"] + build_project = build["repository"]["slug"] + started_at = build["started_at"] + finished_at = build["finished_at"] duration = build["duration"] completed = True if duration else False duration = duration or 0 @@ -96,13 +89,13 @@ def scanJob(KibbleBit, source, bid, token, TLD): buildhash = hashlib.sha224( ( "%s-%s-%s-%s" - % (source["organisation"], source["sourceURL"], bid, buildID) + % (source["organisation"], source["sourceURL"], bid, build_id) ).encode("ascii", errors="replace") ).hexdigest() builddoc = None try: - builddoc = KibbleBit.get("ci_build", buildhash) - except: + builddoc = kibble_bit.get("ci_build", buildhash) + except: # pylint: disable=bare-except pass # If this build already completed, no need to parse it again @@ -110,7 +103,7 @@ def scanJob(KibbleBit, source, bid, token, TLD): # If we're on page > 1 and we've seen a completed build, assume # that we don't need the older ones if pages > 1: - KibbleBit.pprint( + kibble_bit.pprint( "Assuming this is the last page we need (found completed build on page > 1)" ) last_page = True @@ -126,16 +119,16 @@ def scanJob(KibbleBit, source, bid, token, TLD): if build["state"] in ["aborted", "canceled"]: status = "aborted" - FIN = 0 - STA = 0 - if finishedAt: - FIN = datetime.datetime.strptime( - finishedAt, "%Y-%m-%dT%H:%M:%SZ" + fin = 0 + sta = 0 + if finished_at: + fin = datetime.datetime.strptime( + finished_at, "%Y-%m-%dT%H:%M:%SZ" ).timestamp() - if startedAt: - STA = int( + if started_at: + sta = int( datetime.datetime.strptime( - startedAt, "%Y-%m-%dT%H:%M:%SZ" + started_at, "%Y-%m-%dT%H:%M:%SZ" ).timestamp() ) @@ -145,14 +138,14 @@ def scanJob(KibbleBit, source, bid, token, TLD): doc = { # Build specific data "id": buildhash, - "date": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(FIN)), - "buildID": buildID, + "date": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(fin)), + "buildID": build_id, "completed": completed, "duration": duration * 1000, - "job": buildProject, - "jobURL": oURL, + "job": build_project, + "jobURL": o_url, "status": status, - "started": STA, + "started": sta, "ci": "travis", "queuetime": queuetime, # Standard docs values @@ -160,31 +153,31 @@ def scanJob(KibbleBit, source, bid, token, TLD): "organisation": source["organisation"], "upsert": True, } - KibbleBit.append("ci_build", doc) + kibble_bit.append("ci_build", doc) pages += 1 else: # We hit a snag, abort! - KibbleBit.pprint("Travis returned a non-200 response, aborting.") + kibble_bit.pprint("Travis returned a non-200 response, aborting.") return False return True -class travisThread(threading.Thread): +class TravisThread(threading.Thread): """ Generic thread class for scheduling multiple scans at once """ - def __init__(self, block, KibbleBit, source, token, jobs, TLD): - super(travisThread, self).__init__() + def __init__(self, block, kibble_bit, source, token, jobs, TLD): + super(TravisThread, self).__init__() self.block = block - self.KibbleBit = KibbleBit + self.kibble_bit = kibble_bit self.token = token self.source = source self.jobs = jobs self.tld = TLD def run(self): - badOnes = 0 - while len(self.jobs) > 0 and badOnes <= 50: + bad_ones = 0 + while len(self.jobs) > 0 and bad_ones <= 50: self.block.acquire() try: job = self.jobs.pop(0) @@ -195,11 +188,11 @@ def run(self): self.block.release() return self.block.release() - if not scanJob(self.KibbleBit, self.source, job, self.token, self.tld): - self.KibbleBit.pprint("[%s] This borked, trying another one" % job) - badOnes += 1 - if badOnes > 100: - self.KibbleBit.pprint("Too many errors, bailing!") + if not scan_job(self.kibble_bit, self.source, job, self.token, self.tld): + self.kibble_bit.pprint("[%s] This borked, trying another one" % job) + bad_ones += 1 + if bad_ones > 100: + self.kibble_bit.pprint("Too many errors, bailing!") self.source["steps"]["travis"] = { "time": time.time(), "status": "Too many errors while parsing at " @@ -207,38 +200,37 @@ def run(self): "running": False, "good": False, } - self.KibbleBit.updateSource(self.source) + self.kibble_bit.update_source(self.source) return else: - badOnes = 0 + bad_ones = 0 -def sclan(KibbleBit, source): +def scan(kibble_bit, source): # Simple URL check travis = re.match(r"https?://travis-ci\.(org|com)", source["sourceURL"]) if travis: # Is this travs-ci.org or travis-ci.com - we need to know! - TLD = travis.group(1) + tld = travis.group(1) source["steps"]["travis"] = { "time": time.time(), "status": "Parsing Travis job changes...", "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) - pendingJobs = [] - KibbleBit.pprint("Parsing Travis activity at %s" % source["sourceURL"]) + pending_jobs = [] + kibble_bit.pprint("Parsing Travis activity at %s" % source["sourceURL"]) source["steps"]["travis"] = { "time": time.time(), "status": "Downloading changeset", "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) # Travis needs a token - token = None if ( source["creds"] and "token" in source["creds"] @@ -247,7 +239,7 @@ def sclan(KibbleBit, source): ): token = source["creds"]["token"] else: - KibbleBit.pprint("Travis CI requires a token to work!") + kibble_bit.pprint("Travis CI requires a token to work!") return False # Used for pagination @@ -262,15 +254,15 @@ def sclan(KibbleBit, source): stuck = 0 # Ditto avgqueuetime = 0 # Ditto, fake it - maybeQueued = [] + maybe_queued = [] while jobs == 100: - URL = ( + url = ( "https://api.travis-ci.%s/repos?repository.active=true&sort_by=current_build:desc&offset=%u&limit=100&include=repository.last_started_build" - % (TLD, offset) + % (tld, offset) ) offset += 100 r = requests.get( - URL, + url, headers={ "Travis-API-Version": "3", "Authorization": "token %s" % token, @@ -278,7 +270,7 @@ def sclan(KibbleBit, source): ) if r.status_code != 200: - KibbleBit.pprint("Travis did not return a 200 Okay, bad token?!") + kibble_bit.pprint("Travis did not return a 200 Okay, bad token?!") source["steps"]["travis"] = { "time": time.time(), @@ -289,7 +281,7 @@ def sclan(KibbleBit, source): "running": False, "good": False, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) return # For each build job @@ -301,20 +293,20 @@ def sclan(KibbleBit, source): # Is the build currently running? if cb["state"] in ["started", "created", "queued", "pending"]: for job in cb.get("jobs", []): - maybeQueued.append(job["id"]) + maybe_queued.append(job["id"]) # Queue up build jobs for the threaded scanner bid = repo["id"] - pendingJobs.append(bid) + pending_jobs.append(bid) jobs = len(js["repositories"]) - KibbleBit.pprint("Scanned %u jobs..." % total) + kibble_bit.pprint("Scanned %u jobs..." % total) # Find out how many building and pending jobs - for jobID in maybeQueued: - URL = "https://api.travis-ci.%s/job/%u" % (TLD, jobID) + for job_id in maybe_queued: + url = "https://api.travis-ci.%s/job/%u" % (tld, job_id) r = requests.get( - URL, + url, headers={ "Travis-API-Version": "3", "Authorization": "token %s" % token, @@ -324,15 +316,15 @@ def sclan(KibbleBit, source): jobjs = r.json() if jobjs["state"] == "started": building += 1 - KibbleBit.pprint("Job %u is building" % jobID) + kibble_bit.pprint("Job %u is building" % job_id) elif jobjs["state"] in ["created", "queued", "pending"]: queued += 1 blocked += 1 # Queued in Travis generally means a job can't find an executor, and thus is blocked. - KibbleBit.pprint("Job %u is pending" % jobID) - KibbleBit.pprint("%u building, %u queued..." % (building, queued)) + kibble_bit.pprint("Job %u is pending" % job_id) + kibble_bit.pprint("%u building, %u queued..." % (building, queued)) # Save queue snapshot - NOW = int(datetime.datetime.utcnow().timestamp()) + now = int(datetime.datetime.utcnow().timestamp()) queuehash = hashlib.sha224( ( "%s-%s-queue-%s" @@ -343,8 +335,8 @@ def sclan(KibbleBit, source): # Write up a queue doc queuedoc = { "id": queuehash, - "date": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(NOW)), - "time": NOW, + "date": time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(now)), + "time": now, "building": building, "size": queued, "blocked": blocked, @@ -356,15 +348,15 @@ def sclan(KibbleBit, source): "organisation": source["organisation"], "upsert": True, } - KibbleBit.append("ci_queue", queuedoc) + kibble_bit.append("ci_queue", queuedoc) - KibbleBit.pprint("Found %u jobs in Travis" % len(pendingJobs)) + kibble_bit.pprint("Found %u jobs in Travis" % len(pending_jobs)) threads = [] block = threading.Lock() - KibbleBit.pprint("Scanning jobs using 4 sub-threads") + kibble_bit.pprint("Scanning jobs using 4 sub-threads") for i in range(0, 4): - t = travisThread(block, KibbleBit, source, token, pendingJobs, TLD) + t = TravisThread(block, kibble_bit, source, token, pending_jobs, tld) threads.append(t) t.start() @@ -372,7 +364,7 @@ def sclan(KibbleBit, source): t.join() # We're all done, yaay - KibbleBit.pprint("Done scanning %s" % source["sourceURL"]) + kibble_bit.pprint("Done scanning %s" % source["sourceURL"]) source["steps"]["travis"] = { "time": time.time(), @@ -381,4 +373,4 @@ def sclan(KibbleBit, source): "running": False, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) diff --git a/kibble/scanners/scanners/twitter.py b/kibble/scanners/scanners/twitter.py index fd6af5b2..dc87ed2a 100644 --- a/kibble/scanners/scanners/twitter.py +++ b/kibble/scanners/scanners/twitter.py @@ -37,7 +37,7 @@ def accepts(source): return False -def getFollowers(KibbleBit, source, t): +def get_followers(kibble_bit, source, t): """ Get followers of a handle, store them for mapping and trend purposes""" # Get our twitter handle handle = source["sourceURL"] @@ -59,18 +59,18 @@ def getFollowers(KibbleBit, source, t): "followers": no_followers, "date": d, } - KibbleBit.pprint("%s has %u followers currently." % (handle, no_followers)) - KibbleBit.index("twitter_followers", dhash, jst) + kibble_bit.pprint("%s has %u followers currently." % (handle, no_followers)) + kibble_bit.index("twitter_followers", dhash, jst) # Collect list of current followers followers = t.GetFollowers(screen_name=handle) # For each follower, if they're not mapped yet, add them # This has a limitation of 100 new added per run, but meh... - KibbleBit.pprint("Looking up followers of %s" % handle) + kibble_bit.pprint("Looking up followers of %s" % handle) for follower in followers: # id, name, screen_name are useful here - KibbleBit.pprint("Found %s as follower" % follower.screen_name) + kibble_bit.pprint("Found %s as follower" % follower.screen_name) # Store twitter follower profile if not already logged dhash = hashlib.sha224( @@ -78,7 +78,7 @@ def getFollowers(KibbleBit, source, t): "ascii", errors="replace" ) ).hexdigest() - if not KibbleBit.exists("twitter_follow", dhash): + if not kibble_bit.exists("twitter_follow", dhash): jst = { "organisation": source["organisation"], "sourceURL": source["sourceURL"], @@ -91,20 +91,20 @@ def getFollowers(KibbleBit, source, t): "%Y/%m/%d %H:%M:%S", time.gmtime() ), # First time we spotted them following. } - KibbleBit.pprint( + kibble_bit.pprint( "%s is new, recording date and details." % follower.screen_name ) - KibbleBit.index("twitter_follow", dhash, jst) + kibble_bit.index("twitter_follow", dhash, jst) -def scan(KibbleBit, source): +def scan(kibble_bit, source): source["steps"]["twitter"] = { "time": time.time(), "status": "Scanning Twitter activity and status", "running": True, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source) t = None if "creds" in source and source["creds"]: t = twitter.Api( @@ -113,22 +113,22 @@ def scan(KibbleBit, source): consumer_key=source["creds"].get("consumer_key", None), consumer_secret=source["creds"].get("consumer_secret", None), ) - KibbleBit.pprint("Verifying twitter credentials...") + kibble_bit.pprint("Verifying twitter credentials...") try: t.VerifyCredentials() - except: + except: # pylint: disable=bare-except source["steps"]["twitter"] = { "time": time.time(), "status": "Could not verify twitter credentials", "running": False, "good": False, } - KibbleBit.updateSource(source) - KibbleBit.pprint("Could not verify twitter creds, aborting!") + kibble_bit.update_source(source) + kibble_bit.pprint("Could not verify twitter creds, aborting!") return # Start by getting and saving followers try: - getFollowers(KibbleBit, source, t) + get_followers(kibble_bit, source, t) except Exception as err: source["steps"]["twitter"] = { "time": time.time(), @@ -136,8 +136,8 @@ def scan(KibbleBit, source): "running": False, "good": False, } - KibbleBit.updateSource(source) - KibbleBit.pprint("Twitter scan failed: %s" % err) + kibble_bit.update_source(source) + kibble_bit.pprint("Twitter scan failed: %s" % err) # All done, report that! source["steps"]["twitter"] = { @@ -147,4 +147,4 @@ def scan(KibbleBit, source): "running": False, "good": True, } - KibbleBit.updateSource(source) + kibble_bit.update_source(source)