Lots of improvements to GitHub integration. Good titles, issue body a…

…nd a crash reporter label for easy filtering. (tessel#19)

Also, added the ability to resolve crashes via GitHub web hooks.

.gitignore

@@ -1,4 +1,7 @@
+# client secrets
 client_secrets.json
+debug_client_secrets.json
+
 ._DS_Store
 .DS_Store
 *.pyc

github_utils.py

@@ -9,6 +9,7 @@
 
 # constants
 TOKEN_KEY = 'github_token'
+WEBHOOK_SECRET = 'webhook_secret'
 
 DEBUG_OWNER = 'tikurahul'
 DEBUG_REPO = 'sandbox'
@@ -18,6 +19,9 @@
 REPO = 't2-cli'
 CRASH_REPORTER_HOST = 'http://crash-reporter.tessel.io'
 
+DEBUG_CLIENT_SECRETS = 'debug_client_secrets.json'
+CLIENT_SECRETS = 'client_secrets.json'
+
 
 def issue_url(issue_number):
     """
@@ -78,13 +82,13 @@ def manage_github_issue(cls, crash_report):
                     GithubOrchestrator.create_issue_job,
                     crash_report.fingerprint, _queue=GithubOrchestrator.__QUEUE__)
                 logging.info(
-                    'Enqueued job for new issue on GitHub for fingerprint %s' % crash_report.fingerprint)
+                    'Enqueued job for new issue on GitHub for fingerprint {0}'.format(crash_report.fingerprint))
             elif count > 0 and count % GithubOrchestrator.__NOTIFY_FREQUENCY__ == 0:
                 # add comments for an existing crash
                 deferred.defer(
                     GithubOrchestrator.add_comment_job, crash_report.fingerprint, _queue=GithubOrchestrator.__QUEUE__)
                 logging.info(
-                    'Enqueued job for adding comments on GitHub for fingerprint %s' % crash_report.fingerprint)
+                    'Enqueued job for adding comments on GitHub for fingerprint {0}'.format(crash_report.fingerprint))
             else:
                 logging.debug('No pending tasks.')
 
@@ -107,15 +111,17 @@ def create_issue_job(cls, fingerprint):
             if crash_report is not None:
                 # create the github issue
                 issue = github_client.create_issue(crash_report)
-                logging.info('Created GitHub Issue No(%s) for crash (%s)' % (issue.number, crash_report.fingerprint))
+                logging.info(
+                    'Created GitHub Issue No({0}) for crash ({1})'.format(issue.number, crash_report.fingerprint))
                 # update the crash report with the issue id
                 updated_report = CrashReports.update_crash_report(crash_report.fingerprint, {
                     # convert to unicode string
                     'issue': str(issue.number)
                 })
-                logging.info('Updating crash report with fingerprint (%s) complete.' % updated_report.fingerprint)
+                logging.info(
+                    'Updating crash report with fingerprint ({0}) complete.'.format(updated_report.fingerprint))
         except Exception, e:
-            logging.error('Error creating issue for fingerprint (%s) [%s]' % (fingerprint, e.message))
+            logging.error('Error creating issue for fingerprint ({0}) [{1}]'.format(fingerprint, str(e)))
 
     @classmethod
     def add_comment_job(cls, fingerprint):
@@ -128,61 +134,68 @@ def add_comment_job(cls, fingerprint):
             if crash_report is not None:
                 github_client.create_comment(crash_report)
         except Exception, e:
-            logging.error('Error creating comment for fingerprint (%s) [%s]' % (fingerprint, e.message))
+            logging.error('Error creating comment for fingerprint ({0}) [{1}]'.format(fingerprint, str(e)))
 
 
 class GithubClient(object):
     """
     A set of github utilities.
     """
     @classmethod
-    def issue_title(cls, fingerprint):
-        return 'Crash report %s' % fingerprint
+    def issue_title(cls, crash_report=None):
+        crash = crash_report.crash
+        lines = [line for line in crash.splitlines(True) if len(line) > 0]
+        return 'Crash report: {0}'.format(lines[0])
 
-    def issue_body(self, fingerprint):
-        crash_report_uri = '%s%s' % (self.reporter_host, crash_uri(fingerprint))
-        body = 'Full report is at [%s](%s)' % (fingerprint, crash_report_uri)
+    def issue_body(self, crash_report):
+        crash = crash_report.crash.encode('ascii', 'ignore')
+        fingerprint = crash_report.fingerprint
+        crash_report_uri = '{0}{1}'.format(self.reporter_host, crash_uri(fingerprint))
+        body = '{0}\n\nFull report is at [{1}]({2})'.format(crash, fingerprint, crash_report_uri)
         return body
 
     @classmethod
     def issue_comment(cls, count):
-        new_comment = 'More crashes incoming. Current crash count is at `%s`.' % count
+        new_comment = 'More crashes incoming. Current crash count is at `{0}`.'.format(count)
         return new_comment
 
     def __init__(self):
-        with open('client_secrets.json', 'r') as contents:
+        if is_appengine_local():
+            secrets = DEBUG_CLIENT_SECRETS
+        else:
+            secrets = CLIENT_SECRETS
+        with open(secrets, 'r') as contents:
             secrets = json.loads(contents.read())
             github_token = secrets.get(TOKEN_KEY)
+            self.webhook_secret = secrets.get(WEBHOOK_SECRET)
             if is_appengine_local():
                 self.reporter_host = DEBUG_CRASH_REPORTER_HOST
-                self.repo_name = '%s/%s' % (DEBUG_OWNER, DEBUG_REPO)
+                self.repo_name = '{0}/{1}'.format(DEBUG_OWNER, DEBUG_REPO)
             else:
                 self.reporter_host = CRASH_REPORTER_HOST
-                self.repo_name = '%s/%s' % (OWNER, REPO)
+                self.repo_name = '{0}/{1}'.format(OWNER, REPO)
             self.github_client = Github(login_or_token=github_token)
 
     def create_issue(self, crash_report):
         """
         Submits a GitHub issue for a given fingerprint.
         """
-        fingerprint = crash_report.fingerprint
         # get repository
         repository = self.github_client.get_repo(self.repo_name)
-
         # create issue
         issue = repository.create_issue(
-            title=GithubClient.issue_title(fingerprint),
-            body=self.issue_body(fingerprint),
-            labels=[str(fingerprint)])
-
+            title=GithubClient.issue_title(crash_report),
+            body=self.issue_body(crash_report),
+            labels=['crash reporter']
+        )
         return issue
 
     def create_comment(self, crash_report):
         """
         Updates a crash report with the comment.
         """
         count = CrashReport.get_count(crash_report.name)
-        issue_number = int(float(crash_report.issue))
+        issue_number = int(crash_report.issue)
         comment_body = self.issue_comment(count)
 
         # get repo

main.py

@@ -1,5 +1,6 @@
 import StringIO
 import csv
+import json
 import logging
 import urllib
 
@@ -232,6 +233,38 @@ def post(self):
         self.render('update-global-preferences.html')
 
 
+class GitHubWebHooksHandler(webapp2.RequestHandler):
+    @common_request
+    def post(self):
+        """
+        Handle GitHub webhooks
+        """
+        headers = self.request_handler.request.headers
+        event_type = headers.get('X-GitHub-Event')
+        request_body = json.loads(self.request_handler.request.body)
+
+        if event_type is not None and request_body is not None:
+            logging.info('X-GitHub-Event {0}'.format(event_type))
+            if event_type == 'issues':
+                '''
+                We are looking for issue closed event types.
+                https://developer.github.com/v3/activity/events/types/#issuesevent
+                '''
+                action = request_body.get('action')
+                if action == 'closed':
+                    '''
+                    Handle closed event
+                    '''
+                    # the full issue body
+                    issue = request_body.get('issue')
+                    # issue number (treated as a string in the datastore)
+                    number = str(issue.get('number'))
+                    CrashReports.close_github_issue(number)
+                    logging.info('Marked GitHub issue {0} as resolved'.format(number))
+                else:
+                    logging.info('Other action {0}. Ignoring.'.format(action))
+
+
 application = webapp2.WSGIApplication(
     [
         webapp2.Route('/', handler='main.RootHandler', name='home'),
@@ -241,6 +274,7 @@ def post(self):
         webapp2.Route('/trending', handler='main.TrendingCrashesHandler', name='trending_crashes'),
         webapp2.Route('/search', handler='main.SearchCrashesHandler', name='search'),
         webapp2.Route('/preferences/update', handler='main.UpdatePreferencesHandler', name='update_global_preferences'),
+        webapp2.Route('/webhooks/github', handler='main.GitHubWebHooksHandler', name='github_webhooks'),
     ]
     , debug=True
 )

pages/search.html

@@ -26,7 +26,9 @@ <h2>Search<small></small></h2>
               <p>
               Searchable Properties include
                 <code>fingerprint</code>, <code>crash</code>,
-                <code>time</code>, <code>count</code> and Crash <code>state</code>.
+                <code>time</code>, <code>count</code>, GitHub <code>issue</code>
+                and Crash <code>state</code>. <br/>
+                Results are sorted my <code>time</code> (most recent first).
               </p>
               <p>
                 Examples:

util.py

@@ -55,6 +55,17 @@ def update_report_state(cls, fingerprint, new_state):
            'state': new_state
         })
 
+    @classmethod
+    def close_github_issue(cls, issue_number):
+        # find the fingerprint
+        q = CrashReport.all()
+        q.filter('issue = ', issue_number)
+        for entity in q.run(limit=1):
+            fingerprint = entity.fingerprint
+            cls.update_crash_report(fingerprint, {
+                'state': 'resolved'
+            })
+
     @classmethod
     def update_report_issue(cls, fingerprint, issue):
         return CrashReports.update_crash_report(fingerprint, {