TEMP Test Feed Validation Error

Author: rachelekm

deployment/dataproc/gtfs-feed-fetcher/.gitignore

@@ -8,6 +8,8 @@ gtfs/*
 *.html
 patco-gtfs/
 transitfeedcrash.txt
+# temp for testing
+!gtfs/septa_bus.zip
 
 # virtualenv
 env/

deployment/dataproc/gtfs-feed-fetcher/feed_sources/Septa.py

@@ -33,47 +33,48 @@ def __init__(self):
     def fetch(self):
         """Fetch SEPTA bus and rail feeds.
         """
+        self.verify(BUS_FILE)
         # Check GitHub latest release page to see if there is a newer download available.
-        request = requests.get(URL)
-        if request.ok:
-            response = request.json()
-            download_url = response['assets'][0]['browser_download_url']
-            last_updated_str = response['assets'][0]['updated_at']
-            last_updated = datetime.strptime(last_updated_str, LAST_UPDATED_FMT)
-            stat = self.status.get(BUS_FILE)
-            if stat:
-                got_last = datetime.strptime(stat['posted_date'], TIMECHECK_FMT)
-                LOG.debug('SEPTA GTFS last fetched: %s, last updated: %s', got_last, last_updated)
-                if got_last >= last_updated:
-                    LOG.info('No new download available for SEPTA.')
-                    self.update_existing_status(BUS_FILE)
-                    self.update_existing_status(RAIL_FILE)
-                    return
-            else:
-                LOG.info('No previous SEPTA download found. Last update posted: %s', last_updated)
+        # request = requests.get(URL)
+        # if request.ok:
+        #     response = request.json()
+        #     download_url = response['assets'][0]['browser_download_url']
+        #     last_updated_str = response['assets'][0]['updated_at']
+        #     last_updated = datetime.strptime(last_updated_str, LAST_UPDATED_FMT)
+        #     stat = self.status.get(BUS_FILE)
+        #     if stat:
+        #         got_last = datetime.strptime(stat['posted_date'], TIMECHECK_FMT)
+        #         LOG.debug('SEPTA GTFS last fetched: %s, last updated: %s', got_last, last_updated)
+        #         if got_last >= last_updated:
+        #             LOG.info('No new download available for SEPTA.')
+        #             self.update_existing_status(BUS_FILE)
+        #             self.update_existing_status(RAIL_FILE)
+        #             return
+        #     else:
+        #         LOG.info('No previous SEPTA download found. Last update posted: %s', last_updated)
 
-            posted_date = last_updated.strftime(TIMECHECK_FMT)
-            self.set_posted_date(BUS_FILE, posted_date)
-            self.set_posted_date(RAIL_FILE, posted_date)
+        #     posted_date = last_updated.strftime(TIMECHECK_FMT)
+        #     self.set_posted_date(BUS_FILE, posted_date)
+        #     self.set_posted_date(RAIL_FILE, posted_date)
 
 
-            if self.download(DOWNLOAD_FILE_NAME, download_url, do_stream=False):
-                # remove posted date status for parent zip
-                del self.status[DOWNLOAD_FILE_NAME]
-                septa_file = os.path.join(self.ddir, DOWNLOAD_FILE_NAME)
-                if self.extract(septa_file):
-                    self.write_status()
-                # delete download file once the two GTFS zips in it are extracted
-                os.remove(septa_file)
-            else:
-                # clear error for parent septa.zip download, and set statuses for bus and rail feeds
-                err = self.status.get(DOWNLOAD_FILE_NAME, 'Could not download file')
-                self.status = {}
-                self.set_error(BUS_FILE, err)
-                self.set_error(RAIL_FILE, err)
-                self.write_status()
-        else:
-            LOG.error('Could not check GitHub relases page for SEPTA.')
+        #     if self.download(DOWNLOAD_FILE_NAME, download_url, do_stream=False):
+        #         # remove posted date status for parent zip
+        #         del self.status[DOWNLOAD_FILE_NAME]
+        #         septa_file = os.path.join(self.ddir, DOWNLOAD_FILE_NAME)
+        #         if self.extract(septa_file):
+        #             self.write_status()
+        #         # delete download file once the two GTFS zips in it are extracted
+        #         os.remove(septa_file)
+        #     else:
+        #         # clear error for parent septa.zip download, and set statuses for bus and rail feeds
+        #         err = self.status.get(DOWNLOAD_FILE_NAME, 'Could not download file')
+        #         self.status = {}
+        #         self.set_error(BUS_FILE, err)
+        #         self.set_error(RAIL_FILE, err)
+        #         self.write_status()
+        # else:
+        #     LOG.error('Could not check GitHub relases page for SEPTA.')
 
     def extract(self, file_name):
         """Extract bus and rail GTFS files from downloaded zip, then validate each."""

deployment/dataproc/gtfs-feed-fetcher/fetch_cac_feeds.py

@@ -25,6 +25,8 @@ def fetch_all(sources=None):
     # make a copy of the list of all modules in feed_sources;
     # default to use all of them
     sources = list(feed_sources.__all__)
+    #TEMP
+    sources = ['Septa']
 
     LOG.info('Going to fetch feeds from sources: %s', sources)