Update routes for specific dates

backend/models/gtfs.py

@@ -8,6 +8,8 @@
 import gzip
 import hashlib
 import zipfile
+import shutil
+import os
 
 from . import config, util, nextbus, routeconfig, timetables
 
@@ -50,27 +52,40 @@ def get_stop_geometry(stop_xy, shape_lines_xy, shape_cumulative_dist, start_inde
         'offset': int(best_offset) # distance in meters between this stop and the closest line segment of shape
     }
 
-def download_gtfs_data(agency: config.Agency, gtfs_cache_dir):
-    gtfs_url = agency.gtfs_url
+def download_gtfs_data(agency: config.Agency, gtfs_cache_dir, archiving_url=None):
+    cache_dir = Path(gtfs_cache_dir)
+    zip_path = f'{util.get_data_dir()}/gtfs-{agency.id}.zip'
+    if archiving_url == None:
+        gtfs_url = agency.gtfs_url
+    else:
+        gtfs_url = archiving_url
+
+        # need to delete existing zip file and directory in order
+        # to reuse for the archiving passes
+        if cache_dir.exists():	
+            shutil.rmtree(cache_dir)
+            print('removed',cache_dir)
+            os.remove(zip_path)
+            print('removed',zip_path)					
+
     if gtfs_url is None:
         raise Exception(f'agency {agency.id} does not have gtfs_url in config')
 
-    cache_dir = Path(gtfs_cache_dir)
+
     if not cache_dir.exists():
         print(f'downloading gtfs data from {gtfs_url}')
         r = requests.get(gtfs_url)
 
         if r.status_code != 200:
             raise Exception(f"Error fetching {gtfs_url}: HTTP {r.status_code}: {r.text}")
 
-        zip_path = f'{util.get_data_dir()}/gtfs-{agency.id}.zip'
-
         with open(zip_path, 'wb') as f:
             f.write(r.content)
 
         with zipfile.ZipFile(zip_path, 'r') as zip_ref:
             zip_ref.extractall(gtfs_cache_dir)
-
+
+
 def is_subsequence(smaller, bigger):
     smaller_len = len(smaller)
     bigger_len = len(bigger)
@@ -108,15 +123,14 @@ def contains_excluded_stop(shape_stop_ids, excluded_stop_ids):
     return False
 
 class GtfsScraper:
-    def __init__(self, agency: config.Agency):
+    def __init__(self, agency: config.Agency, archiving_url=None):
         self.agency = agency
         self.agency_id = agency_id = agency.id
         gtfs_cache_dir = f'{util.get_data_dir()}/gtfs-{agency_id}'
 
-        download_gtfs_data(agency, gtfs_cache_dir)
+        download_gtfs_data(agency, gtfs_cache_dir, archiving_url=archiving_url)
 
         self.feed = ptg.load_geo_feed(gtfs_cache_dir, {})
-
         self.errors = []
         self.stop_times_by_trip = None
         self.stops_df = None
@@ -261,7 +275,8 @@ def save_timetables(self, save_to_s3=False, skip_existing=False):
         agency_id = self.agency_id
 
         dates_map = self.get_services_by_date()
-
+        ##bri##  print('here\n\n\n',dates_map)
+        ##bri##  exit()
         #
         # Typically, many dates have identical scheduled timetables (with times relative to midnight on that date).
         # Instead of storing redundant timetables for each date, store one timetable per route for each unique set of service_ids.
@@ -1058,7 +1073,7 @@ def get_sort_key(route_data):
             return route_data['title']
         return sorted(routes_data, key=get_sort_key)
 
-    def save_routes(self, save_to_s3, d):
+    def save_routes(self, save_to_s3, d, version_date=None):
         agency = self.agency
         agency_id = agency.id
         routes_df = self.get_gtfs_routes()
@@ -1078,4 +1093,4 @@ def save_routes(self, save_to_s3, d):
 
         routes = [routeconfig.RouteConfig(agency_id, route_data) for route_data in routes_data]
 
-        routeconfig.save_routes(agency_id, routes, save_to_s3=save_to_s3)
+        routeconfig.save_routes(agency_id, routes, save_to_s3=save_to_s3, version_date=version_date)

backend/models/routeconfig.py

@@ -1,4 +1,5 @@
 import re, os, time, requests, json, boto3, gzip
+from pathlib import Path
 from . import util, config
 
 DefaultVersion = 'v3a'
@@ -121,8 +122,17 @@ def get_directions_for_stop(self, stop_id):
             for s in direction['stops'] if s == stop_id
         ]
 
-def get_cache_path(agency_id, version=DefaultVersion):
-    return f'{util.get_data_dir()}/routes_{version}_{agency_id}.json'
+def get_cache_path(agency_id, version=DefaultVersion, version_date=None):
+    # version_date is for saving old versions of routes
+    # It has nothing to do with version=DefaultVersion
+    if version_date == None:
+        return f'{util.get_data_dir()}/routes_{version}_{agency_id}.json'
+    else:
+        return f'{util.get_data_dir()}/routes_{version}_{agency_id}_{version_date}/routes_{version}_{agency_id}_{version_date}.json'
+
+
+    ##bri##return f"{util.get_data_dir()}/datekeys_{version}_{agency_id}/datekeys_{version}_{agency_id}.json"		
+
 
 def get_s3_path(agency_id, version=DefaultVersion):
     return f'routes/{version}/routes_{version}_{agency_id}.json.gz'
@@ -179,14 +189,17 @@ def get_route_config(agency_id, route_id, version=DefaultVersion):
             return route
     return None
 
-def save_routes(agency_id, routes, save_to_s3=False):
+def save_routes(agency_id, routes, save_to_s3=False, version_date=None):
     data_str = json.dumps({
         'version': DefaultVersion,
         'routes': [route.data for route in routes]
     }, separators=(',', ':'))
 
-    cache_path = get_cache_path(agency_id)
-
+    cache_path = get_cache_path(agency_id, version_date=version_date)
+    cache_dir = Path(cache_path).parent
+    if not cache_dir.exists():
+        cache_dir.mkdir(parents = True, exist_ok = True)
+
     with open(cache_path, "w") as f:
         f.write(data_str)
 

backend/save_routes.py

@@ -2,6 +2,8 @@
 from compute_stats import compute_stats_for_dates
 import argparse
 from datetime import date
+import requests
+from secrets import transitfeeds_api_key # you may have to create this
 
 # Downloads and parses the GTFS specification
 # and saves the configuration for all routes to S3.
@@ -34,7 +36,7 @@
 #
 # Currently the script just overwrites the one S3 path, but this process could be extended in the future to
 # store different paths for different dates, to allow fetching historical data for route configurations.
-#
+# UPDATE: We are now saving some older routes in versioned directories in metrics-mvp/backend/data
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Save route configuration from GTFS and possibly Nextbus API')
@@ -52,12 +54,38 @@
 
     save_to_s3 = args.s3
     d = date.today()
-
+    archive_date = date.today()
     errors = []
 
+    limit = '10'
+    urls_feed = 'https://api.transitfeeds.com/v1/getFeedVersions?key=' + transitfeeds_api_key + '&feed=sfmta%2F60&page=1&limit=' + limit + '&err=1&warn=1'
+
+    response = requests.get(urls_feed)
+    data = response.json()
+    archiving_urls = []
+    archiving_dates = []
+    for i in range(len(data['results']['versions'])):
+        archiving_urls.append(data['results']['versions'][i]['url'])	
+        archiving_dates.append(archiving_urls[i].split('/')[6])
+        archiving_dates[i] = archiving_dates[i][:4]+'-'+archiving_dates[i][4:6]+'-'+archiving_dates[i][6:]
+
     for agency in agencies:
-        scraper = gtfs.GtfsScraper(agency)
+        scraper = gtfs.GtfsScraper(agency, archiving_url=None)
         scraper.save_routes(save_to_s3, d)
+        errors += scraper.errors		
+        '''
+        use https://transitfeeds.com/api/swagger/ 
+        to get old routes 
+        and cache them in date versioned folders
+
+        '''
+
+        while(len(archiving_dates) > 0):
+            archiving_date = archiving_dates.pop()
+            archiving_url = archiving_urls.pop()
+            scraper_archiving = gtfs.GtfsScraper(agency, archiving_url=archiving_url)			
+            scraper_archiving.save_routes(False, d, version_date=archiving_date)	
+            errors += scraper_archiving.errors			
 
         if args.timetables:
             timetables_updated = scraper.save_timetables(save_to_s3=save_to_s3, skip_existing=True)
@@ -66,7 +94,6 @@
                 dates = sorted(scraper.get_services_by_date().keys())
                 compute_stats_for_dates(dates, agency, scheduled=True, save_to_s3=save_to_s3)
 
-        errors += scraper.errors
 
     if errors:
         raise Exception("\n".join(errors))