diff --git a/compare_skims.py b/compare_skims.py new file mode 100644 index 00000000..74ea71ec --- /dev/null +++ b/compare_skims.py @@ -0,0 +1,482 @@ +import sys +import os +import argparse +import logging + +import openmatrix as omx +import numpy as np +import pandas as pd + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + # We don't specify handlers here, so default console output is prevented. + # We will explicitly add a StreamHandler and a FileHandler. +) + +logger = logging.getLogger(__name__) # Get the logger for this module + +# Get the root logger +root_logger = logging.getLogger() +# Clear any default handlers that might have been added by basicConfig or previous calls +if root_logger.hasHandlers(): + root_logger.handlers.clear() + +# Define the formatter +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Create a console handler (stdout) +console_handler = logging.StreamHandler(sys.stdout) +console_handler.setLevel(logging.INFO) # Set level for console output +console_handler.setFormatter(formatter) +root_logger.addHandler(console_handler) # Add handler to the root logger + +# Create a file handler +log_file_path = "compare_skims.log" # Define the name of the log file +file_handler = logging.FileHandler(log_file_path) +file_handler.setLevel(logging.INFO) # Set level for file output +file_handler.setFormatter(formatter) +root_logger.addHandler(file_handler) # Add handler to the root logger + +logger.info(f"Logging output to console and file: {log_file_path}") + +def parse_matrix_name(matrix_name): + """Parses a matrix name into path, measure, and time period.""" + # Expected format: PATH_MEASURE__TIMEPERIOD or MEASURE (for non-OD skims like DIST) + parts = matrix_name.rsplit('__', 1) + if len(parts) == 2: + measure_part, time_period = parts + # Try to split measure_part into path and measure + measure_parts = measure_part.rsplit('_', 1) + if len(measure_parts) == 2: + path, measure = measure_parts + else: + # Handle cases like 'DIST__AM' where there's no explicit path + path = None + measure = measure_part # This line was correct + else: + # Handle cases with no time period like 'DIST' + measure_part = matrix_name + time_period = None + measure_parts = measure_part.rsplit('_', 1) + if len(measure_parts) == 2: + # Could be SOV_DIST or SOVTOLL_TOLL + path, measure = measure_parts + else: + # Could be just DIST, or a non-standard name + path = None + measure = measure_part # FIX: Changed 'matrix_part' to 'measure_part' + + return path, measure, time_period + +def calculate_matrix_statistics(data, matrix_name, file_path): + """ + Calculates statistics for a given NumPy array matrix. + + Parameters + ---------- + data : np.ndarray + The matrix data. + matrix_name : str + The name of the matrix. + file_path : str + The path to the OMX file (for logging). + + Returns + ------- + dict + A dictionary containing calculated statistics. + """ + if data is None: + return {"exists": False} + + total_elements = data.size + is_nan = np.isnan(data) + nan_count = np.sum(is_nan) + nan_percent = (nan_count / total_elements) * 100 if total_elements > 0 else 0 + + # Consider zeros as potentially "missing" or invalid for some skims + is_zero = (data == 0) + zero_count = np.sum(is_zero) + zero_percent = (zero_count / total_elements) * 100 if total_elements > 0 else 0 + + # Valid data: not NaN and not zero + is_valid = ~(is_nan | is_zero) + valid_count = np.sum(is_valid) + valid_percent = (valid_count / total_elements) * 100 if total_elements > 0 else 0 + + stats = { + "exists": True, + "shape": data.shape, + "total_elements": total_elements, + "nan_count": nan_count, + "nan_percent": nan_percent, + "zero_count": zero_count, + "zero_percent": zero_percent, + "valid_count": valid_count, + "valid_percent": valid_percent, + "descriptive_stats": {} + } + + if valid_count > 0: + valid_data = data[is_valid] + stats["descriptive_stats"] = { + "mean": np.mean(valid_data), + "median": np.median(valid_data), + "min": np.min(valid_data), + "max": np.max(valid_data), + "std": np.std(valid_data) + } + + return stats + +def compare_skims(path1, path2): + """ + Compares two OMX skim files matrix by matrix. + + Parameters + ---------- + path1 : str + Path to the first OMX file. + path2 : str + Path to the second OMX file. + """ + if not os.path.exists(path1): + logger.error(f"File not found: {path1}") + return + if not os.path.exists(path2): + logger.error(f"File not found: {path2}") + return + + logger.info(f"Comparing skim files:") + logger.info(f" File 1: {path1}") + logger.info(f" File 2: {path2}") + logger.info("-" * 80) + + skims1 = None + skims2 = None + + try: + skims1 = omx.open_file(path1, 'r') + skims2 = omx.open_file(path2, 'r') + + tables1 = set(skims1.list_matrices()) + tables2 = set(skims2.list_matrices()) + + all_tables = sorted(list(tables1.union(tables2))) + + missing_in_2 = sorted(list(tables1 - tables2)) + missing_in_1 = sorted(list(tables2 - tables1)) + + if missing_in_2: + logger.warning(f"Matrices present in '{os.path.basename(path1)}' but not in '{os.path.basename(path2)}': {missing_in_2}") + if missing_in_1: + logger.warning(f"Matrices present in '{os.path.basename(path2)}' but not in '{os.path.basename(path1)}': {missing_in_1}") + + comparison_summary = [] + + for table_name in all_tables: + logger.info(f"\n--- Comparing matrix: {table_name} ---") + + path, measure, period = parse_matrix_name(table_name) + + # --- Basic Statistics (Keep existing logic) --- + data1 = None + data2 = None + stats1 = {"exists": False} + stats2 = {"exists": False} + + # Load data if exists + if table_name in tables1: + try: + data1 = np.array(skims1[table_name]) + stats1 = calculate_matrix_statistics(data1, table_name, path1) + except Exception as e: + logger.error(f"Error reading matrix {table_name} from {path1}: {e}") + stats1 = {"exists": True, "error": str(e)} # Mark as exists but error happened + + if table_name in tables2: + try: + data2 = np.array(skims2[table_name]) + stats2 = calculate_matrix_statistics(data2, table_name, path2) + except Exception as e: + logger.error(f"Error reading matrix {table_name} from {path2}: {e}") + stats2 = {"exists": True, "error": str(e)} # Mark as exists but error happened + + # Report basic statistics for each file (Keep existing logic) + logger.info(f" File 1 ({os.path.basename(path1)}):") + # ... (existing reporting for stats1) ... + if stats1["exists"]: + if "error" in stats1: + logger.info(f" Error: {stats1['error']}") + else: + logger.info(f" Shape: {stats1['shape']}") + logger.info(f" Total Elements: {stats1['total_elements']}") + logger.info(f" NaNs: {stats1['nan_count']} ({stats1['nan_percent']:.2f}%)") + logger.info(f" Zeros: {stats1['zero_count']} ({stats1['zero_percent']:.2f}%)") + logger.info(f" Valid: {stats1['valid_count']} ({stats1['valid_percent']:.2f}%)") + if stats1['valid_count'] > 0: + desc_stats = stats1["descriptive_stats"] + logger.info( + f" Valid Data Stats: Mean={desc_stats['mean']:.4f}, Median={desc_stats['median']:.4f}, Min={desc_stats['min']:.4f}, Max={desc_stats['max']:.4f}, Std Dev={desc_stats['std']:.4f}") + else: + logger.info(" Matrix not found.") + + logger.info(f" File 2 ({os.path.basename(path2)}):") + # ... (existing reporting for stats2) ... + if stats2["exists"]: + if "error" in stats2: + logger.info(f" Error: {stats2['error']}") + else: + logger.info(f" Shape: {stats2['shape']}") + logger.info(f" Total Elements: {stats2['total_elements']}") + logger.info(f" NaNs: {stats2['nan_count']} ({stats2['nan_percent']:.2f}%)") + logger.info(f" Zeros: {stats2['zero_count']} ({stats2['zero_percent']:.2f}%)") + logger.info(f" Valid: {stats2['valid_count']} ({stats2['valid_percent']:.2f}%)") + if stats2['valid_count'] > 0: + desc_stats = stats2["descriptive_stats"] + logger.info( + f" Valid Data Stats: Mean={desc_stats['mean']:.4f}, Median={desc_stats['median']:.4f}, Min={desc_stats['min']:.4f}, Max={desc_stats['max']:.4f}, Std Dev={desc_stats['std']:.4f}") + else: + logger.info(" Matrix not found.") + + # --- Feasibility and Segmented Average Analysis --- + # Only perform this for measures other than TRIPS/FAILURES and if a path and period were parsed + if path and period and measure not in ["TRIPS", "FAILURES"]: + # Determine potential feasibility matrices + feasibility_measures = ["TOTIVT", "IVT", "TIME", "DIST"] # Prioritize time/dist measures + feasibility_matrix_name = None + + for fm in feasibility_measures: + potential_name = f"{path}_{fm}__{period}" + if potential_name in all_tables: + feasibility_matrix_name = potential_name + break # Found a feasibility matrix + + if feasibility_matrix_name: + logger.info(f" Using '{feasibility_matrix_name}' to determine feasible ODs.") + + feasibility_data1 = None + feasibility_data2 = None + mask1 = None + mask2 = None + + # Load feasibility data + if feasibility_matrix_name in tables1: + try: + feasibility_data1 = np.array(skims1[feasibility_matrix_name]) + except Exception as e: + logger.error( + f"Error reading feasibility matrix {feasibility_matrix_name} from {path1}: {e}") + + if feasibility_matrix_name in tables2: + try: + feasibility_data2 = np.array(skims2[feasibility_matrix_name]) + except Exception as e: + logger.error( + f"Error reading feasibility matrix {feasibility_matrix_name} from {path2}: {e}") + + # Create feasibility masks (where value > 0) + shape_match = True + if feasibility_data1 is not None and feasibility_data2 is not None: + if feasibility_data1.shape != feasibility_data2.shape: + logger.warning( + f" Shape mismatch for feasibility matrix {feasibility_matrix_name}: {feasibility_data1.shape} vs {feasibility_data2.shape}. Skipping feasibility analysis.") + shape_match = False + else: + # Ensure data is not NaN before checking > 0 + mask1 = (~np.isnan(feasibility_data1)) & (feasibility_data1 > 0) + mask2 = (~np.isnan(feasibility_data2)) & (feasibility_data2 > 0) + elif feasibility_data1 is not None: + logger.warning( + f" Feasibility matrix {feasibility_matrix_name} only found in {os.path.basename(path1)}. Cannot compare feasibility counts or segmented averages.") + shape_match = False # Cannot do full comparison + elif feasibility_data2 is not None: + logger.warning( + f" Feasibility matrix {feasibility_matrix_name} only found in {os.path.basename(path2)}. Cannot compare feasibility counts or segmented averages.") + shape_match = False # Cannot do full comparison + else: + logger.debug(f" Feasibility matrix {feasibility_matrix_name} not found in either file.") + shape_match = False # Cannot do full comparison + + if mask1 is not None and mask2 is not None and shape_match: + feasible_count1 = np.sum(mask1) + feasible_count2 = np.sum(mask2) + feasible_diff = feasible_count2 - feasible_count1 + + logger.info(f" Feasible ODs (where {feasibility_matrix_name} > 0):") + logger.info(f" File 1: {feasible_count1}") + logger.info(f" File 2: {feasible_count2}") + logger.info(f" Difference (File 2 - File 1): {feasible_diff}") + + if feasible_count1 > 0 or feasible_count2 > 0: + # Define OD categories based on feasibility + mask_both_feasible = mask1 & mask2 + mask_file1_only = mask1 & ~mask2 + mask_file2_only = ~mask1 & mask2 + + both_count = np.sum(mask_both_feasible) + f1_only_count = np.sum(mask_file1_only) + f2_only_count = np.sum(mask_file2_only) + + logger.info(f" OD Category Counts (based on {feasibility_matrix_name} > 0):") + logger.info(f" Feasible in both: {both_count}") + logger.info(f" Feasible in File 1 only: {f1_only_count}") + logger.info(f" Feasible in File 2 only: {f2_only_count}") + + # Calculate and report segmented averages for the CURRENT matrix (table_name) + if stats1["exists"] and stats2[ + "exists"] and "error" not in stats1 and "error" not in stats2: + if stats1["shape"] == stats2["shape"]: + logger.info(f" Segmented Averages for {table_name}:") + + def safe_mean(arr, mask): + # Calculate mean only if mask is not empty and filtered data is not all NaN + filtered_data = arr[mask] + if filtered_data.size > 0 and not np.all(np.isnan(filtered_data)): + return np.nanmean( + filtered_data) # Use nanmean to ignore NaNs in selected slice + return np.nan # Return NaN if no data or all NaN + + # Both feasible + mean1_both = safe_mean(data1, mask_both_feasible) + mean2_both = safe_mean(data2, mask_both_feasible) + logger.info( + f" Feasible in both ({both_count} ODs): File 1 Avg={mean1_both:.4f}, File 2 Avg={mean2_both:.4f}") + + # File 1 only feasible + mean1_f1only = safe_mean(data1, mask_file1_only) + mean2_f1only = safe_mean(data2, + mask_file1_only) # Value in file 2 should ideally be 0 or NaN if not feasible + logger.info( + f" Feasible in File 1 only ({f1_only_count} ODs): File 1 Avg={mean1_f1only:.4f}, File 2 Avg={mean2_f1only:.4f}") + + # File 2 only feasible + mean1_f2only = safe_mean(data1, + mask_file2_only) # Value in file 1 should ideally be 0 or NaN if not feasible + mean2_f2only = safe_mean(data2, mask_file2_only) + logger.info( + f" Feasible in File 2 only ({f2_only_count} ODs): File 1 Avg={mean1_f2only:.4f}, File 2 Avg={mean2_f2only:.4f}") + + else: + logger.warning( + f" Shape mismatch for {table_name} ({stats1['shape']} vs {stats2['shape']}). Cannot calculate segmented averages.") + else: + logger.debug( + f" Skipping segmented average calculation for {table_name}: matrix not found or error occurred in one file.") + # Else: Feasibility matrix existed but had errors or shape mismatch, detailed analysis skipped. + else: + logger.debug( + f" No common feasibility matrix found for {path}_{measure}__{period}. Skipping segmented analysis.") + # Else: Basic statistics already reported. + + # --- Overall Comparison (Keep existing logic, perhaps refine warnings based on detailed analysis) --- + # This section can remain largely the same, providing high-level comparison + # based on basic stats (NaN, Zero, Valid counts, overall mean/median/etc. of Valid data). + # If a detailed segmented analysis was performed, the warnings here about NaN/Zero/Valid counts + # might be less critical than the findings in the segmented analysis, but they are still useful. + if stats1["exists"] and stats2["exists"] and "error" not in stats1 and "error" not in stats2: + if stats1["shape"] != stats2["shape"]: + pass # Shape mismatch already reported above + elif stats1["total_elements"] == 0: + pass # Empty matrices already reported above + else: + # Compare missing data percentages (NaNs, Zeros) + nan_diff = abs(stats1['nan_percent'] - stats2['nan_percent']) + zero_diff = abs(stats1['zero_percent'] - stats2['zero_percent']) + + if nan_diff > 1.0: # Threshold for warning + logger.warning( + f" Overall Significant difference in NaN percentage: {stats1['nan_percent']:.2f}% vs {stats2['nan_percent']:.2f}% (Diff: {nan_diff:.2f}%)") + comparison_summary.append(f"{table_name}: Overall NaN Diff={nan_diff:.2f}%") + + if zero_diff > 1.0: # Threshold for warning + logger.warning( + f" Overall Significant difference in Zero percentage: {stats1['zero_percent']:.2f}% vs {stats2['zero_percent']:.2f}% (Diff: {zero_diff:.2f}%)") + comparison_summary.append(f"{table_name}: Overall Zero Diff={zero_diff:.2f}%") + + # Compare descriptive statistics of *all* valid data (non-NaN, non-zero) + if stats1['valid_count'] > 0 and stats2['valid_count'] > 0: + desc1 = stats1["descriptive_stats"] + desc2 = stats2["descriptive_stats"] + comp_notes = [] + + for stat_name in ["mean", "median", "min", "max", "std"]: + val1 = desc1[stat_name] + val2 = desc2[stat_name] + + if abs(val1) < 1e-9 and abs(val2) < 1e-9: + ratio = 1.0 + elif abs(val1) < 1e-9: + ratio = 0.0 + elif abs(val2) < 1e-9: + ratio = float('inf') + else: + ratio = val2 / val1 + + # Check for significant ratio difference (e.g., outside 0.5x to 2x) + # or large absolute difference for small values + significant_ratio_diff = ratio < 0.5 or ratio > 2.0 + significant_abs_diff = abs(val1 - val2) > 0.01 and ( + abs(val1) > 1e-9 or abs(val2) > 1e-9) # Reduced threshold for absolute diff + + if significant_ratio_diff or significant_abs_diff: + warning_msg = f" Overall Significant difference in {stat_name}: {val1:.4f} vs {val2:.4f}" + if ratio != 0.0 and ratio != float('inf'): + warning_msg += f" (Ratio: {ratio:.2f})" + logger.warning(warning_msg) + comp_notes.append(f"Overall {stat_name} Ratio={ratio:.2f}") + + if comp_notes: + comparison_summary.append(f"{table_name}: {', '.join(comp_notes)}") + + elif stats1['valid_count'] == 0 and stats2['valid_count'] > 0: + logger.warning( + f" Overall Valid data exists in File 2 but not in File 1 ({stats1['valid_count']} vs {stats2['valid_count']}).") + comparison_summary.append(f"{table_name}: Overall NO VALID DATA IN FILE 1") + elif stats1['valid_count'] > 0 and stats2['valid_count'] == 0: + logger.warning( + f" Overall Valid data exists in File 1 but not in File 2 ({stats1['valid_count']} vs {stats2['valid_count']}).") + comparison_summary.append(f"{table_name}: Overall NO VALID DATA IN FILE 2") + elif stats1['valid_count'] == 0 and stats2['valid_count'] == 0: + pass # Both have no valid data, already noted. + + logger.info("\n" + "=" * 80) + logger.info("Comparison Summary:") + logger.info("-" * 80) + if comparison_summary: + for note in comparison_summary: + logger.info(note) + else: + logger.info("No significant differences found.") + logger.info("=" * 80) + + + except Exception as e: + logger.critical(f"An error occurred during skim comparison: {e}") + finally: + if skims1: + try: + skims1.close() + except Exception as e: + logger.error(f"Error closing {path1}: {e}") + if skims2: + try: + skims2.close() + except Exception as e: + logger.error(f"Error closing {path2}: {e}") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="Compare statistics of matrices in two OMX skim files.") + parser.add_argument( + "path1", + help="Path to the first OMX skim file.") + parser.add_argument( + "path2", + help="Path to the second OMX skim file.") + + args = parser.parse_args() + + compare_skims(args.path1, args.path2) \ No newline at end of file diff --git a/copy-seattle-skims.py b/copy-seattle-skims.py new file mode 100644 index 00000000..f3621a2a --- /dev/null +++ b/copy-seattle-skims.py @@ -0,0 +1,310 @@ +import openmatrix as omx +import os +import logging +import sys + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + stream=sys.stdout # Log to console +) +logger = logging.getLogger(__name__) + +# --- File Paths --- +file1_path = "pilates/beam/production/seattle/as-base-skims-seattle-bg.omx" +file2_path = "pilates/beam/production/seattle/as-base-skims-seattle-bg-new.omx" + +# --- Skims to Copy --- +# This list is generated *strictly* based on the analysis of +# compare_skims.log, excluding those with _EXP_ or _HVY_ +# as per the user's criteria for the target file. +skims_to_copy = [ + "BIKE_TRIPS", + "DISTBIKE", + "DISTWALK", + "HOV2TOLL_BTOLL__AM", + "HOV2TOLL_BTOLL__EA", + "HOV2TOLL_BTOLL__EV", + "HOV2TOLL_BTOLL__MD", + "HOV2TOLL_BTOLL__PM", + "HOV2TOLL_DIST__AM", + "HOV2TOLL_DIST__EA", + "HOV2TOLL_DIST__EV", + "HOV2TOLL_DIST__MD", + "HOV2TOLL_DIST__PM", + "HOV2TOLL_TIME__AM", + "HOV2TOLL_TIME__EA", + "HOV2TOLL_TIME__EV", + "HOV2TOLL_TIME__MD", + "HOV2TOLL_TIME__PM", + "HOV2TOLL_VTOLL__AM", + "HOV2TOLL_VTOLL__EA", + "HOV2TOLL_VTOLL__EV", + "HOV2TOLL_VTOLL__MD", + "HOV2TOLL_VTOLL__PM", + "HOV2_BTOLL__AM", + "HOV2_BTOLL__EA", + "HOV2_BTOLL__EV", + "HOV2_BTOLL__MD", + "HOV2_BTOLL__PM", + "HOV2_DIST__AM", + "HOV2_DIST__EA", + "HOV2_DIST__EV", + "HOV2_DIST__MD", + "HOV2_DIST__PM", + "HOV2_TIME__AM", + "HOV2_TIME__EA", + "HOV2_TIME__EV", + "HOV2_TIME__MD", + "HOV2_TIME__PM", + "HOV2_VTOLL__AM", + "HOV2_VTOLL__EA", + "HOV2_VTOLL__EV", + "HOV2_VTOLL__MD", + "HOV2_VTOLL__PM", + "HOV3TOLL_BTOLL__AM", + "HOV3TOLL_BTOLL__EA", + "HOV3TOLL_BTOLL__EV", + "HOV3TOLL_BTOLL__MD", + "HOV3TOLL_BTOLL__PM", + "HOV3TOLL_DIST__AM", + "HOV3TOLL_DIST__EA", + "HOV3TOLL_DIST__EV", + "HOV3TOLL_DIST__MD", + "HOV3TOLL_DIST__PM", + "HOV3TOLL_VTOLL__AM", + "HOV3TOLL_VTOLL__EA", + "HOV3TOLL_VTOLL__EV", + "HOV3TOLL_VTOLL__MD", + "HOV3TOLL_VTOLL__PM", + "HOV3_TIME__AM", + "HOV3_TIME__EA", + "HOV3_TIME__EV", + "HOV3_TIME__MD", + "HOV3_TIME__PM", + "HOV3_VTOLL__AM", + "HOV3_VTOLL__EA", + "HOV3_VTOLL__EV", + "HOV3_VTOLL__MD", + "HOV3_VTOLL__PM", + "RH_POOLED_REJECTIONPROB__AM", + "RH_POOLED_REJECTIONPROB__EA", + "RH_POOLED_REJECTIONPROB__EV", + "RH_POOLED_REJECTIONPROB__MD", + "RH_POOLED_REJECTIONPROB__PM", + "RH_POOLED_TRIPS__AM", + "RH_POOLED_TRIPS__EA", + "RH_POOLED_TRIPS__EV", + "RH_POOLED_TRIPS__MD", + "RH_POOLED_TRIPS__PM", + "RH_POOLED_WAIT__AM", + "RH_POOLED_WAIT__EA", + "RH_POOLED_WAIT__EV", + "RH_POOLED_WAIT__MD", + "RH_POOLED_WAIT__PM", + "RH_SOLO_REJECTIONPROB__AM", + "RH_SOLO_REJECTIONPROB__EA", + "RH_SOLO_REJECTIONPROB__EV", + "RH_SOLO_REJECTIONPROB__MD", + "RH_SOLO_REJECTIONPROB__PM", + "RH_SOLO_TRIPS__AM", + "RH_SOLO_TRIPS__EA", + "RH_SOLO_TRIPS__EV", + "RH_SOLO_TRIPS__MD", + "RH_SOLO_TRIPS__PM", + "RH_SOLO_WAIT__AM", + "RH_SOLO_WAIT__EA", + "RH_SOLO_WAIT__EV", + "RH_SOLO_WAIT__MD", + "RH_SOLO_WAIT__PM", + "SOVTOLL_BTOLL__AM", + "SOVTOLL_BTOLL__EA", + "SOVTOLL_BTOLL__EV", + "SOVTOLL_BTOLL__MD", + "SOVTOLL_BTOLL__PM", + "SOVTOLL_DIST__AM", + "SOVTOLL_DIST__EA", + "SOVTOLL_DIST__EV", + "SOVTOLL_DIST__MD", + "SOVTOLL_DIST__PM", + "SOVTOLL_TIME__AM", + "SOVTOLL_TIME__EA", + "SOVTOLL_TIME__EV", + "SOVTOLL_TIME__MD", + "SOVTOLL_TIME__PM", + "SOVTOLL_VTOLL__AM", + "SOVTOLL_VTOLL__EA", + "SOVTOLL_VTOLL__EV", + "SOVTOLL_VTOLL__MD", + "SOVTOLL_VTOLL__PM", + "SOV_BTOLL__AM", + "SOV_BTOLL__EA", + "SOV_BTOLL__EV", + "SOV_BTOLL__MD", + "SOV_BTOLL__PM", + "SOV_DIST__AM", + "SOV_DIST__EA", + "SOV_DIST__EV", + "SOV_DIST__MD", + "SOV_DIST__PM", + "SOV_TIME__AM", + "SOV_TIME__EA", + "SOV_TIME__EV", + "SOV_TIME__MD", + "SOV_TIME__PM", + "SOV_TRIPS__AM", + "SOV_TRIPS__EA", + "SOV_TRIPS__EV", + "SOV_TRIPS__MD", + "SOV_TRIPS__PM", + "SOV_VTOLL__AM", + "SOV_VTOLL__EA", + "SOV_VTOLL__EV", + "SOV_VTOLL__MD", + "SOV_VTOLL__PM", + "WALK_TRIPS", + "WLK_TRN_WLK_BOARDS__AM", + "WLK_TRN_WLK_BOARDS__EA", + "WLK_TRN_WLK_BOARDS__EV", + "WLK_TRN_WLK_BOARDS__MD", + "WLK_TRN_WLK_BOARDS__PM", + "WLK_TRN_WLK_FAILURES__AM", + "WLK_TRN_WLK_FAILURES__EA", + "WLK_TRN_WLK_FAILURES__EV", + "WLK_TRN_WLK_FAILURES__MD", + "WLK_TRN_WLK_FAILURES__PM", + "WLK_TRN_WLK_FAR__AM", + "WLK_TRN_WLK_FAR__EA", + "WLK_TRN_WLK_FAR__EV", + "WLK_TRN_WLK_FAR__MD", + "WLK_TRN_WLK_FAR__PM", + "WLK_TRN_WLK_IVT__AM", + "WLK_TRN_WLK_IVT__EA", + "WLK_TRN_WLK_IVT__EV", + "WLK_TRN_WLK_IVT__MD", + "WLK_TRN_WLK_IVT__PM", + "WLK_TRN_WLK_IWAIT__AM", + "WLK_TRN_WLK_IWAIT__EA", + "WLK_TRN_WLK_IWAIT__EV", + "WLK_TRN_WLK_IWAIT__MD", + "WLK_TRN_WLK_IWAIT__PM", + "WLK_TRN_WLK_KEYIVT__AM", + "WLK_TRN_WLK_KEYIVT__EA", + "WLK_TRN_WLK_KEYIVT__EV", + "WLK_TRN_WLK_KEYIVT__MD", + "WLK_TRN_WLK_KEYIVT__PM", + "WLK_TRN_WLK_TRIPS__AM", + "WLK_TRN_WLK_TRIPS__EA", + "WLK_TRN_WLK_TRIPS__EV", + "WLK_TRN_WLK_TRIPS__MD", + "WLK_TRN_WLK_TRIPS__PM", + "WLK_TRN_WLK_WACC__AM", + "WLK_TRN_WLK_WACC__EA", + "WLK_TRN_WLK_WACC__EV", + "WLK_TRN_WLK_WACC__MD", + "WLK_TRN_WLK_WACC__PM", + "WLK_TRN_WLK_WAUX__AM", + "WLK_TRN_WLK_WAUX__EA", + "WLK_TRN_WLK_WAUX__EV", + "WLK_TRN_WLK_WAUX__MD", + "WLK_TRN_WLK_WAUX__PM", + "WLK_TRN_WLK_WEGR__AM", + "WLK_TRN_WLK_WEGR__EA", + "WLK_TRN_WLK_WEGR__EV", + "WLK_TRN_WLK_WEGR__MD", + "WLK_TRN_WLK_WEGR__PM", + "WLK_TRN_WLK_XWAIT__AM", + "WLK_TRN_WLK_XWAIT__EA", + "WLK_TRN_WLK_XWAIT__EV", + "WLK_TRN_WLK_XWAIT__MD", + "WLK_TRN_WLK_XWAIT__PM", +] + +def copy_skims(src_path, dest_path, skims_list): + """ + Copies a specified list of skims from a source OMX file to a destination OMX file, + overwriting existing skims in the destination. + + Parameters + ---------- + src_path : str + Path to the source OMX file. + dest_path : str + Path to the destination OMX file. + skims_list : list + A list of skim names (strings) to copy. + """ + if not os.path.exists(src_path): + logger.critical(f"Source file not found: {src_path}") + return + if not os.path.exists(dest_path): + logger.critical(f"Destination file not found: {dest_path}") + return # Or create it, depending on desired behavior. Sticking to copy to existing here. + + skims_src = None + skims_dest = None + + try: + logger.info(f"Opening source file: {src_path}") + skims_src = omx.open_file(src_path, 'r') + + logger.info(f"Opening destination file for append/write: {dest_path}") + # Open in append mode to be able to write/overwrite + skims_dest = omx.open_file(dest_path, 'a') + + logger.info(f"Attempting to copy {len(skims_list)} specified skims.") + + for skim_name in skims_list: + if skim_name in skims_src.list_matrices(): + logger.info(f"Processing skim: '{skim_name}'") + try: + # If the skim exists in the destination, delete it first + if skim_name in skims_dest.list_matrices(): + logger.info(f" Removing existing skim '{skim_name}' from destination.") + del skims_dest[skim_name] + + # Read data from source file + data_to_copy = skims_src[skim_name][:] # Use [:] to get the actual numpy array + + # Write data to destination file + skims_dest[skim_name] = data_to_copy + + logger.info(f" Successfully copied '{skim_name}'.") + + except Exception as e: + logger.error(f" Error copying skim '{skim_name}': {e}") + else: + logger.warning(f" Skim '{skim_name}' not found in source file '{src_path}'. Skipping.") + + logger.info("\n--- Copy Operation Summary ---") + logger.info(f"Attempted to copy {len(skims_list)} skims.") + # You could add counters here for success/failure if desired + logger.info("----------------------------") + + + except FileNotFoundError as e: + logger.critical(f"File not found error: {e}") + except Exception as e: + logger.critical(f"An unexpected error occurred: {e}") + finally: + if skims_src: + try: + skims_src.close() + logger.info(f"Closed source file: {src_path}") + except Exception as e: + logger.error(f"Error closing source file {src_path}: {e}") + if skims_dest: + try: + skims_dest.close() + logger.info(f"Closed destination file: {dest_path}") + except Exception as e: + logger.error(f"Error closing destination file {dest_path}: {e}") + + +if __name__ == '__main__': + # Note: The file paths are hardcoded at the top of the script, + # making this a one-time script tailored to the log analysis. + # You will need to manually run this script. + logger.info("Starting skim copying script.") + copy_skims(file1_path, file2_path, skims_to_copy) + logger.info("Script finished.") \ No newline at end of file diff --git a/environment.yml b/environment.yml index a2e08b9d..e092e192 100644 --- a/environment.yml +++ b/environment.yml @@ -194,6 +194,7 @@ dependencies: - xorg-xproto=7.0.31=h7f98852_1007 - xz=5.2.5=h516909a_1 - yaml=0.2.5=h516909a_0 + - zarr=3.0.6 - zlib=1.2.11=h36c2ea0_1013 - zstd=1.5.0=ha95c52a_0 - pip: diff --git a/hpc/job.sh b/hpc/job.sh index 91c2cb20..40c7b805 100755 --- a/hpc/job.sh +++ b/hpc/job.sh @@ -1,9 +1,27 @@ #!/bin/bash -module load python/3.8.8 -#pip install --user openmatrix -#pip install --user geopandas +module load python/3.10.12 +#python -m pip uninstall --user shapely +#python -m pip install --user shapely +#python -m pip install --user openmatrix +#python -m pip install --user pygeos +#python -m pip install --user geopandas +#python -m pip install --user table +#python -m pip install --user PyYAML export PYTHONPATH=`python -m site --user-site`:$PYTHONPATH cd /global/scratch/users/$USER/sources/PILATES -python run.py -v +echo "$1" + +echo "=== MEMORY INFORMATION ===" +free -h +grep MemTotal /proc/meminfo +grep -i numa /proc/cpuinfo +echo "==========================" + +echo "=== NODE USAGE INFORMATION ===" +squeue -o "%.18i %.9P %.8j %.8u %.8T %.10M %.9l %.6D %R" | grep $(hostname) +echo "==========================" + + +python run.py -c "$1" -S "$2" diff --git a/hpc/job_runner.sh b/hpc/job_runner.sh index d0bc71c7..e550969f 100755 --- a/hpc/job_runner.sh +++ b/hpc/job_runner.sh @@ -4,13 +4,28 @@ RANDOM_PART="$(tr -dc A-Z0-9 [!TIP] +> It's useful to set up a symlink from the home directory (where you're logged in) to the scratch directory (where we want to store big files like the PILATES data) +> ```commandline +> ln -s /global/scratch/users/[username] ./scratch +> ``` + ## Installing PILATES ```commandline cd /global/scratch/users/$USER mkdir sources cd sources -git clone --branch atlas-v2 https://github.com/LBNL-UCB-STI/PILATES.git +git clone --branch copy-working-directory https://github.com/LBNL-UCB-STI/PILATES.git cd PILATES ``` ## Setup Python ```commandline -module load python/3.8.8 -pip install --user openmatrix -pip install --user geopandas +module load python/3.10.12 +python -m pip uninstall --user shapely +python -m pip install --user shapely +python -m pip install --user openmatrix +python -m pip install --user pygeos +python -m pip install --user geopandas +python -m pip install --user table +python -m pip install --user PyYAML export PYTHONPATH=`python -m site --user-site`:$PYTHONPATH ``` @@ -85,4 +108,7 @@ cd ../../../ cd hpc ./job_runner.sh ``` +#### Optional tags: +`-c`: Define a new settings file, defaults to `settings.yaml` +`-s`: Current stage file name of run to restart in the middle of. If none, starts a new run with a new current stage file. Defaults to none. diff --git a/pilates/activitysim/postprocessor.py b/pilates/activitysim/postprocessor.py index 0f141184..b0999307 100644 --- a/pilates/activitysim/postprocessor.py +++ b/pilates/activitysim/postprocessor.py @@ -4,19 +4,19 @@ import os from pilates.utils.io import read_datastore +from workflow_state import WorkflowState logger = logging.getLogger(__name__) -def _load_asim_outputs(settings): +def _load_asim_outputs(settings, output_path): output_tables_settings = settings['asim_output_tables'] prefix = output_tables_settings['prefix'] output_tables = output_tables_settings['tables'] asim_output_dict = {} for table_name in output_tables: file_name = "%s%s.csv" % (prefix, table_name) - file_path = os.path.join( - settings['asim_local_output_folder'], file_name) + file_path = os.path.join(output_path, settings['asim_local_output_folder'], file_name) if table_name == 'persons': index_col = 'person_id' elif table_name == 'households': @@ -50,18 +50,18 @@ def get_usim_datastore_fname(settings, io, year=None): def _prepare_updated_tables( - settings, forecast_year, asim_output_dict, tables_updated_by_asim, + settings, state: WorkflowState, asim_output_dict, tables_updated_by_asim, prefix=None): """ Combines ActivitySim and UrbanSim outputs for tables updated by ActivitySim (e.g. households and persons) """ - data_dir = settings['usim_local_data_folder'] + data_dir = os.path.join(state.full_path, settings['usim_local_mutable_data_folder']) # e.g. model_data_2012.h5 usim_output_store_name = get_usim_datastore_fname( - settings, io='output', year=forecast_year) + settings, io='output', year=state.forecast_year) usim_output_store_path = os.path.join(data_dir, usim_output_store_name) if not os.path.exists(usim_output_store_path): raise ValueError('No output data store found at {0}'.format( @@ -172,7 +172,7 @@ def create_beam_input_data(settings, forecast_year, asim_output_dict): def create_usim_input_data( settings, input_year, forecast_year, asim_output_dict, - tables_updated_by_asim): + tables_updated_by_asim, full_path=None): """ Creates UrbanSim input data for the next iteration. @@ -186,7 +186,9 @@ def create_usim_input_data( """ # parse settings - data_dir = settings['usim_local_data_folder'] + data_dir = settings['usim_local_mutable_data_folder'] + if full_path is not None: + data_dir = os.path.join(full_path, data_dir) # Move UrbanSim input store (e.g. custom_mpo_193482435_model_data.h5) # to archive (e.g. input_data_for_2015_outputs.h5) because otherwise @@ -215,7 +217,7 @@ def create_usim_input_data( if not os.path.exists(usim_output_store_path): raise ValueError('No output data found at {0}.'.format( usim_output_store_path)) - usim_output_store, table_prefix_year = read_datastore(settings, forecast_year) + usim_output_store, table_prefix_year = read_datastore(settings, forecast_year, mutable_data_dir=full_path) logger.info( 'Merging results back into UrbanSim format and storing as .h5!') @@ -270,18 +272,19 @@ def create_usim_input_data( return -def create_next_iter_inputs(settings, year, forecast_year): +def create_next_iter_inputs(settings, year, state: WorkflowState): + forecast_year = state.forecast_year tables_updated_by_asim = ['households', 'persons'] - asim_output_dict = _load_asim_outputs(settings) + asim_output_dict = _load_asim_outputs(settings, state.full_path) asim_output_dict = _prepare_updated_tables( - settings, forecast_year, asim_output_dict, tables_updated_by_asim, + settings, state, asim_output_dict, tables_updated_by_asim, prefix=forecast_year) # if settings['traffic_assignment_enabled']: # create_beam_input_data(settings, forecast_year, asim_output_dict) create_usim_input_data( settings, year, forecast_year, asim_output_dict, - tables_updated_by_asim) + tables_updated_by_asim, state.full_path) return @@ -294,7 +297,7 @@ def update_usim_inputs_after_warm_start( # load usim data if not usim_data_dir: - usim_data_dir = settings['usim_local_data_folder'] + usim_data_dir = settings['usim_local_mutable_data_folder'] datastore_name = get_usim_datastore_fname(settings, io='input') input_store_path = os.path.join(usim_data_dir, datastore_name) if not os.path.exists(input_store_path): diff --git a/pilates/activitysim/preprocessor.py b/pilates/activitysim/preprocessor.py index a9b40e2a..b845dfd1 100644 --- a/pilates/activitysim/preprocessor.py +++ b/pilates/activitysim/preprocessor.py @@ -106,7 +106,7 @@ def read_skims(settings, mode='a', data_dir=None, file_name='skims.omx'): Ignored in read-only mode. """ if data_dir is None: - data_dir = settings['asim_local_input_folder'] + data_dir = settings['asim_local_mutable_data_folder'] path = os.path.join(data_dir, file_name) skims = omx.open_file(path, mode=mode) return skims @@ -195,23 +195,24 @@ def read_zone_geoms(settings, year, asim_param_map = {'random_seed': 'rng_base_seed'} -def update_asim_config(settings, param, valueOverride=None): +def update_asim_config(settings, full_path, param, valueOverride=None): config_header = asim_param_map[param] if valueOverride is None: config_value = settings[param] else: config_value = valueOverride - asim_config_path = os.path.join( - settings['asim_local_configs_folder'], - settings['region'], - 'settings.yaml') + + path_list = [full_path, settings['asim_local_mutable_configs_folder'], + settings.get('asim_main_configs_dir', "configs"), 'settings.yaml'] + + asim_config_path = os.path.join(*path_list) modified = False with open(asim_config_path, 'r') as file: data = file.readlines() with open(asim_config_path, 'w') as file: for line in data: if config_header in line: - if ~modified: + if not modified: indent = line.split(config_header)[0] file.writelines(indent + config_header + ": " + str(config_value) + "\n") modified = True @@ -301,14 +302,14 @@ def _create_skim_object(settings, overwrite=True, output_dir=None): """ if output_dir is None: - output_dir = settings['asim_local_input_folder'] + output_dir = settings['asim_local_mutable_data_folder'] skims_path = os.path.join(output_dir, 'skims.omx') final_skims_exist = os.path.exists(skims_path) skims_fname = settings.get('skims_fname', False) omx_skim_output = skims_fname.endswith('.omx') beam_output_dir = settings['beam_local_output_folder'] - mutable_skims_location = os.path.join(beam_output_dir, skims_fname) + mutable_skims_location = os.path.join(output_dir, "skims.omx") mutable_skims_exist = os.path.exists(mutable_skims_location) should_use_csv_input_skims = mutable_skims_exist & (not omx_skim_output) @@ -323,7 +324,7 @@ def _create_skim_object(settings, overwrite=True, output_dir=None): logger.info("Found existing skims, no need to re-create.") return False, False, False - if (~final_skims_exist | overwrite) & omx_skim_output: + if ((not final_skims_exist) or overwrite) and omx_skim_output: if mutable_skims_exist: return True, should_use_csv_input_skims, False else: @@ -567,7 +568,7 @@ def impute_distances(zones, origin=None, destination=None): return orig.distance(dest).replace({0: 100}).values * (0.621371 / 1000) -def _distance_skims(settings, year, input_skims, order, data_dir=None): +def _distance_skims(settings, year, input_skims, order, data_dir): """ Generates distance matrices for drive, walk and bike modes. Parameters: @@ -580,9 +581,9 @@ def _distance_skims(settings, year, input_skims, order, data_dir=None): """ logger.info("Creating distance skims.") - skims_fname = settings.get('skims_fname', False) - beam_output_dir = settings['beam_local_output_folder'] - mutable_skims_location = os.path.join(beam_output_dir, skims_fname) + skims_fname = 'skims.omx' + # beam_output_dir = settings['beam_local_output_folder'] + mutable_skims_location = os.path.join(data_dir, skims_fname) needToClose = True if input_skims is not None: output_skims = input_skims @@ -750,7 +751,7 @@ def _get_field_or_else_empty(skims: Optional[omx.File], field: str, num_taz: int return np.full((num_taz, num_taz), np.nan, dtype=np.float32), True -def _fill_ridehail_skims(settings, input_skims, order, data_dir=None): +def _fill_ridehail_skims(settings, input_skims, order, data_dir): logger.info("Merging ridehail omx skims.") ridehail_path_map = settings['ridehail_path_map'] @@ -759,9 +760,9 @@ def _fill_ridehail_skims(settings, input_skims, order, data_dir=None): num_taz = len(order) - skims_fname = settings.get('skims_fname', False) - beam_output_dir = settings['beam_local_output_folder'] - mutable_skims_location = os.path.join(beam_output_dir, skims_fname) + skims_fname = 'skims.omx' + # beam_output_dir = settings['beam_local_output_folder'] + mutable_skims_location = os.path.join(data_dir, skims_fname) needToClose = True if input_skims is not None: output_skims = input_skims @@ -814,7 +815,7 @@ def _fill_ridehail_skims(settings, input_skims, order, data_dir=None): output_skims.close() -def _fill_transit_skims(settings, input_skims, order, data_dir=None): +def _fill_transit_skims(settings, input_skims, order, data_dir): logger.info("Merging transit omx skims.") transit_paths = settings['transit_paths'] @@ -823,9 +824,9 @@ def _fill_transit_skims(settings, input_skims, order, data_dir=None): num_taz = len(order) - skims_fname = settings.get('skims_fname', False) - beam_output_dir = settings['beam_local_output_folder'] - mutable_skims_location = os.path.join(beam_output_dir, skims_fname) + skims_fname = "skims.omx" + # beam_output_dir = settings['beam_local_output_folder'] + mutable_skims_location = os.path.join(data_dir, skims_fname) needToClose = True if input_skims is not None: output_skims = input_skims @@ -904,9 +905,9 @@ def _fill_auto_skims(settings, input_skims, order, data_dir=None): num_taz = len(order) - skims_fname = settings.get('skims_fname', False) - beam_output_dir = settings['beam_local_output_folder'] - mutable_skims_location = os.path.join(beam_output_dir, skims_fname) + skims_fname = 'skims.omx' + # beam_output_dir = settings['beam_local_output_folder'] + mutable_skims_location = os.path.join(data_dir, skims_fname) needToClose = True if input_skims is not None: output_skims = input_skims @@ -1044,11 +1045,11 @@ def _create_offset(settings, order, data_dir=None): skims.close() -def create_skims_from_beam(settings, year, +def create_skims_from_beam(settings, state: "WorkflowState", output_dir=None, overwrite=True): if not output_dir: - output_dir = settings['asim_local_input_folder'] + output_dir = os.path.join(state.full_path, settings['asim_local_mutable_data_folder']) # If running in static skims mode and ActivitySim skims already exist # there is no point in recreating them. @@ -1059,42 +1060,42 @@ def create_skims_from_beam(settings, year, new, convertFromCsv, blankSkims = _create_skim_object(settings, overwrite, output_dir=output_dir) validation = settings.get('asim_validation', False) - order = zone_order(settings, year) + order = zone_order(settings, state.forecast_year) if new: tempSkims = _load_raw_beam_skims(settings, convertFromCsv, blankSkims) if isinstance(tempSkims, pd.DataFrame): skims = tempSkims.loc[tempSkims.origin.isin(order) & tempSkims.destination.isin(order), :] - skims = _raw_beam_skims_preprocess(settings, year, skims) + skims = _raw_beam_skims_preprocess(settings, state.year, skims) auto_df, transit_df = _create_skims_by_mode(settings, skims) ridehail_df = _load_raw_beam_origin_skims(settings) - ridehail_df = _raw_beam_origin_skims_preprocess(settings, year, ridehail_df) + ridehail_df = _raw_beam_origin_skims_preprocess(settings, state.year, ridehail_df) # Create skims - _distance_skims(settings, year, auto_df, order, data_dir=output_dir) + _distance_skims(settings, state.year, auto_df, order, data_dir=output_dir) _auto_skims(settings, auto_df, order, data_dir=output_dir) _transit_skims(settings, transit_df, order, data_dir=output_dir) _ridehail_skims(settings, ridehail_df, order, data_dir=output_dir) del auto_df, transit_df else: - beam_output_dir = settings['beam_local_output_folder'] - _distance_skims(settings, year, tempSkims, order, data_dir=beam_output_dir) - _fill_auto_skims(settings, tempSkims, order, data_dir=beam_output_dir) - _fill_transit_skims(settings, tempSkims, order, data_dir=beam_output_dir) - _fill_ridehail_skims(settings, tempSkims, order, data_dir=beam_output_dir) + # beam_output_dir = settings['beam_local_output_folder'] + _distance_skims(settings, state.year, tempSkims, order, data_dir=output_dir) + _fill_auto_skims(settings, tempSkims, order, data_dir=output_dir) + _fill_transit_skims(settings, tempSkims, order, data_dir=output_dir) + _fill_ridehail_skims(settings, tempSkims, order, data_dir=output_dir) if isinstance(tempSkims, omx.File): tempSkims.close() - final_skims_path = os.path.join(settings['asim_local_input_folder'], 'skims.omx') - skims_fname = settings.get('skims_fname', False) - mutable_skims_location = os.path.join(beam_output_dir, skims_fname) - shutil.copyfile(mutable_skims_location, final_skims_path) + # final_skims_path = os.path.join(settings['asim_local_mutable_data_folder'], 'skims.omx') + # skims_fname = settings.get('skims_fname', False) + # mutable_skims_location = os.path.join(beam_output_dir, skims_fname) + # shutil.copyfile(mutable_skims_location, final_skims_path) _create_offset(settings, order, data_dir=output_dir) if validation: - order = zone_order(settings, year) - skim_validations(settings, year, order, data_dir=output_dir) + order = zone_order(settings, state.year) + skim_validations(settings, state.year, order, data_dir=output_dir) def plot_skims(settings, zones, @@ -1241,139 +1242,128 @@ def _get_part_time_enrollment(state_fips): def _update_persons_table(persons, households, unassigned_households, blocks, asim_zone_id_col='TAZ'): - # assign zones - persons.index = persons.index.astype(int) - unassigned_persons = persons.household_id.isin(unassigned_households) - logger.info("Dropping {0} people from {1} households that haven't been assigned locations yet".format( - unassigned_households.shape[0], unassigned_persons.sum())) - persons = persons.loc[~unassigned_persons, :] - persons.loc[:, asim_zone_id_col] = blocks[asim_zone_id_col].reindex( - households['block_id'].reindex(persons['household_id']).values).values - persons.loc[:, asim_zone_id_col] = persons.loc[:, asim_zone_id_col].astype(str) + """Updates person attributes and assigns zones for ActivitySim processing. - # create new column variables - age_mask_1 = persons.age >= 18 - age_mask_2 = persons.age.between(18, 64, inclusive=True) - age_mask_3 = persons.age >= 65 - work_mask = persons.worker == 1 - student_mask = persons.student == 1 - type_1 = ((age_mask_1) & (work_mask) & (~student_mask)) * 1 # Full time - type_4 = ((age_mask_2) & (~work_mask) & (~student_mask)) * 4 - type_5 = ((age_mask_3) & (~work_mask) & (~student_mask)) * 5 - type_3 = ((age_mask_1) & (student_mask)) * 3 - type_6 = (persons.age.between(16, 17, inclusive=True)) * 6 - type_7 = (persons.age.between(6, 16, inclusive=True)) * 7 - type_8 = (persons.age.between(0, 5, inclusive=True)) * 8 - type_list = [ - type_1, type_3, type_4, type_5, type_6, type_7, type_8] - for x in type_list: - type_1.where(type_1 != 0, x, inplace=True) - persons.loc[:, 'ptype'] = type_1 - - pemploy_1 = ((persons.worker == 1) & (persons.age >= 16)) * 1 - pemploy_3 = ((persons.worker == 0) & (persons.age >= 16)) * 3 - pemploy_4 = (persons.age < 16) * 4 - type_list = [pemploy_1, pemploy_3, pemploy_4] - for x in type_list: - pemploy_1.where(pemploy_1 != 0, x, inplace=True) - persons.loc[:, 'pemploy'] = pemploy_1 - - pstudent_1 = (persons.age <= 18) * 1 - pstudent_2 = ((persons.student == 1) & (persons.age > 18)) * 2 - pstudent_3 = (persons.student == 0) * 3 - type_list = [pstudent_1, pstudent_2, pstudent_3] - for x in type_list: - pstudent_1.where(pstudent_1 != 0, x, inplace=True) - persons.loc[:, 'pstudent'] = pstudent_1 - - persons_w_res_blk = pd.merge( - persons, households[['block_id']], - left_on='household_id', right_index=True) - persons_w_xy = pd.merge( - persons_w_res_blk, blocks[['x', 'y']], - left_on='block_id', right_index=True) - persons.loc[:, 'home_x'] = persons_w_xy['x'] - persons.loc[:, 'home_y'] = persons_w_xy['y'] - - del persons_w_res_blk - del persons_w_xy + Args: + persons: DataFrame containing person records + households: DataFrame containing household records + unassigned_households: Series of household IDs without locations + blocks: DataFrame containing block/zone information + asim_zone_id_col: Column name for zone ID (default: 'TAZ') - try: - persons.loc[:, "workplace_taz"] = pd.to_numeric(persons.loc[:, "work_zone_id"].copy(), errors='coerce').fillna( - -1) - except KeyError: - logger.info("Field `workplace_taz` not present in input h5 file. This may be a problem") - try: - persons.loc[:, "school_taz"] = pd.to_numeric(persons.loc[:, "school_zone_id"].copy(), errors='coerce').fillna( - -1) - except KeyError: - logger.info("Field `school_taz` not present in input h5 file. This may be a problem") - persons.loc[:, "worker"] = pd.to_numeric(persons.loc[:, "worker"].copy(), errors='coerce').fillna(0) - persons.loc[:, "student"] = pd.to_numeric(persons.loc[:, "student"].copy(), errors='coerce').fillna(0) - - # clean up dataframe structure - # TODO: move this to annotate_persons.yaml in asim settings - # p_names_dict = {'member_id': 'PNUM'} - # persons = persons.rename(columns=p_names_dict) - - p_null_taz = persons[asim_zone_id_col].isnull() - logger.info("Dropping {0} persons without TAZs".format( - p_null_taz.sum())) - p_newborn = persons['age'] < 1.0 - logger.info("Dropping {0} newborns from this iteration".format( - p_newborn.sum())) - if ("workplace_taz" in persons.columns) & ("school_taz" in persons.columns): - p_badwork = (persons.worker == 1) & ~(persons.workplace_taz >= 0) - p_badschool = (persons.student == 1) & ~(persons.school_taz >= 0) - logger.warn( - "Dropping {0} workers with undefined workplace and {1} students with undefined school".format( - p_badwork.sum(), - p_badschool.sum())) - persons = persons.loc[(~p_null_taz) & (~p_newborn) & (~p_badwork) & (~p_badschool), :] - persons = persons.dropna() - persons.loc[:, 'member_id'] = persons.groupby('household_id')['member_id'].apply(np.argsort) + 1 - - persons.loc[persons['ptype'] == 1, 'school_zone_id'] = -1 - - if ("workplace_taz" in persons.columns) & ("school_taz" in persons.columns): - # Workers with school ID - workers_with_school_id = persons[(persons.ptype == 1) & (persons.school_taz >= 0)].shape - - persons.loc[persons['ptype'] == 1, 'school_taz'] = -1 - workers_with_school_id_post = persons[(persons.ptype == 1) & (persons.school_taz >= 0)].shape - - logger.info(f"Workers with School location: {workers_with_school_id[0]}") - logger.info(f"Workers with School location after cleaning: {workers_with_school_id_post[0]}") - - # Make Sure non-workers and non-students dont't have a school location - non_work_school_with_school_id = persons[(persons.ptype.isin([4, 5])) & (persons.school_taz > 0)].shape - non_work_school_with_work_id = persons[(persons.ptype.isin([4, 5])) & (persons.workplace_taz > 0)].shape - - persons.loc[persons.ptype.isin([4, 5]), 'school_taz'] = -1 - persons.loc[persons.ptype.isin([4, 5]), 'workplace_taz'] = -1 - - non_work_school_with_school_id_post = persons[(persons.ptype.isin([4, 5])) & (persons.school_taz > 0)].shape - logger.info(f"Non-Workers and non-students with School location: {non_work_school_with_school_id[0]}") - logger.info( - f"Non-Workers and non-students with School location after cleaning: {non_work_school_with_school_id_post[0]}") - - non_work_school_with_work_id_post = persons[(persons.ptype.isin([4, 5])) & (persons.school_taz > 0)].shape - logger.info(f"Non-Workers and non-students with Work location: {non_work_school_with_work_id[0]}") - logger.info( - f"Non-Workers and non-students with Work location after cleaning: {non_work_school_with_work_id_post[0]}") - - persons.loc[persons.ptype.isin([4, 5]), 'school_zone_id'] = -1 - - # Make Sure non-workers and non-students dont't have a work location - - persons.loc[persons.ptype.isin([4, 5]), 'work_zone_id'] = -1 - - # assert (persons['work_zone_id'] == persons['workplace_taz']).all() - # assert (persons['school_zone_id'] == persons['school_taz']).all() + Returns: + DataFrame with updated person attributes + """ + # Convert index to int and filter out unassigned persons + persons.index = persons.index.astype(int) + unassigned_mask = persons.household_id.isin(unassigned_households) + logger.info(f"Dropping {unassigned_mask.sum()} people from {unassigned_households.shape[0]} unassigned households") + persons = persons.loc[~unassigned_mask] + + household_zone_map = households['block_id'].map(blocks[asim_zone_id_col]) + persons[asim_zone_id_col] = persons['household_id'].map(household_zone_map).astype(str) + + # Precalculate age ranges and worker/student status + persons['age'] = persons['age'].fillna(-1) + age_ranges = { + 'adult': persons.age >= 18, + 'working_age': persons.age.between(18, 64, inclusive="both"), + 'senior': persons.age >= 65, + 'teen': persons.age.between(16, 17, inclusive="both"), + 'child': persons.age.between(6, 15, inclusive="both"), + 'young_child': persons.age.between(0, 5, inclusive="both") + } + + # Calculate person types more efficiently + conditions = [ + (age_ranges['adult'] & (persons.worker == 1) & (persons.student != 1)), # type 1 + (age_ranges['adult'] & (persons.student == 1)), # type 3 + (age_ranges['working_age'] & (persons.worker != 1) & (persons.student != 1)), # type 4 + (age_ranges['senior'] & (persons.worker != 1) & (persons.student != 1)), # type 5 + age_ranges['teen'], # type 6 + age_ranges['child'], # type 7 + age_ranges['young_child'] # type 8 + ] + values = [1, 3, 4, 5, 6, 7, 8] + persons['ptype'] = np.select(conditions, values, default=0) + + # Calculate employment status + conditions = [ + ((persons.worker == 1) & (persons.age >= 16)), # type 1 + ((persons.worker == 0) & (persons.age >= 16)), # type 3 + (persons.age < 16) # type 4 + ] + values = [1, 3, 4] + persons['pemploy'] = np.select(conditions, values, default=0) + + # Calculate student status + conditions = [ + (persons.age <= 18), # type 1 + ((persons.student == 1) & (persons.age > 18)), # type 2 + (persons.student == 0) # type 3 + ] + values = [1, 2, 3] + persons['pstudent'] = np.select(conditions, values, default=0) + + # Add home coordinates efficiently + home_coords = (households[['block_id']] + .merge(blocks[['x', 'y']], left_on='block_id', right_index=True) + .set_index(households.index)) + persons['home_x'] = persons['household_id'].map(home_coords['x']) + persons['home_y'] = persons['household_id'].map(home_coords['y']) + + # Convert location fields + for field in ['workplace_taz', 'school_taz']: + try: + source_field = 'work_zone_id' if field == 'workplace_taz' else 'school_zone_id' + persons[field] = pd.to_numeric(persons[source_field], errors='coerce').fillna(-1) + except KeyError: + logger.info(f"Field `{field}` not present in input h5 file") + + # Clean numeric fields + persons['worker'] = pd.to_numeric(persons['worker'], errors='coerce').fillna(0) + persons['student'] = pd.to_numeric(persons['student'], errors='coerce').fillna(0) + + # Filter invalid records + mask = ( + ~persons[asim_zone_id_col].isnull() & # Has valid TAZ + (persons['age'] >= 1.0) # Not newborn + ) + + if all(col in persons.columns for col in ['workplace_taz', 'school_taz']): + mask &= ~((persons.worker == 1) & (persons.workplace_taz < 0)) # Valid workplace for workers + mask &= ~((persons.student == 1) & (persons.school_taz < 0)) # Valid school for students + + persons = persons.loc[mask].dropna() + + # Reset member IDs + persons['member_id'] = (persons.groupby('household_id')['member_id'] + .transform(lambda x: np.arange(len(x)) + 1)) + + # Clear school/work locations for specific person types + workers_mask = persons['ptype'] == 1 + nonwork_mask = persons.ptype.isin([4, 5]) + + # Clear school locations for workers + if 'school_taz' in persons.columns: + before_count = (workers_mask & (persons.school_taz >= 0)).sum() + persons.loc[workers_mask, ['school_taz', 'school_zone_id']] = -1 + after_count = (workers_mask & (persons.school_taz >= 0)).sum() + logger.info(f"Workers with school location: {before_count} before, {after_count} after cleaning") + + # Clear work/school locations for non-workers + if all(col in persons.columns for col in ['workplace_taz', 'school_taz']): + before_school = (nonwork_mask & (persons.school_taz > 0)).sum() + before_work = (nonwork_mask & (persons.workplace_taz > 0)).sum() + persons.loc[nonwork_mask, ['school_taz', 'school_zone_id', 'workplace_taz', 'work_zone_id']] = -1 + after_school = (nonwork_mask & (persons.school_taz > 0)).sum() + after_work = (nonwork_mask & (persons.workplace_taz > 0)).sum() + + logger.info(f"Non-workers/students with school location: {before_school} before, {after_school} after") + logger.info(f"Non-workers/students with work location: {before_work} before, {after_work} after") return persons - def _update_households_table(households, blocks, asim_zone_id_col='TAZ'): # assign zones households.index = households.index.astype(int) @@ -1390,6 +1380,7 @@ def _update_households_table(households, blocks, asim_zone_id_col='TAZ'): # create new column variables s = households.persons households.loc[:, 'HHT'] = s.where(s == 1, 4) + households["cars"] = households["cars"].astype(int) # clean up dataframe structure # TODO: move this to annotate_households.yaml in asim settings @@ -1795,18 +1786,85 @@ def _create_land_use_table( zones.loc[:, 'TERMINAL'] = 0 # FIXME zones.loc[:, 'COUNTY'] = 1 # FIXME + logger.info(zones.head()) + logger.info(zones.dtypes) + return zones -def copy_beam_geoms(settings, beam_geoms_location, asim_geoms_location): +def copy_data_to_mutable_location(settings, folder_path): + input_dir = os.path.join(folder_path, settings['asim_local_mutable_data_folder']) + os.makedirs(input_dir, exist_ok=True) + region = settings['region'] + beam_input_dir = settings['beam_local_input_folder'] + beam_output_dir = settings['beam_local_output_folder'] + skims_fname = settings['skims_fname'] + origin_skims_fname = settings['origin_skims_fname'] + beam_geoms_fname = settings['beam_geoms_fname'] + beam_router_directory = settings['beam_router_directory'] + asim_geoms_location = os.path.join(input_dir, beam_geoms_fname) + + input_skims_location = os.path.join(beam_input_dir, region, skims_fname) + mutable_skims_location = os.path.join(input_dir, "skims.omx") + + beam_geoms_location = os.path.join(beam_input_dir, region, beam_router_directory, beam_geoms_fname) + if 'beam_skims_shapefile' in settings: + beam_shape_location = os.path.join(beam_input_dir, region, settings['beam_skims_shapefile']) + else: + logger.warning("Not updating zone_id in beam shapefile, make sure it is correct") + beam_shape_location = None + + # TODO: Handle exception when these dont exist + + if os.path.exists(input_skims_location): + logger.info("Copying input skims from {0} to {1}".format( + input_skims_location, + mutable_skims_location)) + shutil.copyfile(input_skims_location, mutable_skims_location) + else: + if os.path.exists(mutable_skims_location): + logger.info("No input skims at {0}. Proceeding with defaults at {1}".format( + input_skims_location, + mutable_skims_location)) + else: + logger.info("No default skims found anywhere. We will generate defaults instead") + + input_skims_location = os.path.join(beam_input_dir, region, origin_skims_fname) + mutable_skims_location = os.path.join(input_dir, "origin_skims.csv.gz") + + if os.path.exists(input_skims_location): + logger.info("Copying input origin skims from {0} to {1}".format( + input_skims_location, + mutable_skims_location)) + shutil.copyfile(input_skims_location, mutable_skims_location) + else: + if os.path.exists(mutable_skims_location): + logger.info("No input skims at {0}. Proceeding with defaults at {1}".format( + input_skims_location, + mutable_skims_location)) + else: + logger.info("No default input skims found anywhere. We will generate defaults instead") + + logger.info("Copying beam zone geoms from {0} to {1}".format( + beam_geoms_location, + asim_geoms_location)) + + configs_source_dir = os.path.join(settings['asim_local_configs_folder'], settings['region']) + configs_dest_dir = os.path.join(folder_path, settings['asim_local_mutable_configs_folder']) + logger.info("Moving asim configs from {0} to {1}".format(configs_source_dir, configs_dest_dir)) + shutil.copytree(configs_source_dir, configs_dest_dir, dirs_exist_ok=True) + + copy_beam_geoms(settings, beam_geoms_location, asim_geoms_location, beam_shape_location) + + +def copy_beam_geoms(settings, beam_geoms_location, asim_geoms_location, beam_shape_location): zone_type_column = {'block_group': 'BLKGRP', 'taz': 'TAZ', 'block': 'BLK'} beam_geoms_file = pd.read_csv(beam_geoms_location, dtype={'GEOID': str}) - zone_type = settings['skims_zone_type'] + zone_type = settings['skims_zone_type'].lower() zone_id_col = zone_type_column[zone_type] + mapping = geoid_to_zone_map(settings, settings['start_year']) - if 'TAZ' not in beam_geoms_file.columns: - - mapping = geoid_to_zone_map(settings, settings['start_year']) + if zone_id_col not in beam_geoms_file.columns: if zone_type == 'block': logger.info("Mapping block IDs") @@ -1816,46 +1874,61 @@ def copy_beam_geoms(settings, beam_geoms_location, asim_geoms_location): logger.info("Mapping block group IDs to TAZ ids") beam_geoms_file['TAZ'] = beam_geoms_file['GEOID'].astype(str).replace(mapping) + elif zone_type == 'taz': + from_col = settings.get('geoms_index_col', 'zone_id') + to_col = 'TAZ' + logger.info("Renaming TAZ column from {0} to {1}".format(from_col, to_col)) + beam_geoms_file.rename(columns={from_col: to_col}, inplace=True) + else: + logger.error(f"Unrecognized zone type {zone_type}, ASim may fail") + beam_geoms_file.to_csv(asim_geoms_location) + if beam_shape_location is not None: + logger.info("Mapping BEAM geometry geoid column {0} to zone_id column {1}".format( + settings['skim_zone_geoid_col'], + settings['skim_zone_source_id_col'])) + zones = gpd.read_file(beam_shape_location) + zones[settings['skim_zone_source_id_col']] = zones[settings['skim_zone_geoid_col']].astype(str).map(mapping) + logger.info("Re-saving BEAM geometry shapefile with updated zone_id to {0}".format(beam_shape_location)) + zones.to_file(beam_shape_location) + def create_asim_data_from_h5( - settings, year, warm_start=False, output_dir=None): + settings, state: "WorkflowState", warm_start=False, output_dir=None): # warm start: year = start_year # asim_no_usim: year = start_year # normal: year = forecast_year region = settings['region'] - region_id = settings['region_to_region_id'][region] FIPS = settings['FIPS'][region] state_fips = FIPS['state'] county_codes = FIPS['counties'] local_crs = settings['local_crs'][region] - usim_local_data_folder = settings['usim_local_data_folder'] zone_type = settings['skims_zone_type'] if not output_dir: - output_dir = settings['asim_local_input_folder'] + output_dir = os.path.join(state.full_path, settings['asim_local_mutable_data_folder']) asim_zone_id_col = 'TAZ' # TODO: Generalize this or add it to settings.yaml - if region == "sfbay": - input_zone_id_col = 'taz1454' - else: - input_zone_id_col = 'zone_id' + input_zone_id_col = settings.get('geoms_index_col', 'zone_id') # TODO: only call _get_zones_geoms if blocks or colleges or schools # don't already have a zone ID (e.g. TAZ). If they all do then we don't # need zone geoms and we can simply instantiate the zones table from # the unique zone ids in the blocks/persons/households tables. - zones = read_zone_geoms(settings, year, + zones = read_zone_geoms(settings, state.forecast_year, asim_zone_id_col=asim_zone_id_col, default_zone_id_col=input_zone_id_col) store, table_prefix_yr = read_datastore( - settings, year, warm_start=warm_start) + settings, state.forecast_year, warm_start=warm_start, mutable_data_dir=state.full_path) - logger.info("Loading UrbanSim data from .h5") + logger.info( + "Loading UrbanSim data from .h5, with year {0} and warmstart {1}".format(state.forecast_year, warm_start)) + logger.info("Reading households table from {0} in table {1}".format(store._path, + os.path.join(table_prefix_yr, 'households'))) households = store[os.path.join(table_prefix_yr, 'households')] persons = store[os.path.join(table_prefix_yr, 'persons')] try: @@ -1867,7 +1940,7 @@ def create_asim_data_from_h5( # update blocks blocks_cols = blocks.columns.tolist() blocks_to_taz_mapping_updated, blocks = _update_blocks_table( - settings, year, blocks, households, jobs, input_zone_id_col) + settings, state.forecast_year, blocks, households, jobs, 'zone_id') input_zone_id_col = "{0}_zone_id".format(zone_type) if blocks_to_taz_mapping_updated: logger.info( diff --git a/pilates/atlas/postprocessor.py b/pilates/atlas/postprocessor.py index d4346ade..1f63967c 100644 --- a/pilates/atlas/postprocessor.py +++ b/pilates/atlas/postprocessor.py @@ -27,20 +27,21 @@ def _get_usim_datastore_fname(settings, io, year=None): return datastore_name -def atlas_update_h5_vehicle(settings, output_year, warm_start=False): +def atlas_update_h5_vehicle(settings, output_year, state: "WorkflowState", warm_start=False): # use atlas outputs in year provided and update "cars" & "hh_cars" # columns in urbansim h5 files logger.info('ATLAS is updating urbansim outputs for Year {}'.format(output_year)) # read and format atlas vehicle ownership output - atlas_output_path = settings['atlas_host_output_folder'] # 'pilates/atlas/atlas_output' # + atlas_output_path = os.path.join(state.full_path, + settings['atlas_host_output_folder']) # 'pilates/atlas/atlas_output' # fname = 'householdv_{}.csv'.format(output_year) df = pd.read_csv(os.path.join(atlas_output_path, fname)) df = df.rename(columns={'nvehicles': 'cars'}).set_index('household_id').sort_index(ascending=True) df['hh_cars'] = pd.cut(df['cars'], bins=[-0.5, 0.5, 1.5, np.inf], labels=['none', 'one', 'two or more']) # set which h5 file to update - h5path = settings['usim_local_data_folder'] + h5path = os.path.join(state.full_path, settings['usim_local_mutable_data_folder']) if warm_start: h5fname = _get_usim_datastore_fname(settings, io='input') else: @@ -59,8 +60,7 @@ def atlas_update_h5_vehicle(settings, output_year, warm_start=False): key = 'households' olddf = h5[key] - if olddf.index.istype(float): - olddf.index = olddf.index.astype(int) + olddf.index = olddf.index.astype(int) olddf = olddf.reindex(df.index.astype(int)) if olddf.shape[0] != df.shape[0]: @@ -68,17 +68,21 @@ def atlas_update_h5_vehicle(settings, output_year, warm_start=False): else: olddf['cars'] = df['cars'].values olddf['hh_cars'] = df['hh_cars'].values + for col in olddf.columns: + if olddf[col].dtype.name == "category": + logger.info("Converting column {0} from category to str".format(col)) + olddf[col] = olddf[col].astype(str) h5[key] = olddf logger.info('ATLAS update h5 datastore table {0} - done'.format(key)) -def atlas_add_vehileTypeId(settings, output_year): +def atlas_add_vehileTypeId(settings, output_year, state): # add a "vehicleTypeId" column in atlas output vehicles_{$year}.csv, # write as vehicles2_{$year}.csv # which will be read by beam preprocessor # vehicleTypeId = conc "bodytype"-"vintage_category"-"pred_power" - atlas_output_path = settings['atlas_host_output_folder'] + atlas_output_path = os.path.join(state.full_path, settings['atlas_host_output_folder']) fname = 'vehicles_{}.csv'.format(output_year) # read original atlas output "vehicles_*.csv" as dataframe @@ -99,13 +103,13 @@ def atlas_add_vehileTypeId(settings, output_year): df.to_csv(os.path.join(atlas_output_path, 'vehicles2_{}.csv'.format(output_year)), index=False) -def build_beam_vehicles_input(settings, output_year): - atlas_output_path = settings['atlas_host_output_folder'] - atlas_input_path = settings['atlas_host_input_folder'] +def build_beam_vehicles_input(settings, output_year, state): + atlas_output_path = os.path.join(state.full_path, settings['atlas_host_output_folder']) + atlas_input_path = os.path.join(state.full_path, settings['atlas_host_mutable_input_folder']) vehicles = pd.read_csv(os.path.join(atlas_output_path, "vehicles_{0}.csv".format(output_year)), dtype={"householdId": pd.Int64Dtype()}) mapping = pd.read_csv( - os.path.join(atlas_input_path, "vehicle_type_mapping_{0}.csv".format(settings['atlas_adscen']))) + os.path.join(atlas_input_path, "vehicle_type_mapping_{0}.csv".format(settings['atlas_scenario']))) mapping['numberOfVehiclesCreated'] = 0 mapping.set_index(["adopt_fuel", "bodytype", "modelyear", "vehicleTypeId"], inplace=True, drop=True) mapping = mapping.loc[~mapping.index.duplicated(), :] @@ -149,6 +153,7 @@ def build_beam_vehicles_input(settings, output_year): vehiclesSub[['household_id', 'vehicleTypeId']]) outputVehicles = pd.concat(allVehicles).reset_index(drop=True) outputVehicles.rename(columns={"household_id": "householdId"}, inplace=True) + outputVehicles['householdId'] = outputVehicles['householdId'].astype(int) outputVehicles.index.rename("vehicleId", inplace=True) outputVehicles.to_csv(os.path.join(atlas_output_path, 'vehicles_{0}.csv.gz'.format(output_year))) allCounts.loc[allCounts.numberOfVehiclesCreated > 0, :].sort_values(by="numberOfVehiclesCreated", ascending=False)[ diff --git a/pilates/atlas/preprocessor.py b/pilates/atlas/preprocessor.py index f2ce6c01..bd47bb70 100644 --- a/pilates/atlas/preprocessor.py +++ b/pilates/atlas/preprocessor.py @@ -1,5 +1,8 @@ +import glob import logging import os +import shutil +from pathlib import Path import numpy as np import openmatrix as omx @@ -12,6 +15,17 @@ logger = logging.getLogger(__name__) +def copy_data_to_mutable_location(settings, output_dir): + atlas_input_path = settings['atlas_host_input_folder'] + atlas_output_path = settings['atlas_warmstart_input_folder'] + logger.info("Copying atlas inputs from {0} to {1}".format(atlas_input_path, output_dir)) + shutil.copytree(atlas_input_path, output_dir, dirs_exist_ok=True) + if settings['start_year'] == 2017: + logger.info("Copying atlas warmstart outputs from {0} to {1}".format(atlas_output_path, output_dir)) + shutil.copytree(atlas_output_path, output_dir, dirs_exist_ok=True) + # TODO: Download preprocessed inputs from here and unzip to the correct folder: https://storage.googleapis.com/beam-core-outputs/atlas-inputs/freeze_startpoint.zip + + def _get_usim_datastore_fname(settings, io, year=None): # reference: asim postprocessor if io == 'output': @@ -26,25 +40,37 @@ def _get_usim_datastore_fname(settings, io, year=None): return datastore_name -def prepare_atlas_inputs(settings, year, warm_start=False): +def prepare_atlas_inputs(settings, year, state: "WorkflowState", warm_start=False): # set where to find urbansim output - urbansim_output_path = settings['usim_local_data_folder'] + urbansim_output_path = os.path.join(state.full_path, settings['usim_local_mutable_data_folder']) if warm_start: # if warm start, read custom_mpo h5 urbansim_output_fname = _get_usim_datastore_fname(settings, io='input') else: # if in main loop, read urbansim-generated h5 - urbansim_output_fname = _get_usim_datastore_fname(settings, io='output', year=year) + urbansim_output_fname = _get_usim_datastore_fname(settings, io='output', year=state.forecast_year) urbansim_output = os.path.join(urbansim_output_path, urbansim_output_fname) # set where to put atlas csv inputs (processed from urbansim outputs) - atlas_input_path = settings['atlas_host_input_folder'] + "/year{}".format(year) + atlas_input_path = os.path.join(state.full_path, settings['atlas_host_mutable_input_folder'], "year{}".format(year)) # if atlas input path does not exist, create one if not os.path.exists(atlas_input_path): os.makedirs(atlas_input_path) logger.info('ATLAS Input Path Created for Year {}'.format(year)) + if year != state.year: + old_input_path = os.path.join(state.full_path, settings['atlas_host_mutable_input_folder'], + "year{}".format(state.year)) + for f in glob.glob(os.path.join(old_input_path, "*.RData")): + if os.path.exists(os.path.join(atlas_input_path, Path(f).name)): + logger.info( + "Not file {0} to atlas input {1} b/c it exists".format(f, os.path.join(atlas_input_path, + Path(f).name))) + else: + logger.info("Moving file {0} to atlas input for year {1}".format(f, year)) + shutil.copyfile(f, os.path.join(atlas_input_path, Path(f).name)) + # read urbansim h5 outputs with pd.HDFStore(urbansim_output, mode='r') as data: if not warm_start: @@ -179,7 +205,7 @@ def compute_accessibility(path_list, measure_list, settings, year, threshold=500 def _get_time_ODmatrix(settings, path_list, measure_list, threshold): # open skims file - skims_dir = settings['asim_local_input_folder'] + skims_dir = settings['asim_local_mutable_data_folder'] skims = omx.open_file(os.path.join(skims_dir, 'skims.omx'), mode='r') # find the path with minimum time for each o-d diff --git a/pilates/beam/postprocessor.py b/pilates/beam/postprocessor.py index a4e9dcf8..882f380a 100644 --- a/pilates/beam/postprocessor.py +++ b/pilates/beam/postprocessor.py @@ -1,10 +1,17 @@ +import concurrent.futures import logging import os +import shutil import numpy as np import openmatrix as omx import pandas as pd +try: + import xarray as xr +except: + print("FAILED TO LOAD XARRAY") + # import pickle # import cloudpickle # import dill @@ -24,6 +31,11 @@ logger = logging.getLogger(__name__) +TNC_CONSOLIDATION_MAP = { + "TNC_SINGLE": "RH_SOLO", + "TNC_POOLED": "RH_SHARED", +} + def find_latest_beam_iteration(beam_output_dir): iter_dirs = [os.path.join(root, dir) for root, dirs, files in os.walk(beam_output_dir) if @@ -49,22 +61,25 @@ def find_not_taken_dir_name(dir_name): raise RuntimeError(f"Cannot find an appropriate not taken directory for {dir_name}") -def rename_beam_output_directory(settings, year, replanning_iteration_number=0): - beam_output_dir = settings['beam_local_output_folder'] +def rename_beam_output_directory(beam_output_dir, settings, year, replanning_iteration_number=0): iteration_output_directory, _ = find_latest_beam_iteration(beam_output_dir) beam_run_output_dir = os.path.join(*iteration_output_directory.split(os.sep)[:-2]) new_iteration_output_directory = os.path.join(beam_output_dir, settings['region'], "year-{0}-iteration-{1}".format(year, replanning_iteration_number)) if os.path.exists(new_iteration_output_directory): os.rename(new_iteration_output_directory, find_not_taken_dir_name(new_iteration_output_directory)) - os.rename(beam_run_output_dir, new_iteration_output_directory) + try: + os.rename(beam_run_output_dir, new_iteration_output_directory) + except FileNotFoundError: + logger.warning("Files {0} not found. Adding a slash".format(beam_run_output_dir)) + os.rename("/" + str(beam_run_output_dir), new_iteration_output_directory) def find_produced_od_skims(beam_output_dir, suffix="csv.gz"): iteration_dir, it_num = find_latest_beam_iteration(beam_output_dir) if iteration_dir is None: return None - od_skims_path = os.path.join(iteration_dir, "{0}.activitySimODSkims_current.{1}".format(it_num, suffix)) + od_skims_path = os.path.join(iteration_dir, "{0}.skimsActivitySimOD_current.{1}".format(it_num, suffix)) logger.info("expecting skims at {0}".format(od_skims_path)) return od_skims_path @@ -93,42 +108,41 @@ def find_produced_origin_skims(beam_output_dir): return ridehail_skims_path -def _merge_skim(inputMats, outputMats, path, timePeriod, measures): - complete_key = '_'.join([path, 'TRIPS', '', timePeriod]) - failed_key = '_'.join([path, 'FAILURES', '', timePeriod]) +def _merge_skim(inputMats, outputMats, path, timePeriod, measures, transit_scale_factor=100.0): + complete_key = f"{path}_TRIPS__{timePeriod}" + failed_key = f"{path}_FAILURES__{timePeriod}" completed, failed = None, None + if complete_key in inputMats.keys(): completed = np.array(inputMats[complete_key]).copy() - if '_'.join([path, 'TOTIVT', '', timePeriod]) in inputMats.keys(): - shouldNotBeZero = (completed > 0) & (np.array(inputMats['_'.join([path, 'TOTIVT', '', timePeriod])]) == 0) + if f"{path}_TOTIVT__{timePeriod}" in inputMats.keys(): + shouldNotBeZero = (completed > 0) & (np.array(inputMats[f"{path}_TOTIVT__{timePeriod}"]) == 0) if shouldNotBeZero.any(): logger.warning( - "In BEAM outputs for {0} in {1} we have {2} completed trips with " - "time = 0".format(path, timePeriod, shouldNotBeZero.sum())) + f"In BEAM outputs for {path} in {timePeriod} we have {shouldNotBeZero.sum()} completed trips with time = 0" + ) completed[shouldNotBeZero] = 0 failed = np.array(inputMats[failed_key]) - logger.info("Adding {0} valid trips and {1} impossible trips to skim {2}, where {3} had existed before".format( - np.nan_to_num(completed).sum(), - np.nan_to_num(failed).sum(), - complete_key, - np.nan_to_num(np.array(outputMats[complete_key])).sum())) - try: - logger.info("Of the {0} completed trips, {1} were to a previously unobserved " - "OD".format(np.nan_to_num(completed).sum(), - np.nan_to_num(completed[outputMats[complete_key][:] == 0]).sum())) - except: - pass - toPenalize = np.array([0]) - toCancel = np.array([0]) + + logger.info( + f"Adding {np.nan_to_num(completed).sum()} valid trips and {np.nan_to_num(failed).sum()} impossible trips to skim {complete_key}, where {np.nan_to_num(np.array(outputMats[complete_key])).sum()} had existed before" + ) + logger.info( + f"Of the {np.nan_to_num(completed).sum()} completed trips, {np.nan_to_num(completed[outputMats[complete_key][:] == 0]).sum()} were to a previously unobserved OD" + ) + toCancel = [] + toPenalize = [] + for measure in measures: - inputKey = '_'.join([path, measure, '', timePeriod]) + inputKey = f"{path}_{measure}__{timePeriod}" if path in ["WALK", "BIKE"]: if measure == "DIST": - outputKey = path + "DIST" + outputKey = f"{path}DIST" else: - outputKey = '_'.join([path, measure]) + outputKey = f"{path}_{measure}" else: outputKey = inputKey + if (outputKey in outputMats) and (inputKey in inputMats): if measure == "TRIPS": outputMats[outputKey][completed > 0] += completed[completed > 0] @@ -136,15 +150,14 @@ def _merge_skim(inputMats, outputMats, path, timePeriod, measures): outputMats[outputKey][failed > 0] += failed[failed > 0] elif measure == "DIST": outputMats[outputKey][completed > 0] = 0.5 * ( - outputMats[outputKey][completed > 0] + inputMats[inputKey][completed > 0]) + outputMats[outputKey][completed > 0] + inputMats[inputKey][completed > 0] + ) elif measure in ["IWAIT", "XWAIT", "WACC", "WAUX", "WEGR", "DTIM", "DDIST", "FERRYIVT"]: - # NOTE: remember the mtc asim implementation has scaled units for these variables valid = ~np.isnan(inputMats[inputKey][:]) outputMats[outputKey][(completed > 0) & valid] = inputMats[inputKey][ - (completed > 0) & valid] * 100.0 + (completed > 0) & valid] * transit_scale_factor elif measure in ["TOTIVT", "IVT"]: - - inputKeyKEYIVT = '_'.join([path, 'KEYIVT', '', timePeriod]) + inputKeyKEYIVT = f"{path}_KEYIVT__{timePeriod}" outputKeyKEYIVT = inputKeyKEYIVT if (inputKeyKEYIVT in inputMats.keys()) & (outputKeyKEYIVT in outputMats.keys()): additionalFilter = (outputMats[outputKeyKEYIVT][:] > 0) @@ -153,63 +166,53 @@ def _merge_skim(inputMats, outputMats, path, timePeriod, measures): outputTravelTime = np.array(outputMats[outputKey]) toCancel = (failed > 3) & (failed > (6 * completed)) previouslyNonZero = ((outputTravelTime > 0) | additionalFilter) & toCancel - # save this for later so it doesn't get overwritten toPenalize = (failed > completed) & ~toCancel & ((outputTravelTime > 0) | additionalFilter) if toCancel.sum() > 0: logger.info( - "Marking {0} {1} trips completely impossible in {2}. There were {3} completed trips but {4}" - " failed trips in these ODs. Previously, {5} were nonzero".format( - toCancel.sum(), path, timePeriod, completed[toCancel].sum(), failed[toCancel].sum(), - previouslyNonZero.sum())) - logger.info("There are now {0} observed ODs, {1} impossible ODs, and {2} default ODs".format( - ((completed > 0) & (outputTravelTime > 0)).sum(), - (outputTravelTime == 0).sum(), - ((completed == 0) & (outputTravelTime > 0)).sum() - )) + f"Marking {toCancel.sum()} {path} trips completely impossible in {timePeriod}. There were {completed[toCancel].sum()} completed trips but {failed[toCancel].sum()} failed trips in these ODs. Previously, {previouslyNonZero.sum()} were nonzero" + ) + logger.info( + f"There are now {((completed > 0) & (outputTravelTime > 0)).sum()} observed ODs, {(outputTravelTime == 0).sum()} impossible ODs, and {((completed == 0) & (outputTravelTime > 0)).sum()} default ODs" + ) toAllow = ~toCancel & ~toPenalize & ~np.isnan(inputMats[inputKey][:]) outputMats[outputKey][toAllow] = inputMats[inputKey][toAllow] * 100 - # outputMats[outputKey][toCancel] = 0.0 if (inputKeyKEYIVT in inputMats.keys()) & (outputKeyKEYIVT in outputMats.keys()): - # outputMats[outputKeyKEYIVT][toCancel] = 0.0 outputMats[outputKeyKEYIVT][toAllow] = inputMats[inputKeyKEYIVT][toAllow] * 100 - elif not measure.endswith("TOLL"): # hack to avoid overwriting initial tolls + elif not measure.endswith("TOLL"): outputMats[outputKey][completed > 0] = inputMats[inputKey][completed > 0] if path.startswith('SOV_'): for sub in ['SOVTOLL_', 'HOV2_', 'HOV2TOLL_', 'HOV3_', 'HOV3TOLL_']: - newKey = '_'.join([path, measure.replace('SOV_', sub), '', timePeriod]) + newKey = f"{path}_{measure.replace('SOV_', sub)}__{timePeriod}" outputMats[newKey][completed > 0] = inputMats[inputKey][completed > 0] - logger.info("Adding {0} valid trips and {1} impossible trips to skim {2}".format( - np.nan_to_num(completed).sum(), - np.nan_to_num(failed).sum(), - newKey)) + logger.info( + f"Adding {np.nan_to_num(completed).sum()} valid trips and {np.nan_to_num(failed).sum()} impossible trips to skim {newKey}") + badVals = np.sum(np.isnan(outputMats[outputKey][:])) if badVals > 0: - logger.warning("Total number of {0} skim values are NaN for skim {1}".format(badVals, outputKey)) + logger.warning(f"Total number of {badVals} skim values are NaN for skim {outputKey}") elif outputKey in outputMats: - logger.warning("Target skims are missing key {0}".format(outputKey)) + logger.warning(f"Target skims are missing key {outputKey}") else: - logger.warning("BEAM skims are missing key {0}".format(outputKey)) + logger.warning(f"BEAM skims are missing key {outputKey}") if toCancel.sum() > 0: for measure in measures: - if measure not in ["TRIPS, FAILURES"]: - key = '_'.join([path, measure, '', timePeriod]) + if measure not in ["TRIPS", "FAILURES"]: + key = f"{path}_{measure}__{timePeriod}" try: outputMats[key][toCancel] = 0.0 except: - logger.warning( - "Tried to cancel {0} trips for key {1} but couldn't find key".format(toCancel.sum(), key)) + logger.warning(f"Tried to cancel {toCancel.sum()} trips for key {key} but couldn't find key") if ("TOTIVT" in measures) & ("IWAIT" in measures) & ("KEYIVT" in measures): if toPenalize.sum() > 0: - inputKey = '_'.join([path, 'IWAIT', '', timePeriod]) + inputKey = f"{path}_IWAIT__{timePeriod}" outputMats[inputKey][toPenalize] = inputMats[inputKey][toPenalize] * (failed[toPenalize] + 1) / ( - completed[toPenalize] + 1) - # outputSkim.close() - # inputSkim.close() + completed[toPenalize] + 1 + ) else: - logger.info( - "No input skim for mode {0} and time period {1}, with key {2}".format(path, timePeriod, complete_key)) + logger.info(f"No input skim for mode {path} and time period {timePeriod}, with key {complete_key}") + return (path, timePeriod), (completed, failed) @@ -248,17 +251,1946 @@ def simplify(input, timePeriod, mode, utf=False, expand=False): return originalDict -def copy_skims_for_unobserved_modes(mapping, skims): +def copy_skims_for_unobserved_modes(mapping, skims_ds): + """ + Copy skim data from one mode to others based on a mapping. + Operates on the Zarr dataset directly. + + Parameters + ---------- + mapping : dict + Mapping from source mode to list of target modes (e.g., {"SOV": ["SOVTOLL", ...]}). + skims_ds : xarray.Dataset + The target Zarr dataset containing all skims. + """ + logger.info("Copying skims for unobserved modes...") for fromMode, toModes in mapping.items(): - relevantSkimKeys = [key for key in skims.list_matrices() if key.startswith(fromMode + "_") & ~("TOLL" in key)] - for skimKey in relevantSkimKeys: - for toMode in toModes: - toKey = skimKey.replace(fromMode + "_", toMode + "_") - skims[toKey][:] = skims[skimKey][:] - print("Copying values from {0} to {1}".format(skimKey, toKey)) + # Find all skim variables starting with the source mode + # Exclude TRIPS and FAILURES as these represent observed demand, not skim values + # Exclude measures that might have mode-specific meanings or are handled separately + # e.g., "TOLL" skims should not be copied onto non-toll modes, but the mapping is SOV -> SOVTOLL etc. + # The original code copied *all* relevant skim keys. Let's refine this. + # A better approach is to specify which measures should be copied. + # However, sticking to the original behavior of copying most things except TRIPS/FAILURES + # and assuming the measures list in settings covers what's needed. + # Let's copy based on the existence of the target key format in the dataset. + + # Get all variable names in the dataset + all_vars = list(skims_ds.data_vars) + + for var_name in all_vars: + # Check if the variable name starts with the source mode path + # and is not TRIPS, FAILURES, or TNC specific (handled elsewhere) + if var_name.startswith(fromMode + "_") and not any(m in var_name for m in ["_TRIPS", "_FAILURES"]): + # Check if this measure should be copied (avoid copying e.g. SOV_TOLL to HOV) + if "TOLL" in var_name.split('_')[0]: # Check if the mode itself contains TOLL + continue # Don't use TOLL skims as sources to copy FROM + + # Extract the measure name (assuming format SOV_MEASURE or SOVTOLL_MEASURE) + measure_parts = var_name.split('_', 1) + if len(measure_parts) < 2: + continue # Skip if not in expected format + + measure_name = measure_parts[1] # e.g., "TOTIVT" + + # For each target mode, construct the expected target variable name + for toMode in toModes: + target_var_name = f"{toMode}_{measure_name}" + + # Check if the target variable exists in the dataset + if target_var_name in all_vars: + # Perform the copy using .data + # Check shape compatibility - they should be the same (zones, zones, periods) + if skims_ds[var_name].shape == skims_ds[target_var_name].shape: + skims_ds[target_var_name].data[:] = skims_ds[var_name].data[:] + logger.info(f"Copied data from '{var_name}' to '{target_var_name}'") + else: + logger.warning(f"Shape mismatch when copying from '{var_name}' to '{target_var_name}'. Skipping.") + else: + logger.debug(f"Target variable '{target_var_name}' not found in dataset. Skipping copy.") + + logger.info("Completed copying skims for unobserved modes.") + +def _postprocess_tnc_zarr(skims_ds, timePeriods, settings, completed_failed_dict, use_rh_modes=False): + """ + Applies TNC/RH-specific post-processing rules (REJECTIONPROB, IWAIT filling, + DDIST/TOTIVT interpolation, FAR assignment) to Zarr skims. + Operates on either TNC_* or RH_* variables based on `use_rh_modes`. + + Parameters: + ----------- + skims_ds : xarray.Dataset + The target Zarr dataset containing all skims. + timePeriods : list of str + List of time period names. + settings : dict + Settings dictionary, needed for SOV keys and potential FAR value. + completed_failed_dict : dict + Dictionary containing completed and failed trip counts aggregated across + all time periods for each mode path. Keyed by mode path, values are + [completed_trips_3d_array, failed_trips_3d_array]. This dict should + contain counts for the modes being processed (either TNC providers or RH). + use_rh_modes : bool + If True, process RH_* modes. If False, process TNC_* provider modes. + """ + logger.info("Applying TNC/RH-specific post-processing...") + tp_to_idx = {tp: idx for idx, tp in enumerate(timePeriods)} + + if use_rh_modes: + # Process consolidated RH modes + modes_to_process = [ + target_mode for source_mode, target_mode in TNC_CONSOLIDATION_MAP.items() + if f"{target_mode}_TRIPS" in skims_ds.data_vars # Only process if the consolidated variable exists + ] + logger.info(f"Post-processing consolidated RH modes: {modes_to_process}") + else: + # Process original TNC provider modes + # Find provider-specific TNC modes like TNC_SINGLE_UBER, TNC_POOLED_LYFT + all_vars = list(skims_ds.data_vars) + tnc_provider_modes = set() + for var_name in all_vars: + if var_name.startswith("TNC_") and "_TRIPS" in var_name: + parts = var_name.split('_') + if len(parts) >= 3 and parts[2] not in ["TRIPS", "FAILURES", "REJECTIONPROB", "IWAIT", + "DTIM", "DDIST", "TOTIVT", "FAR"]: + tnc_provider_modes.add("_".join(parts[:3])) # e.g., TNC_SINGLE_UBER + elif len(parts) >= 2 and parts[1] in ["SINGLE", "POOLED"]: + key_parts = var_name.split('__') # split off period + if len(key_parts) > 0: + measure_part = key_parts[0] # e.g. TNC_SINGLE_UBER_TRIPS + # Split from the right, looking for measure names + measure_found = False + for common_measure in ["TRIPS", "FAILURES", "REJECTIONPROB", "IWAIT", + "DTIM", "DDIST", "TOTIVT", "FAR"]: + if measure_part.endswith("_" + common_measure): + mode_name = measure_part[:-len("_" + common_measure)] # e.g., TNC_SINGLE_UBER + tnc_provider_modes.add(mode_name) + measure_found = True + break + if not measure_found: + logger.debug(f"Could not parse TNC provider mode from variable: {var_name}") + + modes_to_process = list(tnc_provider_modes) + logger.info(f"Post-processing original TNC provider modes: {modes_to_process}") + + if not modes_to_process: + logger.info("No TNC/RH modes found in skims dataset to post-process.") + return + + # Get SOV skims needed for interpolation ratios + sov_dist_da = skims_ds.get("SOV_DIST") + sov_time_da = skims_ds.get("SOV_TIME") + if (sov_dist_da is None or sov_time_da is None) and any(mode.startswith("RH_") for mode in modes_to_process): + logger.warning("SOV_DIST or SOV_TIME missing, DDIST/TOTIVT interpolation for RH modes will be skipped.") + + + for mode in modes_to_process: + logger.debug(f"Post-processing mode: {mode}") + + # Get completed and failed trip data for this mode (3D arrays) + # These should already be in skims_ds if merge/consolidation happened + completed_key = f"{mode}_TRIPS" + failed_key = f"{mode}_FAILURES" + + completed_da = skims_ds.get(completed_key) + failed_da = skims_ds.get(failed_key) + + if completed_da is None or failed_da is None: + logger.warning(f"Missing {completed_key} or {failed_key} for TNC/RH post-processing. Skipping {mode}.") + continue + + completed_3d = np.nan_to_num(completed_da.data) + failed_3d = np.nan_to_num(failed_da.data) + + # Get other relevant TNC/RH data arrays + rejection_prob_da = skims_ds.get(f"{mode}_REJECTIONPROB") + iwait_da = skims_ds.get(f"{mode}_IWAIT") + ddist_da = skims_ds.get(f"{mode}_DDIST") + totivt_da = skims_ds.get(f"{mode}_TOTIVT") + far_da = skims_ds.get(f"{mode}_FAR") + + # --- REJECTIONPROB --- + if rejection_prob_da is not None: + logger.debug(f"Calculating REJECTIONPROB for {mode}") + total_trips_3d = completed_3d + failed_3d + # Calculate OD-level probability where total trips > 0 + valid_ods_mask_3d = total_trips_3d > 0 + + rejection_prob_3d = np.zeros_like(completed_3d, dtype=np.float32) + np.divide(failed_3d, total_trips_3d, + out=rejection_prob_3d, + where=valid_ods_mask_3d) + + # Apply origin-level probability where origin total trips > 0 (This matches original OMX logic) + completed_sum_by_origin_3d = completed_3d.sum(axis=1) # Sum over destination dimension + failed_sum_by_origin_3d = failed_3d.sum(axis=1) # Sum over destination dimension + total_trips_origin_3d = completed_sum_by_origin_3d + failed_sum_by_origin_3d + valid_origins_mask_3d = total_trips_origin_3d > 0 # Mask is (zones, periods) + + # Calculate origin-level probs + origin_probs_3d = np.zeros_like(completed_sum_by_origin_3d, dtype=np.float32) + np.divide(failed_sum_by_origin_3d, total_trips_origin_3d, + out=origin_probs_3d, + where=valid_origins_mask_3d) # origin_probs_3d shape (zones, periods) + + # Apply origin-level probability to rows where origin total > 0 + for tp_idx in range(len(timePeriods)): + valid_origins_tp = valid_origins_mask_3d[:, tp_idx] # (zones,) + origin_probs_tp = origin_probs_3d[:, tp_idx] # (zones,) + rejection_prob_da.data[valid_origins_tp, :, tp_idx] = origin_probs_tp[valid_origins_tp, None] # Apply row-wise + + logger.debug(f"Updated REJECTIONPROB for {mode}") + + + # --- IWAIT --- + if iwait_da is not None: + logger.debug(f"Processing IWAIT for {mode}") + + # Calculate weighted mean wait time by origin *using the TRIPS data from skims_ds* + # Sum waitTime * completed per origin, divide by sum completed per origin + # Use the data already in the Zarr slice (`iwait_da.data`) which was merged/consolidated + # Apply scaling (100) if needed - _merge_zarr_skim does this. If consolidate logic does it, + # the data should already be scaled. Assume data is already scaled (e.g. in 0.01 minutes). + # The postprocessing logic *should* happen on scaled data. + + current_iwait_data_3d = np.nan_to_num(iwait_da.data) # Data should be in 0.01 minutes from merge/consolidate + completed_3d_scaled = completed_3d # Completed is count, no scaling needed for weight + + sum_weighted_wait_3d = np.nansum(current_iwait_data_3d * completed_3d_scaled, axis=1) # Shape (zones, periods) + sum_completed_origin_3d = np.nansum(completed_3d_scaled, axis=1) # Shape (zones, periods) + + # Handle division by zero - origins with no completed trips will have NaN mean initially + weighted_mean_by_origin_3d = np.full_like(sum_completed_origin_3d, np.nan, dtype=np.float32) + valid_origins_for_mean_3d = sum_completed_origin_3d != 0 + np.divide(sum_weighted_wait_3d, sum_completed_origin_3d, + out=weighted_mean_by_origin_3d, + where=valid_origins_for_mean_3d) # Shape (zones, periods) + + # Identify cells to fill: where completed is 0 AND the origin had some completed trips + # Apply this filling logic period by period for clarity + for tp_idx in range(len(timePeriods)): + completed_tp = completed_3d[:, :, tp_idx] # (zones, zones) + valid_origins_tp = valid_origins_for_mean_3d[:, tp_idx] # (zones,) + weighted_mean_tp = weighted_mean_by_origin_3d[:, tp_idx] # (zones,) + + # Mask for cells where completed == 0 AND the origin has at least one completed trip *in this period* + mask_to_fill_with_average = (completed_tp == 0) & (valid_origins_tp[:, None]) # (zones, zones) + + # Apply the weighted mean by origin + if mask_to_fill_with_average.any(): + # Get the origin index for each cell to fill + origin_indices_to_fill = np.where(mask_to_fill_with_average)[0] # Row indices (origins) + iwait_da.data[mask_to_fill_with_average, tp_idx] = weighted_mean_tp[origin_indices_to_fill] + + logger.debug(f"Filled {mask_to_fill_with_average.sum()} {mode} IWAIT values in {timePeriods[tp_idx]} with origin weighted average.") + + + # Handle bad values (NaNs) that might still exist if an origin had no completed trips at all + # These remain 0 after _merge_zarr_skim or _consolidate_tnc_data + # Check if any NaNs were introduced or remain (they shouldn't be if init was zeros/nan) + # iwait_da.data[:, :, tp_idx] = iwait_slice # Update Zarr slice - not needed if modifying .data in place + + logger.debug(f"Updated IWAIT for {mode}") + + + # --- DDIST and TOTIVT (Interpolation using SOV) --- + # Requires SOV_DIST and SOV_TIME skims to be present in skims_ds + + # DDIST + if ddist_da is not None and sov_dist_da is not None: + logger.debug(f"Processing DDIST for {mode}") + current_ddist_data_3d = np.nan_to_num(ddist_da.data) # Data should be in miles/feet? Assume consistent units + sov_dist_data_3d = np.nan_to_num(sov_dist_da.data) + + # Cells where we can calculate the ratio: SOV_DIST > 0, MODE_TRIPS > 0, MODE_DDIST > 0 + # Use 3D arrays for this calculation across all periods + mask_for_ratio_3d = (sov_dist_data_3d > 0) & (completed_3d > 0) & (current_ddist_data_3d > 0) + + ratio = np.nan + # Calculate overall weighted average ratio if there are any valid points across all periods + if mask_for_ratio_3d.any(): + # Calculate weighted average ratio: (MODE_DDIST * Completed).sum() / (SOV_DIST * Completed).sum() + weighted_mode_ddist_3d = current_ddist_data_3d[mask_for_ratio_3d] * completed_3d[mask_for_ratio_3d] + weighted_sov_dist_3d = sov_dist_data_3d[mask_for_ratio_3d] * completed_3d[mask_for_ratio_3d] + sum_weighted_sov = np.sum(weighted_sov_dist_3d) + if sum_weighted_sov > 0: + ratio = np.sum(weighted_mode_ddist_3d) / sum_weighted_sov + + ratios_individual = current_ddist_data_3d[mask_for_ratio_3d] / sov_dist_data_3d[mask_for_ratio_3d] + logger.info( + f"Observed {mode} DDIST/SOV DIST ratio of {ratio:2.3f} ({np.nanpercentile(ratios_individual, 10):2.3f} - {np.nanpercentile(ratios_individual, 90):2.3f}) for {mode}. " + f"Interpolating {np.sum(~mask_for_ratio_3d):.0f} missing values." + ) + else: + logger.info(f"No data points available to calculate {mode} DDIST/SOV DIST ratio for {mode}.") + + # Apply minimum ratio check (from feature branch logic) + min_ratio = 0.8 + if not np.isnan(ratio) and ratio < min_ratio: + logger.warning(f"Calculated {mode} DDIST/SOV DIST ratio ({ratio:2.3f}) is below {min_ratio}. Setting ratio to {min_ratio}.") + ratio = min_ratio + + # Interpolate where ratio was calculated and cells were NOT used for ratio calculation + mask_to_interpolate_3d = ~mask_for_ratio_3d + if not np.isnan(ratio) and mask_to_interpolate_3d.any(): + # Interpolate using SOV_DIST where needed + # Ensure we don't use SOV_DIST values that are zero for interpolation + mask_interpolation_valid_sov_3d = mask_to_interpolate_3d & (sov_dist_data_3d > 0) + # Apply interpolated DDIST value (distance measure, no 100x scaling) + ddist_da.data[mask_interpolation_valid_sov_3d] = sov_dist_data_3d[mask_interpolation_valid_sov_3d] * ratio # Assumes SOV_DIST is in consistent units (miles/feet) + logger.debug(f"Interpolated {mask_interpolation_valid_sov_3d.sum()} {mode} DDIST values using SOV_DIST ratio.") + + # Handle remaining NaNs if any (e.g., where SOV_DIST was also 0 or ratio wasn't calculable) + if np.isnan(ddist_da.data).any(): + nan_count = np.isnan(ddist_da.data).sum() + ddist_da.data[np.isnan(ddist_da.data)] = 0.0 # Default remaining NaNs to 0 + logger.debug(f"Set {nan_count} remaining NaN {mode} DDIST values to 0.") + + logger.debug(f"Updated DDIST for {mode}") + elif ddist_da is not None: + logger.warning(f"{mode} DDIST cannot be interpolated: SOV_DIST is missing from skims.") + + + # TOTIVT + if totivt_da is not None and sov_time_da is not None: + logger.debug(f"Processing TOTIVT for {mode}") + current_totivt_data_3d = np.nan_to_num(totivt_da.data) + sov_time_data_3d = np.nan_to_num(sov_time_da.data) + + # Cells where we can calculate the ratio: SOV_TIME > 0, MODE_TRIPS > 0, MODE_TOTIVT > 0 + # Use 3D arrays for this calculation across all periods + mask_for_ratio_3d = (sov_time_data_3d > 0) & (completed_3d > 0) & (current_totivt_data_3d > 0) + + ratio = np.nan + # Calculate overall weighted average ratio if there are any valid points across all periods + if mask_for_ratio_3d.any(): + # Calculate weighted average ratio: (MODE_TOTIVT * Completed).sum() / (SOV_TIME * Completed).sum() + weighted_mode_totivt_3d = current_totivt_data_3d[mask_for_ratio_3d] * completed_3d[mask_for_ratio_3d] + weighted_sov_time_3d = sov_time_data_3d[mask_for_ratio_3d] * completed_3d[mask_for_ratio_3d] + sum_weighted_sov = np.sum(weighted_sov_time_3d) + if sum_weighted_sov > 0: + ratio = np.sum(weighted_mode_totivt_3d) / sum_weighted_sov + + ratios_individual = current_totivt_data_3d[mask_for_ratio_3d] / sov_time_data_3d[mask_for_ratio_3d] + logger.info( + f"Observed {mode} TOTIVT/SOV TIME ratio of {ratio:2.3f} ({np.nanpercentile(ratios_individual, 10):2.3f} - {np.nanpercentile(ratios_individual, 90):2.3f}) for {mode}. " + f"Interpolating {np.sum(~mask_for_ratio_3d):.0f} missing values." + ) + else: + logger.info(f"No data points available to calculate {mode} TOTIVT/SOV TIME ratio for {mode}.") + + + # Apply minimum ratio check (from feature branch logic) + min_ratio = 0.8 # Same minimum as DDIST in feature branch + if not np.isnan(ratio) and ratio < min_ratio: + logger.warning(f"Calculated {mode} TOTIVT/SOV TIME ratio ({ratio:2.3f}) is below {min_ratio}. Setting ratio to {min_ratio}.") + ratio = min_ratio + + # Interpolate where ratio was calculated and cells were NOT used for ratio calculation + mask_to_interpolate_3d = ~mask_for_ratio_3d + if not np.isnan(ratio) and mask_to_interpolate_3d.any(): + # Interpolate using SOV_TIME where needed + # Ensure we don't use SOV_TIME values that are zero for interpolation + mask_interpolation_valid_sov_3d = mask_to_interpolate_3d & (sov_time_data_3d > 0) + totivt_da.data[mask_interpolation_valid_sov_3d] = sov_time_data_3d[mask_interpolation_valid_sov_3d] * ratio + logger.debug(f"Interpolated {mask_interpolation_valid_sov_3d.sum()} {mode} TOTIVT values using SOV_TIME ratio.") + + # Handle remaining NaNs if any (e.g., where SOV_TIME was also 0 or ratio wasn't calculable) + if np.isnan(totivt_da.data).any(): + nan_count = np.isnan(totivt_da.data).sum() + totivt_da.data[np.isnan(totivt_da.data)] = 0.0 # Default remaining NaNs to 0 + logger.debug(f"Set {nan_count} remaining NaN {mode} TOTIVT values to 0.") + + + logger.debug(f"Updated TOTIVT for {mode}") + elif totivt_da is not None: + logger.warning(f"{mode} TOTIVT cannot be interpolated: SOV_TIME is missing from skims.") + + # --- FAR --- + if far_da is not None: + logger.debug(f"Processing FAR for {mode}") + # Assuming FAR should be set to a default value where it wasn't observed or is zero + # Original logic seems to set FAR to 0 except where explicitly calculated, or set to a fixed value. + # The OMX code did not explicitly modify FAR, but the postprocessor had commented out code. + # Let's default to 0 where TRIPS are zero, or a fixed value if settings provide one. + default_far = settings.get('tnc_default_far', 0.0) # Add this setting + + # Set FAR to default_far where TRIPS are zero or NaN + mask_zero_trips = (completed_3d + failed_3d) == 0 + mask_nan_far = np.isnan(far_da.data) + mask_to_set = mask_zero_trips | mask_nan_far + + if mask_to_set.any(): + far_da.data[mask_to_set] = default_far + logger.debug(f"Set {mask_to_set.sum()} {mode} FAR values to default {default_far}") + + # Ensure FAR is not negative + negative_far = far_da.data < 0 + if negative_far.any(): + logger.warning(f"Found {negative_far.sum()} negative FAR values for {mode}. Setting to 0.") + far_da.data[negative_far] = 0.0 + + logger.debug(f"Updated FAR for {mode}") + + + logger.info("Completed TNC/RH-specific post-processing.") + + +def clear_skim_cache(asim_local_output_dir): + skims_path = os.path.join(asim_local_output_dir, "cache") + if os.path.exists(skims_path): + logger.info("Deleting skims cache at {0}. Eventually we should modify it in place".format(skims_path)) + shutil.rmtree(skims_path) + else: + logger.warning("Did not find skim cache to delete") + +def _accumulate_all_completed_failed_trips(partialSkims, timePeriods): + """ + Accumulates completed and failed trip counts from all modes/providers + in the partial OMX skim file. + + Parameters: + ----------- + partialSkims : omx.open_file object + The OMX file containing the partial skims from BEAM. + timePeriods : list of str + List of time period names. + + Returns: + -------- + dict + Dictionary where keys are mode paths (e.g., "SOV", "TNC_SINGLE_UBER", "WLK_TRN_WLK") + and values are tuples: (completed_trips_3d_array, failed_trips_3d_array). + """ + logger.info("Accumulating completed and failed trip counts from partial skims...") + completed_failed_dict = {} + + try: + # Get shape from a known 3D matrix, fallback to 2D shape if needed + sample_matrix = next((m for m in partialSkims.list_matrices() if m.endswith("__" + timePeriods[0])), None) + if sample_matrix: + out_array_shape_2d = partialSkims[sample_matrix].shape + out_array_shape_3d = list(out_array_shape_2d) + [len(timePeriods)] + else: + # Fallback if no matrices found with period suffix + logger.warning("No partial skim matrices found with period suffix. Cannot determine shape for trip counts.") + return {} + + except Exception as e: + logger.error(f"Error determining partial skim shape: {e}. Cannot accumulate trip counts.") + return {} + + + for omx_key in partialSkims.list_matrices(): + # Split key into measure_part and period + parts = omx_key.split('__') + if len(parts) != 2: + continue # Skip keys not in PATH_MEASURE__PERIOD format + + measure_part = parts[0] # e.g. SOV_TRIPS or TNC_SINGLE_UBER_TRIPS + period = parts[1] # e.g. AM + + if period not in timePeriods: + continue # Skip keys for irrelevant periods + + # Check if it's a TRIPS or FAILURES matrix + is_trips = measure_part.endswith("_TRIPS") + is_failures = measure_part.endswith("_FAILURES") + + if is_trips or is_failures: + # Extract the mode path from the measure_part + # e.g., SOV from SOV_TRIPS, TNC_SINGLE_UBER from TNC_SINGLE_UBER_TRIPS + if is_trips: + mode = measure_part[:-len("_TRIPS")] + else: # is_failures + mode = measure_part[:-len("_FAILURES")] + + if mode not in completed_failed_dict: + completed_failed_dict[mode] = [ + np.zeros(out_array_shape_3d, dtype=np.float32), # Completed trips 3D + np.zeros(out_array_shape_3d, dtype=np.float32) # Failed trips 3D + ] + + try: + tp_idx = timePeriods.index(period) + data_2d = np.nan_to_num(partialSkims[omx_key][:]) # Get 2D numpy array for this period + + # Ensure shape compatibility + if data_2d.shape != out_array_shape_2d: + logger.warning(f"Shape mismatch for partial skim {omx_key}. Expected {out_array_shape_2d}, got {data_2d.shape}. Skipping accumulation.") + continue + + if is_trips: + completed_failed_dict[mode][0][:, :, tp_idx] = data_2d + else: # is_failures + completed_failed_dict[mode][1][:, :, tp_idx] = data_2d + + except ValueError: + logger.warning(f"Period '{period}' from key '{omx_key}' not found in settings periods. Skipping.") + except Exception as e: + logger.error(f"Error reading partial skim {omx_key} for trip count accumulation: {e}") + + + logger.info(f"Accumulated trip counts for {len(completed_failed_dict)} modes/providers.") + # logger.debug(f"Modes with trip counts: {list(completed_failed_dict.keys())}") # Can be very verbose + + return completed_failed_dict + +def _accumulate_completed_failed_trips(partialSkims, timePeriods): + completed_failed_dict = {} + out_array_shape = list(partialSkims.shape()) + [len(timePeriods)] + + for tpIdx, tp in enumerate(timePeriods): + for key in partialSkims.list_matrices(): + if key.endswith(f"_TRIPS__{tp}"): + mode = key.rsplit('_', 3)[0] # Extract mode from key (e.g., WLK_TRN_WLK) + if mode not in completed_failed_dict: + completed_failed_dict[mode] = [np.zeros(out_array_shape, dtype=np.float32), + np.zeros(out_array_shape, dtype=np.float32)] + completed_failed_dict[mode][0][:, :, tpIdx] = np.nan_to_num(partialSkims[key][:]) + elif key.endswith(f"_FAILURES__{tp}"): + mode = key.rsplit('_', 3)[0] # Extract mode from key (e.g., WLK_TRN_WLK) + if mode not in completed_failed_dict: + completed_failed_dict[mode] = [np.zeros(out_array_shape, dtype=np.float32), + np.zeros(out_array_shape, dtype=np.float32)] + completed_failed_dict[mode][1][:, :, tpIdx] = np.nan_to_num(partialSkims[key][:]) + + return completed_failed_dict + + +def _transform_measure(input_vals, completed, failed, measure, path, transit_scale_factor=100.0): + """ + Transforms skim values based on measure type, path, and trip completion statistics. + This is applied to a single 2D slice (one time period). + + Parameters: + ----------- + input_vals : ndarray (2D) + The new observed values for a single time period from the partial skim. + completed : ndarray (2D) + Count of completed trips for each OD pair for the current time period (2D slice). + failed : ndarray (2D) + Count of failed trips for each OD pair for the current time period (2D slice). + measure : str + The measure name (e.g., "IWAIT", "TOTIVT"). + path : str + The mode path (e.g., "SOV", "WLK_TRN_WLK", "TNC_SINGLE_UBER", "RH_SOLO"). + + Returns: + -------- + tuple (mask, vals, to_cancel) + mask: Boolean array (2D) indicating which cells to update with `vals`. + vals: Values (1D, matching mask.sum()) to assign to the masked cells. + to_cancel: Boolean array (2D) indicating cells to zero out due to high failure rate (for IVT/TOTIVT). + Returns None for other measures. + """ + # Basic mask - where there are completed trips and valid input values + valid = ~np.isnan(input_vals) + completed_mask = (completed > 0) + basic_mask = valid & completed_mask + # Handle negative values - seems reasonable to treat as 0 + negative = (input_vals < 0) + if np.any(negative): + logger.debug(f"Found {np.sum(negative)} negative values in input_vals for measure {measure} and path {path} in one period. Setting them to 0.") + input_vals[negative] = 0.0 + + measures_is_scaled_for_transit = {"TOTIVT", "IVT", "WACC", "IWAIT", "XWAIT", "WAUX", "WEGR", "DTIM", "FERRYIVT", "KEYIVT", "FAR"} + # Apply 100x scaling for specific measures only if the mode is NOT TNC/RH + scaling = transit_scale_factor if measure in measures_is_scaled_for_transit and not (path.startswith("TNC_") or path.startswith("RH_")) else 1.0 + + + # Handle measures that need scaling + if measure in measures_is_scaled_for_transit: + return basic_mask, input_vals[basic_mask] * scaling, None + + # Handle travel time measures with penalty logic (only TOTIVT/IVT) for NON-TNC/RH + elif measure in ["TOTIVT", "IVT"]: + # The penalty logic applies only to non-TNC/RH modes (like transit) + + if path.startswith("TNC_") or path.startswith("RH_"): + # TNC/RH TOTIVT/IVT is handled by the interpolation logic in post-processing + + is_tnc_rh = path.startswith("TNC_") or path.startswith("RH_") + + if is_tnc_rh: + return basic_mask, input_vals[basic_mask], None + else: + # Apply penalty logic for non-TNC/RH modes (like transit) + to_cancel = (failed > 5) & (failed > (1 * completed)) + + # To allow: NOT canceled, NOT penalized, valid input, and some completed trips + to_allow = valid & completed_mask & ~to_cancel + + # Prepare result values for cells to update + result_vals = np.zeros_like(input_vals[basic_mask]) # Only allocate for cells covered by basic_mask + + # Use regular values where allowed (within the basic_mask) + mask_allow_subset = to_allow[basic_mask] # Subset of basic_mask that is also to_allow + if mask_allow_subset.any(): + result_vals[mask_allow_subset] = input_vals[basic_mask][mask_allow_subset] * scaling + + # The update mask is the basic_mask itself, but the values are calculated based on penalty/allowance + return basic_mask, result_vals, to_cancel + + + # Handle DIST (simple assignment where completed > 0) + elif measure == "DIST": + # TNC DDIST interpolation is handled in post-processing + if path.startswith("TNC_") or path.startswith("RH_"): + # For TNC/RH, DIST is handled by interpolation + return np.zeros_like(input_vals, dtype=bool), np.array([]), None + else: + # For others, just apply direct value where completed > 0 + return basic_mask, input_vals[basic_mask], None + + + # Handle non-scaled measures (simple assignment where completed > 0) + # Includes COST, TOLLs, FAR, REJECTIONPROB etc. + # TNC FAR and REJECTIONPROB are handled in post-processing + elif measure not in ["REJECTIONPROB", "FAR"]: # Exclude measures handled in post-processing + if path.startswith("TNC_") or path.startswith("RH_"): + if measure == "COST": + # For TNC/RH COST, simple assignment where completed > 0 + return basic_mask, input_vals[basic_mask], None + else: + # Other non-excluded measures for TNC/RH (e.g. TOLLs) - simple assignment where completed > 0 + return basic_mask, input_vals[basic_mask], None + + else: + # For non-TNC/RH, simple assignment where completed > 0 + return basic_mask, input_vals[basic_mask], None + + + # Default case (e.g., REJECTIONPROB, FAR - handled in post-processing) + # Should not be reached for measures intended to be processed here. + logger.warning(f"Measure {measure} for path {path} fell through _transform_measure logic.") + return np.zeros_like(input_vals, dtype=bool), np.array([]), None + +def _merge_one_zarr_measure(partialSkims, skims_ds, path, measure, timePeriods, completed_3d, failed_3d): + """ + Merges a single measure's data for a given path and all time periods + from OMX partial skims into a Zarr DataArray, applying measure-specific transformations. + + Parameters: + ----------- + partialSkims : omx.open_file object + The OMX file containing the partial skims from BEAM. + skims_ds : xarray.Dataset + The target Zarr dataset containing all skims. + path : str + The mode path (e.g., "SOV", "WLK_TRN_WLK"). + measure : str + The measure name (e.g., "TOTIVT", "DIST"). + timePeriods : list of str + List of time period names. + completed_3d : ndarray (3D) + Completed trip counts for this path across all periods [zones, zones, periods]. + failed_3d : ndarray (3D) + Failed trip counts for this path across all periods [zones, zones, periods]. + + Returns: + -------- + tuple (path, measure_name) + The path and measure name of the skim that was processed. + """ + target_var_name = f"{path}_{measure}" # e.g., SOV_TOTIVT + + # Ensure the target variable exists in skims_ds + # Use SOV_TRIPS shape/coords as a default if target var doesn't exist + if target_var_name not in skims_ds.data_vars: + logger.warning(f"Target Zarr variable '{target_var_name}' not found. Creating with SOV_TRIPS shape.") + try: + zarr_shape = skims_ds['SOV_TRIPS'].shape + zarr_coords = skims_ds['SOV_TRIPS'].coords + zarr_dims = skims_ds['SOV_TRIPS'].dims + skims_ds[target_var_name] = xr.DataArray( + np.zeros(zarr_shape, dtype=np.float32), + coords=zarr_coords, + dims=zarr_dims, + name=target_var_name + ) + except KeyError: + logger.error(f"Cannot create variable {target_var_name}: SOV_TRIPS not found to get shape/coords.") + return None, None # Indicate skipping + target_da = skims_ds[target_var_name] + + + # Iterate through each time period slice + for tpIdx, tp in enumerate(timePeriods): + # Construct the key for the partial skims in OMX format + partial_key = f"{path}_{measure}__{tp}" + + if partial_key in partialSkims: + try: + input_vals_tp = partialSkims[partial_key][:] # Get 2D numpy array for this period + + # Ensure target DA slice exists for this period + if target_da.ndim < 3 or target_da.shape[-1] <= tpIdx: + logger.warning(f"Target Zarr variable {target_var_name} does not have enough time period dimensions for period {tp} (index {tpIdx}). Skipping merge for this period.") + continue # Skip this period + + # Ensure shape compatibility + if input_vals_tp.shape != target_da.shape[:2]: + logger.warning(f"Shape mismatch for partial skim {partial_key}. Expected {target_da.shape[:2]}, got {input_vals_tp.shape}. Skipping merge for this period.") + continue + + # Get the 2D completed/failed slices for the current time period + completed_tp = completed_3d[:, :, tpIdx] + failed_tp = failed_3d[:, :, tpIdx] + + # Apply transformations using the helper function + # _transform_measure expects 2D completed/failed arrays + mask_update, vals_update, to_cancel_tp = _transform_measure( + input_vals_tp, completed_tp, failed_tp, measure, path # Pass path here + ) + + # Apply updates to the Zarr DataArray slice using the mask + current_slice = target_da.data[:, :, tpIdx] + if mask_update.any(): + current_slice[mask_update] = vals_update + + + # Handle cancellations for IVT/TOTIVT + if to_cancel_tp is not None and to_cancel_tp.any(): + cancellation_count = to_cancel_tp.sum() + current_slice[to_cancel_tp] = 0.0 # Set canceled values to 0 + logger.debug(f" Canceled {cancellation_count} ODs for {partial_key}") + + # Update the slice in the DataArray's data + target_da.data[:, :, tpIdx] = current_slice + + except Exception as e: + logger.error(f"Error processing skim slice {partial_key}: {e}") + # Continue to the next time period + + else: + logger.debug(f"Partial skims missing key {partial_key}. Skipping merge for this period.") + + # Note: SOV/HOV copying logic removed from here - handled by copy_skims_for_unobserved_modes + + return path, measure + + +def _merge_zarr_trip_counts(allSkims, path, completed, failed): + key_completed = f"{path}_TRIPS" + key_failed = f"{path}_FAILURES" + any_observed = (completed + failed) > 0 + # Ensure keys exist before attempting to access .data + if key_completed in allSkims and key_failed in allSkims: + prev_completed = np.nan_to_num(allSkims[key_completed].data) + prev_failed = np.nan_to_num(allSkims[key_failed].data) + logger.info(f"For {path} previously had {prev_completed[any_observed].sum():.0f} completed trips and {prev_failed[any_observed].sum():.0f} failed trips") + prev_completed[any_observed] = 0.5 * prev_completed[any_observed] + completed[any_observed] + prev_failed[any_observed] = 0.5 * prev_completed[any_observed] + failed[any_observed] + allSkims[key_completed].data[:] = prev_completed # Use [:] to modify data in place + allSkims[key_failed].data[:] = prev_failed # Use [:] to modify data in place + logger.info(f"Now we have {prev_completed[any_observed].sum():.0f} completed trips and {prev_failed[any_observed].sum():.0f} failed trips") + else: + logger.warning(f"Skipping trip counts merge for {path} as {key_completed} or {key_failed} does not exist in target skims file") + + + +def _merge_zarr_skim(partialSkims, skims_da, completed_failed_dict, timePeriods): + """ + Merges a single measure's data for all time periods from OMX partial skims + into a Zarr DataArray, applying measure-specific transformations. + + Parameters: + ----------- + partialSkims : omx.open_file object + The OMX file containing the partial skims from BEAM. + skims_da : xarray.DataArray + The target Zarr DataArray for the current measure (e.g., skims['SOV_TOTIVT']). + Expected dimensions: (origin, destination, time_period). + completed_failed_dict : dict + Dictionary containing completed and failed trip counts aggregated across + all time periods for each mode path. Keyed by mode path, values are + [completed_trips_3d_array, failed_trips_3d_array]. + timePeriods : list of str + List of time period names. + + Returns: + -------- + tuple (path, measure_name) + The path and measure name of the skim that was processed. + """ + # Extract path and measure name from the DataArray name + # Expected name format: "PATH_MEASURE" (e.g., "SOV_TOTIVT") + path_measure_name = skims_da.name + parts = path_measure_name.rsplit('_', 1) + if len(parts) != 2: + logger.warning(f"Skipping skim '{path_measure_name}' due to unexpected name format.") + return None, None # Indicate skipping + + path, measure_name = path_measure_name.rsplit('_', 1) # Re-parse to handle names like WLK_TRN_WLK_TOTIVT + + # Skip TNC-specific measures that are handled in post-processing + if path.startswith("TNC") and measure_name in ["REJECTIONPROB", "IWAIT", "DDIST", "TOTIVT"]: + # These will be calculated in the post-processing step + return path, measure_name # Indicate processed, but no data merged here + + # Get the completed and failed trips for the current mode path + # These are 3D arrays [zones, zones, time periods] + completed_3d, failed_3d = completed_failed_dict.get(path, (None, None)) + + if completed_3d is None or failed_3d is None: + logger.debug(f"No completed/failed trip data found for mode path {path}. Skipping merge for {path_measure_name}.") + return path, measure_name # Indicate processed, but no data merged + + weighted_avgs = {} + cancellation_counts = {} + + # Iterate through each time period slice in the DataArray + for tpIdx, tp in enumerate(timePeriods): + # Construct the key for the partial skims in OMX format + partial_key = f"{path}_{measure_name}__{tp}" + + # Get the 2D completed/failed slices for the current time period + completed_tp = completed_3d[:, :, tpIdx] + failed_tp = failed_3d[:, :, tpIdx] + + if partial_key in partialSkims: + try: + input_vals_tp = partialSkims[partial_key][:] # Get 2D numpy array for this period + + # Apply transformations using the helper function + # _transform_measure expects 2D completed/failed arrays + mask_update, vals_update, to_cancel_tp = _transform_measure( + input_vals_tp, completed_tp, failed_tp, measure_name, path + ) + + # Apply updates to the Zarr DataArray slice using the mask + if mask_update.any(): + # Use .data for direct, in-place modification + current_slice = skims_da.data[:, :, tpIdx] + + # Calculate weighted averages for logging *before* update + # Use completed trips for the weight + weights = completed_tp[mask_update] + before_vals = current_slice[mask_update] + after_vals = vals_update + + if np.sum(weights) > 0: + weighted_avg_before = np.average(before_vals, weights=weights) + weighted_avg_after = np.average(after_vals, weights=weights) + else: + weighted_avg_before = np.nan + weighted_avg_after = np.nan + weighted_avgs[tp] = (weighted_avg_before, weighted_avg_after) + + # Apply the updated values + current_slice[mask_update] = after_vals + skims_da.data[:, :, tpIdx] = current_slice # Ensure changes are reflected (might be redundant with [:] but safer) + + + # Handle cancellations for IVT/TOTIVT + if to_cancel_tp is not None and to_cancel_tp.any(): + cancellation_counts[tp] = to_cancel_tp.sum() + current_slice = skims_da.data[:, :, tpIdx] + current_slice[to_cancel_tp] = 0.0 # Set canceled values to 0 + skims_da.data[:, :, tpIdx] = current_slice # Ensure changes reflected + + except Exception as e: + logger.error(f"Error processing skim slice {partial_key}: {e}") + # Continue to the next time period + + else: + logger.debug(f"Partial skims missing key {partial_key}. Skipping merge for this period.") + + # Log weighted averages summary if any updates occurred + if weighted_avgs: + summary = "; ".join( + f"{tp}: before={before:.2f}, after={after:.2f}" + for tp, (before, after) in weighted_avgs.items() + if not np.isnan(before) or not np.isnan(after) + ) + if summary: + logger.info(f"Weighted average update for {path_measure_name} (by completed trips): {summary}") + else: + logger.debug(f"No weighted average updates for {path_measure_name} (no completed trips in updated cells).") + + # Log cancellation summary if any occurred + if cancellation_counts: + summary = "; ".join( + f"{tp}: {count} ODs" for tp, count in cancellation_counts.items() if count > 0 + ) + if summary: + logger.info(f"Canceled ODs for {path_measure_name}: {summary}") + + # Note: SOV/HOV copying logic removed from here. + # Note: TNC specific logic removed from here. + + return path, measure_name + + +def _handle_transit_mode_availability(skims_ds, timePeriods): + """ + Handles transit mode availability by checking specific transit modes. + + For OD pairs with 50+ successful general transit trips, any specific transit mode + with 0 successful trips will be marked as unavailable (TOTIVT = 0). + Only marks OD pairs where TOTIVT is currently > 0 or np.nan. + Operates directly on the Zarr dataset. + + Parameters: + ----------- + skims_ds : xarray.Dataset + The target Zarr dataset containing all skims. + timePeriods : list of str + List of time period names. + """ + # General transit mode + general_transit_path = "WLK_TRN_WLK" + + # List of specific transit modes to check + specific_transit_modes = [ + "WLK_LOC_WLK", # Local bus + "WLK_HVY_WLK", # Heavy rail + "WLK_COM_WLK", # Commuter rail + "WLK_EXP_WLK", # Express bus + # Add any other specific transit modes as needed + ] + + # Create a summary table header using logger + logger.info(f"{'=' * 80}") + logger.info(f"Transit Mode Availability Analysis") + logger.info(f"{'=' * 80}") + logger.info( + f"{'Period':<6} | {'Mode':<12} | {'ODs w/50+ Transit':<18} | {'ODs w/0 Mode Trips':<18} | {'Changed':<10} | {'% Changed':<10}") + logger.info(f"{'-' * 6}-+-{'-' * 12}-+-{'-' * 18}-+-{'-' * 18}-+-{'-' * 10}-+-{'-' * 10}") + + tp_to_idx = {tp: idx for idx, tp in enumerate(timePeriods)} + + # Process each time period + for tp in timePeriods: + tp_idx = tp_to_idx[tp] + + # Check if general transit trip counts exist + general_trips_key = f"{general_transit_path}_TRIPS" # Zarr key format + general_trips_da = skims_ds.get(general_trips_key) + + if general_trips_da is None: + logger.info( + f"{tp:<6} | {'ALL MODES':<12} | {'NO DATA':<18} | {'NO DATA':<18} | {'NO DATA':<10} | {'N/A':<10}") + continue + + # Find OD pairs with at least 50 successful transit trips for this period slice + general_transit_trips = general_trips_da.data[:, :, tp_idx] # Get 2D slice + mask_significant_transit = general_transit_trips >= 50 + + significant_count = np.sum(mask_significant_transit) + + if not np.any(mask_significant_transit): + # No significant transit trips for this period, nothing to check + logger.info(f"{tp:<6} | {'ALL MODES':<12} | {0:<18} | {0:<18} | {0:<10} | {0:<10.1f}") + continue + + # Check each specific transit mode + for mode in specific_transit_modes: + # Get trip counts for this mode for this period slice + mode_trips_key = f"{mode}_TRIPS" # Zarr key format + mode_trips_da = skims_ds.get(mode_trips_key) + + if mode_trips_da is None: + logger.info( + f"{tp:<6} | {mode:<12} | {significant_count:<18} | {'NO DATA':<18} | {'NO DATA':<10} | {'N/A':<10}") + continue + + mode_trips = mode_trips_da.data[:, :, tp_idx] # Get 2D slice + + # Find OD pairs where general transit has 50+ trips but this mode has 0 + mask_mode_unused = mask_significant_transit & (mode_trips == 0) + + unused_count = np.sum(mask_mode_unused) + + # Mark this mode as unavailable for these OD pairs by setting TOTIVT = 0 + totivt_key = f"{mode}_TOTIVT" # Zarr key format + totivt_da = skims_ds.get(totivt_key) + changed_count = 0 + + if totivt_da is not None and unused_count > 0: + # Get current TOTIVT values slice + current_totivt_slice = totivt_da.data[:, :, tp_idx] + + # Only mark as unavailable if current TOTIVT is > 0 or np.nan within the mask_mode_unused area + mask_to_change = mask_mode_unused & ((current_totivt_slice > 0) | np.isnan(current_totivt_slice)) + changed_count = np.sum(mask_to_change) + # Update values directly using .data slice + if changed_count > 0: + current_totivt_slice[mask_to_change] = 0.0 + totivt_da.data[:, :, tp_idx] = current_totivt_slice # Ensure changes are reflected -def merge_current_omx_od_skims(all_skims_path, previous_skims_path, beam_output_dir, settings): + percent_changed = (changed_count / significant_count) * 100 if significant_count > 0 else 0 + + logger.info( + f"{tp:<6} | {mode:<12} | {significant_count:<18} | {unused_count:<18} | {changed_count:<10} | {percent_changed:<10.1f}") + + logger.info(f"{'=' * 80}") + + +def _consolidate_tnc_data_zarr(partialSkims, skims_ds, timePeriods, completed_failed_dict_providers, settings): + """ + Consolidates skims from multiple TNC providers (e.g., TNC_SINGLE_UBER, TNC_SINGLE_LYFT) + into single consolidated skims (e.g., RH_SOLO, RH_SHARED) within the Zarr dataset. + Reads from partialSkims (OMX) and writes to skims_ds (Zarr). + + Parameters: + ----------- + partialSkims : omx.open_file object + The OMX file containing the partial skims from BEAM (provider-specific). + skims_ds : xarray.Dataset + The target Zarr dataset (main skims file). Consolidated data is written here. + timePeriods : list of str + List of time period names. + completed_failed_dict_providers : dict + Dictionary containing completed and failed trip counts (3D arrays) + for each TNC *provider* mode (e.g., TNC_SINGLE_UBER, TNC_POOLED_LYFT). + Used for weighted averages. + settings : dict + Settings dictionary. + + Returns: + -------- + dict + A dictionary containing consolidated completed/failed trip counts (3D arrays) + for the RH_SOLO and RH_SHARED modes. + """ + logger.info("Starting TNC fleet consolidation from OMX to Zarr...") + + all_partial_vars = partialSkims.list_matrices() + tp_to_idx = {tp: idx for idx, tp in enumerate(timePeriods)} + # Get Zarr shape/coords from an existing 3D variable + try: + zarr_shape = skims_ds['SOV_TRIPS'].shape + zarr_coords = skims_ds['SOV_TRIPS'].coords + zarr_dims = skims_ds['SOV_TRIPS'].dims + if zarr_shape[-1] != len(timePeriods): + logger.warning(f"Zarr time period dimension ({zarr_shape[-1]}) does not match settings periods ({len(timePeriods)}). Using Zarr shape.") + # Update tp_to_idx to match Zarr if necessary, or raise error. Let's assume settings is correct. + # For safety, ensure the Zarr has enough space for all periods in settings. + if zarr_shape[-1] < len(timePeriods): + logger.error(f"Zarr shape {zarr_shape} has fewer time periods than in settings ({len(timePeriods)}). Cannot consolidate.") + return {} # Indicate failure or no consolidation + # If Zarr has more periods, we only populate those listed in settings. + except KeyError: + logger.error("Zarr dataset does not contain 'SOV_TRIPS'. Cannot determine shape and coordinates for new variables.") + return {} # Indicate failure or no consolidation + except Exception as e: + logger.error(f"Error getting Zarr shape/coords: {e}. Cannot consolidate.") + return {} + + + # Group TNC provider variables from partialSkims by (base_mode, measure) + grouped_sources = {} # Key: (base_mode, measure), Value: list of (provider, period, omx_key) + + for omx_key in all_partial_vars: + # Split key into measure_part and period (e.g., TNC_SINGLE_UBER_IWAIT__AM -> TNC_SINGLE_UBER_IWAIT, AM) + parts = omx_key.split('__') + if len(parts) != 2: + continue # Skip keys not in PATH_MEASURE__PERIOD format + + measure_part = parts[0] # e.g. TNC_SINGLE_UBER_IWAIT + period = parts[1] # e.g. AM + + if not measure_part.startswith("TNC_") or period not in timePeriods: + continue # Only process TNC keys for relevant periods + + # Parse measure_part: TNC_BASEMODE_PROVIDER_MEASURE or TNC_BASEMODE_MEASURE + measure_part_parts = measure_part.split('_') + if len(measure_part_parts) < 3: + logger.debug(f"Skipping TNC key with unexpected format (too few parts): {omx_key}") + continue + + tnc_prefix = measure_part_parts[0] # Should be "TNC" + base_mode = measure_part_parts[1] # e.g. "SINGLE", "POOLED" + # Check if base_mode maps to a consolidated RH mode + if f"{tnc_prefix}_{base_mode}" not in TNC_CONSOLIDATION_MAP: + logger.debug(f"Skipping TNC key with unknown base mode pattern: {omx_key}") + continue + + # Identify provider and measure. The first part after TNC_BASEMODE that is NOT a known measure prefix + # is likely part of the provider name, UNLESS the measure name itself has underscores. + # Let's try to find the measure name by splitting from the right, checking against common measures. + measure = None + provider_parts = [] + remaining_parts = list(measure_part_parts[2:]) # Parts after TNC_BASEMODE + + # Reverse the remaining parts and check if the rightmost form a known measure + reversed_remaining = remaining_parts[::-1] + for i in range(len(reversed_remaining)): + potential_measure_parts = reversed_remaining[:i+1][::-1] # e.g., ['IWAIT'] or ['KEY', 'IVT'] + potential_measure = "_".join(potential_measure_parts) + + # Check against common measures or specific ones with underscores + if potential_measure in ["TRIPS", "FAILURES", "REJECTIONPROB", "IWAIT", "FAR", "DDIST", "TOTIVT"]: + measure = potential_measure + provider_parts = remaining_parts[:len(remaining_parts) - (i+1)] # Remaining parts are the provider + break # Found the measure + + if measure is None: + logger.warning(f"Could not identify measure name in TNC key: {omx_key}. Skipping.") + continue + + provider = "_".join(provider_parts) if provider_parts else "DEFAULT" # Use "DEFAULT" if no provider part + base_mode_key = f"{tnc_prefix}_{base_mode}" # e.g. "TNC_SINGLE" + + if (base_mode_key, measure) not in grouped_sources: + grouped_sources[(base_mode_key, measure)] = [] + + grouped_sources[(base_mode_key, measure)].append((provider, period, omx_key)) + + logger.debug(f"Grouped TNC sources from OMX: {grouped_sources}") + + consolidated_completed_failed_dict = {} # Store consolidated counts for RH_ modes + + # First, consolidate TRIPS and FAILURES to get total counts per RH_ mode + for base_mode_key, measure in [(k[0], k[1]) for k in grouped_sources.keys()]: + if measure not in ["TRIPS", "FAILURES"]: + continue # Only process trips/failures first + + target_mode = TNC_CONSOLIDATION_MAP[base_mode_key] # e.g. "RH_SOLO" + target_var_name = f"{target_mode}_{measure}" + + # Ensure target variable exists in Zarr + if target_var_name not in skims_ds.data_vars: + logger.debug(f"Creating target variable '{target_var_name}' in Zarr dataset.") + skims_ds[target_var_name] = xr.DataArray( + np.zeros(zarr_shape, dtype=np.float32), + coords=zarr_coords, + dims=zarr_dims, + name=target_var_name + ) + target_da = skims_ds[target_var_name] + + # Initialize accumulation array for this measure + consolidated_data_3d = np.zeros(zarr_shape, dtype=np.float32) + + # Accumulate data from all relevant provider sources + for provider, period, omx_key in grouped_sources.get((base_mode_key, measure), []): + if omx_key in partialSkims: + try: + tp_idx = tp_to_idx[period] + input_vals_tp = np.nan_to_num(partialSkims[omx_key][:]) # Get 2D numpy array for this period + # Ensure shape compatibility before summing + if input_vals_tp.shape == zarr_shape[:2]: # Compare (zones, zones) + consolidated_data_3d[:, :, tp_idx] += input_vals_tp + else: + logger.warning(f"Shape mismatch for partial skim {omx_key}. Expected {zarr_shape[:2]}, got {input_vals_tp.shape}. Skipping sum.") + except Exception as e: + logger.error(f"Error reading partial skim {omx_key} for consolidation: {e}") + else: + logger.debug(f"Partial skim {omx_key} not found. Skipping.") + + # Assign the summed data to the target Zarr variable + target_da.data[:] = consolidated_data_3d + logger.debug(f"Summed {measure} into '{target_var_name}'. Total: {consolidated_data_3d.sum():.0f}") + + # Store the consolidated counts + if target_mode not in consolidated_completed_failed_dict: + # Initialize with zeros of the correct shape + consolidated_completed_failed_dict[target_mode] = [ + np.zeros(zarr_shape, dtype=np.float32), + np.zeros(zarr_shape, dtype=np.float32) + ] + + if measure == "TRIPS": + consolidated_completed_failed_dict[target_mode][0][:] = consolidated_data_3d + elif measure == "FAILURES": + consolidated_completed_failed_dict[target_mode][1][:] = consolidated_data_3d + + # Now, consolidate other measures using weighted average + for (base_mode_key, measure), sources in grouped_sources.items(): + if measure in ["TRIPS", "FAILURES", "REJECTIONPROB"]: + continue # Already handled TRIPS/FAILURES, REJECTIONPROB calculated later + + target_mode = TNC_CONSOLIDATION_MAP[base_mode_key] # e.g. "RH_SOLO" + target_var_name = f"{target_mode}_{measure}" + + # Ensure target variable exists in Zarr + if target_var_name not in skims_ds.data_vars: + logger.debug(f"Creating target variable '{target_var_name}' in Zarr dataset.") + skims_ds[target_var_name] = xr.DataArray( + np.zeros(zarr_shape, dtype=np.float32), # Initialize with zeros + coords=zarr_coords, + dims=zarr_dims, + name=target_var_name + ) + target_da = skims_ds[target_var_name] + + # Initialize accumulation arrays (weighted sum and sum of weights) for this measure + sum_weighted_values_3d = np.zeros(zarr_shape, dtype=np.float32) + sum_weights_3d = np.zeros(zarr_shape, dtype=np.float32) # Sum of completed trips + + # Accumulate data from all relevant provider sources + for provider, period, omx_key in sources: + if omx_key in partialSkims: + try: + tp_idx = tp_to_idx[period] + input_vals_tp = np.nan_to_num(partialSkims[omx_key][:]) # Get 2D data + + # Get corresponding TRIPS data for this provider and period from the provider dict + provider_mode_key = f"TNC_{base_mode}_{provider}" # e.g. TNC_SINGLE_UBER + if provider_mode_key in completed_failed_dict_providers: + provider_completed_3d, _ = completed_failed_dict_providers[provider_mode_key] + completed_tp = np.nan_to_num(provider_completed_3d[:, :, tp_idx]) # Get 2D completed trips for this provider/period + + # Ensure shapes match + if input_vals_tp.shape == zarr_shape[:2] and completed_tp.shape == zarr_shape[:2]: + # Handle negative values in input_vals_tp before weighting + input_vals_tp[input_vals_tp < 0] = 0.0 + + # Weighted value = measure_value * completed_trips + weighted_values_tp = input_vals_tp * completed_tp + sum_weighted_values_3d[:, :, tp_idx] += weighted_values_tp + sum_weights_3d[:, :, tp_idx] += completed_tp # Sum of completed trips are the weights + else: + logger.warning(f"Shape mismatch for partial skim {omx_key} or its trips data. Skipping weighted average contribution.") + else: + logger.debug(f"Completed trip data for provider mode {provider_mode_key} not found in provider dict. Skipping weighted average contribution for {omx_key}.") + except Exception as e: + logger.error(f"Error processing partial skim {omx_key} for weighted consolidation: {e}") + else: + logger.debug(f"Partial skim {omx_key} not found. Skipping.") + + # Calculate the weighted average for the entire 3D array + consolidated_data_3d = np.zeros(zarr_shape, dtype=np.float32) # Initialize with zeros + valid_weights_mask_3d = sum_weights_3d != 0 + # Use np.divide with 'where' and 'out' for safe division + np.divide(sum_weighted_values_3d, sum_weights_3d, + out=consolidated_data_3d, # Write results into the consolidated data array + where=valid_weights_mask_3d) + + # Assign the calculated data to the target Zarr variable + target_da.data[:] = consolidated_data_3d + mean_weighted_avg = np.nanmean(consolidated_data_3d[valid_weights_mask_3d]) if valid_weights_mask_3d.any() else np.nan + logger.debug(f"Calculated weighted average for {measure} into '{target_var_name}'. Mean (weighted): {mean_weighted_avg:.4f}") + + logger.info("Completed TNC fleet consolidation from OMX to Zarr.") + + # Return consolidated counts for use in post-processing + return consolidated_completed_failed_dict + + +# New function to transfer TNC provider data from OMX to Zarr without consolidation +def _transfer_tnc_provider_data_zarr(partialSkims, skims_ds, timePeriods, completed_failed_dict_providers, settings): + """ + Transfers provider-specific TNC skims from partialSkims (OMX) to skims_ds (Zarr). + Does NOT consolidate. This is for the case where consolidation is disabled. + + Parameters: + ----------- + partialSkims : omx.open_file object + The OMX file containing the partial skims from BEAM (provider-specific). + skims_ds : xarray.Dataset + The target Zarr dataset (main skims file). Provider data is written here. + timePeriods : list of str + List of time period names. + completed_failed_dict_providers : dict + Dictionary containing completed and failed trip counts (3D arrays) + for each TNC *provider* mode. Used by _merge_one_zarr_measure. + settings : dict + Settings dictionary. + + Returns: + -------- + list + List of Zarr variable names that were created/updated (TNC provider skims). + """ + logger.info("Starting TNC provider data transfer from OMX to Zarr...") + + all_partial_vars = partialSkims.list_matrices() + tp_to_idx = {tp: idx for idx, tp in enumerate(timePeriods)} + transferred_vars = set() + + # Group TNC provider variables from partialSkims by (mode, measure) (where mode is provider specific) + # Key: (mode, measure), Value: list of (period, omx_key) + grouped_sources = {} + + for omx_key in all_partial_vars: + # Split key into measure_part and period + parts = omx_key.split('__') + if len(parts) != 2: + continue # Skip keys not in PATH_MEASURE__PERIOD format + + measure_part = parts[0] # e.g. TNC_SINGLE_UBER_IWAIT + period = parts[1] # e.g. AM + + if not measure_part.startswith("TNC_") or period not in timePeriods: + continue # Only process TNC keys for relevant periods + + # Parse measure_part to get the full provider mode name and the measure + measure = None + mode = None + + measure_part_parts = measure_part.split('_') + if len(measure_part_parts) < 3: + logger.debug(f"Skipping TNC key with unexpected format (too few parts): {omx_key}") + continue + + # Assume format TNC_BASEMODE_PROVIDER_MEASURE or TNC_BASEMODE_MEASURE + # Let's try to find the measure name by splitting from the right + remaining_parts = list(measure_part_parts[2:]) + reversed_remaining = remaining_parts[::-1] + for i in range(len(reversed_remaining)): + potential_measure_parts = reversed_remaining[:i+1][::-1] + potential_measure = "_".join(potential_measure_parts) + + if potential_measure in ["TRIPS", "FAILURES", "REJECTIONPROB", "IWAIT", "XWAIT", "WACC", + "WAUX", "WEGR", "DTIM", "DDIST", "TOTIVT", "FAR", "COST"] or \ + potential_measure in TNC_MEASURES_WITH_UNDERSCORES: + measure = potential_measure + mode_parts = measure_part_parts[:len(measure_part_parts) - (i+1)] # Parts before the measure + mode = "_".join(mode_parts) # e.g., TNC_SINGLE_UBER + break + + if mode is None or measure is None: + logger.warning(f"Could not parse TNC provider mode or measure from key: {omx_key}. Skipping.") + continue + + if (mode, measure) not in grouped_sources: + grouped_sources[(mode, measure)] = [] + + grouped_sources[(mode, measure)].append((period, omx_key)) + + + # Get Zarr shape/coords from an existing 3D variable + try: + zarr_shape = skims_ds['SOV_TRIPS'].shape + zarr_coords = skims_ds['SOV_TRIPS'].coords + zarr_dims = skims_ds['SOV_TRIPS'].dims + if zarr_shape[-1] < len(timePeriods): + logger.error(f"Zarr shape {zarr_shape} has fewer time periods than in settings ({len(timePeriods)}). Cannot transfer TNC data.") + return [] # Indicate failure + except KeyError: + logger.error("Zarr dataset does not contain 'SOV_TRIPS'. Cannot determine shape and coordinates for new TNC provider variables.") + return [] # Indicate failure + except Exception as e: + logger.error(f"Error getting Zarr shape/coords: {e}. Cannot transfer TNC data.") + return [] + + + # Transfer data for each mode/measure pair + for (mode, measure), sources in grouped_sources.items(): + target_var_name = f"{mode}_{measure}" # e.g. TNC_SINGLE_UBER_IWAIT + + # Ensure target variable exists in Zarr + if target_var_name not in skims_ds.data_vars: + logger.debug(f"Creating target variable '{target_var_name}' in Zarr dataset.") + # Check if the mode exists in the completed_failed_dict_providers + # This is needed to get the shape. Fallback to SOV_TRIPS shape. + try: + # Use the shape from the provider counts if available + provider_completed_shape = completed_failed_dict_providers[mode][0].shape + if provider_completed_shape[:2] != zarr_shape[:2] or provider_completed_shape[-1] < len(timePeriods): + logger.warning(f"Provider completed counts shape {provider_completed_shape} for mode {mode} is incompatible with Zarr shape {zarr_shape} or settings periods {len(timePeriods)}. Falling back to SOV_TRIPS shape.") + raise KeyError # Fallback + current_zarr_shape = provider_completed_shape + except KeyError: + # Fallback to default Zarr shape if provider shape isn't reliable + current_zarr_shape = zarr_shape + logger.debug(f"Using default Zarr shape {current_zarr_shape} for mode {mode}") + + skims_ds[target_var_name] = xr.DataArray( + np.zeros(current_zarr_shape, dtype=np.float32), # Initialize with zeros + coords=zarr_coords, # Use SOV_TRIPS coords as a default + dims=zarr_dims, # Use SOV_TRIPS dims as a default + name=target_var_name + ) + target_da = skims_ds[target_var_name] + transferred_vars.add(target_var_name) + + # Get completed/failed counts for this mode from the provider dict + completed_3d, failed_3d = completed_failed_dict_providers.get(mode, (None, None)) + if completed_3d is None or failed_3d is None: + logger.warning(f"Completed/failed trip data for mode {mode} not found in provider dict. Skipping merge for {target_var_name}.") + continue # Cannot merge without counts + + # Merge data from all relevant periods + for period, omx_key in sources: + if omx_key in partialSkims: + try: + tp_idx = tp_to_idx[period] + # Ensure the target DA has enough dimensions for this period + if target_da.ndim < 3 or target_da.shape[-1] <= tp_idx: + logger.warning(f"Target Zarr variable {target_var_name} does not have enough time period dimensions for period {period} (index {tp_idx}). Skipping merge for this period.") + continue # Skip this period if target DA is too small + + input_vals_tp = partialSkims[omx_key][:] # Get 2D numpy array for this period + + # Ensure shape compatibility before merging + if input_vals_tp.shape != target_da.shape[:2]: + logger.warning(f"Shape mismatch for partial skim {omx_key}. Expected {target_da.shape[:2]}, got {input_vals_tp.shape}. Skipping merge for this period.") + continue + + # Apply transformations using the helper function _transform_measure + # _transform_measure expects 2D completed/failed arrays + completed_tp = completed_3d[:, :, tp_idx] + failed_tp = failed_3d[:, :, tp_idx] + + mask_update, vals_update, to_cancel_tp = _transform_measure( + input_vals_tp, completed_tp, failed_tp, measure, mode # Pass mode name here + ) + + # Apply updates to the Zarr DataArray slice using the mask + current_slice = target_da.data[:, :, tp_idx] + if mask_update.any(): + current_slice[mask_update] = vals_update + + # Handle cancellations for IVT/TOTIVT + if to_cancel_tp is not None and to_cancel_tp.any(): + cancellation_count = to_cancel_tp.sum() + current_slice[to_cancel_tp] = 0.0 # Set canceled values to 0 + logger.debug(f"Canceled {cancellation_count} ODs for {omx_key}") + + # Update the slice in the DataArray's data + target_da.data[:, :, tp_idx] = current_slice + + + except Exception as e: + logger.error(f"Error reading partial skim {omx_key} for transfer: {e}") + else: + logger.debug(f"Partial skim {omx_key} not found. Skipping.") + + logger.info("Completed TNC provider data transfer from OMX to Zarr.") + return list(transferred_vars) + + +def write_zarr_skim_as_omx(all_skims_path, settings, new_skim_name, exclude_tables=None): + """ + Write the skims from the Zarr format to an OMX format. + + Parameters + ---------- + all_skims_path : str + Path to the main skims Zarr store. + settings : dict + Settings dictionary, needed for 'region' and 'beam_local_input_folder'. + new_skim_name : str + Name of the new OMX skim file to be created (e.g., 'skims.omx'). + exclude_tables : list, optional + A list of variable names (strings) from the Zarr dataset + to exclude from being written to the OMX file. Defaults to None. + + Returns + ------- + str or None + The path to the newly created OMX file if successful, otherwise None. + """ + logger.info(f"Starting conversion of Zarr skims to OMX at {all_skims_path}") + + region = settings.get('region') + beam_input_dir = settings.get('beam_local_input_folder') + + if not region or not beam_input_dir: + logger.error("Settings 'region' or 'beam_local_input_folder' are not defined. Cannot determine output path.") + return None + + target_skims_path = os.path.join(beam_input_dir, region, new_skim_name) + + if exclude_tables is None: + exclude_tables = [] + + skims_ds = None # Initialize Zarr dataset variable + new_omx_file = None # Initialize OMX file variable + + try: + # Open the Zarr dataset + skims_ds = xr.open_zarr(all_skims_path) + logger.info(f"Opened Zarr skims file: {all_skims_path}") + + # --- Get Zone IDs and Time Periods from Zarr coordinates --- + # ActivitySim Zarr skims typically have 'otaz' and 'time_period' coordinates + zone_ids = None + if 'otaz' in skims_ds.coords: + # Ensure zone_ids are integers compatible with OMX mapping + zone_ids = skims_ds.coords['otaz'].values + # Convert to a basic list or array type that openmatrix likes, if necessary + # numpy array should be fine + logger.info(f"Retrieved {len(zone_ids)} zone IDs from Zarr coordinates.") + else: + logger.warning("Zarr skims file does not have 'otaz' coordinate. Zone mapping will NOT be added to OMX.") + + time_periods = [] + if 'time_period' in skims_ds.coords: + # Ensure time periods are strings for OMX keys + time_periods = [str(s) for s in skims_ds.time_period.values] + logger.info(f"Found {len(time_periods)} time periods in Zarr: {time_periods}") + else: + logger.warning("Zarr skims file does not have 'time_period' coordinate. 3D variables cannot be written with period suffixes.") + + # --- Prepare output file --- + logger.info(f"Target output OMX path: {target_skims_path}") + if os.path.exists(target_skims_path): + logger.info(f"Deleting existing file: {target_skims_path}") + try: + os.remove(target_skims_path) + except OSError as e: + logger.error(f"Error deleting existing file {target_skims_path}: {e}. Aborting.") + return None + + # Create the new OMX file using 'w' mode as we just deleted it + new_omx_file = omx.open_file(target_skims_path, 'w') + logger.info(f"Created new OMX file: {target_skims_path}") + + # --- Add Zone Mapping FIRST --- + if zone_ids is not None and len(zone_ids) > 0: + try: + # Add the 'zone_id' mapping to the OMX file + new_omx_file.create_mapping('zone_id', zone_ids, overwrite=True) + logger.info(f"Created 'zone_id' mapping in OMX file with {len(zone_ids)} zones.") + except Exception as e: + logger.error(f"Error creating zone mapping in OMX file: {e}.") + + # --- Write Matrices from Zarr to OMX --- + logger.info("Writing matrices to OMX file...") + written_count = 0 + for key in skims_ds.data_vars: # Iterate through data variables + if key in exclude_tables: + logger.info(f"Skipping excluded variable: {key}") + continue + + try: + data_array = skims_ds[key] + data = data_array.values # Load data into memory + logger.debug(f"Processing variable '{key}' with shape {data.shape}, dtype {data.dtype}.") + + if data_array.ndim == 2: + # Ensure shape matches expected (zones, zones) if zones were found + expected_shape_2d = (len(zone_ids), len(zone_ids)) if zone_ids is not None else None + if expected_shape_2d and data_array.shape != expected_shape_2d: + logger.warning(f"Shape mismatch for 2D variable '{key}': {data_array.shape} vs expected {expected_shape_2d}. Skipping.") + continue + + # Write 2D matrix directly + new_omx_file[key] = np.nan_to_num(data) + logger.debug(f" Wrote 2D matrix '{key}'") + written_count += 1 + + elif data_array.ndim == 3: + # Ensure shape matches expected (zones, zones, periods) if zones and periods were found + expected_shape_3d_prefix = (len(zone_ids), len(zone_ids)) if zone_ids is not None else None + if expected_shape_3d_prefix and data_array.shape[:2] != expected_shape_3d_prefix: + logger.warning(f"Shape prefix mismatch for 3D variable '{key}': {data_array.shape[:2]} vs expected {expected_shape_3d_prefix}. Skipping.") + continue + + if not time_periods or data_array.shape[-1] != len(time_periods): + logger.warning(f"Period dimension mismatch for 3D variable '{key}': {data_array.shape[-1]} vs expected {len(time_periods)}. Cannot write as OMX slices. Skipping.") + continue + + # Write slices for each time period + for t_idx, tp in enumerate(time_periods): + new_key = f"{key}__{tp}" # Standard OMX format for 3D -> 2D slices + slice_data = data[:, :, t_idx] + new_omx_file[new_key] = np.nan_to_num(slice_data) + # logger.debug(f" Wrote slice '{new_key}'") # Too verbose for debug + strsplit = key.rsplit('_',1) + new_omx_file[new_key].attrs['measure'] = strsplit[-1] + new_omx_file[new_key].attrs['timePeriod'] = tp + if len(strsplit) == 2: + new_omx_file[new_key].attrs['mode'] = strsplit[0] + written_count += 1 + logger.debug(f" Wrote {len(time_periods)} slices for 3D variable '{key}'") + + else: + logger.warning(f"Skipping variable '{key}' with unexpected dimension count: {data_array.ndim}") + + except Exception as e: + logger.error(f"Error writing variable '{key}' to OMX: {e}. Continuing with next variable.") + + logger.info(f"Finished writing {written_count} matrices to OMX file {target_skims_path}.") + + # Return the path on success + return target_skims_path + + except Exception as e: + logger.critical(f"An unexpected error occurred during Zarr to OMX conversion: {e}") + return None # Indicate failure + + finally: + # Ensure files are closed + if new_omx_file: + try: + new_omx_file.close() + logger.info("Closed OMX file.") + except Exception as e: + logger.error(f"Error closing OMX file {target_skims_path}: {e}") + + if skims_ds: + try: + skims_ds.close() + logger.info("Closed Zarr dataset.") + except Exception as e: + logger.error(f"Error closing Zarr dataset {all_skims_path}: {e}") + + + +def merge_current_zarr_od_skims(all_skims_path, beam_output_dir, settings, override=None): + """ + Merges current BEAM OMX skims into the main Zarr skims file. + Handles TNC consolidation if enabled. + """ + logger.info(f"Starting merge of current BEAM OMX skims into Zarr at {all_skims_path}") + + if override is None: + current_omx_skims_path = find_produced_od_skims(beam_output_dir, "omx") + else: + current_omx_skims_path = override + + if current_omx_skims_path is None or not os.path.exists(current_omx_skims_path): + logger.warning(f"No current OMX skims found at {current_omx_skims_path}. Skipping merge.") + # Check if the Zarr file exists; if not, ActivitySim won't run, so this is a failure state. + # Assume Zarr was created by asim_pre.create_zarr_skims + if not os.path.exists(all_skims_path): + logger.error(f"Target Zarr skims file not found at {all_skims_path} and no new OMX skims available. Cannot proceed.") + return None + try: + skims_ds = xr.open_zarr(all_skims_path) # Open existing for possible post-processing if it's a replan iteration + skims_ds.close() # Just check if it's readable + except Exception as e: + logger.error(f"Failed to open existing Zarr skims file at {all_skims_path}: {e}. Cannot proceed.") + return None + + logger.info("Existing Zarr skims file found, but no new OMX skims to merge. Proceeding with post-processing on existing Zarr if needed.") + partialSkims = None # Ensure partialSkims is None if file not found + + else: + try: + partialSkims = omx.open_file(current_omx_skims_path, mode='r') + logger.info(f"Opened partial skims file: {current_omx_skims_path}") + except Exception as e: + logger.error(f"Failed to open partial skims file {current_omx_skims_path}: {e}. Skipping merge.") + partialSkims = None # Ensure partialSkims is None if opening fails + if not os.path.exists(all_skims_path): + logger.error(f"Target Zarr skims file not found at {all_skims_path} and failed to open new OMX skims. Cannot proceed.") + return None # Indicate failure + + # Open the Zarr dataset in append mode to modify it in place + try: + skims_ds = xr.open_zarr(all_skims_path) + logger.info(f"Opened Zarr skims file: {all_skims_path}") + except Exception as e: + logger.error(f"Failed to open target Zarr skims file {all_skims_path}: {e}. Cannot proceed.") + if partialSkims: + partialSkims.close() + return None # Indicate failure + + + timePeriods = settings["periods"] + consolidate_tnc_fleets = settings.get('consolidate_tnc_fleets', True) + + # Step 1: Accumulate completed and failed trips from partial skims (all modes/providers) + completed_failed_dict_all = {} + if partialSkims: + completed_failed_dict_all = _accumulate_all_completed_failed_trips(partialSkims, timePeriods) + + for path, (completed, failed) in completed_failed_dict_all.items(): + _merge_zarr_trip_counts(skims_ds, path, completed, failed) + + + + # Step 2: Process TNC/RH skims (either consolidate or transfer) and run their specific post-processing + # This step handles creating/updating TNC_* or RH_* variables in skims_ds and running their post-processing. + # It needs the provider-level completed/failed counts if not consolidating. + tnc_modes_processed = set() # Keep track of modes handled by the TNC/RH logic + if partialSkims: + # If consolidating, _process_all_tnc_logic will use provider counts internally + # and run postprocessing on consolidated RH modes. + # If not consolidating, it will transfer provider data and run postprocessing + # on TNC_* provider modes. + try: + tnc_modes_processed = _process_all_tnc_logic(partialSkims, skims_ds, timePeriods, settings, completed_failed_dict_all) + except AttributeError as e: + logger.error(f"Failed to process TNC logic at {all_skims_path}: {e}") + else: + logger.warning("No new OMX skims found. Reconstructing TNC/RH trip counts from existing Zarr data for post-processing.") + tnc_modes_for_postprocess = set() + all_zarr_vars = list(skims_ds.data_vars) + if consolidate_tnc_fleets: + for target_mode in TNC_CONSOLIDATION_MAP.values(): + if f"{target_mode}_TRIPS" in all_zarr_vars: + tnc_modes_for_postprocess.add(target_mode) + else: + # Find existing TNC provider modes in Zarr + for var_name in all_zarr_vars: + if var_name.startswith("TNC_") and "_TRIPS" in var_name: + mode = var_name.rsplit('_TRIPS', 1)[0] # Extract mode name + tnc_modes_for_postprocess.add(mode) + + # Reconstruct completed_failed_dict for these modes from Zarr + completed_failed_dict_from_zarr = {} + for mode in tnc_modes_for_postprocess: + trips_key = f"{mode}_TRIPS" + failures_key = f"{mode}_FAILURES" + if trips_key in skims_ds.data_vars and failures_key in skims_ds.data_vars: + completed_failed_dict_from_zarr[mode] = ( + np.nan_to_num(skims_ds[trips_key].data), + np.nan_to_num(skims_ds[failures_key].data) + ) + if tnc_modes_for_postprocess: + # Run post-processing on existing TNC/RH data in Zarr + _postprocess_tnc_zarr(skims_ds, timePeriods, settings, completed_failed_dict_from_zarr, use_rh_modes=consolidate_tnc_fleets) + else: + logger.info("No existing TNC/RH modes found in Zarr for post-processing.") + + + # Step 3: Process non-TNC/non-RH skims and run standard merging + if partialSkims: # Only merge if we have new partial skims + # Group partial skims by (path, measure) + grouped_partial_sources = {} # Key: (path, measure), Value: list of omx_key + + for omx_key in partialSkims.list_matrices(): + # Split key into measure_part and period + parts = omx_key.split('__') + if len(parts) != 2: + continue # Skip keys not in PATH_MEASURE__PERIOD format + + measure_part = parts[0] + period = parts[1] + + if period not in timePeriods: + continue + + # Determine path and measure + # Try parsing from right for standard measures + measure = None + path = None + measure_part_parts = measure_part.split('_') + reversed_parts = measure_part_parts[::-1] + + for i in range(len(reversed_parts)): + potential_measure_parts = reversed_parts[:i+1][::-1] + potential_measure = "_".join(potential_measure_parts) + potential_path_parts = measure_part_parts[:len(measure_part_parts) - (i+1)] + potential_path = "_".join(potential_path_parts) + + # Check if the potential measure is a common one + if potential_measure in ["TRIPS", "FAILURES", "REJECTIONPROB", "IWAIT", "XWAIT", "WACC", + "WAUX", "WEGR", "DTIM", "DDIST", "TOTIVT", "FAR", "COST", "DIST", "TIME", + "BTOLL", "VTOLL", "KEYIVT", "FERRYIVT", + ]: + measure = potential_measure + path = potential_path + break + + if path is None or measure is None: + logger.debug(f"Could not parse path/measure from key {omx_key}. Skipping.") + continue + + # Skip TNC/RH paths if they were handled by consolidation/transfer + if path.startswith("TNC_") or path.startswith("RH_"): + # If consolidation was on, TNC_* and RH_* were handled by _process_all_tnc_logic + # If consolidation was off, TNC_* were handled by _process_all_tnc_logic + # In either case, skip them here. + if consolidate_tnc_fleets: + # If consolidating, skip original TNC provider paths and the new RH paths + # Need to check if 'path' is an original provider path or a consolidated RH path + is_provider_path = path.startswith("TNC_") # e.g. TNC_SINGLE_UBER + is_consolidated_path = path in TNC_CONSOLIDATION_MAP.values() # e.g. RH_SOLO + if is_provider_path or is_consolidated_path: + logger.debug(f"Skipping OMX key {omx_key} for generic merge: Path {path} is TNC/RH and handled elsewhere (consolidation enabled).") + continue + else: + # If not consolidating, skip original TNC provider paths (handled by transfer) + if path.startswith("TNC_"): + logger.debug(f"Skipping OMX key {omx_key} for generic merge: Path {path} is TNC and handled elsewhere (consolidation disabled).") + continue + + # Add to the grouping for non-TNC/non-RH paths + if (path, measure) not in grouped_partial_sources: + grouped_partial_sources[(path, measure)] = [] + grouped_partial_sources[(path, measure)].append(omx_key) + + + logger.info(f"Merging {len(grouped_partial_sources)} non-TNC/non-RH (path, measure) groups from partial skims.") + + # Process each non-TNC/non-RH (path, measure) group + processed_non_tnc_vars = set() + for (path, measure), omx_keys in grouped_partial_sources.items(): + # Ensure we have completed/failed counts for this path + if path not in completed_failed_dict_all: + logger.warning(f"Completed/failed trip data for path {path} not found. Skipping merge for measure {measure}.") + continue + + completed_3d, failed_3d = completed_failed_dict_all[path] + + # Call the generic merge function for this measure/path + processed_path, processed_measure = _merge_one_zarr_measure( + partialSkims, skims_ds, path, measure, timePeriods, completed_3d, failed_3d + ) + if processed_path is not None: + processed_non_tnc_vars.add(f"{processed_path}_{processed_measure}") + + logger.info(f"Completed merging {len(processed_non_tnc_vars)} non-TNC/non-RH variables.") + + # Step 4: Handle transit mode availability (uses TRIPS/FAILURES data already in skims_ds) + # This should run regardless of whether new OMX skims were merged, to ensure consistency. + logger.info("Applying transit mode availability rules.") + _handle_transit_mode_availability(skims_ds, timePeriods) + + + # Step 5: Copy skims for unobserved modes (e.g., SOV -> HOV/SOVTOLL) + # This should run after the base skims (like SOV) are finalized. + # This should run regardless of whether new OMX skims were merged. + logger.info("Copying skims for unobserved modes based on mapping.") + mapping = settings.get('unobserved_skim_copy_map', {"SOV": ["SOVTOLL", "HOV2", "HOV2TOLL", "HOV3", "HOV3TOLL"]}) # Add setting + copy_skims_for_unobserved_modes(mapping, skims_ds) + + # Step 6: Save the updated Zarr dataset + logger.info(f"Started writing updated zarr skims to {all_skims_path}") + # Use mode='w' to overwrite with the updated data, consolidated=True for performance + # Ensure zarr_version=2 for compatibility + try: + skims_ds.to_zarr(all_skims_path, mode='w', consolidated=True, zarr_version=2) + logger.info("Completed writing zarr skims successfully.") + merge_successful = True # Indicate merge/post-processing was attempted successfully + except Exception as e: + logger.error(f"FAILED to write updated zarr skims to {all_skims_path}: {e}") + merge_successful = False # Indicate failure + + # Close the datasets + skims_ds.close() + if partialSkims: + partialSkims.close() + + # Return the path to the new OMX skims *if* a new one was found and processed. + # This is used by the caller to check if skims were updated. + if partialSkims is not None and merge_successful: + return current_omx_skims_path + else: # partialSkims is None or writing failed + logger.warning("No new OMX skims found, returning None.") + return None # No new skims to indicate success/failure for + + + +# New function to orchestrate TNC/RH processing +def _process_all_tnc_logic(partialSkims, skims_ds, timePeriods, settings, completed_failed_dict_all): + """ + Handles all TNC/RH related skim processing: + - If consolidate_tnc_fleets is True: Consolidates provider data -> RH_* in Zarr, then post-processes RH_*. + - If consolidate_tnc_fleets is False: Transfers provider data -> TNC_* in Zarr, then post-processes TNC_*. + + Parameters: + ----------- + partialSkims : omx.open_file object + The OMX file containing the partial skims from BEAM. + skims_ds : xarray.Dataset + The target Zarr dataset containing all skims. + timePeriods : list of str + List of time period names. + settings : dict + Settings dictionary. + completed_failed_dict_all : dict + Dictionary containing completed and failed trip counts (3D arrays) + for ALL modes/providers in the partial skims. + + Returns: + -------- + set + A set of Zarr variable names that were created/updated by this function (TNC provider or RH consolidated). + """ + consolidate_tnc_fleets = settings.get('consolidate_tnc_fleets', True) + processed_vars = set() + tnc_modes_to_postprocess = set() + completed_failed_dict_for_postprocess = {} # Counts for the modes that will be post-processed + + if consolidate_tnc_fleets: + logger.info("TNC fleet consolidation enabled.") + # Step 2a: Consolidate TNC provider data into RH_* variables in skims_ds + # This function also returns the consolidated RH counts + consolidated_completed_failed_dict = _consolidate_tnc_data_zarr( + partialSkims, skims_ds, timePeriods, completed_failed_dict_all, settings + ) + # Add consolidated RH modes to the list for post-processing + tnc_modes_to_postprocess.update(consolidated_completed_failed_dict.keys()) + # Use the consolidated counts for post-processing + completed_failed_dict_for_postprocess = consolidated_completed_failed_dict + + # The consolidated variables (RH_*) are now in skims_ds. + # We need to get their names to return them as handled. + for target_mode in consolidated_completed_failed_dict.keys(): + for measure in skims_ds.data_vars: # Iterate through ALL skims_ds vars to find the ones belonging to this mode + if measure.startswith(f"{target_mode}_"): + processed_vars.add(measure) + + + else: + logger.info("TNC fleet consolidation disabled. Transferring provider data.") + # Step 2a: Transfer TNC provider data into TNC_* variables in skims_ds + # This function returns the names of variables transferred. + transferred_vars = _transfer_tnc_provider_data_zarr( + partialSkims, skims_ds, timePeriods, completed_failed_dict_all, settings + ) + processed_vars.update(transferred_vars) + + # Identify the TNC provider modes that were transferred + # Iterate through the transferred variable names to find the modes + for var_name in transferred_vars: + parts = var_name.split('_') + if len(parts) >= 3 and parts[0] == "TNC": + # Assuming format TNC_BASEMODE_PROVIDER_MEASURE or TNC_BASEMODE_MEASURE + # Need to parse the mode name correctly again + measure_part = var_name.rsplit('_', 1)[0] # Remove measure at the end for 3D vars like RH_SOLO_TRIPS + if measure_part.endswith("__"): # Handle 2D vars like DIST__AM + measure_part = var_name.split('__')[0] + elif "_" in var_name.split("__")[0]: # Handle 3D vars + measure_part = var_name.split("__")[0] + + + # Find the mode name from the beginning of the measure_part + mode_name = None + measure_name_check = measure_part + while '_' in measure_name_check: + potential_mode = measure_name_check + potential_measure_end = measure_name_check.rsplit('_', 1)[-1] + if potential_measure_end.upper() in ["TRIPS", "FAILURES", "REJECTIONPROB", "IWAIT", "DDIST", "TOTIVT", "FAR"]: + # Found a potential measure end, the rest is the mode + mode_name = potential_mode.rsplit('_', 1)[0] + break + measure_name_check = potential_mode.rsplit('_', 1)[0] # Chop off the last part and try again + + if mode_name and mode_name.startswith("TNC_"): + tnc_modes_to_postprocess.add(mode_name) + # Use the provider counts for post-processing + if mode_name in completed_failed_dict_all: + completed_failed_dict_for_postprocess[mode_name] = completed_failed_dict_all[mode_name] + else: + logger.warning(f"Completed/failed counts not found for provider mode {mode_name} in completed_failed_dict_all. Cannot post-process.") + + + logger.info(f"Identified {len(tnc_modes_to_postprocess)} TNC provider modes for post-processing.") + + + # Step 2b: Run TNC/RH specific post-processing on the data now in skims_ds + # This function needs the completed/failed counts for the modes it's processing. + if tnc_modes_to_postprocess: + _postprocess_tnc_zarr( + skims_ds, timePeriods, settings, completed_failed_dict_for_postprocess, + use_rh_modes=consolidate_tnc_fleets + ) + else: + logger.info("No TNC/RH modes to post-process after consolidation/transfer step.") + + return processed_vars + +def merge_current_omx_od_skims(all_skims_path, beam_output_dir, settings): + """ + Merge current OMX skims from BEAM into the main skims file. + + Parameters + ---------- + all_skims_path : str + Path to the main skims file + beam_output_dir : str + Path to the BEAM output directory + settings : dict + Settings dictionary + + Returns + ------- + str + Path to the current skims file + """ skims = omx.open_file(all_skims_path, 'a') current_skims_path = find_produced_od_skims(beam_output_dir, "omx") partialSkims = omx.open_file(current_skims_path, mode='r') @@ -278,9 +2210,9 @@ def merge_current_omx_od_skims(all_skims_path, previous_skims_path, beam_output_ simplify(skims, timePeriod, path, False), path, timePeriod, vals) for (path, timePeriod, vals) in iterable] - discover_impossible_ods(result, skims) + discover_impossible_ods(result, skims, skims.list_matrices()) mapping = {"SOV": ["SOVTOLL", "HOV2", "HOV2TOLL", "HOV3", "HOV3TOLL"]} - copy_skims_for_unobserved_modes(mapping, skims) + copy_skims_for_unobserved_modes(mapping, skims, skims.list_matrices()) order = zone_order(settings, settings['start_year']) zone_id = np.arange(1, len(order) + 1) @@ -292,11 +2224,163 @@ def merge_current_omx_od_skims(all_skims_path, previous_skims_path, beam_output_ partialSkims.close() return current_skims_path +def trim_inaccessible_ods_zarr(all_skims_path, settings): + """ + Zero out inaccessible ODs in Zarr-format skims, similar to trim_inaccessible_ods for OMX. + Uses direct .data access for in-place modification. + """ + logger.info("Starting trim of inaccessible ODs in Zarr skims.") + try: + skims = xr.open_zarr(all_skims_path) # Open in append/write mode + except Exception as e: + logger.error(f"Failed to open skims file {all_skims_path} for trimming inaccessible ODs: {e}") + return + + try: + order = zone_order(settings, settings.get('start_year', 2015)) # Use .get for start_year safety + periods = settings["periods"] + transit_paths_settings = settings.get('transit_paths', {}) # Use .get for safety + except Exception as e: + logger.error(f"Error reading settings for trimming: {e}. Skipping trim.") + skims.close() + return + + if not transit_paths_settings: + logger.warning("No 'transit_paths' defined in settings for trimming inaccessible ODs. Skipping trim.") + skims.close() + return + + tp_to_idx = {p: i for i, p in enumerate(periods)} + num_zones = len(order) + + # Calculate total trips per OD pair across all non-RH transit/walk/bike modes for each period + # Initialize totalTrips as a 3D array [zones, zones, periods] + totalTrips = np.zeros((num_zones, num_zones, len(periods)), dtype=np.float32) -def trim_inaccessible_ods(settings): - all_skims_path = os.path.join(settings['asim_local_input_folder'], "skims.omx") + all_vars = list(skims.data_vars) + + for var_name in all_vars: + parts = var_name.split('__') + if len(parts) == 2: # Variable name is in PATH_MEASURE__PERIOD format (old OMX style, sometimes kept in Zarr) + measure_part, tp = parts + if tp in periods: + # Check if it's a TRIPS matrix for a non-RH mode + if 'TRIPS' in measure_part and not measure_part.startswith('RH_'): + try: + tp_idx = periods.index(tp) + if skims[var_name].ndim == 2 and skims[var_name].shape == (num_zones, num_zones): + totalTrips[:, :, tp_idx] += np.nan_to_num(skims[var_name].data) + else: + logger.debug(f"Skipping '{var_name}' for total trips calculation due to unexpected shape {skims[var_name].shape}.") + except ValueError: + logger.warning(f"Period '{tp}' from key '{var_name}' not found in settings periods. Skipping.") + except Exception as e: + logger.error(f"Error reading '{var_name}' for total trips calculation: {e}") + + elif skims[var_name].ndim == 3: # Variable name is in PATH_MEASURE format (new Zarr style) + # Check if it's a TRIPS matrix for a non-RH mode + if var_name.endswith("_TRIPS") and not var_name.startswith("RH_"): + try: + # Assuming 3D shape is [zones, zones, periods] + if skims[var_name].shape[:2] == (num_zones, num_zones) and skims[var_name].shape[-1] >= len(periods): + # Sum across all periods we care about (as defined by settings) + for tp_idx in range(len(periods)): + totalTrips[:, :, tp_idx] += np.nan_to_num(skims[var_name].data[:, :, tp_idx]) + else: + logger.debug(f"Skipping '{var_name}' for total trips calculation due to unexpected shape {skims[var_name].shape}.") + except Exception as e: + logger.error(f"Error reading '{var_name}' for total trips calculation: {e}") + + # Calculate total trips BY ZONE (sum across rows and columns for each zone) for each period + completedAllTripsByOandD_3d = totalTrips.sum(axis=1) + totalTrips.sum(axis=0) # Shape (zones, periods) + + + # Now, check transit modes for trimming + for path, metrics in transit_paths_settings.items(): + trip_name_zarr = f"{path}_TRIPS" # Zarr key format + fail_name_zarr = f"{path}_FAILURES" # Zarr key format + + trip_da = skims.get(trip_name_zarr) + fail_da = skims.get(fail_name_zarr) + + if trip_da is None or fail_da is None: + logger.debug(f"Skipping trim for {path}: missing {trip_name_zarr} or {fail_name_zarr} in skims_ds.") + continue + + # Get 3D completed/failed transit trips for this mode + try: + completedTransitTrips_3d = np.nan_to_num(trip_da.data[:, :, :len(periods)]) # Slice to match settings periods + failedTransitTrips_3d = np.nan_to_num(fail_da.data[:, :, :len(periods)]) # Slice to match settings periods + if completedTransitTrips_3d.shape[:2] != (num_zones, num_zones) or completedTransitTrips_3d.shape[-1] != len(periods): + logger.warning(f"Shape mismatch for {trip_name_zarr} or {fail_name_zarr} ({completedTransitTrips_3d.shape}) vs expected ({num_zones}, {num_zones}, {len(periods)}). Skipping trim for {path}.") + continue + except Exception as e: + logger.error(f"Error reading trip data for {path} from Zarr: {e}. Skipping trim for this path.") + continue + + + # Calculate completed and failed transit trips BY ZONE for each period + completedTransitTripsByOandD_3d = completedTransitTrips_3d.sum(axis=1) + completedTransitTrips_3d.sum(axis=0) # Shape (zones, periods) + failedTransitTripsByOandD_3d = failedTransitTrips_3d.sum(axis=1) + failedTransitTrips_3d.sum(axis=0) # Shape (zones, periods) + + + # Determine which zones to delete for each period + # Condition: (Total trips > 1000) & (Failed transit trips > 200) & (Completed transit trips == 0) + toDelete_3d = (completedAllTripsByOandD_3d > 1000) & (failedTransitTripsByOandD_3d > 200) & (completedTransitTripsByOandD_3d == 0) # Shape (zones, periods) + + + # Apply trimming for each period where zones need deletion + for tpIdx, period in enumerate(periods): + zones_to_delete_tp = toDelete_3d[:, tpIdx] # Boolean array (zones,) for this period + num_zones_to_delete_tp = np.sum(zones_to_delete_tp) + + if num_zones_to_delete_tp > 0: + logger.info(f"Trimming {path} service for {num_zones_to_delete_tp} zones in {period} because no completed transit trips were observed for these zones (conditions met).") + + # Apply deletion mask to metrics for this path and period + for metric in metrics: + name_zarr = f"{path}_{metric}" # Zarr key format + metric_da = skims.get(name_zarr) + + if metric_da is not None: + try: + # Ensure the DataArray has enough dimensions and size for the period + if metric_da.ndim == 3 and metric_da.shape[-1] > tpIdx and metric_da.shape[:2] == (num_zones, num_zones): + # Get the 2D slice for this period + arr_slice = metric_da.data[:, :, tpIdx] + # Apply the deletion mask. This sets values to 0 where the origin OR destination zone is in `zones_to_delete_tp`. + arr_slice[zones_to_delete_tp[:, None] | zones_to_delete_tp[None, :]] = 0.0 + # Update the data in the Zarr array + metric_da.data[:, :, tpIdx] = arr_slice + logger.debug(f"Trimmed {name_zarr} for {period} for {num_zones_to_delete_tp} zones.") + elif metric_da.ndim == 2 and len(periods) == 1 and periods[0] == period and metric_da.shape == (num_zones, num_zones): + # Handle 2D case (single period) + arr_slice = metric_da.data[:, :] + arr_slice[zones_to_delete_tp[:, None] | zones_to_delete_tp[None, :]] = 0.0 + metric_da.data[:, :] = arr_slice + logger.debug(f"Trimmed {name_zarr} (2D) for {period} for {num_zones_to_delete_tp} zones.") + else: + logger.warning(f"Skipping trim for metric {name_zarr} in {period} due to unexpected shape {metric_da.shape} or period mismatch.") + except Exception as e: + logger.error(f"Error applying trim mask for metric {name_zarr} in {period}: {e}. Skipping.") + else: + logger.debug(f"Skipping trim for metric {name_zarr} in {period}: variable not found in skims_ds.") + + # Save the updated Zarr dataset after trimming + logger.info(f"Started writing zarr skims after trimming inaccessible ODs to {all_skims_path}") + try: + skims.to_zarr(all_skims_path, mode='w', consolidated=True, zarr_version=2) # Use mode='w' to overwrite + logger.info("Completed writing zarr skims after trimming successfully.") + except Exception as e: + logger.error(f"FAILED to write zarr skims after trimming inaccessible ODs to {all_skims_path}: {e}") + + skims.close() + + +def trim_inaccessible_ods(settings, working_dir): + all_skims_path = os.path.join(working_dir, settings['asim_local_mutable_data_folder'], "skims.omx") order = zone_order(settings, settings['start_year']) - skims = omx.open_file(all_skims_path, "a") + skims = omx.open_file(str(all_skims_path), "a") all_mats = skims.list_matrices() totalTrips = dict() for period in settings["periods"]: @@ -326,9 +2410,9 @@ def trim_inaccessible_ods(settings): skims.close() -def discover_impossible_ods(result, skims): +def discover_impossible_ods(result, skims, mats): # return (path, timePeriod), (completed, failed) - allMats = skims.list_matrices() + allMats = mats metricsPerPath = dict() for (path, tp), _ in result: if path not in metricsPerPath.keys(): @@ -433,7 +2517,7 @@ def merge_current_omx_origin_skims(all_skims_path, previous_skims_path, beam_out cur_skims = pd.read_csv(current_skims_path, dtype=rawInputSchema, na_values=["∞"]) cur_skims['timePeriod'] = cur_skims['hour'].apply(hourToTimeBin) cur_skims.rename(columns={'tazId': 'origin'}, inplace=True) - cur_skims['completedRequests'] = cur_skims['observations'] * (1. - cur_skims['unmatchedRequestsPercent'] / 100.) + cur_skims['completedRequests'] = cur_skims['observations'] * (1. - cur_skims['unmatchedRequestsPercent'] / 100.0) cur_skims = cur_skims.groupby(['timePeriod', 'reservationType', 'origin']).apply(aggregateInTimePeriod) cur_skims['failures'] = cur_skims['observations'] - cur_skims['completedRequests'] skims = omx.open_file(all_skims_path, 'a') diff --git a/pilates/beam/preprocessor.py b/pilates/beam/preprocessor.py index 14b2aba7..1e162993 100644 --- a/pilates/beam/preprocessor.py +++ b/pilates/beam/preprocessor.py @@ -4,16 +4,47 @@ import shutil import pandas as pd import numpy as np +import glob logger = logging.getLogger(__name__) beam_param_map = {'beam_sample': 'beam.agentsim.agentSampleSizeAsFractionOfPopulation', 'beam_replanning_portion': 'beam.agentsim.agents.plans.merge.fraction', - 'max_plans_memory': 'beam.replanning.maxAgentPlanMemorySize' + 'max_plans_memory': 'beam.replanning.maxAgentPlanMemorySize', + 'skim_zone_geoid_col': 'beam.agentsim.taz.tazIdFieldName', } -def update_beam_config(settings, param, valueOverride=None): +def copy_data_to_mutable_location(settings, output_dir): + beam_config_path = os.path.join( + settings['beam_local_input_folder'], + settings['region']) + dest = os.path.join(output_dir, settings['region']) + logger.info("Copying beam inputs from {0} to {1}".format(beam_config_path, dest)) + + ## TODO: Update configs not to rely on beam.inputDirectory or update BEAM to have it be relative to code location and/or configurable + + # for file in os.listdir(beam_config_path): + # pathname = os.path.join(beam_config_path, file) + # if file.endswith(".conf"): + # shutil.copy(pathname, output_subdir) + # elif file.startswith("r5") & os.path.isdir(pathname): + # shutil.copytree(pathname, os.path.join(output_subdir, file)) + # elif file == "urbansim": + # shutil.copytree(pathname, os.path.join(output_subdir, file)) + + shutil.copytree(beam_config_path, dest) + common_config_path = os.path.join(settings['beam_local_input_folder'], 'common') + shutil.copytree(common_config_path, os.path.join(output_dir, 'common')) + if 'beam_skims_shapefile' in settings: + logger.info("Updating beam config to use zone id of {0}".format(settings['skim_zone_geoid_col'])) + update_beam_config(settings, + os.path.split(os.path.split(os.path.split(output_dir)[0])[0])[0], # Sorry... + 'skim_zone_geoid_col', + settings['skim_zone_geoid_col']) + + +def update_beam_config(settings, working_dir, param, valueOverride=None): if param in settings: config_header = beam_param_map[param] if valueOverride is None: @@ -21,7 +52,8 @@ def update_beam_config(settings, param, valueOverride=None): else: config_value = valueOverride beam_config_path = os.path.join( - settings['beam_local_input_folder'], + working_dir, + settings['beam_local_mutable_data_folder'], settings['region'], settings['beam_config']) modified = False @@ -30,7 +62,7 @@ def update_beam_config(settings, param, valueOverride=None): with open(beam_config_path, 'w') as file: for line in data: if config_header in line: - if ~modified: + if not modified: file.writelines(config_header + " = " + str(config_value) + "\n") modified = True else: @@ -54,52 +86,107 @@ def make_archive(source, destination): shutil.move('%s.%s' % (name, fmt), destination) -def copy_vehicles_from_atlas(settings, year): +def copy_vehicles_from_atlas(settings, state: "WorkflowState"): beam_scenario_folder = os.path.join( - settings['beam_local_input_folder'], + state.full_path, + settings['beam_local_mutable_data_folder'], settings['region'], settings['beam_scenario_folder']) beam_vehicles_path = os.path.join(beam_scenario_folder, 'vehicles.csv.gz') - atlas_output_data_dir = settings['atlas_host_output_folder'] - atlas_vehicle_file_loc = os.path.join(atlas_output_data_dir, "vehicles_{0}.csv.gz".format(year)) + atlas_output_data_dir = os.path.join(state.full_path, settings['atlas_host_output_folder']) + atlas_vehicle_file_loc = os.path.join(atlas_output_data_dir, "vehicles_{0}.csv.gz".format(state.forecast_year)) if not os.path.exists(atlas_vehicle_file_loc): - atlas_vehicle_file_loc = os.path.join(atlas_output_data_dir, "vehicles_{0}.csv.gz".format(year - 1)) + atlas_vehicle_file_loc = os.path.join(atlas_output_data_dir, + "vehicles_{0}.csv.gz".format(state.forecast_year - 1)) + logger.info("Copying atlas vehicles file from {0} to {1}".format(atlas_vehicle_file_loc, beam_vehicles_path)) shutil.copy(atlas_vehicle_file_loc, beam_vehicles_path) -def copy_plans_from_asim(settings, year, replanning_iteration_number=0): - asim_output_data_dir = settings['asim_local_output_folder'] +def copy_plans_from_asim(settings, state: "WorkflowState", replanning_iteration_number=0): + asim_output_data_dir = os.path.join(state.full_path, settings['asim_local_output_folder']) beam_scenario_folder = os.path.join( - settings['beam_local_input_folder'], + state.full_path, + settings['beam_local_mutable_data_folder'], settings['region'], settings['beam_scenario_folder']) - def copy_with_compression_asim_file_to_beam(asim_file_name, beam_file_name): - asim_file_path = os.path.join(asim_output_data_dir, asim_file_name) - beam_file_path = os.path.join(beam_scenario_folder, beam_file_name) - logger.info("Copying asim file %s to beam input scenario file %s", asim_file_path, beam_file_path) - - if os.path.exists(asim_file_path): - df = pd.read_csv(asim_file_path, dtype={"household_id": pd.Int64Dtype(), - "person_id": pd.Int64Dtype(), - "trip_id": pd.Int64Dtype(), - "cars": pd.Int64Dtype(), - "VEHICL": pd.Int64Dtype(), - "age": pd.Int64Dtype(), - "sex": pd.Int64Dtype()} - ).rename(columns={"VEHICL": "cars"}).rename(columns={"auto_ownership": "cars"}) - df = df.loc[:, ~df.columns.duplicated()].copy() - df.to_csv(beam_file_path, compression="gzip") - # with open(asim_file_path, 'rb') as f_in, gzip.open( - # beam_file_path, 'wb') as f_out: - # f_out.writelines(f_in) - - def copy_with_compression_asim_file_to_asim_archive(file_path, file_name, year, replanning_iteration_number): + def locate_asim_file(file_name, fmt): + if fmt == "csv": + return os.path.join(asim_output_data_dir, "final_" + file_name + ".csv") + elif fmt == "parquet": + if file_name == "plans": + a_n = "beam_plans" + else: + a_n = file_name + return os.path.join(asim_output_data_dir, "final_pipeline", a_n, "final.parquet") + elif fmt is None: + return os.path.join(asim_output_data_dir, file_name) + + def locate_beam_file(file_name, fmt): + if fmt == "csv": + return os.path.join(beam_scenario_folder, file_name + ".csv.gz") + elif fmt == "parquet": + return os.path.join(beam_scenario_folder, file_name + ".parquet") + + def copy_with_compression_asim_file_to_beam(asim_file_name, beam_file_name, file_format): + """ + TODO: Switch this to polars for better performance + def copy_with_compression_asim_file_to_beam(asim_file_name, beam_file_name): + import polars as pl + asim_file_path = os.path.join(asim_output_data_dir, asim_file_name) + beam_file_path = os.path.join(beam_scenario_folder, beam_file_name) + logger.info("Copying asim file %s to beam input scenario file %s", asim_file_path, beam_file_path) + + if os.path.exists(asim_file_path): + df = pl.scan_csv(asim_file_path).with_columns([ + pl.col("household_id").cast(pl.Int64), + pl.col("person_id").cast(pl.Int64), + pl.col("trip_id").cast(pl.Int64), + pl.col("cars").cast(pl.Int64), + pl.col("VEHICL").cast(pl.Int64).alias("cars"), + pl.col("auto_ownership").cast(pl.Int64).alias("cars"), + pl.col("age").cast(pl.Int64), + pl.col("sex").cast(pl.Int64) + ]).select(pl.col("*").exclude_duplicates()).collect() + df.write_csv(beam_file_path, compression="gzip") + """ + if file_format == "csv": + asim_file_path = locate_asim_file(asim_file_name, file_format) + beam_file_path = locate_beam_file(beam_file_name, file_format) + # asim_file_path = os.path.join(asim_output_data_dir, "final_" + asim_file_name + ".csv") + # beam_file_path = os.path.join(beam_scenario_folder, beam_file_name + ".csv.gz") + logger.info("Copying asim file %s to beam input scenario file %s", asim_file_path, beam_file_path) + + if os.path.exists(asim_file_path): + df = pd.read_csv(asim_file_path, dtype={"household_id": pd.Int64Dtype(), + "person_id": pd.Int64Dtype(), + "trip_id": pd.Int64Dtype(), + "cars": pd.Int64Dtype(), + "VEHICL": pd.Int64Dtype(), + "age": pd.Int64Dtype(), + "sex": pd.Int64Dtype()} + ).rename(columns={"VEHICL": "cars"}).rename(columns={"auto_ownership": "cars"}) + df = df.loc[:, ~df.columns.duplicated()].copy() + df.to_csv(beam_file_path, compression="gzip") + # with open(asim_file_path, 'rb') as f_in, gzip.open( + # beam_file_path, 'wb') as f_out: + # f_out.writelines(f_in) + elif file_format == "parquet": + asim_file_path = locate_asim_file(asim_file_name, file_format) + beam_file_path = locate_beam_file(beam_file_name, file_format) + logger.info("Copying asim file %s to beam input scenario file %s", asim_file_path, beam_file_path) + df = pd.read_parquet(asim_file_path).rename(columns={"VEHICL": "cars"}).rename( + columns={"auto_ownership": "cars"}).rename(columns={"tripId": "trip_id"}) + if "household_id" in df.columns: + df = df.astype({"household_id": pd.Int64Dtype()}) + df.loc[:, ~df.columns.duplicated()].to_parquet(beam_file_path) + + def copy_with_compression_asim_file_to_asim_archive(file_path, file_name, year, replanning_iteration_number, fmt): iteration_folder_name = "year-{0}-iteration-{1}".format(year, replanning_iteration_number) iteration_folder_path = os.path.join(asim_output_data_dir, iteration_folder_name) - if ~os.path.exists(os.path.abspath(iteration_folder_path)): + if not os.path.exists(os.path.abspath(iteration_folder_path)): os.makedirs(iteration_folder_path, exist_ok=True) - input_file_path = os.path.join(file_path, file_name) + input_file_path = locate_asim_file(file_name, fmt) target_file_path = os.path.join(iteration_folder_path, file_name) if target_file_path.endswith('.csv'): target_file_path += '.gz' @@ -113,34 +200,46 @@ def copy_with_compression_asim_file_to_asim_archive(file_path, file_name, year, shutil.copy(input_file_path, target_file_path) def merge_only_updated_households(): - asim_plans_path = os.path.join(asim_output_data_dir, 'final_plans.csv') - asim_households_path = os.path.join(asim_output_data_dir, 'final_households.csv') - asim_persons_path = os.path.join(asim_output_data_dir, 'final_persons.csv') - beam_plans_path = os.path.join(beam_scenario_folder, 'plans.csv.gz') - beam_households_path = os.path.join(beam_scenario_folder, 'households.csv.gz') - beam_persons_path = os.path.join(beam_scenario_folder, 'persons.csv.gz') + asim_plans_path = locate_asim_file("plans", file_format) + asim_households_path = locate_asim_file("households", file_format) + asim_persons_path = locate_asim_file("persons", file_format) + beam_plans_path = locate_beam_file("plans", file_format) + beam_households_path = locate_beam_file("households", file_format) + beam_persons_path = locate_beam_file("persons", file_format) if os.path.exists(beam_plans_path): logger.info("Merging asim outputs with existing beam input scenario files") - original_households = pd.read_csv(beam_households_path, dtype={"household_id": pd.Int64Dtype(), - "cars": pd.Int64Dtype(), - "auto_ownership": pd.Int64Dtype()} + if file_format == "csv": + original_households = pd.read_csv(beam_households_path, dtype={"household_id": pd.Int64Dtype(), + "cars": pd.Int64Dtype(), + "auto_ownership": pd.Int64Dtype()} + ) + updated_households = pd.read_csv(asim_households_path, dtype={"household_id": pd.Int64Dtype(), + "VEHICL": pd.Int64Dtype(), + "auto_ownership": pd.Int64Dtype()} + ).rename(columns={"VEHICL": "cars"}).rename( + columns={"auto_ownership": "cars"}) + updated_households = updated_households.loc[:, ~updated_households.columns.duplicated()].copy() + original_persons = pd.read_csv(beam_persons_path, dtype={"household_id": pd.Int64Dtype(), + "person_id": pd.Int64Dtype(), + "age": pd.Int64Dtype(), + "sex": pd.Int64Dtype()} + ) + updated_persons = pd.read_csv(asim_persons_path, dtype={"household_id": pd.Int64Dtype(), + "person_id": pd.Int64Dtype(), + "age": pd.Int64Dtype(), + "sex": pd.Int64Dtype()} ) - updated_households = pd.read_csv(asim_households_path, dtype={"household_id": pd.Int64Dtype(), - "VEHICL": pd.Int64Dtype(), - "auto_ownership": pd.Int64Dtype()} - ).rename(columns={"VEHICL": "cars"}).rename( - columns={"auto_ownership": "cars"}) - updated_households = updated_households.loc[:, ~updated_households.columns.duplicated()].copy() - original_persons = pd.read_csv(beam_persons_path, dtype={"household_id": pd.Int64Dtype(), - "person_id": pd.Int64Dtype(), - "age": pd.Int64Dtype(), - "sex": pd.Int64Dtype()} - ) - updated_persons = pd.read_csv(asim_persons_path, dtype={"household_id": pd.Int64Dtype(), - "person_id": pd.Int64Dtype(), - "age": pd.Int64Dtype(), - "sex": pd.Int64Dtype()} - ) + original_plans = pd.read_csv(beam_plans_path).rename(columns={'tripId': 'trip_id'}) + updated_plans = pd.read_csv(asim_plans_path) + elif file_format == "parquet": + original_households = pd.read_parquet(beam_households_path) + updated_households = pd.read_parquet(asim_households_path) + original_persons = pd.read_parquet(beam_persons_path) + updated_persons = pd.read_parquet(asim_persons_path) + original_plans = pd.read_parquet(beam_plans_path).rename(columns={'tripId': 'trip_id'}) + updated_plans = pd.read_parquet(asim_plans_path) + else: + raise NotImplementedError per_o = original_persons.person_id.unique() per_u = updated_persons.person_id.unique() overlap = np.in1d(per_u.astype(float), per_o.astype(float)).sum() @@ -159,15 +258,12 @@ def merge_only_updated_households(): "person_id": pd.Int64Dtype(), "age": pd.Int64Dtype(), "sex": pd.Int64Dtype()}) - persons_final.to_csv(beam_persons_path, index=False, compression='gzip') + households_final = pd.concat([updated_households, original_households.loc[ ~original_households.household_id.isin(hh_u), :]]) households_final = households_final.astype({"household_id": pd.Int64Dtype(), "cars": pd.Int64Dtype()}) - households_final.to_csv(beam_households_path, index=False, compression='gzip') - original_plans = pd.read_csv(beam_plans_path).rename(columns={'tripId': 'trip_id'}) - updated_plans = pd.read_csv(asim_plans_path) unchanged_plans = original_plans.loc[~original_plans.person_id.isin(per_u), :] logger.info("Adding %s new plan elements after and keeping %s from previous iteration", len(updated_plans), len(unchanged_plans)) @@ -175,40 +271,62 @@ def merge_only_updated_households(): persons_with_plans = np.in1d(persons_final.person_id.unique().astype(float), plans_final.person_id.unique().astype(float)).sum() logger.info("Of %s persons, %s of them have plans", len(persons_final), persons_with_plans) - plans_final.to_csv(beam_plans_path, compression='gzip', index=False) + if file_format == "csv": + persons_final.to_csv(beam_persons_path, index=False, compression='gzip') + households_final.to_csv(beam_households_path, index=False, compression='gzip') + plans_final.to_csv(beam_plans_path, compression='gzip', index=False) + else: + persons_final.to_parquet(beam_persons_path, index=False) + households_final.to_parquet(beam_households_path, index=False) + plans_final.to_parquet(beam_plans_path, index=False) else: logger.info("No plans existed already so copying them directly. THIS IS BAD") pd.read_csv(asim_plans_path).to_csv(beam_plans_path, compression='gzip') - if replanning_iteration_number < 0: - copy_with_compression_asim_file_to_beam('final_plans.csv', 'plans.csv.gz') - copy_with_compression_asim_file_to_beam('final_households.csv', 'households.csv.gz') - copy_with_compression_asim_file_to_beam('final_persons.csv', 'persons.csv.gz') - # copy_with_compression_asim_file_to_beam('final_land_use.csv', 'land_use.csv.gz') - # copy_with_compression_asim_file_to_beam('final_tours.csv', 'tours.csv.gz') - # copy_with_compression_asim_file_to_beam('final_trips.csv', 'trips.csv.gz') - # copy_with_compression_asim_file_to_beam('final_joint_tour_participants.csv', - # 'joint_tour_participants.csv.gz') - else: - merge_only_updated_households() - - if settings.get('final_asim_plans_folder', False): - # This first one not currently necessary when asim-lite is replanning all households - # copy_with_compression_asim_file_to_asim_archive(asim_output_data_dir, 'final_plans.csv', year, - # replanning_iteration_number) - copy_with_compression_asim_file_to_asim_archive(beam_scenario_folder, 'plans.csv.gz', year, - replanning_iteration_number) - copy_with_compression_asim_file_to_asim_archive(beam_scenario_folder, 'households.csv.gz', year, - replanning_iteration_number) - copy_with_compression_asim_file_to_asim_archive(beam_scenario_folder, 'persons.csv.gz', year, - replanning_iteration_number) - copy_with_compression_asim_file_to_asim_archive(asim_output_data_dir, 'final_land_use.csv', year, - replanning_iteration_number) - copy_with_compression_asim_file_to_asim_archive(asim_output_data_dir, 'final_tours.csv', year, - replanning_iteration_number) - copy_with_compression_asim_file_to_asim_archive(asim_output_data_dir, 'final_trips.csv', year, - replanning_iteration_number) - copy_with_compression_asim_file_to_asim_archive(asim_output_data_dir, 'trip_mode_choice', year, - replanning_iteration_number) + + if settings.get('copy_plans_from_asim_outputs', True): + logging.info("You have chosen to use final ASIM plans. Will attempt to read files from:") + logging.info(f"- Beam scenario folder: {beam_scenario_folder}") + logging.info(f"- ASIM output data directory: {asim_output_data_dir}") + file_format = settings.get("file_format", "parquet") + if replanning_iteration_number < 0: + copy_with_compression_asim_file_to_beam('plans', 'plans', file_format) + copy_with_compression_asim_file_to_beam('households', 'households', file_format) + copy_with_compression_asim_file_to_beam('persons', 'persons', file_format) + else: + merge_only_updated_households() + + # Files from beam_scenario_folder + if os.path.exists(beam_scenario_folder): + file_format = settings.get("file_format", "parquet") + try: + copy_with_compression_asim_file_to_asim_archive(beam_scenario_folder, 'plans', state.year, + replanning_iteration_number, file_format) + copy_with_compression_asim_file_to_asim_archive(beam_scenario_folder, 'households', state.year, + replanning_iteration_number, file_format) + copy_with_compression_asim_file_to_asim_archive(beam_scenario_folder, 'persons', state.year, + replanning_iteration_number, file_format) + except: + logger.error("Error copying asim files to asim archive") + else: + logging.warning( + f"Warning: Directory {beam_scenario_folder} does not exist. Cannot copy beam scenario files.") + + # Files from asim_output_data_dir + if os.path.exists(asim_output_data_dir): + copy_with_compression_asim_file_to_asim_archive(asim_output_data_dir, 'land_use', state.year, + replanning_iteration_number, file_format) + copy_with_compression_asim_file_to_asim_archive(asim_output_data_dir, 'tours', state.year, + replanning_iteration_number, file_format) + copy_with_compression_asim_file_to_asim_archive(asim_output_data_dir, 'trips', state.year, + replanning_iteration_number, file_format) + try: + copy_with_compression_asim_file_to_asim_archive(asim_output_data_dir, 'trip_mode_choice', state.year, + replanning_iteration_number, None) + except FileNotFoundError: + logger.info("Note: trip_mode_choice utilities not found. Skipping.") + + else: + logging.warning(f"Assuming that asim plans have already been generated and are stored in the beam input data") return diff --git a/pilates/polaris/travel_model.py b/pilates/polaris/travel_model.py index f8ffdfc4..f6b8db15 100644 --- a/pilates/polaris/travel_model.py +++ b/pilates/polaris/travel_model.py @@ -14,228 +14,230 @@ logger = logging.getLogger(__name__) + def all_subdirs_of(out_name, b='.'): - result = [] - # for d in os.listdir(b): - # bd = os.path.join(b, d) - # if os.path.isdir(bd): result.append(bd) - search_path = '{0}/{1}'.format(b, out_name + '*') - result = glob.glob(search_path) - return result + result = [] + # for d in os.listdir(b): + # bd = os.path.join(b, d) + # if os.path.isdir(bd): result.append(bd) + search_path = '{0}/{1}'.format(b, out_name + '*') + result = glob.glob(search_path) + return result + def get_latest_polaris_output(out_name, data_dir='.'): - # all_subdirs = [d for d in os.listdir(data_dir) if os.path.isdir(d)] - all_subdirs = all_subdirs_of(out_name, data_dir) - latest_subdir = Path(max(all_subdirs, key=os.path.getmtime)) - return latest_subdir - + # all_subdirs = [d for d in os.listdir(data_dir) if os.path.isdir(d)] + all_subdirs = all_subdirs_of(out_name, data_dir) + latest_subdir = Path(max(all_subdirs, key=os.path.getmtime)) + return latest_subdir + + def update_scenario_file(base_scenario, forecast_year): - s = base_scenario.replace('.json', '_' + str(forecast_year) + '.json') - return s + s = base_scenario.replace('.json', '_' + str(forecast_year) + '.json') + return s + def modify_scenario(scenario_file, parameter, value): - f = open(scenario_file,'r') - filedata = f.read() - datalist = [] - f.close() + f = open(scenario_file, 'r') + filedata = f.read() + datalist = [] + f.close() - for d in filedata.split(','): - datalist.append(d.strip()) + for d in filedata.split(','): + datalist.append(d.strip()) - find_str = '"' + parameter + '"*' - find_match_list = fnmatch.filter(datalist,find_str) + find_str = '"' + parameter + '"*' + find_match_list = fnmatch.filter(datalist, find_str) - if len(find_match_list)==0: - print('Could not find parameter: ' + find_str + ' in scenario file: ' + scenario_file) - print(datalist) - sys.exit() - - find_match = find_match_list[len(find_match_list)-1] + if len(find_match_list) == 0: + print('Could not find parameter: ' + find_str + ' in scenario file: ' + scenario_file) + print(datalist) + sys.exit() - newstr = '"' + parameter + '" : ' + str(value) - newdata = filedata.replace(find_match,newstr) + find_match = find_match_list[len(find_match_list) - 1] - f = open(scenario_file,'w') - f.write(newdata) - f.close() + newstr = '"' + parameter + '" : ' + str(value) + newdata = filedata.replace(find_match, newstr) -def run_polaris(forecast_year, usim_settings, warm_start=False): + f = open(scenario_file, 'w') + f.write(newdata) + f.close() - logger.info('**** RUNNING POLARIS FOR FORECAST YEAR {0}, WARM START MODE = {1}'.format(forecast_year, warm_start)) - - # read settings from config file - with open('pilates/polaris/polaris_settings.yaml') as file: - polaris_settings = yaml.load(file, Loader=yaml.FullLoader) - data_dir = Path(polaris_settings.get('data_dir')) - backup_dir = Path(polaris_settings.get('backup_dir')) - scripts_dir = Path(polaris_settings.get('scripts_dir')) - db_name = polaris_settings.get('db_name') - out_name = polaris_settings.get('out_name') - polaris_exe = polaris_settings.get('polaris_exe') - scenario_init_file = polaris_settings.get('scenario_main_init', None) - scenario_main_file = polaris_settings.get('scenario_main') - vehicle_file_base = polaris_settings.get('vehicle_file_basename', None) - vehicle_file_fleet_base = polaris_settings.get('fleet_vehicle_file_basename', None) - num_threads = polaris_settings.get('num_threads') - num_abm_runs = polaris_settings.get('num_abm_runs') - block_loc_file_name = polaris_settings.get('block_loc_file_name') - population_scale_factor = polaris_settings.get('population_scale_factor') - archive_dir = polaris_settings.get('archive_dir') - db_supply = "{0}/{1}-Supply.sqlite".format(str(data_dir), db_name) - db_demand = "{0}/{1}-Demand.sqlite".format(str(data_dir), db_name) - block_loc_file = "{0}/{1}".format(str(data_dir), block_loc_file_name) - vot_level = polaris_settings.get('vot_level') - - usim_output_dir = os.path.abspath(usim_settings['usim_local_data_folder']) - - # store the original inputs - supply_db_name = db_name + "-Supply.sqlite" - demand_db_name = db_name + "-Demand.sqlite" - result_db_name = db_name + "-Result.sqlite" - highway_skim_file_name = "highway_skim_file.bin" - transit_skim_file_name = "transit_skim_file.bin" - - - cwd = os.getcwd() - os.chdir(data_dir) - - #check if warm start and initialize changes - if warm_start: - num_abm_runs = 1 - - # start with fresh demand database from backup (only for warm start - PR.copyreplacefile(backup_dir / demand_db_name, data_dir) - - # load the urbansim population for the init run - preprocessor.preprocess_usim_for_polaris(forecast_year, usim_output_dir, block_loc_file, db_supply, db_demand, 1.0, usim_settings) - - # update the vehicles table with new costs - if forecast_year: - veh_script = "vehicle_operating_cost_" + str(forecast_year) + ".sql" - PR.execute_sql_script(data_dir / demand_db_name, data_dir / veh_script) - - fail_count = 0 - loop = 0 - - while loop < int(num_abm_runs): - scenario_file = '' - - # set vehicle distribution file name based on forecast year - veh_file_name = '"' + vehicle_file_base + '_{0}.txt"'.format(forecast_year) - fleet_veh_file_name = '"' + vehicle_file_fleet_base + '_{0}.txt"'.format(forecast_year) - if not forecast_year: - veh_file_name = '"' + vehicle_file_base + '.txt"'.format(forecast_year) - fleet_veh_file_name = '"' + vehicle_file_fleet_base + 'txt"'.format(forecast_year) - - if loop == 0: - if forecast_year: - scenario_file = PR.update_scenario_file(scenario_init_file, forecast_year) - else: - scenario_file = scenario_init_file - - PR.modify_scenario(scenario_file, "time_dependent_routing_weight_factor", 1.0) - PR.modify_scenario(scenario_file, "read_population_from_database", 'true') - - # set warm_start specific settings (that are also modified by loop...) - if warm_start: - PR.modify_scenario(scenario_file, "percent_to_synthesize", 1.0) - PR.modify_scenario(scenario_file, "read_population_from_urbansim", 'true') - PR.modify_scenario(scenario_file, "warm_start_mode", 'true') - PR.modify_scenario(scenario_file, "time_dependent_routing", 'false') - PR.modify_scenario(scenario_file, "multimodal_routing", 'false') - PR.modify_scenario(scenario_file, "use_tnc_system", 'false') - PR.modify_scenario(scenario_file, "output_moe_for_assignment_interval", 'false') - PR.modify_scenario(scenario_file, "output_link_moe_for_simulation_interval" , 'false') - PR.modify_scenario(scenario_file, "output_link_moe_for_assignment_interval" , 'false') - PR.modify_scenario(scenario_file, "output_turn_movement_moe_for_assignment_interval" , 'false') - PR.modify_scenario(scenario_file, "output_turn_movement_moe_for_simulation_interval" , 'false') - PR.modify_scenario(scenario_file, "output_network_moe_for_simulation_interval" , 'false') - PR.modify_scenario(scenario_file, "write_skim_tables" , 'false') - PR.modify_scenario(scenario_file, "write_vehicle_trajectory" , 'false') - PR.modify_scenario(scenario_file, "write_transit_trajectory" , 'false') - PR.modify_scenario(scenario_file, "demand_reduction_factor", 1.0) - PR.modify_scenario(scenario_file, "traffic_scale_factor", 1.0) - else: - PR.modify_scenario(scenario_file, "percent_to_synthesize", population_scale_factor) - PR.modify_scenario(scenario_file, "read_population_from_urbansim", 'false') - PR.modify_scenario(scenario_file, "warm_start_mode", 'false') - PR.modify_scenario(scenario_file, "time_dependent_routing", 'false') - PR.modify_scenario(scenario_file, "multimodal_routing", 'true') - PR.modify_scenario(scenario_file, "use_tnc_system", 'true') - PR.modify_scenario(scenario_file, "output_link_moe_for_assignment_interval" , 'true') - PR.modify_scenario(scenario_file, "output_turn_movement_moe_for_assignment_interval" , 'true') - PR.modify_scenario(scenario_file, "write_skim_tables" , 'true') - PR.modify_scenario(scenario_file, "write_vehicle_trajectory" , 'true') - PR.modify_scenario(scenario_file, "demand_reduction_factor", population_scale_factor) - PR.modify_scenario(scenario_file, "traffic_scale_factor", population_scale_factor) - - if warm_start and not forecast_year: - PR.modify_scenario(scenario_file, "replan_workplaces", 'true') - else: - PR.modify_scenario(scenario_file, "replan_workplaces", 'false') - else: - if forecast_year: - scenario_file = PR.update_scenario_file(scenario_main_file, forecast_year) - PR.modify_scenario(scenario_file, "time_dependent_routing_weight_factor", 1.0/int(loop)) - PR.modify_scenario(scenario_file, "percent_to_synthesize", 1.0) - PR.modify_scenario(scenario_file, "demand_reduction_factor", 1.0) - PR.modify_scenario(scenario_file, "traffic_scale_factor", population_scale_factor) - PR.modify_scenario(scenario_file, "read_population_from_urbansim", 'false') - PR.modify_scenario(scenario_file, "read_population_from_database", 'true') - PR.modify_scenario(scenario_file, "replan_workplaces", 'false') - - PR.modify_scenario(scenario_file, "vehicle_distribution_file_name", veh_file_name) - PR.modify_scenario(scenario_file, "fleet_vehicle_distribution_file_name", fleet_veh_file_name) - - arguments = '{0} {1}'.format(scenario_file, str(num_threads)) - logger.info(f'Executing \'{str(polaris_exe)} {arguments}\'') - - # run executable - success = PR.run_polaris_instance(data_dir, polaris_exe, scenario_file, num_threads, None) - # get output directory and write files into working dir for next run - output_dir = PR.get_latest_polaris_output(out_name, data_dir) - - if success: - PR.copyreplacefile(output_dir / demand_db_name, data_dir) - PR.execute_sql_script(data_dir / demand_db_name, scripts_dir / "clean_db_after_abm_for_abm.sql") - # skip all of the file storage and analysis for warm start runs - only need the demand file - if not warm_start: - fail_count = 0 - # copy the network outputs back to main data directory - PR.copyreplacefile(output_dir / result_db_name, data_dir) - PR.copyreplacefile(output_dir / highway_skim_file_name, data_dir) - PR.copyreplacefile(data_dir / supply_db_name, output_dir) - loop += 1 - else: - fail_count += 1 - if fail_count >= 3: - logger.info("POLARIS crashed three times in a row") - sys.exit() - else: - shutil.rmtree(output_dir) - logger.info(f"Deleting failed results directory for attempt: {loop}") - - - os.chdir(cwd) - # find the latest output - output_dir = PR.get_latest_polaris_output(out_name, data_dir) - # db_supply = "{0}/{1}-Supply.sqlite".format(output_dir, db_name) - # db_demand = "{0}/{1}-Demand.sqlite".format(output_dir, db_name) - # db_result = "{0}/{1}-Result.sqlite".format(output_dir, db_name) - # auto_skim = "{0}/{1}".format(output_dir, polaris_settings.get('auto_skim_file')) - # transit_skim = "{0}/{1}".format(output_dir, polaris_settings.get('transit_skim_file')) - - # postprocessor.generate_polaris_skims_for_usim( 'pilates/polaris/data', db_name, db_supply, db_demand, db_result, auto_skim, transit_skim, vot_level) - - if warm_start: - postprocessor.update_usim_after_polaris(forecast_year, usim_output_dir, db_demand, usim_settings) - # store the updated full population demand database for the next warm start round - PR.copyreplacefile(data_dir / demand_db_name, backup_dir ) - else: - archive_dir = postprocessor.archive_polaris_output(db_name, forecast_year, output_dir, data_dir) - postprocessor.archive_and_generate_usim_skims(forecast_year, db_name, output_dir, vot_level) - # only run the analysis script for the final iteration of the loop and process asynchronously to save time - p1 = Thread(target=PR.execute_sql_script_with_attach, args=(archive_dir / demand_db_name, scripts_dir / "wtf_baseline_analysis.sql", archive_dir / supply_db_name)) - p1.start() - +def run_polaris(forecast_year, usim_settings, warm_start=False): + logger.info('**** RUNNING POLARIS FOR FORECAST YEAR {0}, WARM START MODE = {1}'.format(forecast_year, warm_start)) + + # read settings from config file + with open('pilates/polaris/polaris_settings.yaml') as file: + polaris_settings = yaml.load(file, Loader=yaml.FullLoader) + data_dir = Path(polaris_settings.get('data_dir')) + backup_dir = Path(polaris_settings.get('backup_dir')) + scripts_dir = Path(polaris_settings.get('scripts_dir')) + db_name = polaris_settings.get('db_name') + out_name = polaris_settings.get('out_name') + polaris_exe = polaris_settings.get('polaris_exe') + scenario_init_file = polaris_settings.get('scenario_main_init', None) + scenario_main_file = polaris_settings.get('scenario_main') + vehicle_file_base = polaris_settings.get('vehicle_file_basename', None) + vehicle_file_fleet_base = polaris_settings.get('fleet_vehicle_file_basename', None) + num_threads = polaris_settings.get('num_threads') + num_abm_runs = polaris_settings.get('num_abm_runs') + block_loc_file_name = polaris_settings.get('block_loc_file_name') + population_scale_factor = polaris_settings.get('population_scale_factor') + archive_dir = polaris_settings.get('archive_dir') + db_supply = "{0}/{1}-Supply.sqlite".format(str(data_dir), db_name) + db_demand = "{0}/{1}-Demand.sqlite".format(str(data_dir), db_name) + block_loc_file = "{0}/{1}".format(str(data_dir), block_loc_file_name) + vot_level = polaris_settings.get('vot_level') + + usim_output_dir = os.path.abspath(usim_settings['usim_local_mutable_data_folder']) + + # store the original inputs + supply_db_name = db_name + "-Supply.sqlite" + demand_db_name = db_name + "-Demand.sqlite" + result_db_name = db_name + "-Result.sqlite" + highway_skim_file_name = "highway_skim_file.bin" + transit_skim_file_name = "transit_skim_file.bin" + + cwd = os.getcwd() + os.chdir(data_dir) + + # check if warm start and initialize changes + if warm_start: + num_abm_runs = 1 + + # start with fresh demand database from backup (only for warm start + PR.copyreplacefile(backup_dir / demand_db_name, data_dir) + + # load the urbansim population for the init run + preprocessor.preprocess_usim_for_polaris(forecast_year, usim_output_dir, block_loc_file, db_supply, db_demand, + 1.0, usim_settings) + + # update the vehicles table with new costs + if forecast_year: + veh_script = "vehicle_operating_cost_" + str(forecast_year) + ".sql" + PR.execute_sql_script(data_dir / demand_db_name, data_dir / veh_script) + + fail_count = 0 + loop = 0 + + while loop < int(num_abm_runs): + scenario_file = '' + + # set vehicle distribution file name based on forecast year + veh_file_name = '"' + vehicle_file_base + '_{0}.txt"'.format(forecast_year) + fleet_veh_file_name = '"' + vehicle_file_fleet_base + '_{0}.txt"'.format(forecast_year) + if not forecast_year: + veh_file_name = '"' + vehicle_file_base + '.txt"'.format(forecast_year) + fleet_veh_file_name = '"' + vehicle_file_fleet_base + 'txt"'.format(forecast_year) + + if loop == 0: + if forecast_year: + scenario_file = PR.update_scenario_file(scenario_init_file, forecast_year) + else: + scenario_file = scenario_init_file + + PR.modify_scenario(scenario_file, "time_dependent_routing_weight_factor", 1.0) + PR.modify_scenario(scenario_file, "read_population_from_database", 'true') + + # set warm_start specific settings (that are also modified by loop...) + if warm_start: + PR.modify_scenario(scenario_file, "percent_to_synthesize", 1.0) + PR.modify_scenario(scenario_file, "read_population_from_urbansim", 'true') + PR.modify_scenario(scenario_file, "warm_start_mode", 'true') + PR.modify_scenario(scenario_file, "time_dependent_routing", 'false') + PR.modify_scenario(scenario_file, "multimodal_routing", 'false') + PR.modify_scenario(scenario_file, "use_tnc_system", 'false') + PR.modify_scenario(scenario_file, "output_moe_for_assignment_interval", 'false') + PR.modify_scenario(scenario_file, "output_link_moe_for_simulation_interval", 'false') + PR.modify_scenario(scenario_file, "output_link_moe_for_assignment_interval", 'false') + PR.modify_scenario(scenario_file, "output_turn_movement_moe_for_assignment_interval", 'false') + PR.modify_scenario(scenario_file, "output_turn_movement_moe_for_simulation_interval", 'false') + PR.modify_scenario(scenario_file, "output_network_moe_for_simulation_interval", 'false') + PR.modify_scenario(scenario_file, "write_skim_tables", 'false') + PR.modify_scenario(scenario_file, "write_vehicle_trajectory", 'false') + PR.modify_scenario(scenario_file, "write_transit_trajectory", 'false') + PR.modify_scenario(scenario_file, "demand_reduction_factor", 1.0) + PR.modify_scenario(scenario_file, "traffic_scale_factor", 1.0) + else: + PR.modify_scenario(scenario_file, "percent_to_synthesize", population_scale_factor) + PR.modify_scenario(scenario_file, "read_population_from_urbansim", 'false') + PR.modify_scenario(scenario_file, "warm_start_mode", 'false') + PR.modify_scenario(scenario_file, "time_dependent_routing", 'false') + PR.modify_scenario(scenario_file, "multimodal_routing", 'true') + PR.modify_scenario(scenario_file, "use_tnc_system", 'true') + PR.modify_scenario(scenario_file, "output_link_moe_for_assignment_interval", 'true') + PR.modify_scenario(scenario_file, "output_turn_movement_moe_for_assignment_interval", 'true') + PR.modify_scenario(scenario_file, "write_skim_tables", 'true') + PR.modify_scenario(scenario_file, "write_vehicle_trajectory", 'true') + PR.modify_scenario(scenario_file, "demand_reduction_factor", population_scale_factor) + PR.modify_scenario(scenario_file, "traffic_scale_factor", population_scale_factor) + + if warm_start and not forecast_year: + PR.modify_scenario(scenario_file, "replan_workplaces", 'true') + else: + PR.modify_scenario(scenario_file, "replan_workplaces", 'false') + else: + if forecast_year: + scenario_file = PR.update_scenario_file(scenario_main_file, forecast_year) + PR.modify_scenario(scenario_file, "time_dependent_routing_weight_factor", 1.0 / int(loop)) + PR.modify_scenario(scenario_file, "percent_to_synthesize", 1.0) + PR.modify_scenario(scenario_file, "demand_reduction_factor", 1.0) + PR.modify_scenario(scenario_file, "traffic_scale_factor", population_scale_factor) + PR.modify_scenario(scenario_file, "read_population_from_urbansim", 'false') + PR.modify_scenario(scenario_file, "read_population_from_database", 'true') + PR.modify_scenario(scenario_file, "replan_workplaces", 'false') + + PR.modify_scenario(scenario_file, "vehicle_distribution_file_name", veh_file_name) + PR.modify_scenario(scenario_file, "fleet_vehicle_distribution_file_name", fleet_veh_file_name) + + arguments = '{0} {1}'.format(scenario_file, str(num_threads)) + logger.info(f'Executing \'{str(polaris_exe)} {arguments}\'') + + # run executable + success = PR.run_polaris_instance(data_dir, polaris_exe, scenario_file, num_threads, None) + # get output directory and write files into working dir for next run + output_dir = PR.get_latest_polaris_output(out_name, data_dir) + + if success: + PR.copyreplacefile(output_dir / demand_db_name, data_dir) + PR.execute_sql_script(data_dir / demand_db_name, scripts_dir / "clean_db_after_abm_for_abm.sql") + # skip all of the file storage and analysis for warm start runs - only need the demand file + if not warm_start: + fail_count = 0 + # copy the network outputs back to main data directory + PR.copyreplacefile(output_dir / result_db_name, data_dir) + PR.copyreplacefile(output_dir / highway_skim_file_name, data_dir) + PR.copyreplacefile(data_dir / supply_db_name, output_dir) + loop += 1 + else: + fail_count += 1 + if fail_count >= 3: + logger.info("POLARIS crashed three times in a row") + sys.exit() + else: + shutil.rmtree(output_dir) + logger.info(f"Deleting failed results directory for attempt: {loop}") + + os.chdir(cwd) + # find the latest output + output_dir = PR.get_latest_polaris_output(out_name, data_dir) + # db_supply = "{0}/{1}-Supply.sqlite".format(output_dir, db_name) + # db_demand = "{0}/{1}-Demand.sqlite".format(output_dir, db_name) + # db_result = "{0}/{1}-Result.sqlite".format(output_dir, db_name) + # auto_skim = "{0}/{1}".format(output_dir, polaris_settings.get('auto_skim_file')) + # transit_skim = "{0}/{1}".format(output_dir, polaris_settings.get('transit_skim_file')) + + # postprocessor.generate_polaris_skims_for_usim( 'pilates/polaris/data', db_name, db_supply, db_demand, db_result, auto_skim, transit_skim, vot_level) + + if warm_start: + postprocessor.update_usim_after_polaris(forecast_year, usim_output_dir, db_demand, usim_settings) + # store the updated full population demand database for the next warm start round + PR.copyreplacefile(data_dir / demand_db_name, backup_dir) + else: + archive_dir = postprocessor.archive_polaris_output(db_name, forecast_year, output_dir, data_dir) + postprocessor.archive_and_generate_usim_skims(forecast_year, db_name, output_dir, vot_level) + # only run the analysis script for the final iteration of the loop and process asynchronously to save time + p1 = Thread(target=PR.execute_sql_script_with_attach, args=( + archive_dir / demand_db_name, scripts_dir / "wtf_baseline_analysis.sql", archive_dir / supply_db_name)) + p1.start() diff --git a/pilates/postprocessing/.DS_Store b/pilates/postprocessing/.DS_Store index fe9903bd..b7f3a881 100644 Binary files a/pilates/postprocessing/.DS_Store and b/pilates/postprocessing/.DS_Store differ diff --git a/pilates/postprocessing/postprocessor.py b/pilates/postprocessing/postprocessor.py index 0e98b9e2..f8feaf8b 100644 --- a/pilates/postprocessing/postprocessor.py +++ b/pilates/postprocessing/postprocessor.py @@ -64,7 +64,7 @@ def copy_with_compression_asim_file_to_mep(asim_file_name, mep_file_name): f_out.writelines(f_in) def copy_urbansim_outputs_to_mep(): - data_dir = settings['usim_local_data_folder'] + data_dir = settings['usim_local_mutable_data_folder'] usim_output_store_name = get_usim_datastore_fname( settings, io='input', year=year) usim_output_store_path = os.path.join(data_dir, usim_output_store_name) diff --git a/pilates/urbansim/postprocessor.py b/pilates/urbansim/postprocessor.py index 6f17fa40..320de74b 100644 --- a/pilates/urbansim/postprocessor.py +++ b/pilates/urbansim/postprocessor.py @@ -7,7 +7,6 @@ def _get_usim_datastore_fname(settings, io, year=None): - if io == 'output': datastore_name = settings['usim_formattable_output_file_name'].format( year=year) @@ -16,59 +15,64 @@ def _get_usim_datastore_fname(settings, io, year=None): region_id = settings['region_to_region_id'][region] usim_base_fname = settings['usim_formattable_input_file_name'] datastore_name = usim_base_fname.format(region_id=region_id) + else: + raise ValueError(f"Invalid io parameter: {io}. Must be either 'input' or 'output'") return datastore_name -def create_next_iter_usim_data(settings, year, forecast_year): - - data_dir = settings['usim_local_data_folder'] +def create_next_iter_usim_data(settings, year, forecast_year, full_path): + data_dir = settings['usim_local_mutable_data_folder'] # Move UrbanSim input store (e.g. custom_mpo_193482435_model_data.h5) # to archive (e.g. input_data_for_2015_outputs.h5) because otherwise # it will be overwritten in the next step. input_datastore_name = _get_usim_datastore_fname(settings, io='input') input_store_path = os.path.join(data_dir, input_datastore_name) - if os.path.exists(input_store_path): - archive_fname = 'input_data_for_{0}_outputs.h5'.format(forecast_year) - logger.info( - "Moving urbansim inputs from the previous iteration to {0}".format( - archive_fname)) - new_input_store_path = input_store_path.replace( - input_datastore_name, archive_fname) - os.rename(input_store_path, new_input_store_path) - og_input_store = pd.HDFStore(new_input_store_path) - new_input_store = pd.HDFStore(input_store_path) - assert len(new_input_store.keys()) == 0 - updated_tables = [] + # First check if urbansim model is activated + if settings.get('land_use_model') == 'urbansim': + if os.path.exists(input_store_path): + archive_fname = 'input_data_for_{0}_outputs.h5'.format(forecast_year) + logger.info("Moving urbansim inputs from the previous iteration to {0}".format(archive_fname)) + new_input_store_path = input_store_path.replace(input_datastore_name, archive_fname) + os.rename(input_store_path, new_input_store_path) + og_input_store = pd.HDFStore(str(new_input_store_path)) + new_input_store = pd.HDFStore(str(input_store_path)) + assert len(new_input_store.keys()) == 0 + updated_tables = [] - # load last iter output data - output_datastore_name = _get_usim_datastore_fname(settings, 'output', forecast_year) - output_store_path = os.path.join(data_dir, output_datastore_name) - - # copy usim outputs into new input data store - logger.info( - 'Merging results back into UrbanSim and storing as .h5!') - output_store, table_prefix_year = read_datastore(settings, forecast_year) + # load last iter output data + # output_datastore_name = _get_usim_datastore_fname(settings, 'output', forecast_year) + # output_store_path = os.path.join(data_dir, output_datastore_name) - for h5_key in output_store.keys(): - table_name = h5_key.split('/')[-1] - if os.path.join('/', table_prefix_year, table_name) == h5_key: - updated_tables.append(table_name) - new_input_store[table_name] = output_store[h5_key] - - # copy missing tables from original usim inputs into new input data store - for h5_key in og_input_store.keys(): - table_name = h5_key.split('/')[-1] - if table_name not in updated_tables: + # copy usim outputs into new input data store logger.info( - "Copying {0} input table to output store!".format( - table_name)) - new_input_store[table_name] = og_input_store[h5_key] + 'Merging results back into UrbanSim and storing as .h5!') + output_store, table_prefix_year = read_datastore(settings, forecast_year, mutable_data_dir=full_path) + + for h5_key in output_store.keys(): + table_name = h5_key.split('/')[-1] + if os.path.join('/', table_prefix_year, table_name) == h5_key: + updated_tables.append(table_name) + new_input_store[table_name] = output_store[h5_key] + + # copy missing tables from original usim inputs into new input data store + for h5_key in og_input_store.keys(): + table_name = h5_key.split('/')[-1] + if table_name not in updated_tables: + logger.info( + "Copying {0} input table to output store!".format( + table_name)) + new_input_store[table_name] = og_input_store[h5_key] - assert new_input_store.keys() == og_input_store.keys() - og_input_store.close() - new_input_store.close() - output_store.close() - + assert new_input_store.keys() == og_input_store.keys() + og_input_store.close() + new_input_store.close() + output_store.close() + else: + logger.error(f"Input store path {input_store_path} does not exist") + return # or handle this case appropriately + else: + # If urbansim is not activated, just log an info message and continue + logger.info("Urbansim model is not activated, skipping input store path check") \ No newline at end of file diff --git a/pilates/urbansim/preprocessor.py b/pilates/urbansim/preprocessor.py index d3c527e9..f9f8a683 100644 --- a/pilates/urbansim/preprocessor.py +++ b/pilates/urbansim/preprocessor.py @@ -31,7 +31,7 @@ } -def _load_raw_skims(settings, skim_format): +def _load_raw_skims(settings, asim_data_dir, skim_format): skims_fname = settings.get('skims_fname', False) try: @@ -52,14 +52,8 @@ def _load_raw_skims(settings, skim_format): 'destination': 'to_zone_id', 'TOTIVT_IVT_minutes': 'SOV_AM_IVT_mins'}) elif skims_fname.endswith('omx'): - beam_output_dir = settings['beam_local_output_folder'] - skims_fname = settings['skims_fname'] - mutable_skims_location = os.path.join(beam_output_dir, skims_fname) - region_id = settings['region_to_region_id'][settings['region']] - input_skims_location = "pilates/urbansim/data/skims_mpo_{0}.omx".format(region_id) - logger.info( - "Copying skims from {0} to {1} for urbansim".format(mutable_skims_location, input_skims_location)) - shutil.copyfile(mutable_skims_location, input_skims_location) + skims_fname = "skims.omx" + mutable_skims_location = os.path.join(asim_data_dir, skims_fname) skims = omx.open_file(mutable_skims_location, 'r') zone_ids = skims.mapping('zone_id').keys() index = pd.Index(zone_ids, name="from_zone_id", dtype=str) @@ -104,27 +98,67 @@ def _load_raw_skims(settings, skim_format): return skims -def usim_model_data_fname(region_id): - return 'custom_mpo_{0}_model_data.h5'.format(region_id) - - -def add_skims_to_model_data(settings, data_dir=None): +def copy_data_to_mutable_location(settings, output_dir): + region = settings['region'] + region_id = settings['region_to_region_id'][region] + year_specific_model_data_fname = settings.get('usim_formattable_input_file_name_year', '').format( + region_id=region_id, + start_year=settings[ + 'start_year']) + model_data_fname = settings['usim_formattable_input_file_name'].format(region_id=region_id) + ## TODO: Copy over other file if start year is 2017 + data_dir = settings['usim_local_data_input_folder'] + if os.path.exists(os.path.join(data_dir, year_specific_model_data_fname)) & ( + settings.get('usim_formattable_input_file_name_year') is not None): + src = os.path.join(data_dir, year_specific_model_data_fname) + else: + src = os.path.join(data_dir, model_data_fname) + dest = os.path.join(output_dir, model_data_fname) + logger.info("Copying input urbansim data from {0} to {1}".format(src, dest)) + shutil.copyfile(src, dest) + other_data_fnames = [ + "hsize_ct_{0}.csv".format(region_id), + "income_rates_{0}.csv".format(region_id), + "relmap_{0}.csv".format(region_id), + "schools_2010.csv", + "blocks_school_districts_2010.csv" + ] + for fname in other_data_fnames: + src = os.path.join(data_dir, fname) + dest = os.path.join(output_dir, fname) + if os.path.exists(src): + logger.info("Copying input urbansim file from {0} to {1}".format(src, dest)) + shutil.copyfile(src, dest) + + +def add_skims_to_model_data(settings, data_dir=None, skims_dir=None): mapping = geoid_to_zone_map(settings) - output_geoid_loc = "pilates/urbansim/data/geoid_to_zone.csv" + if not data_dir: + data_dir = settings['usim_local_data_input_folder'] + output_geoid_loc = os.path.join(data_dir, "geoid_to_zone.csv") logger.info("Writing zone mapping to {0}".format(output_geoid_loc)) pd.Series(mapping).to_frame("zone_id").rename_axis("GEOID").to_csv(output_geoid_loc) # load skims logger.info("Loading skims from disk") region = settings['region'] - skim_format = settings['travel_model'] - df = _load_raw_skims(settings, skim_format=skim_format) + region_id = settings['region_to_region_id'][region] + skim_format = settings['travel_model'] or "beam" + df = _load_raw_skims(settings, skims_dir, skim_format=skim_format) + if skims_dir is not None: + source = os.path.join( + data_dir.replace(settings['usim_local_mutable_data_folder'], settings['asim_local_mutable_data_folder']), + "skims.omx") + dest = os.path.join(data_dir, "skims_mpo_{0}.omx".format(region_id)) + shutil.copyfile(source, dest) + logger.info("Copying skims from {0} to {1}".format(source, dest)) + + ## IF SKIMS DON"T EXIST IN USIM DIRECTORY COPY THEM OVER # load datastore - region_id = settings['region_to_region_id'][region] + model_data_fname = settings['usim_formattable_input_file_name'].format(region_id=region_id) - if not data_dir: - data_dir = settings['usim_local_data_folder'] + model_data_fpath = os.path.join(data_dir, model_data_fname) if not os.path.exists(model_data_fpath): raise ValueError('No input data found at {0}'.format( diff --git a/pilates/utils/geog.py b/pilates/utils/geog.py index f37b9e45..223421aa 100644 --- a/pilates/utils/geog.py +++ b/pilates/utils/geog.py @@ -1,3 +1,5 @@ +import time + import geopandas as gpd import pandas as pd import logging @@ -65,7 +67,8 @@ def get_taz_geoms(settings, taz_id_col_in='taz1454', zone_id_col_out='zone_id', ## FIX ME: other regions taz should be here - only sfbay for now print(url) - gdf = gpd.read_file(url, crs="EPSG:4326") + gdf = gpd.read_file(url) + gdf = gdf.set_crs("EPSG:4326", allow_override=True) print(list(gdf)) if region == 'austin': mapping = geoid_to_zone_map(settings) @@ -109,14 +112,16 @@ def get_county_block_geoms( blocks_remaining = True all_features = [] page = 0 - while blocks_remaining: + max_pages = 100 + while blocks_remaining and page < max_pages: offset = page * result_size url = base_url.format(state_fips, county_fips, result_size, offset) result = requests.get(url) try: features = result.json()['features'] - except KeyError: - logger.error("No features returned. Try a smaller result size.") + except Exception as e: + logger.error(f"Error parsing features: {e}") + break all_features += features if 'exceededTransferLimit' in result.json().keys(): if result.json()['exceededTransferLimit']: @@ -128,6 +133,10 @@ def get_county_block_geoms( blocks_remaining = False else: page += 1 + time.sleep(0.2) # be nice to the API + + if page == max_pages: + logger.warning("Reached max_pages limit in get_county_block_geoms, possible infinite loop avoided.") df = pd.DataFrame() for feature in all_features: @@ -300,7 +309,7 @@ def geoid_to_zone_map(settings, year=None): zone_type = settings['skims_zone_type'] travel_model = settings.get('travel_model', 'beam') zone_id_col = 'zone_id' - geoid_to_zone_folder = os.path.join("pilates", "utils", "data", region, travel_model) + geoid_to_zone_folder = os.path.join("pilates", "utils", "data", region, travel_model or "beam") if not os.path.exists(geoid_to_zone_folder): os.makedirs(geoid_to_zone_folder) diff --git a/pilates/utils/io.py b/pilates/utils/io.py index e1a03a5f..b1f26b66 100644 --- a/pilates/utils/io.py +++ b/pilates/utils/io.py @@ -33,8 +33,11 @@ def parse_args_and_settings(settings_file='settings.yaml'): '"l" for land use, "a" for activity demand, ' '"t" for traffic assignment. Can specify multiple (e.g. "at")')) parser.add_argument( - '-c', '--config', action='store', + '-c', '--config', default='settings.yaml', help='config file name') + parser.add_argument( + '-S', '--stage', default="current_stage.yaml", + help='current state file to pick up from') args = parser.parse_args() if args.config: @@ -48,7 +51,9 @@ def parse_args_and_settings(settings_file='settings.yaml'): settings.update({ 'static_skims': args.static_skims, 'warm_start_skims': args.warm_start_skims, - 'asim_validation': args.figures}) + 'asim_validation': args.figures, + 'state_file_loc': args.stage, + 'settings_file': settings_file}) # override .yaml settings with command-line values if command-line # values are not False/None @@ -104,12 +109,13 @@ def parse_args_and_settings(settings_file='settings.yaml'): return settings -def read_datastore(settings, year=None, warm_start=False): +def read_datastore(settings, year=None, warm_start=False, mutable_data_dir=None): """ Access to the land use .H5 data store """ # If `year` is the start year of the whole simulation, or `warm_start` is - # True, then land use forecasting has been skipped. This is useful for + # True, or if urbansim is turned off entirely, then land use forecasting + # has been skipped. This is useful for # generating "warm start" skims for the base year. In this case, the # ActivitySim inputs must be created from the base year land use *inputs* # since no land use outputs have been created yet. @@ -119,26 +125,40 @@ def read_datastore(settings, year=None, warm_start=False): region = settings['region'] region_id = settings['region_to_region_id'][region] - usim_local_data_folder = settings['usim_local_data_folder'] + if mutable_data_dir is None: + data_loc = settings['usim_local_data_input_folder'] + else: + data_loc = os.path.join(mutable_data_dir, settings['usim_local_mutable_data_folder']) + urbansim_enabled = settings.get('land_use_model') is not None - if (year == settings['start_year']) or (warm_start): + if (year == settings['start_year']) or warm_start or not urbansim_enabled: + logger.info( + "Year {0}, start year {1}, warm_start {2}, urbansim_enabled {3}".format(year, settings['start_year'], + warm_start, urbansim_enabled)) table_prefix_yr = '' # input data store tables have no year prefix usim_datastore = settings['usim_formattable_input_file_name'].format( region_id=region_id) + usim_datastore_fpath = os.path.join(data_loc, usim_datastore) + store = pd.HDFStore(usim_datastore_fpath) + if "households" not in store: + table_prefix_yr = str(year) + if "{0}/households".format(table_prefix_yr) not in store: + raise KeyError("No households table of either format found in {0}. Tables: {1}".format(usim_datastore_fpath, store.keys())) + else: + logger.info("Using {0}/households table".format(table_prefix_yr)) # Otherwise we read from the land use outputs else: usim_datastore = settings['usim_formattable_output_file_name'].format( year=year) table_prefix_yr = str(year) + usim_datastore_fpath = os.path.join(data_loc, usim_datastore) + store = pd.HDFStore(usim_datastore_fpath) - usim_datastore_fpath = os.path.join(usim_local_data_folder, usim_datastore) logger.info("Opening urbansim datastore at {0}".format(usim_datastore)) if not os.path.exists(usim_datastore_fpath): raise ValueError('No land use data found at {0}!'.format( usim_datastore_fpath)) - store = pd.HDFStore(usim_datastore_fpath) - return store, table_prefix_yr diff --git a/run.py b/run.py index b32df0d1..65d0de6a 100644 --- a/run.py +++ b/run.py @@ -1,12 +1,12 @@ +import argparse import random import warnings +from datetime import datetime import pandas as pd import tables from tables import HDF5ExtError -from pilates.activitysim.preprocessor import copy_beam_geoms - warnings.simplefilter(action='ignore', category=FutureWarning) from workflow_state import WorkflowState @@ -106,106 +106,91 @@ def find_latest_beam_iteration(beam_output_dir): print(iter_dirs) -def setup_beam_skims(settings): - region = settings['region'] - region_id = settings['region_to_region_id'][region] - beam_input_dir = settings['beam_local_input_folder'] - beam_output_dir = settings['beam_local_output_folder'] - skims_fname = settings['skims_fname'] - origin_skims_fname = settings['origin_skims_fname'] - beam_geoms_fname = settings['beam_geoms_fname'] - beam_router_directory = settings['beam_router_directory'] - asim_geoms_location = os.path.join(settings['asim_local_input_folder'], beam_geoms_fname) - - input_skims_location = os.path.join(beam_input_dir, region, skims_fname) - mutable_skims_location = os.path.join(beam_output_dir, skims_fname) - - beam_geoms_location = os.path.join(beam_input_dir, region, beam_router_directory, beam_geoms_fname) - - # TODO: Handle exception when these dont exist - - if os.path.exists(input_skims_location): - logger.info("Copying input skims from {0} to {1}".format( - input_skims_location, - mutable_skims_location)) - shutil.copyfile(input_skims_location, mutable_skims_location) - else: - if os.path.exists(mutable_skims_location): - logger.info("No input skims at {0}. Proceeding with defaults at {1}".format( - input_skims_location, - mutable_skims_location)) - else: - logger.info("No default skims found anywhere. We will generate defaults instead") - - input_skims_location = os.path.join(beam_input_dir, region, origin_skims_fname) - mutable_skims_location = os.path.join(beam_output_dir, origin_skims_fname) - - if os.path.exists(input_skims_location): - logger.info("Copying input origin skims from {0} to {1}".format( - input_skims_location, - mutable_skims_location)) - shutil.copyfile(input_skims_location, mutable_skims_location) - else: - if os.path.exists(mutable_skims_location): - logger.info("No input skims at {0}. Proceeding with defaults at {1}".format( - input_skims_location, - mutable_skims_location)) - else: - logger.info("No default input skims found anywhere. We will generate defaults instead") - - logger.info("Copying beam zone geoms from {0} to {1}".format( - beam_geoms_location, - asim_geoms_location)) - - copy_beam_geoms(settings, beam_geoms_location, asim_geoms_location) - - -def get_base_asim_cmd(settings, household_sample_size=None): +def get_base_asim_cmd(settings, household_sample_size=None, num_processes=None): formattable_asim_cmd = settings['asim_formattable_command'] if not household_sample_size: household_sample_size = settings.get('household_sample_size', 0) - num_processes = settings.get('num_processes', multiprocessing.cpu_count() - 1) + num_processes = num_processes or settings.get('num_processes', multiprocessing.cpu_count() - 1) chunk_size = settings.get('chunk_size', 0) # default no chunking base_asim_cmd = formattable_asim_cmd.format( household_sample_size, num_processes, chunk_size) return base_asim_cmd - -def get_asim_docker_vols(settings): +def get_asim_additional_args(asim_docker_vols, compile): + additional_args = [] + if settings.get("file_format", "parquet") == "parquet": + additional_args.append("--persist-sharrow-cache") + for local, d in asim_docker_vols.items(): + if "data" in d['bind']: + additional_args.append('-d') + additional_args.append('"{0}"'.format(d['bind'])) + elif "output" in d['bind']: + additional_args.append('-o') + additional_args.append('"{0}"'.format(d['bind'])) + elif "compile" in d['bind']: + if compile: + additional_args.append('-c') + additional_args.append('"{0}"'.format(d['bind'])) + elif "configs" in d['bind']: + additional_args.append('-c') + additional_args.append('"{0}"'.format(d['bind'])) + return additional_args + +def get_asim_docker_vols(settings, working_dir=None): region = settings['region'] asim_subdir = settings['region_to_asim_subdir'][region] asim_remote_workdir = os.path.join('/activitysim', asim_subdir) - asim_local_input_folder = os.path.abspath( - settings['asim_local_input_folder']) - asim_local_output_folder = os.path.abspath( - settings['asim_local_output_folder']) - asim_local_configs_folder = os.path.abspath( - os.path.join(settings['asim_local_configs_folder'], region)) + if working_dir is not None: + asim_local_mutable_data_folder = os.path.abspath( + os.path.join(working_dir, settings['asim_local_mutable_data_folder'])) + asim_local_output_folder = os.path.abspath( + os.path.join(working_dir, settings['asim_local_output_folder'])) + asim_local_configs_folder = os.path.abspath( + os.path.join(working_dir, settings['asim_local_mutable_configs_folder'], settings.get('asim_main_configs_dir', "configs"))) + asim_local_configs_compile_folder = os.path.abspath( + os.path.join(working_dir, settings['asim_local_mutable_configs_folder'], "configs_sh_compile")) + else: + asim_local_mutable_data_folder = os.path.abspath( + settings['asim_local_mutable_data_folder']) + asim_local_output_folder = os.path.abspath( + settings['asim_local_output_folder']) + asim_local_configs_folder = os.path.abspath( + os.path.join(settings['asim_local_configs_folder'], region, "configs")) + asim_local_configs_compile_folder = os.path.abspath( + os.path.join(settings['asim_local_configs_folder'], region, "configs_sh_compile")) asim_remote_input_folder = os.path.join( asim_remote_workdir, 'data') asim_remote_output_folder = os.path.join( asim_remote_workdir, 'output') asim_remote_configs_folder = os.path.join( asim_remote_workdir, 'configs') + asim_remote_configs_compile_folder = os.path.join( + asim_remote_workdir, 'configs_sh_compile') asim_docker_vols = { - asim_local_input_folder: { + asim_local_mutable_data_folder: { 'bind': asim_remote_input_folder, 'mode': 'rw'}, asim_local_output_folder: { 'bind': asim_remote_output_folder, 'mode': 'rw'}, + asim_local_configs_compile_folder: { + 'bind': asim_remote_configs_compile_folder, + 'mode': 'rw'}, asim_local_configs_folder: { 'bind': asim_remote_configs_folder, - 'mode': 'rw'}} + 'mode': 'rw'} + } return asim_docker_vols -def get_usim_docker_vols(settings): +def get_usim_docker_vols(settings, output_dir=None): usim_remote_data_folder = settings['usim_client_data_folder'] - usim_local_data_folder = os.path.abspath( - settings['usim_local_data_folder']) + if output_dir is None: + output_dir = settings['usim_local_data_input_folder'] + usim_local_mutable_data_folder = os.path.abspath( + output_dir) usim_docker_vols = { - usim_local_data_folder: { + usim_local_mutable_data_folder: { 'bind': usim_remote_data_folder, 'mode': 'rw'}} return usim_docker_vols @@ -224,9 +209,13 @@ def get_usim_cmd(settings, year, forecast_year): ## Atlas vehicle ownership model volume mount defintion, equivalent to ## docker run -v atlas_host_input_folder:atlas_container_input_folder -def get_atlas_docker_vols(settings): - atlas_host_input_folder = os.path.abspath(settings['atlas_host_input_folder']) - atlas_host_output_folder = os.path.abspath(settings['atlas_host_output_folder']) +def get_atlas_docker_vols(settings, working_dir=None): + if working_dir is None: + atlas_host_input_folder = os.path.abspath(settings['atlas_host_input_folder']) + else: + atlas_host_input_folder = os.path.abspath( + os.path.join(working_dir, settings['atlas_host_mutable_input_folder'])) + atlas_host_output_folder = os.path.abspath(os.path.join(working_dir or "", settings['atlas_host_output_folder'])) atlas_container_input_folder = os.path.abspath(settings['atlas_container_input_folder']) atlas_container_output_folder = os.path.abspath(settings['atlas_container_output_folder']) atlas_docker_vols = { @@ -258,7 +247,8 @@ def warm_start_activities(settings, year, client): activity_demand_model, activity_demand_image = get_model_and_image(settings, 'activity_demand_model') if activity_demand_model == 'polaris': - run_polaris(None, settings, warm_start=True) + # run_polaris(None, settings, warm_start=True) + logger.info("POLARIS module is not activated due to missing polarisruntime library") elif activity_demand_model == 'activitysim': # 1. PARSE SETTINGS @@ -286,9 +276,9 @@ def warm_start_activities(settings, year, client): logger.info("Creating {0} input data from {1} outputs".format( activity_demand_model, land_use_model).upper()) - if not os.path.exists(os.path.join(settings['asim_local_input_folder'], 'skims.omx')): - asim_pre.create_skims_from_beam(settings, year, overwrite=False) - asim_pre.create_asim_data_from_h5(settings, year, warm_start=True) + if not os.path.exists(os.path.join(settings['asim_local_mutable_data_folder'], 'skims.omx')): + asim_pre.create_skims_from_beam(settings, state, overwrite=False) + asim_pre.create_asim_data_from_h5(settings, state, warm_start=True) # 3. RUN ACTIVITYSIM IN WARM START MODE logger.info("Running {0} in warm start mode".format( @@ -307,26 +297,32 @@ def warm_start_activities(settings, year, client): return -def forecast_land_use(settings, year, forecast_year, client, container_manager): - run_land_use(settings, year, forecast_year, client) +def forecast_land_use(settings, year, workflow_state: WorkflowState, client, container_manager): + run_land_use(settings, year, workflow_state, client) # check for outputs, exit if none - usim_local_data_folder = settings['usim_local_data_folder'] usim_output_store = settings['usim_formattable_output_file_name'].format( - year=forecast_year) - usim_datastore_fpath = os.path.join(usim_local_data_folder, usim_output_store) + year=workflow_state.forecast_year) + output_dir = os.path.join(workflow_state.output_path, workflow_state.folder_name, + settings['usim_local_mutable_data_folder']) + usim_datastore_fpath = os.path.join(output_dir, usim_output_store) if not os.path.exists(usim_datastore_fpath): logger.critical( - "No UrbanSim output data found. It probably did not finish successfully.") + "No UrbanSim output data found at {0}. It probably did not finish successfully.".format( + usim_datastore_fpath)) sys.exit(1) -def run_land_use(settings, year, forecast_year, client): +def run_land_use(settings, year, workflow_state: WorkflowState, client): logger.info("Running land use") # 1. PARSE SETTINGS + output_dir = os.path.join(workflow_state.output_path, workflow_state.folder_name, + settings['usim_local_mutable_data_folder']) + os.makedirs(output_dir, exist_ok=True) land_use_model, land_use_image = get_model_and_image(settings, "land_use_model") - usim_docker_vols = get_usim_docker_vols(settings) + usim_docker_vols = get_usim_docker_vols(settings, output_dir) + forecast_year = workflow_state.forecast_year usim_cmd = get_usim_cmd(settings, year, forecast_year) # 2. PREPARE URBANSIM DATA @@ -334,7 +330,11 @@ def run_land_use(settings, year, forecast_year, client): "Preparing {0} input data for land use development simulation.".format( year)) formatted_print(print_str) - usim_pre.add_skims_to_model_data(settings) + + # usim_pre.copy_data_to_mutable_location(settings, output_dir) + asim_output_dir = os.path.join(workflow_state.output_path, workflow_state.folder_name, + settings['asim_local_mutable_data_folder']) + usim_pre.add_skims_to_model_data(settings, output_dir, asim_output_dir) if is_already_opened_in_write_mode(usim_data_path): logger.warning( @@ -356,11 +356,16 @@ def run_land_use(settings, year, forecast_year, client): ## Atlas: evolve household vehicle ownership -def run_atlas(settings, output_year, client, warm_start_atlas, atlas_run_count=1): +def run_atlas(settings, state: WorkflowState, client, warm_start_atlas, forecast=False, atlas_run_count=1): # warm_start: warm_start_atlas = True, output_year = year = start_year # asim_no_usim: warm_start_atlas = True, output_year = year (should = start_year) # normal: warm_start_atlas = False, output_year = forecast_year + if forecast: + yr = state.forecast_year + else: + yr = state.start_year + # 1. PARSE SETTINGS vehicle_ownership_model, atlas_image = get_model_and_image(settings, "vehicle_ownership_model") freq = settings.get('vehicle_ownership_freq', False) @@ -372,18 +377,18 @@ def run_atlas(settings, output_year, client, warm_start_atlas, atlas_run_count=1 rebfactor = settings.get('atlas_rebfactor', 0) taxfactor = settings.get('atlas_taxfactor', 0) discIncent = settings.get('atlas_discIncent', 0) - atlas_docker_vols = get_atlas_docker_vols(settings) - atlas_cmd = get_atlas_cmd(settings, freq, output_year, npe, nsample, beamac, mod, adscen, rebfactor, taxfactor, + atlas_docker_vols = get_atlas_docker_vols(settings, state.full_path) + atlas_cmd = get_atlas_cmd(settings, freq, yr, npe, nsample, beamac, mod, adscen, rebfactor, taxfactor, discIncent) docker_stdout = settings.get('docker_stdout', False) # 2. PREPARE ATLAS DATA if warm_start_atlas: print_str = ( - "Preparing input data for warm start vehicle ownership simulation for {0}.".format(output_year)) + "Preparing input data for warm start vehicle ownership simulation for {0}.".format(yr)) else: print_str = ( - "Preparing input data for vehicle ownership simulation for {0}.".format(output_year)) + "Preparing input data for vehicle ownership simulation for {0}.".format(yr)) formatted_print(print_str) # create skims.omx (lines moved from warm_start_activities) @@ -395,7 +400,12 @@ def run_atlas(settings, output_year, client, warm_start_atlas, atlas_run_count=1 # prepare atlas inputs from urbansim h5 output # preprocessed csv input files saved in "atlas/atlas_inputs/year{}/" - atlas_pre.prepare_atlas_inputs(settings, output_year, warm_start=warm_start_atlas) + if forecast: + yrs = [y + 2 for y in range(state.year, yr, 2)] + else: + yrs = [yr] + for yr_it in yrs: + atlas_pre.prepare_atlas_inputs(settings, yr_it, state, warm_start=warm_start_atlas) # calculate accessibility if atlas_beamac != 0 if beamac > 0: @@ -408,22 +418,22 @@ def run_atlas(settings, output_year, client, warm_start_atlas, atlas_run_count=1 # 'WLK_COM_DRV', 'WLK_EXP_DRV', 'WLK_HVY_DRV', 'WLK_LOC_DRV', 'WLK_LRF_DRV', # 'DRV_COM_WLK', 'DRV_EXP_WLK', 'DRV_HVY_WLK', 'DRV_LOC_WLK', 'DRV_LRF_WLK'] # measure_list = ['WACC','IWAIT','XWAIT','TOTIVT','WEGR','DTIM'] - atlas_pre.compute_accessibility(path_list, measure_list, settings, output_year) + atlas_pre.compute_accessibility(path_list, measure_list, settings, state.forecast_year) # 3. RUN ATLAS via docker container client print_str = ( "Simulating vehicle ownership for {0} " "with frequency {1}, npe {2} nsample {3} beamac {4}".format( - output_year, freq, npe, nsample, beamac)) + yr, freq, npe, nsample, beamac)) formatted_print(print_str) run_container(client, settings, atlas_image, atlas_docker_vols, atlas_cmd, working_dir='/') # 4. ATLAS OUTPUT -> UPDATE USIM OUTPUT CARS & HH_CARS - atlas_post.atlas_update_h5_vehicle(settings, output_year, warm_start=warm_start_atlas) + atlas_post.atlas_update_h5_vehicle(settings, yr, state, warm_start=warm_start_atlas) # 5. ATLAS OUTPUT -> ADD A VEHICLETYPEID COL FOR BEAM - atlas_post.atlas_add_vehileTypeId(settings, output_year) - atlas_post.build_beam_vehicles_input(settings, output_year) + atlas_post.atlas_add_vehileTypeId(settings, yr, state) + atlas_post.build_beam_vehicles_input(settings, yr, state) logger.info('Atlas Done!') @@ -434,20 +444,24 @@ def run_atlas(settings, output_year, client, warm_start_atlas, atlas_run_count=1 # run_atlas_auto is a run_atlas upgraded version, which will run_atlas again if # outputs are not generated. This is mainly for preventing crash due to parellel # computiing errors that can be resolved by a simple resubmission -def run_atlas_auto(settings, output_year, client, warm_start_atlas): - atlas_output_path = settings['atlas_host_output_folder'] - fname = 'vehicles_{}.csv'.format(output_year) +def run_atlas_auto(settings, state: WorkflowState, client, warm_start_atlas, forecast=False): + if forecast: + yr = state.forecast_year + else: + yr = state.start_year + atlas_output_path = os.path.join(state.full_path, settings['atlas_host_output_folder']) + fname = 'vehicles_{}.csv'.format(yr) if os.path.exists(os.path.join(atlas_output_path, fname)) & warm_start_atlas: logger.info( "Running in warm start mode but warm started files for year {0} already exist. Assuming we can skip this " "step and move on to the forecast year".format( - output_year)) + yr)) return # run atlas atlas_run_count = 1 # try: - run_atlas(settings, output_year, client, warm_start_atlas, atlas_run_count) + run_atlas(settings, state, client, warm_start_atlas, forecast, atlas_run_count) # except: # logger.error('ATLAS RUN #{} FAILED'.format(atlas_run_count)) @@ -457,7 +471,7 @@ def run_atlas_auto(settings, output_year, client, warm_start_atlas): if not os.path.exists(os.path.join(atlas_output_path, fname)): logger.error('LAST ATLAS RUN FAILED -> RE-LAUNCHING ATLAS RUN #{} BELOW'.format(atlas_run_count)) try: - run_atlas(settings, output_year, client, warm_start_atlas, atlas_run_count) + run_atlas(settings, state, client, warm_start_atlas, forecast, atlas_run_count) except: logger.error('ATLAS RUN #{} FAILED'.format(atlas_run_count)) @@ -465,7 +479,7 @@ def run_atlas_auto(settings, output_year, client, warm_start_atlas): def generate_activity_plans( - settings, year, forecast_year, client, + settings, year, state: WorkflowState, client, resume_after=None, warm_start=False, overwrite_skims=True, @@ -485,12 +499,13 @@ def generate_activity_plans( if settings.get('regenerate_seed', True): new_seed = random.randint(0, int(1e9)) logger.info("Re-seeding asim with new seed {0}".format(new_seed)) - asim_pre.update_asim_config(settings, "random_seed", new_seed) + asim_pre.update_asim_config(settings, state.full_path, "random_seed", new_seed) activity_demand_model, activity_demand_image = get_model_and_image(settings, 'activity_demand_model') if activity_demand_model == 'polaris': - run_polaris(forecast_year, settings, warm_start=True) + # run_polaris(state.forecast_year, settings, warm_start=True) + logger.info("POLARIS module is not activated due to missing polarisruntime library") elif activity_demand_model == 'activitysim': @@ -499,9 +514,9 @@ def generate_activity_plans( land_use_model = settings['land_use_model'] region = settings['region'] asim_subdir = settings['region_to_asim_subdir'][region] - asim_workdir = os.path.join('/activitysim', asim_subdir) - asim_docker_vols = get_asim_docker_vols(settings) - asim_cmd = get_base_asim_cmd(settings) + asim_workdir = os.path.join('activitysim', asim_subdir) + asim_docker_vols = get_asim_docker_vols(settings, state.full_path) + docker_stdout = settings.get('docker_stdout', False) # If this is the first iteration, skims should only exist because @@ -517,24 +532,45 @@ def generate_activity_plans( land_use_model) formatted_print(print_str) asim_pre.create_skims_from_beam( - settings, year=forecast_year, overwrite=overwrite_skims) - asim_pre.create_asim_data_from_h5(settings, year=forecast_year, warm_start=warm_start) + settings, state=state, overwrite=overwrite_skims) + asim_pre.create_asim_data_from_h5(settings, state=state, warm_start=warm_start) # 3. GENERATE ACTIVITY PLANS print_str = ( "Generating activity plans for the year " "{0} with {1}".format( - forecast_year, activity_demand_model)) + state.forecast_year, activity_demand_model)) + + if not state.asim_compiled: + asim_cmd = get_base_asim_cmd(settings, household_sample_size=2500, num_processes=1) + if resume_after: + asim_cmd += ' -r {0}'.format(resume_after) + + additional_args = get_asim_additional_args(asim_docker_vols, True) + success = run_container(client, settings, + activity_demand_image, + working_dir=asim_workdir, + volumes=asim_docker_vols, + command=asim_cmd, + args=additional_args) + logger.info("ASIM Compilation success: {0}".format(success)) + # if not success: + # raise RuntimeError("ASim Compilation failed") + state.compile_asim() + asim_cmd = get_base_asim_cmd(settings) if resume_after: asim_cmd += ' -r {0}'.format(resume_after) print_str += ". Picking up after {0}".format(resume_after) formatted_print(print_str) + additional_args = get_asim_additional_args(asim_docker_vols, False) + run_container(client, settings, activity_demand_image, working_dir=asim_workdir, volumes=asim_docker_vols, - command=asim_cmd) + command=asim_cmd, + args=additional_args) # 4. COPY ACTIVITY DEMAND OUTPUTS --> LAND USE INPUTS # If generating activities for the base year (i.e. warm start), @@ -544,9 +580,9 @@ def generate_activity_plans( print_str = ( "Generating {0} {1} input data from " "{2} outputs".format( - forecast_year, land_use_model, activity_demand_model)) + state.forecast_year, land_use_model, activity_demand_model)) formatted_print(print_str) - asim_post.create_next_iter_inputs(settings, year, forecast_year) + asim_post.create_next_iter_inputs(settings, year, state) logger.info('Done!') @@ -554,30 +590,41 @@ def generate_activity_plans( def run_traffic_assignment( - settings, year, forecast_year, client, replanning_iteration_number=0): + settings, year, state: WorkflowState, client, replanning_iteration_number=0): """ This step will run the traffic simulation platform and generate new skims with updated congested travel times. """ + logger.info("===== STARTING TRAFFIC ASSIGNMENT =====") travel_model, travel_model_image = get_model_and_image(settings, 'travel_model') + logger.info(f"Travel model: {travel_model}, Image: {travel_model_image}") + if travel_model == 'polaris': - run_polaris(forecast_year, settings, warm_start=False) + # run_polaris(state.forecast_year, settings, warm_start=False) + logger.info("POLARIS module is not activated due to missing polarisruntime library") elif travel_model == 'beam': # 1. PARSE SETTINGS beam_config = settings['beam_config'] region = settings['region'] - path_to_beam_config = '/app/input/{0}/{1}'.format( - region, beam_config) - beam_local_input_folder = settings['beam_local_input_folder'] - abs_beam_input = os.path.abspath(beam_local_input_folder) - beam_local_output_folder = settings['beam_local_output_folder'] - abs_beam_output = os.path.abspath(beam_local_output_folder) + path_to_beam_config = '/app/input/{0}/{1}'.format(region, beam_config) + run_path = state.full_path + beam_local_mutable_data_folder = os.path.join(run_path, settings['beam_local_mutable_data_folder']) + abs_beam_input = os.path.abspath(str(beam_local_mutable_data_folder)) + logger.info(f"Absolute path to BEAM input: {abs_beam_input} -> Container: /app/input (rw)") + + beam_local_output_folder = os.path.join(run_path, settings['beam_local_output_folder']) + abs_beam_output = os.path.abspath(str(beam_local_output_folder)) + logger.info(f"Absolute path to BEAM output: {abs_beam_output} -> Container: /app/output (rw)") + activity_demand_model = settings.get('activity_demand_model', False) + logger.info(f"Activity demand model: {activity_demand_model}") + docker_stdout = settings['docker_stdout'] skims_fname = settings['skims_fname'] origin_skims_fname = settings['origin_skims_fname'] beam_memory = settings.get('beam_memory', str(int(psutil.virtual_memory().total / (1024. ** 3)) - 2) + 'g') + logger.info(f"BEAM memory allocation: {beam_memory}") # remember the last produced skims in order to detect that # beam didn't work properly during this run @@ -589,7 +636,8 @@ def run_traffic_assignment( logger.error("Invalid skim format {0}".format(skims_fname)) previous_od_skims = beam_post.find_produced_od_skims(beam_local_output_folder, skimFormat) previous_origin_skims = beam_post.find_produced_origin_skims(beam_local_output_folder) - logger.info("Found skims from the previous beam run: %s", previous_od_skims) + if previous_origin_skims: + logger.info(f"Found skims from the previous BEAM run: {previous_od_skims}") # 2. COPY ACTIVITY DEMAND OUTPUTS --> TRAFFIC ASSIGNMENT INPUTS if settings['traffic_assignment_enabled']: @@ -598,13 +646,21 @@ def run_traffic_assignment( "{2} outputs".format( year, travel_model, activity_demand_model)) formatted_print(print_str) + logger.info("Copying plans from ActivitySim to BEAM") beam_pre.copy_plans_from_asim( - settings, year, replanning_iteration_number) + settings, state, replanning_iteration_number) # 3. RUN BEAM - logger.info( - "Starting beam container, input: %s, output: %s, config: %s", - abs_beam_input, abs_beam_output, beam_config) + logger.info("Starting BEAM container, input: %s, output: %s, config: %s", abs_beam_input, abs_beam_output, + beam_config) + + # Check if the beam config file exists + expected_config_path = os.path.join(abs_beam_input, region, beam_config) + if os.path.exists(expected_config_path): + logger.info(f"BEAM config file exists at host path: {expected_config_path}") + else: + logger.warning(f"BEAM config file NOT FOUND at expected host path: {expected_config_path}") + run_container( client, settings, @@ -626,8 +682,11 @@ def run_traffic_assignment( # 4. POSTPROCESS path_to_mutable_od_skims = os.path.join(abs_beam_output, skims_fname) path_to_origin_skims = os.path.join(abs_beam_output, origin_skims_fname) + logger.info(f"Path to mutable OD skims: {path_to_mutable_od_skims}") + logger.info(f"Path to origin skims: {path_to_origin_skims}") if skimFormat == "csv.gz": + logger.info("Processing CSV.GZ format skims") current_od_skims = beam_post.merge_current_od_skims( path_to_mutable_od_skims, previous_od_skims, beam_local_output_folder) if current_od_skims == previous_od_skims: @@ -640,21 +699,49 @@ def run_traffic_assignment( beam_post.merge_current_origin_skims( path_to_origin_skims, previous_origin_skims, beam_local_output_folder) else: - asim_data_dir = settings['asim_local_input_folder'] - asim_skims_path = os.path.join(asim_data_dir, 'skims.omx') - current_od_skims = beam_post.merge_current_omx_od_skims(asim_skims_path, previous_od_skims, - beam_local_output_folder, settings) + logger.info("Processing OMX format skims") + + # Check if ActivitySim is enabled - only proceed with ActivitySim integration if it's enabled + asim_enabled = activity_demand_model and activity_demand_model == 'activitysim' + beam_asim_ridehail_measure_map = settings['beam_asim_ridehail_measure_map'] + if not asim_enabled: + logger.info("ActivitySim is not enabled, skipping skim merging for ActivitySim") + # Still check if BEAM produced skims + current_od_skims = beam_post.find_produced_od_skims(beam_local_output_folder, "omx") + if current_od_skims == previous_od_skims and replanning_iteration_number > 0: + logger.error( + "BEAM hasn't produced the new skims at {0} for some reason. " + "Please check beamLog.out for errors in the directory {1}".format(current_od_skims, + abs_beam_output) + ) + return + elif settings["file_format"] == "parquet": + asim_data_dir = os.path.join(run_path, settings['asim_local_output_folder'], "cache") + asim_skims_path = os.path.join(asim_data_dir, 'skims.zarr') + current_od_skims = beam_post.merge_current_zarr_od_skims(asim_skims_path, + beam_local_output_folder, settings) + logger.warning("RIDEHAIL SKIM MERGING NOT YET IMPLEMENTED FOR PARQUET FILES") + else: + asim_data_dir = os.path.join(state.full_path, settings['asim_local_mutable_data_folder']) + asim_skims_path = os.path.join(asim_data_dir, 'skims.omx') + current_od_skims = beam_post.merge_current_omx_od_skims(asim_skims_path, previous_od_skims, + beam_local_output_folder, settings) + beam_post.merge_current_omx_origin_skims( + asim_skims_path, previous_origin_skims, beam_local_output_folder, + beam_asim_ridehail_measure_map) + logger.info(f"ActivitySim data directory: {asim_data_dir}") + logger.info(f"ActivitySim skims path: {asim_skims_path}") + if current_od_skims == previous_od_skims: logger.error( "BEAM hasn't produced the new skims at {0} for some reason. " "Please check beamLog.out for errors in the directory {1}".format(current_od_skims, abs_beam_output) ) sys.exit(1) - beam_asim_ridehail_measure_map = settings['beam_asim_ridehail_measure_map'] - beam_post.merge_current_omx_origin_skims( - asim_skims_path, previous_origin_skims, beam_local_output_folder, - beam_asim_ridehail_measure_map) - beam_post.rename_beam_output_directory(settings, year, replanning_iteration_number) + + logger.info(f"Renaming BEAM output directory for year {year}, iteration {replanning_iteration_number}") + beam_post.rename_beam_output_directory(abs_beam_output, settings, year, replanning_iteration_number) + logger.info("===== COMPLETED TRAFFIC ASSIGNMENT =====") return @@ -687,7 +774,7 @@ def initialize_asim_for_replanning(settings, forecast_year): region = settings['region'] asim_subdir = settings['region_to_asim_subdir'][region] asim_workdir = os.path.join('/activitysim', asim_subdir) - asim_docker_vols = get_asim_docker_vols(settings) + asim_docker_vols = get_asim_docker_vols(settings, state.full_path) base_asim_cmd = get_base_asim_cmd(settings, replan_hh_samp_size) docker_stdout = settings.get('docker_stdout', False) @@ -702,19 +789,19 @@ def initialize_asim_for_replanning(settings, forecast_year): command=base_asim_cmd) -def run_replanning_loop(settings, forecast_year): +def run_replanning_loop(state: WorkflowState): replan_iters = settings['replan_iters'] replan_hh_samp_size = settings['replan_hh_samp_size'] activity_demand_model, activity_demand_image = get_model_and_image(settings, 'activity_demand_model') region = settings['region'] asim_subdir = settings['region_to_asim_subdir'][region] asim_workdir = os.path.join('/activitysim', asim_subdir) - asim_docker_vols = get_asim_docker_vols(settings) + asim_docker_vols = get_asim_docker_vols(settings, state.full_path) base_asim_cmd = get_base_asim_cmd(settings, replan_hh_samp_size) docker_stdout = settings.get('docker_stdout', False) last_asim_step = settings['replan_after'] - for i in range(replan_iters): + for i in range(state.iteration, replan_iters): replanning_iteration_number = i + 1 print_str = ( 'Replanning Iteration {0}'.format(replanning_iteration_number)) @@ -723,10 +810,10 @@ def run_replanning_loop(settings, forecast_year): if settings.get('regenerate_seed', True): new_seed = random.randint(0, int(1e9)) logger.info("Re-seeding asim with new seed {0}".format(new_seed)) - asim_pre.update_asim_config(settings, "random_seed", new_seed) + asim_pre.update_asim_config(settings, state.full_path,"random_seed", new_seed) # a) format new skims for asim - asim_pre.create_skims_from_beam(settings, forecast_year, overwrite=False) + asim_pre.create_skims_from_beam(settings, state, overwrite=False) # b) replan with asim print_str = ( @@ -742,12 +829,12 @@ def run_replanning_loop(settings, forecast_year): # e) run BEAM if replanning_iteration_number < replan_iters: - beam_pre.update_beam_config(settings, 'beam_replanning_portion') - beam_pre.update_beam_config(settings, 'max_plans_memory') - else: - beam_pre.update_beam_config(settings, 'beam_replanning_portion', 1.0) + beam_pre.update_beam_config(settings, working_dir, 'beam_replanning_portion') + beam_pre.update_beam_config(settings, working_dir, 'max_plans_memory') + # else: + # beam_pre.update_beam_config(settings, working_dir, 'beam_replanning_portion', 1.0) run_traffic_assignment( - settings, year, forecast_year, client, replanning_iteration_number) + settings, year, state, client, replanning_iteration_number) return @@ -783,7 +870,7 @@ def to_singularity_env(env): def run_container(client, settings: dict, image: str, volumes: dict, command: str, - working_dir=None, environment=None): + working_dir=None, environment=None, args=None) -> bool: """ Executes container using docker or singularity :param client: the docker client. If it's provided then docker is used, otherwise singularity is used @@ -798,6 +885,7 @@ def run_container(client, settings: dict, image: str, volumes: dict, command: st (or image layers at the docker hub) and find the last WORKDIR instruction or by issuing a command: docker run -it --entrypoint /bin/bash ghcr.io/lbnl-science-it/atlas:v1.0.7 -c "env | grep PWD" :param environment: a dictionary that contains environment variables that needs to be set to the container + :param args: additional arguments to the command """ if client: docker_stdout = settings.get('docker_stdout', False) @@ -819,18 +907,20 @@ def run_container(client, settings: dict, image: str, volumes: dict, command: st print(log) container.remove() logger.info("Finished docker container: %s, command: %s", image, command) + return True else: for local_folder in volumes: os.makedirs(local_folder, exist_ok=True) singularity_volumes = to_singularity_volumes(volumes) proc = ["singularity", "run", "--cleanenv", "--writable-tmpfs"] \ + (["--env", to_singularity_env(environment)] if environment else []) \ - + (["--pwd", working_dir] if working_dir else []) \ - + ["-B", singularity_volumes, image] \ + + (["--pwd", working_dir] if working_dir else []) \ + + ["-B", singularity_volumes, image] + (args if args else []) \ + command.split() logger.info("Running command: %s", " ".join(proc)) - subprocess.run(proc) - logger.info("Finished command: %s", " ".join(proc)) + result = subprocess.run(proc) + logger.info("Finished command: %s with exit code %s", " ".join(proc), result.returncode) + return str(result) == "0" def get_model_and_image(settings: dict, model_type: str): @@ -851,6 +941,11 @@ def get_model_and_image(settings: dict, model_type: str): if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--config", help="Path to the config file to be used", + action="store_true") + parser.add_argument("-s", "--state", help="Path to the current_state file to be used", + action="store_true") logger = logging.getLogger(__name__) @@ -866,6 +961,8 @@ def get_model_and_image(settings: dict, model_type: str): # load args and settings settings = parse_args_and_settings() + logger.info("Using config file {}".format(settings['settings_file'])) + # parse scenario settings start_year = settings['start_year'] end_year = settings['end_year'] @@ -884,6 +981,9 @@ def get_model_and_image(settings: dict, model_type: str): replanning_enabled = settings['replanning_enabled'] container_manager = settings['container_manager'] + state = WorkflowState.from_settings(settings) + working_dir = state.full_path + if not land_use_enabled: print("LAND USE MODEL DISABLED") if not activity_demand_enabled: @@ -892,7 +992,7 @@ def get_model_and_image(settings: dict, model_type: str): print("TRAFFIC ASSIGNMENT MODEL DISABLED") if traffic_assignment_enabled: - beam_pre.update_beam_config(settings, 'beam_sample') + beam_pre.update_beam_config(settings, state.full_path, 'beam_sample') if warm_start_skims: formatted_print('"WARM START SKIMS" MODE ENABLED') @@ -901,9 +1001,6 @@ def get_model_and_image(settings: dict, model_type: str): formatted_print('"STATIC SKIMS" MODE ENABLED') logger.info('Using the same set of skims for every iteration.') - if settings.get('travel_model') == 'beam': - setup_beam_skims(settings) - # start docker client if container_manager == 'docker': client = initialize_docker_client(settings) @@ -913,71 +1010,94 @@ def get_model_and_image(settings: dict, model_type: str): ################################# # RUN THE SIMULATION WORKFLOW # ################################# - state = WorkflowState.from_settings(settings) + for year in state: # 1. FORECAST LAND USE if state.should_do(WorkflowState.Stage.land_use): - # hack: make sure that the usim datastore isn't open - usim_data_path = os.path.join(settings['usim_local_data_folder'], - settings['usim_formattable_input_file_name'].format( - region_id=settings['region_to_region_id'][settings['region']])) - if is_already_opened_in_write_mode(usim_data_path): - logger.warning( - "Closing h5 files {0} because they were left open. You should really " - "figure out where this happened".format(tables.file._open_files.filenames)) - tables.file._open_files.close_all() - - # 1a. IF START YEAR, WARM START MANDATORY ACTIVITIES - if (state.is_start_year()) and warm_start_activities_enabled: - # IF ATLAS ENABLED, UPDATE USIM INPUT H5 - if vehicle_ownership_model_enabled: - run_atlas_auto(settings, year, client, warm_start_atlas=True) - warm_start_activities(settings, year, client) - - # 1b. RUN LAND USE SIMULATION - forecast_land_use(settings, year, state.forecast_year, client, container_manager) - state.complete(WorkflowState.Stage.land_use) + # Skip if land use model is not enabled + if not land_use_enabled: + logger.info("Skipping land use stage: land use model not enabled") + state.complete(WorkflowState.Stage.land_use) + else: + # hack: make sure that the usim datastore isn't open + usim_data_path = os.path.join(settings['usim_local_data_input_folder'], + settings['usim_formattable_input_file_name'].format( + region_id=settings['region_to_region_id'][settings['region']])) + if is_already_opened_in_write_mode(usim_data_path): + logger.warning( + "Closing h5 files {0} because they were left open. You should really " + "figure out where this happened".format(tables.file._open_files.filenames)) + tables.file._open_files.close_all() + + # 1a. IF START YEAR, WARM START MANDATORY ACTIVITIES + if (state.is_start_year()) and warm_start_activities_enabled: + # IF ATLAS ENABLED, UPDATE USIM INPUT H5 + if vehicle_ownership_model_enabled: + run_atlas_auto(settings, state, client, warm_start_atlas=True) + warm_start_activities(settings, year, client) + + # 1b. RUN LAND USE SIMULATION + forecast_land_use(settings, year, state, client, container_manager) + state.complete(WorkflowState.Stage.land_use) # 2. RUN ATLAS (HOUSEHOLD VEHICLE OWNERSHIP) if state.should_do(WorkflowState.Stage.vehicle_ownership_model): - # If the forecast year is the same as the base year of this - # iteration, then land use forecasting has not been run. In this - # case, atlas need to update urbansim *inputs* before activitysim - # reads it in the next step. - if state.forecast_year == year: - run_atlas_auto(settings, year, client, warm_start_atlas=True) - - # If urbansim has been called, ATLAS will read, run, and update - # vehicle ownership info in urbansim *outputs* h5 datastore. - elif state.is_start_year(): - run_atlas_auto(settings, state.start_year, client, warm_start_atlas=True) - run_atlas_auto(settings, state.forecast_year, client, warm_start_atlas=False) + if state.forecast_year > 2017: + # If the forecast year is the same as the base year of this + # iteration, then land use forecasting has not been run. In this + # case, atlas need to update urbansim *inputs* before activitysim + # reads it in the next step. + if state.forecast_year == year: + run_atlas_auto(settings, state, client, warm_start_atlas=True) + + # If urbansim has been called, ATLAS will read, run, and update + # vehicle ownership info in urbansim *outputs* h5 datastore. + elif state.is_start_year(): + run_atlas_auto(settings, state, client, warm_start_atlas=True) + run_atlas_auto(settings, state, client, warm_start_atlas=False, forecast=True) + else: + run_atlas_auto(settings, state, client, warm_start_atlas=False, forecast=True) + else: + logger.info("Skipping atlas in year {0} because we can't start until 2017".format(state.year)) state.complete(WorkflowState.Stage.vehicle_ownership_model) # 3. GENERATE ACTIVITIES if state.should_do(WorkflowState.Stage.activity_demand): - # If the forecast year is the same as the base year of this - # iteration, then land use forecasting has not been run. In this - # case we have to read from the land use *inputs* because no - # *outputs* have been generated yet. This is usually only the case - # for generating "warm start" skims, so we treat it the same even - # if the "warm_start_skims" setting was not set to True at runtime - generate_activity_plans( - settings, year, state.forecast_year, client, warm_start=warm_start_skims or not land_use_enabled) + activity_demand_model = settings.get('activity_demand_model', False) + if activity_demand_model and activity_demand_enabled: + # If the forecast year is the same as the base year of this + # iteration, then land use forecasting has not been run. In this + # case we have to read from the land use *inputs* because no + # *outputs* have been generated yet. This is usually only the case + # for generating "warm start" skims, so we treat it the same even + # if the "warm_start_skims" setting was not set to True at runtime + generate_activity_plans( + settings, year, state, client, warm_start=warm_start_skims or not land_use_enabled) + else: + logger.info("Skipping activity demand generation: activity demand model not enabled") state.complete(WorkflowState.Stage.activity_demand) # 5. INITIALIZE ASIM LITE IF BEAM REPLANNING ENABLED # have to re-run asim all the way through on sample to shrink the # cache for use in re-planning, otherwise cache will use entire pop if state.should_do(WorkflowState.Stage.initialize_asim_for_replanning): - initialize_asim_for_replanning(settings, state.forecast_year) + activity_demand_model = settings.get('activity_demand_model', False) + if activity_demand_model == 'activitysim' and activity_demand_enabled and replanning_enabled: + initialize_asim_for_replanning(settings, state.forecast_year) + else: + logger.info("Skipping asim initialization for replanning: conditions not met") state.complete(WorkflowState.Stage.initialize_asim_for_replanning) if state.should_do(WorkflowState.Stage.activity_demand_directly_from_land_use): - # If not generating activities with a separate ABM (e.g. - # ActivitySim), then we need to create the next iteration of land - # use data directly from the last set of land use outputs. - usim_post.create_next_iter_usim_data(settings, year, state.forecast_year) + # Skip if land use model is not enabled + land_use_model = settings.get('land_use_model', False) + if not settings.get('land_use_enabled', False) or not land_use_model: + logger.info("Skipping direct activity generation from land use: land use model not enabled") + else: + # If not generating activities with a separate ABM (e.g. + # ActivitySim), then we need to create the next iteration of land + # use data directly from the last set of land use outputs. + usim_post.create_next_iter_usim_data(settings, year, state.forecast_year, state.full_path) state.complete(WorkflowState.Stage.activity_demand_directly_from_land_use) # DO traffic assignment - but skip if using polaris as this is done along @@ -986,25 +1106,45 @@ def get_model_and_image(settings: dict, model_type: str): # 4. RUN TRAFFIC ASSIGNMENT if settings['discard_plans_every_year']: - beam_pre.update_beam_config(settings, 'max_plans_memory', 0) + beam_pre.update_beam_config(settings, working_dir, 'max_plans_memory', 0) else: - beam_pre.update_beam_config(settings, 'max_plans_memory') - beam_pre.update_beam_config(settings, 'beam_replanning_portion', 1.0) + beam_pre.update_beam_config(settings, working_dir, 'max_plans_memory') + # beam_pre.update_beam_config(settings, working_dir, 'beam_replanning_portion', 1.0) if vehicle_ownership_model_enabled: - beam_pre.copy_vehicles_from_atlas(settings, state.forecast_year) - run_traffic_assignment(settings, year, state.forecast_year, client, -1) + beam_pre.copy_vehicles_from_atlas(settings, state) + run_traffic_assignment(settings, year, state, client, -1) state.complete(WorkflowState.Stage.traffic_assignment) # 5. REPLAN if state.should_do(WorkflowState.Stage.traffic_assignment_replan): - if replanning_enabled > 0: - run_replanning_loop(settings, state.forecast_year) - process_event_file(settings, year, settings['replan_iters']) - copy_outputs_to_mep(settings, year, settings['replan_iters']) - else: - process_event_file(settings, year, -1) - copy_outputs_to_mep(settings, year, -1) - beam_post.trim_inaccessible_ods(settings) + activity_demand_model = settings.get('activity_demand_model', False) + if activity_demand_model and activity_demand_model == 'activitysim': + if replanning_enabled > 0: + run_replanning_loop(state) + try: + process_event_file(settings, year, settings['replan_iters']) + copy_outputs_to_mep(settings, year, settings['replan_iters']) + except: + print("Skipping post") + else: + try: + process_event_file(settings, year, -1) + copy_outputs_to_mep(settings, year, -1) + except: + print("Skipping post") + state.complete(WorkflowState.Stage.traffic_assignment_replan) + activity_demand_model = settings.get('activity_demand_model', "") + if (activity_demand_model.lower() == "activitysim") and activity_demand_enabled: + if settings['file_format'] == "parquet": + try: + asim_data_dir = os.path.join(working_dir, settings['asim_local_output_folder'], "cache") + asim_skims_path = os.path.join(asim_data_dir, 'skims.zarr') + current_od_skims = beam_post.trim_inaccessible_ods_zarr(asim_skims_path, settings) + except Exception as e: + logger.error(f"Error trimming inaccessible ODs: {e}") + else: + beam_post.trim_inaccessible_ods(settings, working_dir) + logger.info("Finished") diff --git a/scenarios/settings--sfbay-pilates-base--jdeq.yaml b/scenarios/settings--sfbay-pilates-base--jdeq.yaml new file mode 100644 index 00000000..8236186b --- /dev/null +++ b/scenarios/settings--sfbay-pilates-base--jdeq.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: sfbay +scenario: base +start_year: 2017 +end_year: 2018 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://haitamlaarabi/beam:1.0.beta.1 + activitysim: docker://zaneedell/activitysim:0.1.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: haitamlaarabi/beam:1.0.beta.1 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: taz # one of [taz, block_group, block] +skims_fname: as-base-skims-sfbay-taz.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. +beam_router_directory: r5/r5-simple-no-local +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: pilates-base--jdeq + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 275000 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: scenarios/sfbay-pilates-base--jdeq.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "200g" +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 4 +copy_plans_from_asim_outputs: False +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 275000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings--sfbay-pilates-network--cbg2855-jdeq-400g-zn.yaml b/scenarios/settings--sfbay-pilates-network--cbg2855-jdeq-400g-zn.yaml new file mode 100644 index 00000000..3df1cda4 --- /dev/null +++ b/scenarios/settings--sfbay-pilates-network--cbg2855-jdeq-400g-zn.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: sfbay +scenario: base +start_year: 2017 +end_year: 2018 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://zaneedell/beam:1.0.1 + activitysim: docker://zaneedell/activitysim:0.1.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: zaneedell/beam:1.0.1 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: taz # one of [taz, block_group, block] +skims_fname: as-base-skims-sfbay-taz.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. +beam_router_directory: r5/sfbay-area-cbg2855-network +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: pilates-network--cbg2855-jdeq-400g-zn + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 275000 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: scenarios/network-calibration/sfbay-pilates-network--cbg2855-jdeq.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "400g" +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 4 +copy_plans_from_asim_outputs: False +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 275000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings--sfbay-pilates-network--cbg2855-jdeq-400g.yaml b/scenarios/settings--sfbay-pilates-network--cbg2855-jdeq-400g.yaml new file mode 100644 index 00000000..703dc81f --- /dev/null +++ b/scenarios/settings--sfbay-pilates-network--cbg2855-jdeq-400g.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: sfbay +scenario: base +start_year: 2017 +end_year: 2018 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://haitamlaarabi/beam:1.0.beta.1 + activitysim: docker://zaneedell/activitysim:0.1.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: haitamlaarabi/beam:1.0.beta.1 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: taz # one of [taz, block_group, block] +skims_fname: as-base-skims-sfbay-taz.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. +beam_router_directory: r5/sfbay-area-cbg2855-network +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: pilates-network--cbg2855-jdeq-400g + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 275000 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: scenarios/network-calibration/sfbay-pilates-network--cbg2855-jdeq.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "400g" +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 4 +copy_plans_from_asim_outputs: False +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 275000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings--sfbay-pilates-network--cbg2855-jdeq.yaml b/scenarios/settings--sfbay-pilates-network--cbg2855-jdeq.yaml new file mode 100644 index 00000000..2ddce130 --- /dev/null +++ b/scenarios/settings--sfbay-pilates-network--cbg2855-jdeq.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: sfbay +scenario: base +start_year: 2017 +end_year: 2018 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://haitamlaarabi/beam:1.0.beta.1 + activitysim: docker://zaneedell/activitysim:0.1.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: haitamlaarabi/beam:1.0.beta.1 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: taz # one of [taz, block_group, block] +skims_fname: as-base-skims-sfbay-taz.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. +beam_router_directory: r5/sfbay-area-cbg2855-network +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: pilates-network--cbg2855-jdeq + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 275000 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: scenarios/network-calibration/sfbay-pilates-network--cbg2855-jdeq.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "200g" +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 4 +copy_plans_from_asim_outputs: False +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 275000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings--sfbay-pilates-network--cbg6000-jdeq.yaml b/scenarios/settings--sfbay-pilates-network--cbg6000-jdeq.yaml new file mode 100644 index 00000000..c5e5fc65 --- /dev/null +++ b/scenarios/settings--sfbay-pilates-network--cbg6000-jdeq.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: sfbay +scenario: base +start_year: 2017 +end_year: 2018 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://haitamlaarabi/beam:1.0.beta.1 + activitysim: docker://zaneedell/activitysim:0.1.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: haitamlaarabi/beam:1.0.beta.1 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: taz # one of [taz, block_group, block] +skims_fname: as-base-skims-sfbay-taz.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. +beam_router_directory: r5/sfbay-area-cbg6000-network +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: pilates-network--cbg6000-jdeq + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 275000 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: scenarios/network-calibration/sfbay-pilates-network--cbg6000-jdeq.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "200g" +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 4 +copy_plans_from_asim_outputs: False +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 275000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings--sfbay-pilates-network--cbg9000-jdeq.yaml b/scenarios/settings--sfbay-pilates-network--cbg9000-jdeq.yaml new file mode 100644 index 00000000..88f0e840 --- /dev/null +++ b/scenarios/settings--sfbay-pilates-network--cbg9000-jdeq.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: sfbay +scenario: base +start_year: 2017 +end_year: 2018 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://haitamlaarabi/beam:1.0.beta.1 + activitysim: docker://zaneedell/activitysim:0.1.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: haitamlaarabi/beam:1.0.beta.1 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: taz # one of [taz, block_group, block] +skims_fname: as-base-skims-sfbay-taz.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. +beam_router_directory: r5/sfbay-area-cbg9000-network +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: pilates-network--cbg9000-jdeq + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 275000 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: scenarios/network-calibration/sfbay-pilates-network--cbg9000-jdeq.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "200g" +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 4 +copy_plans_from_asim_outputs: False +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 275000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings--sfbay-pilates-network--psimpl-sfres-jdeq-400g-hl.yaml b/scenarios/settings--sfbay-pilates-network--psimpl-sfres-jdeq-400g-hl.yaml new file mode 100644 index 00000000..6fe4b40a --- /dev/null +++ b/scenarios/settings--sfbay-pilates-network--psimpl-sfres-jdeq-400g-hl.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: sfbay +scenario: base +start_year: 2017 +end_year: 2018 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://haitamlaarabi/beam:1.0.beta.1 + activitysim: docker://zaneedell/activitysim:0.1.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: haitamlaarabi/beam:1.0.beta.1 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: taz # one of [taz, block_group, block] +skims_fname: as-base-skims-sfbay-taz.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. +beam_router_directory: r5/r5-partially-simplified-sfres +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: pilates-network--psimpl-sfres-jdeq-400g-hl + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 275000 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: scenarios/network-calibration/sfbay-pilates-network--psimpl-sfres-jdeq.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "400g" +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 4 +copy_plans_from_asim_outputs: False +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 275000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings--sfbay-pilates-network--psimpl-sfres-jdeq-400g-zn.yaml b/scenarios/settings--sfbay-pilates-network--psimpl-sfres-jdeq-400g-zn.yaml new file mode 100644 index 00000000..c0e99b12 --- /dev/null +++ b/scenarios/settings--sfbay-pilates-network--psimpl-sfres-jdeq-400g-zn.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: sfbay +scenario: base +start_year: 2017 +end_year: 2018 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://zaneedell/beam:1.0.1 + activitysim: docker://zaneedell/activitysim:0.1.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: zaneedell/beam:1.0.1 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: taz # one of [taz, block_group, block] +skims_fname: as-base-skims-sfbay-taz.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. +beam_router_directory: r5/r5-partially-simplified-sfres +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: pilates-network--psimpl-sfres-jdeq-400g-zn + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 275000 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: scenarios/network-calibration/sfbay-pilates-network--psimpl-sfres-jdeq.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "400g" +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 4 +copy_plans_from_asim_outputs: False +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 275000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/settings-campo.yaml b/scenarios/settings-campo.yaml similarity index 100% rename from settings-campo.yaml rename to scenarios/settings-campo.yaml diff --git a/scenarios/settings-s1.yaml b/scenarios/settings-s1.yaml new file mode 100644 index 00000000..5809908a --- /dev/null +++ b/scenarios/settings-s1.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: sfbay +scenario: base +start_year: 2017 +end_year: 2050 +land_use_freq: 1 +travel_model_freq: 6 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: urbansim +travel_model: beam +activity_demand_model: activitysim +vehicle_ownership_model: atlas + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://zaneedell/beam:0.9.13-beta-v4 + activitysim: docker://zaneedell/activitysim:0.1.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: zaneedell/beam:0.9.10 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: taz # one of [taz, block_group, block] +skims_fname: as-base-skims-sfbay-taz.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. +beam_router_directory: r5-simplified-sfres +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: atlas-ESS + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 0 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: ESS_const_220_price # ess_cons; zev_mandate; baseline +atlas_adscen: ess_cons # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: sfbay-pilates-network--simp-jdeq-07-storage-5-atlas-s1.conf # OR austin-pilates-base.conf +beam_sample: 0.1 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "200g" +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 3 +final_asim_plans_folder: pilates/activitysim/output/final_plans +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 0 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings-s2.yaml b/scenarios/settings-s2.yaml new file mode 100644 index 00000000..a8a07ca3 --- /dev/null +++ b/scenarios/settings-s2.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: sfbay +scenario: base +start_year: 2017 +end_year: 2050 +land_use_freq: 1 +travel_model_freq: 6 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: urbansim +travel_model: beam +activity_demand_model: activitysim +vehicle_ownership_model: atlas + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://zaneedell/beam:0.9.14-v5 + activitysim: docker://zaneedell/activitysim:0.1.13 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: zaneedell/beam:0.9.10 + activitysim: zaneedell/activitysim:0.1.7 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: taz # one of [taz, block_group, block] +skims_fname: as-base-skims-sfbay-taz.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. +beam_router_directory: r5-simplified-sfres +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: atlas-mandate + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 0 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: evMandForced2 # ess_cons; zev_mandate; baseline +atlas_adscen: zev_mandate # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: sfbay-pilates-network--simp-jdeq-07-storage-5-atlas-s2.conf # OR austin-pilates-base.conf +beam_sample: 0.1 +beam_replanning_portion: 1.0 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "210g" +skim_zone_source_id_col: objectid +replan_iters: 0 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: True +max_plans_memory: 3 +final_asim_plans_folder: pilates/activitysim/output/final_plans +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 0 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings-seattle-freight-parquet.yaml b/scenarios/settings-seattle-freight-parquet.yaml new file mode 100644 index 00000000..bed1295b --- /dev/null +++ b/scenarios/settings-seattle-freight-parquet.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: seattle +scenario: base +start_year: 2017 +end_year: 2023 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://zaneedell/beam:1.0.6 + activitysim: docker://zaneedell/activitysim:0.2.5 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: zaneedell/beam:1.0.6 + activitysim: zaneedell/activitysim:0.2.5 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: block_group # one of [taz, block_group, block] +skims_fname: as-base-skims-seattle-bg.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-seattle-bg.csv.gz #file or folder name. +beam_router_directory: r5/r5-residential-partiallysimplified-ferry +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: GEOID + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: seattle-freight-test-parquet + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 275000 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + +# BEAM +beam_config: seattle-base-pilates-parquet.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "200g" +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 4 +copy_plans_from_asim_outputs: False +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 +file_format: "parquet" + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 275000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings-seattle-freight.yaml b/scenarios/settings-seattle-freight.yaml new file mode 100644 index 00000000..bf35d705 --- /dev/null +++ b/scenarios/settings-seattle-freight.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: seattle +scenario: base +start_year: 2017 +end_year: 2023 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://haitamlaarabi/beam:1.0.beta.1 + activitysim: docker://zaneedell/activitysim:0.1.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: haitamlaarabi/beam:1.0.beta.1 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: block_group # one of [taz, block_group, block] +skims_fname: as-base-skims-seattle-bg.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-seattle-bg.csv.gz #file or folder name. +beam_router_directory: r5/r5-residential-partiallysimplified-ferry +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: GEOID + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: seattle-freight-teste + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 275000 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + +# BEAM +beam_config: seattle-base-pilates.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "200g" +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 4 +copy_plans_from_asim_outputs: False +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 +file_format: "csv.gz" + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 275000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings-seattle-newplans.yaml b/scenarios/settings-seattle-newplans.yaml new file mode 100644 index 00000000..753e8c77 --- /dev/null +++ b/scenarios/settings-seattle-newplans.yaml @@ -0,0 +1,456 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: seattle +scenario: base +start_year: 2010 +end_year: 2012 +land_use_freq: 1 +travel_model_freq: 2 +vehicle_ownership_freq: 2 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: activitysim +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://zaneedell/beam:0.9.13-beta-v2 + activitysim: docker://zaneedell/activitysim:0.1.7 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: zaneedell/beam:0.9.10 + activitysim: zaneedell/activitysim:0.1.1 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: block_group # one of [taz, block_group, block] +skims_fname: as-base-skims-seattle-bg.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-seattle-bg.csv.gz #file or folder name. +beam_router_directory: r5-dense +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: GEOID + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: fill-skims-final + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_host_output_folder: pilates/atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 0 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: seattle.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: 480g +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 3 +final_asim_plans_folder: pilates/activitysim/output/final_plans +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 22695 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: bay_area +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings-seattle-null.yaml b/scenarios/settings-seattle-null.yaml new file mode 100644 index 00000000..412592a3 --- /dev/null +++ b/scenarios/settings-seattle-null.yaml @@ -0,0 +1,457 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: seattle +scenario: base +start_year: 2010 +end_year: 2024 +land_use_freq: 1 +travel_model_freq: 2 +vehicle_ownership_freq: 2 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: activitysim +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://zaneedell/beam:1.0.1 + activitysim: docker://zaneedell/activitysim:0.1.12 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: zaneedell/beam:0.9.10 + activitysim: zaneedell/activitysim:0.1.1 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: block_group # one of [taz, block_group, block] +skims_fname: as-base-skims-seattle-bg.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-seattle-bg.csv.gz #file or folder name. +beam_router_directory: r5/r5-residential-partiallysimplified-ferry +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: GEOID + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: baseline-tourmode-null + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_host_output_folder: pilates/atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 0 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: seattle.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.25 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: 225g +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: True +max_plans_memory: 1 +final_asim_plans_folder: pilates/activitysim/output/final_plans +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 226950 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: bay_area +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings-seattle.yaml b/scenarios/settings-seattle.yaml new file mode 100644 index 00000000..259dd0ae --- /dev/null +++ b/scenarios/settings-seattle.yaml @@ -0,0 +1,457 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: seattle +scenario: base +start_year: 2010 +end_year: 2024 +land_use_freq: 1 +travel_model_freq: 2 +vehicle_ownership_freq: 2 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: activitysim +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://zaneedell/beam:1.0.1 + activitysim: docker://zaneedell/activitysim:0.1.12 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: zaneedell/beam:0.9.10 + activitysim: zaneedell/activitysim:0.1.1 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: block_group # one of [taz, block_group, block] +skims_fname: as-base-skims-seattle-bg.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-seattle-bg.csv.gz #file or folder name. +beam_router_directory: r5/r5-residential-partiallysimplified-ferry +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: GEOID + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: baseline-tourmode + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_host_output_folder: pilates/atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 0 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: seattle.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.25 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: 215g +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 3 +final_asim_plans_folder: pilates/activitysim/output/final_plans +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 226950 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: bay_area +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings-sfbay-freight--20241106--2018-Baseline.yaml b/scenarios/settings-sfbay-freight--20241106--2018-Baseline.yaml new file mode 100644 index 00000000..ea32ccfb --- /dev/null +++ b/scenarios/settings-sfbay-freight--20241106--2018-Baseline.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: sfbay +scenario: base +start_year: 2017 +end_year: 2023 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://haitamlaarabi/beam:1.0.beta.1 + activitysim: docker://zaneedell/activitysim:0.1.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: haitamlaarabi/beam:1.0.beta.1 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: taz # one of [taz, block_group, block] +skims_fname: as-base-skims-sfbay-taz.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. +beam_router_directory: r5/r5-simple-no-local +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: sfbay-freight--20241106--2018-Baseline + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 275000 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: scenarios/freight/sfbay-pilates-freight--20241106--2018-Baseline.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "200g" +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 4 +copy_plans_from_asim_outputs: False +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 275000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/scenarios/settings-sfbay-network.yaml b/scenarios/settings-sfbay-network.yaml new file mode 100644 index 00000000..ce1d9ef0 --- /dev/null +++ b/scenarios/settings-sfbay-network.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: sfbay +scenario: base +start_year: 2017 +end_year: 2023 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://haitamlaarabi/beam:1.0.beta.1 + activitysim: docker://zaneedell/activitysim:0.1.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: haitamlaarabi/beam:1.0.beta.1 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: taz # one of [taz, block_group, block] +skims_fname: as-base-skims-sfbay-taz.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. +beam_router_directory: r5/sfbay_residential_2855pop_network +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: pilates-network--2855pop-jdeq + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 275000 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: scenarios/network-calibration/sfbay-pilates-network--2855pop-jdeq.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "200g" +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 4 +copy_plans_from_asim_outputs: False +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 275000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/settings-new-asim-seattle-null.yaml b/settings-new-asim-seattle-null.yaml new file mode 100644 index 00000000..c3b075ec --- /dev/null +++ b/settings-new-asim-seattle-null.yaml @@ -0,0 +1,464 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: seattle +scenario: base +start_year: 2017 +end_year: 2025 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 1 + +# simulation platforms (leave blank to turn off) +land_use_model: # urbansim +travel_model: beam +activity_demand_model: activitysim +vehicle_ownership_model: # atlas + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + activitysim: docker://zaneedell/activitysim:0.2.5 + beam: docker://zaneedell/beam:1.0.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: zaneedell/beam:0.9.10 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: block_group # one of [taz, block_group, block] +skims_fname: as-base-skims-seattle-bg.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-seattle-bg.csv.gz #file or folder name. +beam_router_directory: r5/seattle-area-cbg412-ferry-network +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: GEOID + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: new-asim-new-map-v5-null + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 0 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: seattle-pilates.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 1.0 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "180g" +skim_zone_source_id_col: objectid +replan_iters: 0 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: True +max_plans_memory: 1 +final_asim_plans_folder: pilates/activitysim/output/final_plans +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 250000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +file_format: parquet +num_processes: 25 # 94 +asim_subdir: configs +asim_main_configs_dir: configs_extended +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "--households_sample_size {0} -m {1} -g {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: activitysim/examples/prototype_mtc_clean + seattle: activitysim/examples/prototype_mtc_clean +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/settings-new-asim-seattle.yaml b/settings-new-asim-seattle.yaml new file mode 100644 index 00000000..919395b4 --- /dev/null +++ b/settings-new-asim-seattle.yaml @@ -0,0 +1,466 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: seattle +scenario: base +start_year: 2010 +end_year: 2020 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 1 + +# simulation platforms (leave blank to turn off) +land_use_model: # urbansim +travel_model: beam +activity_demand_model: activitysim +vehicle_ownership_model: # atlas + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + activitysim: docker://zaneedell/activitysim:0.2.5 + beam: docker://zaneedell/beam:1.0.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: zaneedell/beam:0.9.10 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: block_group # one of [taz, block_group, block] +skims_fname: as-base-skims-seattle-bg.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-seattle-bg.csv.gz #file or folder name. +beam_router_directory: r5/seattle-area-cbg412-ferry-network +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: GEOID + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: new-asim-new-map-v5 + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 0 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: seattle-pilates.conf # OR austin-pilates-base.conf +beam_skims_shapefile: shape/block-groups-32048.shp +skim_zone_source_id_col: sk_zone +skim_zone_geoid_col: geoid10 +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "180g" +replan_iters: 0 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 5 +final_asim_plans_folder: pilates/activitysim/output/final_plans +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 250000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +file_format: parquet +num_processes: 25 # 94 +asim_subdir: configs +asim_main_configs_dir: configs_extended +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "--households_sample_size {0} -m {1} -g {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: activitysim/examples/prototype_mtc_clean + seattle: activitysim/examples/prototype_mtc_clean +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/settings-new-asim.yaml b/settings-new-asim.yaml new file mode 100644 index 00000000..b759cc6a --- /dev/null +++ b/settings-new-asim.yaml @@ -0,0 +1,464 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: sfbay +scenario: base +start_year: 2018 +end_year: 2025 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 1 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: activitysim +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://zaneedell/beam:1.0.11 + activitysim: docker://zaneedell/activitysim:0.2.5 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: zaneedell/beam:0.9.10 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: taz # one of [taz, block_group, block] +skims_fname: as-base-skims-sfbay-taz.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. +beam_router_directory: r5/r5-simple-no-local +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: new-asim + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 0 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: sfbay-pilates-base-omx.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "180g" +skim_zone_source_id_col: objectid +replan_iters: 0 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 5 +final_asim_plans_folder: pilates/activitysim/output/final_plans +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 250000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +file_format: parquet +num_processes: 25 # 94 +asim_subdir: configs +asim_main_configs_dir: configs_extended +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "--households_sample_size {0} -m {1} -g {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: activitysim/examples/prototype_mtc_clean + seattle: activitysim/examples/prototype_mtc_clean +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/settings-newbeam.yaml b/settings-newbeam.yaml new file mode 100644 index 00000000..3ceb5aac --- /dev/null +++ b/settings-newbeam.yaml @@ -0,0 +1,461 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: sfbay +scenario: base +start_year: 2017 +end_year: 2023 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 6 + +# simulation platforms (leave blank to turn off) +land_use_model: +travel_model: beam +activity_demand_model: activitysim +vehicle_ownership_model: + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + beam: docker://zaneedell/beam:1.0.beta + activitysim: docker://zaneedell/activitysim:0.1.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: zaneedell/beam:0.9.10 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: taz # one of [taz, block_group, block] +skims_fname: as-base-skims-sfbay-taz.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. +beam_router_directory: r5-simplified-sfres +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: new-sfbay-baseline + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 275000 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: sfbay-pilates-network--simp-jdeq-07-storage-5.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "200g" +skim_zone_source_id_col: objectid +replan_iters: 2 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 4 +final_asim_plans_folder: pilates/activitysim/output/final_plans +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 275000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +#num_processes: 90 # 94 +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "-h {0} -n {1} -c {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: bay_area + seattle: seattle +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/settings-seattle-fasterrail.yaml b/settings-seattle-fasterrail.yaml new file mode 100644 index 00000000..d9a3225e --- /dev/null +++ b/settings-seattle-fasterrail.yaml @@ -0,0 +1,464 @@ +######################################################################################################### + +############################ +# PILATES SETTINGS # +############################ + +# scenario defs +region: seattle +scenario: base +start_year: 2017 +end_year: 2025 +land_use_freq: 1 +travel_model_freq: 1 +vehicle_ownership_freq: 1 + +# simulation platforms (leave blank to turn off) +land_use_model: # urbansim +travel_model: beam +activity_demand_model: activitysim +vehicle_ownership_model: # atlas + +# docker or singularity +container_manager: singularity +#singularity settings +singularity_images: + urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 + atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 + activitysim: docker://zaneedell/activitysim:0.2.5 + beam: docker://zaneedell/beam:1.0.11 + polaris: + +# docker settings +docker_images: + urbansim: jdcaicedo/urbansim_demos:v0.1.1 + atlas: ghcr.io/lbnl-science-it/atlas:v1.0.7 + beam: zaneedell/beam:0.9.10 + activitysim: zaneedell/activitysim:0.1.4 + polaris: tbd +docker_stdout: False +pull_latest: False + +# skim settings +skims_zone_type: block_group # one of [taz, block_group, block] +skims_fname: as-base-skims-seattle-bg.omx +# skims_fname: as-base-skims-austin-bg.csv.gz +origin_skims_fname: as-origin-skims-seattle-bg.csv.gz #file or folder name. +beam_router_directory: r5-fasterrail/seattle-area-cbg412-ferry-network +beam_geoms_fname: clipped_tazs.csv +geoms_index_col: GEOID + +# region_zone_type: +# austin: block_groups # one of [taz, block_groups, block] +# detroit: block_groups # one of [taz, block_groups, block] +# sfbay: taz # one of [taz, block_groups, block] + +######################################################################################################### + +############################### +# SUB-PLATFORM SETTINGS # +############################### + +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: new-asim-new-map-v5-fasterrail + +# URBANSIM +region_to_region_id: + sfbay: "06197001" + austin: "48197301" + seattle: "53199100" +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ +usim_client_base_folder: /base/demos_urbansim +usim_client_data_folder: /base/demos_urbansim/data/ +usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" +usim_formattable_output_file_name: "model_data_{year}.h5" +usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" +warm_start_activities: False + +## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan +## Docker bind mounts: host == local; container == remote == client +atlas_host_input_folder: pilates/atlas/atlas_input +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output +atlas_container_input_folder: /atlas_input +atlas_container_output_folder: /atlas_output +atlas_basedir: / +atlas_codedir: / +atlas_sample_size: 0 # zero means no sampling, whole pop. +atlas_num_processes: 40 # suggested <9 for hima +atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc +atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline +atlas_adscen: baseline # ess_cons; zev_mandate; baseline +atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES +atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES +atlas_discIncent: 0 # whether to discount incentive dollars, 0 = NO (default), 1 = YES +atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --basedir {4} --codedir {5} --beamac {6} --mod {7} --adscen {8} --rebfactor {9} --taxfactor {10} --discIncent {11}" + + + +# BEAM +beam_config: seattle-faster-rail.conf # OR austin-pilates-base.conf +beam_sample: 1.0 +beam_replanning_portion: 0.4 +beam_local_input_folder: pilates/beam/production/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ +beam_scenario_folder: urbansim/ +beam_memory: "180g" +skim_zone_source_id_col: objectid +replan_iters: 0 +replan_hh_samp_size: 0 +replan_after: non_mandatory_tour_scheduling # resume asim after this step +discard_plans_every_year: False +max_plans_memory: 5 +final_asim_plans_folder: pilates/activitysim/output/final_plans +beam_simulated_hwy_paths: + - SOV + - HOV2 + - HOV3 + +# POLARIS +polaris_local_data_folder: pilates/polaris/data +polaris_skim_keys: + base: 'auto_skims' + t: 't4' # 8-9am + impedances: + - ivtt + - cost + +# MEP +mep_local_output_folder: pilates/postprocessing/MEP/ + +# ACTIVITYSIM +household_sample_size: 250000 # zero means no sampling, whole pop. +chunk_size: 12_000_000_000 # 4000000000 +file_format: parquet +num_processes: 25 # 94 +asim_subdir: configs +asim_main_configs_dir: configs_extended +asim_local_input_folder: pilates/activitysim/data/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ +asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ +asim_validation_folder: pilates/activitysim/validation +asim_formattable_command: "--households_sample_size {0} -m {1} -g {2}" +region_to_asim_subdir: + austin: austin + detroit: detroit + sfbay: activitysim/examples/prototype_mtc_clean + seattle: activitysim/examples/prototype_mtc_clean +region_to_asim_bucket: + austin: austin-activitysim + detroit: detroit-activitysim + sfbay: bayarea-activitysim + seattle: seattle-activitysim +periods: + - EA + - AM + - MD + - PM + - EV +transit_paths: + DRV_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + DRV_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_HVY_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LOC_DRV: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_LRF_DRV: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_EXP_DRV: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - DTIM + - BOARDS + - DDIST + - WAUX + WLK_COM_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_HVY_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_LOC_WLK: + - TOTIVT + - FAR + - XWAIT + - IWAIT + - BOARDS + - WAUX + WLK_LRF_WLK: + - TOTIVT + - FERRYIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_EXP_WLK: + - TOTIVT + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX + WLK_TRN_WLK: + - IVT + - WACC + - WEGR + - FAR + - XWAIT + - KEYIVT + - IWAIT + - BOARDS + - WAUX +hwy_paths: + - SOV + - SOVTOLL + - HOV2 + - HOV2TOLL + - HOV3 + - HOV3TOLL +ridehail_path_map: + RH_SOLO: Solo + RH_POOLED: Pooled +beam_asim_hwy_measure_map: + TIME: TIME_minutes + DIST: DIST_miles + BTOLL: null + VTOLL: VTOLL_FAR +beam_asim_transit_measure_map: + WAIT: null + IVT: TOTIVT_IVT_minutes + TOTIVT: TOTIVT_IVT_minutes + KEYIVT: KEYIVT_minutes + FERRYIVT: FERRYIVT_minutes + FAR: VTOLL_FAR + DTIM: DTIM_minutes + DDIST: DDIST_miles + WAUX: WAUX_minutes + WEGR: WEGR_minutes + WACC: WACC_minutes + IWAIT: IWAIT_minutes + XWAIT: XWAIT_minutes + BOARDS: BOARDS +beam_asim_ridehail_measure_map: + WAIT: waitTimeInMinutes + REJECTIONPROB: unmatchedRequestPortion +asim_output_tables: + prefix: final_ + tables: + - checkpoints + - land_use + - households + - persons + - tours + - trips + - joint_tour_participants + - plans +asim_from_usim_col_maps: + households: + persons: PERSONS # asim preproc + PERSONS: hhsize # asim settings.yaml + cars: auto_ownership # asim preproc + # VEHICL: auto_ownership # asim annotate_households.csv + workers: num_workers # asim settings.yaml + persons: + member_id: PNUM # asim preproc + land_use: + ZONE: TAZ # asim settings.yaml + COUNTY: county_id # asim settings.yaml +asim_to_usim_col_maps: + households: + hhsize: persons # asim postproc + num_workers: workers # asim postproc + auto_ownership: cars # asim postproc + persons: + PNUM: member_id # asim postproc + workplace_taz: work_zone_id # asim postproc + school_taz: school_taz_id # asim postproc + + +# POSTPROCESSING +postprocessing_output_folder: pilates/postprocessing/output +scenario_definitions: + name: "baseline" + lever: "default" + lever_position: 1.0 + + +######################################################################################################### + +################################ +# MISC. PILATES SETTINGS # +################################ + +# GEOGRAPHY SETTINGS +FIPS: + sfbay: + state: "06" + counties: + - "001" + - "013" + - "041" + - "055" + - "075" + - "081" + - "085" + - "095" + - "097" + austin: + state: "48" + counties: + - "021" + - "053" + - "055" + - "209" + - "453" + - "491" + seattle: + state: "53" + counties: + - "061" + - "035" + - "033" + - "053" +local_crs: + sfbay: EPSG:7131 + austin: EPSG:32140 + seattle: EPSG:32048 + +# VALIDATION METRIC LIBRARY +validation_metrics: + growth: + - population + - employment + - residential_units + commute_dists: + mode_shares: + auto_ownership: + density_gradients: + rent: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + jobs: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + population: + access: + var: employment + cost: minutes + levels: + - 15 + - 45 + diff --git a/settings.yaml b/settings.yaml index b5f1acb5..fe0dd753 100644 --- a/settings.yaml +++ b/settings.yaml @@ -8,10 +8,10 @@ region: sfbay scenario: base start_year: 2017 -end_year: 2021 +end_year: 2050 land_use_freq: 1 -travel_model_freq: 2 -vehicle_ownership_freq: 2 +travel_model_freq: 6 +vehicle_ownership_freq: 6 # simulation platforms (leave blank to turn off) land_use_model: urbansim @@ -25,8 +25,8 @@ container_manager: singularity singularity_images: urbansim: docker://jdcaicedo/urbansim_demos:v0.1.1 atlas: docker://ghcr.io/lbnl-science-it/atlas:sfb-v2.0.12 - beam: docker://zaneedell/beam:0.9.12 - activitysim: docker://zaneedell/activitysim:0.1.4 + beam: docker://zaneedell/beam:0.9.13-beta-v4 + activitysim: docker://zaneedell/activitysim:0.1.11 polaris: # docker settings @@ -44,8 +44,9 @@ skims_zone_type: taz # one of [taz, block_group, block] skims_fname: as-base-skims-sfbay-taz.omx # skims_fname: as-base-skims-austin-bg.csv.gz origin_skims_fname: as-origin-skims-sfbay-taz.csv.gz #file or folder name. -beam_router_directory: r5-simple-no-local +beam_router_directory: r5-simplified-sfres beam_geoms_fname: clipped_tazs.csv +geoms_index_col: taz1454 # region_zone_type: # austin: block_groups # one of [taz, block_groups, block] @@ -58,15 +59,20 @@ beam_geoms_fname: clipped_tazs.csv # SUB-PLATFORM SETTINGS # ############################### +output_directory: /global/scratch/users/$USER/pilates-output +output_run_name: atlas-baseline + # URBANSIM region_to_region_id: sfbay: "06197001" austin: "48197301" seattle: "53199100" -usim_local_data_folder: pilates/urbansim/data/ +usim_local_data_input_folder: pilates/urbansim/data/ +usim_local_mutable_data_folder: urbansim/data/ usim_client_base_folder: /base/demos_urbansim usim_client_data_folder: /base/demos_urbansim/data/ usim_formattable_input_file_name: "custom_mpo_{region_id}_model_data_2017.h5" +usim_formattable_input_file_name_year: "custom_mpo_{region_id}_model_data_{start_year}.h5" usim_formattable_output_file_name: "model_data_{year}.h5" usim_formattable_command: "-r {0} -i {1} -y {2} -f {3} -t {4}" warm_start_activities: False @@ -74,7 +80,9 @@ warm_start_activities: False ## ATLAS: vehicle simulation (may call usim somewhere) - Ling/Tin/Yuhan ## Docker bind mounts: host == local; container == remote == client atlas_host_input_folder: pilates/atlas/atlas_input -atlas_host_output_folder: pilates/atlas/atlas_output +atlas_warmstart_input_folder: pilates/atlas/atlas_input +atlas_host_mutable_input_folder: atlas/atlas_input +atlas_host_output_folder: atlas/atlas_output atlas_container_input_folder: /atlas_input atlas_container_output_folder: /atlas_output atlas_basedir: / @@ -83,6 +91,7 @@ atlas_sample_size: 0 # zero means no sampling, whole pop. atlas_num_processes: 40 # suggested <9 for hima atlas_beamac: 0 # 0 if use pre-processed accessibility RData, otherwise inline calc atlas_mod: 2 # 1 (static) or 2 (dynamic) +atlas_scenario: baseline # ess_cons; zev_mandate; baseline atlas_adscen: baseline # ess_cons; zev_mandate; baseline atlas_rebfactor: 1 # whether to turn on rebate incentive, 0 = NO, 1 = YES atlas_taxfactor: 1 # whether to turn on tax credit incentive, 0 = NO, 1 = YES @@ -92,18 +101,19 @@ atlas_formattable_command: "--freq {0} --outyear {1} --npe {2} --nsample {3} --b # BEAM -beam_config: sfbay-pilates-baseline2_0-omx.conf # OR austin-pilates-base.conf +beam_config: sfbay-pilates-network--simp-jdeq-07-storage-5.conf # OR austin-pilates-base.conf beam_sample: 0.1 beam_replanning_portion: 0.4 beam_local_input_folder: pilates/beam/production/ -beam_local_output_folder: pilates/beam/beam_output/ +beam_local_mutable_data_folder: beam/input/ +beam_local_output_folder: beam/beam_output/ beam_scenario_folder: urbansim/ -#beam_memory: "650g" +beam_memory: "200g" skim_zone_source_id_col: objectid replan_iters: 2 replan_hh_samp_size: 0 replan_after: non_mandatory_tour_scheduling # resume asim after this step -discard_plans_every_year: True +discard_plans_every_year: False max_plans_memory: 3 final_asim_plans_folder: pilates/activitysim/output/final_plans beam_simulated_hwy_paths: @@ -128,8 +138,10 @@ household_sample_size: 0 # zero means no sampling, whole pop. chunk_size: 12_000_000_000 # 4000000000 #num_processes: 90 # 94 asim_local_input_folder: pilates/activitysim/data/ -asim_local_output_folder: pilates/activitysim/output/ +asim_local_mutable_data_folder: activitysim/data/ +asim_local_output_folder: activitysim/output/ asim_local_configs_folder: pilates/activitysim/configs/ +asim_local_mutable_configs_folder: activitysim/configs/ asim_validation_folder: pilates/activitysim/validation asim_formattable_command: "-h {0} -n {1} -c {2}" region_to_asim_subdir: diff --git a/upload_via_globus.sh b/upload_via_globus.sh index 2f30df6f..0eb6ab4f 100755 --- a/upload_via_globus.sh +++ b/upload_via_globus.sh @@ -1,21 +1,34 @@ #!/bin/sh -if [ $# -eq 2 ] +if [ $# -eq 3 ] then - echo "Uploading results to GCloud in Region: $1 via Globus"; - echo "Output directory: $2"; + file=$1 + region=$2 + output_dir=$3 + + echo "Reading run state from file $file" + + folder_name=$(awk -F ':' '/folder_name/ {print $2=$2;}' $file | xargs) + path=$(awk -F ':' '/path/ {print $2=$2;}' $file | xargs) + + echo "Looking for output files at $path/$folder_name" + + echo "Uploading results to GCloud in Region: $region via Globus"; + echo "Output directory: $output_dir"; # Globus Tutorial Collection IDs - FROM="5791d5ee-c85a-4753-91f0-502a80d050d7:/global/scratch/users/zaneedell/sources/PILATES" + FROM="5791d5ee-c85a-4753-91f0-502a80d050d7:$path/$folder_name" TO="54047297-0b17-4dd9-ba50-ba1dc2063468:beam-core-outputs" - globus mkdir "$TO/$2" - globus mkdir "$TO/$2/beam" - globus mkdir "$TO/$2/activitysim" - globus mkdir "$TO/$2/activitysim/data" - globus transfer "$FROM/pilates/beam/beam_output/$1/" "$TO/$2/beam/" --recursive --label "BEAM Outputs" --exclude "*xml*" - globus transfer "$FROM/pilates/activitysim/output/" "$TO/$2/activitysim/" --recursive --label "ASim Outputs" --include "final*" --include "year*" --exclude "*" - globus transfer "$FROM/pilates/activitysim/data/" "$TO/$2/activitysim/data/" --recursive --label "ASim Inputs" + find "$FROM/activitysim/output/" -type f \( -name "households" -o -name "land_use" -o -name "persons" -o -name "plans" -o -name "tours" -o -name "trips" \) -exec sh -c 'echo "Renaming: $1 -> $1.parquet"; mv "$1" "$1.parquet"' _ {} \; + + globus mkdir "$TO/$output_dir" + globus mkdir "$TO/$output_dir/beam" + globus mkdir "$TO/$output_dir/activitysim" + globus mkdir "$TO/$output_dir/activitysim/data" + globus transfer "$FROM/beam/beam_output/$region/" "$TO/$output_dir/beam/" -s size --recursive --label "BEAM Outputs" --exclude "*xml*" + globus transfer "$FROM/activitysim/output/" "$TO/$output_dir/activitysim/" -s size --recursive --label "ASim Outputs" --include "year*" --exclude "*" + globus transfer "$FROM/activitysim/data/" "$TO/$output_dir/activitysim/data/" -s size --recursive --label "ASim Inputs" else echo "Please provide a region (e.g. 'austin' or 'sfbay') and S3 directory name" fi diff --git a/validate_skims.py b/validate_skims.py index 1bc019e2..fa0a5be0 100644 --- a/validate_skims.py +++ b/validate_skims.py @@ -3,23 +3,8 @@ import openmatrix as omx import numpy as np -if __name__ == '__main__': - if len(sys.argv) == 1: - path = "pilates/activitysim/sfbay/data/skims.omx" - dry_run = True - else: - path = sys.argv[1] - if len(sys.argv) >= 3: - dry_run = (sys.argv[2].lower() == 'true') - else: - print("Doing a try run. Add another argument of `false` to do a real run.") - dry_run = True - - if dry_run: - method = 'r' - else: - method = 'a' +def validate_skim(path, dry_run): sk = omx.open_file(path, method) tables = sk.list_matrices() @@ -54,8 +39,8 @@ sz_valid = sz - ignore.sum().sum() print("Finding {:0.2%} of speeds are 0, leaving them alone".format(ignore.sum().sum() / sz)) print("Finding {:0.2%} of ODs have valid observations, leaving them alone".format(nonzero.sum().sum() / sz)) - print("Finding {:0.2%} of remaining speeds are nan, replacing them with 30 mph".format( - np.isnan(spd_raw[~ignore]).sum() / sz_valid)) + print("Finding {:0.2%} of remaining {:} speeds are nan, replacing them with 30 mph".format( + np.isnan(spd_raw[~ignore]).sum() / sz_valid, sz_valid)) spd_raw[np.isnan(spd_raw)] = 30.0 print("Finding {0:0.2%} of remaining speeds are too fast, replacing them with {1} mph".format( (spd_raw[~ignore] > modeMaxSpeed).sum() / sz_valid, modeMaxSpeed)) @@ -102,3 +87,23 @@ sk[table][:] = newToll sk.close() + + +if __name__ == '__main__': + if len(sys.argv) == 1: + path = "pilates/activitysim/sfbay/data/skims.omx" + dry_run = True + else: + path = sys.argv[1] + if len(sys.argv) >= 3: + dry_run = (sys.argv[2].lower() == 'true') + else: + print("Doing a try run. Add another argument of `false` to do a real run.") + dry_run = True + + if dry_run: + method = 'r' + else: + method = 'a' + + validate_skim(path, dry_run) diff --git a/workflow_state.py b/workflow_state.py index e676e5b0..90c25687 100644 --- a/workflow_state.py +++ b/workflow_state.py @@ -1,23 +1,41 @@ +from datetime import datetime from enum import Enum import os import yaml import logging +from pilates.activitysim import preprocessor as asim_pre +from pilates.urbansim import preprocessor as usim_pre +from pilates.beam import preprocessor as beam_pre +from pilates.atlas import preprocessor as atlas_pre ## logger = logging.getLogger(__name__) + class WorkflowState: - Stage = Enum('WorkflowStage', ['land_use', 'vehicle_ownership_model', 'activity_demand', 'initialize_asim_for_replanning', 'activity_demand_directly_from_land_use', 'traffic_assignment', 'traffic_assignment_replan']) + Stage = Enum('WorkflowStage', + ['land_use', 'vehicle_ownership_model', 'activity_demand', 'initialize_asim_for_replanning', + 'activity_demand_directly_from_land_use', 'traffic_assignment', 'traffic_assignment_replan']) def __init__(self, start_year, end_year, travel_model_freq, land_use_enabled, vehicle_ownership_model_enabled, - activity_demand_enabled, traffic_assignment_enabled, replanning_enabled, year, stage): + activity_demand_enabled, traffic_assignment_enabled, replanning_enabled, year, stage, iteration, + output_path, folder_name, file_loc, asim_compiled=False): self.iteration_started = False self.start_year = start_year self.end_year = end_year self.travel_model_freq = travel_model_freq self.year = year self.stage = stage + self.iteration = iteration self.forecast_year = None self.enabled_stages = set([]) + self.folder_name = folder_name + self.output_path = output_path + self.file_loc = file_loc + self.__asim_compiled = asim_compiled + if year == 2010: + self.initial_step = 7 + else: + self.initial_step = None if land_use_enabled: self.enabled_stages.add(WorkflowState.Stage.land_use) if vehicle_ownership_model_enabled: @@ -32,6 +50,89 @@ def __init__(self, start_year, end_year, travel_model_freq, land_use_enabled, ve self.enabled_stages.add(WorkflowState.Stage.traffic_assignment) self.enabled_stages.add(WorkflowState.Stage.traffic_assignment_replan) + @property + def full_path(self): + return os.path.join(self.output_path, self.folder_name) + + @property + def asim_compiled(self): + if self.__asim_compiled: + logger.info("ActivitySim already compiled in year %s", self.year) + else: + logger.info("ActivitySim not compiled in year %s, so running compilation (this will take longer)", + self.year) + return self.__asim_compiled + + def compile_asim(self): + self.__asim_compiled = True + logger.info("Completed compiling activitysim in year %s", self.year) + WorkflowState.write_stage(self.year, self.stage, self.file_loc, self.output_path, self.folder_name, + self.iteration, self.__asim_compiled) + + def _create_output_dir(self, settings: dict): + dt = datetime.now().strftime("%Y%m%d-%H%M%S") + base_loc = os.path.expandvars(settings['output_directory']) + run_name = settings['output_run_name'] + folder_name = "{0}-{1}-{2}".format(settings['region'], run_name, dt) + folder_path = os.path.join(base_loc, folder_name) + os.makedirs(folder_path, exist_ok=True) + + have_not_copied_usim_data = True + + for model in ['travel_model', 'activity_demand_model', 'vehicle_ownership_model', 'land_use_model']: + if settings.get(model) is not None: + model_name = settings[model] + os.makedirs(os.path.join(folder_path, model_name), exist_ok=True) + if (model_name == "urbansim") | ((model_name == "activitysim") & have_not_copied_usim_data): + output_dir = os.path.join(folder_path, settings['usim_local_mutable_data_folder']) + os.makedirs(output_dir, exist_ok=True) + usim_pre.copy_data_to_mutable_location(settings, output_dir) + have_not_copied_usim_data = False + if model_name == "beam": + input_dir = os.path.join(folder_path, settings['beam_local_mutable_data_folder']) + os.makedirs(input_dir, exist_ok=True) + beam_pre.copy_data_to_mutable_location(settings, input_dir) + output_dir = os.path.join(folder_path, settings['beam_local_output_folder']) + os.makedirs(output_dir, exist_ok=True) + if model_name == "atlas": + input_dir = os.path.join(folder_path, settings['atlas_host_mutable_input_folder']) + os.makedirs(input_dir, exist_ok=True) + atlas_pre.copy_data_to_mutable_location(settings, input_dir) + output_dir = os.path.join(folder_path, settings['atlas_host_output_folder']) + os.makedirs(output_dir, exist_ok=True) + if model_name == "activitysim": + asim_pre.copy_data_to_mutable_location(settings, folder_path) + output_dir = os.path.join(folder_path, settings['asim_local_output_folder']) + os.makedirs(output_dir, exist_ok=True) + + self.output_path = base_loc + self.folder_name = folder_name + + print('STOP') + + @classmethod + def write_stage(cls, year: int, current_stage: Stage, file_loc, path, folder_name, iteration, asim_compiled): + to_save = {"year": year, "stage": current_stage.name if current_stage else None, "path": path, + "folder_name": folder_name, "iteration": iteration, "asim_compiled": asim_compiled} + with open(file_loc, mode="w", encoding="utf-8") as f: + yaml.dump(to_save, f) + + @classmethod + def read_current_stage(cls, file_loc): + if not os.path.exists(file_loc): + logger.info("Creating new stage info at {}".format(file_loc)) + return [None, None, None, None, None, False] + with open(file_loc, encoding="utf-8") as f: + data = yaml.load(f, Loader=yaml.FullLoader) + data = data if data is not None else {} + year = data.get('year', None) + stage_str = data.get('stage', 'null') + stage = None if stage_str == 'null' else WorkflowState.Stage[stage_str] + path = data.get('path', None) + folder_name = data.get('folder_name', None) + iteration = data.get('iteration', 0) or 0 + asim_compiled = data.get('asim_compiled', False) + return [year, stage, iteration, path, folder_name, asim_compiled] @classmethod def from_settings(cls, settings): @@ -43,44 +144,43 @@ def from_settings(cls, settings): activity_demand_enabled = settings['activity_demand_enabled'] traffic_assignment_enabled = settings['traffic_assignment_enabled'] replanning_enabled = settings['replanning_enabled'] - [year, stage] = cls.read_current_stage() + file_loc = settings['state_file_loc'] + copy_files = settings.get("copy_files", True) + [year, stage, iteration, path, folder_name, asim_compiled] = cls.read_current_stage(file_loc) if year: - logger.info("Found unfinished run: year=%s, stage=%s)", year, stage) + logger.info("Found unfinished run: year=%s, stage=%s, filename=%s)", year, stage, file_loc) year = year or start_year - return WorkflowState(start_year, end_year, travel_model_freq, land_use_enabled, vehicle_ownership_model_enabled, - activity_demand_enabled, traffic_assignment_enabled, replanning_enabled, year, stage) - - @classmethod - def write_stage(cls, year: int, current_stage: Stage): - to_save = {"year": year, "stage": current_stage.name if current_stage else None} - with open('current_stage.yaml', mode="w", encoding="utf-8") as f: - yaml.dump(to_save, f) - - @classmethod - def read_current_stage(cls): - if not os.path.exists('current_stage.yaml'): - return [None, None] - with open('current_stage.yaml', encoding="utf-8") as f: - data = yaml.load(f, Loader=yaml.FullLoader) - data = data if data is not None else {} - year = data.get('year', None) - stage_str = data.get('stage', 'null') - stage = None if stage_str == 'null' else WorkflowState.Stage[stage_str] - return [year, stage] + out = WorkflowState(start_year, end_year, travel_model_freq, land_use_enabled, vehicle_ownership_model_enabled, + activity_demand_enabled, traffic_assignment_enabled, replanning_enabled, year, stage, + iteration, path, folder_name, file_loc, asim_compiled) + if ((path is None) | (folder_name is None)) & copy_files: + out._create_output_dir(settings) + if not copy_files: + out.output_path = "" + out.folder_name = "pilates" + if year: + out.forecast_year = min(year + (out.initial_step or travel_model_freq), + end_year) if land_use_enabled else start_year + return out def enabled(self, stage) -> bool: return stage in self.enabled_stages def should_continue(self) -> bool: - next_year = self.year + self.travel_model_freq if self.iteration_started else self.year + if self.initial_step is not None: + step = self.initial_step + self.initial_step = None + else: + step = None + next_year = self.year + (step or self.travel_model_freq) if self.iteration_started else self.year if next_year >= self.end_year: return False self.year = next_year logger.info("Started year %d", self.year) self.iteration_started = True self.forecast_year = \ - min(self.year + self.travel_model_freq, self.end_year) if self.enabled(WorkflowState.Stage.land_use)\ - else self.start_year + min(self.year + (step or self.travel_model_freq), self.end_year) if self.enabled( + WorkflowState.Stage.land_use) else self.start_year return self.year < self.end_year def is_start_year(self): @@ -99,9 +199,16 @@ def complete(self, stage): self.stage = None [year, next_stage] = self.next_stage(self.year, stage) if year: - WorkflowState.write_stage(year, next_stage) + WorkflowState.write_stage(year, next_stage, self.file_loc, self.output_path, self.folder_name, 0, + self.__asim_compiled) else: - os.remove('current_stage.yaml') + os.remove(self.file_loc) + + def complete_iteration(self, iteration): + logger.info("Completed iteration %d of stage %s of %d", iteration, self.stage, self.year) + self.iteration += 1 + WorkflowState.write_stage(self.year, self.stage, self.file_loc, self.output_path, self.folder_name, + self.iteration, self.__asim_compiled) def next_stage(self, year: int, stage: Stage): next_enabled_stage = next(filter(self.enabled, list(WorkflowState.Stage)[stage.value:]), None) @@ -119,4 +226,4 @@ def __next__(self): if self.should_continue(): return self.year else: - raise StopIteration \ No newline at end of file + raise StopIteration