access to global constants and removing chooser col filtering

Author: dhensle

activitysim/abm/models/location_choice.py

@@ -161,6 +161,7 @@ def _location_sample(
         "reindex": reindex,
         "land_use": state.get_dataframe("land_use"),
     }
+    locals_d.update(state.get_global_constants())
     locals_d.update(model_settings.CONSTANTS or {})
 
     # preprocess choosers table
@@ -232,9 +233,7 @@ def location_sample(
     chunk_tag,
     trace_label,
 ):
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-    choosers = persons_merged[chooser_columns]
+    choosers = persons_merged
 
     # create wrapper with keys for this lookup - in this case there is a home_zone_id in the choosers
     # and a zone_id in the alternatives which get merged during interaction
@@ -385,12 +384,7 @@ def location_presample(
         HOME_TAZ in persons_merged
     )  # 'TAZ' should already be in persons_merged from land_use
 
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    # FIXME we don't actually require that land_use provide a TAZ crosswalk
-    # FIXME maybe we should add it for multi-zone (from maz_taz) if missing?
-    chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-    chooser_columns = [HOME_TAZ if c == HOME_MAZ else c for c in chooser_columns]
-    choosers = persons_merged[chooser_columns]
+    choosers = persons_merged
 
     # create wrapper with keys for this lookup - in this case there is a HOME_TAZ in the choosers
     # and a DEST_TAZ in the alternatives which get merged during interaction
@@ -555,11 +549,6 @@ def run_location_logsums(
         mandatory=False,
     )
 
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    persons_merged_df = logsum.filter_chooser_columns(
-        persons_merged_df, logsum_settings, model_settings
-    )
-
     logger.info(f"Running {trace_label} with {len(location_sample_df.index)} rows")
 
     choosers = location_sample_df.join(persons_merged_df, how="left")
@@ -618,9 +607,7 @@ def run_location_simulate(
     """
     assert not persons_merged.empty
 
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-    choosers = persons_merged[chooser_columns]
+    choosers = persons_merged
 
     alt_dest_col_name = model_settings.ALT_DEST_COL_NAME
 
@@ -651,6 +638,7 @@ def run_location_simulate(
         "reindex": reindex,
         "land_use": state.get_dataframe("land_use"),
     }
+    locals_d.update(state.get_global_constants())
     locals_d.update(model_settings.CONSTANTS or {})
 
     # preprocess choosers table

activitysim/abm/models/school_escorting.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import logging
+import warnings
 from typing import Any, Literal
 
 import numpy as np
@@ -335,8 +336,6 @@ class SchoolEscortSettings(BaseLogitComponentSettings, extra="forbid"):
     GENDER_WEIGHT: float = 10.0
     AGE_WEIGHT: float = 1.0
 
-    SIMULATE_CHOOSER_COLUMNS: list[str] | None = None
-
     SPEC: None = None
     """The school escort model does not use this setting."""
 
@@ -369,6 +368,21 @@ class SchoolEscortSettings(BaseLogitComponentSettings, extra="forbid"):
         Multinomial logit model.
     """
 
+    @property
+    def SIMULATE_CHOOSER_COLUMNS(self) -> None:
+        """Was used to help reduce the memory needed for the model.
+        Setting is now obsolete and doesn't do anything.
+        Functionality was replaced by util.drop_unused_columns
+
+        .. deprecated:: 1.4
+        """
+        warnings.warn(
+            "SIMULATE_CHOOSER_COLUMNS is deprecated and replaced by util.drop_unused_columns",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return None
+
 
 @workflow.step
 def school_escorting(
@@ -469,12 +483,6 @@ def school_escorting(
         # else:
         #     locals_dict.pop("_sharrow_skip", None)
 
-        # reduce memory by limiting columns if selected columns are supplied
-        chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-        if chooser_columns is not None:
-            chooser_columns = chooser_columns + participant_columns
-            choosers = choosers[chooser_columns]
-
         # add previous data to stage
         if stage_num >= 1:
             choosers = add_prev_choices_to_choosers(

activitysim/abm/models/util/logsums.py

@@ -16,41 +16,6 @@
 logger = logging.getLogger(__name__)
 
 
-def filter_chooser_columns(
-    choosers, logsum_settings: dict | PydanticBase, model_settings: dict | PydanticBase
-):
-    try:
-        chooser_columns = logsum_settings.LOGSUM_CHOOSER_COLUMNS
-    except AttributeError:
-        chooser_columns = logsum_settings.get("LOGSUM_CHOOSER_COLUMNS", [])
-
-    if (
-        isinstance(model_settings, dict)
-        and "CHOOSER_ORIG_COL_NAME" in model_settings
-        and model_settings["CHOOSER_ORIG_COL_NAME"] not in chooser_columns
-    ):
-        chooser_columns.append(model_settings["CHOOSER_ORIG_COL_NAME"])
-    if (
-        isinstance(model_settings, PydanticBase)
-        and hasattr(model_settings, "CHOOSER_ORIG_COL_NAME")
-        and model_settings.CHOOSER_ORIG_COL_NAME
-        and model_settings.CHOOSER_ORIG_COL_NAME not in chooser_columns
-    ):
-        chooser_columns.append(model_settings.CHOOSER_ORIG_COL_NAME)
-
-    missing_columns = [c for c in chooser_columns if c not in choosers]
-    if missing_columns:
-        logger.debug(
-            "logsum.filter_chooser_columns missing_columns %s" % missing_columns
-        )
-
-    # ignore any columns not appearing in choosers df
-    chooser_columns = [c for c in chooser_columns if c in choosers]
-
-    choosers = choosers[chooser_columns]
-    return choosers
-
-
 def compute_location_choice_logsums(
     state: workflow.State,
     choosers: pd.DataFrame,

activitysim/abm/models/util/tour_destination.py

@@ -113,6 +113,7 @@ def _destination_sample(
         "dest_col_name": skims.dest_key,  # added for sharrow flows
         "timeframe": "timeless",
     }
+    locals_d.update(state.get_global_constants())
     constants = model_settings.CONSTANTS
     if constants is not None:
         locals_d.update(constants)
@@ -619,18 +620,9 @@ def run_destination_sample(
     chunk_size,
     trace_label,
 ):
-    # FIXME - MEMORY HACK - only include columns actually used in spec (omit them pre-merge)
-    chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
 
     # if special person id is passed
     chooser_id_column = model_settings.CHOOSER_ID_COLUMN
-
-    persons_merged = persons_merged[
-        [c for c in persons_merged.columns if c in chooser_columns]
-    ]
-    tours = tours[
-        [c for c in tours.columns if c in chooser_columns or c == chooser_id_column]
-    ]
     choosers = pd.merge(
         tours, persons_merged, left_on=chooser_id_column, right_index=True, how="left"
     )
@@ -726,11 +718,6 @@ def run_destination_logsums(
 
     chunk_tag = "tour_destination.logsums"
 
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    persons_merged = logsum.filter_chooser_columns(
-        persons_merged, logsum_settings, model_settings
-    )
-
     # merge persons into tours
     choosers = pd.merge(
         destination_sample,
@@ -793,18 +780,8 @@ def run_destination_simulate(
         coefficients_file_name=model_settings.COEFFICIENTS,
     )
 
-    # FIXME - MEMORY HACK - only include columns actually used in spec (omit them pre-merge)
-    chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-
     # if special person id is passed
     chooser_id_column = model_settings.CHOOSER_ID_COLUMN
-
-    persons_merged = persons_merged[
-        [c for c in persons_merged.columns if c in chooser_columns]
-    ]
-    tours = tours[
-        [c for c in tours.columns if c in chooser_columns or c == chooser_id_column]
-    ]
     choosers = pd.merge(
         tours, persons_merged, left_on=chooser_id_column, right_index=True, how="left"
     )
@@ -846,6 +823,7 @@ def run_destination_simulate(
         "dest_col_name": skims.dest_key,  # added for sharrow flows
         "timeframe": "timeless",
     }
+    locals_d.update(state.get_global_constants())
     if constants is not None:
         locals_d.update(constants)
 

activitysim/abm/models/util/tour_od.py

@@ -178,6 +178,7 @@ def _od_sample(
         "orig_col_name": ORIG_TAZ,
         "dest_col_name": DEST_TAZ,
     }
+    locals_d.update(state.get_global_constants())
     constants = model_settings.CONSTANTS
     if constants is not None:
         locals_d.update(constants)
@@ -690,9 +691,6 @@ def run_od_sample(
     )
 
     choosers = tours
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-    choosers = choosers[chooser_columns]
 
     # interaction_sample requires that choosers.index.is_monotonic_increasing
     if not choosers.index.is_monotonic_increasing:
@@ -769,11 +767,6 @@ def run_od_logsums(
     dest_id_col = model_settings.DEST_COL_NAME
     tour_od_id_col = get_od_id_col(origin_id_col, dest_id_col)
 
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    tours_merged_df = logsum.filter_chooser_columns(
-        tours_merged_df, logsum_settings, model_settings
-    )
-
     # merge ods into choosers table
     choosers = od_sample.join(tours_merged_df, how="left")
     choosers[tour_od_id_col] = (
@@ -949,10 +942,6 @@ def run_od_simulate(
     # merge persons into tours
     choosers = tours
 
-    # FIXME - MEMORY HACK - only include columns actually used in spec
-    chooser_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-    choosers = choosers[chooser_columns]
-
     # interaction_sample requires that choosers.index.is_monotonic_increasing
     if not choosers.index.is_monotonic_increasing:
         logger.debug(
@@ -1003,6 +992,7 @@ def run_od_simulate(
         "orig_col_name": origin_col_name,
         "dest_col_name": dest_col_name,
     }
+    locals_d.update(state.get_global_constants())
     if constants is not None:
         locals_d.update(constants)
 

activitysim/abm/models/util/tour_scheduling.py

@@ -24,24 +24,6 @@ def run_tour_scheduling(
     trace_label: str,
 ):
 
-    if model_settings.LOGSUM_SETTINGS:
-        logsum_settings = TourModeComponentSettings.read_settings_file(
-            state.filesystem,
-            str(model_settings.LOGSUM_SETTINGS),
-            mandatory=False,
-        )
-        logsum_columns = logsum_settings.LOGSUM_CHOOSER_COLUMNS
-    else:
-        logsum_columns = []
-
-    # - filter chooser columns for both logsums and simulate
-    model_columns = model_settings.SIMULATE_CHOOSER_COLUMNS
-    chooser_columns = logsum_columns + [
-        c for c in model_columns if c not in logsum_columns
-    ]
-
-    persons_merged = expressions.filter_chooser_columns(persons_merged, chooser_columns)
-
     timetable = state.get_injectable("timetable")
 
     # - run preprocessor to annotate choosers

activitysim/abm/models/util/vectorize_tour_scheduling.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import logging
+import warnings
 from collections import OrderedDict
 from pathlib import Path
 from typing import Any
@@ -42,7 +43,6 @@ class TourSchedulingSettings(LogitComponentSettings, extra="forbid"):
     it is assumed to be an unsegmented preprocessor.  Otherwise, the dict keys
     give the segements.
     """
-    SIMULATE_CHOOSER_COLUMNS: list[str] | None = None
 
     SPEC_SEGMENTS: dict[str, LogitComponentSettings] = {}
 
@@ -63,6 +63,21 @@ class TourSchedulingSettings(LogitComponentSettings, extra="forbid"):
     If less than 1, use this fraction of the total number of rows.
     """
 
+    @property
+    def SIMULATE_CHOOSER_COLUMNS(self) -> None:
+        """Was used to help reduce the memory needed for the model.
+        Setting is now obsolete and doesn't do anything.
+        Functionality was replaced by util.drop_unused_columns
+
+        .. deprecated:: 1.4
+        """
+        warnings.warn(
+            "SIMULATE_CHOOSER_COLUMNS is deprecated and replaced by util.drop_unused_columns",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return None
+
 
 def skims_for_logsums(
     state: workflow.State,
@@ -822,6 +837,7 @@ def _schedule_tours(
 
     # - make choices
     locals_d = {"tt": timetable.attach_state(state)}
+    locals_d.update(state.get_global_constants())
     constants = config.get_model_constants(model_settings)
     if constants is not None:
         locals_d.update(constants)

activitysim/core/configuration/logit.py

@@ -261,7 +261,6 @@ class TourLocationComponentSettings(LocationComponentSettings, extra="forbid"):
     SEGMENT_IDS: dict[str, int] | dict[str, str] | dict[str, bool] | None = None
     SHADOW_PRICE_TABLE: str | None = None
     MODELED_SIZE_TABLE: str | None = None
-    SIMULATE_CHOOSER_COLUMNS: list[str] | None = None
     ALT_DEST_COL_NAME: str
     LOGSUM_TOUR_PURPOSE: str | dict[str, str] | None = None
     MODEL_SELECTOR: str | None = None
@@ -271,6 +270,21 @@ class TourLocationComponentSettings(LocationComponentSettings, extra="forbid"):
     ORIG_ZONE_ID: str | None = None
     """This setting appears to do nothing..."""
 
+    @property
+    def SIMULATE_CHOOSER_COLUMNS(self) -> None:
+        """Was used to help reduce the memory needed for the model.
+        Setting is now obsolete and doesn't do anything.
+        Functionality was replaced by util.drop_unused_columns
+
+        .. deprecated:: 1.4
+        """
+        warnings.warn(
+            "SIMULATE_CHOOSER_COLUMNS is deprecated and replaced by util.drop_unused_columns",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return None
+
 
 class TourModeComponentSettings(TemplatedLogitComponentSettings, extra="forbid"):
     MODE_CHOICE_LOGSUM_COLUMN_NAME: str | None = None
@@ -281,4 +295,18 @@ class TourModeComponentSettings(TemplatedLogitComponentSettings, extra="forbid")
     nontour_preprocessor: PreprocessorSettings | list[
         PreprocessorSettings
     ] | None = None
-    LOGSUM_CHOOSER_COLUMNS: list[str] = []
+
+    @property
+    def LOGSUM_CHOOSER_COLUMNS(self) -> None:
+        """Was used to help reduce the memory needed for the model.
+        Setting is now obsolete and doesn't do anything.
+        Functionality was replaced by util.drop_unused_columns
+
+        .. deprecated:: 1.4
+        """
+        warnings.warn(
+            "LOGSUM_CHOOSER_COLUMNS is deprecated and replaced by util.drop_unused_columns",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return None