Fixup

.github/workflows/build_pipeline.yml

@@ -6,11 +6,11 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v1
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: 3.7
-    - uses: actions/cache@v1
+        python-version: 3.8
+    - uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
@@ -76,7 +76,7 @@ jobs:
         aws s3 cp app.zip "s3://$AWS_S3_CODE_BUCKET/$repo_slug.zip"
     - name: Send build success notification
       if: success()
-      uses: rtCamp/action-slack-notify@v2.0.0
+      uses: rtCamp/action-slack-notify@v2.2.0
       env:
         SLACK_MESSAGE: ${{ github.repository }} build ${{ github.run_number }} launched by ${{ github.actor }} has succeeded
         SLACK_TITLE: Build Success
@@ -86,7 +86,7 @@ jobs:
         SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
     - name: Send build failure notification
       if: failure()
-      uses: rtCamp/action-slack-notify@v2.0.0
+      uses: rtCamp/action-slack-notify@v2.2.0
       env:
         SLACK_COLOR: '#FF0000'
         SLACK_MESSAGE: ${{ github.repository }} build ${{ github.run_number }} launched by ${{ github.actor }} has failed

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8-slim-bullseye
+FROM python:3.13-slim-bullseye
 
 RUN apt-get update \
 && apt-get upgrade -y \

README.md

@@ -37,6 +37,20 @@ git clone [email protected]:arup-group/elara.git
 
 Once available locally, navigate to the folder to install Elara and its dependencies. Using a virtual environment is highly recommended.
 
+### Mamba
+
+Using [mamba](https://mamba.readthedocs.io/en/latest/index.html).
+
+```{sh}
+git clone [email protected]:arup-group/elara.git
+cd elara
+mamba install -n elara --file requirements.txt python=3.13
+mamba activate elara
+pip install --no-deps -e .
+pytest
+elara --help
+```
+
 ### OSX
 
 ```{sh}

elara/benchmarking.py

@@ -1,4 +1,3 @@
-from pandas.core import groupby
 import pandas as pd
 import os
 import numpy as np
@@ -95,7 +94,7 @@ def build(self, resources: dict, write_path: Optional[str] = None) -> dict:
         bm_df = pd.concat([benchmarks_df[self.value_field],
                           simulation_df[self.value_field]], axis=1)
         bm_df.columns = self.output_value_fields
-        bm_df.dropna(0, inplace=True)
+        bm_df.dropna(axis=0, inplace=True)
 
         self.plot_comparisons(bm_df)
 
@@ -314,7 +313,7 @@ def __init__(
             **kwargs
         )
         self.groupby_person_attribute = groupby_person_attribute
-        self.simulation_name = f"trip_modes_all"
+        self.simulation_name = "trip_modes_all"
         if groupby_person_attribute is not None:
             self.logger.debug(
                 f"Found 'groupby_person_attribute': {groupby_person_attribute}")
@@ -348,7 +347,7 @@ def __init__(
             **kwargs
         )
         self.groupby_person_attribute = groupby_person_attribute
-        self.simulation_name = f"trip_modes_all"
+        self.simulation_name = "trip_modes_all"
         if groupby_person_attribute is not None:
             self.logger.debug(
                 f"Found 'groupby_person_attribute': {groupby_person_attribute}")
@@ -372,7 +371,7 @@ def __init__(self, config, mode, **kwargs):
         super().__init__(config, mode=mode, **kwargs)
         destination_activities = kwargs.get("destination_activity_filters", [])
         groupby_person_attribute = kwargs.get("groupby_person_attribute")
-        self.simulation_name = f"trip_activity_modes_all"
+        self.simulation_name = "trip_activity_modes_all"
         for act in destination_activities:
             self.simulation_name += f"_{act}"
         if groupby_person_attribute is not None:
@@ -398,7 +397,7 @@ def __init__(self, config, mode, **kwargs):
         super().__init__(config, mode=mode, **kwargs)
         destination_activities = kwargs.get("destination_activity_filters", [])
         groupby_person_attribute = kwargs.get("groupby_person_attribute")
-        self.simulation_name = f"trip_activity_modes_all"
+        self.simulation_name = "trip_activity_modes_all"
         for act in destination_activities:
             self.simulation_name += f"_{act}"
         if groupby_person_attribute is not None:
@@ -434,7 +433,7 @@ def __init__(
             **kwargs
         )
         self.groupby_person_attribute = groupby_person_attribute
-        self.simulation_name = f"plan_modes_all"
+        self.simulation_name = "plan_modes_all"
         if groupby_person_attribute is not None:
             self.logger.debug(
                 f"Found 'groupby_person_attribute': {groupby_person_attribute}")
@@ -468,7 +467,7 @@ def __init__(
             **kwargs
         )
         self.groupby_person_attribute = groupby_person_attribute
-        self.simulation_name = f"plan_modes_all"
+        self.simulation_name = "plan_modes_all"
         if groupby_person_attribute is not None:
             self.logger.debug(
                 f"Found 'groupby_person_attribute': {groupby_person_attribute}")
@@ -492,7 +491,7 @@ def __init__(self, config, mode, **kwargs):
         super().__init__(config, mode=mode, **kwargs)
         destination_activities = kwargs.get("destination_activity_filters", [])
         groupby_person_attribute = kwargs.get("groupby_person_attribute")
-        self.simulation_name = f"plan_activity_modes_all"
+        self.simulation_name = "plan_activity_modes_all"
         for act in destination_activities:
             self.simulation_name += f"_{act}"
         if groupby_person_attribute is not None:
@@ -518,7 +517,7 @@ def __init__(self, config, mode, **kwargs):
         super().__init__(config, mode=mode, **kwargs)
         destination_activities = kwargs.get("destination_activity_filters", [])
         groupby_person_attribute = kwargs.get("groupby_person_attribute")
-        self.simulation_name = f"plan_activity_modes_all"
+        self.simulation_name = "plan_activity_modes_all"
         for act in destination_activities:
             self.simulation_name += f"_{act}"
         if groupby_person_attribute is not None:
@@ -653,7 +652,7 @@ def __init__(self, config, mode, benchmark_data_path=None, **kwargs) -> None:
                 if not links:
                     missing_counters += 1
                     self.logger.warning(
-                        f"Benchmark data has no links - suggests error with Bench (i.e. MATSIM network has not matched to BM)."
+                        "Benchmark data has no links - suggests error with Bench (i.e. MATSIM network has not matched to BM)."
                     )
 
         # Check for number of missing BM links. Note this is a fault with the BM (ie missing links)
@@ -919,7 +918,7 @@ def __init__(self, config, mode, benchmark_data_path=None, **kwargs) -> None:
                 if not stops:
                     missing_counters += 1
                     self.logger.debug(
-                        f"Benchmark data has no stop/s - suggests error with Bench (i.e. MATSIM network has not matched to BM)."
+                        "Benchmark data has no stop/s - suggests error with Bench (i.e. MATSIM network has not matched to BM)."
                     )
 
         # Check for number of missing BM stops. Note this is a fault with the BM (ie missing stops)
@@ -994,7 +993,7 @@ def build(self, resource: dict, write_path: Optional[str] = None) -> dict:
                 sim_result = np.array([0.0 for _ in range(len(bm_hours))])
 
                 # check if direction available
-                if not direction in model_results:
+                if direction not in model_results:
                     raise UserWarning(
                         f"Direction: {direction} not available in model results"
                     )
@@ -2311,8 +2310,16 @@ def merge_summary_stats(bm_results_summary):
     for record in data:
         record_type = record['source']
 
-        record.pop("score")
-        record.pop("source")
+        record.pop("score", None)
+        record.pop("source", None)
+        record.pop("mode", None)
+        record.pop("counter_id", None)
+        record.pop("line", None)
+        record.pop("o", None)
+        record.pop("d", None)
+        record.pop("direction", None)
+        record.pop("link_id", None)
+        # absolute trash, but it works
 
         if record_type != "difference":
             for measurement in list(record):

elara/event_handlers.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import pickle
 from math import floor
 from typing import Optional, Tuple, Union
 
@@ -231,7 +232,10 @@ def finalise(self):
         del self.veh_occupancy
         name = f"{str(self)}.pkl"
         path = os.path.join(self.config.output_path, name)
-        nx.write_gpickle(self.graph, path)
+
+        with open(path, 'wb') as f:
+            pickle.dump(self.graph, f, pickle.HIGHEST_PROTOCOL)
+
         del self.graph
 
 
@@ -435,7 +439,7 @@ def build(self, resources: dict, write_path: Optional[str] = None) -> None:
                 )
         else:
             self.logger.debug(f'Selecting links for mode:{self.mode}.')
-            self.elem_gdf = self.elem_gdf.loc[links, :]
+            self.elem_gdf = self.elem_gdf.loc[list(links), :]
 
         self.elem_ids, self.elem_indices = self.generate_elem_ids(self.elem_gdf)
 
@@ -492,7 +496,7 @@ def finalise(self) -> None:
             counts_df = pd.DataFrame(self.counts.flatten(), index=index)[0]
             counts_df = counts_df.unstack(level='hour').sort_index()
 
-            counts_df['total'] = counts_df.sum(1)
+            counts_df['total'] = counts_df.sum(1, numeric_only=True)
             counts_df = counts_df.reset_index().set_index('elem')
 
             key = f"{self.name}_{self.groupby_person_attribute}"
@@ -510,7 +514,7 @@ def finalise(self) -> None:
         totals_df = pd.DataFrame(
             data=self.counts, index=self.elem_ids, columns=range(0, self.config.time_periods)
         ).sort_index()
-        totals_df['total'] = totals_df.sum(1)
+        totals_df['total'] = totals_df.sum(1, numeric_only=True)
 
         del self.counts
 
@@ -590,7 +594,7 @@ def build(self, resources: dict, write_path: Optional[str] = None) -> None:
                 )
         else:
             self.logger.debug(f'Selecting links for mode:{self.mode}.')
-            self.elem_gdf = self.elem_gdf.loc[links, :]
+            self.elem_gdf = self.elem_gdf.loc[list(links), :]
 
         self.elem_ids, self.elem_indices = self.generate_elem_ids(self.elem_gdf)
 
@@ -649,7 +653,7 @@ def finalise(self) -> None:
             counts_df = counts_df.unstack(level='hour').sort_index()
             # counts_df = counts_df.reset_index().set_index(['elem', self.groupby_person_attribute])
 
-            counts_df['total'] = counts_df.sum(1)
+            counts_df['total'] = counts_df.sum(1, numeric_only=True)
             counts_df = counts_df.reset_index().set_index('elem')
 
             key = f"{self.name}_{self.groupby_person_attribute}"
@@ -667,7 +671,7 @@ def finalise(self) -> None:
         totals_df = pd.DataFrame(
             data=self.counts, index=self.elem_ids, columns=range(0, self.config.time_periods)
         ).sort_index()
-        totals_df['total'] = totals_df.sum(1)
+        totals_df['total'] = totals_df.sum(1, numeric_only=True)
 
         del self.counts
 
@@ -740,7 +744,7 @@ def build(self, resources: dict, write_path: Optional[str] = None) -> None:
                 )
         else:
             self.logger.debug(f'Selecting links for mode:{self.mode}.')
-            self.elem_gdf = self.elem_gdf.loc[links, :]
+            self.elem_gdf = self.elem_gdf.loc[list(links), :]
 
         self.elem_ids, self.elem_indices = self.generate_elem_ids(self.elem_gdf)
 
@@ -1038,7 +1042,7 @@ def build(self, resources: dict, write_path: Optional[str] = None) -> None:
                 )
         else:
             self.logger.debug(f'Selecting links for mode:{self.mode}.')
-            self.elem_gdf = self.elem_gdf.loc[links, :]
+            self.elem_gdf = self.elem_gdf.loc[list(links), :]
 
         self.elem_ids, self.elem_indices = self.generate_elem_ids(self.elem_gdf)
 
@@ -1150,7 +1154,7 @@ def finalise(self):
             counts_df = pd.DataFrame(self.counts.flatten(), index=index)[0]
             counts_df = counts_df.unstack(level='hour').sort_index()
             counts_df = counts_df.reset_index().set_index(['elem', self.groupby_person_attribute])
-            counts_df['total'] = counts_df.sum(1)
+            counts_df['total'] = counts_df.sum(1, numeric_only=True)
             counts_df = counts_df.reset_index().set_index('elem')
             key = f"{self.name}_{self.groupby_person_attribute}"
             counts_df = self.elem_gdf.join(
@@ -1168,7 +1172,7 @@ def finalise(self):
         totals_df = pd.DataFrame(
             data=self.counts, index=self.elem_ids, columns=range(0, self.config.time_periods)
         ).sort_index()
-        totals_df['total'] = totals_df.sum(1)
+        totals_df['total'] = totals_df.sum(1, numeric_only=True)
 
         del self.counts
 
@@ -1435,7 +1439,7 @@ def build(self, resources: dict, write_path: Optional[str] = None) -> None:
                 )
         else:
             self.logger.debug(f'Filtering stops for mode:{self.mode}.')
-            self.elem_gdf = self.elem_gdf.loc[viable_stops, :]
+            self.elem_gdf = self.elem_gdf.loc[list(viable_stops), :]
 
         self.elem_ids, self.elem_indices = self.generate_elem_ids(self.elem_gdf)
 
@@ -1527,7 +1531,7 @@ def finalise(self):
                 counts_df = pd.DataFrame(data.flatten(), index=index)[0]
                 counts_df = counts_df.unstack(level='hour').sort_index()
                 counts_df = counts_df.reset_index().set_index(['elem', self.groupby_person_attribute])
-                counts_df['total'] = counts_df.sum(1)
+                counts_df['total'] = counts_df.sum(1, numeric_only=True)
                 counts_df = counts_df.reset_index().set_index('elem')
 
                 # Create volume counts output
@@ -1544,7 +1548,7 @@ def finalise(self):
             totals_df = pd.DataFrame(
                 data=data, index=self.elem_ids, columns=range(0, self.config.time_periods)
             ).sort_index()
-            totals_df['total'] = totals_df.sum(1)
+            totals_df['total'] = totals_df.sum(1, numeric_only=True)
 
             del data
 
@@ -1621,7 +1625,7 @@ def build(self, resources: dict, write_path: Optional[str] = None) -> None:
                 )
         else:
             self.logger.debug(f'Filtering stops for mode:{self.mode}.')
-            self.elem_gdf = self.elem_gdf.loc[viable_stops, :]
+            self.elem_gdf = self.elem_gdf.loc[list(viable_stops), :]
 
         self.elem_ids, self.elem_indices = self.generate_elem_ids(self.elem_gdf)
 
@@ -1751,7 +1755,7 @@ def finalise(self):
             counts_df.index.name = n
 
         counts_df = counts_df.reset_index().set_index(['origin', 'destination', str(self.groupby_person_attribute)])
-        counts_df['total'] = counts_df.sum(1)
+        counts_df['total'] = counts_df.sum(1, numeric_only=True)
 
         counts_df['geometry'] = [
             LineString([o, d]) for o, d in zip(counts_df.origin_geometry, counts_df.destination_geometry)
@@ -1769,7 +1773,7 @@ def finalise(self):
         # calc sum across all recorded attribute classes
         totals_df = counts_df.reset_index().groupby(
             ['origin', 'destination']
-            ).sum().reset_index().set_index(['origin', 'destination'])
+            ).sum(numeric_only=True).reset_index().set_index(['origin', 'destination'])
 
         # Join stop data and build geometry
         for n in ("origin", "destination"):
@@ -1867,7 +1871,7 @@ def build(self, resources: dict, write_path: Optional[str] = None) -> None:
                 )
         else:
             self.logger.debug(f'Filtering stops for mode:{self.mode}.')
-            self.elem_gdf = self.elem_gdf.loc[viable_stops, :]
+            self.elem_gdf = self.elem_gdf.loc[list(viable_stops), :]
 
         self.elem_ids, self.elem_indices = self.generate_elem_ids(self.elem_gdf)
 
@@ -2001,7 +2005,7 @@ def finalise(self):
 
         counts_df = counts_df.reset_index().set_index(names+['veh_counts'])
         counts_df['route'] = counts_df.index.get_level_values('veh_id').map(self.veh_route)
-        counts_df['total'] = counts_df.sum(1)
+        counts_df['total'] = counts_df.sum(1, numeric_only=True)
 
         counts_df['geometry'] = [LineString([o, d]) for o, d in zip(
             counts_df.from_stop_geometry, counts_df.to_stop_geometry)]
@@ -2022,7 +2026,7 @@ def finalise(self):
             self.result_dfs[key] = counts_df
 
         # calc sum across all recorded attribute classes
-        totals_df = counts_df.reset_index().groupby(['from_stop', 'to_stop', 'veh_id', 'route', 'veh_counts']).sum()
+        totals_df = counts_df.reset_index().groupby(['from_stop', 'to_stop', 'veh_id', 'route', 'veh_counts']).sum(numeric_only=True)
 
         # Join stop data and build geometry
         for n in ("from_stop", "to_stop"):