diff --git a/.coveragerc b/.coveragerc index 70674c543..846588761 100644 --- a/.coveragerc +++ b/.coveragerc @@ -19,4 +19,7 @@ exclude_lines = if __name__ == .__main__.: raise NotImplementedError @ray.remote + @abstractmethod def policy_mapping_fn* + def main(args)* + pragma: no cover diff --git a/.gitignore b/.gitignore index 29d788c27..6d9ff7a71 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,5 @@ flow.ang *.ang.old *.sang +# local configuration file for data pipeline +**/data_pipeline_config diff --git a/.travis.yml b/.travis.yml index b2cb1a897..a802f5d23 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,22 +38,33 @@ before_install: # Set up requirements for flow - conda env create -f environment.yml - - source activate flow + - source activate circles # [sumo] dependencies and binaries - - pushd $HOME/build/flow-project - - ./flow/scripts/setup_libsumo_ubuntu.sh + - pushd $HOME/build/CIRCLES-consortium + - ./flow/scripts/setup_sumo_ubuntu1604.sh - popd - source ~/.bashrc # [aimsun] install the conda env and update the path to the env - - pushd $HOME/build/flow-project + - pushd $HOME/build/CIRCLES-consortium - ./flow/scripts/setup_aimsun.sh - popd - source ~/.bashrc - export AIMSUN_SITEPACKAGES="/home/travis/miniconda/envs/aimsun_flow" - export AIMSUN_NEXT_PATH="/home/user/Aimsun_Next_XXX" + # Install stable-baselines + - pip install stable_baselines==2.7.0 + + # Install h-baselines + - pushd $HOME + - git clone https://github.com/AboudyKreidieh/h-baselines.git + - pushd h-baselines + - pip install -e . + - popd + - popd + - ls ../ install: @@ -62,7 +73,6 @@ install: - pip install -e . - pip install coveralls - pip install jupyter - - pip install stable_baselines==2.7.0 script: - nose2 --with-coverage diff --git a/CODEOWNERS b/CODEOWNERS index 175783157..1a0ba0d97 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -2,7 +2,7 @@ # Each line is a file pattern followed by one or more owners. # These owners will be the default owners for everything in the repo. -* @cathywu @eugenevinitsky @AboudyKreidieh @kanaadp +* @eugenevinitsky @AboudyKreidieh @kjang96 # Order is important. The last matching pattern has the most precedence. # So if a pull request only touches javascript files, only these owners diff --git a/docs/Prius_EnergyModel.pdf b/docs/Prius_EnergyModel.pdf new file mode 100644 index 000000000..4884fb5cc Binary files /dev/null and b/docs/Prius_EnergyModel.pdf differ diff --git a/docs/Tacoma_EnergyModel.pdf b/docs/Tacoma_EnergyModel.pdf new file mode 100644 index 000000000..8fe16d05f Binary files /dev/null and b/docs/Tacoma_EnergyModel.pdf differ diff --git a/docs/libsumo_mac.md b/docs/libsumo_mac.md new file mode 100644 index 000000000..7a02d3823 --- /dev/null +++ b/docs/libsumo_mac.md @@ -0,0 +1,23 @@ +# How to install Libsumo for Mac OS + +This is adapted from an email exchange with the SUMO staff. + + + +To install libsumo requires re-building and installing SUMO from source. + +## Steps + +- **Install swig:** ```brew install swig``` +- **Clone the repo:** ```git clone https://github.com/eclipse/sumo.git``` +- **Create a “cmake-build” directory inside sumo/build/ and navigate to it:** ```mkdir build/cmake-build && cd build/cmake-build``` + +**The next 3 steps are inside that directory** + +- ```cmake ../..``` +- ```make``` +- ```make install``` + +## Additional Notes +- You can test if libsumo has been built looking at (./testlibsumo) inside the sumo/bin/ directory. +- Bear in mind to use libsumo with the same Python version with which CMake built SUMO. \ No newline at end of file diff --git a/docs/source/flow_setup.rst b/docs/source/flow_setup.rst index 606a9d6d4..6dab3cd6b 100644 --- a/docs/source/flow_setup.rst +++ b/docs/source/flow_setup.rst @@ -2,7 +2,7 @@ .. contents:: Table of contents Local Installation of Flow -================== +========================== To get Flow running, you need three things: Flow, @@ -61,10 +61,13 @@ install everything you will need from SUMO, run one of the below scripts from the Flow main directory. Choose the script that matches the operating system you are running. + For Ubuntu 14.04: -:: +**WARNING**: +Libsumo is not supported on Ubuntu 14.04. This script will setup a sumo without libsumo. +:: scripts/setup_sumo_ubuntu1404.sh For Ubuntu 16.04: @@ -108,7 +111,25 @@ Note that, if the above commands did not work, you may need to run ``source ~/.bashrc`` or open a new terminal to update your $PATH variable. *Troubleshooting*: -If you are a Mac user and the above command gives you the error ``FXApp:openDisplay: unable to open display :0.0``, make sure to open the application XQuartz. +If you are a Mac user and the above command gives you the error +``FXApp:openDisplay: unable to open display :0.0``, make sure to open the +application XQuartz. + +*Troubleshooting*: +If you are a Mac user and the above command gives you the error +``Segmentation fault: 11``, make sure to reinstall ``fox`` using brew. +:: + + # Uninstall Catalina bottle of fox: + $ brew uninstall --ignore-dependencies fox + + # Edit brew Formula of fox: + $ brew edit fox + + # Comment out or delete the following line: sha256 "c6697be294c9a0458580564d59f8db32791beb5e67a05a6246e0b969ffc068bc" => :catalina + # Install Mojave bottle of fox: + $ brew install fox + Testing your SUMO and Flow installation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -151,7 +172,10 @@ during the execution of various tasks. The path should look something like: export AIMSUN_NEXT_PATH="/home/user/Aimsun_Next_X_Y_Z/" # Linux export AIMSUN_NEXT_PATH="/Applications/Aimsun Next.app/Contents/MacOS/" # OS X -`Note for Mac users:` when you download Aimsun, you will get a folder named "Programming". You need to rename it to "programming" (all lowercase) and to move it inside the "Aimsun Next.app/Contents/MacOS/" directory so that the python API can work. +`Note for Mac users:` when you download Aimsun, you will get a folder named +"Programming". You need to rename it to "programming" (all lowercase) and to +move it inside the "Aimsun Next.app/Contents/MacOS/" directory so that the +python API can work. In addition, being that Aimsun's python API is written to support Python 2.7.4, we will need to create a Python 2.7.4 conda environment that Aimsun can refer @@ -198,8 +222,11 @@ to activate the `flow` env. Type: source activate flow python examples/simulate.py ring --aimsun -*Troubleshootig for Ubuntu users with Aimsun 8.4*: when you run the above example, you may get a subprocess.Popen error ``OSError: [Errno 8] Exec format error:``. -To fix this, go to the `Aimsun Next` main directory, open the `Aimsun_Next` binary with a text editor and add the shebang to the first line of the script ``#!/bin/sh``. +*Troubleshootig for Ubuntu users with Aimsun 8.4*: when you run the above +example, you may get a subprocess.Popen error ``OSError: [Errno 8] Exec format error:``. +To fix this, go to the `Aimsun Next` main directory, open the `Aimsun_Next` +binary with a text editor and add the shebang to the first line of the script +``#!/bin/sh``. (Optional) Install Ray RLlib ---------------------------- @@ -211,10 +238,11 @@ RLlib is one such library. First visit and install the required packages. -If you are not intending to develop RL algorithms or customize rllib you don't need to do anything, -Ray was installed when you created the conda environment. +If you are not intending to develop RL algorithms or customize rllib you don't +need to do anything, Ray was installed when you created the conda environment. -If you are intending to modify Ray, the installation process for this library is as follows: +If you are intending to modify Ray, the installation process for this library +is as follows: :: @@ -249,6 +277,34 @@ In order to test run an Flow experiment in RLlib, try the following command: If it does not fail, this means that you have Flow properly configured with RLlib. + +Visualizing with Tensorboard +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To visualize the training progress: + +:: + + tensorboard --logdir=~/ray_results + +If tensorboard is not installed, you can install with pip: + +:: + + pip install tensorboard + +For information on how to deploy a cluster, refer to the +`Ray instructions `_. +The basic workflow is running the following locally, ssh-ing into the host +machine, and starting jobs from there. + +:: + + pip install boto3 + ray create-or-update scripts/ray_autoscale.yaml + ray teardown scripts/ray_autoscale.yaml + + (Optional) Install Stable Baselines ----------------------------------- @@ -267,33 +323,29 @@ You can test your installation by running python examples/train.py singleagent_ring --rl_trainer Stable-Baselines +(Optional) Install h-baselines +------------------------------ -(Optional) Visualizing with Tensorboard ---------------------------------------- - -To visualize the training progress: +h-baselines is another variant of stable-baselines that support the use of +single-agent, multiagent, and hierarchical policies. To install h-baselines, +run the following commands: :: - tensorboard --logdir=~/ray_results - -If tensorboard is not installed, you can install with pip: + git clone https://github.com/AboudyKreidieh/h-baselines.git + cd h-baselines + source activate flow # if using a Flow environment + pip install -e . -:: - pip install tensorboard +Testing your h-baselines installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -For information on how to deploy a cluster, refer to the `Ray instructions `_. -The basic workflow is running the following locally, ssh-ing into the host machine, and starting -jobs from there. +You can test your installation by running :: - pip install boto3 - ray create-or-update scripts/ray_autoscale.yaml - ray teardown scripts/ray_autoscale.yaml - - + python examples/train.py singleagent_ring --rl_trainer h-baselines (Optional) Direct install of SUMO from GitHub @@ -358,16 +410,22 @@ If you have Ubuntu 14.04+, run the following command Virtual installation of Flow (using docker containers) -================================ +====================================================== To install a containerized Flow stack, run: + :: + docker run -d -p 5901:5901 -p 6901:6901 fywu85/flow-desktop:latest To access the docker container, go to the following URL and enter the default password `password`: + :: + http://localhost:6901/vnc.html To use the Jupyter Notebook inside the container, run: + :: + jupyter notebook --ip=127.0.0.1 diff --git a/docs/sumo_wheels.md b/docs/sumo_wheels.md index 0d3eda3e1..b937c4236 100644 --- a/docs/sumo_wheels.md +++ b/docs/sumo_wheels.md @@ -62,7 +62,7 @@ repository is migrating to a new SUMO version. cd /path/to/sumo/bin mkdir data cp -r ../data/* data - tar -cJf binaries-.tar.xz !(Makefile*|start-command-line.bat) + tar -cJf binaries-.tar.gz !(Makefile*|start-command-line.bat) 5. Create a `sumotools` wheel from the python-related packages. This only needs to be done once for all Ubuntu and Mac, as the python tools are distribution diff --git a/environment.yml b/environment.yml index f57c8d33d..005a5807e 100644 --- a/environment.yml +++ b/environment.yml @@ -1,18 +1,17 @@ -name: flow +name: circles dependencies: - - python==3.6.8 - - scipy==1.1.0 - - lxml==4.4.1 - - six==1.11.0 - - path.py - - python-dateutil==2.7.3 - - pip>=18.0 - - tensorflow==1.9.0 - - cloudpickle==1.2.1 - - setuptools==41.0.0 - - plotly==2.4.0 + - python==3.7.3 - pip: + - scipy==1.1.0 + - lxml==4.4.1 + - six==1.11.0 + - path.py + - python-dateutil==2.7.3 + - pip>=18.0 + - tensorflow==1.15.2 + - setuptools==41.0.0 + - plotly==2.4.0 - gym==0.14.0 - pyprind==2.11.2 - nose2==0.8.0 @@ -21,9 +20,11 @@ dependencies: - matplotlib==3.0.0 - dill - lz4 - - ray==0.7.3 + - ray==0.8.0 - setproctitle - psutil - opencv-python - - boto3==1.4.8 + - boto3==1.10.45 - redis~=2.10.6 + - tabulate + - pytz diff --git a/examples/README.md b/examples/README.md index f25f488c5..8156831fe 100644 --- a/examples/README.md +++ b/examples/README.md @@ -10,8 +10,8 @@ within the Flow framework on a variety of traffic problems. These examples are python files that may be executed either from terminal or via a text editor. For example, in order to execute the non-RL Ring example we run: -```shell -python simulate.py ring +```shell script +python simulate.py "ring" ``` The examples are categorized into the following 3 sections: @@ -24,58 +24,168 @@ micro-simulator sumo and traffic macro-simulator Aimsun. To execute these examples, run -```shell +```shell script python simulate.py EXP_CONFIG ``` -where `EXP_CONFIG` is the name of the experiment configuration file, as located in -`exp_configs/non_rl.` +where `EXP_CONFIG` is the name of the experiment configuration file, as located +in `exp_configs/non_rl.` There are several *optional* arguments that can be added to the above command: -```shell +```shell script python simulate.py EXP_CONFIG --num_runs n --no_render --aimsun --gen_emission ``` -where `--num_runs` indicates the number of simulations to run (default of `n` is 1), `--no_render` indicates whether to deactivate the simulation GUI during runtime (by default simulation GUI is active), `--aimsun` indicates whether to run the simulation using the simulator Aimsun (the default simulator is SUMO), and `--gen_emission` indicates whether to generate an emission file from the simulation. +where `--num_runs` indicates the number of simulations to run (default of `n` +is 1), `--no_render` indicates whether to deactivate the simulation GUI during +runtime (by default simulation GUI is active), `--aimsun` indicates whether to +run the simulation using the simulator Aimsun (the default simulator is SUMO), +and `--gen_emission` indicates whether to generate an emission file from the +simulation. -## RL examples based on RLlib +## RL examples -These examples are similar networks as those mentioned in *non-RL examples*, but in the -presence of autonomous vehicle (AV) or traffic light agents -being trained through RL algorithms provided by *RLlib*. +### RLlib + +These examples are similar networks as those mentioned in *non-RL examples*, +but in the presence of autonomous vehicle (AV) or traffic light agents being +trained through RL algorithms provided by *RLlib*. To execute these examples, run -```shell - python train.py EXP_CONFIG - (or python train.py EXP_CONFIG --rl_trainer RLlib) +```shell script +python train.py EXP_CONFIG --rl_trainer "rllib" --algorithm ``` -where `EXP_CONFIG` is the name of the experiment configuration file, as located in -`exp_configs/rl/singleagent` or `exp_configs/rl/multiagent.` - +where `EXP_CONFIG` is the name of the experiment configuration file, as located +in `exp_configs/rl/singleagent` or `exp_configs/rl/multiagent.` Here `` +should be the name of your desired algorithm. Currently we support PPO and TD3. -## RL examples based on stable_baselines +### stable-baselines These examples provide similar networks as those -mentioned in *non-RL examples*, but in the presence of autonomous vehicle (AV) or traffic -light agents being trained through RL algorithms provided by OpenAI *stable -baselines*. +mentioned in *non-RL examples*, but in the presence of autonomous vehicle (AV) +or traffic light agents being trained through RL algorithms provided by OpenAI +*stable-baselines*. To execute these examples, run -```shell - python train.py EXP_CONFIG --rl_trainer Stable-Baselines +```shell script +python train.py EXP_CONFIG --rl_trainer "stable-baselines" ``` -where `EXP_CONFIG` is the name of the experiment configuration file, as located in -`exp_configs/rl/singleagent.` +where `EXP_CONFIG` is the name of the experiment configuration file, as located +in `exp_configs/rl/singleagent.` Note that, currently, multiagent experiments are only supported through RLlib. There are several *optional* arguments that can be added to the above command: -```shell - python train.py EXP_CONFIG --rl_trainer Stable-Baselines --num_cpus n1 --num_steps n2 --rollout_size r +```shell script +python train.py EXP_CONFIG --rl_trainer "stable-baselines" --num_cpus n1 --num_steps n2 --rollout_size r +``` +where `--num_cpus` indicates the number of CPUs to use (default of `n1` is 1), +`--num_steps` indicates the total steps to perform the learning (default of +`n2` is 5000), and `--rollout_size` indicates the number of steps in a training +batch (default of `r` is 1000) + +### h-baselines + +A third RL algorithms package supported by the `train.py` script is +[h-baselines](https://github.com/AboudyKreidieh/h-baselines). In order to use +the algorithms supported by this package, begin by installing h-baselines by +following the setup instructions located +[here](https://flow.readthedocs.io/en/latest/flow_setup.html#optional-install-h-baselines). +A policy can be trained using one of the exp_configs as follows: + +```shell script +python examples/train.py singleagent_ring --rl_trainer h-baselines +``` + +**Logging:** + +The above script executes a training operation and begins logging training and +testing data under the path: *training_data/singleagent_ring/*. + +To visualize the statistics of various tensorflow operations in tensorboard, +type: + +```shell script +tensorboard --logdir /examples/training_data/singleagent_ring/ ``` -where `--num_cpus` indicates the number of CPUs to use (default of `n1` is 1), `--num_steps` indicates the total steps to perform the learning (default of `n2` is 5000), and `--rollout_size` indicates the number of steps in a training batch (default of `r` is 1000) + +Moreover, as training progressive, per-iteration and cumulative statistics are +printed as a table on your terminal. These statistics are stored under the csv +files *train.csv* and *eval.csv* (if also using an evaluation environment) +within the same directory. + +**Hyperparameters:** + +When using h-baseline, multiple new command-line arguments can be passed to +adjust the choice of algorithm and variable hyperparameters of the algorithms. +These new arguments are as follows: + +* `--alg` (*str*): The algorithm to use. Must be one of [TD3, SAC]. Defaults to + 'TD3'. +* `--evaluate` (*store_true*): whether to add an evaluation environment. The + evaluation environment is similar to the training environment, but with + `env_params.evaluate` set to True. +* `--n_training` (*int*): Number of training operations to perform. Each + training operation is performed on a new seed. Defaults to 1. +* `--total_steps` (*int*): Total number of timesteps used during training. + Defaults to 1000000. +* `--seed` (*int*): Sets the seed for numpy, tensorflow, and random. Defaults + to 1. +* `--log_interval` (*int*): the number of training steps before logging + training results. Defaults to 2000. +* `--eval_interval` (*int*): number of simulation steps in the training + environment before an evaluation is performed. Only relevant if `--evaluate` + is called. Defaults to 50000. +* `--save_interval` (int): number of simulation steps in the training + environment before the model is saved. Defaults to 50000. +* `--initial_exploration_steps` (*int*): number of timesteps that the policy is + run before training to initialize the replay buffer with samples. Defaults to + 10000. +* `--nb_train_steps` (*int*): the number of training steps. Defaults to 1. +* `--nb_rollout_steps` (*int*): the number of rollout steps. Defaults to 1. +* `--nb_eval_episodes` (*int*): the number of evaluation episodes. Only + relevant if `--evaluate` is called. Defaults to 50. +* `--reward_scale` (*float*): the value the reward should be scaled by. + Defaults to 1. +* `--buffer_size` (*int*): the max number of transitions to store. Defaults to + 200000. +* `--batch_size` (*int*): the size of the batch for learning the policy. + Defaults to 128. +* `--actor_lr` (*float*): the actor learning rate. Defaults to 3e-4. +* `--critic_lr` (*float*): the critic learning rate. Defaults to 3e-4. +* `--tau` (*float*): the soft update coefficient (keep old values, between 0 + and 1). Defatuls to 0.005. +* `--gamma` (*float*): the discount rate. Defaults to 0.99. +* `--layer_norm` (*store_true*): enable layer normalisation +* `--use_huber` (*store_true*): specifies whether to use the huber distance + function as the loss for the critic. If set to False, the mean-squared error + metric is used instead") +* `--actor_update_freq` (*int*): number of training steps per actor policy + update step. The critic policy is updated every training step. Only used when + the algorithm is set to "TD3". Defaults to 2. +* `--noise` (*float*): scaling term to the range of the action space, that is + subsequently used as the standard deviation of Gaussian noise added to the + action if `apply_noise` is set to True in `get_action`". Only used when the + algorithm is set to "TD3". Defaults to 0.1. +* `--target_policy_noise` (*float*): standard deviation term to the noise from + the output of the target actor policy. See TD3 paper for more. Only used when + the algorithm is set to "TD3". Defaults to 0.2. +* `--target_noise_clip` (*float*): clipping term for the noise injected in the + target actor policy. Only used when the algorithm is set to "TD3". Defaults + to 0.5. +* `--target_entropy` (*float*): target entropy used when learning the entropy + coefficient. If set to None, a heuristic value is used. Only used when the + algorithm is set to "SAC". Defaults to None. + +Additionally, the following arguments can be passed when training a multiagent +policy: + +* `--shared` (*store_true*): whether to use a shared policy for all agents +* `--maddpg` (*store_true*): whether to use an algorithm-specific variant of + the MADDPG algorithm + ## Simulated Examples diff --git a/examples/exp_configs/non_rl/bay_bridge.py b/examples/exp_configs/non_rl/bay_bridge.py index d7d78360f..f3e0c465f 100644 --- a/examples/exp_configs/non_rl/bay_bridge.py +++ b/examples/exp_configs/non_rl/bay_bridge.py @@ -48,7 +48,7 @@ lc_pushy=0.8, lc_speed_gain=4.0, model="LC2013", - lane_change_mode="no_lat_collide", + lane_change_mode="no_lc_safe", # lcKeepRight=0.8 ), num_vehicles=1400) diff --git a/examples/exp_configs/non_rl/bay_bridge_toll.py b/examples/exp_configs/non_rl/bay_bridge_toll.py index 1b8268aeb..0941823cb 100644 --- a/examples/exp_configs/non_rl/bay_bridge_toll.py +++ b/examples/exp_configs/non_rl/bay_bridge_toll.py @@ -46,7 +46,7 @@ model="LC2013", lcCooperative=0.2, lcSpeedGain=15, - lane_change_mode="no_lat_collide", + lane_change_mode="no_lc_safe", ), num_vehicles=50) diff --git a/examples/exp_configs/non_rl/highway_single.py b/examples/exp_configs/non_rl/highway_single.py new file mode 100644 index 000000000..86011ea02 --- /dev/null +++ b/examples/exp_configs/non_rl/highway_single.py @@ -0,0 +1,153 @@ +"""Example of an open network with human-driven vehicles and a wave.""" + +import numpy as np + +from flow.controllers import IDMController +from flow.controllers.velocity_controllers import FollowerStopper +from flow.core.params import EnvParams +from flow.core.params import NetParams +from flow.core.params import InitialConfig +from flow.core.params import InFlows +from flow.core.params import VehicleParams +from flow.core.params import SumoParams +from flow.core.params import SumoLaneChangeParams +from flow.core.rewards import instantaneous_mpg +from flow.core.params import SumoCarFollowingParams +from flow.networks import HighwayNetwork +from flow.envs import TestEnv +from flow.networks.highway import ADDITIONAL_NET_PARAMS + +# the speed of vehicles entering the network +TRAFFIC_SPEED = 24.1 +# the maximum speed at the downstream boundary edge +END_SPEED = 6.0 +# the inflow rate of vehicles +TRAFFIC_FLOW = 2215 +# the simulation time horizon (in steps) +HORIZON = 1500 +# whether to include noise in the car-following models +INCLUDE_NOISE = True +# fraction of vehicles that are follower-stoppers. 0.10 corresponds to 10% +PENETRATION_RATE = 0.0 + +additional_net_params = ADDITIONAL_NET_PARAMS.copy() +additional_net_params.update({ + # length of the highway + "length": 2500, + # number of lanes + "lanes": 1, + # speed limit for all edges + "speed_limit": 30, + # number of edges to divide the highway into + "num_edges": 2, + # whether to include a ghost edge. This edge is provided a different speed + # limit. + "use_ghost_edge": True, + # speed limit for the ghost edge + "ghost_speed_limit": END_SPEED, + # length of the downstream ghost edge with the reduced speed limit + "boundary_cell_length": 300, +}) + +vehicles = VehicleParams() +vehicles.add( + "human", + acceleration_controller=(IDMController, { + 'a': 1.3, + 'b': 2.0, + 'noise': 0.3 if INCLUDE_NOISE else 0.0 + }), + car_following_params=SumoCarFollowingParams( + min_gap=0.5 + ), + lane_change_params=SumoLaneChangeParams( + model="SL2015", + lc_sublane=2.0, + ), +) + +if PENETRATION_RATE > 0.0: + vehicles.add( + "av", + color='red', + num_vehicles=0, + acceleration_controller=(FollowerStopper, { + "v_des": 5.0, + "control_length": [500, 2300] + }), + ) + +inflows = InFlows() + +inflows.add( + veh_type="human", + edge="highway_0", + vehs_per_hour=int(TRAFFIC_FLOW * (1 - PENETRATION_RATE)), + depart_lane="free", + depart_speed=TRAFFIC_SPEED, + name="idm_highway_inflow") + +if PENETRATION_RATE > 0.0: + inflows.add( + veh_type="av", + edge="highway_0", + vehs_per_hour=int(TRAFFIC_FLOW * PENETRATION_RATE), + depart_lane="free", + depart_speed=TRAFFIC_SPEED, + name="av_highway_inflow") + +# SET UP FLOW PARAMETERS + +flow_params = dict( + # name of the experiment + exp_tag='highway-single', + + # name of the flow environment the experiment is running on + env_name=TestEnv, + + # name of the network class the experiment is running on + network=HighwayNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=HORIZON, + warmup_steps=500, + sims_per_step=3, + ), + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + sim_step=0.4, + render=False, + use_ballistic=True, + restart_instance=False + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflows, + additional_params=additional_net_params + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig(), +) + +custom_callables = { + "avg_merge_speed": lambda env: np.nan_to_num(np.mean( + env.k.vehicle.get_speed(env.k.vehicle.get_ids()))), + "avg_outflow": lambda env: np.nan_to_num( + env.k.vehicle.get_outflow_rate(120)), + "miles_per_gallon": lambda env: np.nan_to_num( + instantaneous_mpg(env, env.k.vehicle.get_ids(), gain=1.0) + ) +} diff --git a/examples/exp_configs/non_rl/i210_subnetwork.py b/examples/exp_configs/non_rl/i210_subnetwork.py index dd85c56cf..0c66f42e7 100644 --- a/examples/exp_configs/non_rl/i210_subnetwork.py +++ b/examples/exp_configs/non_rl/i210_subnetwork.py @@ -1,9 +1,10 @@ """I-210 subnetwork example.""" import os - import numpy as np from flow.controllers.car_following_models import IDMController +from flow.controllers.velocity_controllers import FollowerStopper +from flow.controllers.routing_controllers import I210Router from flow.core.params import SumoParams from flow.core.params import EnvParams from flow.core.params import NetParams @@ -11,48 +12,133 @@ from flow.core.params import VehicleParams from flow.core.params import InitialConfig from flow.core.params import InFlows -import flow.config as config +from flow.core.rewards import instantaneous_mpg +from flow.networks import I210SubNetwork +from flow.networks.i210_subnetwork import EDGES_DISTRIBUTION from flow.envs import TestEnv -from flow.networks.i210_subnetwork import I210SubNetwork, EDGES_DISTRIBUTION +import flow.config as config + +# =========================================================================== # +# Specify some configurable constants. # +# =========================================================================== # + +# whether to include the upstream ghost edge in the network +WANT_GHOST_CELL = True +# whether to include the downstream slow-down edge in the network +WANT_DOWNSTREAM_BOUNDARY = True +# whether to include vehicles on the on-ramp +ON_RAMP = False +# the inflow rate of vehicles (in veh/hr) +INFLOW_RATE = 2050 +# on-ramp inflow_rate +ON_RAMP_INFLOW_RATE = 500 +# the speed of inflowing vehicles from the main edge (in m/s) +INFLOW_SPEED = 25.5 +# fraction of vehicles that are follower-stoppers. 0.10 corresponds to 10% +PENETRATION_RATE = 0.0 +# desired speed of the follower stopper vehicles +V_DES = 5.0 +# horizon over which to run the env +HORIZON = 1500 +# steps to run before follower-stopper is allowed to take control +WARMUP_STEPS = 600 + +# =========================================================================== # +# Specify the path to the network template. # +# =========================================================================== # + +if WANT_DOWNSTREAM_BOUNDARY: + NET_TEMPLATE = os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_" + "downstream.xml") +elif WANT_GHOST_CELL: + NET_TEMPLATE = os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/i210_with_ghost_cell.xml") +else: + NET_TEMPLATE = os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/test2.net.xml") + +# If the ghost cell is not being used, remove it from the initial edges that +# vehicles can be placed on. +edges_distribution = EDGES_DISTRIBUTION.copy() +if not WANT_GHOST_CELL: + edges_distribution.remove("ghost0") + +# =========================================================================== # +# Specify vehicle-specific information and inflows. # +# =========================================================================== # -# create the base vehicle type that will be used for inflows vehicles = VehicleParams() + vehicles.add( "human", num_vehicles=0, lane_change_params=SumoLaneChangeParams( - lane_change_mode="strategic", + lane_change_mode="sumo_default", ), acceleration_controller=(IDMController, { - "a": 0.3, "b": 2.0, "noise": 0.5 + "a": 1.3, + "b": 2.0, + "noise": 0.3, + }), + routing_controller=(I210Router, {}) if ON_RAMP else None, +) + +vehicles.add( + "av", + num_vehicles=0, + color="red", + acceleration_controller=(FollowerStopper, { + "v_des": V_DES, + "no_control_edges": ["ghost0", "119257908#3"] }), + routing_controller=(I210Router, {}) if ON_RAMP else None, ) inflow = InFlows() + # main highway -inflow.add( - veh_type="human", - edge="119257914", - vehs_per_hour=8378, - departLane="random", - departSpeed=23) +highway_start_edge = "ghost0" if WANT_GHOST_CELL else "119257914" + +for lane in [0, 1, 2, 3, 4]: + inflow.add( + veh_type="human", + edge=highway_start_edge, + vehs_per_hour=INFLOW_RATE * (1 - PENETRATION_RATE), + depart_lane=lane, + depart_speed=INFLOW_SPEED) + + if PENETRATION_RATE > 0.0: + inflow.add( + veh_type="av", + edge=highway_start_edge, + vehs_per_hour=INFLOW_RATE * PENETRATION_RATE, + depart_lane=lane, + depart_speed=INFLOW_SPEED) + # on ramp -# inflow.add( -# veh_type="human", -# edge="27414345", -# vehs_per_hour=321, -# departLane="random", -# departSpeed=20) -# inflow.add( -# veh_type="human", -# edge="27414342#0", -# vehs_per_hour=421, -# departLane="random", -# departSpeed=20) - -NET_TEMPLATE = os.path.join( - config.PROJECT_PATH, - "examples/exp_configs/templates/sumo/test2.net.xml") +if ON_RAMP: + inflow.add( + veh_type="human", + edge="27414345", + vehs_per_hour=int(ON_RAMP_INFLOW_RATE * (1 - PENETRATION_RATE)), + depart_speed=10, + ) + + if PENETRATION_RATE > 0.0: + inflow.add( + veh_type="av", + edge="27414345", + vehs_per_hour=int(ON_RAMP_INFLOW_RATE * PENETRATION_RATE), + depart_lane="random", + depart_speed=10) + +# =========================================================================== # +# Generate the flow_params dict with all relevant simulation information. # +# =========================================================================== # flow_params = dict( # name of the experiment @@ -69,22 +155,28 @@ # simulation-related parameters sim=SumoParams( - sim_step=0.5, + sim_step=0.4, render=False, - color_by_speed=True, + color_by_speed=False, use_ballistic=True ), # environment related parameters (see flow.core.params.EnvParams) env=EnvParams( - horizon=4500, + horizon=HORIZON, + warmup_steps=WARMUP_STEPS, + sims_per_step=3 ), # network-related parameters (see flow.core.params.NetParams and the # network's documentation or ADDITIONAL_NET_PARAMS component) net=NetParams( inflows=inflow, - template=NET_TEMPLATE + template=NET_TEMPLATE, + additional_params={ + "on_ramp": ON_RAMP, + "ghost_edge": WANT_GHOST_CELL, + } ), # vehicles to be placed in the network at the start of a rollout (see @@ -94,19 +186,30 @@ # parameters specifying the positioning of vehicles upon initialization/ # reset (see flow.core.params.InitialConfig) initial=InitialConfig( - edges_distribution=EDGES_DISTRIBUTION, + edges_distribution=edges_distribution, ), ) +# =========================================================================== # +# Specify custom callable that is logged during simulation runtime. # +# =========================================================================== # + edge_id = "119257908#1-AddedOnRampEdge" + + +def valid_ids(env, veh_ids): + """Return the names of vehicles within the controllable edges.""" + return [ + veh_id for veh_id in veh_ids + if env.k.vehicle.get_edge(veh_id) not in ["ghost0", "119257908#3"] + ] + + custom_callables = { "avg_merge_speed": lambda env: np.nan_to_num(np.mean( - env.k.vehicle.get_speed(env.k.vehicle.get_ids_by_edge(edge_id)))), + env.k.vehicle.get_speed(valid_ids(env, env.k.vehicle.get_ids())))), "avg_outflow": lambda env: np.nan_to_num( env.k.vehicle.get_outflow_rate(120)), - # we multiply by 5 to account for the vehicle length and by 1000 to convert - # into veh/km - "avg_density": lambda env: 5 * 1000 * len(env.k.vehicle.get_ids_by_edge( - edge_id)) / (env.k.network.edge_length(edge_id) - * env.k.network.num_lanes(edge_id)), + "mpg": lambda env: instantaneous_mpg( + env, valid_ids(env, env.k.vehicle.get_ids()), gain=1.0), } diff --git a/examples/exp_configs/non_rl/i210_subnetwork_sweep.py b/examples/exp_configs/non_rl/i210_subnetwork_sweep.py deleted file mode 100644 index 28cba81ce..000000000 --- a/examples/exp_configs/non_rl/i210_subnetwork_sweep.py +++ /dev/null @@ -1,151 +0,0 @@ -"""I-210 subnetwork example. - -In this case flow_params is a list of dicts. This is to test the effects of -multiple human-driver model parameters on the flow traffic. -""" -from collections import OrderedDict -from copy import deepcopy -import itertools -import os -import numpy as np - -from flow.core.params import SumoParams -from flow.core.params import EnvParams -from flow.core.params import NetParams -from flow.core.params import SumoLaneChangeParams -from flow.core.params import VehicleParams -from flow.core.params import InitialConfig -from flow.core.params import InFlows -import flow.config as config -from flow.envs import TestEnv -from flow.networks.i210_subnetwork import I210SubNetwork, EDGES_DISTRIBUTION - -# the default parameters for all lane change parameters -default_dict = { - "lane_change_mode": "strategic", - "model": "LC2013", - "lc_strategic": 1.0, - "lc_cooperative": 1.0, - "lc_speed_gain": 1.0, - "lc_keep_right": 1.0, - "lc_look_ahead_left": 2.0, - "lc_speed_gain_right": 1.0, - "lc_sublane": 1.0, - "lc_pushy": 0, - "lc_pushy_gap": 0.6, - "lc_assertive": 1, - "lc_accel_lat": 1.0 -} - -# values to sweep through for some lane change parameters -sweep_dict = OrderedDict({ - "lc_strategic": [1.0, 2.0, 4.0, 8.0], - "lc_cooperative": [1.0, 2.0], - "lc_look_ahead_left": [2.0, 4.0] -}) - -# Create a list of possible lane change parameter combinations. -all_names = sorted(sweep_dict) -combinations = itertools.product(*(sweep_dict[name] for name in all_names)) -combination_list = list(combinations) -res = [] -for val in combination_list: - curr_dict = {} - for elem, name in zip(val, all_names): - curr_dict[name] = elem - res.append(curr_dict) - -# Create a list of all possible flow_params dictionaries to sweep through the -# different lane change parameters. -flow_params = [] - -for lane_change_dict in res: - # no vehicles in the network. The lane change parameters of inflowing - # vehicles are updated here. - vehicles = VehicleParams() - update_dict = deepcopy(default_dict) - update_dict.update(lane_change_dict) - vehicles.add( - "human", - num_vehicles=0, - lane_change_params=SumoLaneChangeParams(**update_dict) - ) - - inflow = InFlows() - # main highway - inflow.add( - veh_type="human", - edge="119257914", - vehs_per_hour=8378, - # probability=1.0, - departLane="random", - departSpeed=20) - # on ramp - inflow.add( - veh_type="human", - edge="27414345", - vehs_per_hour=321, - departLane="random", - departSpeed=20) - inflow.add( - veh_type="human", - edge="27414342#0", - vehs_per_hour=421, - departLane="random", - departSpeed=20) - - NET_TEMPLATE = os.path.join( - config.PROJECT_PATH, - "examples/exp_configs/templates/sumo/test2.net.xml") - - params = dict( - # name of the experiment - exp_tag='I-210_subnetwork', - - # name of the flow environment the experiment is running on - env_name=TestEnv, - - # name of the network class the experiment is running on - network=I210SubNetwork, - - # simulator that is used by the experiment - simulator='traci', - - # simulation-related parameters - sim=SumoParams( - sim_step=0.8, - render=True, - color_by_speed=True - ), - - # environment related parameters (see flow.core.params.EnvParams) - env=EnvParams( - horizon=4500, # one hour of run time - ), - - # network-related parameters (see flow.core.params.NetParams and the - # network's documentation or ADDITIONAL_NET_PARAMS component) - net=NetParams( - inflows=inflow, - template=NET_TEMPLATE - ), - - # vehicles to be placed in the network at the start of a rollout (see - # flow.core.params.VehicleParams) - veh=vehicles, - - # parameters specifying the positioning of vehicles upon - # initialization/reset (see flow.core.params.InitialConfig) - initial=InitialConfig( - edges_distribution=EDGES_DISTRIBUTION, - ), - ) - - # Store the next flow_params dict. - flow_params.append(params) - - -custom_callables = { - "avg_merge_speed": lambda env: np.mean(env.k.vehicle.get_speed( - env.k.vehicle.get_ids_by_edge("119257908#1-AddedOnRampEdge"))) -} diff --git a/examples/exp_configs/non_rl/minicity.py b/examples/exp_configs/non_rl/minicity.py index 23b232480..35d5edbce 100644 --- a/examples/exp_configs/non_rl/minicity.py +++ b/examples/exp_configs/non_rl/minicity.py @@ -18,7 +18,7 @@ speed_mode=1, ), lane_change_params=SumoLaneChangeParams( - lane_change_mode="no_lat_collide", + lane_change_mode="no_lc_safe", ), initial_speed=0, num_vehicles=90) diff --git a/examples/exp_configs/rl/multiagent/adversarial_figure_eight.py b/examples/exp_configs/rl/multiagent/adversarial_figure_eight.py index 4fb81ce97..a7834c6cd 100644 --- a/examples/exp_configs/rl/multiagent/adversarial_figure_eight.py +++ b/examples/exp_configs/rl/multiagent/adversarial_figure_eight.py @@ -8,7 +8,7 @@ # the negative of the AV reward from copy import deepcopy -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from flow.controllers import ContinuousRouter from flow.controllers import IDMController from flow.controllers import RLController diff --git a/examples/exp_configs/rl/multiagent/lord_of_the_rings.py b/examples/exp_configs/rl/multiagent/lord_of_the_rings.py index e7688c87d..866d915ce 100644 --- a/examples/exp_configs/rl/multiagent/lord_of_the_rings.py +++ b/examples/exp_configs/rl/multiagent/lord_of_the_rings.py @@ -3,7 +3,7 @@ Creates a set of stabilizing the ring experiments to test if more agents -> fewer needed batches """ -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from flow.controllers import ContinuousRouter from flow.controllers import IDMController from flow.controllers import RLController diff --git a/examples/exp_configs/rl/multiagent/multiagent_figure_eight.py b/examples/exp_configs/rl/multiagent/multiagent_figure_eight.py index 0579bb978..2b18bf61b 100644 --- a/examples/exp_configs/rl/multiagent/multiagent_figure_eight.py +++ b/examples/exp_configs/rl/multiagent/multiagent_figure_eight.py @@ -1,5 +1,5 @@ """Figure eight example.""" -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from ray.tune.registry import register_env from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams diff --git a/examples/exp_configs/rl/multiagent/multiagent_highway.py b/examples/exp_configs/rl/multiagent/multiagent_highway.py index cec0b3fba..353eccb2a 100644 --- a/examples/exp_configs/rl/multiagent/multiagent_highway.py +++ b/examples/exp_configs/rl/multiagent/multiagent_highway.py @@ -3,7 +3,7 @@ Trains a non-constant number of agents, all sharing the same policy, on the highway with ramps network. """ -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from flow.controllers import RLController from flow.core.params import EnvParams, NetParams, InitialConfig, InFlows, \ VehicleParams, SumoParams, \ diff --git a/examples/exp_configs/rl/multiagent/multiagent_i210.py b/examples/exp_configs/rl/multiagent/multiagent_i210.py index 94f709ff4..3a8207eb8 100644 --- a/examples/exp_configs/rl/multiagent/multiagent_i210.py +++ b/examples/exp_configs/rl/multiagent/multiagent_i210.py @@ -4,114 +4,217 @@ highway with ramps network. """ import os +import numpy as np -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy from ray.tune.registry import register_env +from flow.controllers import RLController +from flow.controllers.routing_controllers import I210Router +from flow.controllers.car_following_models import IDMController import flow.config as config -from flow.controllers.rlcontroller import RLController from flow.core.params import EnvParams from flow.core.params import NetParams from flow.core.params import InitialConfig from flow.core.params import InFlows from flow.core.params import VehicleParams from flow.core.params import SumoParams +from flow.core.params import SumoCarFollowingParams from flow.core.params import SumoLaneChangeParams -from flow.networks.i210_subnetwork import I210SubNetwork, EDGES_DISTRIBUTION +from flow.core.rewards import energy_consumption from flow.envs.multiagent.i210 import I210MultiEnv, ADDITIONAL_ENV_PARAMS from flow.utils.registry import make_create_env +from flow.networks.i210_subnetwork import I210SubNetwork, EDGES_DISTRIBUTION + +# =========================================================================== # +# Specify some configurable constants. # +# =========================================================================== # -# SET UP PARAMETERS FOR THE SIMULATION +# whether to include the downstream slow-down edge in the network as well as a +# ghost cell at the upstream edge +WANT_BOUNDARY_CONDITIONS = True +# whether to include vehicles on the on-ramp +ON_RAMP = False +# the inflow rate of vehicles (in veh/hr) +INFLOW_RATE = 2050 +# the inflow rate on the on-ramp (in veh/hr) +ON_RAMP_INFLOW_RATE = 500 +# the speed of inflowing vehicles from the main edge (in m/s) +INFLOW_SPEED = 25.5 +# fraction of vehicles that are RL vehicles. 0.10 corresponds to 10% +PENETRATION_RATE = 0.05 +# desired speed of the vehicles in the network +V_DES = 5.0 +# horizon over which to run the env +HORIZON = 1000 +# steps to run before follower-stopper is allowed to take control +WARMUP_STEPS = 600 +# whether to turn off the fail safes for the human-driven vehicles +ALLOW_COLLISIONS = False -# number of training iterations -N_TRAINING_ITERATIONS = 200 -# number of rollouts per training iteration -N_ROLLOUTS = 2 -# number of steps per rollout -HORIZON = 500 -# number of parallel workers -N_CPUS = 1 +# =========================================================================== # +# Specify the path to the network template. # +# =========================================================================== # -# percentage of autonomous vehicles compared to human vehicles on highway -PENETRATION_RATE = 10 +if WANT_BOUNDARY_CONDITIONS: + NET_TEMPLATE = os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_" + "downstream.xml") +else: + NET_TEMPLATE = os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/test2.net.xml") +edges_distribution = EDGES_DISTRIBUTION.copy() + +# =========================================================================== # +# Set up parameters for the environment. # +# =========================================================================== # -# SET UP PARAMETERS FOR THE ENVIRONMENT additional_env_params = ADDITIONAL_ENV_PARAMS.copy() additional_env_params.update({ - 'max_accel': 1, - 'max_decel': 1, - # configure the observation space. Look at the I210MultiEnv class for more info. + 'max_accel': 2.6, + 'max_decel': 4.5, + + # configure the observation space. Look at the I210MultiEnv class for more + # info. 'lead_obs': True, + # whether to add in a reward for the speed of nearby vehicles + "local_reward": True, + # whether to use the MPG reward. Otherwise, defaults to a target velocity + # reward + "mpg_reward": False, + # whether to use the MPJ reward. Otherwise, defaults to a target velocity + # reward + "mpj_reward": False, + # how many vehicles to look back for any reward + "look_back_length": 3, + # whether to reroute vehicles once they have exited + "reroute_on_exit": False, + 'target_velocity': 5.0, + # how many AVs there can be at once (this is only for centralized critics) + "max_num_agents": 10, + # which edges we shouldn't apply control on + "no_control_edges": ["ghost0", "119257908#3"], + + # whether to add a slight reward for opening up a gap that will be annealed + # out N iterations in + "headway_curriculum": False, + # how many timesteps to anneal the headway curriculum over + "headway_curriculum_iters": 100, + # weight of the headway reward + "headway_reward_gain": 2.0, + # desired time headway + "min_time_headway": 2.0, + + # whether to add a slight reward for traveling at a desired speed + "speed_curriculum": True, + # how many timesteps to anneal the headway curriculum over + "speed_curriculum_iters": 20, + # weight of the headway reward + "speed_reward_gain": 5.0, + # penalize stopped vehicles + "penalize_stops": False, + "stop_penalty": 0.01, + + # penalize accels + "penalize_accel": False, + "accel_penalty": (1 / 400.0) }) -# CREATE VEHICLE TYPES AND INFLOWS -# no vehicles in the network +# =========================================================================== # +# Specify vehicle-specific information and inflows. # +# =========================================================================== # + +# create the base vehicle types that will be used for inflows vehicles = VehicleParams() -vehicles.add( - "human", - num_vehicles=0, - lane_change_params=SumoLaneChangeParams( - lane_change_mode="strategic", +if ON_RAMP: + vehicles.add( + "human", + num_vehicles=0, + routing_controller=(I210Router, {}), + acceleration_controller=(IDMController, { + 'a': 1.3, + 'b': 2.0, + 'noise': 0.3 + }), + car_following_params=SumoCarFollowingParams( + speed_mode=19 if ALLOW_COLLISIONS else 'right_of_way' + ), + lane_change_params=SumoLaneChangeParams( + lane_change_mode="sumo_default", + ), + ) +else: + vehicles.add( + "human", + num_vehicles=0, + acceleration_controller=(IDMController, { + 'a': 1.3, + 'b': 2.0, + 'noise': 0.3 + }), + car_following_params=SumoCarFollowingParams( + speed_mode=19 if ALLOW_COLLISIONS else 'right_of_way' + ), + lane_change_params=SumoLaneChangeParams( + lane_change_mode="sumo_default", + ), ) -) vehicles.add( "av", - acceleration_controller=(RLController, {}), num_vehicles=0, + acceleration_controller=(RLController, {}), ) inflow = InFlows() -# main highway -pen_rate = PENETRATION_RATE / 100 -assert pen_rate < 1.0, "your penetration rate is over 100%" -assert pen_rate > 0.0, "your penetration rate should be above zero" -inflow.add( - veh_type="human", - edge="119257914", - vehs_per_hour=8378 * pen_rate, - # probability=1.0, - departLane="random", - departSpeed=20) -# on ramp -# inflow.add( -# veh_type="human", -# edge="27414345", -# vehs_per_hour=321 * pen_rate, -# departLane="random", -# departSpeed=20) -# inflow.add( -# veh_type="human", -# edge="27414342#0", -# vehs_per_hour=421 * pen_rate, -# departLane="random", -# departSpeed=20) - -# Now add the AVs -# main highway -inflow.add( - veh_type="av", - edge="119257914", - vehs_per_hour=int(8378 * pen_rate), - # probability=1.0, - departLane="random", - departSpeed=20) -# # on ramp -# inflow.add( -# veh_type="av", -# edge="27414345", -# vehs_per_hour=int(321 * pen_rate), -# departLane="random", -# departSpeed=20) -# inflow.add( -# veh_type="av", -# edge="27414342#0", -# vehs_per_hour=int(421 * pen_rate), -# departLane="random", -# departSpeed=20) - -NET_TEMPLATE = os.path.join( - config.PROJECT_PATH, - "examples/exp_configs/templates/sumo/test2.net.xml") +for lane in [0, 1, 2, 3, 4]: + if WANT_BOUNDARY_CONDITIONS: + # Add the inflows from the main highway. + inflow.add( + veh_type="human", + edge="ghost0", + vehs_per_hour=int(INFLOW_RATE * (1 - PENETRATION_RATE)), + departLane=lane, + departSpeed=INFLOW_SPEED) + inflow.add( + veh_type="av", + edge="ghost0", + vehs_per_hour=int(INFLOW_RATE * PENETRATION_RATE), + departLane=lane, + departSpeed=INFLOW_SPEED) + else: + # Add the inflows from the main highway. + inflow.add( + veh_type="human", + edge="119257914", + vehs_per_hour=int(INFLOW_RATE * (1 - PENETRATION_RATE)), + departLane=lane, + departSpeed=INFLOW_SPEED) + inflow.add( + veh_type="av", + edge="119257914", + vehs_per_hour=int(INFLOW_RATE * PENETRATION_RATE), + departLane=lane, + departSpeed=INFLOW_SPEED) + + # Add the inflows from the on-ramps. + if ON_RAMP: + inflow.add( + veh_type="human", + edge="27414345", + vehs_per_hour=int(ON_RAMP_INFLOW_RATE * (1 - PENETRATION_RATE)), + departLane="random", + departSpeed=10) + inflow.add( + veh_type="human", + edge="27414342#0", + vehs_per_hour=int(ON_RAMP_INFLOW_RATE * (1 - PENETRATION_RATE)), + departLane="random", + departSpeed=10) + +# =========================================================================== # +# Generate the flow_params dict with all relevant simulation information. # +# =========================================================================== # flow_params = dict( # name of the experiment @@ -128,24 +231,32 @@ # simulation-related parameters sim=SumoParams( - sim_step=0.8, + sim_step=0.4, render=False, - color_by_speed=True, - restart_instance=True + color_by_speed=False, + restart_instance=True, + use_ballistic=True, + disable_collisions=True ), # environment related parameters (see flow.core.params.EnvParams) env=EnvParams( horizon=HORIZON, - sims_per_step=1, + sims_per_step=3, + warmup_steps=WARMUP_STEPS, additional_params=additional_env_params, + done_at_exit=not additional_env_params["reroute_on_exit"] ), # network-related parameters (see flow.core.params.NetParams and the # network's documentation or ADDITIONAL_NET_PARAMS component) net=NetParams( inflows=inflow, - template=NET_TEMPLATE + template=NET_TEMPLATE, + additional_params={ + "on_ramp": ON_RAMP, + "ghost_edge": WANT_BOUNDARY_CONDITIONS + } ), # vehicles to be placed in the network at the start of a rollout (see @@ -155,23 +266,25 @@ # parameters specifying the positioning of vehicles upon initialization/ # reset (see flow.core.params.InitialConfig) initial=InitialConfig( - edges_distribution=EDGES_DISTRIBUTION, + edges_distribution=edges_distribution, ), ) -# SET UP RLLIB MULTI-AGENT FEATURES +# =========================================================================== # +# Set up rllib multi-agent features. # +# =========================================================================== # create_env, env_name = make_create_env(params=flow_params, version=0) # register as rllib env register_env(env_name, create_env) -# multiagent configuration +# multi-agent configuration test_env = create_env() obs_space = test_env.observation_space act_space = test_env.action_space -POLICY_GRAPHS = {'av': (PPOTFPolicy, obs_space, act_space, {})} +POLICY_GRAPHS = {'av': (None, obs_space, act_space, {})} POLICIES_TO_TRAIN = ['av'] @@ -179,3 +292,14 @@ def policy_mapping_fn(_): """Map a policy in RLlib.""" return 'av' + + +custom_callables = { + "avg_speed": lambda env: np.mean([speed for speed in + env.k.vehicle.get_speed(env.k.vehicle.get_ids()) if speed >= 0]), + "avg_outflow": lambda env: np.nan_to_num(env.k.vehicle.get_outflow_rate(120)), + "avg_energy": lambda env: -1 * energy_consumption(env, 0.1), + "avg_per_step_energy": lambda env: -1 * energy_consumption(env, 0.1) / env.k.vehicle.num_vehicles + if env.k.vehicle.num_vehicles > 0 + else 0, +} diff --git a/examples/exp_configs/rl/multiagent/multiagent_merge.py b/examples/exp_configs/rl/multiagent/multiagent_merge.py index bfc9fb3b7..312d3a0dd 100644 --- a/examples/exp_configs/rl/multiagent/multiagent_merge.py +++ b/examples/exp_configs/rl/multiagent/multiagent_merge.py @@ -3,7 +3,7 @@ Trains a a small percentage of rl vehicles to dissipate shockwaves caused by on-ramp merge to a single lane open highway network. """ -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from ray.tune.registry import register_env from flow.core.params import SumoParams, EnvParams, InitialConfig diff --git a/examples/exp_configs/rl/multiagent/multiagent_ring.py b/examples/exp_configs/rl/multiagent/multiagent_ring.py index a789174f4..c2fc52f14 100644 --- a/examples/exp_configs/rl/multiagent/multiagent_ring.py +++ b/examples/exp_configs/rl/multiagent/multiagent_ring.py @@ -3,7 +3,7 @@ Trains a number of autonomous vehicles to stabilize the flow of 22 vehicles in a variable length ring road. """ -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from ray.tune.registry import register_env from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams diff --git a/examples/exp_configs/rl/multiagent/multiagent_straight_road.py b/examples/exp_configs/rl/multiagent/multiagent_straight_road.py new file mode 100644 index 000000000..e0726b059 --- /dev/null +++ b/examples/exp_configs/rl/multiagent/multiagent_straight_road.py @@ -0,0 +1,211 @@ +"""Multi-agent highway with ramps example. + +Trains a non-constant number of agents, all sharing the same policy, on the +highway with ramps network. +""" +from flow.controllers import RLController, IDMController +from flow.core.params import EnvParams, NetParams, InitialConfig, InFlows, \ + VehicleParams, SumoParams, SumoLaneChangeParams, SumoCarFollowingParams +from flow.networks import HighwayNetwork +from flow.envs.ring.accel import ADDITIONAL_ENV_PARAMS +from flow.envs.multiagent import MultiStraightRoad +from flow.networks.highway import ADDITIONAL_NET_PARAMS +from flow.utils.registry import make_create_env +from ray.tune.registry import register_env + +# SET UP PARAMETERS FOR THE SIMULATION + +# the speed of vehicles entering the network +TRAFFIC_SPEED = 24.1 +# the maximum speed at the downstream boundary edge +END_SPEED = 6.0 +# the inflow rate of vehicles +HIGHWAY_INFLOW_RATE = 2215 +# the simulation time horizon (in steps) +HORIZON = 1000 +# whether to include noise in the car-following models +INCLUDE_NOISE = True + +PENETRATION_RATE = 10.0 + +WARMUP_STEPS = 500 + +additional_net_params = ADDITIONAL_NET_PARAMS.copy() +additional_net_params.update({ + # length of the highway + "length": 2500, + # number of lanes + "lanes": 1, + # speed limit for all edges + "speed_limit": 30, + # number of edges to divide the highway into + "num_edges": 2, + # whether to include a ghost edge + "use_ghost_edge": True, + # speed limit for the ghost edge + "ghost_speed_limit": END_SPEED, + # length of the cell imposing a boundary + "boundary_cell_length": 300, +}) + + +# SET UP PARAMETERS FOR THE ENVIRONMENT + +additional_env_params = ADDITIONAL_ENV_PARAMS.copy() +additional_env_params.update({ + 'max_accel': 2.6, + 'max_decel': 4.5, + 'target_velocity': 6.0, + 'local_reward': True, + 'lead_obs': True, + 'control_range': [500, 2300], + # whether to reroute vehicles once they have exited + "reroute_on_exit": True, + # whether to use the MPG reward. Otherwise, defaults to a target velocity reward + "mpg_reward": False, + # whether to use the joules reward. Otherwise, defaults to a target velocity reward + "mpj_reward": False, + # how many vehicles to look back for the MPG reward + "look_back_length": 3, + # how many AVs there can be at once (this is only for centralized critics) + "max_num_agents": 10, + + # whether to add a slight reward for opening up a gap that will be annealed out N iterations in + "headway_curriculum": False, + # how many timesteps to anneal the headway curriculum over + "headway_curriculum_iters": 100, + # weight of the headway reward + "headway_reward_gain": 2.0, + # desired time headway + "min_time_headway": 2.0, + + # whether to add a slight reward for traveling at a desired speed + "speed_curriculum": True, + # how many timesteps to anneal the headway curriculum over + "speed_curriculum_iters": 20, + # weight of the headway reward + "speed_reward_gain": 1.0, + + # penalize stopped vehicles + "penalize_stops": True, + "stop_penalty": 0.05, + + # penalize accels + "penalize_accel": True, + "accel_penalty": 0.05, + +}) + + +# CREATE VEHICLE TYPES AND INFLOWS + +vehicles = VehicleParams() +inflows = InFlows() +vehicles.add( + "human", + acceleration_controller=(IDMController, { + 'a': 1.3, + 'b': 2.0, + 'noise': 0.3 if INCLUDE_NOISE else 0.0 + }), + car_following_params=SumoCarFollowingParams( + min_gap=0.5 + ), + lane_change_params=SumoLaneChangeParams( + model="SL2015", + lc_sublane=2.0, + ), +) + +# autonomous vehicles +vehicles.add( + color='red', + veh_id='rl', + acceleration_controller=(RLController, {})) + +# add human vehicles on the highway +inflows.add( + veh_type="human", + edge="highway_0", + vehs_per_hour=int(HIGHWAY_INFLOW_RATE * (1 - PENETRATION_RATE / 100)), + depart_lane="free", + depart_speed=TRAFFIC_SPEED, + name="idm_highway_inflow") + +# add autonomous vehicles on the highway +# they will stay on the highway, i.e. they won't exit through the off-ramps +inflows.add( + veh_type="rl", + edge="highway_0", + vehs_per_hour=int(HIGHWAY_INFLOW_RATE * (PENETRATION_RATE / 100)), + depart_lane="free", + depart_speed=TRAFFIC_SPEED, + name="rl_highway_inflow") + +flow_params = dict( + # name of the experiment + exp_tag='multiagent_highway', + + # name of the flow environment the experiment is running on + env_name=MultiStraightRoad, + + # name of the network class the experiment is running on + network=HighwayNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=HORIZON, + warmup_steps=WARMUP_STEPS, + sims_per_step=3, + additional_params=additional_env_params + ), + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + sim_step=0.4, + render=False, + restart_instance=True, + use_ballistic=True + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflows, + additional_params=additional_net_params + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig(), +) + + +# SET UP RLLIB MULTI-AGENT FEATURES + +create_env, env_name = make_create_env(params=flow_params, version=0) + +# register as rllib env +register_env(env_name, create_env) + +# multiagent configuration +test_env = create_env() +obs_space = test_env.observation_space +act_space = test_env.action_space + + +POLICY_GRAPHS = {'av': (None, obs_space, act_space, {})} + +POLICIES_TO_TRAIN = ['av'] + + +def policy_mapping_fn(_): + """Map a policy in RLlib.""" + return 'av' diff --git a/examples/exp_configs/rl/multiagent/multiagent_traffic_light_grid.py b/examples/exp_configs/rl/multiagent/multiagent_traffic_light_grid.py index b8293f638..308dfa0d7 100644 --- a/examples/exp_configs/rl/multiagent/multiagent_traffic_light_grid.py +++ b/examples/exp_configs/rl/multiagent/multiagent_traffic_light_grid.py @@ -1,6 +1,6 @@ """Multi-agent traffic light example (single shared policy).""" -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from flow.envs.multiagent import MultiTrafficLightGridPOEnv from flow.networks import TrafficLightGridNetwork from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams diff --git a/examples/exp_configs/rl/singleagent/singleagent_straight_road.py b/examples/exp_configs/rl/singleagent/singleagent_straight_road.py new file mode 100644 index 000000000..efd56214a --- /dev/null +++ b/examples/exp_configs/rl/singleagent/singleagent_straight_road.py @@ -0,0 +1,164 @@ +"""Multi-agent highway with ramps example. + +Trains a non-constant number of agents, all sharing the same policy, on the +highway with ramps network. +""" +from flow.controllers import RLController, IDMController +from flow.core.params import EnvParams, NetParams, InitialConfig, InFlows, \ + VehicleParams, SumoParams, SumoLaneChangeParams +from flow.envs.ring.accel import ADDITIONAL_ENV_PARAMS +from flow.networks import HighwayNetwork +from flow.envs import SingleStraightRoad +from flow.networks.highway import ADDITIONAL_NET_PARAMS +from flow.utils.registry import make_create_env +from ray.tune.registry import register_env + + +# SET UP PARAMETERS FOR THE SIMULATION + +# number of steps per rollout +HORIZON = 2000 + +# inflow rate on the highway in vehicles per hour +HIGHWAY_INFLOW_RATE = 10800 / 5 +# percentage of autonomous vehicles compared to human vehicles on highway +PENETRATION_RATE = 10 + + +# SET UP PARAMETERS FOR THE NETWORK + +additional_net_params = ADDITIONAL_NET_PARAMS.copy() +additional_net_params.update({ + # length of the highway + "length": 2000, + # number of lanes + "lanes": 1, + # speed limit for all edges + "speed_limit": 30, + # number of edges to divide the highway into + "num_edges": 2 +}) + + +# SET UP PARAMETERS FOR THE ENVIRONMENT + +additional_env_params = ADDITIONAL_ENV_PARAMS.copy() +additional_env_params.update({ + 'max_accel': 2.6, + 'max_decel': 4.5, + 'target_velocity': 18.0, + 'local_reward': True, + 'lead_obs': True, + "terminate_on_wave": False, + # the environment is not allowed to terminate below this horizon length + 'wave_termination_horizon': 1000, + # the speed below which we consider a wave to have occured + 'wave_termination_speed': 10.0, + # whether the vehicle continues to acquire reward after it exits the system. This causes it to have incentive + # to leave the network in a good state after it leaves + 'reward_after_exit': True +}) + + +# CREATE VEHICLE TYPES AND INFLOWS + +vehicles = VehicleParams() +inflows = InFlows() + +# human vehicles +vehicles.add( + "human", + num_vehicles=0, + lane_change_params=SumoLaneChangeParams( + lane_change_mode="strategic", + ), + acceleration_controller=(IDMController, {"a": .3, "b": 2.0, "noise": 0.5}), +) + +# autonomous vehicles +vehicles.add( + veh_id='rl', + acceleration_controller=(RLController, {})) + +# add human vehicles on the highway +inflows.add( + veh_type="human", + edge="highway_0", + vehs_per_hour=int(HIGHWAY_INFLOW_RATE * (1 - PENETRATION_RATE / 100)), + depart_lane="free", + depart_speed="23.0", + name="idm_highway_inflow") + +# add autonomous vehicles on the highway +# they will stay on the highway, i.e. they won't exit through the off-ramps +inflows.add( + veh_type="rl", + edge="highway_0", + vehs_per_hour=int(HIGHWAY_INFLOW_RATE * (PENETRATION_RATE / 100)), + depart_lane="free", + depart_speed="23.0", + name="rl_highway_inflow") + +# SET UP FLOW PARAMETERS +done_at_exit = True +if additional_env_params['reward_after_exit']: + done_at_exit = False + +flow_params = dict( + # name of the experiment + exp_tag='singleagent_highway', + + # name of the flow environment the experiment is running on + env_name=SingleStraightRoad, + + # name of the network class the experiment is running on + network=HighwayNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=HORIZON, + warmup_steps=500, + sims_per_step=1, # do not put more than one + done_at_exit=done_at_exit, + additional_params=additional_env_params, + ), + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + sim_step=0.5, + render=False, + use_ballistic=True, + restart_instance=True + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflows, + additional_params=additional_net_params + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig(), +) + + +# SET UP RLLIB MULTI-AGENT FEATURES + +create_env, env_name = make_create_env(params=flow_params, version=0) + +# register as rllib env +register_env(env_name, create_env) + +# multiagent configuration +test_env = create_env() +obs_space = test_env.observation_space +act_space = test_env.action_space diff --git a/examples/exp_configs/templates/sumo/i210_with_ghost_cell.xml b/examples/exp_configs/templates/sumo/i210_with_ghost_cell.xml new file mode 100644 index 000000000..66e5a1131 --- /dev/null +++ b/examples/exp_configs/templates/sumo/i210_with_ghost_cell.xml @@ -0,0 +1,5719 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml b/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml new file mode 100644 index 000000000..ee508b730 --- /dev/null +++ b/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml @@ -0,0 +1,5719 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/simulate.py b/examples/simulate.py index 100a27ca2..b8a12f0b3 100644 --- a/examples/simulate.py +++ b/examples/simulate.py @@ -5,8 +5,15 @@ """ import argparse import sys +import json +import os from flow.core.experiment import Experiment +from flow.core.params import AimsunParams +from flow.utils.rllib import FlowParamsEncoder + +from flow.data_pipeline.data_pipeline import collect_metadata_from_config + def parse_args(args): """Parse training options user can specify in command line. @@ -17,7 +24,6 @@ def parse_args(args): the output parser object """ parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, description="Parse argument used when running a Flow simulation.", epilog="python simulate.py EXP_CONFIG --num_runs INT --no_render") @@ -49,6 +55,17 @@ def parse_args(args): '--libsumo', action='store_true', help='Whether to run with libsumo') + parser.add_argument( + '--to_aws', + type=str, nargs='?', default=None, const="default", + help='Specifies the name of the partition to store the output' + 'file on S3. Putting not None value for this argument' + 'automatically set gen_emission to True.') + parser.add_argument( + '--is_baseline', + action='store_true', + help='specifies whether this is a baseline run' + ) return parser.parse_known_args(args)[0] @@ -58,17 +75,19 @@ def parse_args(args): assert not (flags.libsumo and flags.aimsun), "Cannot enable both libsumo and aimsun!" + flags.gen_emission = flags.gen_emission or flags.to_aws + # Get the flow_params object. module = __import__("exp_configs.non_rl", fromlist=[flags.exp_config]) - flow_params = getattr(module, flags.exp_config).flow_params + config_obj = getattr(module, flags.exp_config) + flow_params = config_obj.flow_params # Get the custom callables for the runner. - if hasattr(getattr(module, flags.exp_config), "custom_callables"): - callables = getattr(module, flags.exp_config).custom_callables - else: - callables = None + callables = getattr(config_obj, "custom_callables", None) + + # load some metadata from the exp_config file + supplied_metadata = collect_metadata_from_config(config_obj) - # Update some variables based on inputs. flow_params['sim'].render = not flags.no_render if flags.libsumo: print("Running with libsumo! Make sure you have it installed!") @@ -77,12 +96,26 @@ def parse_args(args): flow_params['sim'].use_libsumo = flags.libsumo flow_params['simulator'] = 'aimsun' if flags.aimsun else 'traci' + # If Aimsun is being called, replace SumoParams with AimsunParams. + if flags.aimsun: + sim_params = AimsunParams() + sim_params.__dict__.update(flow_params['sim'].__dict__) + flow_params['sim'] = sim_params + # Specify an emission path if they are meant to be generated. if flags.gen_emission: flow_params['sim'].emission_path = "./data" + # Create the flow_params object + fp_ = flow_params['exp_tag'] + dir_ = flow_params['sim'].emission_path + with open(os.path.join(dir_, "{}.json".format(fp_)), 'w') as outfile: + json.dump(flow_params, outfile, + cls=FlowParamsEncoder, sort_keys=True, indent=4) + # Create the experiment object. exp = Experiment(flow_params, callables) # Run for the specified number of rollouts. - exp.run(flags.num_runs, convert_to_csv=flags.gen_emission) + exp.run(flags.num_runs, convert_to_csv=flags.gen_emission, to_aws=flags.to_aws, + is_baseline=flags.is_baseline, supplied_metadata=supplied_metadata) diff --git a/examples/train.py b/examples/train.py index 050351449..45a01fb7e 100644 --- a/examples/train.py +++ b/examples/train.py @@ -6,30 +6,22 @@ Usage python train.py EXP_CONFIG """ - import argparse +from datetime import datetime import json import os import sys from time import strftime - -from stable_baselines.common.vec_env import DummyVecEnv, SubprocVecEnv -from stable_baselines import PPO2 - -import ray -from ray import tune -from ray.tune import run_experiments -from ray.tune.registry import register_env -from flow.utils.registry import make_create_env -try: - from ray.rllib.agents.agent import get_agent_class -except ImportError: - from ray.rllib.agents.registry import get_agent_class from copy import deepcopy +import numpy as np +import pytz from flow.core.util import ensure_dir +from flow.core.rewards import instantaneous_mpg from flow.utils.registry import env_constructor from flow.utils.rllib import FlowParamsEncoder, get_flow_params +from flow.utils.registry import make_create_env +from flow.replay.transfer_tests import create_parser, generate_graphs def parse_args(args): @@ -55,31 +47,69 @@ def parse_args(args): parser.add_argument( '--rl_trainer', type=str, default="rllib", help='the RL trainer to use. either rllib or Stable-Baselines') - + parser.add_argument( + '--algorithm', type=str, default="PPO", + help='RL algorithm to use. Options are PPO, TD3, and CENTRALIZEDPPO (which uses a centralized value function)' + ' right now.' + ) parser.add_argument( '--num_cpus', type=int, default=1, help='How many CPUs to use') parser.add_argument( '--num_steps', type=int, default=5000, - help='How many total steps to perform learning over') + help='How many total steps to perform learning over. Relevant for stable-baselines') + parser.add_argument( + '--grid_search', action='store_true', default=False, + help='Whether to grid search over hyperparams') + parser.add_argument( + '--num_iterations', type=int, default=200, + help='How many iterations are in a training run.') + parser.add_argument( + '--checkpoint_freq', type=int, default=20, + help='How often to checkpoint.') + parser.add_argument( + '--num_rollouts', type=int, default=1, + help='How many rollouts are in a training batch') parser.add_argument( '--rollout_size', type=int, default=1000, help='How many steps are in a training batch.') + parser.add_argument('--use_s3', action='store_true', default=False, + help='If true, upload results to s3') + parser.add_argument('--local_mode', action='store_true', default=False, + help='If true only 1 CPU will be used') + parser.add_argument('--render', action='store_true', default=False, + help='If true, we render the display') parser.add_argument( '--checkpoint_path', type=str, default=None, help='Directory with checkpoint to restore training from.') parser.add_argument( '--libsumo', action='store_true', help='Whether to run with libsumo') + parser.add_argument( + '--exp_title', type=str, default=None, + help='Name of experiment that results will be stored in') + parser.add_argument('--multi_node', action='store_true', + help='Set to true if this will be run in cluster mode.' + 'Relevant for rllib') + parser.add_argument( + '--upload_graphs', type=str, nargs=2, + help='Whether to generate and upload graphs to leaderboard at the end of training.' + 'Arguments are name of the submitter and name of the strategy.' + 'Only relevant for i210 training on rllib') return parser.parse_known_args(args)[0] -def run_model_stablebaseline(flow_params, num_cpus=1, rollout_size=50, num_steps=50): +def run_model_stablebaseline(flow_params, + num_cpus=1, + rollout_size=50, + num_steps=50): """Run the model for num_steps if provided. Parameters ---------- + flow_params : dict + flow-specific parameters num_cpus : int number of CPUs used during training rollout_size : int @@ -93,6 +123,8 @@ def run_model_stablebaseline(flow_params, num_cpus=1, rollout_size=50, num_steps stable_baselines.* the trained model """ + from stable_baselines.common.vec_env import DummyVecEnv, SubprocVecEnv + from stable_baselines import PPO2 if num_cpus == 1: constructor = env_constructor(params=flow_params, version=0)() # The algorithms require a vectorized environment to run @@ -109,9 +141,11 @@ def run_model_stablebaseline(flow_params, num_cpus=1, rollout_size=50, num_steps def setup_exps_rllib(flow_params, n_cpus, n_rollouts, + flags, policy_graphs=None, policy_mapping_fn=None, - policies_to_train=None): + policies_to_train=None, + ): """Return the relevant components of an RLlib experiment. Parameters @@ -122,13 +156,14 @@ def setup_exps_rllib(flow_params, number of CPUs to run the experiment over n_rollouts : int number of rollouts per training iteration + flags : TODO + custom arguments policy_graphs : dict, optional TODO policy_mapping_fn : function, optional TODO policies_to_train : list of str, optional TODO - Returns ------- str @@ -138,22 +173,149 @@ def setup_exps_rllib(flow_params, dict training configuration parameters """ - horizon = flow_params['env'].horizon + from ray import tune + from ray.tune.registry import register_env + from ray.rllib.env.group_agents_wrapper import _GroupAgentsWrapper + try: + from ray.rllib.agents.agent import get_agent_class + except ImportError: + from ray.rllib.agents.registry import get_agent_class - alg_run = "PPO" + horizon = flow_params['env'].horizon - agent_cls = get_agent_class(alg_run) - config = deepcopy(agent_cls._default_config) + alg_run = flags.algorithm.upper() + + if alg_run == "PPO": + from flow.algorithms.custom_ppo import CustomPPOTrainer + from ray.rllib.agents.ppo import DEFAULT_CONFIG + alg_run = CustomPPOTrainer + config = deepcopy(DEFAULT_CONFIG) + + config["num_workers"] = n_cpus + config["horizon"] = horizon + config["model"].update({"fcnet_hiddens": [32, 32]}) + config["train_batch_size"] = horizon * n_rollouts + config["gamma"] = 0.995 # discount rate + config["use_gae"] = True + config["no_done_at_end"] = True + config["lambda"] = 0.97 + config["kl_target"] = 0.02 + config["num_sgd_iter"] = 10 + if flags.grid_search: + config["lambda"] = tune.grid_search([0.5, 0.9]) + config["lr"] = tune.grid_search([5e-4, 5e-5]) + elif alg_run == "CENTRALIZEDPPO": + from flow.algorithms.centralized_PPO import CCTrainer, CentralizedCriticModel + from ray.rllib.agents.ppo import DEFAULT_CONFIG + from ray.rllib.models import ModelCatalog + alg_run = CCTrainer + config = deepcopy(DEFAULT_CONFIG) + config['model']['custom_model'] = "cc_model" + config["model"]["custom_options"]["max_num_agents"] = flow_params['env'].additional_params['max_num_agents'] + config["model"]["custom_options"]["central_vf_size"] = 100 + + ModelCatalog.register_custom_model("cc_model", CentralizedCriticModel) + + config["num_workers"] = n_cpus + config["horizon"] = horizon + config["model"].update({"fcnet_hiddens": [32, 32]}) + config["train_batch_size"] = horizon * n_rollouts + config["gamma"] = 0.995 # discount rate + config["use_gae"] = True + config["lambda"] = 0.97 + config["kl_target"] = 0.02 + config["num_sgd_iter"] = 10 + if flags.grid_search: + config["lambda"] = tune.grid_search([0.5, 0.9]) + config["lr"] = tune.grid_search([5e-4, 5e-5]) + + elif alg_run == "TD3": + alg_run = get_agent_class(alg_run) + config = deepcopy(alg_run._default_config) + + config["num_workers"] = n_cpus + config["horizon"] = horizon + config["learning_starts"] = 10000 + config["buffer_size"] = 20000 # reduced to test if this is the source of memory problems + if flags.grid_search: + config["prioritized_replay"] = tune.grid_search(['True', 'False']) + config["actor_lr"] = tune.grid_search([1e-3, 1e-4]) + config["critic_lr"] = tune.grid_search([1e-3, 1e-4]) + config["n_step"] = tune.grid_search([1, 10]) - config["num_workers"] = n_cpus - config["train_batch_size"] = horizon * n_rollouts - config["gamma"] = 0.999 # discount rate - config["model"].update({"fcnet_hiddens": [32, 32, 32]}) - config["use_gae"] = True - config["lambda"] = 0.97 - config["kl_target"] = 0.02 - config["num_sgd_iter"] = 10 - config["horizon"] = horizon + else: + sys.exit("We only support PPO, TD3, right now.") + + # define some standard and useful callbacks + def on_episode_start(info): + episode = info["episode"] + episode.user_data["avg_speed"] = [] + episode.user_data["avg_speed_avs"] = [] + episode.user_data["avg_energy"] = [] + episode.user_data["inst_mpg"] = [] + episode.user_data["num_cars"] = [] + episode.user_data["avg_accel_human"] = [] + episode.user_data["avg_accel_avs"] = [] + + def on_episode_step(info): + episode = info["episode"] + env = info["env"].get_unwrapped()[0] + if isinstance(env, _GroupAgentsWrapper): + env = env.env + if hasattr(env, 'no_control_edges'): + veh_ids = [ + veh_id for veh_id in env.k.vehicle.get_ids() + if env.k.vehicle.get_speed(veh_id) >= 0 + and env.k.vehicle.get_edge(veh_id) not in env.no_control_edges + ] + rl_ids = [ + veh_id for veh_id in env.k.vehicle.get_rl_ids() + if env.k.vehicle.get_speed(veh_id) >= 0 + and env.k.vehicle.get_edge(veh_id) not in env.no_control_edges + ] + else: + veh_ids = [veh_id for veh_id in env.k.vehicle.get_ids() if env.k.vehicle.get_speed(veh_id) >= 0] + rl_ids = [veh_id for veh_id in env.k.vehicle.get_rl_ids() if env.k.vehicle.get_speed(veh_id) >= 0] + + speed = np.mean([speed for speed in env.k.vehicle.get_speed(veh_ids)]) + if not np.isnan(speed): + episode.user_data["avg_speed"].append(speed) + av_speed = np.mean([speed for speed in env.k.vehicle.get_speed(rl_ids) if speed >= 0]) + if not np.isnan(av_speed): + episode.user_data["avg_speed_avs"].append(av_speed) + episode.user_data["inst_mpg"].append(instantaneous_mpg(env, veh_ids, gain=1.0)) + episode.user_data["num_cars"].append(len(env.k.vehicle.get_ids())) + episode.user_data["avg_accel_human"].append(np.nan_to_num(np.mean( + [np.abs((env.k.vehicle.get_speed(veh_id) - env.k.vehicle.get_previous_speed(veh_id))/env.sim_step) for + veh_id in veh_ids if veh_id in env.k.vehicle.previous_speeds.keys()] + ))) + episode.user_data["avg_accel_avs"].append(np.nan_to_num(np.mean( + [np.abs((env.k.vehicle.get_speed(veh_id) - env.k.vehicle.get_previous_speed(veh_id))/env.sim_step) for + veh_id in rl_ids if veh_id in env.k.vehicle.previous_speeds.keys()] + ))) + + def on_episode_end(info): + episode = info["episode"] + avg_speed = np.mean(episode.user_data["avg_speed"]) + episode.custom_metrics["avg_speed"] = avg_speed + avg_speed_avs = np.mean(episode.user_data["avg_speed_avs"]) + episode.custom_metrics["avg_speed_avs"] = avg_speed_avs + episode.custom_metrics["avg_accel_avs"] = np.mean(episode.user_data["avg_accel_avs"]) + episode.custom_metrics["avg_energy_per_veh"] = np.mean(episode.user_data["avg_energy"]) + episode.custom_metrics["avg_mpg_per_veh"] = np.mean(episode.user_data["inst_mpg"]) + episode.custom_metrics["num_cars"] = np.mean(episode.user_data["num_cars"]) + + def on_train_result(info): + """Store the mean score of the episode, and increment or decrement the iteration number for curriculum.""" + trainer = info["trainer"] + trainer.workers.foreach_worker( + lambda ev: ev.foreach_env( + lambda env: env.set_iteration_num())) + + config["callbacks"] = {"on_episode_start": tune.function(on_episode_start), + "on_episode_step": tune.function(on_episode_step), + "on_episode_end": tune.function(on_episode_end), + "on_train_result": tune.function(on_train_result)} # save the flow params for replay flow_json = json.dumps( @@ -163,7 +325,6 @@ def setup_exps_rllib(flow_params, # multiagent configuration if policy_graphs is not None: - print("policy_graphs", policy_graphs) config['multiagent'].update({'policies': policy_graphs}) if policy_mapping_fn is not None: config['multiagent'].update({'policy_mapping_fn': tune.function(policy_mapping_fn)}) @@ -172,24 +333,245 @@ def setup_exps_rllib(flow_params, create_env, gym_name = make_create_env(params=flow_params) - # Register as rllib env register_env(gym_name, create_env) return alg_run, gym_name, config -if __name__ == "__main__": - flags = parse_args(sys.argv[1:]) +def train_rllib(submodule, flags): + """Train policies using the PPO algorithm in RLlib.""" + import ray + from ray import tune - # import relevant information from the exp_config script - module = __import__("exp_configs.rl.singleagent", fromlist=[flags.exp_config]) - module_ma = __import__("exp_configs.rl.multiagent", fromlist=[flags.exp_config]) + flow_params = submodule.flow_params + flow_params['sim'].render = flags.render + policy_graphs = getattr(submodule, "POLICY_GRAPHS", None) + policy_mapping_fn = getattr(submodule, "policy_mapping_fn", None) + policies_to_train = getattr(submodule, "policies_to_train", None) + + alg_run, gym_name, config = setup_exps_rllib( + flow_params, flags.num_cpus, flags.num_rollouts, flags, + policy_graphs, policy_mapping_fn, policies_to_train) + + config['num_workers'] = flags.num_cpus + config['env'] = gym_name + + # create a custom string that makes looking at the experiment names easier + def trial_str_creator(trial): + return "{}_{}".format(trial.trainable_name, trial.experiment_tag) + + if flags.multi_node: + ray.init(redis_address='localhost:6379') + elif flags.local_mode: + ray.init(local_mode=True) + else: + ray.init(num_cpus=flags.num_cpus + 1) + + exp_dict = { + "run_or_experiment": alg_run, + "name": flags.exp_title or flow_params['exp_tag'], + "config": config, + "checkpoint_freq": flags.checkpoint_freq, + "checkpoint_at_end": True, + 'trial_name_creator': trial_str_creator, + "max_failures": 0, + "stop": { + "training_iteration": flags.num_iterations, + }, + } + date = datetime.now(tz=pytz.utc) + date = date.astimezone(pytz.timezone('US/Pacific')).strftime("%m-%d-%Y") + if flags.use_s3: + s3_string = "s3://i210.experiments/i210/" \ + + date + '/' + flags.exp_title + exp_dict['upload_dir'] = s3_string + tune.run(**exp_dict, queue_trials=False, raise_on_failed_trial=False) + + if flags.upload_graphs: + print('Generating experiment graphs and uploading them to leaderboard') + submitter_name, strategy_name = flags.upload_graphs + + # reset ray + ray.shutdown() + if flags.local_mode: + ray.init(local_mode=True) + else: + ray.init() + + # grab checkpoint path + for (dirpath, _, _) in os.walk(os.path.expanduser("~/ray_results")): + if "checkpoint_{}".format(flags.checkpoint_freq) in dirpath \ + and dirpath.split('/')[-3] == flags.exp_title: + checkpoint_path = os.path.dirname(dirpath) + checkpoint_number = -1 + for name in os.listdir(checkpoint_path): + if name.startswith('checkpoint'): + cp = int(name.split('_')[1]) + checkpoint_number = max(checkpoint_number, cp) + + # create dir for graphs output + output_dir = os.path.join(checkpoint_path, 'output_graphs') + if not os.path.exists(output_dir): + os.mkdir(output_dir) + + # run graph generation script + parser = create_parser() + + strategy_name_full = str(strategy_name) + if flags.grid_search: + strategy_name_full += '__' + dirpath.split('/')[-2] + + args = parser.parse_args([ + '-r', checkpoint_path, '-c', str(checkpoint_number), + '--gen_emission', '--use_s3', '--num_cpus', str(flags.num_cpus), + '--output_dir', output_dir, + '--submitter_name', submitter_name, + '--strategy_name', strategy_name_full.replace(',', '_').replace(';', '_') + ]) + generate_graphs(args) + + +def train_h_baselines(env_name, args, multiagent): + """Train policies using SAC and TD3 with h-baselines.""" + from hbaselines.algorithms import OffPolicyRLAlgorithm + from hbaselines.utils.train import parse_options, get_hyperparameters + + # Get the command-line arguments that are relevant here + args = parse_options(description="", example_usage="", args=args) + + # the base directory that the logged data will be stored in + base_dir = "training_data" + + for i in range(args.n_training): + # value of the next seed + seed = args.seed + i + + # The time when the current experiment started. + now = strftime("%Y-%m-%d-%H:%M:%S") + + # Create a save directory folder (if it doesn't exist). + dir_name = os.path.join(base_dir, '{}/{}'.format(args.env_name, now)) + ensure_dir(dir_name) + + # Get the policy class. + if args.alg == "TD3": + if multiagent: + from hbaselines.multi_fcnet.td3 import MultiFeedForwardPolicy + policy = MultiFeedForwardPolicy + else: + from hbaselines.fcnet.td3 import FeedForwardPolicy + policy = FeedForwardPolicy + elif args.alg == "SAC": + if multiagent: + from hbaselines.multi_fcnet.sac import MultiFeedForwardPolicy + policy = MultiFeedForwardPolicy + else: + from hbaselines.fcnet.sac import FeedForwardPolicy + policy = FeedForwardPolicy + else: + raise ValueError("Unknown algorithm: {}".format(args.alg)) + + # Get the hyperparameters. + hp = get_hyperparameters(args, policy) + + # Add the seed for logging purposes. + params_with_extra = hp.copy() + params_with_extra['seed'] = seed + params_with_extra['env_name'] = args.env_name + params_with_extra['policy_name'] = policy.__name__ + params_with_extra['algorithm'] = args.alg + params_with_extra['date/time'] = now + + # Add the hyperparameters to the folder. + with open(os.path.join(dir_name, 'hyperparameters.json'), 'w') as f: + json.dump(params_with_extra, f, sort_keys=True, indent=4) + + # Create the algorithm object. + alg = OffPolicyRLAlgorithm( + policy=policy, + env="flow:{}".format(env_name), + eval_env="flow:{}".format(env_name) if args.evaluate else None, + **hp + ) + + # Perform training. + alg.learn( + total_steps=args.total_steps, + log_dir=dir_name, + log_interval=args.log_interval, + eval_interval=args.eval_interval, + save_interval=args.save_interval, + initial_exploration_steps=args.initial_exploration_steps, + seed=seed, + ) + + +def train_stable_baselines(submodule, flags): + """Train policies using the PPO algorithm in stable-baselines.""" + from stable_baselines.common.vec_env import DummyVecEnv + from stable_baselines import PPO2 + flow_params = submodule.flow_params + # Path to the saved files + exp_tag = flow_params['exp_tag'] + result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S")) + + # Perform training. + print('Beginning training.') + model = run_model_stablebaseline( + flow_params, flags.num_cpus, flags.rollout_size, flags.num_steps) + + # Save the model to a desired folder and then delete it to demonstrate + # loading. + print('Saving the trained model!') + path = os.path.realpath(os.path.expanduser('~/baseline_results')) + ensure_dir(path) + save_path = os.path.join(path, result_name) + model.save(save_path) + + # dump the flow params + with open(os.path.join(path, result_name) + '.json', 'w') as outfile: + json.dump(flow_params, outfile, + cls=FlowParamsEncoder, sort_keys=True, indent=4) + + # Replay the result by loading the model + print('Loading the trained model and testing it out!') + model = PPO2.load(save_path) + flow_params = get_flow_params(os.path.join(path, result_name) + '.json') + flow_params['sim'].render = True + env = env_constructor(params=flow_params, version=0)() + # The algorithms require a vectorized environment to run + eval_env = DummyVecEnv([lambda: env]) + obs = eval_env.reset() + reward = 0 + for _ in range(flow_params['env'].horizon): + action, _states = model.predict(obs) + obs, rewards, dones, info = eval_env.step(action) + reward += rewards + print('the final reward is {}'.format(reward)) + + +def main(args): + """Perform the training operations.""" + # Parse script-level arguments (not including package arguments). + flags = parse_args(args) + + # Import relevant information from the exp_config script. + module = __import__( + "exp_configs.rl.singleagent", fromlist=[flags.exp_config]) + module_ma = __import__( + "exp_configs.rl.multiagent", fromlist=[flags.exp_config]) + + # Import the sub-module containing the specified exp_config and determine + # whether the environment is single agent or multi-agent. if hasattr(module, flags.exp_config): submodule = getattr(module, flags.exp_config) + multiagent = False elif hasattr(module_ma, flags.exp_config): submodule = getattr(module_ma, flags.exp_config) - assert flags.rl_trainer.lower() == "RLlib".lower(), \ + assert flags.rl_trainer.lower() in ["rllib", "h-baselines"], \ "Currently, multiagent experiments are only supported through "\ - "RLlib. Try running this experiment using RLlib: 'python train.py EXP_CONFIG'" + "RLlib. Try running this experiment using RLlib: " \ + "'python train.py EXP_CONFIG'" + multiagent = True else: assert False, "Unable to find experiment config!" @@ -200,72 +582,17 @@ def setup_exps_rllib(flow_params, assert libsumo.isLibsumo(), "Failed to load libsumo" flow_params['sim'].use_libsumo = flags.libsumo + # Perform the training operation. if flags.rl_trainer.lower() == "rllib": - n_cpus = submodule.N_CPUS - n_rollouts = submodule.N_ROLLOUTS - policy_graphs = getattr(submodule, "POLICY_GRAPHS", None) - policy_mapping_fn = getattr(submodule, "policy_mapping_fn", None) - policies_to_train = getattr(submodule, "policies_to_train", None) - - alg_run, gym_name, config = setup_exps_rllib( - flow_params, n_cpus, n_rollouts, - policy_graphs, policy_mapping_fn, policies_to_train) - - ray.init(num_cpus=n_cpus + 1, object_store_memory=200 * 1024 * 1024) - exp_config = { - "run": alg_run, - "env": gym_name, - "config": { - **config - }, - "checkpoint_freq": 20, - "checkpoint_at_end": True, - "max_failures": 999, - "stop": { - "training_iteration": flags.num_steps, - }, - } - - if flags.checkpoint_path is not None: - exp_config['restore'] = flags.checkpoint_path - trials = run_experiments({flow_params["exp_tag"]: exp_config}) - - elif flags.rl_trainer == "Stable-Baselines": - # Path to the saved files - exp_tag = flow_params['exp_tag'] - result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S")) - - # Perform training. - print('Beginning training.') - model = run_model_stablebaseline(flow_params, flags.num_cpus, flags.rollout_size, flags.num_steps) - - # Save the model to a desired folder and then delete it to demonstrate - # loading. - print('Saving the trained model!') - path = os.path.realpath(os.path.expanduser('~/baseline_results')) - ensure_dir(path) - save_path = os.path.join(path, result_name) - model.save(save_path) - - # dump the flow params - with open(os.path.join(path, result_name) + '.json', 'w') as outfile: - json.dump(flow_params, outfile, - cls=FlowParamsEncoder, sort_keys=True, indent=4) - - # Replay the result by loading the model - print('Loading the trained model and testing it out!') - model = PPO2.load(save_path) - flow_params = get_flow_params(os.path.join(path, result_name) + '.json') - flow_params['sim'].render = True - env_constructor = env_constructor(params=flow_params, version=0)() - # The algorithms require a vectorized environment to run - eval_env = DummyVecEnv([lambda: env_constructor]) - obs = eval_env.reset() - reward = 0 - for _ in range(flow_params['env'].horizon): - action, _states = model.predict(obs) - obs, rewards, dones, info = eval_env.step(action) - reward += rewards - print('the final reward is {}'.format(reward)) + train_rllib(submodule, flags) + elif flags.rl_trainer.lower() == "stable-baselines": + train_stable_baselines(submodule, flags) + elif flags.rl_trainer.lower() == "h-baselines": + train_h_baselines(flags.exp_config, args, multiagent) else: - assert False, "rl_trainer should be either 'RLlib' or 'Stable-Baselines'!" + raise ValueError("rl_trainer should be either 'rllib', 'h-baselines', " + "or 'stable-baselines'.") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/flow/algorithms/centralized_PPO.py b/flow/algorithms/centralized_PPO.py new file mode 100644 index 000000000..133d7c8bf --- /dev/null +++ b/flow/algorithms/centralized_PPO.py @@ -0,0 +1,553 @@ +"""An example of customizing PPO to leverage a centralized critic.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np + +from ray.rllib.agents.ppo.ppo import PPOTrainer +from flow.algorithms.custom_ppo import CustomPPOTFPolicy, KLCoeffMixin +from ray.rllib.evaluation.postprocessing import compute_advantages, \ + Postprocessing +from ray.rllib.policy.sample_batch import SampleBatch +from ray.rllib.policy.tf_policy import LearningRateSchedule, \ + EntropyCoeffSchedule, ACTION_LOGP +from ray.rllib.models.modelv2 import ModelV2 +from ray.rllib.models.tf.tf_modelv2 import TFModelV2 +from ray.rllib.models.tf.recurrent_tf_modelv2 import RecurrentTFModelV2 +from ray.rllib.utils.annotations import override +from ray.rllib.models.tf.fcnet_v2 import FullyConnectedNetwork +from ray.rllib.utils.explained_variance import explained_variance +from ray.rllib.utils import try_import_tf + +tf = try_import_tf() + +# Frozen logits of the policy that computed the action +BEHAVIOUR_LOGITS = "behaviour_logits" + +CENTRAL_OBS = "central_obs" +OPPONENT_ACTION = "opponent_action" + +parser = argparse.ArgumentParser() +parser.add_argument("--stop", type=int, default=100000) + + +class CentralizedCriticModel(TFModelV2): + """Multi-agent model that implements a centralized VF.""" + + # TODO(@evinitsky) make this work with more than boxes + + def __init__(self, obs_space, action_space, num_outputs, model_config, + name): + super(CentralizedCriticModel, self).__init__( + obs_space, action_space, num_outputs, model_config, name) + # Base of the model + self.model = FullyConnectedNetwork(obs_space, action_space, + num_outputs, model_config, name) + self.register_variables(self.model.variables()) + + # Central VF maps (obs, opp_ops, opp_act) -> vf_pred + self.max_num_agents = model_config['custom_options']['max_num_agents'] + self.obs_space_shape = obs_space.shape[0] + self.obs_space = obs_space + other_obs = tf.keras.layers.Input( + shape=(obs_space.shape[0] * self.max_num_agents,), + name="central_obs") + central_vf_dense = tf.keras.layers.Dense( + model_config['custom_options']['central_vf_size'], + activation=tf.nn.tanh, name="c_vf_dense")(other_obs) + central_vf_out = tf.keras.layers.Dense( + 1, activation=None, name="c_vf_out")(central_vf_dense) + self.central_vf = tf.keras.Model( + inputs=[other_obs], outputs=central_vf_out) + self.register_variables(self.central_vf.variables) + + def forward(self, input_dict, state, seq_lens): + """Run forward inference.""" + return self.model.forward(input_dict, state, seq_lens) + + def central_value_function(self, central_obs): + """Compute the centralized value function.""" + return tf.reshape( + self.central_vf( + [central_obs]), [-1]) + + def value_function(self): + """Compute the normal value function; this is only here to make the code run.""" + return self.model.value_function() # not used + + +# TODO(@evinitsky) support recurrence +class CentralizedCriticModelRNN(RecurrentTFModelV2): + """Example of using the Keras functional API to define a RNN model.""" + + def __init__(self, + obs_space, + action_space, + num_outputs, + model_config, + name, + hiddens_size=64, + cell_size=64): + super(CentralizedCriticModelRNN, self).__init__( + obs_space, action_space, num_outputs, model_config, name) + self.cell_size = cell_size + + # Define input layers + input_layer = tf.keras.layers.Input( + shape=(None, obs_space.shape[0]), name="inputs") + state_in_h = tf.keras.layers.Input(shape=(cell_size,), name="h") + state_in_c = tf.keras.layers.Input(shape=(cell_size,), name="c") + seq_in = tf.keras.layers.Input(shape=(), name="seq_in") + + # Preprocess observation with a hidden layer and send to LSTM cell + dense1 = tf.keras.layers.Dense( + hiddens_size, activation=tf.nn.relu, name="dense1")(input_layer) + lstm_out, state_h, state_c = tf.keras.layers.LSTM( + cell_size, return_sequences=True, return_state=True, name="lstm")( + inputs=dense1, + mask=tf.sequence_mask(seq_in), + initial_state=[state_in_h, state_in_c]) + + # Postprocess LSTM output with another hidden layer and compute values + logits = tf.keras.layers.Dense( + self.num_outputs, + activation=tf.keras.activations.linear, + name="logits")(lstm_out) + values = tf.keras.layers.Dense( + 1, activation=None, name="values")(lstm_out) + + # Create the RNN model + self.model = tf.keras.Model( + inputs=[input_layer, seq_in, state_in_h, state_in_c], + outputs=[logits, values, state_h, state_c]) + self.register_variables(self.model.variables) + self.model.summary() + + # TODO(@evinitsky) add layer sharing to the VF + # Create the centralized VF + # Central VF maps (obs, opp_ops, opp_act) -> vf_pred + self.max_num_agents = model_config.get("max_num_agents", 120) + self.obs_space_shape = obs_space.shape[0] + other_obs = tf.keras.layers.Input( + shape=(obs_space.shape[0] * self.max_num_agents,), + name="all_agent_obs") + central_vf_dense = tf.keras.layers.Dense( + model_config.get("central_vf_size", 64), activation=tf.nn.tanh, + name="c_vf_dense")(other_obs) + central_vf_dense2 = tf.keras.layers.Dense( + model_config.get("central_vf_size", 64), activation=tf.nn.tanh, + name="c_vf_dense")(central_vf_dense) + central_vf_out = tf.keras.layers.Dense( + 1, activation=None, name="c_vf_out")(central_vf_dense2) + self.central_vf = tf.keras.Model( + inputs=[other_obs], outputs=central_vf_out) + self.register_variables(self.central_vf.variables) + + @override(RecurrentTFModelV2) + def forward_rnn(self, inputs, state, seq_lens): + """Forward inference on the RNN.""" + model_out, self._value_out, h, c = self.model( + [inputs, seq_lens] + state) + return model_out, [h, c] + + @override(ModelV2) + def get_initial_state(self): + """Set up the initial RNN state.""" + return [ + np.zeros(self.cell_size, np.float32), + np.zeros(self.cell_size, np.float32), + ] + + def central_value_function(self, central_obs): + """Compute the central value function.""" + return tf.reshape( + self.central_vf( + [central_obs]), [-1]) + + def value_function(self): + """Compute the normal value function; this is only here to make the code run.""" + return tf.reshape(self._value_out, [-1]) # not used + + +class CentralizedValueMixin(object): + """Add methods to evaluate the central value function from the model.""" + + def __init__(self): + # TODO(@evinitsky) clean up naming + self.central_value_function = self.model.central_value_function( + self.get_placeholder(CENTRAL_OBS) + ) + + def compute_central_vf(self, central_obs): + """Run forward inference on the model.""" + feed_dict = { + self.get_placeholder(CENTRAL_OBS): central_obs, + } + return self.get_session().run(self.central_value_function, feed_dict) + + +def centralized_critic_postprocessing(policy, + sample_batch, + other_agent_batches=None, + episode=None): + """Find all other agents that overlapped with you and stack their obs to be passed to the central VF.""" + if policy.loss_initialized(): + assert other_agent_batches is not None + + # time_span = (sample_batch['t'][0], sample_batch['t'][-1]) + # # there's a new problem here, namely that a segment might not be continuous due to the rerouting + # other_agent_timespans = {agent_id: + # (other_agent_batches[agent_id][1]["t"][0], + # other_agent_batches[agent_id][1]["t"][-1]) + # for agent_id in other_agent_batches.keys()} + other_agent_times = {agent_id: other_agent_batches[agent_id][1]["t"] + for agent_id in other_agent_batches.keys()} + agent_time = sample_batch['t'] + # # find agents whose time overlaps with the current agent + rel_agents = {agent_id: other_agent_time for agent_id, other_agent_time + in other_agent_times.items()} + # if len(rel_agents) > 0: + other_obs = { + agent_id: other_agent_batches[agent_id][1]["obs"].copy() + for agent_id in other_agent_batches.keys() + } + padded_agent_obs = { + agent_id: fill_missing( + agent_time, + other_agent_times[agent_id], + other_obs[agent_id] + ) + for agent_id, rel_agent_time in rel_agents.items() + } + # okay, now we need to stack and sort + central_obs_list = [padded_obs for padded_obs in + padded_agent_obs.values()] + try: + central_obs_batch = np.hstack( + (sample_batch["obs"], np.hstack(central_obs_list))) + except Exception as e: + print("Error in centralized PPO: ", e) + # TODO(@ev) this is a bug and needs to be fixed + central_obs_batch = sample_batch["obs"] + max_vf_agents = policy.model.max_num_agents + num_agents = len(rel_agents) + 1 + if num_agents < max_vf_agents: + diff = max_vf_agents - num_agents + zero_pad = np.zeros((central_obs_batch.shape[0], + policy.model.obs_space_shape * diff)) + central_obs_batch = np.hstack((central_obs_batch, + zero_pad)) + elif num_agents > max_vf_agents: + print("Too many agents!") + + # also record the opponent obs and actions in the trajectory + sample_batch[CENTRAL_OBS] = central_obs_batch + + # overwrite default VF prediction with the central VF + sample_batch[SampleBatch.VF_PREDS] = policy.compute_central_vf( + sample_batch[CENTRAL_OBS]) + else: + # policy hasn't initialized yet, use zeros + # TODO(evinitsky) put in the right shape + obs_shape = sample_batch[SampleBatch.CUR_OBS].shape[1] + obs_shape = (1, obs_shape * (policy.model.max_num_agents)) + sample_batch[CENTRAL_OBS] = np.zeros(obs_shape) + # TODO(evinitsky) put in the right shape. Will break if actions aren't 1 + sample_batch[SampleBatch.VF_PREDS] = np.zeros(1, dtype=np.float32) + + # TODO (ak): this was not being used, so commented + # completed = sample_batch["dones"][-1] + + # if not completed and policy.loss_initialized(): + # last_r = 0.0 + # else: + # next_state = [] + # for i in range(policy.num_state_tensors()): + # next_state.append([sample_batch["state_out_{}".format(i)][-1]]) + # last_r = policy.compute_central_vf(sample_batch[CENTRAL_OBS][-1][np.newaxis, ...])[0] + + batch = compute_advantages( + sample_batch, + 0.0, + policy.config["gamma"], + policy.config["lambda"], + use_gae=policy.config["use_gae"]) + return batch + + +def time_overlap(time_span, agent_time): + """Check if agent_time overlaps with time_span.""" + if agent_time[0] <= time_span[1] and agent_time[1] >= time_span[0]: + return True + else: + return False + + +def fill_missing(agent_time, other_agent_time, obs): + """Pad the obs to the appropriate length for agents that don't overlap perfectly in time.""" + # shortcut, the two overlap perfectly + if np.sum(agent_time == other_agent_time) == agent_time.shape[0]: + return obs + new_obs = np.zeros((agent_time.shape[0], obs.shape[1])) + other_agent_time_set = set(other_agent_time) + for i, time in enumerate(agent_time): + if time in other_agent_time_set: + new_obs[i] = obs[np.where(other_agent_time == time)] + return new_obs + + +def overlap_and_pad_agent(time_span, agent_time, obs): + """Take the part of obs that overlaps, pad to length time_span. + + Parameters + ---------- + time_span : tuple + tuple of the first and last time that the agent of interest is in the + system + agent_time : tuple + tuple of the first and last time that the agent whose obs we are + padding is in the system + obs : array_like + observations of the agent whose time is agent_time + """ + assert time_overlap(time_span, agent_time) + print(time_span) + print(agent_time) + # FIXME(ev) some of these conditions can be combined + # no padding needed + if agent_time[0] == time_span[0] and agent_time[1] == time_span[1]: + return obs + # agent enters before time_span starts and exits before time_span end + if agent_time[0] < time_span[0] and agent_time[1] < time_span[1]: + non_overlap_time = time_span[0] - agent_time[0] + missing_time = time_span[1] - agent_time[1] + overlap_obs = obs[non_overlap_time:] + padding = np.zeros((missing_time, obs.shape[1])) + obs_concat = np.concatenate((overlap_obs, padding)) + return obs_concat + # agent enters after time_span starts and exits after time_span ends + elif agent_time[0] > time_span[0] and agent_time[1] > time_span[1]: + non_overlap_time = agent_time[1] - time_span[1] + overlap_obs = obs[:-non_overlap_time] + missing_time = agent_time[0] - time_span[0] + padding = np.zeros((missing_time, obs.shape[1])) + obs_concat = np.concatenate((padding, overlap_obs)) + return obs_concat + # agent time is entirely contained in time_span + elif agent_time[0] >= time_span[0] and agent_time[1] <= time_span[1]: + missing_left = agent_time[0] - time_span[0] + missing_right = time_span[1] - agent_time[1] + obs_concat = obs + if missing_left > 0: + padding = np.zeros((missing_left, obs.shape[1])) + obs_concat = np.concatenate((padding, obs_concat)) + if missing_right > 0: + padding = np.zeros((missing_right, obs.shape[1])) + obs_concat = np.concatenate((obs_concat, padding)) + return obs_concat + # agent time totally contains time_span + elif agent_time[0] <= time_span[0] and agent_time[1] >= time_span[1]: + non_overlap_left = time_span[0] - agent_time[0] + non_overlap_right = agent_time[1] - time_span[1] + overlap_obs = obs + if non_overlap_left > 0: + overlap_obs = overlap_obs[non_overlap_left:] + if non_overlap_right > 0: + overlap_obs = overlap_obs[:-non_overlap_right] + return overlap_obs + + +def loss_with_central_critic(policy, model, dist_class, train_batch): + """Set up the PPO loss but replace the VF loss with the centralized VF loss.""" + CentralizedValueMixin.__init__(policy) + + logits, state = model.from_batch(train_batch) + action_dist = dist_class(logits, model) + + policy.loss_obj = PPOLoss( + policy.action_space, + dist_class, + model, + train_batch[Postprocessing.VALUE_TARGETS], + train_batch[Postprocessing.ADVANTAGES], + train_batch[SampleBatch.ACTIONS], + train_batch[BEHAVIOUR_LOGITS], + train_batch[ACTION_LOGP], + train_batch[SampleBatch.VF_PREDS], + action_dist, + policy.central_value_function, + policy.kl_coeff, + tf.ones_like(train_batch[Postprocessing.ADVANTAGES], dtype=tf.bool), + entropy_coeff=policy.entropy_coeff, + clip_param=policy.config["clip_param"], + vf_clip_param=policy.config["vf_clip_param"], + vf_loss_coeff=policy.config["vf_loss_coeff"], + use_gae=policy.config["use_gae"], + model_config=policy.config["model"]) + + return policy.loss_obj.loss + + +class PPOLoss(object): + """Object containing the PPO loss function.""" + + def __init__(self, + action_space, + dist_class, + model, + value_targets, + advantages, + actions, + prev_logits, + prev_actions_logp, + vf_preds, + curr_action_dist, + value_fn, + cur_kl_coeff, + valid_mask, + entropy_coeff=0, + clip_param=0.1, + vf_clip_param=0.1, + vf_loss_coeff=1.0, + use_gae=True, + model_config=None): + """Construct the loss for Proximal Policy Objective. + + Parameters + ---------- + action_space : TODO + Environment observation space specification. + dist_class : TODO + action distribution class for logits. + value_targets : tf.placeholder + Placeholder for target values; used for GAE. + actions : tf.placeholder + Placeholder for actions taken from previous model evaluation. + advantages : tf.placeholder + Placeholder for calculated advantages from previous model + evaluation. + prev_logits : tf.placeholder + Placeholder for logits output from previous model evaluation. + prev_actions_logp : tf.placeholder + Placeholder for prob output from previous model evaluation. + vf_preds : tf.placeholder + Placeholder for value function output from previous model + evaluation. + curr_action_dist : ActionDistribution + ActionDistribution of the current model. + value_fn : tf.Tensor + Current value function output Tensor. + cur_kl_coeff : tf.Variable + Variable holding the current PPO KL coefficient. + valid_mask : tf.Tensor + A bool mask of valid input elements (#2992). + entropy_coeff : float + Coefficient of the entropy regularizer. + clip_param : float + Clip parameter + vf_clip_param : float + Clip parameter for the value function + vf_loss_coeff : float + Coefficient of the value function loss + use_gae : bool + If true, use the Generalized Advantage Estimator. + model_config : dict, optional + model config for use in specifying action distributions. + """ + + def reduce_mean_valid(t): + return tf.reduce_mean(tf.boolean_mask(t, valid_mask)) + + prev_dist = dist_class(prev_logits, model) + # Make loss functions. + logp_ratio = tf.exp(curr_action_dist.logp(actions) - prev_actions_logp) + action_kl = prev_dist.kl(curr_action_dist) + self.mean_kl = reduce_mean_valid(action_kl) + + curr_entropy = curr_action_dist.entropy() + self.mean_entropy = reduce_mean_valid(curr_entropy) + + surrogate_loss = tf.minimum( + advantages * logp_ratio, + advantages * tf.clip_by_value(logp_ratio, 1 - clip_param, + 1 + clip_param)) + self.mean_policy_loss = reduce_mean_valid(-surrogate_loss) + + if use_gae: + vf_loss1 = tf.square(value_fn - value_targets) + vf_clipped = vf_preds + tf.clip_by_value( + value_fn - vf_preds, -vf_clip_param, vf_clip_param) + vf_loss2 = tf.square(vf_clipped - value_targets) + vf_loss = tf.maximum(vf_loss1, vf_loss2) + self.mean_vf_loss = reduce_mean_valid(vf_loss) + loss = reduce_mean_valid( + -surrogate_loss + + vf_loss_coeff * vf_loss - entropy_coeff * curr_entropy) + else: + self.mean_vf_loss = tf.constant(0.0) + loss = reduce_mean_valid(-surrogate_loss - + entropy_coeff * curr_entropy) + self.loss = loss + + +def new_ppo_surrogate_loss(policy, model, dist_class, train_batch): + """Return the PPO loss with the centralized value function.""" + loss = loss_with_central_critic(policy, model, dist_class, train_batch) + return loss + + +def setup_mixins(policy, obs_space, action_space, config): + """Construct additional classes that add on to PPO.""" + KLCoeffMixin.__init__(policy, config) + + EntropyCoeffSchedule.__init__(policy, config["entropy_coeff"], + config["entropy_coeff_schedule"]) + LearningRateSchedule.__init__(policy, config["lr"], config["lr_schedule"]) + # hack: put in a noop VF so some of the inherited PPO code runs + policy.value_function = tf.zeros( + tf.shape(policy.get_placeholder(SampleBatch.CUR_OBS))[0]) + + +def central_vf_stats(policy, train_batch, grads): + """Report the explained variance of the centralized value function.""" + return { + "vf_explained_var": explained_variance( + train_batch[Postprocessing.VALUE_TARGETS], + policy.central_value_function), + } + + +def kl_and_loss_stats(policy, train_batch): + """Trianing stats to pass to the tensorboard.""" + return { + "cur_kl_coeff": tf.cast(policy.kl_coeff, tf.float64), + "cur_lr": tf.cast(policy.cur_lr, tf.float64), + "total_loss": policy.loss_obj.loss, + "policy_loss": policy.loss_obj.mean_policy_loss, + "vf_loss": policy.loss_obj.mean_vf_loss, + "vf_explained_var": explained_variance( + train_batch[Postprocessing.VALUE_TARGETS], + policy.model.value_function()), + "vf_preds": train_batch[Postprocessing.VALUE_TARGETS], + "kl": policy.loss_obj.mean_kl, + "entropy": policy.loss_obj.mean_entropy, + "entropy_coeff": tf.cast(policy.entropy_coeff, tf.float64), + } + + +CCPPO = CustomPPOTFPolicy.with_updates( + name="CCPPO", + postprocess_fn=centralized_critic_postprocessing, + loss_fn=new_ppo_surrogate_loss, + stats_fn=kl_and_loss_stats, + before_loss_init=setup_mixins, + grad_stats_fn=central_vf_stats, + mixins=[ + LearningRateSchedule, EntropyCoeffSchedule, + CentralizedValueMixin, KLCoeffMixin + ]) + +CCTrainer = PPOTrainer.with_updates(name="CCPPOTrainer", default_policy=CCPPO) diff --git a/flow/algorithms/custom_ppo.py b/flow/algorithms/custom_ppo.py new file mode 100644 index 000000000..47a4459aa --- /dev/null +++ b/flow/algorithms/custom_ppo.py @@ -0,0 +1,344 @@ +"""PPO but without the adaptive KL term that RLlib added.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import logging + +import ray +from ray.rllib.evaluation.postprocessing import compute_advantages, \ + Postprocessing +from ray.rllib.policy.sample_batch import SampleBatch +from ray.rllib.policy.tf_policy import LearningRateSchedule, \ + EntropyCoeffSchedule, ACTION_LOGP +from ray.rllib.policy.tf_policy_template import build_tf_policy +from ray.rllib.utils.explained_variance import explained_variance +from ray.rllib.utils.tf_ops import make_tf_callable +from ray.rllib.utils import try_import_tf + +from ray.rllib.agents.trainer_template import build_trainer +from ray.rllib.agents.ppo.ppo import choose_policy_optimizer, DEFAULT_CONFIG +from ray.rllib.agents.ppo.ppo import warn_about_bad_reward_scales + +tf = try_import_tf() + +logger = logging.getLogger(__name__) + +# Frozen logits of the policy that computed the action +BEHAVIOUR_LOGITS = "behaviour_logits" + + +class PPOLoss(object): + """PPO Loss object.""" + + def __init__(self, + action_space, + dist_class, + model, + value_targets, + advantages, + actions, + prev_logits, + prev_actions_logp, + vf_preds, + curr_action_dist, + value_fn, + cur_kl_coeff, + valid_mask, + entropy_coeff=0, + clip_param=0.1, + vf_clip_param=0.1, + vf_loss_coeff=1.0, + use_gae=True, + model_config=None): + """Construct the loss for Proximal Policy Objective. + + Parameters + ---------- + action_space : TODO + Environment observation space specification. + dist_class : TODO + action distribution class for logits. + value_targets : tf.placeholder + Placeholder for target values; used for GAE. + actions : tf.placeholder + Placeholder for actions taken from previous model evaluation. + advantages : tf.placeholder + Placeholder for calculated advantages from previous model + evaluation. + prev_logits : tf.placeholder + Placeholder for logits output from previous model evaluation. + prev_actions_logp : tf.placeholder + Placeholder for prob output from previous model evaluation. + vf_preds : tf.placeholder + Placeholder for value function output from previous model + evaluation. + curr_action_dist : ActionDistribution + ActionDistribution of the current model. + value_fn : tf.Tensor + Current value function output Tensor. + cur_kl_coeff : tf.Variable + Variable holding the current PPO KL coefficient. + valid_mask : tf.Tensor + A bool mask of valid input elements (#2992). + entropy_coeff : float + Coefficient of the entropy regularizer. + clip_param : float + Clip parameter + vf_clip_param : float + Clip parameter for the value function + vf_loss_coeff : float + Coefficient of the value function loss + use_gae : bool + If true, use the Generalized Advantage Estimator. + model_config : dict, optional + model config for use in specifying action distributions. + """ + + def reduce_mean_valid(t): + return tf.reduce_mean(tf.boolean_mask(t, valid_mask)) + + prev_dist = dist_class(prev_logits, model) + # Make loss functions. + logp_ratio = tf.exp(curr_action_dist.logp(actions) - prev_actions_logp) + action_kl = prev_dist.kl(curr_action_dist) + self.mean_kl = reduce_mean_valid(action_kl) + + curr_entropy = curr_action_dist.entropy() + self.mean_entropy = reduce_mean_valid(curr_entropy) + + surrogate_loss = tf.minimum( + advantages * logp_ratio, + advantages * tf.clip_by_value(logp_ratio, 1 - clip_param, + 1 + clip_param)) + self.mean_policy_loss = reduce_mean_valid(-surrogate_loss) + + if use_gae: + vf_loss1 = tf.square(value_fn - value_targets) + vf_clipped = vf_preds + tf.clip_by_value( + value_fn - vf_preds, -vf_clip_param, vf_clip_param) + vf_loss2 = tf.square(vf_clipped - value_targets) + vf_loss = tf.maximum(vf_loss1, vf_loss2) + self.mean_vf_loss = reduce_mean_valid(vf_loss) + loss = reduce_mean_valid( + -surrogate_loss + + vf_loss_coeff * vf_loss - entropy_coeff * curr_entropy) + else: + self.mean_vf_loss = tf.constant(0.0) + loss = reduce_mean_valid(-surrogate_loss - entropy_coeff * curr_entropy) + self.loss = loss + + +def ppo_surrogate_loss(policy, model, dist_class, train_batch): + """Construct and return the PPO loss.""" + logits, state = model.from_batch(train_batch) + action_dist = dist_class(logits, model) + + if state: + max_seq_len = tf.reduce_max(train_batch["seq_lens"]) + mask = tf.sequence_mask(train_batch["seq_lens"], max_seq_len) + mask = tf.reshape(mask, [-1]) + else: + mask = tf.ones_like( + train_batch[Postprocessing.ADVANTAGES], dtype=tf.bool) + + policy.loss_obj = PPOLoss( + policy.action_space, + dist_class, + model, + train_batch[Postprocessing.VALUE_TARGETS], + train_batch[Postprocessing.ADVANTAGES], + train_batch[SampleBatch.ACTIONS], + train_batch[BEHAVIOUR_LOGITS], + train_batch[ACTION_LOGP], + train_batch[SampleBatch.VF_PREDS], + action_dist, + model.value_function(), + policy.kl_coeff, + mask, + entropy_coeff=policy.entropy_coeff, + clip_param=policy.config["clip_param"], + vf_clip_param=policy.config["vf_clip_param"], + vf_loss_coeff=policy.config["vf_loss_coeff"], + use_gae=policy.config["use_gae"], + model_config=policy.config["model"]) + + return policy.loss_obj.loss + + +def kl_and_loss_stats(policy, train_batch): + """Return statistics for the tensorboard.""" + return { + "cur_kl_coeff": tf.cast(policy.kl_coeff, tf.float64), + "cur_lr": tf.cast(policy.cur_lr, tf.float64), + "total_loss": policy.loss_obj.loss, + "policy_loss": policy.loss_obj.mean_policy_loss, + "vf_loss": policy.loss_obj.mean_vf_loss, + "vf_explained_var": explained_variance( + train_batch[Postprocessing.VALUE_TARGETS], + policy.model.value_function()), + "vf_preds": train_batch[Postprocessing.VALUE_TARGETS], + "kl": policy.loss_obj.mean_kl, + "entropy": policy.loss_obj.mean_entropy, + "entropy_coeff": tf.cast(policy.entropy_coeff, tf.float64), + "advantages": train_batch[Postprocessing.ADVANTAGES], + "rewards": train_batch["rewards"] + } + + +def vf_preds_and_logits_fetches(policy): + """Add value function and logits outputs to experience train_batches.""" + return { + SampleBatch.VF_PREDS: policy.model.value_function(), + BEHAVIOUR_LOGITS: policy.model.last_output(), + } + + +def postprocess_ppo_gae(policy, + sample_batch, + other_agent_batches=None, + episode=None): + """Add the policy logits, VF preds, and advantages to the trajectory.""" + completed = sample_batch["dones"][-1] + if completed: + last_r = 0.0 + else: + next_state = [] + for i in range(policy.num_state_tensors()): + next_state.append([sample_batch["state_out_{}".format(i)][-1]]) + last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1], + sample_batch[SampleBatch.ACTIONS][-1], + sample_batch[SampleBatch.REWARDS][-1], + *next_state) + + batch = compute_advantages( + sample_batch, + last_r, + policy.config["gamma"], + policy.config["lambda"], + use_gae=policy.config["use_gae"]) + return batch + + +def clip_gradients(policy, optimizer, loss): + """If grad_clip is not None, clip the gradients.""" + variables = policy.model.trainable_variables() + if policy.config["grad_clip"] is not None: + grads_and_vars = optimizer.compute_gradients(loss, variables) + grads = [g for (g, v) in grads_and_vars] + policy.grads, _ = tf.clip_by_global_norm(grads, + policy.config["grad_clip"]) + clipped_grads = list(zip(policy.grads, variables)) + return clipped_grads + else: + return optimizer.compute_gradients(loss, variables) + + +class ValueNetworkMixin(object): + """Construct the value function.""" + + def __init__(self, obs_space, action_space, config): + if config["use_gae"]: + + @make_tf_callable(self.get_session()) + def value(ob, prev_action, prev_reward, *state): + model_out, _ = self.model({ + SampleBatch.CUR_OBS: tf.convert_to_tensor([ob]), + SampleBatch.PREV_ACTIONS: tf.convert_to_tensor( + [prev_action]), + SampleBatch.PREV_REWARDS: tf.convert_to_tensor( + [prev_reward]), + "is_training": tf.convert_to_tensor(False), + }, [tf.convert_to_tensor([s]) for s in state], + tf.convert_to_tensor([1])) + return self.model.value_function()[0] + + else: + + @make_tf_callable(self.get_session()) + def value(ob, prev_action, prev_reward, *state): + return tf.constant(0.0) + + self._value = value + + +def setup_config(policy, obs_space, action_space, config): + """Add additional custom options from the config.""" + # auto set the model option for layer sharing + config["model"]["vf_share_layers"] = config["vf_share_layers"] + + +def setup_mixins(policy, obs_space, action_space, config): + """Construct additional classes that add on to PPO.""" + KLCoeffMixin.__init__(policy, config) + ValueNetworkMixin.__init__(policy, obs_space, action_space, config) + EntropyCoeffSchedule.__init__(policy, config["entropy_coeff"], + config["entropy_coeff_schedule"]) + LearningRateSchedule.__init__(policy, config["lr"], config["lr_schedule"]) + + +class KLCoeffMixin(object): + """Update the KL Coefficient. This is intentionally disabled to match the PPO paper better.""" + + def __init__(self, config): + # KL Coefficient + self.kl_coeff_val = config["kl_coeff"] + self.kl_target = config["kl_target"] + self.kl_coeff = tf.get_variable( + initializer=tf.constant_initializer(self.kl_coeff_val), + name="kl_coeff", + shape=(), + trainable=False, + dtype=tf.float32) + + def update_kl(self, blah): + """Disabled to match the PPO paper better.""" + pass + + +CustomPPOTFPolicy = build_tf_policy( + name="CustomPPOTFPolicy", + get_default_config=lambda: ray.rllib.agents.ppo.ppo.DEFAULT_CONFIG, + loss_fn=ppo_surrogate_loss, + stats_fn=kl_and_loss_stats, + extra_action_fetches_fn=vf_preds_and_logits_fetches, + postprocess_fn=postprocess_ppo_gae, + gradients_fn=clip_gradients, + before_init=setup_config, + before_loss_init=setup_mixins, + mixins=[ + LearningRateSchedule, EntropyCoeffSchedule, + ValueNetworkMixin, KLCoeffMixin + ]) + + +def validate_config(config): + """Check that the config is set up properly.""" + if config["entropy_coeff"] < 0: + raise DeprecationWarning("entropy_coeff must be >= 0") + if isinstance(config["entropy_coeff"], int): + config["entropy_coeff"] = float(config["entropy_coeff"]) + if config["batch_mode"] == "truncate_episodes" and not config["use_gae"]: + raise ValueError( + "Episode truncation is not supported without a value " + "function. Consider setting batch_mode=complete_episodes.") + if config["multiagent"]["policies"] and not config["simple_optimizer"]: + logger.info( + "In multi-agent mode, policies will be optimized sequentially " + "by the multi-GPU optimizer. Consider setting " + "simple_optimizer=True if this doesn't work for you.") + if config["simple_optimizer"]: + logger.warning( + "Using the simple minibatch optimizer. This will significantly " + "reduce performance, consider simple_optimizer=False.") + elif tf and tf.executing_eagerly(): + config["simple_optimizer"] = True # multi-gpu not supported + + +CustomPPOTrainer = build_trainer( + name="CustomPPOTrainer", + default_config=DEFAULT_CONFIG, + default_policy=CustomPPOTFPolicy, + make_policy_optimizer=choose_policy_optimizer, + validate_config=validate_config, + after_train_result=warn_about_bad_reward_scales) diff --git a/flow/benchmarks/README.md b/flow/benchmarks/README.md index 963ad5b70..bbcba9414 100644 --- a/flow/benchmarks/README.md +++ b/flow/benchmarks/README.md @@ -38,12 +38,12 @@ inflow = 300 veh/hour/lane S=(915,), A=(25,), T=400. this problem is to learn to avoid the *capacity drop* that is characteristic to bottleneck structures in transportation networks, and maximize the total outflow in a mixed-autonomy setting. -- `flow.benchmarks.bottleneck0` 4 lanes, inflow = 1900 veh/hour, 10% CAV +- `flow.benchmarks.bottleneck0` 4 lanes, inflow = 2500 veh/hour, 10% CAV penetration, no vehicles are allowed to lane change, S=(141,), A=(20,), T=1000. -- `flow.benchmarks.bottleneck1` 4 lanes, inflow = 1900 veh/hour, 10% CAV +- `flow.benchmarks.bottleneck1` 4 lanes, inflow = 2500 veh/hour, 10% CAV penetration, the human drivers follow the standard lane changing model in the simulator, S=(141,), A=(20,), T=1000. -- `flow.benchmarks.bottleneck2` 8 lanes, inflow = 3800 veh/hour, 10% CAV +- `flow.benchmarks.bottleneck2` 8 lanes, inflow = 5000 veh/hour, 10% CAV penetration, no vehicles are allowed to lane change, S=(281,), A=(40,), T=1000. ## Training on Custom Algorithms diff --git a/flow/benchmarks/bottleneck0.py b/flow/benchmarks/bottleneck0.py index b0e86844c..b07947ad7 100644 --- a/flow/benchmarks/bottleneck0.py +++ b/flow/benchmarks/bottleneck0.py @@ -66,7 +66,7 @@ } # flow rate -flow_rate = 2000 * SCALING +flow_rate = 2500 * SCALING # percentage of flow coming out of each lane inflow = InFlows() diff --git a/flow/benchmarks/bottleneck1.py b/flow/benchmarks/bottleneck1.py index 26ae6527a..9c8d9c192 100644 --- a/flow/benchmarks/bottleneck1.py +++ b/flow/benchmarks/bottleneck1.py @@ -66,7 +66,7 @@ } # flow rate -flow_rate = 2000 * SCALING +flow_rate = 2500 * SCALING # percentage of flow coming out of each lane inflow = InFlows() diff --git a/flow/benchmarks/bottleneck2.py b/flow/benchmarks/bottleneck2.py index 5052b3b88..4651d448b 100644 --- a/flow/benchmarks/bottleneck2.py +++ b/flow/benchmarks/bottleneck2.py @@ -66,7 +66,7 @@ } # flow rate -flow_rate = 2000 * SCALING +flow_rate = 2500 * SCALING # percentage of flow coming out of each lane inflow = InFlows() diff --git a/flow/benchmarks/grid0.py b/flow/benchmarks/grid0.py index 1655c3b3c..5c4ee5349 100644 --- a/flow/benchmarks/grid0.py +++ b/flow/benchmarks/grid0.py @@ -4,7 +4,7 @@ - **Observation Dimension**: (339, ) - **Horizon**: 400 steps """ -from flow.envs import TrafficLightGridPOEnv +from flow.envs import TrafficLightGridBenchmarkEnv from flow.networks import TrafficLightGridNetwork from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams, \ InFlows, SumoCarFollowingParams @@ -68,7 +68,7 @@ exp_tag="grid_0", # name of the flow environment the experiment is running on - env_name=TrafficLightGridPOEnv, + env_name=TrafficLightGridBenchmarkEnv, # name of the network class the experiment is running on network=TrafficLightGridNetwork, diff --git a/flow/benchmarks/grid1.py b/flow/benchmarks/grid1.py index ec2a27454..83055adfd 100644 --- a/flow/benchmarks/grid1.py +++ b/flow/benchmarks/grid1.py @@ -4,7 +4,7 @@ - **Observation Dimension**: (915, ) - **Horizon**: 400 steps """ -from flow.envs import TrafficLightGridPOEnv +from flow.envs import TrafficLightGridBenchmarkEnv from flow.networks import TrafficLightGridNetwork from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams, \ InFlows, SumoCarFollowingParams @@ -68,7 +68,7 @@ exp_tag="grid_1", # name of the flow environment the experiment is running on - env_name=TrafficLightGridPOEnv, + env_name=TrafficLightGridBenchmarkEnv, # name of the network class the experiment is running on network=TrafficLightGridNetwork, diff --git a/flow/controllers/__init__.py b/flow/controllers/__init__.py index 6cb20077a..a2106a21a 100755 --- a/flow/controllers/__init__.py +++ b/flow/controllers/__init__.py @@ -14,7 +14,8 @@ from flow.controllers.base_controller import BaseController from flow.controllers.car_following_models import CFMController, \ BCMController, OVMController, LinearOVM, IDMController, \ - SimCarFollowingController, LACController, GippsController + SimCarFollowingController, LACController, GippsController, \ + BandoFTLController from flow.controllers.velocity_controllers import FollowerStopper, \ PISaturation, NonLocalFollowerStopper @@ -22,12 +23,12 @@ from flow.controllers.base_lane_changing_controller import \ BaseLaneChangeController from flow.controllers.lane_change_controllers import StaticLaneChanger, \ - SimLaneChangeController + SimLaneChangeController, AILaneChangeController # routing controllers from flow.controllers.base_routing_controller import BaseRouter from flow.controllers.routing_controllers import ContinuousRouter, \ - GridRouter, BayBridgeRouter + GridRouter, BayBridgeRouter, I210Router __all__ = [ "RLController", "BaseController", "BaseLaneChangeController", "BaseRouter", @@ -35,5 +36,6 @@ "IDMController", "SimCarFollowingController", "FollowerStopper", "PISaturation", "StaticLaneChanger", "SimLaneChangeController", "ContinuousRouter", "GridRouter", "BayBridgeRouter", "LACController", - "GippsController", "NonLocalFollowerStopper" + "GippsController", "NonLocalFollowerStopper", "BandoFTLController", + "AILaneChangeController", "I210Router" ] diff --git a/flow/controllers/base_controller.py b/flow/controllers/base_controller.py index 41780826b..5eab1fb41 100755 --- a/flow/controllers/base_controller.py +++ b/flow/controllers/base_controller.py @@ -1,9 +1,10 @@ """Contains the base acceleration controller class.""" +from abc import ABCMeta, abstractmethod import numpy as np -class BaseController: +class BaseController(metaclass=ABCMeta): """Base class for flow-controlled acceleration behavior. Instantiates a controller and forces the user to pass a @@ -33,8 +34,12 @@ class BaseController: specified to in this model are as desired. delay : int delay in applying the action (time) - fail_safe : str - Should be either "instantaneous" or "safe_velocity" + fail_safe : list of str or str + List of failsafes which can be "instantaneous", "safe_velocity", + "feasible_accel", or "obey_speed_limit". The order of applying the + falsafes will be based on the order in the list. + display_warnings : bool + Flag for toggling on/off printing failsafe warnings to screen. noise : double variance of the gaussian from which to sample a noisy acceleration """ @@ -44,6 +49,7 @@ def __init__(self, car_following_params, delay=0, fail_safe=None, + display_warnings=True, noise=0): """Instantiate the base class for acceleration behavior.""" self.veh_id = veh_id @@ -55,7 +61,29 @@ def __init__(self, self.delay = delay # longitudinal failsafe used by the vehicle - self.fail_safe = fail_safe + if isinstance(fail_safe, str): + failsafe_list = [fail_safe] + elif isinstance(fail_safe, list) or fail_safe is None: + failsafe_list = fail_safe + else: + failsafe_list = None + raise ValueError("fail_safe should be string or list of strings. Setting fail_safe to None\n") + + failsafe_map = { + 'instantaneous': self.get_safe_action_instantaneous, + 'safe_velocity': self.get_safe_velocity_action, + 'feasible_accel': lambda _, accel: self.get_feasible_action(accel), + 'obey_speed_limit': self.get_obey_speed_limit_action + } + self.failsafes = [] + if failsafe_list: + for check in failsafe_list: + if check in failsafe_map: + self.failsafes.append(failsafe_map.get(check)) + else: + raise ValueError('Skipping {}, as it is not a valid failsafe.'.format(check)) + + self.display_warnings = display_warnings self.max_accel = car_following_params.controller_params['accel'] # max deaccel should always be a positive @@ -63,9 +91,34 @@ def __init__(self, self.car_following_params = car_following_params + @abstractmethod def get_accel(self, env): """Return the acceleration of the controller.""" - raise NotImplementedError + pass + + @abstractmethod + def get_custom_accel(self, this_vel, lead_vel, h): + """Return the custom computed acceleration of the controller. + + This method computes acceleration based on custom state information, + while get_accel() method compute acceleration based on the current state + information that are obtained from the environment. + + Parameters + ---------- + this_vel : float + this vehicle's velocity + lead_vel : float + leading vehicle's velocity + h : float + headway to leading vehicle + + Returns + ------- + float + the custom acceleration of the controller + """ + pass def get_action(self, env): """Convert the get_accel() acceleration into an action. @@ -75,8 +128,8 @@ def get_action(self, env): time step. This method also augments the controller with the desired level of - stochastic noise, and utlizes the "instantaneous" or "safe_velocity" - failsafes if requested. + stochastic noise, and utlizes the "instantaneous", "safe_velocity", + "feasible_accel", and/or "obey_speed_limit" failsafes if requested. Parameters ---------- @@ -88,6 +141,12 @@ def get_action(self, env): float the modified form of the acceleration """ + # clear the current stored accels of this vehicle to None + env.k.vehicle.update_accel(self.veh_id, None, noise=False, failsafe=False) + env.k.vehicle.update_accel(self.veh_id, None, noise=False, failsafe=True) + env.k.vehicle.update_accel(self.veh_id, None, noise=True, failsafe=False) + env.k.vehicle.update_accel(self.veh_id, None, noise=True, failsafe=True) + # this is to avoid abrupt decelerations when a vehicle has just entered # a network and it's data is still not subscribed if len(env.k.vehicle.get_edge(self.veh_id)) == 0: @@ -105,16 +164,26 @@ def get_action(self, env): if accel is None: return None + # store the acceleration without noise to each vehicle + # run fail safe if requested + env.k.vehicle.update_accel(self.veh_id, accel, noise=False, failsafe=False) + accel_no_noise_with_failsafe = accel + + for failsafe in self.failsafes: + accel_no_noise_with_failsafe = failsafe(env, accel_no_noise_with_failsafe) + + env.k.vehicle.update_accel(self.veh_id, accel_no_noise_with_failsafe, noise=False, failsafe=True) + # add noise to the accelerations, if requested if self.accel_noise > 0: - accel += np.random.normal(0, self.accel_noise) + accel += np.sqrt(env.sim_step) * np.random.normal(0, self.accel_noise) + env.k.vehicle.update_accel(self.veh_id, accel, noise=True, failsafe=False) - # run the failsafes, if requested - if self.fail_safe == 'instantaneous': - accel = self.get_safe_action_instantaneous(env, accel) - elif self.fail_safe == 'safe_velocity': - accel = self.get_safe_velocity_action(env, accel) + # run the fail-safes, if requested + for failsafe in self.failsafes: + accel = failsafe(env, accel) + env.k.vehicle.update_accel(self.veh_id, accel, noise=True, failsafe=True) return accel def get_safe_action_instantaneous(self, env, action): @@ -160,6 +229,13 @@ def get_safe_action_instantaneous(self, env, action): # if the vehicle will crash into the vehicle ahead of it in the # next time step (assuming the vehicle ahead of it is not # moving), then stop immediately + if self.display_warnings: + print( + "=====================================\n" + "Vehicle {} is about to crash. Instantaneous acceleration " + "clipping applied.\n" + "=====================================".format(self.veh_id)) + return -this_vel / sim_step else: # if the vehicle is not in danger of crashing, continue with @@ -221,8 +297,8 @@ def safe_velocity(self, env): Returns ------- float - maximum safe velocity given a maximum deceleration and delay in - performing the breaking action + maximum safe velocity given a maximum deceleration, delay in + performing the breaking action, and speed limit """ lead_id = env.k.vehicle.get_leader(self.veh_id) lead_vel = env.k.vehicle.get_speed(lead_id) @@ -233,4 +309,97 @@ def safe_velocity(self, env): v_safe = 2 * h / env.sim_step + dv - this_vel * (2 * self.delay) + # check for speed limit FIXME: this is not called + # this_edge = env.k.vehicle.get_edge(self.veh_id) + # edge_speed_limit = env.k.network.speed_limit(this_edge) + + if this_vel > v_safe: + if self.display_warnings: + print( + "=====================================\n" + "Speed of vehicle {} is greater than safe speed. Safe velocity " + "clipping applied.\n" + "=====================================".format(self.veh_id)) + return v_safe + + def get_obey_speed_limit_action(self, env, action): + """Perform the "obey_speed_limit" failsafe action. + + Checks if the computed acceleration would put us above edge speed limit. + If it would, output the acceleration that would put at the speed limit + velocity. + + Parameters + ---------- + env : flow.envs.Env + current environment, which contains information of the state of the + network at the current time step + action : float + requested acceleration action + + Returns + ------- + float + the requested action clipped by the speed limit + """ + # check for speed limit + this_edge = env.k.vehicle.get_edge(self.veh_id) + edge_speed_limit = env.k.network.speed_limit(this_edge) + + this_vel = env.k.vehicle.get_speed(self.veh_id) + sim_step = env.sim_step + + if this_vel + action * sim_step > edge_speed_limit: + if edge_speed_limit > 0: + if self.display_warnings: + print( + "=====================================\n" + "Speed of vehicle {} is greater than speed limit. Obey " + "speed limit clipping applied.\n" + "=====================================".format(self.veh_id)) + return (edge_speed_limit - this_vel) / sim_step + else: + return -this_vel / sim_step + else: + return action + + def get_feasible_action(self, action): + """Perform the "feasible_accel" failsafe action. + + Checks if the computed acceleration would put us above maximum + acceleration or deceleration. If it would, output the acceleration + equal to maximum acceleration or deceleration. + + Parameters + ---------- + action : float + requested acceleration action + + Returns + ------- + float + the requested action clipped by the feasible acceleration or + deceleration. + """ + if action > self.max_accel: + action = self.max_accel + + if self.display_warnings: + print( + "=====================================\n" + "Acceleration of vehicle {} is greater than the max " + "acceleration. Feasible acceleration clipping applied.\n" + "=====================================".format(self.veh_id)) + + if action < -self.max_deaccel: + action = -self.max_deaccel + + if self.display_warnings: + print( + "=====================================\n" + "Deceleration of vehicle {} is greater than the max " + "deceleration. Feasible acceleration clipping applied.\n" + "=====================================".format(self.veh_id)) + + return action diff --git a/flow/controllers/base_lane_changing_controller.py b/flow/controllers/base_lane_changing_controller.py index af009f992..eb2b566f5 100755 --- a/flow/controllers/base_lane_changing_controller.py +++ b/flow/controllers/base_lane_changing_controller.py @@ -1,7 +1,9 @@ """Contains the base lane change controller class.""" +from abc import ABCMeta, abstractmethod -class BaseLaneChangeController: + +class BaseLaneChangeController(metaclass=ABCMeta): """Base class for lane-changing controllers. Instantiates a controller and forces the user to pass a @@ -36,6 +38,7 @@ def __init__(self, veh_id, lane_change_params=None): self.veh_id = veh_id self.lane_change_params = lane_change_params + @abstractmethod def get_lane_change_action(self, env): """Specify the lane change action to be performed. @@ -55,7 +58,7 @@ def get_lane_change_action(self, env): float or int requested lane change action """ - raise NotImplementedError + pass def get_action(self, env): """Return the action of the lane change controller. diff --git a/flow/controllers/base_routing_controller.py b/flow/controllers/base_routing_controller.py index b001dc62e..17048ce7d 100755 --- a/flow/controllers/base_routing_controller.py +++ b/flow/controllers/base_routing_controller.py @@ -1,7 +1,9 @@ """Contains the base routing controller class.""" +from abc import ABCMeta, abstractmethod -class BaseRouter: + +class BaseRouter(metaclass=ABCMeta): """Base class for routing controllers. These controllers are used to dynamically change the routes of vehicles @@ -30,6 +32,7 @@ def __init__(self, veh_id, router_params): self.veh_id = veh_id self.router_params = router_params + @abstractmethod def choose_route(self, env): """Return the routing method implemented by the controller. @@ -45,4 +48,4 @@ def choose_route(self, env): is returned, the vehicle performs no routing action in the current time step. """ - raise NotImplementedError + pass diff --git a/flow/controllers/car_following_models.py b/flow/controllers/car_following_models.py index f86c546e8..192442980 100755 --- a/flow/controllers/car_following_models.py +++ b/flow/controllers/car_following_models.py @@ -56,7 +56,8 @@ def __init__(self, v_des=8, time_delay=0.0, noise=0, - fail_safe=None): + fail_safe=None, + display_warnings=True): """Instantiate a CFM controller.""" BaseController.__init__( self, @@ -64,7 +65,9 @@ def __init__(self, car_following_params, delay=time_delay, fail_safe=fail_safe, - noise=noise) + noise=noise, + display_warnings=display_warnings, + ) self.veh_id = veh_id self.k_d = k_d @@ -87,6 +90,10 @@ def get_accel(self, env): return self.k_d*(d_l - self.d_des) + self.k_v*(lead_vel - this_vel) + \ self.k_c*(self.v_des - this_vel) + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + class BCMController(BaseController): """Bilateral car-following model controller. @@ -132,7 +139,8 @@ def __init__(self, v_des=8, time_delay=0.0, noise=0, - fail_safe=None): + fail_safe=None, + display_warnings=True): """Instantiate a Bilateral car-following model controller.""" BaseController.__init__( self, @@ -140,7 +148,9 @@ def __init__(self, car_following_params, delay=time_delay, fail_safe=fail_safe, - noise=noise) + noise=noise, + display_warnings=display_warnings, + ) self.veh_id = veh_id self.k_d = k_d @@ -175,6 +185,10 @@ def get_accel(self, env): self.k_v * ((lead_vel - this_vel) - (this_vel - trail_vel)) + \ self.k_c * (self.v_des - this_vel) + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + class LACController(BaseController): """Linear Adaptive Cruise Control. @@ -212,7 +226,8 @@ def __init__(self, a=0, time_delay=0.0, noise=0, - fail_safe=None): + fail_safe=None, + display_warnings=True): """Instantiate a Linear Adaptive Cruise controller.""" BaseController.__init__( self, @@ -220,7 +235,9 @@ def __init__(self, car_following_params, delay=time_delay, fail_safe=fail_safe, - noise=noise) + noise=noise, + display_warnings=display_warnings, + ) self.veh_id = veh_id self.k_1 = k_1 @@ -244,6 +261,10 @@ def get_accel(self, env): return self.a + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + class OVMController(BaseController): """Optimal Vehicle Model controller. @@ -289,7 +310,8 @@ def __init__(self, v_max=30, time_delay=0, noise=0, - fail_safe=None): + fail_safe=None, + display_warnings=True): """Instantiate an Optimal Vehicle Model controller.""" BaseController.__init__( self, @@ -297,7 +319,9 @@ def __init__(self, car_following_params, delay=time_delay, fail_safe=fail_safe, - noise=noise) + noise=noise, + display_warnings=display_warnings, + ) self.veh_id = veh_id self.v_max = v_max self.alpha = alpha @@ -327,6 +351,10 @@ def get_accel(self, env): return self.alpha * (v_h - this_vel) + self.beta * h_dot + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + class LinearOVM(BaseController): """Linear OVM controller. @@ -364,7 +392,8 @@ def __init__(self, h_st=5, time_delay=0.0, noise=0, - fail_safe=None): + fail_safe=None, + display_warnings=True): """Instantiate a Linear OVM controller.""" BaseController.__init__( self, @@ -372,7 +401,9 @@ def __init__(self, car_following_params, delay=time_delay, fail_safe=fail_safe, - noise=noise) + noise=noise, + display_warnings=display_warnings, + ) self.veh_id = veh_id # 4.8*1.85 for case I, 3.8*1.85 for case II, per Nakayama self.v_max = v_max @@ -396,6 +427,10 @@ def get_accel(self, env): return (v_h - this_vel) / self.adaptation + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + class IDMController(BaseController): """Intelligent Driver Model (IDM) controller. @@ -445,6 +480,7 @@ def __init__(self, time_delay=0.0, noise=0, fail_safe=None, + display_warnings=True, car_following_params=None): """Instantiate an IDM controller.""" BaseController.__init__( @@ -453,7 +489,9 @@ def __init__(self, car_following_params, delay=time_delay, fail_safe=fail_safe, - noise=noise) + noise=noise, + display_warnings=display_warnings, + ) self.v0 = v0 self.T = T self.a = a @@ -481,6 +519,21 @@ def get_accel(self, env): return self.a * (1 - (v / self.v0)**self.delta - (s_star / h)**2) + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + # in order to deal with ZeroDivisionError + if abs(h) < 1e-3: + h = 1e-3 + + if lead_vel is None: # no car ahead + s_star = 0 + else: + s_star = self.s0 + max( + 0, this_vel * self.T + this_vel * (this_vel - lead_vel) / + (2 * np.sqrt(self.a * self.b))) + + return self.a * (1 - (this_vel / self.v0)**self.delta - (s_star / h)**2) + class SimCarFollowingController(BaseController): """Controller whose actions are purely defined by the simulator. @@ -496,6 +549,10 @@ def get_accel(self, env): """See parent class.""" return None + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + class GippsController(BaseController): """Gipps' Model controller. @@ -546,7 +603,8 @@ def __init__(self, tau=1, delay=0, noise=0, - fail_safe=None): + fail_safe=None, + display_warnings=True): """Instantiate a Gipps' controller.""" BaseController.__init__( self, @@ -554,8 +612,9 @@ def __init__(self, car_following_params, delay=delay, fail_safe=fail_safe, - noise=noise - ) + noise=noise, + display_warnings=display_warnings, + ) self.v_desired = v0 self.acc = acc @@ -580,3 +639,97 @@ def get_accel(self, env): v_next = min(v_acc, v_safe, self.v_desired) return (v_next-v)/env.sim_step + + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError + + +class BandoFTLController(BaseController): + """Bando follow-the-leader controller. + + Usage + ----- + See BaseController for usage example. + + Attributes + ---------- + veh_id : str + Vehicle ID for SUMO identification + car_following_params : flow.core.params.SumoCarFollowingParams + see parent class + alpha : float + gain on desired velocity to current velocity difference + (default: 0.6) + beta : float + gain on lead car velocity and self velocity difference + (default: 0.9) + h_st : float + headway for stopping (default: 5) + h_go : float + headway for full speed (default: 35) + v_max : float + max velocity (default: 30) + time_delay : float + time delay (default: 0.5) + noise : float + std dev of normal perturbation to the acceleration (default: 0) + fail_safe : str + type of flow-imposed failsafe the vehicle should posses, defaults + to no failsafe (None) + """ + + def __init__(self, + veh_id, + car_following_params, + alpha=.5, + beta=20, + h_st=2, + h_go=10, + v_max=32, + want_max_accel=False, + time_delay=0, + noise=0, + fail_safe=None, + display_warnings=True): + """Instantiate an Bando controller.""" + BaseController.__init__( + self, + veh_id, + car_following_params, + delay=time_delay, + fail_safe=fail_safe, + noise=noise, + display_warnings=display_warnings, + ) + self.veh_id = veh_id + self.v_max = v_max + self.alpha = alpha + self.beta = beta + self.h_st = h_st + self.h_go = h_go + self.want_max_accel = want_max_accel + + def get_accel(self, env): + """See parent class.""" + # without generating waves. + lead_id = env.k.vehicle.get_leader(self.veh_id) + if not lead_id: # no car ahead + if self.want_max_accel: + return self.max_accel + + v_l = env.k.vehicle.get_speed(lead_id) + v = env.k.vehicle.get_speed(self.veh_id) + s = env.k.vehicle.get_headway(self.veh_id) + return self.accel_func(v, v_l, s) + + def accel_func(self, v, v_l, s): + """Compute the acceleration function.""" + v_h = self.v_max * ((np.tanh(s/self.h_st-2)+np.tanh(2))/(1+np.tanh(2))) + s_dot = v_l - v + u = self.alpha * (v_h - v) + self.beta * s_dot/(s**2) + return u + + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError diff --git a/flow/controllers/lane_change_controllers.py b/flow/controllers/lane_change_controllers.py index 7f7a16ff3..cc268bd3b 100755 --- a/flow/controllers/lane_change_controllers.py +++ b/flow/controllers/lane_change_controllers.py @@ -24,3 +24,167 @@ class StaticLaneChanger(BaseLaneChangeController): def get_lane_change_action(self, env): """See parent class.""" return 0 + + +class AILaneChangeController(BaseLaneChangeController): + """A lane-changing controller based on acceleration incentive model. + + Usage + ----- + See base class for usage example. + + Attributes + ---------- + veh_id : str + Vehicle ID for SUMO/Aimsun identification + lane_change_params : flow.core.param.SumoLaneChangeParams + see parent class + left_delta : float + used for the incentive criterion for left lane change (default: 2.6) + right_delta : float + used for the incentive criterion for right lane change (default: 2.7) + left_beta : float + used for the incentive criterion for left lane change (default: 2.6) + right_beta : float + used for the incentive criterion for right lane change (default: 2.7) + """ + + def __init__(self, + veh_id, + lane_change_params=None, + left_delta=2.6, + right_delta=2.7, + left_beta=2.6, + right_beta=2.7): + """Instantiate an AI lane-change controller.""" + BaseLaneChangeController.__init__( + self, + veh_id, + lane_change_params, + ) + + self.veh_id = veh_id + self.left_delta = left_delta + self.right_delta = right_delta + self.left_beta = left_beta + self.right_beta = right_beta + + def get_lane_change_action(self, env): + """See parent class.""" + # acceleration if the ego vehicle remains in current lane. + ego_accel_controller = env.k.vehicle.get_acc_controller(self.veh_id) + acc_in_present_lane = ego_accel_controller.get_accel(env) + + # get ego vehicle lane number, and velocity + ego_lane = env.k.vehicle.get_lane(self.veh_id) + ego_vel = env.k.vehicle.get_speed(self.veh_id) + + # get lane leaders, followers, headways, and tailways + lane_leaders = env.k.vehicle.get_lane_leaders(self.veh_id) + lane_followers = env.k.vehicle.get_lane_followers(self.veh_id) + lane_headways = env.k.vehicle.get_lane_headways(self.veh_id) + lane_tailways = env.k.vehicle.get_lane_tailways(self.veh_id) + + # determine left and right lane number + this_edge = env.k.vehicle.get_edge(self.veh_id) + num_lanes = env.k.network.num_lanes(this_edge) + l_lane = ego_lane - 1 if ego_lane > 0 else None + r_lane = ego_lane + 1 if ego_lane < num_lanes - 1 else None + + # compute ego and new follower accelerations if moving to left lane + if l_lane is not None: + # get left leader and follower vehicle ID + l_l = lane_leaders[l_lane] + l_f = lane_followers[l_lane] + + # ego acceleration if the ego vehicle is in the lane to the left + if l_l not in ['', None]: + # left leader velocity and headway + l_l_vel = env.k.vehicle.get_speed(l_l) + l_l_headway = lane_headways[l_lane] + acc_in_left_lane = ego_accel_controller.get_custom_accel( + this_vel=ego_vel, + lead_vel=l_l_vel, + h=l_l_headway) + else: # if left lane exists but left leader does not exist + # in this case we assign None to the leader velocity and + # large number to headway + l_l_vel = None + l_l_headway = 1000 + acc_in_left_lane = ego_accel_controller.get_custom_accel( + this_vel=ego_vel, + lead_vel=l_l_vel, + h=l_l_headway) + + # follower acceleration if the ego vehicle is in the left lane + if l_f not in ['', None]: + # left follower velocity and headway + l_f_vel = env.k.vehicle.get_speed(l_f) + l_f_tailway = lane_tailways[l_lane] + l_f_accel_controller = env.k.vehicle.get_acc_controller(l_f) + left_lane_follower_acc = l_f_accel_controller.get_custom_accel( + this_vel=l_f_vel, + lead_vel=ego_vel, + h=l_f_tailway) + else: # if left lane exists but left follower does not exist + # in this case we assign maximum acceleration + left_lane_follower_acc = ego_accel_controller.max_accel + else: + acc_in_left_lane = None + left_lane_follower_acc = None + + # compute ego and new follower accelerations if moving to right lane + if r_lane is not None: + # get right leader and follower vehicle ID + r_l = lane_leaders[r_lane] + r_f = lane_followers[r_lane] + + # ego acceleration if the ego vehicle is in the lane to the right + if r_l not in ['', None]: + # right leader velocity and headway + r_l_vel = env.k.vehicle.get_speed(r_l) + r_l_headway = lane_headways[r_lane] + acc_in_right_lane = ego_accel_controller.get_custom_accel( + this_vel=ego_vel, + lead_vel=r_l_vel, + h=r_l_headway) + else: # if right lane exists but right leader does not exist + # in this case we assign None to the leader velocity and + # large number to headway + r_l_vel = None + r_l_headway = 1000 + acc_in_right_lane = ego_accel_controller.get_custom_accel( + this_vel=ego_vel, + lead_vel=r_l_vel, + h=r_l_headway) + + # follower acceleration if the ego vehicle is in the right lane + if r_f not in ['', None]: + # right follower velocity and headway + r_f_vel = env.k.vehicle.get_speed(r_f) + r_f_headway = lane_tailways[r_lane] + r_f_accel_controller = env.k.vehicle.get_acc_controller(r_f) + right_lane_follower_acc = r_f_accel_controller.get_custom_accel( + this_vel=r_f_vel, + lead_vel=ego_vel, + h=r_f_headway) + else: # if right lane exists but right follower does not exist + # assign maximum acceleration + right_lane_follower_acc = ego_accel_controller.max_accel + else: + acc_in_right_lane = None + right_lane_follower_acc = None + + # determine lane change action + if l_lane is not None and acc_in_left_lane >= - self.left_beta and \ + left_lane_follower_acc >= -self.left_beta and \ + acc_in_left_lane >= acc_in_present_lane + self.left_delta: + action = 1 + elif r_lane is not None and acc_in_right_lane >= - self.right_beta and \ + right_lane_follower_acc >= -self.right_beta and \ + acc_in_right_lane >= acc_in_present_lane + self.right_delta: + action = -1 + else: + action = 0 + + return action diff --git a/flow/controllers/rlcontroller.py b/flow/controllers/rlcontroller.py index 61f53f11a..f0d922cfa 100755 --- a/flow/controllers/rlcontroller.py +++ b/flow/controllers/rlcontroller.py @@ -37,3 +37,11 @@ def __init__(self, veh_id, car_following_params): self, veh_id, car_following_params) + + def get_accel(self, env): + """Pass, as this is never called; required to override abstractmethod.""" + pass + + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError diff --git a/flow/controllers/routing_controllers.py b/flow/controllers/routing_controllers.py index e6ccdde78..02aa34cb4 100755 --- a/flow/controllers/routing_controllers.py +++ b/flow/controllers/routing_controllers.py @@ -124,3 +124,29 @@ def choose_route(self, env): new_route = super().choose_route(env) return new_route + + +class I210Router(ContinuousRouter): + """Assists in choosing routes in select cases for the I-210 sub-network. + + Extension to the Continuous Router. + + Usage + ----- + See base class for usage example. + """ + + def choose_route(self, env): + """See parent class.""" + edge = env.k.vehicle.get_edge(self.veh_id) + lane = env.k.vehicle.get_lane(self.veh_id) + + # vehicles on these edges in lanes 4 and 5 are not going to be able to + # make it out in time + if edge == "119257908#1-AddedOffRampEdge" and lane in [5, 4, 3]: + new_route = env.available_routes[ + "119257908#1-AddedOffRampEdge"][0][0] + else: + new_route = super().choose_route(env) + + return new_route diff --git a/flow/controllers/velocity_controllers.py b/flow/controllers/velocity_controllers.py index 2e4b7c22a..acebdb5ff 100644 --- a/flow/controllers/velocity_controllers.py +++ b/flow/controllers/velocity_controllers.py @@ -26,10 +26,12 @@ def __init__(self, veh_id, car_following_params, v_des=15, - danger_edges=None): + danger_edges=None, + control_length=None, + no_control_edges=None): """Instantiate FollowerStopper.""" BaseController.__init__( - self, veh_id, car_following_params, delay=1.0, + self, veh_id, car_following_params, delay=0.0, fail_safe='safe_velocity') # desired speed of the vehicle @@ -45,7 +47,10 @@ def __init__(self, self.d_1 = 1.5 self.d_2 = 1.0 self.d_3 = 0.5 + self.danger_edges = danger_edges if danger_edges else {} + self.control_length = control_length + self.no_control_edges = no_control_edges def find_intersection_dist(self, env): """Find distance to intersection. @@ -74,46 +79,57 @@ def find_intersection_dist(self, env): def get_accel(self, env): """See parent class.""" - lead_id = env.k.vehicle.get_leader(self.veh_id) - this_vel = env.k.vehicle.get_speed(self.veh_id) - lead_vel = env.k.vehicle.get_speed(lead_id) - - if self.v_des is None: + if env.time_counter < env.env_params.warmup_steps * env.env_params.sims_per_step: return None - - if lead_id is None: - v_cmd = self.v_des else: - dx = env.k.vehicle.get_headway(self.veh_id) - dv_minus = min(lead_vel - this_vel, 0) - - dx_1 = self.dx_1_0 + 1 / (2 * self.d_1) * dv_minus**2 - dx_2 = self.dx_2_0 + 1 / (2 * self.d_2) * dv_minus**2 - dx_3 = self.dx_3_0 + 1 / (2 * self.d_3) * dv_minus**2 - v = min(max(lead_vel, 0), self.v_des) - # compute the desired velocity - if dx <= dx_1: - v_cmd = 0 - elif dx <= dx_2: - v_cmd = v * (dx - dx_1) / (dx_2 - dx_1) - elif dx <= dx_3: - v_cmd = v + (self.v_des - this_vel) * (dx - dx_2) \ - / (dx_3 - dx_2) - else: - v_cmd = self.v_des + lead_id = env.k.vehicle.get_leader(self.veh_id) + this_vel = env.k.vehicle.get_speed(self.veh_id) + lead_vel = env.k.vehicle.get_speed(lead_id) - edge = env.k.vehicle.get_edge(self.veh_id) + if self.v_des is None: + return None - if edge == "": - return None + if lead_id is None: + v_cmd = self.v_des + else: + dx = env.k.vehicle.get_headway(self.veh_id) + dv_minus = min(lead_vel - this_vel, 0) + + dx_1 = self.dx_1_0 + 1 / (2 * self.d_1) * dv_minus**2 + dx_2 = self.dx_2_0 + 1 / (2 * self.d_2) * dv_minus**2 + dx_3 = self.dx_3_0 + 1 / (2 * self.d_3) * dv_minus**2 + v = min(max(lead_vel, 0), self.v_des) + # compute the desired velocity + if dx <= dx_1: + v_cmd = 0 + elif dx <= dx_2: + v_cmd = v * (dx - dx_1) / (dx_2 - dx_1) + elif dx <= dx_3: + v_cmd = v + (self.v_des - this_vel) * (dx - dx_2) \ + / (dx_3 - dx_2) + else: + v_cmd = self.v_des + + edge = env.k.vehicle.get_edge(self.veh_id) + + if edge == "": + return None + + if (self.find_intersection_dist(env) <= 10 and + env.k.vehicle.get_edge(self.veh_id) in self.danger_edges) or \ + env.k.vehicle.get_edge(self.veh_id)[0] == ":" \ + or (self.control_length and (env.k.vehicle.get_x_by_id(self.veh_id) < self.control_length[0] + or env.k.vehicle.get_x_by_id(self.veh_id) > self.control_length[1])) \ + or (self.no_control_edges is not None and len(self.no_control_edges) > 0 + and edge in self.no_control_edges): + return None + else: + # compute the acceleration from the desired velocity + return np.clip((v_cmd - this_vel) / env.sim_step, -np.abs(self.max_deaccel), self.max_accel) - if self.find_intersection_dist(env) <= 10 and \ - env.k.vehicle.get_edge(self.veh_id) in self.danger_edges or \ - env.k.vehicle.get_edge(self.veh_id)[0] == ":": - return None - else: - # compute the acceleration from the desired velocity - return (v_cmd - this_vel) / env.sim_step + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError class NonLocalFollowerStopper(FollowerStopper): @@ -154,11 +170,6 @@ def get_accel(self, env): if edge == "": return None - - if self.find_intersection_dist(env) <= 10 and \ - env.k.vehicle.get_edge(self.veh_id) in self.danger_edges or \ - env.k.vehicle.get_edge(self.veh_id)[0] == ":": - return None else: # compute the acceleration from the desired velocity return (v_cmd - this_vel) / env.sim_step @@ -184,7 +195,7 @@ class PISaturation(BaseController): def __init__(self, veh_id, car_following_params): """Instantiate PISaturation.""" - BaseController.__init__(self, veh_id, car_following_params, delay=1.0) + BaseController.__init__(self, veh_id, car_following_params, delay=0.0) # maximum achievable acceleration by the vehicle self.max_accel = car_following_params.controller_params['accel'] @@ -238,3 +249,7 @@ def get_accel(self, env): accel = (self.v_cmd - this_vel) / env.sim_step return min(accel, self.max_accel) + + def get_custom_accel(self, this_vel, lead_vel, h): + """See parent class.""" + raise NotImplementedError diff --git a/flow/core/experiment.py b/flow/core/experiment.py index 69a78cb0e..c23f5359e 100755 --- a/flow/core/experiment.py +++ b/flow/core/experiment.py @@ -1,11 +1,21 @@ """Contains an experiment class for running simulations.""" -from flow.core.util import emission_to_csv from flow.utils.registry import make_create_env -import datetime +from flow.core.util import ensure_dir +from flow.data_pipeline.data_pipeline import upload_to_s3 +from flow.data_pipeline.data_pipeline import get_configuration +from flow.data_pipeline.data_pipeline import generate_trajectory_table +from flow.data_pipeline.data_pipeline import write_dict_to_csv +from flow.data_pipeline.leaderboard_utils import network_name_translate +from flow.core.rewards import veh_energy_consumption +from flow.visualize.time_space_diagram import tsd_main +from collections import defaultdict +from datetime import timezone +from datetime import datetime import logging import time -import os import numpy as np +import uuid +import os class Experiment: @@ -59,7 +69,10 @@ class can generate csv files from emission files produced by sumo. These the environment object the simulator will run """ - def __init__(self, flow_params, custom_callables=None): + def __init__(self, + flow_params, + custom_callables=None, + register_with_ray=False): """Instantiate the Experiment class. Parameters @@ -71,21 +84,39 @@ def __init__(self, flow_params, custom_callables=None): want to extract from the environment. The lambda will be called at each step to extract information from the env and it will be stored in a dict keyed by the str. + register_with_ray : bool + whether the environment is meant to be registered with ray. If set + to True, the environment is including as part of `self.env` + separately (i.e. not here). """ self.custom_callables = custom_callables or {} # Get the env name and a creator for the environment. - create_env, _ = make_create_env(flow_params) - - # Create the environment. - self.env = create_env() - - logging.info(" Starting experiment {} at {}".format( - self.env.network.name, str(datetime.datetime.utcnow()))) + create_env, env_name = make_create_env(flow_params) - logging.info("Initializing environment.") + # record env_name and create_env, need it to register for ray + self.env_name = env_name + self.create_env = create_env - def run(self, num_runs, rl_actions=None, convert_to_csv=False): + # Create the environment. + if not register_with_ray: + self.env = create_env() + + logging.info(" Starting experiment {} at {}".format( + self.env.network.name, str(datetime.utcnow()))) + + logging.info("Initializing environment.") + + def run(self, + num_runs, + rl_actions=None, + convert_to_csv=False, + to_aws=None, + is_baseline=False, + multiagent=False, + rets=None, + policy_map_fn=None, + supplied_metadata=None): """Run the given network for a set number of runs. Parameters @@ -98,6 +129,20 @@ def run(self, num_runs, rl_actions=None, convert_to_csv=False): convert_to_csv : bool Specifies whether to convert the emission file created by sumo into a csv file + to_aws: str + Specifies the S3 partition you want to store the output file, + will be used to later for query. If NONE, won't upload output + to S3. + is_baseline: bool + Specifies whether this is a baseline run. + multiagent : bool + whether the policy is multi-agent + rets : dict + a dictionary to store the rewards for multiagent simulation + policy_map_fn : function + a mapping from each agent to their respective policy + supplied_metadata: dict (str: list) + metadata provided by the caller Returns ------- @@ -119,12 +164,23 @@ def run(self, num_runs, rl_actions=None, convert_to_csv=False): 'output should be generated. If you do not wish to generate ' 'emissions, set the convert_to_csv parameter to False.') + # Make sure the emission path directory exists, and if not, create it. + if self.env.sim_params.emission_path is not None: + ensure_dir(self.env.sim_params.emission_path) + # used to store info_dict = { - "returns": [], "velocities": [], "outflows": [], + "avg_trip_energy": [], + "avg_trip_time": [], + "total_completed_trips": [] } + if not multiagent: + info_dict["returns"] = [] + all_trip_energy_distribution = defaultdict(lambda: []) + all_trip_time_distribution = defaultdict(lambda: []) + info_dict.update({ key: [] for key in self.custom_callables.keys() }) @@ -137,11 +193,59 @@ def rl_actions(*_): t = time.time() times = [] + if convert_to_csv and self.env.simulator == "traci": + # data pipeline + source_id = 'flow_{}'.format(uuid.uuid4().hex) + metadata = defaultdict(lambda: []) + + # collect current time + cur_datetime = datetime.now(timezone.utc) + cur_date = cur_datetime.date().isoformat() + cur_time = cur_datetime.time().isoformat() + + if to_aws: + # collecting information for metadata table + metadata['source_id'].append(source_id) + metadata['submission_time'].append(cur_time) + metadata['network'].append( + network_name_translate(self.env.network.name.split('_20')[0])) + metadata['is_baseline'].append(str(is_baseline)) + if supplied_metadata is not None and 'name' in supplied_metadata and 'strategy' in supplied_metadata: + name = supplied_metadata['name'] + strategy = supplied_metadata['strategy'] + else: + name, strategy = get_configuration() + metadata['submitter_name'].append(name) + metadata['strategy'].append(strategy) + metadata.update(supplied_metadata) + + # emission-specific parameters + dir_path = self.env.sim_params.emission_path + trajectory_table_path = os.path.join( + dir_path, '{}.csv'.format(source_id)) + metadata_table_path = os.path.join( + dir_path, '{}_METADATA.csv'.format(source_id)) + else: + source_id = None + trajectory_table_path = None + metadata_table_path = None + metadata = None + cur_date = None + + emission_files = [] for i in range(num_runs): - ret = 0 + if rets and multiagent: + ret = {key: [0] for key in rets.keys()} + else: + ret = 0 vel = [] + per_vehicle_energy_trace = defaultdict(lambda: []) + completed_veh_types = {} + completed_vehicle_avg_energy = {} + completed_vehicle_travel_time = {} custom_vals = {key: [] for key in self.custom_callables.keys()} state = self.env.reset() + initial_vehicles = set(self.env.k.vehicle.get_ids()) for j in range(num_steps): t0 = time.time() state, reward, done, _ = self.env.step(rl_actions(state)) @@ -151,24 +255,68 @@ def rl_actions(*_): # Compute the velocity speeds and cumulative returns. veh_ids = self.env.k.vehicle.get_ids() vel.append(np.mean(self.env.k.vehicle.get_speed(veh_ids))) - ret += reward + if rets and multiagent: + for actor, rew in reward.items(): + ret[policy_map_fn(actor)][0] += rew + elif not multiagent: + ret += reward # Compute the results for the custom callables. for (key, lambda_func) in self.custom_callables.items(): custom_vals[key].append(lambda_func(self.env)) - if done: + # Compute the results for energy metrics + for past_veh_id in per_vehicle_energy_trace.keys(): + if past_veh_id not in veh_ids and past_veh_id not in completed_vehicle_avg_energy: + all_trip_energy_distribution[completed_veh_types[past_veh_id]].append( + np.sum(per_vehicle_energy_trace[past_veh_id])) + all_trip_time_distribution[completed_veh_types[past_veh_id]].append( + len(per_vehicle_energy_trace[past_veh_id])) + completed_vehicle_avg_energy[past_veh_id] = np.sum(per_vehicle_energy_trace[past_veh_id]) + completed_vehicle_travel_time[past_veh_id] = len(per_vehicle_energy_trace[past_veh_id]) + + # Update the stored energy metrics calculation results + for veh_id in veh_ids: + if veh_id not in initial_vehicles: + if veh_id not in per_vehicle_energy_trace: + # we have to skip the first step's energy calculation + per_vehicle_energy_trace[veh_id].append(0) + completed_veh_types[veh_id] = self.env.k.vehicle.get_type(veh_id) + else: + per_vehicle_energy_trace[veh_id].append(-1 * veh_energy_consumption(self.env, veh_id)) + + if multiagent and done['__all__']: + break + if type(done) is dict and done['__all__'] or done is True: break + if rets and multiagent: + for key in rets.keys(): + rets[key].append(ret[key]) + # Store the information from the run in info_dict. outflow = self.env.k.vehicle.get_outflow_rate(int(500)) - info_dict["returns"].append(ret) + if not multiagent: + info_dict["returns"].append(ret) info_dict["velocities"].append(np.mean(vel)) info_dict["outflows"].append(outflow) + info_dict["avg_trip_energy"].append(np.mean(list(completed_vehicle_avg_energy.values()))) + info_dict["avg_trip_time"].append(np.mean(list(completed_vehicle_travel_time.values()))) + info_dict["total_completed_trips"].append(len(list(completed_vehicle_avg_energy.values()))) for key in custom_vals.keys(): info_dict[key].append(np.mean(custom_vals[key])) - print("Round {0}, return: {1}".format(i, ret)) + if rets and multiagent: + for agent_id, rew in rets.items(): + print('Round {}, Return: {} for agent {}'.format( + i, ret, agent_id)) + elif not multiagent: + print('Round {}, Return: {}'.format(i, ret)) + + # Save emission data at the end of every rollout. This is skipped + # by the internal method if no emission path was specified. + if self.env.simulator == "traci": + emission_files.append(self.env.k.simulation.save_emission(run_id=i)) # Print the averages/std for all variables in the info_dict. for key in info_dict.keys(): @@ -179,20 +327,39 @@ def rl_actions(*_): print("steps/second:", np.mean(times)) self.env.terminate() - if convert_to_csv and self.env.simulator == "traci": - # wait a short period of time to ensure the xml file is readable - time.sleep(0.1) - - # collect the location of the emission file - dir_path = self.env.sim_params.emission_path - emission_filename = \ - "{0}-emission.xml".format(self.env.network.name) - emission_path = os.path.join(dir_path, emission_filename) - - # convert the emission file into a csv - emission_to_csv(emission_path) - - # Delete the .xml version of the emission file. - os.remove(emission_path) + if to_aws: + write_dict_to_csv(metadata_table_path, metadata, True) + generate_trajectory_table(emission_files, trajectory_table_path, source_id) + tsd_main( + emission_files[0], + { + 'network': self.env.network.__class__, + 'env': self.env.env_params, + 'sim': self.env.sim_params + }, + min_speed=0, + max_speed=10 + ) + upload_to_s3( + 'circles.data.pipeline', + 'metadata_table/date={0}/partition_name={1}_METADATA/' + '{1}_METADATA.csv'.format(cur_date, source_id), + metadata_table_path + ) + upload_to_s3( + 'circles.data.pipeline', + 'fact_vehicle_trace/date={0}/partition_name={1}/' + '{1}.csv'.format(cur_date, source_id), + trajectory_table_path, + {'network': metadata['network'][0], + 'is_baseline': metadata['is_baseline'][0]} + ) + upload_to_s3( + 'circles.data.pipeline', + 'time_space_diagram/date={0}/partition_name={1}/' + '{1}.png'.format(cur_date, source_id), + emission_files[0].replace('csv', 'png') + ) + os.remove(trajectory_table_path) return info_dict diff --git a/flow/core/kernel/network/aimsun.py b/flow/core/kernel/network/aimsun.py index 0378d45a9..89971bf58 100644 --- a/flow/core/kernel/network/aimsun.py +++ b/flow/core/kernel/network/aimsun.py @@ -262,7 +262,9 @@ def close(self): cur_dir = os.path.join(config.PROJECT_PATH, 'flow/core/kernel/network') os.remove(os.path.join(cur_dir, 'data_%s.json' % self.sim_params.port)) - os.remove('%s_%s' % (self.network.net_params.template, self.sim_params.port)) + if self.network.net_params.template is not None: + os.remove('%s_%s' % (self.network.net_params.template, + self.sim_params.port)) ########################################################################### # State acquisition methods # diff --git a/flow/core/kernel/network/traci.py b/flow/core/kernel/network/traci.py index 0921e2272..f22550645 100644 --- a/flow/core/kernel/network/traci.py +++ b/flow/core/kernel/network/traci.py @@ -47,7 +47,7 @@ def __init__(self, master_kernel, sim_params): master_kernel : flow.core.kernel.Kernel the higher level kernel (used to call methods from other sub-kernels) - sim_params : flow.core.params.SimParams + sim_params : flow.core.params.SumoParams simulation-specific parameters """ super(TraCIKernelNetwork, self).__init__(master_kernel, sim_params) @@ -849,6 +849,11 @@ def generate_cfg(self, net_params, traffic_lights, routes): # check collisions at intersections cfg.append(E("collision.check-junctions", value="true")) + # disable all collisions and teleporting in the simulation. + if self.sim_params.disable_collisions: + cfg.append(E("collision.mingap-factor", value="0")) + cfg.append(E("collision.action", value=str("none"))) + printxml(cfg, self.cfg_path + self.sumfn) return self.sumfn diff --git a/flow/core/kernel/simulation/traci.py b/flow/core/kernel/simulation/traci.py index e2501bb3a..5d28b134e 100644 --- a/flow/core/kernel/simulation/traci.py +++ b/flow/core/kernel/simulation/traci.py @@ -1,16 +1,18 @@ """Script containing the TraCI simulation kernel class.""" -from flow.core.kernel.simulation import KernelSimulation -import flow.config as config -import traci.constants as tc -import traci import traceback import os import time import logging import subprocess import signal +import csv +import flow.config as config +from flow.core.kernel.simulation import KernelSimulation +from flow.core.util import ensure_dir +import traci.constants as tc +import traci # Number of retries on restarting SUMO before giving up RETRIES_ON_ERROR = 10 @@ -20,6 +22,32 @@ class TraCISimulation(KernelSimulation): """Sumo simulation kernel. Extends flow.core.kernel.simulation.KernelSimulation + + Attributes + ---------- + sumo_proc : subprocess.Popen + contains the subprocess.Popen instance used to start traci + sim_step : float + seconds per simulation step + emission_path : str or None + Path to the folder in which to create the emissions output. Emissions + output is not generated if this value is not specified + time : float + used to internally keep track of the simulation time + stored_data : dict >> + a dict object used to store additional data if an emission file is + provided. The first key corresponds to the name of the vehicle, the + second corresponds to the time the sample was issued, and the final + keys represent the additional data stored at every given time for every + vehicle, and consists of the following keys: + + * acceleration (no noise): the accelerations issued to the vehicle, + excluding noise + * acceleration (requested): the requested acceleration by the vehicle, + including noise + * acceleration (actual): the actual acceleration by the vehicle, + collected by computing the difference between the speeds of the + vehicle and dividing it by the sim_step term """ def __init__(self, master_kernel): @@ -32,8 +60,12 @@ def __init__(self, master_kernel): sub-kernels) """ KernelSimulation.__init__(self, master_kernel) - # contains the subprocess.Popen instance used to start traci + self.sumo_proc = None + self.sim_step = None + self.emission_path = None + self.time = 0 + self.stored_data = dict() def pass_api(self, kernel_api): """See parent class. @@ -45,9 +77,14 @@ def pass_api(self, kernel_api): # subscribe some simulation parameters needed to check for entering, # exiting, and colliding vehicles self.kernel_api.simulation.subscribe([ - tc.VAR_DEPARTED_VEHICLES_IDS, tc.VAR_ARRIVED_VEHICLES_IDS, - tc.VAR_TELEPORT_STARTING_VEHICLES_IDS, tc.VAR_TIME_STEP, - tc.VAR_DELTA_T + tc.VAR_DEPARTED_VEHICLES_IDS, + tc.VAR_ARRIVED_VEHICLES_IDS, + tc.VAR_TELEPORT_STARTING_VEHICLES_IDS, + tc.VAR_TIME_STEP, + tc.VAR_DELTA_T, + tc.VAR_LOADED_VEHICLES_NUMBER, + tc.VAR_DEPARTED_VEHICLES_NUMBER, + tc.VAR_ARRIVED_VEHICLES_NUMBER ]) def simulation_step(self): @@ -56,10 +93,61 @@ def simulation_step(self): def update(self, reset): """See parent class.""" - pass + if reset: + self.time = 0 + else: + self.time += self.sim_step + + # Collect the additional data to store in the emission file. + if self.emission_path is not None: + kv = self.master_kernel.vehicle + for veh_id in self.master_kernel.vehicle.get_ids(): + t = round(self.time, 2) + + # some miscellaneous pre-processing + position = kv.get_2d_position(veh_id) + + # Make sure dictionaries corresponding to the vehicle and + # time are available. + if veh_id not in self.stored_data.keys(): + self.stored_data[veh_id] = dict() + if t not in self.stored_data[veh_id].keys(): + self.stored_data[veh_id][t] = dict() + + # Add the speed, position, and lane data. + self.stored_data[veh_id][t].update({ + "speed": kv.get_speed(veh_id), + "lane_number": kv.get_lane(veh_id), + "edge_id": kv.get_edge(veh_id), + "relative_position": kv.get_position(veh_id), + "x": position[0], + "y": position[1], + "headway": kv.get_headway(veh_id), + "leader_id": kv.get_leader(veh_id), + "follower_id": kv.get_follower(veh_id), + "leader_rel_speed": + kv.get_speed(kv.get_leader(veh_id)) + - kv.get_speed(veh_id), + "target_accel_with_noise_with_failsafe": + kv.get_accel(veh_id, noise=True, failsafe=True), + "target_accel_no_noise_no_failsafe": + kv.get_accel(veh_id, noise=False, failsafe=False), + "target_accel_with_noise_no_failsafe": + kv.get_accel(veh_id, noise=True, failsafe=False), + "target_accel_no_noise_with_failsafe": + kv.get_accel(veh_id, noise=False, failsafe=True), + "realized_accel": + kv.get_realized_accel(veh_id), + "road_grade": kv.get_road_grade(veh_id), + "distance": kv.get_distance(veh_id), + }) def close(self): """See parent class.""" + # Save the emission data to a csv. + if self.emission_path is not None: + self.save_emission() + self.kernel_api.close() def check_collision(self): @@ -69,10 +157,24 @@ def check_collision(self): def start_simulation(self, network, sim_params): """Start a sumo simulation instance. - This method uses the configuration files created by the network class - to initialize a sumo instance. Also initializes a traci connection to - interface with sumo from Python. + This method performs the following operations: + + 1. It collect the simulation step size and the emission path + information. If an emission path is specifies, it ensures that the + path exists. + 2. It also uses the configuration files created by the network class to + initialize a sumo instance. + 3. Finally, It initializes a traci connection to interface with sumo + from Python and returns the connection. """ + # Save the simulation step size (for later use). + self.sim_step = sim_params.sim_step + + # Update the emission path term. + self.emission_path = sim_params.emission_path + if self.emission_path is not None: + ensure_dir(self.emission_path) + error = None for _ in range(RETRIES_ON_ERROR): try: @@ -94,12 +196,10 @@ def start_simulation(self, network, sim_params): if sim_params.num_clients > 1: logging.info(" Num clients are" + str(sim_params.num_clients)) + logging.debug(" Emission file: " + str(self.emission_path)) logging.debug(" Step length: " + str(sim_params.sim_step)) - try: - use_libsumo = sim_params.use_libsumo - except AttributeError: - use_libsumo = False + use_libsumo = getattr(sim_params, "use_libsumo", False) if sim_params.render or not use_libsumo: # Opening the I/O thread to SUMO @@ -141,3 +241,74 @@ def teardown_sumo(self): os.killpg(self.sumo_proc.pid, signal.SIGTERM) except Exception as e: print("Error during teardown: {}".format(e)) + + def save_emission(self, run_id=0): + """Save any collected emission data to a csv file. + + If no data was collected, nothing happens. Moreover, any internally + stored data by this class is cleared whenever data is stored. + + Parameters + ---------- + run_id : int + the rollout number, appended to the name of the emission file. Used + to store emission files from multiple rollouts run sequentially. + + Returns + ------- + emission_file_path: str + the relative path of the emission file + """ + # If there is no stored data, ignore this operation. This is to ensure + # that data isn't deleted if the operation is called twice. + if len(self.stored_data) == 0: + return + + # Get a csv name for the emission file. + name = "{}-{}_emission.csv".format( + self.master_kernel.network.network.name, run_id) + + # The name of all stored data-points (excluding id and time) + stored_ids = [ + "x", + "y", + "speed", + "headway", + "leader_id", + "follower_id", + "leader_rel_speed", + "target_accel_with_noise_with_failsafe", + "target_accel_no_noise_no_failsafe", + "target_accel_with_noise_no_failsafe", + "target_accel_no_noise_with_failsafe", + "realized_accel", + "road_grade", + "edge_id", + "lane_number", + "distance", + "relative_position", + ] + + # Update the stored data to push to the csv file. + final_data = {"time": [], "id": []} + final_data.update({key: [] for key in stored_ids}) + + for veh_id in self.stored_data.keys(): + for t in self.stored_data[veh_id].keys(): + final_data['time'].append(t) + final_data['id'].append(veh_id) + for key in stored_ids: + final_data[key].append(self.stored_data[veh_id][t][key]) + + emission_file_path = os.path.join(self.emission_path, name) + with open(emission_file_path, "w") as f: + print(emission_file_path, self.emission_path) + writer = csv.writer(f, delimiter=',') + writer.writerow(final_data.keys()) + writer.writerows(zip(*final_data.values())) + + # Clear all memory from the stored data. This is useful if this + # function is called in between resets. + self.stored_data.clear() + + return emission_file_path diff --git a/flow/core/kernel/vehicle/aimsun.py b/flow/core/kernel/vehicle/aimsun.py index 3320d1515..16c94558a 100644 --- a/flow/core/kernel/vehicle/aimsun.py +++ b/flow/core/kernel/vehicle/aimsun.py @@ -65,6 +65,7 @@ def __init__(self, # number of vehicles to exit the network for every time-step self._num_arrived = [] self._arrived_ids = [] + self._arrived_rl_ids = [] # contains conversion from Flow-ID to Aimsun-ID self._id_aimsun2flow = {} @@ -174,11 +175,17 @@ def update(self, reset): added_vehicles = self.kernel_api.get_entered_ids() exited_vehicles = self.kernel_api.get_exited_ids() + # keep track of arrived rl vehicles + arrived_rl_ids = [] + # add the new vehicles if they should be tracked for aimsun_id in added_vehicles: veh_type = self.kernel_api.get_vehicle_type_name(aimsun_id) if veh_type in self.tracked_vehicle_types: self._add_departed(aimsun_id) + if aimsun_id in self.get_rl_ids(): + arrived_rl_ids.append(aimsun_id) + self._arrived_rl_ids.append(arrived_rl_ids) # remove the exited vehicles if they were tracked if not reset: @@ -403,7 +410,7 @@ def add(self, veh_id, type_id, edge, pos, lane, speed): def reset(self): """See parent class.""" - pass + raise NotImplementedError def remove(self, aimsun_id): """See parent class.""" @@ -517,7 +524,7 @@ def choose_routes(self, veh_id, route_choices): edge the vehicle is currently on. If a value of None is provided, the vehicle does not update its route """ - pass # FIXME + raise NotImplementedError # FIXME # for i, veh_id in enumerate(veh_ids): # if route_choices[i] is not None: # aimsun_id = self._id_flow2aimsun[veh_id] @@ -525,6 +532,10 @@ def choose_routes(self, veh_id, route_choices): # self.kernel_api.AKIVehTrackedModifyNextSections( # aimsun_id, size_next_sections, route_choices[i]) + def set_max_speed(self, veh_id, max_speed): + """See parent class.""" + raise NotImplementedError + ########################################################################### # Methods to visually distinguish vehicles by {RL, observed, unobserved} # ########################################################################### @@ -560,10 +571,30 @@ def get_observed_ids(self): """Return the list of observed vehicles.""" return self.__observed_ids + def get_color(self, veh_id): + """See parent class.""" + raise NotImplementedError + + def set_color(self, veh_id, color): + """See parent class.""" + raise NotImplementedError + ########################################################################### # State acquisition methods # ########################################################################### + def get_orientation(self, veh_id): + """See parent class.""" + raise NotImplementedError + + def get_timestep(self, veh_id): + """See parent class.""" + raise NotImplementedError + + def get_timedelta(self, veh_id): + """See parent class.""" + raise NotImplementedError + def get_ids(self): """See parent class.""" return self.__ids @@ -611,6 +642,32 @@ def get_num_arrived(self): else: return 0 + def get_arrived_ids(self): + """See parent class.""" + raise NotImplementedError + + def get_arrived_rl_ids(self, k=1): + """See parent class.""" + if len(self._arrived_rl_ids) > 0: + arrived = [] + for arr in self._arrived_rl_ids[-k:]: + arrived.extend(arr) + return arrived + else: + return 0 + + def get_departed_ids(self): + """See parent class.""" + raise NotImplementedError + + def get_num_not_departed(self): + """See parent class.""" + raise NotImplementedError + + def get_fuel_consumption(self): + """See parent class.""" + raise NotImplementedError + def get_type(self, veh_id): """See parent class.""" if isinstance(veh_id, (list, np.ndarray)): @@ -627,6 +684,10 @@ def get_speed(self, veh_id, error=-1001): return [self.get_speed(veh, error) for veh in veh_id] return self.__vehicles[veh_id]['tracking_info'].CurrentSpeed / 3.6 + def get_default_speed(self, veh_id, error=-1001): + """See parent class.""" + raise NotImplementedError + def get_position(self, veh_id, error=-1001): """See parent class.""" if isinstance(veh_id, (list, np.ndarray)): @@ -806,3 +867,7 @@ def get_lane_followers_speed(self, veh_id, error=None): def get_lane_leaders_speed(self, veh_id, error=None): """See parent class.""" raise NotImplementedError + + def get_max_speed(self, veh_id, error): + """See parent class.""" + raise NotImplementedError diff --git a/flow/core/kernel/vehicle/base.py b/flow/core/kernel/vehicle/base.py index 072a37eab..5b4ba0622 100644 --- a/flow/core/kernel/vehicle/base.py +++ b/flow/core/kernel/vehicle/base.py @@ -1,7 +1,9 @@ """Script containing the base vehicle kernel class.""" +from abc import ABCMeta, abstractmethod -class KernelVehicle(object): + +class KernelVehicle(object, metaclass=ABCMeta): """Flow vehicle kernel. This kernel sub-class is used to interact with the simulator with regards @@ -66,6 +68,7 @@ def pass_api(self, kernel_api): # Methods for interacting with the simulator # ########################################################################### + @abstractmethod def update(self, reset): """Update the vehicle kernel with data from the current time step. @@ -78,8 +81,9 @@ def update(self, reset): specifies whether the simulator was reset in the last simulation step """ - raise NotImplementedError + pass + @abstractmethod def add(self, veh_id, type_id, edge, pos, lane, speed): """Add a vehicle to the network. @@ -98,12 +102,14 @@ def add(self, veh_id, type_id, edge, pos, lane, speed): speed : float starting speed of the added vehicle """ - raise NotImplementedError + pass + @abstractmethod def reset(self): """Reset any additional state that needs to be reset.""" - raise NotImplementedError + pass + @abstractmethod def remove(self, veh_id): """Remove a vehicle. @@ -119,20 +125,29 @@ def remove(self, veh_id): veh_id : str unique identifier of the vehicle to be removed """ - raise NotImplementedError + pass - def apply_acceleration(self, veh_id, acc): + @abstractmethod + def apply_acceleration(self, veh_id, acc, smooth_duration=0): """Apply the acceleration requested by a vehicle in the simulator. + In SUMO, this function applies setSpeed for smooth_duration=0, otherwise + the slowDown method applies acceleration smoothly over the smooth_duration + time (in seconds). For more information, see: + https://sumo.dlr.de/pydoc/traci._vehicle.html#VehicleDomain-slowDown + Parameters ---------- veh_id : str or list of str list of vehicle identifiers acc : float or array_like requested accelerations from the vehicles + smooth_duration : float + duration in seconds over which acceleration should be smoothly applied, default: 0 """ - raise NotImplementedError + pass + @abstractmethod def apply_lane_change(self, veh_id, direction): """Apply an instantaneous lane-change to a set of vehicles. @@ -155,8 +170,9 @@ def apply_lane_change(self, veh_id, direction): ValueError If any of the direction values are not -1, 0, or 1. """ - raise NotImplementedError + pass + @abstractmethod def choose_routes(self, veh_id, route_choices): """Update the route choice of vehicles in the network. @@ -169,8 +185,9 @@ def choose_routes(self, veh_id, route_choices): edge the vehicle is currently on. If a value of None is provided, the vehicle does not update its route """ - raise NotImplementedError + pass + @abstractmethod def set_max_speed(self, veh_id, max_speed): """Update the maximum allowable speed by a vehicles in the network. @@ -181,115 +198,179 @@ def set_max_speed(self, veh_id, max_speed): max_speed : float desired max speed by the vehicle """ - raise NotImplementedError + pass ########################################################################### # Methods to visually distinguish vehicles by {RL, observed, unobserved} # ########################################################################### + @abstractmethod def update_vehicle_colors(self): """Modify the color of vehicles if rendering is active.""" - raise NotImplementedError + pass + @abstractmethod def set_observed(self, veh_id): """Add a vehicle to the list of observed vehicles.""" - raise NotImplementedError + pass + @abstractmethod def remove_observed(self, veh_id): """Remove a vehicle from the list of observed vehicles.""" - raise NotImplementedError + pass + @abstractmethod def get_observed_ids(self): """Return the list of observed vehicles.""" - raise NotImplementedError + pass + @abstractmethod def get_color(self, veh_id): """Return and RGB tuple of the color of the specified vehicle.""" - raise NotImplementedError + pass + @abstractmethod def set_color(self, veh_id, color): """Set the color of the specified vehicle with the RGB tuple.""" - raise NotImplementedError + pass ########################################################################### # State acquisition methods # ########################################################################### + @abstractmethod def get_orientation(self, veh_id): """Return the orientation of the vehicle of veh_id.""" - raise NotImplementedError + pass + @abstractmethod def get_timestep(self, veh_id): """Return the time step of the vehicle of veh_id.""" - raise NotImplementedError + pass + @abstractmethod def get_timedelta(self, veh_id): """Return the simulation time delta of the vehicle of veh_id.""" - raise NotImplementedError + pass + @abstractmethod def get_type(self, veh_id): """Return the type of the vehicle of veh_id.""" - raise NotImplementedError + pass + @abstractmethod def get_ids(self): """Return the names of all vehicles currently in the network.""" - raise NotImplementedError + pass + @abstractmethod def get_human_ids(self): """Return the names of all non-rl vehicles currently in the network.""" - raise NotImplementedError + pass + @abstractmethod def get_controlled_ids(self): """Return the names of all flow acceleration-controlled vehicles. This only include vehicles that are currently in the network. """ - raise NotImplementedError + pass + @abstractmethod def get_controlled_lc_ids(self): """Return the names of all flow lane change-controlled vehicles. This only include vehicles that are currently in the network. """ - raise NotImplementedError + pass + @abstractmethod def get_rl_ids(self): """Return the names of all rl-controlled vehicles in the network.""" - raise NotImplementedError + pass + @abstractmethod def get_ids_by_edge(self, edges): """Return the names of all vehicles in the specified edge. If no vehicles are currently in the edge, then returns an empty list. """ - raise NotImplementedError + pass + @abstractmethod def get_inflow_rate(self, time_span): """Return the inflow rate (in veh/hr) of vehicles from the network. This value is computed over the specified **time_span** seconds. """ - raise NotImplementedError + pass + @abstractmethod def get_outflow_rate(self, time_span): """Return the outflow rate (in veh/hr) of vehicles from the network. This value is computed over the specified **time_span** seconds. """ - raise NotImplementedError + pass + @abstractmethod def get_num_arrived(self): """Return the number of vehicles that arrived in the last time step.""" - raise NotImplementedError + pass + @abstractmethod def get_arrived_ids(self): """Return the ids of vehicles that arrived in the last time step.""" - raise NotImplementedError + pass + @abstractmethod def get_departed_ids(self): """Return the ids of vehicles that departed in the last time step.""" - raise NotImplementedError + pass + + @abstractmethod + def get_num_not_departed(self): + """Return the number of vehicles not departed in the last time step. + + This includes vehicles that were loaded but not departed. + """ + pass + + @abstractmethod + def get_fuel_consumption(self, veh_id, error=-1001): + """Return the mpg / s of the specified vehicle. + + Parameters + ---------- + veh_id : str or list of str + vehicle id, or list of vehicle ids + error : any, optional + value that is returned if the vehicle is not found + + Returns + ------- + float + """ + pass + + @abstractmethod + def get_energy_model(self, veh_id, error=""): + """Return the energy model class object of the specified vehicle. + + Parameters + ---------- + veh_id : str or list of str + vehicle id, or list of vehicle ids + error : str + value that is returned if the vehicle is not found + + Returns + ------- + subclass of BaseEnergyModel + """ + pass + @abstractmethod def get_speed(self, veh_id, error=-1001): """Return the speed of the specified vehicle. @@ -304,8 +385,26 @@ def get_speed(self, veh_id, error=-1001): ------- float """ - raise NotImplementedError + pass + @abstractmethod + def get_default_speed(self, veh_id, error=-1001): + """Return the expected speed if no control were applied. + + Parameters + ---------- + veh_id : str or list of str + vehicle id, or list of vehicle ids + error : any, optional + value that is returned if the vehicle is not found + + Returns + ------- + float + """ + pass + + @abstractmethod def get_position(self, veh_id, error=-1001): """Return the position of the vehicle relative to its current edge. @@ -320,8 +419,9 @@ def get_position(self, veh_id, error=-1001): ------- float """ - raise NotImplementedError + pass + @abstractmethod def get_edge(self, veh_id, error=""): """Return the edge the specified vehicle is currently on. @@ -336,8 +436,9 @@ def get_edge(self, veh_id, error=""): ------- str """ - raise NotImplementedError + pass + @abstractmethod def get_lane(self, veh_id, error=-1001): """Return the lane index of the specified vehicle. @@ -352,8 +453,9 @@ def get_lane(self, veh_id, error=-1001): ------- int """ - raise NotImplementedError + pass + @abstractmethod def get_route(self, veh_id, error=list()): """Return the route of the specified vehicle. @@ -368,8 +470,9 @@ def get_route(self, veh_id, error=list()): ------- list of str """ - raise NotImplementedError + pass + @abstractmethod def get_length(self, veh_id, error=-1001): """Return the length of the specified vehicle. @@ -384,8 +487,9 @@ def get_length(self, veh_id, error=-1001): ------- float """ - raise NotImplementedError + pass + @abstractmethod def get_leader(self, veh_id, error=""): """Return the leader of the specified vehicle. @@ -400,8 +504,9 @@ def get_leader(self, veh_id, error=""): ------- str """ - raise NotImplementedError + pass + @abstractmethod def get_follower(self, veh_id, error=""): """Return the follower of the specified vehicle. @@ -416,8 +521,9 @@ def get_follower(self, veh_id, error=""): ------- str """ - raise NotImplementedError + pass + @abstractmethod def get_headway(self, veh_id, error=-1001): """Return the headway of the specified vehicle(s). @@ -432,8 +538,9 @@ def get_headway(self, veh_id, error=-1001): ------- float """ - raise NotImplementedError + pass + @abstractmethod def get_last_lc(self, veh_id, error=-1001): """Return the last time step a vehicle changed lanes. @@ -452,8 +559,9 @@ def get_last_lc(self, veh_id, error=-1001): ------- int """ - raise NotImplementedError + pass + @abstractmethod def get_acc_controller(self, veh_id, error=None): """Return the acceleration controller of the specified vehicle. @@ -468,8 +576,9 @@ def get_acc_controller(self, veh_id, error=None): ------- object """ - raise NotImplementedError + pass + @abstractmethod def get_lane_changing_controller(self, veh_id, error=None): """Return the lane changing controller of the specified vehicle. @@ -484,8 +593,9 @@ def get_lane_changing_controller(self, veh_id, error=None): ------- object """ - raise NotImplementedError + pass + @abstractmethod def get_routing_controller(self, veh_id, error=None): """Return the routing controller of the specified vehicle. @@ -500,8 +610,9 @@ def get_routing_controller(self, veh_id, error=None): ------- object """ - raise NotImplementedError + pass + @abstractmethod def get_lane_headways(self, veh_id, error=list()): """Return the lane headways of the specified vehicles. @@ -519,8 +630,9 @@ def get_lane_headways(self, veh_id, error=list()): ------- list of float """ - raise NotImplementedError + pass + @abstractmethod def get_lane_leaders_speed(self, veh_id, error=list()): """Return the speed of the leaders of the specified vehicles. @@ -540,8 +652,9 @@ def get_lane_leaders_speed(self, veh_id, error=list()): ------- list of float """ - raise NotImplementedError + pass + @abstractmethod def get_lane_followers_speed(self, veh_id, error=list()): """Return the speed of the followers of the specified vehicles. @@ -561,8 +674,9 @@ def get_lane_followers_speed(self, veh_id, error=list()): ------- list of float """ - raise NotImplementedError + pass + @abstractmethod def get_lane_leaders(self, veh_id, error=list()): """Return the leaders for the specified vehicle in all lanes. @@ -577,8 +691,9 @@ def get_lane_leaders(self, veh_id, error=list()): ------- lis of str """ - raise NotImplementedError + pass + @abstractmethod def get_lane_tailways(self, veh_id, error=list()): """Return the lane tailways of the specified vehicle. @@ -596,8 +711,9 @@ def get_lane_tailways(self, veh_id, error=list()): ------- list of float """ - raise NotImplementedError + pass + @abstractmethod def get_lane_followers(self, veh_id, error=list()): """Return the followers for the specified vehicle in all lanes. @@ -612,8 +728,9 @@ def get_lane_followers(self, veh_id, error=list()): ------- list of str """ - raise NotImplementedError + pass + @abstractmethod def get_x_by_id(self, veh_id): """Provide a 1-D representation of the position of a vehicle. @@ -630,8 +747,9 @@ def get_x_by_id(self, veh_id): ------- float """ - raise NotImplementedError + pass + @abstractmethod def get_max_speed(self, veh_id, error): """Return the max speed of the specified vehicle. @@ -646,4 +764,33 @@ def get_max_speed(self, veh_id, error): ------- float """ - raise NotImplementedError + pass + + ########################################################################### + # Methods for Datapipeline # + ########################################################################### + + @abstractmethod + def get_accel(self, veh_id, noise=True, failsafe=True): + """Return the acceleration of vehicle with veh_id.""" + pass + + @abstractmethod + def update_accel(self, veh_id, accel, noise=True, failsafe=True): + """Update stored acceleration of vehicle with veh_id.""" + pass + + @abstractmethod + def get_2d_position(self, veh_id, error=-1001): + """Return (x, y) position of vehicle with veh_id.""" + pass + + @abstractmethod + def get_realized_accel(self, veh_id): + """Return the acceleration that the vehicle actually make.""" + pass + + @abstractmethod + def get_road_grade(self, veh_id): + """Return the road-grade of the vehicle with veh_id.""" + pass diff --git a/flow/core/kernel/vehicle/traci.py b/flow/core/kernel/vehicle/traci.py index abd6cbc89..d3f6fbe80 100644 --- a/flow/core/kernel/vehicle/traci.py +++ b/flow/core/kernel/vehicle/traci.py @@ -10,9 +10,18 @@ from flow.controllers.car_following_models import SimCarFollowingController from flow.controllers.rlcontroller import RLController from flow.controllers.lane_change_controllers import SimLaneChangeController +from flow.controllers.lane_change_controllers import AILaneChangeController from bisect import bisect_left import itertools from copy import deepcopy +import platform +import os + +try: + import libsumo +except ImportError: + warnings.warn('You do not have libsumo installed. If you are trying to use libsumo for speed and it isn\'t ' + 'working, that\'s why.') # colors for vehicles WHITE = (255, 255, 255) @@ -22,7 +31,8 @@ STEPS = 10 rdelta = 255 / STEPS # smoothly go from red to green as the speed increases -color_bins = [[int(255 - rdelta * i), int(rdelta * i), 0] for i in range(STEPS + 1)] +color_bins = [[int(255 - rdelta * i), int(rdelta * i), 0] for i in + range(STEPS + 1)] class TraCIVehicle(KernelVehicle): @@ -57,6 +67,8 @@ def __init__(self, self.num_vehicles = 0 # number of rl vehicles in the network self.num_rl_vehicles = 0 + # number of vehicles loaded but not departed vehicles + self.num_not_departed = 0 # contains the parameters associated with each type of vehicle self.type_parameters = {} @@ -69,21 +81,24 @@ def __init__(self, # number of vehicles that entered the network for every time-step self._num_departed = [] - self._departed_ids = [] + self._departed_ids = 0 # number of vehicles to exit the network for every time-step self._num_arrived = [] - self._arrived_ids = [] + self._arrived_ids = 0 self._arrived_rl_ids = [] # checks whether the instance is being rendered self.render = sim_params.render # whether to use libsumo - try: - self.use_libsumo = sim_params.use_libsumo - except AttributeError: - self.use_libsumo = False + self.use_libsumo = getattr(sim_params, "use_libsumo", False) + if self.use_libsumo: + relese = float(os.popen('lsb_release -r -s').read()) + if platform.system() == 'Linux' and \ + relese not in [float(18.04), float(16.04)]: + warnings.warn('Libsumo currently supported only on Ubuntu ' + '18.04 and 16.04 !') # whether or not to automatically color vehicles try: @@ -95,6 +110,13 @@ def __init__(self, # old speeds used to compute accelerations self.previous_speeds = {} + # The time that previous speed is recorded, used to calculate + # realized_accel + self.previous_time = 0 + + # flag to collect lane leaders/followers/headways/tailways for all + self.collect_info_all = False + def initialize(self, vehicles): """Initialize vehicle state information. @@ -111,6 +133,7 @@ def initialize(self, vehicles): self.minGap = vehicles.minGap self.num_vehicles = 0 self.num_rl_vehicles = 0 + self.num_not_departed = 0 self.__vehicles.clear() for typ in vehicles.initial: @@ -197,14 +220,15 @@ def update(self, reset): self.prev_last_lc[veh_id] = -float("inf") self._num_departed.clear() self._num_arrived.clear() - self._departed_ids.clear() - self._arrived_ids.clear() + self._departed_ids = 0 + self._arrived_ids = 0 self._arrived_rl_ids.clear() + self.num_not_departed = 0 # add vehicles from a network template, if applicable if hasattr(self.master_kernel.network.network, "template_vehicles"): - for veh_id in self.master_kernel.network.network.\ + for veh_id in self.master_kernel.network.network. \ template_vehicles: vals = deepcopy(self.master_kernel.network.network. template_vehicles[veh_id]) @@ -223,11 +247,14 @@ def update(self, reset): self.__vehicles[veh_id]["last_lc"] = self.time_counter # updated the list of departed and arrived vehicles - self._num_departed.append( - len(sim_obs[tc.VAR_DEPARTED_VEHICLES_IDS])) - self._num_arrived.append(len(sim_obs[tc.VAR_ARRIVED_VEHICLES_IDS])) - self._departed_ids.append(sim_obs[tc.VAR_DEPARTED_VEHICLES_IDS]) - self._arrived_ids.append(sim_obs[tc.VAR_ARRIVED_VEHICLES_IDS]) + self._num_departed.append(sim_obs[tc.VAR_LOADED_VEHICLES_NUMBER]) + self._num_arrived.append(sim_obs[tc.VAR_ARRIVED_VEHICLES_NUMBER]) + self._departed_ids = sim_obs[tc.VAR_DEPARTED_VEHICLES_IDS] + self._arrived_ids = sim_obs[tc.VAR_ARRIVED_VEHICLES_IDS] + + # update the number of not departed vehicles + self.num_not_departed += sim_obs[tc.VAR_LOADED_VEHICLES_NUMBER] - \ + sim_obs[tc.VAR_DEPARTED_VEHICLES_NUMBER] # update the "headway", "leader", and "follower" variables for veh_id in self.__ids: @@ -289,6 +316,10 @@ def _add_departed(self, veh_id, veh_type): # specify the type self.__vehicles[veh_id]["type"] = veh_type + # specify energy model + self.__vehicles[veh_id]["energy_model"] = self.type_parameters[ + veh_type]["energy_model"]() + car_following_params = \ self.type_parameters[veh_type]["car_following_params"] @@ -306,6 +337,11 @@ def _add_departed(self, veh_id, veh_type): self.__vehicles[veh_id]["lane_changer"] = \ lc_controller[0](veh_id=veh_id, **lc_controller[1]) + # if lane changer is AILaneChangeController, set collect info flag True + if lc_controller[0] == AILaneChangeController and \ + not self.collect_info_all: + self.collect_info_all = True + # specify the routing controller class rt_controller = self.type_parameters[veh_type]["routing_controller"] if rt_controller is not None: @@ -318,7 +354,6 @@ def _add_departed(self, veh_id, veh_type): if accel_controller[0] == RLController: if veh_id not in self.__rl_ids: self.__rl_ids.append(veh_id) - self.num_rl_vehicles += 1 else: if veh_id not in self.__human_ids: self.__human_ids.append(veh_id) @@ -330,8 +365,16 @@ def _add_departed(self, veh_id, veh_type): # subscribe the new vehicle and get its subscription results if self.render or not self.use_libsumo: self.kernel_api.vehicle.subscribe(veh_id, [ - tc.VAR_LANE_INDEX, tc.VAR_LANEPOSITION, tc.VAR_ROAD_ID, - tc.VAR_SPEED, tc.VAR_EDGES, tc.VAR_POSITION, tc.VAR_ANGLE + tc.VAR_LANE_INDEX, + tc.VAR_LANEPOSITION, + tc.VAR_ROAD_ID, + tc.VAR_SPEED, + tc.VAR_EDGES, + tc.VAR_POSITION, + tc.VAR_ANGLE, + tc.VAR_SPEED_WITHOUT_TRACI, + tc.VAR_FUELCONSUMPTION, + tc.VAR_DISTANCE, ]) self.kernel_api.vehicle.subscribeLeader(veh_id, 2000) new_obs = self.kernel_api.vehicle.getSubscriptionResults(veh_id) @@ -369,9 +412,12 @@ def _add_departed(self, veh_id, veh_type): self.kernel_api.vehicle.getLaneIndex(veh_id) self.__sumo_obs[veh_id][tc.VAR_SPEED] = \ self.kernel_api.vehicle.getSpeed(veh_id) + self.__sumo_obs[veh_id][tc.VAR_FUELCONSUMPTION] = \ + self.kernel_api.vehicle.getFuelConsumption(veh_id) # make sure that the order of rl_ids is kept sorted self.__rl_ids.sort() + self.num_rl_vehicles = len(self.__rl_ids) return new_obs @@ -513,24 +559,42 @@ def get_num_arrived(self): def get_arrived_ids(self): """See parent class.""" - if len(self._arrived_ids) > 0: - return self._arrived_ids[-1] - else: - return 0 + return self._arrived_ids - def get_arrived_rl_ids(self): + def get_arrived_rl_ids(self, k=1): """See parent class.""" if len(self._arrived_rl_ids) > 0: - return self._arrived_rl_ids[-1] + arrived = [] + for arr in self._arrived_rl_ids[-k:]: + arrived.extend(arr) + return arrived else: return 0 def get_departed_ids(self): """See parent class.""" - if len(self._departed_ids) > 0: - return self._departed_ids[-1] - else: - return 0 + return self._departed_ids + + def get_num_not_departed(self): + """See parent class.""" + return self.num_not_departed + + def get_fuel_consumption(self, veh_id, error=-1001): + """Return fuel consumption in gallons/s.""" + ml_to_gallons = 0.000264172 + if isinstance(veh_id, (list, np.ndarray)): + return [self.get_fuel_consumption(vehID, error) for vehID in veh_id] + return self.__sumo_obs.get(veh_id, {}).get(tc.VAR_FUELCONSUMPTION, error) * ml_to_gallons + + def get_energy_model(self, veh_id, error=""): + """See parent class.""" + if isinstance(veh_id, (list, np.ndarray)): + return [self.get_energy_model(vehID) for vehID in veh_id] + try: + return self.__vehicles.get(veh_id, {'energy_model': error})['energy_model'] + except KeyError: + print("Energy model not specified for vehicle {}".format(veh_id)) + raise def get_previous_speed(self, veh_id, error=-1001): """See parent class.""" @@ -544,6 +608,13 @@ def get_speed(self, veh_id, error=-1001): return [self.get_speed(vehID, error) for vehID in veh_id] return self.__sumo_obs.get(veh_id, {}).get(tc.VAR_SPEED, error) + def get_default_speed(self, veh_id, error=-1001): + """See parent class.""" + if isinstance(veh_id, (list, np.ndarray)): + return [self.get_default_speed(vehID, error) for vehID in veh_id] + return self.__sumo_obs.get(veh_id, {}).get(tc.VAR_SPEED_WITHOUT_TRACI, + error) + def get_position(self, veh_id, error=-1001): """See parent class.""" if isinstance(veh_id, (list, np.ndarray)): @@ -703,8 +774,8 @@ def _multi_lane_headways(self): self.master_kernel.network.get_junction_list())) # maximum number of lanes in the network - max_lanes = max([self.master_kernel.network.num_lanes(edge_id) - for edge_id in tot_list]) + max_lanes = max(self.master_kernel.network.num_lanes(edge_id) + for edge_id in tot_list) # Key = edge id # Element = list, with the ith element containing tuples with the name @@ -729,7 +800,11 @@ def _multi_lane_headways(self): for lane in range(max_lanes): edge_dict[edge][lane].sort(key=lambda x: x[1]) - for veh_id in self.get_rl_ids(): + # get tracked vehicles IDs, for these vehicles info will be collected + tracked_vehs = self.get_ids() if self.collect_info_all \ + else self.get_rl_ids() # FIXME (yf) better collect tracked vehicles + + for veh_id in tracked_vehs: # collect the lane leaders, followers, headways, and tailways for # each vehicle edge = self.get_edge(veh_id) @@ -931,18 +1006,22 @@ def _prev_edge_followers(self, veh_id, edge_dict, lane, num_edges): return tailway, follower - def apply_acceleration(self, veh_ids, acc): + def apply_acceleration(self, veh_ids, acc, smooth_duration=0): """See parent class.""" - # to hand the case of a single vehicle + # to handle the case of a single vehicle if type(veh_ids) == str: veh_ids = [veh_ids] acc = [acc] for i, vid in enumerate(veh_ids): if acc[i] is not None and vid in self.get_ids(): + self.__vehicles[vid]["accel"] = acc[i] this_vel = self.get_speed(vid) next_vel = max([this_vel + acc[i] * self.sim_step, 0.]) - self.kernel_api.vehicle.slowDown(vid, next_vel, 1e-3) + if smooth_duration: + self.kernel_api.vehicle.slowDown(vid, next_vel, smooth_duration) + else: + self.kernel_api.vehicle.setSpeed(vid, next_vel) def apply_lane_change(self, veh_ids, direction): """See parent class.""" @@ -972,7 +1051,7 @@ def apply_lane_change(self, veh_ids, direction): # perform the requested lane action action in TraCI if target_lane != this_lane: self.kernel_api.vehicle.changeLane( - veh_id, int(target_lane), 100000) + veh_id, int(target_lane), self.sim_step) if veh_id in self.get_rl_ids(): self.prev_last_lc[veh_id] = \ @@ -991,6 +1070,8 @@ def choose_routes(self, veh_ids, route_choices): def get_x_by_id(self, veh_id): """See parent class.""" + if isinstance(veh_id, (list, np.ndarray)): + return [self.get_x_by_id(vehID) for vehID in veh_id] if self.get_edge(veh_id) == '': # occurs when a vehicle crashes is teleported for some other reason return 0. @@ -1008,7 +1089,8 @@ def update_vehicle_colors(self): for veh_id in self.get_rl_ids(): try: # If vehicle is already being colored via argument to vehicles.add(), don't re-color it. - if self._force_color_update or 'color' not in self.type_parameters[self.get_type(veh_id)]: + if self._force_color_update or 'color' not in \ + self.type_parameters[self.get_type(veh_id)]: # color rl vehicles red self.set_color(veh_id=veh_id, color=RED) except (FatalTraCIError, TraCIException) as e: @@ -1019,7 +1101,8 @@ def update_vehicle_colors(self): try: color = CYAN if veh_id in self.get_observed_ids() else WHITE # If vehicle is already being colored via argument to vehicles.add(), don't re-color it. - if self._force_color_update or 'color' not in self.type_parameters[self.get_type(veh_id)]: + if self._force_color_update or 'color' not in \ + self.type_parameters[self.get_type(veh_id)]: self.set_color(veh_id=veh_id, color=color) except (FatalTraCIError, TraCIException) as e: print('Error when updating human vehicle colors:', e) @@ -1029,7 +1112,8 @@ def update_vehicle_colors(self): if 'av' in veh_id: color = RED # If vehicle is already being colored via argument to vehicles.add(), don't re-color it. - if self._force_color_update or 'color' not in self.type_parameters[self.get_type(veh_id)]: + if self._force_color_update or 'color' not in \ + self.type_parameters[self.get_type(veh_id)]: self.set_color(veh_id=veh_id, color=color) except (FatalTraCIError, TraCIException) as e: print('Error when updating human vehicle colors:', e) @@ -1042,7 +1126,8 @@ def update_vehicle_colors(self): veh_speed = self.get_speed(veh_id) bin_index = np.digitize(veh_speed, speed_ranges) # If vehicle is already being colored via argument to vehicles.add(), don't re-color it. - if self._force_color_update or 'color' not in self.type_parameters[self.get_type(veh_id)]: + if self._force_color_update or 'color' not in \ + self.type_parameters[self.get_type(veh_id)]: self.set_color(veh_id=veh_id, color=color_bins[bin_index]) # clear the list of observed vehicles @@ -1098,7 +1183,6 @@ def set_max_speed(self, veh_id, max_speed): def _get_libsumo_subscription_results(self, veh_id): """Create a traci-style subscription result in the case of libsumo.""" - import libsumo as libsumo try: res = { tc.VAR_LANE_INDEX: @@ -1109,7 +1193,17 @@ def _get_libsumo_subscription_results(self, veh_id): tc.VAR_SPEED: self.kernel_api.vehicle.getSpeed(veh_id), tc.VAR_EDGES: self.kernel_api.vehicle.getRoute(veh_id), tc.VAR_LEADER: - self.kernel_api.vehicle.getLeader(veh_id, dist=2000) + self.kernel_api.vehicle.getLeader(veh_id, dist=2000), + tc.VAR_ANGLE: + self.kernel_api.vehicle.getAngle(veh_id), + tc.VAR_DISTANCE: + self.kernel_api.vehicle.getDistance(veh_id), + tc.VAR_FUELCONSUMPTION: + self.kernel_api.vehicle.getFuelConsumption(veh_id), + tc.VAR_POSITION: + self.kernel_api.vehicle.getPosition(veh_id), + tc.VAR_SPEED_WITHOUT_TRACI: + self.kernel_api.vehicle.getSpeedWithoutTraCI(veh_id), } except (TraCIException, FatalTraCIError, libsumo.TraCIException): # This is in case a vehicle exited the network and has not been @@ -1117,3 +1211,51 @@ def _get_libsumo_subscription_results(self, veh_id): res = None return res + + def get_accel(self, veh_id, noise=True, failsafe=True): + """See parent class.""" + metric_name = 'accel' + if noise: + metric_name += '_with_noise' + else: + metric_name += '_no_noise' + if failsafe: + metric_name += '_with_failsafe' + else: + metric_name += '_no_failsafe' + + return self.__vehicles[veh_id].get(metric_name, None) \ + or self.get_realized_accel(veh_id) + + def update_accel(self, veh_id, accel, noise=True, failsafe=True): + """See parent class.""" + metric_name = 'accel' + if noise: + metric_name += '_with_noise' + else: + metric_name += '_no_noise' + if failsafe: + metric_name += '_with_failsafe' + else: + metric_name += '_no_failsafe' + + self.__vehicles[veh_id][metric_name] = accel + + def get_realized_accel(self, veh_id): + """See parent class.""" + if self.get_distance(veh_id) == 0: + return 0 + return (self.get_speed(veh_id) - self.get_previous_speed(veh_id)) / self.sim_step + + def get_2d_position(self, veh_id, error=-1001): + """See parent class.""" + return self.__sumo_obs.get(veh_id, {}).get(tc.VAR_POSITION, error) + + def get_distance(self, veh_id, error=-1001): + """See parent class.""" + return self.__sumo_obs.get(veh_id, {}).get(tc.VAR_DISTANCE, error) + + def get_road_grade(self, veh_id): + """See parent class.""" + # TODO : Brent + return 0 diff --git a/flow/core/params.py b/flow/core/params.py index dd10b85ca..53ae68329 100755 --- a/flow/core/params.py +++ b/flow/core/params.py @@ -1,12 +1,18 @@ """Objects that define the various meta-parameters of an experiment.""" -import logging import collections +import logging +import sys +from sys import platform from flow.utils.flow_warnings import deprecated_attribute from flow.controllers.car_following_models import SimCarFollowingController from flow.controllers.rlcontroller import RLController from flow.controllers.lane_change_controllers import SimLaneChangeController +from flow.energy_models.toyota_energy import PriusEnergy +from flow.energy_models.toyota_energy import TacomaEnergy +from flow.energy_models.power_demand import PDMCombustionEngine +from flow.energy_models.power_demand import PDMElectric SPEED_MODES = { @@ -17,7 +23,30 @@ "all_checks": 31 } -LC_MODES = {"aggressive": 0, "no_lat_collide": 512, "strategic": 1621} +LC_MODES = { + "no_lc_safe": 512, + "no_lc_aggressive": 0, + "sumo_default": 1621, + "no_strategic_aggressive": 1108, + "no_strategic_safe": 1620, + "only_strategic_aggressive": 1, + "only_strategic_safe": 513, + "no_cooperative_aggressive": 1105, + "no_cooperative_safe": 1617, + "only_cooperative_aggressive": 4, + "only_cooperative_safe": 516, + "no_speed_gain_aggressive": 1093, + "no_speed_gain_safe": 1605, + "only_speed_gain_aggressive": 16, + "only_speed_gain_safe": 528, + "no_right_drive_aggressive": 1045, + "no_right_drive_safe": 1557, + "only_right_drive_aggressive": 64, + "only_right_drive_safe": 576 +} + +ENERGY_MODELS = set([PriusEnergy, TacomaEnergy, PDMCombustionEngine, PDMElectric]) +DEFAULT_ENERGY_MODEL = PDMCombustionEngine # Traffic light defaults PROGRAM_ID = 1 @@ -242,6 +271,7 @@ def add(self, num_vehicles=0, car_following_params=None, lane_change_params=None, + energy_model=DEFAULT_ENERGY_MODEL, color=None): """Add a sequence of vehicles to the list of vehicles in the network. @@ -278,6 +308,12 @@ def add(self, # FIXME: depends on simulator lane_change_params = SumoLaneChangeParams() + if energy_model not in ENERGY_MODELS: + print('{} for vehicle {} is not a valid energy model. Defaulting to {}\n'.format(energy_model, + veh_id, + DEFAULT_ENERGY_MODEL)) + energy_model = DEFAULT_ENERGY_MODEL + type_params = {} type_params.update(car_following_params.controller_params) type_params.update(lane_change_params.controller_params) @@ -291,7 +327,8 @@ def add(self, "routing_controller": routing_controller, "initial_speed": initial_speed, "car_following_params": car_following_params, - "lane_change_params": lane_change_params} + "lane_change_params": lane_change_params, + "energy_model": energy_model} if color: type_params['color'] = color @@ -314,7 +351,9 @@ def add(self, "car_following_params": car_following_params, "lane_change_params": - lane_change_params + lane_change_params, + "energy_model": + energy_model }) # This is used to return the actual headways from the vehicles class. @@ -396,7 +435,7 @@ class SimParams(object): def __init__(self, sim_step=0.1, render=False, - restart_instance=False, + restart_instance=True, emission_path=None, save_render=False, sight_radius=25, @@ -406,7 +445,7 @@ def __init__(self, """Instantiate SimParams.""" self.sim_step = sim_step self.render = render - self.restart_instance = restart_instance + self.restart_instance = True # restart_instance self.emission_path = emission_path self.save_render = save_render self.sight_radius = sight_radius @@ -490,6 +529,7 @@ def __init__(self, centroid_config_name=None, subnetwork_name=None): """Instantiate AimsunParams.""" + restart_instance = True super(AimsunParams, self).__init__( sim_step, render, restart_instance, emission_path, save_render, sight_radius, show_radius, pxpm) @@ -512,9 +552,9 @@ class SumoParams(SimParams): Attributes ---------- - libsumo : bool, optional + use_libsumo : bool, optional Whether to enable libsumo speedup. Note SUMO must be compiled with - libsumo for this to work. Is not currently supported on OSX machines. + libsumo for this to work. port : int, optional Port for Traci to connect to; finds an empty port by default sim_step : float optional @@ -571,6 +611,8 @@ class SumoParams(SimParams): current time step use_ballistic: bool, optional If true, use a ballistic integration step instead of an euler step + disable_collisions: bool, optional + If true, disables explicit collision checking and teleporting in SUMO """ def __init__(self, @@ -593,12 +635,16 @@ def __init__(self, teleport_time=-1, num_clients=1, color_by_speed=False, - use_ballistic=False): + use_ballistic=False, + disable_collisions=False): """Instantiate SumoParams.""" + restart_instance = True super(SumoParams, self).__init__( sim_step, render, restart_instance, emission_path, save_render, sight_radius, show_radius, pxpm, force_color_update) self.use_libsumo = use_libsumo + if platform == "darwin" and self.use_libsumo: + sys.exit("Libsumo does not currently work on macs") self.port = port self.lateral_resolution = lateral_resolution self.no_step_log = no_step_log @@ -609,6 +655,7 @@ def __init__(self, self.num_clients = num_clients self.color_by_speed = color_by_speed self.use_ballistic = use_ballistic + self.disable_collisions = disable_collisions class EnvParams: @@ -642,6 +689,9 @@ class EnvParams: specifies whether to clip actions from the policy by their range when they are inputted to the reward function. Note that the actions are still clipped before they are provided to `apply_rl_actions`. + done_at_exit : bool, optional + If true, done is returned as True when the vehicle exits. This is only + applied to multi-agent environments. """ def __init__(self, @@ -650,7 +700,8 @@ def __init__(self, warmup_steps=0, sims_per_step=1, evaluate=False, - clip_actions=True): + clip_actions=True, + done_at_exit=True): """Instantiate EnvParams.""" self.additional_params = \ additional_params if additional_params is not None else {} @@ -659,6 +710,7 @@ def __init__(self, self.sims_per_step = sims_per_step self.evaluate = evaluate self.clip_actions = clip_actions + self.done_at_exit = done_at_exit def get_additional_param(self, key): """Return a variable from additional_params.""" @@ -902,14 +954,71 @@ class SumoLaneChangeParams: ---------- lane_change_mode : str or int, optional may be one of the following: + * "no_lc_safe" (default): Disable all SUMO lane changing but still + handle safety checks (collision avoidance and safety-gap enforcement) + in the simulation. Binary is [001000000000] + * "no_lc_aggressive": SUMO lane changes are not executed, collision + avoidance and safety-gap enforcement are off. + Binary is [000000000000] + + * "sumo_default": Execute all changes requested by a custom controller + unless in conflict with TraCI. Binary is [011001010101]. + + * "no_strategic_aggressive": Execute all changes except strategic + (routing) lane changes unless in conflict with TraCI. Collision + avoidance and safety-gap enforcement are off. Binary is [010001010100] + * "no_strategic_safe": Execute all changes except strategic + (routing) lane changes unless in conflict with TraCI. Collision + avoidance and safety-gap enforcement are on. Binary is [011001010100] + * "only_strategic_aggressive": Execute only strategic (routing) lane + changes unless in conflict with TraCI. Collision avoidance and + safety-gap enforcement are off. Binary is [000000000001] + * "only_strategic_safe": Execute only strategic (routing) lane + changes unless in conflict with TraCI. Collision avoidance and + safety-gap enforcement are on. Binary is [001000000001] + + * "no_cooperative_aggressive": Execute all changes except cooperative + (change in order to allow others to change) lane changes unless in + conflict with TraCI. Collision avoidance and safety-gap enforcement + are off. Binary is [010001010001] + * "no_cooperative_safe": Execute all changes except cooperative + lane changes unless in conflict with TraCI. Collision avoidance and + safety-gap enforcement are on. Binary is [011001010001] + * "only_cooperative_aggressive": Execute only cooperative lane changes + unless in conflict with TraCI. Collision avoidance and safety-gap + enforcement are off. Binary is [000000000100] + * "only_cooperative_safe": Execute only cooperative lane changes + unless in conflict with TraCI. Collision avoidance and safety-gap + enforcement are on. Binary is [001000000100] + + * "no_speed_gain_aggressive": Execute all changes except speed gain (the + other lane allows for faster driving) lane changes unless in conflict + with TraCI. Collision avoidance and safety-gap enforcement are off. + Binary is [010001000101] + * "no_speed_gain_safe": Execute all changes except speed gain + lane changes unless in conflict with TraCI. Collision avoidance and + safety-gap enforcement are on. Binary is [011001000101] + * "only_speed_gain_aggressive": Execute only speed gain lane changes + unless in conflict with TraCI. Collision avoidance and safety-gap + enforcement are off. Binary is [000000010000] + * "only_speed_gain_safe": Execute only speed gain lane changes + unless in conflict with TraCI. Collision avoidance and safety-gap + enforcement are on. Binary is [001000010000] + + * "no_right_drive_aggressive": Execute all changes except right drive + (obligation to drive on the right) lane changes unless in conflict + with TraCI. Collision avoidance and safety-gap enforcement are off. + Binary is [010000010101] + * "no_right_drive_safe": Execute all changes except right drive + lane changes unless in conflict with TraCI. Collision avoidance and + safety-gap enforcement are on. Binary is [011000010101] + * "only_right_drive_aggressive": Execute only right drive lane changes + unless in conflict with TraCI. Collision avoidance and safety-gap + enforcement are off. Binary is [000001000000] + * "only_right_drive_safe": Execute only right drive lane changes + unless in conflict with TraCI. Collision avoidance and safety-gap + enforcement are on. Binary is [001001000000] - * "no_lat_collide" (default): Human cars will not make lane - changes, RL cars can lane change into any space, no matter how - likely it is to crash - * "strategic": Human cars make lane changes in accordance with SUMO - to provide speed boosts - * "aggressive": RL cars are not limited by sumo with regard to - their lane-change actions, and can crash longitudinally * int values may be used to define custom lane change modes for the given vehicles, specified at: http://sumo.dlr.de/wiki/TraCI/Change_Vehicle_State#lane_change_mode_.280xb6.29 @@ -948,7 +1057,7 @@ class SumoLaneChangeParams: """ def __init__(self, - lane_change_mode="no_lat_collide", + lane_change_mode="no_lc_safe", model="LC2013", lc_strategic=1.0, lc_cooperative=1.0, @@ -1056,7 +1165,7 @@ def __init__(self, elif not (isinstance(lane_change_mode, int) or isinstance(lane_change_mode, float)): logging.error("Setting lane change mode to default.") - lane_change_mode = LC_MODES["no_lat_collide"] + lane_change_mode = LC_MODES["no_lc_safe"] self.lane_change_mode = lane_change_mode diff --git a/flow/core/rewards.py b/flow/core/rewards.py index 6de472af2..20ed1c6a7 100755 --- a/flow/core/rewards.py +++ b/flow/core/rewards.py @@ -307,26 +307,94 @@ def punish_rl_lane_changes(env, penalty=1): def energy_consumption(env, gain=.001): + """Calculate power consumption for all vehicle. + + Assumes vehicle is an average sized vehicle. + The power calculated here is the lower bound of the actual power consumed + by a vehicle. + + Parameters + ---------- + env : flow.envs.Env + the environment variable, which contains information on the current + state of the system. + gain : float + scaling factor for the reward + """ + veh_ids = env.k.vehicle.get_ids() + return veh_energy_consumption(env, veh_ids, gain) + + +def veh_energy_consumption(env, veh_ids=None, gain=.001): """Calculate power consumption of a vehicle. Assumes vehicle is an average sized vehicle. The power calculated here is the lower bound of the actual power consumed by a vehicle. + + Parameters + ---------- + env : flow.envs.Env + the environment variable, which contains information on the current + state of the system. + veh_ids : [list] or str + list of veh_ids or single veh_id to compute the reward over + gain : float + scaling factor for the reward """ + if veh_ids is None: + veh_ids = env.k.vehicle.get_ids() + elif not isinstance(veh_ids, list): + veh_ids = [veh_ids] + power = 0 + for veh_id in veh_ids: + if veh_id not in env.k.vehicle.previous_speeds: + continue + energy_model = env.k.vehicle.get_energy_model(veh_id) + if energy_model != "": + speed = env.k.vehicle.get_speed(veh_id) + accel = env.k.vehicle.get_accel(veh_id, noise=False, failsafe=True) + grade = env.k.vehicle.get_road_grade(veh_id) + power += energy_model.get_instantaneous_power(accel, speed, grade) - M = 1200 # mass of average sized vehicle (kg) - g = 9.81 # gravitational acceleration (m/s^2) - Cr = 0.005 # rolling resistance coefficient - Ca = 0.3 # aerodynamic drag coefficient - rho = 1.225 # air density (kg/m^3) - A = 2.6 # vehicle cross sectional area (m^2) - for veh_id in env.k.vehicle.get_ids(): - speed = env.k.vehicle.get_speed(veh_id) - prev_speed = env.k.vehicle.get_previous_speed(veh_id) + return -gain * power - accel = abs(speed - prev_speed) / env.sim_step - power += M * speed * accel + M * g * Cr * speed + 0.5 * rho * A * Ca * speed ** 3 +def instantaneous_mpg(env, veh_ids=None, gain=.001): + """Calculate the instantaneous mpg for every simulation step specific to the vehicle type. - return -gain * power + Parameters + ---------- + env : flow.envs.Env + the environment variable, which contains information on the current + state of the system. + veh_ids : [list] or str + list of veh_ids or single veh_id to compute the reward over + gain : float + scaling factor for the reward + """ + if veh_ids is None: + veh_ids = env.k.vehicle.get_ids() + elif not isinstance(veh_ids, list): + veh_ids = [veh_ids] + + cumulative_gallons = 0 + cumulative_distance = 0 + for veh_id in veh_ids: + energy_model = env.k.vehicle.get_energy_model(veh_id) + if energy_model != "": + speed = env.k.vehicle.get_speed(veh_id) + accel = env.k.vehicle.get_accel(veh_id, noise=False, failsafe=True) + grade = env.k.vehicle.get_road_grade(veh_id) + gallons_per_hr = energy_model.get_instantaneous_fuel_consumption(accel, speed, grade) + if speed >= 0.0: + cumulative_gallons += gallons_per_hr + cumulative_distance += speed + + cumulative_gallons /= 3600.0 + cumulative_distance /= 1609.34 + # miles / gallon is (distance_dot * \delta t) / (gallons_dot * \delta t) + mpg = cumulative_distance / (cumulative_gallons + 1e-6) + + return mpg * gain diff --git a/flow/core/util.py b/flow/core/util.py index 1821a76a5..c0c31f811 100755 --- a/flow/core/util.py +++ b/flow/core/util.py @@ -4,7 +4,6 @@ import errno import os from lxml import etree -from xml.etree import ElementTree def makexml(name, nsl): @@ -47,42 +46,39 @@ def emission_to_csv(emission_path, output_path=None): path to the csv file that will be generated, default is the same directory as the emission file, with the same name """ - parser = etree.XMLParser(recover=True) - tree = ElementTree.parse(emission_path, parser=parser) - root = tree.getroot() - - # parse the xml data into a dict + context = etree.iterparse(emission_path, recover=True) out_data = [] - for time in root.findall('timestep'): - t = float(time.attrib['time']) - - for car in time: - out_data.append(dict()) - try: - out_data[-1]['time'] = t - out_data[-1]['CO'] = float(car.attrib['CO']) - out_data[-1]['y'] = float(car.attrib['y']) - out_data[-1]['CO2'] = float(car.attrib['CO2']) - out_data[-1]['electricity'] = float(car.attrib['electricity']) - out_data[-1]['type'] = car.attrib['type'] - out_data[-1]['id'] = car.attrib['id'] - out_data[-1]['eclass'] = car.attrib['eclass'] - out_data[-1]['waiting'] = float(car.attrib['waiting']) - out_data[-1]['NOx'] = float(car.attrib['NOx']) - out_data[-1]['fuel'] = float(car.attrib['fuel']) - out_data[-1]['HC'] = float(car.attrib['HC']) - out_data[-1]['x'] = float(car.attrib['x']) - out_data[-1]['route'] = car.attrib['route'] - out_data[-1]['relative_position'] = float(car.attrib['pos']) - out_data[-1]['noise'] = float(car.attrib['noise']) - out_data[-1]['angle'] = float(car.attrib['angle']) - out_data[-1]['PMx'] = float(car.attrib['PMx']) - out_data[-1]['speed'] = float(car.attrib['speed']) - out_data[-1]['edge_id'] = car.attrib['lane'].rpartition('_')[0] - out_data[-1]['lane_number'] = car.attrib['lane'].\ - rpartition('_')[-1] - except KeyError: - del out_data[-1] + for event, elem in context: + if elem.tag == "timestep": + t = float(elem.attrib['time']) + for car in elem: + out_data.append(dict()) + try: + out_data[-1]['time'] = t + out_data[-1]['CO'] = float(car.attrib['CO']) + out_data[-1]['y'] = float(car.attrib['y']) + out_data[-1]['CO2'] = float(car.attrib['CO2']) + out_data[-1]['electricity'] = float(car.attrib['electricity']) + out_data[-1]['type'] = car.attrib['type'] + out_data[-1]['id'] = car.attrib['id'] + out_data[-1]['eclass'] = car.attrib['eclass'] + out_data[-1]['waiting'] = float(car.attrib['waiting']) + out_data[-1]['NOx'] = float(car.attrib['NOx']) + out_data[-1]['fuel'] = float(car.attrib['fuel']) + out_data[-1]['HC'] = float(car.attrib['HC']) + out_data[-1]['x'] = float(car.attrib['x']) + out_data[-1]['route'] = car.attrib['route'] + out_data[-1]['relative_position'] = float(car.attrib['pos']) + out_data[-1]['noise'] = float(car.attrib['noise']) + out_data[-1]['angle'] = float(car.attrib['angle']) + out_data[-1]['PMx'] = float(car.attrib['PMx']) + out_data[-1]['speed'] = float(car.attrib['speed']) + out_data[-1]['edge_id'] = car.attrib['lane'].rpartition('_')[0] + out_data[-1]['lane_number'] = car.attrib['lane']. \ + rpartition('_')[-1] + except KeyError: + del out_data[-1] + elem.clear() # sort the elements of the dictionary by the vehicle id out_data = sorted(out_data, key=lambda k: k['id']) diff --git a/flow/data_pipeline/README.md b/flow/data_pipeline/README.md new file mode 100644 index 000000000..65aeb8d49 --- /dev/null +++ b/flow/data_pipeline/README.md @@ -0,0 +1,12 @@ +To run a simulation with output stored locally only: + + `python simulate.py EXP_CONFIG --gen_emission` + +To run a simulation and upload output to pipeline: + + `python simulate.py EXP_CONFIG --to_aws` + +To run a simulation, upload output to pipeline, and mark it as baseline: + + `python simulate.py EXP_CONFIG --to_aws --is_baseline` + diff --git a/flow/data_pipeline/__init__.py b/flow/data_pipeline/__init__.py new file mode 100644 index 000000000..d9d6a6573 --- /dev/null +++ b/flow/data_pipeline/__init__.py @@ -0,0 +1 @@ +"""Empty init file to ensure that data_pipeline is recognized as a package.""" diff --git a/flow/data_pipeline/data_pipeline.py b/flow/data_pipeline/data_pipeline.py new file mode 100644 index 000000000..587ce6e70 --- /dev/null +++ b/flow/data_pipeline/data_pipeline.py @@ -0,0 +1,469 @@ +"""contains class and helper functions for the data pipeline.""" +import pandas as pd +import boto3 +from botocore.exceptions import ClientError +from flow.data_pipeline.query import QueryStrings, prerequisites, tables +from time import time +from datetime import date +import csv +from io import StringIO +import json +import collections + + +def generate_trajectory_table(emission_files, trajectory_table_path, source_id): + """Generate desired output for the trajectory_table based on SUMO emissions. + + Parameters + ---------- + emission_files : list + paths to the SUMO emission + trajectory_table_path : str + path to the file for S3 upload only + source_id : str + a unique id for the simulation that generate these emissions + """ + for i in range(len(emission_files)): + # 1000000 rows are approximately 260 MB, which is an appropriate size to load into memory at once + emission_output = pd.read_csv(emission_files[i], iterator=True, chunksize=1000000) + chunk_count = 0 + for chunk in emission_output: + chunk['source_id'] = source_id + chunk['run_id'] = "run_{}".format(i) + # add header row to the file only at the first run (when i==0) and the first chunk (chunk_count==0) + chunk.to_csv(trajectory_table_path, mode='a+', index=False, header=(chunk_count == 0) and (i == 0)) + chunk_count += 1 + + +def write_dict_to_csv(data_path, extra_info, include_header=False): + """Write extra to the CSV file at data_path, create one if not exist. + + Parameters + ---------- + data_path : str + output file path + extra_info: dict + extra information needed in the trajectory table, collected from flow + include_header: bool + whether or not to include the header in the output, this should be set to + True for the first write to the a empty or newly created CSV, and set to + False for subsequent appends. + """ + extra_info = pd.DataFrame.from_dict(extra_info) + extra_info.to_csv(data_path, mode='a+', index=False, header=include_header) + + +def upload_to_s3(bucket_name, bucket_key, file_path, metadata={}): + """Upload a file to S3 bucket. + + Parameters + ---------- + bucket_name : str + the bucket to upload to + bucket_key: str + the key within the bucket for the file + file_path: str + the path of the file to be uploaded + metadata: dict + all the metadata that should be attached to this simulation + """ + s3 = boto3.resource("s3") + s3.Bucket(bucket_name).upload_file(file_path, bucket_key, + ExtraArgs={"Metadata": metadata}) + return + + +def get_extra_info(veh_kernel, extra_info, veh_ids, source_id, run_id): + """Get all the necessary information for the trajectory output from flow.""" + for vid in veh_ids: + extra_info["time_step"].append(veh_kernel.get_timestep(vid) / 1000) + extra_info["id"].append(vid) + position = veh_kernel.get_2d_position(vid) + extra_info["x"].append(position[0]) + extra_info["y"].append(position[1]) + extra_info["speed"].append(veh_kernel.get_speed(vid)) + extra_info["headway"].append(veh_kernel.get_headway(vid)) + extra_info["leader_id"].append(veh_kernel.get_leader(vid)) + extra_info["follower_id"].append(veh_kernel.get_follower(vid)) + extra_info["leader_rel_speed"].append(veh_kernel.get_speed( + veh_kernel.get_leader(vid)) - veh_kernel.get_speed(vid)) + extra_info["target_accel_with_noise_with_failsafe"].append(veh_kernel.get_accel(vid)) + extra_info["target_accel_no_noise_no_failsafe"].append( + veh_kernel.get_accel(vid, noise=False, failsafe=False)) + extra_info["target_accel_with_noise_no_failsafe"].append( + veh_kernel.get_accel(vid, noise=True, failsafe=False)) + extra_info["target_accel_no_noise_with_failsafe"].append( + veh_kernel.get_accel(vid, noise=False, failsafe=True)) + extra_info["realized_accel"].append(veh_kernel.get_realized_accel(vid)) + extra_info["road_grade"].append(veh_kernel.get_road_grade(vid)) + extra_info["edge_id"].append(veh_kernel.get_edge(vid)) + extra_info["lane_id"].append(veh_kernel.get_lane(vid)) + extra_info["distance"].append(veh_kernel.get_distance(vid)) + extra_info["relative_position"].append(veh_kernel.get_position(vid)) + extra_info["source_id"].append(source_id) + extra_info["run_id"].append(run_id) + + +def get_configuration(submitter_name=None, strategy_name=None): + """Get configuration for the metadata table.""" + try: + config_df = pd.read_csv('./data_pipeline_config') + except FileNotFoundError: + config_df = pd.DataFrame(data={"submitter_name": [""], "strategy": [""]}) + + if not config_df['submitter_name'][0]: + if submitter_name: + name = submitter_name + else: + name = input("Please enter your name:").strip() + while not name: + name = input("Please enter a non-empty name:").strip() + config_df['submitter_name'] = [name] + + if strategy_name: + strategy = strategy_name + else: + strategy = input( + "Please enter strategy name (current: \"{}\"):".format(config_df["strategy"][0])).strip() + if strategy: + config_df['strategy'] = [strategy] + + config_df.to_csv('./data_pipeline_config', index=False) + + return config_df['submitter_name'][0], config_df['strategy'][0] + + +def delete_obsolete_data(s3, latest_key, table, bucket="circles.data.pipeline"): + """Delete the obsolete data on S3.""" + keys = list_object_keys(s3, bucket=bucket, prefixes=table, suffix='.csv') + keys.remove(latest_key) + for key in keys: + s3.delete_object(Bucket=bucket, Key=key) + + +def update_baseline(s3, baseline_network, baseline_source_id): + """Update the baseline table on S3 if new baseline run is added.""" + obj = s3.get_object(Bucket='circles.data.pipeline', Key='baseline_table/baselines.csv')['Body'] + original_str = obj.read().decode() + reader = csv.DictReader(StringIO(original_str)) + new_str = StringIO() + writer = csv.DictWriter(new_str, fieldnames=['network', 'source_id']) + writer.writeheader() + writer.writerow({'network': baseline_network, 'source_id': baseline_source_id}) + for row in reader: + if row['network'] != baseline_network: + writer.writerow(row) + s3.put_object(Bucket='circles.data.pipeline', Key='baseline_table/baselines.csv', + Body=new_str.getvalue().replace('\r', '').encode()) + + +def get_completed_queries(s3, source_id): + """Return the deserialized set of completed queries from S3.""" + try: + completed_queries_obj = \ + s3.get_object(Bucket='circles.data.pipeline', Key='lambda_temp/{}'.format(source_id))['Body'] + completed_queries = json.loads(completed_queries_obj.read().decode('utf-8')) + except ClientError as e: + if e.response['Error']['Code'] == 'NoSuchKey': + completed_queries = set() + else: + raise + return set(completed_queries) + + +def put_completed_queries(s3, source_id, completed_queries_set): + """Put the completed queries list into S3 as in a serialized json format.""" + completed_queries_list = list(completed_queries_set) + completed_queries_json = json.dumps(completed_queries_list) + s3.put_object(Bucket='circles.data.pipeline', Key='lambda_temp/{}'.format(source_id), + Body=completed_queries_json.encode('utf-8')) + + +def get_ready_queries(completed_queries, new_query): + """Return queries whose prerequisite queries are completed.""" + readied_queries = [] + unfinished_queries = prerequisites.keys() - completed_queries + upadted_completed_queries = completed_queries.copy() + upadted_completed_queries.add(new_query) + for query_name in unfinished_queries: + if not prerequisites[query_name][1].issubset(completed_queries): + if prerequisites[query_name][1].issubset(upadted_completed_queries): + readied_queries.append((query_name, prerequisites[query_name][0])) + return readied_queries + + +def list_object_keys(s3, bucket='circles.data.pipeline', prefixes='', suffix=''): + """Return all keys in the given bucket that start with prefix and end with suffix. Not limited by 1000.""" + contents = [] + if not isinstance(prefixes, collections.Iterable) or type(prefixes) is str: + prefixes = [prefixes] + for prefix in prefixes: + response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix) + if 'Contents' in response: + contents.extend(response['Contents']) + while response['IsTruncated']: + response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, + ContinuationToken=response['NextContinuationToken']) + contents.extend(response['Contents']) + keys = [content['Key'] for content in contents if content['Key'].endswith(suffix)] + return keys + + +def delete_table(s3, bucket='circles.data.pipeline', only_query_result=True, table='', source_id=''): + """Delete the specified data files in S3.""" + queries = ["lambda_temp"] + if table: + queries.append(table) + else: + queries = tables + if only_query_result: + queries.remove('fact_vehicle_trace') + queries.remove('metadata_table') + else: + confirmation = input("Are you sure you want to delete the simulation emission file? This process is" + "irreversible. (Y/N)").strip() + if not (confirmation in ['y', 'Y', 'yes', 'Yes']): + return + if source_id: + queries.remove('leaderboard_chart_agg') + queries.remove('fact_top_scores') + keys = list_object_keys(s3, bucket=bucket, prefixes=queries) + if source_id: + keys = [key for key in keys if source_id in key] + for key in keys: + s3.delete_object(Bucket=bucket, Key=key) + + +def rerun_query(s3, queue_url, bucket='circles.data.pipeline', source_id=''): + """Re-run queries for simulation datas that has been uploaded to s3, will delete old data before re-run.""" + vehicle_trace_keys = list_object_keys(s3, bucket=bucket, prefixes="fact_vehicle_trace", suffix='.csv') + delete_table(s3, bucket=bucket, source_id=source_id) + if source_id: + vehicle_trace_keys = [key for key in vehicle_trace_keys if source_id in key] + sqs_client = boto3.client('sqs') + # A s3 put event message template, used to trigger lambda for an existing submission without uploading it again + event_template = """ + {{ + "Records": [ + {{ + "eventVersion": "2.0", + "eventSource": "aws:s3", + "awsRegion": "us-west-2", + "eventTime": "1970-01-01T00:00:00.000Z", + "eventName": "ObjectCreated:Put", + "userIdentity": {{ + "principalId": "EXAMPLE" + }}, + "requestParameters": {{ + "sourceIPAddress": "127.0.0.1" + }}, + "responseElements": {{ + "x-amz-request-id": "EXAMPLE123456789", + "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH" + }}, + "s3": {{ + "s3SchemaVersion": "1.0", + "configurationId": "testConfigRule", + "bucket": {{ + "name": "{bucket}", + "ownerIdentity": {{ + "principalId": "EXAMPLE" + }}, + "arn": "arn:aws:s3:::{bucket}" + }}, + "object": {{ + "key": "{key}", + "size": 1024, + "eTag": "0123456789abcdef0123456789abcdef", + "sequencer": "0A1B2C3D4E5F678901" + }} + }} + }} + ] + }}""" + for key in vehicle_trace_keys: + sqs_client.send_message(QueueUrl=queue_url, + MessageBody=event_template.format(bucket=bucket, key=key)) + + +def list_source_ids(s3, bucket='circles.data.pipeline'): + """Return a list of the source_id of all simulations which has been uploaded to s3.""" + vehicle_trace_keys = list_object_keys(s3, bucket=bucket, prefixes="fact_vehicle_trace", suffix='csv') + source_ids = ['flow_{}'.format(key.split('/')[2].split('=')[1].split('_')[1]) for key in vehicle_trace_keys] + return source_ids + + +def collect_metadata_from_config(config_obj): + """Collect the metadata from the exp_config files.""" + supplied_metadata = dict() + supplied_metadata['version'] = [getattr(config_obj, 'VERSION', '2.0')] + supplied_metadata['on_ramp'] = [str(getattr(config_obj, 'ON_RAMP', False))] + supplied_metadata['penetration_rate'] = [str(100.0 * getattr(config_obj, 'PENETRATION_RATE', 0.0))] + supplied_metadata['road_grade'] = [str(getattr(config_obj, 'ROAD_GRADE', False))] + return supplied_metadata + + +class AthenaQuery: + """Class used to run queries. + + Act as a query engine, maintains an open session with AWS Athena. + + Attributes + ---------- + MAX_WAIT : int + maximum number of seconds to wait before declares time-out + client : boto3.client + the athena client that is used to run the query + existing_partitions : list + a list of partitions that is already recorded in Athena's datalog, + this is obtained through query at the initialization of this class + instance. + """ + + def __init__(self): + """Initialize AthenaQuery instance. + + initialize a client session with AWS Athena, + query Athena to obtain extisting_partition. + """ + self.MAX_WAIT = 60 + self.client = boto3.client("athena") + self.existing_partitions = {} + + def get_existing_partitions(self, table): + """Return the existing partitions in the S3 bucket. + + Returns + ------- + partitions: a list of existing partitions on S3 bucket + """ + response = self.client.start_query_execution( + QueryString='SHOW PARTITIONS {}'.format(table), + QueryExecutionContext={ + 'Database': 'circles' + }, + WorkGroup='primary' + ) + if self.wait_for_execution(response['QueryExecutionId']): + raise RuntimeError("get current partitions timed out") + response = self.client.get_query_results( + QueryExecutionId=response['QueryExecutionId'], + MaxResults=1000 + ) + return [data['Data'][0]['VarCharValue'] for data in response['ResultSet']['Rows']] + + def check_status(self, execution_id): + """Return the status of the execution with given id. + + Parameters + ---------- + execution_id : string + id of the execution that is checked for + Returns + ------- + status: str + QUEUED|RUNNING|SUCCEEDED|FAILED|CANCELLED + """ + response = self.client.get_query_execution( + QueryExecutionId=execution_id + ) + return response['QueryExecution']['Status']['State'] + + def wait_for_execution(self, execution_id): + """Wait for the execution to finish or time-out. + + Parameters + ---------- + execution_id : str + id of the execution this is watiing for + Returns + ------- + time_out: bool + True if time-out, False if success + Raises + ------ + RuntimeError: if execution failed or get canceled + """ + start = time() + while time() - start < self.MAX_WAIT: + state = self.check_status(execution_id) + if state == 'FAILED' or state == 'CANCELLED': + raise RuntimeError("update partition failed") + elif state == 'SUCCEEDED': + return False + return True + + def update_partition(self, table, submission_date, partition): + """Load the given partition to the trajectory_table on Athena. + + Parameters + ---------- + table : str + the name of the table to update + submission_date : str + the new partition date that needs to be loaded + partition : str + the new partition that needs to be loaded + """ + response = self.client.start_query_execution( + QueryString=QueryStrings['UPDATE_PARTITION'].value.format(table=table, date=submission_date, + partition=partition), + QueryExecutionContext={ + 'Database': 'circles' + }, + WorkGroup='primary' + ) + if self.wait_for_execution(response['QueryExecutionId']): + raise RuntimeError("update partition timed out") + self.existing_partitions[table].append("date={}/partition_name={}".format(submission_date, partition)) + return + + def repair_partition(self, table, submission_date, partition): + """Load the missing partitions.""" + if table not in self.existing_partitions.keys(): + self.existing_partitions[table] = self.get_existing_partitions(table) + if "date={}/partition_name={}".format(submission_date, partition) not in \ + self.existing_partitions[table]: + self.update_partition(table, submission_date, partition) + + def run_query(self, query_name, result_location="s3://circles.data.pipeline/result/", + submission_date="today", partition="default", **kwargs): + """Start the execution of a query, does not wait for it to finish. + + Parameters + ---------- + query_name : str + name of the query in QueryStrings enum that will be run + result_location: str, optional + location on the S3 bucket where the result will be stored + submission_date : str + name of the partition date to run this query on + partition: str, optional + name of the partition to run this query on + Returns + ------- + execution_id: str + the execution id of the execution started by this method + Raises + ------ + ValueError: if tries to run a query not existed in QueryStrings enum + """ + if query_name not in QueryStrings.__members__: + raise ValueError("query not existed: please add it to query.py") + + if submission_date == "today": + submission_date = date.today().isoformat() + + source_id = "flow_{}".format(partition.split('_')[1]) + + response = self.client.start_query_execution( + QueryString=QueryStrings[query_name].value.format(date=submission_date, partition=source_id, **kwargs), + QueryExecutionContext={ + 'Database': 'circles' + }, + ResultConfiguration={ + 'OutputLocation': result_location, + }, + WorkGroup='primary' + ) + return response['QueryExecutionId'] diff --git a/flow/data_pipeline/lambda_function.py b/flow/data_pipeline/lambda_function.py new file mode 100644 index 000000000..d82e6e341 --- /dev/null +++ b/flow/data_pipeline/lambda_function.py @@ -0,0 +1,89 @@ +"""lambda function on AWS Lambda.""" +import boto3 +import json +from urllib.parse import unquote_plus +from flow.data_pipeline.data_pipeline import AthenaQuery, delete_obsolete_data, update_baseline, \ + get_ready_queries, get_completed_queries, put_completed_queries +from flow.data_pipeline.query import tables, network_filters, summary_tables, triggers, max_decel, leader_max_decel + +s3 = boto3.client('s3') +queryEngine = AthenaQuery() +sqs = boto3.client('sqs') + + +def lambda_handler(event, context): + """Handle S3 put event on AWS Lambda.""" + # stores all lists of completed query for each source_id + completed = {} + records = [] + event_records = [] + # do a pre-sweep to put all s3 records in one list + for event_record in event['Records']: + if event_record["eventSource"] == "aws:s3": + event_records.append(event_record) + elif event_record['eventSource'] == "aws:sqs": + s3_event = json.loads(event_record['body']) + event_records.extend(s3_event['Records']) + # do a pre-sweep to handle tasks other than initalizing a query + for record in event_records: + bucket = record['s3']['bucket']['name'] + key = unquote_plus(record['s3']['object']['key']) + table = key.split('/')[0] + if table not in tables: + continue + # delete unwanted metadata files + s3.delete_object(Bucket=bucket, Key=(key + '.metadata')) + # load the partition for newly added table + query_date = key.split('/')[-3].split('=')[-1] + partition = key.split('/')[-2].split('=')[-1] + source_id = "flow_{}".format(partition.split('_')[1]) + if table == "fact_vehicle_trace": + query_name = "FACT_VEHICLE_TRACE" + else: + query_name = partition.replace(source_id, "")[1:] + queryEngine.repair_partition(table, query_date, partition) + # delete obsolete data + if table in summary_tables: + delete_obsolete_data(s3, key, table) + # add table that need to start a query to list + if query_name in triggers: + records.append((bucket, key, table, query_name, query_date, partition, source_id)) + + # initialize the queries + for bucket, key, table, query_name, query_date, partition, source_id in records: + # retrieve the set of completed query for this source_id if not already available + if source_id not in completed.keys(): + completed[source_id] = get_completed_queries(s3, source_id) + # if query already recorded before, skip it. This is to tolerate repetitive execution by Lambda + if query_name in completed[source_id]: + continue + # retrieve metadata and use it to determine the right location filters + metadata_key = "fact_vehicle_trace/date={0}/partition_name={1}/{1}.csv".format(query_date, source_id) + response = s3.head_object(Bucket=bucket, Key=metadata_key) + if 'network' in response["Metadata"]: + network = response["Metadata"]['network'] + inflow_filter = network_filters[network]['inflow_filter'] + outflow_filter = network_filters[network]['outflow_filter'] + start_filter = network_filters[network]['warmup_steps'] + + # update baseline if needed + if table == 'fact_vehicle_trace' \ + and 'is_baseline' in response['Metadata'] and response['Metadata']['is_baseline'] == 'True': + update_baseline(s3, network, source_id) + + readied_queries = get_ready_queries(completed[source_id], query_name) + completed[source_id].add(query_name) + # stores the updated list of completed queries back to S3 + put_completed_queries(s3, source_id, completed[source_id]) + # initialize queries and store them at appropriate locations + for readied_query_name, table_name in readied_queries: + result_location = 's3://circles.data.pipeline/{}/date={}/partition_name={}_{}'.format(table_name, + query_date, + source_id, + readied_query_name) + message_body = (readied_query_name, result_location, query_date, partition, inflow_filter, outflow_filter, + start_filter, max_decel, leader_max_decel) + message_body = json.dumps(message_body) + sqs.send_message( + QueueUrl="", + MessageBody=message_body) diff --git a/flow/data_pipeline/leaderboard_utils.py b/flow/data_pipeline/leaderboard_utils.py new file mode 100644 index 000000000..dd7055f8b --- /dev/null +++ b/flow/data_pipeline/leaderboard_utils.py @@ -0,0 +1,166 @@ +"""APIs for the leader board front end.""" +import os +import boto3 +import pandas as pd +from io import StringIO + + +network_name_map = {"highway": "Single-Lane Straight Road", + "highway_single": "Single-Lane Straight Road", + "ring": "Single-Lane Ring Road", + "I-210_subnetwork": "I-210 without Ramps", + "I_210_subnetwork": "I-210 without Ramps"} + + +def network_name_translate(network_name): + """Translate network name to a human friendly name for the leaderboard.""" + return network_name_map.get(network_name, network_name) + + +def key_to_name(key): + """Return the standard formatted file name from object key.""" + k_list = key.split("/") + date = k_list[1].replace("date=", "") + name = k_list[2].replace("partition_name=", "") + index = name.find("_", 5) + source_id = name + query_name = "" + if index != -1: + source_id = name[0:index] + query_name = "_" + name[index+1:].replace("_", "-") + return "{}_{}{}.csv".format(date, source_id.replace("_", "-"), query_name) + + +def get_table_disk(table_name="fact_vehicle_trace", bucket="circles.data.pipeline"): + """Fetch tables from s3 and store in ./result directory. + + Parameters + ---------- + table_name : str + The name of table to retrieve from S3, the current available tables are: + fact_vehicle_trace + fact_energy_trace + fact_network_throughput_agg + fact_network_inflows_outflows + fact_vehicle_fuel_efficiency_agg + fact_network_metrics_by_distance_agg + fact_network_metrics_by_time_agg + fact_network_fuel_efficiency_agg + leaderboard_chart + leaderboard_chart_agg + Note that leaderboard_chart_agg is a combination of all previous + learderboard_chart entries in one CSV file. It's only used to + avoid burdening the web server with more calculation. The date + and source_id in its name is always going to reflect the latest + leaderboard_chart entry. + bucket : str + the S3 bucket that holds these tables + """ + try: + os.makedirs("result/{}".format(table_name)) + except FileExistsError: + pass + s3 = boto3.client("s3") + response = s3.list_objects_v2(Bucket=bucket) + keys = [e["Key"] for e in response["Contents"] if e["Key"].find(table_name) == 0 and e["Key"][-4:] == ".csv"] + names = [key_to_name(k) for k in keys] + existing_results = os.listdir("./result/{}".format(table_name)) + updated = False + for index in range(len(keys)): + if names[index] not in existing_results: + updated = True + s3.download_file(bucket, keys[index], "./result/{}/{}".format(table_name, names[index])) + if table_name == "leaderboard_chart_agg" and updated: + for p in existing_results: + os.remove("./result/{}/{}".format(table_name, p)) + + +def get_table_memory(table_name="fact_vehicle_trace", bucket="circles.data.pipeline", existing_results=()): + """Fetch tables from s3 and return them as in-memory pandas dataframe objects. + + Parameters + ---------- + bucket: str + the S3 bucket that holds the tables + table_name: str + the name of the name to retrieve from S3, for detail see get_table_disk + existing_results: list + tables that should not be fetched, + the names must follow the convention: + {date}_{source_id(no run number)}_{query_name}.csv + + Returns + ------- + file_list: dict + a dictionary of pandas dataframes, each contains a table from S3 + The dataframs are keyed by their name: {source_id(no run number)}_{query_name}.csv + + """ + s3 = boto3.client("s3") + response = s3.list_objects_v2(Bucket=bucket) + keys = [e["Key"] for e in response["Contents"] if e["Key"].find(table_name) == 0 and e["Key"][-4:] == ".csv"] + names = [key_to_name(k) for k in keys] + results = dict() + for index in range(len(keys)): + if names[index] not in existing_results: + obj = s3.get_object(Bucket=bucket, Key=keys[index])["Body"] + obj_str = obj.read().decode("utf-8") + results[names[index]] = pd.read_csv(StringIO(obj_str)) + return results + + +def get_table_url(table_name="fact_vehicle_trace", bucket="circles.data.pipeline", existing_results=()): + """Fetch tables from s3 and return as urls, requires the bucket to have public access. + + Parameters + ---------- + bucket: str + the S3 bucket that holds the tables + table_name: str + the name of the name to retrieve from S3, for detail see get_table_disk + existing_results: list + tables that should not be fetched, + the names must follow the convention: + {date}_{source_id(no run number)}_{query_name}.csv + + Returns + ------- + file_list: dict + a dictionary of urls, each contains a table from S3 + The urls are keyed by their name: {source_id(no run number)}_{query_name}.csv + + """ + s3 = boto3.client("s3") + response = s3.list_objects_v2(Bucket=bucket) + keys = [e["Key"] for e in response["Contents"] if e["Key"].find(table_name) == 0 and e["Key"][-4:] == ".csv"] + names = [key_to_name(k) for k in keys] + results = dict() + for index in range(len(keys)): + if names[index] not in existing_results: + results[names[index]] = "https://{}.s3.{}.amazonaws.com/{}".format(bucket, "us-west-2", keys[index]) + return results + + +def get_metadata(name, bucket="circles.data.pipeline"): + """Get the metadata by name. + + Parameters + ---------- + name: str + the name of the table whose metadata will be returned + bucket: str + the bucket that hold the table + + Returns + ------- + metadata: dict + a dictionary of all the metadata, there is no guarantee + for which keys are included + """ + s3 = boto3.client("s3") + name_list = name.split('_') + source_id = name_list[1].replace('.csv', "").replace('-', '_') + response = s3.head_object(Bucket=bucket, + Key="fact_vehicle_trace/date={0}/partition_name={1}/{1}.csv".format(name_list[0], + source_id)) + return response["Metadata"] diff --git a/flow/data_pipeline/query.py b/flow/data_pipeline/query.py new file mode 100644 index 000000000..e7747a9a2 --- /dev/null +++ b/flow/data_pipeline/query.py @@ -0,0 +1,932 @@ +"""stores all the pre-defined query strings.""" +from collections import defaultdict +from enum import Enum + +# tags for different queries +prerequisites = { + "TACOMA_FIT_DENOISED_ACCEL": ( + "fact_energy_trace", {"FACT_VEHICLE_TRACE"} + ), + "PRIUS_FIT_DENOISED_ACCEL": ( + "fact_energy_trace", {"FACT_VEHICLE_TRACE"} + ), + "FACT_SAFETY_METRICS_2D": ( + "fact_safety_metrics", {"FACT_VEHICLE_TRACE"} + ), + "FACT_SAFETY_METRICS_3D": ( + "fact_safety_metrics", {"FACT_VEHICLE_TRACE"} + ), + "FACT_NETWORK_THROUGHPUT_AGG": ( + "fact_network_throughput_agg", {"FACT_VEHICLE_TRACE"} + ), + "FACT_NETWORK_INFLOWS_OUTFLOWS": ( + "fact_network_inflows_outflows", {"FACT_VEHICLE_TRACE"} + ), + "FACT_NETWORK_SPEED": ( + "fact_network_speed", {"FACT_VEHICLE_TRACE"} + ), + "FACT_VEHICLE_COUNTS_BY_TIME": ( + "fact_vehicle_counts_by_time", {"FACT_VEHICLE_TRACE"} + ), + "FACT_VEHICLE_FUEL_EFFICIENCY_AGG": ( + "fact_vehicle_fuel_efficiency_agg", {"FACT_VEHICLE_TRACE", + "TACOMA_FIT_DENOISED_ACCEL", + "PRIUS_FIT_DENOISED_ACCEL"} + ), + "FACT_NETWORK_METRICS_BY_DISTANCE_AGG": ( + "fact_network_metrics_by_distance_agg", {"FACT_VEHICLE_TRACE", + "TACOMA_FIT_DENOISED_ACCEL", + "PRIUS_FIT_DENOISED_ACCEL"} + ), + "FACT_NETWORK_METRICS_BY_TIME_AGG": ( + "fact_network_metrics_by_time_agg", {"FACT_VEHICLE_TRACE", + "TACOMA_FIT_DENOISED_ACCEL", + "PRIUS_FIT_DENOISED_ACCEL"} + ), + "FACT_VEHICLE_FUEL_EFFICIENCY_BINNED": ( + "fact_vehicle_fuel_efficiency_binned", {"FACT_VEHICLE_FUEL_EFFICIENCY_AGG"} + ), + "FACT_NETWORK_FUEL_EFFICIENCY_AGG": ( + "fact_network_fuel_efficiency_agg", {"FACT_VEHICLE_FUEL_EFFICIENCY_AGG"} + ), + "FACT_SAFETY_METRICS_AGG": ( + "fact_safety_metrics_agg", {"FACT_SAFETY_METRICS_3D"} + ), + "FACT_SAFETY_METRICS_BINNED": ( + "fact_safety_metrics_binned", {"FACT_SAFETY_METRICS_3D"} + ), + "LEADERBOARD_CHART": ( + "leaderboard_chart", {"FACT_NETWORK_THROUGHPUT_AGG", + "FACT_NETWORK_SPEED", + "FACT_NETWORK_FUEL_EFFICIENCY_AGG", + "FACT_SAFETY_METRICS_AGG"} + ), + "LEADERBOARD_CHART_AGG": ( + "leaderboard_chart_agg", {"LEADERBOARD_CHART"} + ), + "FACT_TOP_SCORES": ( + "fact_top_scores", {"LEADERBOARD_CHART_AGG"} + ), +} + +triggers = [ + "FACT_VEHICLE_TRACE", + "TACOMA_FIT_DENOISED_ACCEL", + "PRIUS_FIT_DENOISED_ACCEL", + "FACT_VEHICLE_FUEL_EFFICIENCY_AGG", + "FACT_SAFETY_METRICS_3D", + "FACT_NETWORK_THROUGHPUT_AGG", + "FACT_NETWORK_SPEED", + "FACT_NETWORK_FUEL_EFFICIENCY_AGG", + "FACT_SAFETY_METRICS_AGG", + "LEADERBOARD_CHART", + "LEADERBOARD_CHART_AGG" +] + +tables = [ + "fact_vehicle_trace", + "fact_energy_trace", + "fact_vehicle_counts_by_time", + "fact_safety_metrics", + "fact_safety_metrics_agg", + "fact_safety_metrics_binned", + "fact_network_throughput_agg", + "fact_network_inflows_outflows", + "fact_network_speed", + "fact_vehicle_fuel_efficiency_agg", + "fact_vehicle_fuel_efficiency_binned", + "fact_network_metrics_by_distance_agg", + "fact_network_metrics_by_time_agg", + "fact_network_fuel_efficiency_agg", + "leaderboard_chart", + "leaderboard_chart_agg", + "fact_top_scores", + "metadata_table" +] + +summary_tables = ["leaderboard_chart_agg", "fact_top_scores"] + +network_filters = defaultdict(lambda: { + 'inflow_filter': "x > 500", + 'outflow_filter': "x < 2300", + 'warmup_steps': 500 * 3 * 0.4 + }) +network_filters['I-210 without Ramps'] = { + 'inflow_filter': "edge_id != 'ghost0'", + 'outflow_filter': "edge_id != '119257908#3'", + 'warmup_steps': 600 * 3 * 0.4 + } + +max_decel = -1.0 +leader_max_decel = -2.0 + +TACOMA_FIT_FINAL_SELECT = """ + SELECT + id, + time_step, + speed, + acceleration, + road_grade, + GREATEST(0, 2041 * acceleration * speed + + 3405.5481762 + + 83.12392997 * speed + + 6.7650718327 * POW(speed,2) + + 0.7041355229 * POW(speed,3) + ) + GREATEST(0, 4598.7155 * acceleration + 975.12719 * acceleration * speed) AS power, + 'TACOMA_FIT_DENOISED_ACCEL' AS energy_model_id, + source_id + FROM {} + ORDER BY id, time_step + """ + +PRIUS_FIT_FINAL_SELECT = """ + , pmod_calculation AS ( + SELECT + id, + time_step, + speed, + acceleration, + road_grade, + GREATEST(1663 * acceleration * speed + + 1.046 + + 119.166 * speed + + 0.337 * POW(speed,2) + + 0.383 * POW(speed,3) + + GREATEST(0, 296.66 * acceleration * speed)) AS p_mod, + source_id + FROM {} + ) + SELECT + id, + time_step, + speed, + acceleration, + road_grade, + GREATEST(p_mod, 0.869 * p_mod, -2338 * speed) AS power, + 'PRIUS_FIT_DENOISED_ACCEL' AS energy_model_id, + source_id + FROM pmod_calculation + ORDER BY id, time_step + """ + +DENOISED_ACCEL = """ + WITH denoised_accel_cte AS ( + SELECT + id, + time_step, + speed, + COALESCE (target_accel_no_noise_with_failsafe, + target_accel_no_noise_no_failsafe, + realized_accel) AS acceleration, + road_grade, + source_id + FROM fact_vehicle_trace + WHERE 1 = 1 + AND date = \'{{date}}\' + AND partition_name=\'{{partition}}\' + ) + {}""" + + +class QueryStrings(Enum): + """An enumeration of all the pre-defined query strings.""" + + SAMPLE = """ + SELECT * + FROM trajectory_table + WHERE date = \'{date}\' + AND partition_name=\'{partition}\' + LIMIT 15; + """ + + UPDATE_PARTITION = """ + ALTER TABLE {table} + ADD IF NOT EXISTS PARTITION (date = \'{date}\', partition_name=\'{partition}\'); + """ + + TACOMA_FIT_DENOISED_ACCEL = \ + DENOISED_ACCEL.format(TACOMA_FIT_FINAL_SELECT.format('denoised_accel_cte')) + + PRIUS_FIT_DENOISED_ACCEL = \ + DENOISED_ACCEL.format(PRIUS_FIT_FINAL_SELECT.format('denoised_accel_cte')) + + FACT_SAFETY_METRICS_2D = """ + SELECT + vt.id, + vt.time_step, + COALESCE(( + value_lower_left*(headway_upper-headway)*(rel_speed_upper-leader_rel_speed) + + value_lower_right*(headway-headway_lower)*(rel_speed_upper-leader_rel_speed) + + value_upper_left*(headway_upper-headway)*(leader_rel_speed-rel_speed_lower) + + value_upper_right*(headway-headway_lower)*(leader_rel_speed-rel_speed_lower) + ) / ((headway_upper-headway_lower)*(rel_speed_upper-rel_speed_lower)), 200.0) AS safety_value, + 'v2D_HJI' AS safety_model, + vt.source_id + FROM fact_vehicle_trace vt + LEFT OUTER JOIN fact_safety_matrix sm ON 1 = 1 + AND vt.leader_rel_speed BETWEEN sm.rel_speed_lower AND sm.rel_speed_upper + AND vt.headway BETWEEN sm.headway_lower AND sm.headway_upper + WHERE 1 = 1 + AND vt.date = \'{date}\' + AND vt.partition_name = \'{partition}\' + AND vt.time_step >= {start_filter} + AND vt.{inflow_filter} + AND vt.{outflow_filter} + ; + """ + + FACT_SAFETY_METRICS_3D = """ + SELECT + id, + time_step, + headway + (CASE + WHEN -speed/{max_decel} > -(speed+leader_rel_speed)/{leader_max_decel} THEN + -0.5*POW(leader_rel_speed, 2)/{leader_max_decel} + + -0.5*POW(speed,2)/{leader_max_decel} + + -speed*leader_rel_speed/{leader_max_decel} + + 0.5*POW(speed,2)/{max_decel} + ELSE + -leader_rel_speed*speed/{max_decel} + + 0.5*POW(speed,2)*{leader_max_decel}/POW({max_decel},2) + + -0.5*POW(speed,2)/{max_decel} + END) AS safety_value, + 'v3D' AS safety_model, + source_id + FROM fact_vehicle_trace + WHERE 1 = 1 + AND date = \'{date}\' + AND partition_name = \'{partition}\' + AND leader_id IS NOT NULL + AND time_step >= {start_filter} + AND {inflow_filter} + AND {outflow_filter} + ; + """ + + FACT_SAFETY_METRICS_AGG = """ + SELECT + source_id, + SUM(CASE WHEN safety_value > 0 THEN 1.0 ELSE 0.0 END) * 100.0 / COUNT() safety_rate, + MIN(safety_value) AS safety_value_max + FROM fact_safety_metrics + WHERE 1 = 1 + AND date = \'{date}\' + AND partition_name = \'{partition}_FACT_SAFETY_METRICS_3D\' + AND safety_model = 'v3D' + GROUP BY 1 + ; + """ + + FACT_SAFETY_METRICS_BINNED = """ + WITH unfilter_bins AS ( + SELECT + ROW_NUMBER() OVER() - 51 AS lb, + ROW_NUMBER() OVER() - 50 AS ub + FROM fact_safety_matrix + ), bins AS ( + SELECT + lb, + ub + FROM unfilter_bins + WHERE 1=1 + AND lb >= -5 + AND ub <= 15 + ) + SELECT + CONCAT('[', CAST(bins.lb AS VARCHAR), ', ', CAST(bins.ub AS VARCHAR), ')') AS safety_value_bin, + COUNT() AS count + FROM bins + LEFT JOIN fact_safety_metrics fsm ON 1 = 1 + AND fsm.date = \'{date}\' + AND fsm.partition_name = \'{partition}_FACT_SAFETY_METRICS_3D\' + AND fsm.safety_value >= bins.lb + AND fsm.safety_value < bins.ub + AND fsm.safety_model = 'v3D' + GROUP BY 1 + ; + """ + + FACT_NETWORK_THROUGHPUT_AGG = """ + WITH min_time AS ( + SELECT + source_id, + id, + MIN(time_step) AS enter_time + FROM fact_vehicle_trace + WHERE 1 = 1 + AND date = \'{date}\' + AND partition_name = \'{partition}\' + AND {inflow_filter} + GROUP BY 1, 2 + ), agg AS ( + SELECT + source_id, + COUNT(DISTINCT id) AS n_vehicles, + MAX(enter_time) - MIN(enter_time) AS total_time_seconds + FROM min_time + WHERE 1 = 1 + AND enter_time >= {start_filter} + GROUP BY 1 + ) + SELECT + source_id, + n_vehicles * 3600 / total_time_seconds AS throughput_per_hour + FROM agg + ;""" + + FACT_VEHICLE_FUEL_EFFICIENCY_AGG = """ + WITH sub_fact_vehicle_trace AS ( + SELECT + v.id, + v.source_id, + e.energy_model_id, + MAX(distance) - MIN(distance) AS distance_meters, + (MAX(e.time_step) - MIN(e.time_step)) / (COUNT(DISTINCT e.time_step) - 1) AS time_step_size_seconds, + SUM(e.power) AS power_watts + FROM fact_vehicle_trace v + JOIN fact_energy_trace AS e ON 1 = 1 + AND e.id = v.id + AND e.time_step = v.time_step + AND e.source_id = v.source_id + AND e.date = \'{date}\' + AND e.partition_name LIKE \'{partition}_%\' + AND e.time_step >= {start_filter} + WHERE 1 = 1 + AND v.date = \'{date}\' + AND v.partition_name = \'{partition}\' + AND v.{inflow_filter} + AND v.{outflow_filter} + GROUP BY 1, 2, 3 + HAVING 1 = 1 + AND MAX(distance) - MIN(distance) > 10 + AND COUNT(DISTINCT e.time_step) > 10 + ) + SELECT + id, + source_id, + energy_model_id, + distance_meters, + power_watts * time_step_size_seconds AS energy_joules, + distance_meters / (power_watts * time_step_size_seconds) AS efficiency_meters_per_joules, + 33554.13 * distance_meters / (power_watts * time_step_size_seconds) AS efficiency_miles_per_gallon + FROM sub_fact_vehicle_trace + WHERE 1 = 1 + AND power_watts * time_step_size_seconds != 0 + ; + """ + + FACT_VEHICLE_FUEL_EFFICIENCY_BINNED = """ + WITH unfilter_bins AS ( + SELECT + ROW_NUMBER() OVER() - 1 AS lb, + ROW_NUMBER() OVER() AS ub + FROM fact_safety_matrix + ), tacoma_binned AS ( + SELECT + bins.lb, + bins.ub, + COUNT() AS count + FROM unfilter_bins bins + LEFT JOIN fact_vehicle_fuel_efficiency_agg agg ON 1 = 1 + AND agg.date = \'{date}\' + AND agg.partition_name = \'{partition}_FACT_VEHICLE_FUEL_EFFICIENCY_AGG\' + AND agg.efficiency_miles_per_gallon >= bins.lb + AND agg.efficiency_miles_per_gallon < bins.ub + AND agg.energy_model_id = 'TACOMA_FIT_DENOISED_ACCEL' + GROUP BY 1, 2 + ), prius_binned AS ( + SELECT + bins.lb, + bins.ub, + COUNT() AS count + FROM unfilter_bins bins + LEFT JOIN fact_vehicle_fuel_efficiency_agg agg ON 1 = 1 + AND agg.date = \'{date}\' + AND agg.partition_name = \'{partition}_FACT_VEHICLE_FUEL_EFFICIENCY_AGG\' + AND agg.efficiency_miles_per_gallon >= bins.lb + AND agg.efficiency_miles_per_gallon < bins.ub + AND agg.energy_model_id = 'PRIUS_FIT_DENOISED_ACCEL' + GROUP BY 1, 2 + ), tacoma_ratio_to_report AS ( + SELECT + lb, + ub, + 100.0 * count / (SUM(count) OVER()) AS count + FROM tacoma_binned + ORDER BY lb + ), prius_ratio_to_report AS ( + SELECT + lb, + ub, + 100.0 * count / (SUM(count) OVER()) AS count + FROM prius_binned + ORDER BY lb + ) + SELECT + 'TACOMA_FIT_DENOISED_ACCEL' AS energy_model_id, + CONCAT('[', CAST(lb AS VARCHAR), ', ', CAST(ub AS VARCHAR), ')') AS fuel_efficiency_bin, + count + FROM tacoma_ratio_to_report + WHERE 1 = 1 + AND lb >= 0 + AND ub <= 60 + UNION ALL + SELECT + 'PRIUS_FIT_DENOISED_ACCEL' AS energy_model_id, + CONCAT('[', CAST(lb AS VARCHAR), ', ', CAST(ub AS VARCHAR), ')') AS fuel_efficiency_bin, + count + FROM prius_ratio_to_report + WHERE 1 = 1 + AND lb >= 130 + AND ub <= 190 + ;""" + + FACT_NETWORK_FUEL_EFFICIENCY_AGG = """ + WITH aggs AS ( + SELECT + source_id, + energy_model_id, + SUM(distance_meters) AS distance_meters, + SUM(energy_joules) AS energy_joules + FROM fact_vehicle_fuel_efficiency_agg + WHERE 1 = 1 + AND date = \'{date}\' + AND partition_name = \'{partition}_FACT_VEHICLE_FUEL_EFFICIENCY_AGG\' + GROUP BY 1, 2 + ) + SELECT + source_id, + energy_model_id, + 1000 * distance_meters / energy_joules AS efficiency_meters_per_kilojoules, + (CASE energy_model_id + WHEN 'TACOMA_FIT_DENOISED_ACCEL' THEN 33554.13 + WHEN 'PRIUS_FIT_DENOISED_ACCEL' THEN 75384.94 + END) * distance_meters / energy_joules AS efficiency_miles_per_gallon + FROM aggs + WHERE 1 = 1 + AND energy_joules != 0 + ;""" + + FACT_NETWORK_SPEED = """ + WITH vehicle_agg AS ( + SELECT + id, + source_id, + AVG(speed) AS vehicle_avg_speed, + COUNT(DISTINCT time_step) AS n_steps, + MAX(time_step) - MIN(time_step) AS time_delta, + MAX(distance) - MIN(distance) AS distance_delta + FROM fact_vehicle_trace + WHERE 1 = 1 + AND date = \'{date}\' + AND partition_name = \'{partition}\' + AND {inflow_filter} + AND {outflow_filter} + AND time_step >= {start_filter} + GROUP BY 1, 2 + ) + SELECT + source_id, + SUM(vehicle_avg_speed * n_steps) / SUM(n_steps) AS avg_instantaneous_speed, + SUM(distance_delta) / SUM(time_delta) AS avg_network_speed + FROM vehicle_agg + GROUP BY 1 + ;""" + + LEADERBOARD_CHART = """ + SELECT + nt.source_id, + nt.throughput_per_hour, + ns.avg_instantaneous_speed, + ns.avg_network_speed, + sm.safety_rate, + sm.safety_value_max, + AVG(CASE + WHEN fe.energy_model_id = 'PRIUS_FIT_DENOISED_ACCEL' + THEN fe.efficiency_meters_per_kilojoules END) AS prius_efficiency_meters_per_kilojoules, + AVG(CASE + WHEN fe.energy_model_id = 'TACOMA_FIT_DENOISED_ACCEL' + THEN fe.efficiency_meters_per_kilojoules END) AS tacoma_efficiency_meters_per_kilojoules, + AVG(CASE + WHEN fe.energy_model_id = 'PRIUS_FIT_DENOISED_ACCEL' + THEN fe.efficiency_miles_per_gallon END) AS prius_efficiency_miles_per_gallon, + AVG(CASE + WHEN fe.energy_model_id = 'TACOMA_FIT_DENOISED_ACCEL' + THEN fe.efficiency_miles_per_gallon END) AS tacoma_efficiency_miles_per_gallon + FROM fact_network_throughput_agg AS nt + JOIN fact_network_speed AS ns ON 1 = 1 + AND ns.date = \'{date}\' + AND ns.partition_name = \'{partition}_FACT_NETWORK_SPEED\' + AND nt.source_id = ns.source_id + JOIN fact_network_fuel_efficiency_agg AS fe ON 1 = 1 + AND fe.date = \'{date}\' + AND fe.partition_name = \'{partition}_FACT_NETWORK_FUEL_EFFICIENCY_AGG\' + AND nt.source_id = fe.source_id + JOIN fact_safety_metrics_agg AS sm ON 1 = 1 + AND sm.date = \'{date}\' + AND sm.partition_name = \'{partition}_FACT_SAFETY_METRICS_AGG\' + AND nt.source_id = sm.source_id + WHERE 1 = 1 + AND nt.date = \'{date}\' + AND nt.partition_name = \'{partition}_FACT_NETWORK_THROUGHPUT_AGG\' + GROUP BY 1, 2, 3, 4, 5, 6 + ;""" + + FACT_NETWORK_INFLOWS_OUTFLOWS = """ + WITH in_out_time_step AS ( + SELECT + id, + source_id, + MIN(CASE WHEN {inflow_filter} THEN time_step - {start_filter} ELSE NULL END) AS inflow_time_step, + MIN(CASE WHEN {outflow_filter} THEN NULL ELSE time_step - {start_filter} END) AS outflow_time_step + FROM fact_vehicle_trace + WHERE 1 = 1 + AND date = \'{date}\' + AND partition_name = \'{partition}\' + GROUP BY 1, 2 + ), inflows AS ( + SELECT + CAST(inflow_time_step / 60 AS INTEGER) * 60 AS time_step, + source_id, + 60 * COUNT(DISTINCT id) AS inflow_rate + FROM in_out_time_step + GROUP BY 1, 2 + ), outflows AS ( + SELECT + CAST(outflow_time_step / 60 AS INTEGER) * 60 AS time_step, + source_id, + 60 * COUNT(DISTINCT id) AS outflow_rate + FROM in_out_time_step + GROUP BY 1, 2 + ) + SELECT + COALESCE(i.time_step, o.time_step) AS time_step, + COALESCE(i.source_id, o.source_id) AS source_id, + COALESCE(i.inflow_rate, 0) AS inflow_rate, + COALESCE(o.outflow_rate, 0) AS outflow_rate + FROM inflows i + FULL OUTER JOIN outflows o ON 1 = 1 + AND i.time_step = o.time_step + AND i.source_id = o.source_id + WHERE 1 = 1 + AND COALESCE(i.time_step, o.time_step) >= 0 + ORDER BY time_step + ;""" + + FACT_NETWORK_METRICS_BY_DISTANCE_AGG = """ + WITH joined_trace AS ( + SELECT + vt.id, + vt.source_id, + vt.time_step, + vt.distance - FIRST_VALUE(vt.distance) + OVER (PARTITION BY vt.id, vt.source_id ORDER BY vt.time_step ASC) AS distance_meters, + energy_model_id, + et.speed, + et.acceleration, + vt.time_step - LAG(vt.time_step, 1) + OVER (PARTITION BY vt.id, vt.source_id ORDER BY vt.time_step ASC) AS sim_step, + SUM(power) + OVER (PARTITION BY vt.id, vt.source_id ORDER BY vt.time_step ASC + ROWS BETWEEN UNBOUNDED PRECEDING and CURRENT ROW) AS cumulative_power + FROM fact_vehicle_trace vt + JOIN fact_energy_trace et ON 1 = 1 + AND et.date = \'{date}\' + AND et.partition_name = \'{partition}_TACOMA_FIT_DENOISED_ACCEL\' + AND vt.id = et.id + AND vt.source_id = et.source_id + AND vt.time_step = et.time_step + AND et.energy_model_id = 'TACOMA_FIT_DENOISED_ACCEL' + WHERE 1 = 1 + AND vt.date = \'{date}\' + AND vt.partition_name = \'{partition}\' + AND vt.{inflow_filter} + AND vt.{outflow_filter} + AND vt.time_step >= {start_filter} + ), cumulative_energy AS ( + SELECT + id, + source_id, + time_step, + distance_meters, + energy_model_id, + speed, + acceleration, + cumulative_power * sim_step AS energy_joules + FROM joined_trace + ), binned_cumulative_energy AS ( + SELECT + source_id, + CAST(distance_meters/10 AS INTEGER) * 10 AS distance_meters_bin, + AVG(speed) AS speed_avg, + AVG(speed) + STDDEV(speed) AS speed_upper_bound, + AVG(speed) - STDDEV(speed) AS speed_lower_bound, + AVG(acceleration) AS accel_avg, + AVG(acceleration) + STDDEV(acceleration) AS accel_upper_bound, + AVG(acceleration) - STDDEV(acceleration) AS accel_lower_bound, + AVG(energy_joules) AS cumulative_energy_avg, + AVG(energy_joules) + STDDEV(energy_joules) AS cumulative_energy_upper_bound, + AVG(energy_joules) - STDDEV(energy_joules) AS cumulative_energy_lower_bound + FROM cumulative_energy + GROUP BY 1, 2 + HAVING 1 = 1 + AND COUNT(DISTINCT time_step) > 1 + ), binned_energy_start_end AS ( + SELECT DISTINCT + source_id, + id, + CAST(distance_meters/10 AS INTEGER) * 10 AS distance_meters_bin, + FIRST_VALUE(energy_joules) + OVER (PARTITION BY id, CAST(distance_meters/10 AS INTEGER) * 10 + ORDER BY time_step ASC) AS energy_start, + LAST_VALUE(energy_joules) + OVER (PARTITION BY id, CAST(distance_meters/10 AS INTEGER) * 10 + ORDER BY time_step ASC) AS energy_end + FROM cumulative_energy + ), binned_energy AS ( + SELECT + source_id, + distance_meters_bin, + AVG(energy_end - energy_start) AS instantaneous_energy_avg, + AVG(energy_end - energy_start) + STDDEV(energy_end - energy_start) AS instantaneous_energy_upper_bound, + AVG(energy_end - energy_start) - STDDEV(energy_end - energy_start) AS instantaneous_energy_lower_bound + FROM binned_energy_start_end + GROUP BY 1, 2 + ) + SELECT + bce.source_id AS source_id, + bce.distance_meters_bin AS distance_meters_bin, + bce.cumulative_energy_avg, + bce.cumulative_energy_lower_bound, + bce.cumulative_energy_upper_bound, + bce.speed_avg, + bce.speed_upper_bound, + bce.speed_lower_bound, + bce.accel_avg, + bce.accel_upper_bound, + bce.accel_lower_bound, + COALESCE(be.instantaneous_energy_avg, 0) AS instantaneous_energy_avg, + COALESCE(be.instantaneous_energy_upper_bound, 0) AS instantaneous_energy_upper_bound, + COALESCE(be.instantaneous_energy_lower_bound, 0) AS instantaneous_energy_lower_bound + FROM binned_cumulative_energy bce + JOIN binned_energy be ON 1 = 1 + AND bce.source_id = be.source_id + AND bce.distance_meters_bin = be.distance_meters_bin + ORDER BY distance_meters_bin ASC + ;""" + + FACT_NETWORK_METRICS_BY_TIME_AGG = """ + WITH joined_trace AS ( + SELECT + vt.id, + vt.source_id, + vt.time_step - FIRST_VALUE(vt.time_step) + OVER (PARTITION BY vt.id, vt.source_id ORDER BY vt.time_step ASC) AS time_step, + energy_model_id, + et.speed, + et.acceleration, + vt.time_step - LAG(vt.time_step, 1) + OVER (PARTITION BY vt.id, vt.source_id ORDER BY vt.time_step ASC) AS sim_step, + SUM(power) + OVER (PARTITION BY vt.id, vt.source_id ORDER BY vt.time_step ASC + ROWS BETWEEN UNBOUNDED PRECEDING and CURRENT ROW) AS cumulative_power + FROM fact_vehicle_trace vt + JOIN fact_energy_trace et ON 1 = 1 + AND et.date = \'{date}\' + AND et.partition_name = \'{partition}_TACOMA_FIT_DENOISED_ACCEL\' + AND vt.id = et.id + AND vt.source_id = et.source_id + AND vt.time_step = et.time_step + AND et.energy_model_id = 'TACOMA_FIT_DENOISED_ACCEL' + WHERE 1 = 1 + AND vt.date = \'{date}\' + AND vt.partition_name = \'{partition}\' + AND vt.{inflow_filter} + AND vt.{outflow_filter} + AND vt.time_step >= {start_filter} + ), cumulative_energy AS ( + SELECT + id, + source_id, + time_step, + energy_model_id, + speed, + acceleration, + cumulative_power * sim_step AS energy_joules + FROM joined_trace + ), binned_cumulative_energy AS ( + SELECT + source_id, + CAST(time_step/10 AS INTEGER) * 10 AS time_seconds_bin, + AVG(speed) AS speed_avg, + AVG(speed) + STDDEV(speed) AS speed_upper_bound, + AVG(speed) - STDDEV(speed) AS speed_lower_bound, + AVG(acceleration) AS accel_avg, + AVG(acceleration) + STDDEV(acceleration) AS accel_upper_bound, + AVG(acceleration) - STDDEV(acceleration) AS accel_lower_bound, + AVG(energy_joules) AS cumulative_energy_avg, + AVG(energy_joules) + STDDEV(energy_joules) AS cumulative_energy_upper_bound, + AVG(energy_joules) - STDDEV(energy_joules) AS cumulative_energy_lower_bound + FROM cumulative_energy + GROUP BY 1, 2 + HAVING 1 = 1 + AND COUNT(DISTINCT time_step) > 1 + ), binned_energy_start_end AS ( + SELECT DISTINCT + source_id, + id, + CAST(time_step/10 AS INTEGER) * 10 AS time_seconds_bin, + FIRST_VALUE(energy_joules) + OVER (PARTITION BY id, CAST(time_step/10 AS INTEGER) * 10 + ORDER BY time_step ASC) AS energy_start, + LAST_VALUE(energy_joules) + OVER (PARTITION BY id, CAST(time_step/10 AS INTEGER) * 10 + ORDER BY time_step ASC) AS energy_end + FROM cumulative_energy + ), binned_energy AS ( + SELECT + source_id, + time_seconds_bin, + AVG(energy_end - energy_start) AS instantaneous_energy_avg, + AVG(energy_end - energy_start) + STDDEV(energy_end - energy_start) AS instantaneous_energy_upper_bound, + AVG(energy_end - energy_start) - STDDEV(energy_end - energy_start) AS instantaneous_energy_lower_bound + FROM binned_energy_start_end + GROUP BY 1, 2 + ) + SELECT + bce.source_id AS source_id, + bce.time_seconds_bin AS time_seconds_bin, + bce.cumulative_energy_avg, + bce.cumulative_energy_lower_bound, + bce.cumulative_energy_upper_bound, + bce.speed_avg, + bce.speed_upper_bound, + bce.speed_lower_bound, + bce.accel_avg, + bce.accel_upper_bound, + bce.accel_lower_bound, + COALESCE(be.instantaneous_energy_avg, 0) AS instantaneous_energy_avg, + COALESCE(be.instantaneous_energy_upper_bound, 0) AS instantaneous_energy_upper_bound, + COALESCE(be.instantaneous_energy_lower_bound, 0) AS instantaneous_energy_lower_bound + FROM binned_cumulative_energy bce + JOIN binned_energy be ON 1 = 1 + AND bce.source_id = be.source_id + AND bce.time_seconds_bin = be.time_seconds_bin + ORDER BY time_seconds_bin ASC + ;""" + + FACT_VEHICLE_COUNTS_BY_TIME = """ + WITH counts AS ( + SELECT + vt.source_id, + vt.time_step, + COUNT(DISTINCT vt.id) AS vehicle_count + FROM fact_vehicle_trace vt + WHERE 1 = 1 + AND vt.date = \'{date}\' + AND vt.partition_name = \'{partition}\' + AND vt.{inflow_filter} + AND vt.{outflow_filter} + AND vt.time_step >= {start_filter} + GROUP BY 1, 2 + ) + SELECT + source_id, + time_step - FIRST_VALUE(time_step) + OVER (PARTITION BY source_id ORDER BY time_step ASC) AS time_step, + vehicle_count + FROM counts + ; + """ + + LEADERBOARD_CHART_AGG = """ + WITH agg AS ( + SELECT + l.date AS submission_date, + m.submission_time, + l.source_id, + m.submitter_name, + m.strategy, + m.network, + m.is_baseline, + COALESCE (m.penetration_rate, 'x') AS penetration_rate, + COALESCE (m.version, '2.0') AS version, + COALESCE (m.road_grade, 'False') AS road_grade, + COALESCE (m.on_ramp, 'False') AS on_ramp, + l.prius_efficiency_meters_per_kilojoules, + l.tacoma_efficiency_meters_per_kilojoules, + l.prius_efficiency_miles_per_gallon, + l.tacoma_efficiency_miles_per_gallon, + l.throughput_per_hour, + l.avg_instantaneous_speed, + l.avg_network_speed, + l.safety_rate, + l.safety_value_max, + b.source_id AS baseline_source_id + FROM leaderboard_chart AS l, metadata_table AS m, baseline_table as b + WHERE 1 = 1 + AND l.source_id = m.source_id + AND m.network = b.network + AND (m.is_baseline='False' + OR (m.is_baseline='True' + AND m.source_id = b.source_id)) + ), joined_cols AS ( + SELECT + agg.submission_date, + agg.source_id, + agg.submitter_name, + agg.strategy, + agg.network || ';' || + ' v' || agg.version || ';' || + ' PR: ' || agg.penetration_rate || '%;' || + CASE agg.on_ramp WHEN + 'True' THEN ' with ramps;' + ELSE ' no ramps;' END || + CASE agg.road_grade WHEN + 'True' THEN ' with grade;' + ELSE ' no grade;' END AS network, + agg.is_baseline, + agg.prius_efficiency_miles_per_gallon, + agg.tacoma_efficiency_miles_per_gallon, + 100 * (1 - + baseline.prius_efficiency_meters_per_kilojoules / agg.prius_efficiency_meters_per_kilojoules) + AS prius_fuel_economy_improvement, + 100 * (1 - + baseline.tacoma_efficiency_meters_per_kilojoules / agg.tacoma_efficiency_meters_per_kilojoules) + AS tacoma_fuel_economy_improvement, + agg.throughput_per_hour, + 100 * (agg.throughput_per_hour - baseline.throughput_per_hour) / baseline.throughput_per_hour + AS throughput_change, + agg.avg_network_speed, + 100 * (agg.avg_network_speed - baseline.avg_network_speed) / baseline.avg_network_speed + AS speed_change, + agg.safety_rate, + agg.safety_value_max + FROM agg + JOIN agg AS baseline ON 1 = 1 + AND agg.network = baseline.network + AND agg.version = baseline.version + AND agg.on_ramp = baseline.on_ramp + AND agg.road_grade = baseline.road_grade + AND baseline.is_baseline = 'True' + AND agg.baseline_source_id = baseline.source_id + ) + SELECT + submission_date, + source_id, + submitter_name, + strategy, + network, + is_baseline, + tacoma_efficiency_miles_per_gallon, + prius_efficiency_miles_per_gallon, + CAST (ROUND(tacoma_efficiency_miles_per_gallon, 1) AS VARCHAR) || + '; ' || CAST (ROUND(prius_efficiency_miles_per_gallon, 1) AS VARCHAR) || + ' (' || (CASE WHEN SIGN(tacoma_fuel_economy_improvement) = 1 THEN '+' ELSE '' END) || + CAST (ROUND(tacoma_fuel_economy_improvement, 1) AS VARCHAR) || '%; ' || + (CASE WHEN SIGN(prius_fuel_economy_improvement) = 1 THEN '+' ELSE '' END) || + CAST (ROUND(prius_fuel_economy_improvement, 1) AS VARCHAR) || '%)' AS efficiency, + CAST (ROUND(throughput_per_hour, 1) AS VARCHAR) || + ' (' || (CASE WHEN SIGN(throughput_change) = 1 THEN '+' ELSE '' END) || + CAST (ROUND(throughput_change, 1) AS VARCHAR) || '%)' AS inflow, + CAST (ROUND(avg_network_speed, 1) AS VARCHAR) || + ' (' || (CASE WHEN SIGN(speed_change) = 1 THEN '+' ELSE '' END) || + CAST (ROUND(speed_change, 1) AS VARCHAR) || '%)' AS speed, + ROUND(safety_rate, 1) AS safety_rate, + ROUND(safety_value_max, 1) AS safety_value_max + FROM joined_cols + ;""" + + FACT_TOP_SCORES = """ + WITH curr_max AS ( + SELECT + network, + submission_date, + MAX(tacoma_efficiency_miles_per_gallon) + OVER (PARTITION BY network ORDER BY submission_date ASC + ROWS BETWEEN UNBOUNDED PRECEDING and CURRENT ROW) AS tacoma_max_score, + MAX(prius_efficiency_miles_per_gallon) + OVER (PARTITION BY network ORDER BY submission_date ASC + ROWS BETWEEN UNBOUNDED PRECEDING and CURRENT ROW) AS prius_max_score + FROM leaderboard_chart_agg + WHERE 1 = 1 + AND is_baseline = 'False' + ), prev_max AS ( + SELECT + network, + submission_date, + LAG(tacoma_max_score, 1) OVER (PARTITION BY network ORDER BY submission_date ASC) AS tacoma_max_score, + LAG(prius_max_score, 1) OVER (PARTITION BY network ORDER BY submission_date ASC) AS prius_max_score + FROM curr_max + ), unioned AS ( + SELECT * FROM curr_max + UNION ALL + SELECT * FROM prev_max + ) + SELECT DISTINCT * + FROM unioned + WHERE 1 = 1 + AND tacoma_max_score IS NOT NULL + AND prius_max_score IS NOT NULL + ORDER BY 1, 2, 3 + ;""" diff --git a/flow/data_pipeline/run_query.py b/flow/data_pipeline/run_query.py new file mode 100644 index 000000000..1eb802205 --- /dev/null +++ b/flow/data_pipeline/run_query.py @@ -0,0 +1,35 @@ +"""runner script for invoking query manually.""" +import argparse +from flow.data_pipeline.data_pipeline import AthenaQuery +from flow.data_pipeline.query import QueryStrings + +parser = argparse.ArgumentParser(prog="run_query", description="runs query on AWS Athena and stores the result to" + "a S3 location") +parser.add_argument("--run", type=str, nargs="+") +parser.add_argument("--result_location", type=str, nargs='?', default="s3://circles.data.pipeline/query-result/") +parser.add_argument("--partition", type=str, nargs='?', default="default") +parser.add_argument("--list_partitions", action="store_true") +parser.add_argument("--check_status", type=str, nargs='+') +parser.add_argument("--list_queries", action="store_true") +parser.add_argument("--test_query", nargs=1) + + +if __name__ == "__main__": + args = parser.parse_args() + queryEngine = AthenaQuery() + + if args.run: + execution_ids = [] + for query_name in args.run: + execution_ids.append(queryEngine.run_query(query_name, args.result_location, partition=args.partition)) + print(execution_ids) + if args.list_partitions: + print(queryEngine.existing_partitions) + if args.check_status: + status = dict() + for execution_id in args.check_status: + status[execution_id] = queryEngine.check_status(execution_id) + print(status) + if args.list_queries: + for q in QueryStrings: + print(q) diff --git a/flow/energy_models/base_energy.py b/flow/energy_models/base_energy.py new file mode 100644 index 000000000..ba5da5080 --- /dev/null +++ b/flow/energy_models/base_energy.py @@ -0,0 +1,62 @@ +"""Script containing the base vehicle energy class.""" +from abc import ABCMeta, abstractmethod + + +class BaseEnergyModel(metaclass=ABCMeta): + """Base energy model class. + + Calculate the instantaneous power consumption of a vehicle in + the network. It returns the power in Watts regardless of the + vehicle type: whether EV or Combustion Engine, Toyota Prius or Tacoma + or non-Toyota vehicles. Non-Toyota vehicles are set by default + to be an averaged-size vehicle. + + Note: road grade is included as an input parameter, but the + functional dependence on road grade is not yet implemented. + """ + + def __init__(self): + # 15 kilowatts = 1 gallon/hour conversion factor + self.conversion = 15e3 + + @abstractmethod + def get_instantaneous_power(self, accel, speed, grade): + """Calculate the instantaneous power consumption of a vehicle. + + Must be implemented by child classes. + + Parameters + ---------- + accel : float + Instantaneous acceleration of the vehicle + speed : float + Instantaneous speed of the vehicle + grade : float + Instantaneous road grade of the vehicle + + Returns + ------- + float + """ + pass + + def get_instantaneous_fuel_consumption(self, accel, speed, grade): + """Calculate the instantaneous fuel consumption of a vehicle. + + Fuel consumption is reported in gallons per hour, with the conversion + rate of 15kW = 1 gallon/hour. + + Parameters + ---------- + accel : float + Instantaneous acceleration of the vehicle + speed : float + Instantaneous speed of the vehicle + grade : float + Instantaneous road grade of the vehicle + + Returns + ------- + float + """ + return self.get_instantaneous_power(accel, speed, grade) / self.conversion diff --git a/flow/energy_models/power_demand.py b/flow/energy_models/power_demand.py new file mode 100644 index 000000000..2fe34d1ed --- /dev/null +++ b/flow/energy_models/power_demand.py @@ -0,0 +1,139 @@ +"""Script containing the vehicle power demand model energy classes.""" +from abc import ABCMeta +import numpy as np + +from flow.energy_models.base_energy import BaseEnergyModel + + +class PowerDemandModel(BaseEnergyModel, metaclass=ABCMeta): + """Vehicle Power Demand base energy model class. + + Calculate power consumption of a vehicle based on physics + derivation. Assumes some vehicle characteristics. The + power calculated here is the lower bound of the actual + power consumed by the vehicle plus a bilinear polynomial + function used as a correction factor. + """ + + def __init__(self, + mass=2041, + idle_coeff=3405.5481762, + linear_friction_coeff=83.123929917, + quadratic_friction_coeff=6.7650718327, + drag_coeff=0.7041355229, + p0_correction=0, + p1_correction=0, + p2_correction=0, + p3_correction=0): + super(PowerDemandModel, self).__init__() + + self.mass = mass + self.phys_power_coeffs = np.array([idle_coeff, + linear_friction_coeff, + quadratic_friction_coeff, + drag_coeff]) + self.power_correction_coeffs = np.array([p0_correction, + p1_correction, + p2_correction, + p3_correction]) + + def calculate_phys_power(self, accel, speed, grade): + """Calculate the instantaneous power from physics-based derivation. + + Parameters + ---------- + accel : float + Instantaneous acceleration of the vehicle + speed : float + Instantaneous speed of the vehicle + grade : float + Instantaneous road grade of the vehicle + Returns + ------- + float + """ + state_variables = np.array([1, speed, speed**2, speed**3]) + power_0 = np.dot(self.phys_power_coeffs, state_variables) + return self.mass * accel * speed + power_0 + + def get_power_correction_factor(self, accel, speed, grade): + """Calculate the instantaneous power correction of a vehicle. + + Parameters + ---------- + accel : float + Instantaneous acceleration of the vehicle + speed : float + Instantaneous speed of the vehicle + grade : float + Instantaneous road grade of the vehicle + + Returns + ------- + float + """ + state_variables = np.array([1, accel, speed, accel * speed]) + return max(0, np.dot(self.power_correction_coeffs, state_variables)) + + def get_instantaneous_power(self, accel, speed, grade): + """See parent class.""" + phys_power = self.calculate_phys_power(accel, speed, grade) + power_correction_factor = self.get_power_correction_factor(accel, speed, grade) + return phys_power + power_correction_factor + + +class PDMCombustionEngine(PowerDemandModel): + """Power Demand Model for a combustion engine vehicle. + + For more information, see docs/Tacoma_EnergyModel.pdf + """ + + def __init__(self, + mass=2041, + idle_coeff=3405.5481762, + linear_friction_coeff=83.123929917, + quadratic_friction_coeff=6.7650718327, + drag_coeff=0.7041355229, + p1_correction=4598.7155, + p3_correction=975.12719): + super(PDMCombustionEngine, self).__init__(mass=mass, + idle_coeff=idle_coeff, + linear_friction_coeff=linear_friction_coeff, + quadratic_friction_coeff=quadratic_friction_coeff, + drag_coeff=drag_coeff, + p1_correction=p1_correction, + p3_correction=p3_correction) + + def calculate_phys_power(self, accel, speed, grade): + """See parent class.""" + return max(super(PDMCombustionEngine, self).calculate_phys_power(accel, speed, grade), 0) + + +class PDMElectric(PowerDemandModel): + """Power Demand Model for an electric vehicle. + + For more information, see docs/Prius_EnergyModel.pdf + """ + + def __init__(self, + mass=1663, + idle_coeff=1.046, + linear_friction_coeff=119.166, + quadratic_friction_coeff=0.337, + drag_coeff=0.383, + p3_correction=296.66, + alpha=0.869, + beta=2338): + super(PDMElectric, self).__init__(mass=mass, + idle_coeff=idle_coeff, + linear_friction_coeff=linear_friction_coeff, + quadratic_friction_coeff=quadratic_friction_coeff, + drag_coeff=drag_coeff, + p3_correction=p3_correction) + self.alpha = alpha + self.beta = beta + + def get_instantaneous_power(self, accel, speed, grade): + """See parent class.""" + mod_power = super(PDMElectric, self).get_instantaneous_power(accel, speed, grade) + return max(mod_power, self.alpha * mod_power, -self.beta * speed) diff --git a/flow/energy_models/toyota_energy.py b/flow/energy_models/toyota_energy.py new file mode 100644 index 000000000..397610089 --- /dev/null +++ b/flow/energy_models/toyota_energy.py @@ -0,0 +1,66 @@ +"""Script containing the Toyota energy classes.""" +from abc import ABCMeta, abstractmethod +import dill as pickle +import boto3 +import os + +from flow.energy_models.base_energy import BaseEnergyModel + + +class ToyotaModel(BaseEnergyModel, metaclass=ABCMeta): + """Base Toyota Energy model class.""" + + def __init__(self, filename): + super(ToyotaModel, self).__init__() + + # download file from s3 bucket + s3 = boto3.client('s3') + s3.download_file('toyota.restricted', filename, 'temp.pkl') + + with open('temp.pkl', 'rb') as file: + try: + self.toyota_energy = pickle.load(file) + # delete pickle file + os.remove('temp.pkl') + except TypeError: + print('Must use Python version 3.6.8 to unpickle') + # delete pickle file + os.remove('temp.pkl') + raise + + @abstractmethod + def get_instantaneous_power(self, accel, speed, grade): + """See parent class.""" + pass + + +class PriusEnergy(ToyotaModel): + """Toyota Prius (EV) energy model class.""" + + def __init__(self, sim_step, soc=0.9): + super(PriusEnergy, self).__init__(filename='prius_ev.pkl') + self.sim_step = sim_step + self.soc = soc + + def get_instantaneous_power(self, accel, speed, grade): + """See parent class.""" + socdot = self.toyota_energy(self.soc, accel, speed, grade) + self.soc -= socdot * self.sim_step + # FIXME (Joy): convert socdot to power + return socdot + + +class TacomaEnergy(ToyotaModel): + """Toyota Tacoma energy model class.""" + + def __init__(self): + super(TacomaEnergy, self).__init__(filename='tacoma.pkl') + + def get_instantaneous_power(self, accel, speed, grade): + """See parent class.""" + return self.get_instantaneous_fuel_consumption(accel, speed, grade) * self.conversion + + def get_instantaneous_fuel_consumption(self, accel, speed, grade): + """See parent class.""" + fc = self.toyota_energy(accel, speed, grade) + return fc * 3600.0 / 3217.25 diff --git a/flow/envs/__init__.py b/flow/envs/__init__.py index 5befe6a33..8bea3dd4f 100755 --- a/flow/envs/__init__.py +++ b/flow/envs/__init__.py @@ -4,13 +4,14 @@ from flow.envs.bottleneck import BottleneckAccelEnv, BottleneckEnv, \ BottleneckDesiredVelocityEnv from flow.envs.traffic_light_grid import TrafficLightGridEnv, \ - TrafficLightGridPOEnv, TrafficLightGridTestEnv + TrafficLightGridPOEnv, TrafficLightGridTestEnv, TrafficLightGridBenchmarkEnv from flow.envs.ring.lane_change_accel import LaneChangeAccelEnv, \ LaneChangeAccelPOEnv from flow.envs.ring.accel import AccelEnv from flow.envs.ring.wave_attenuation import WaveAttenuationEnv, \ WaveAttenuationPOEnv from flow.envs.merge import MergePOEnv +from flow.envs.straightroad_env import SingleStraightRoad from flow.envs.test import TestEnv # deprecated classes whose names have changed @@ -33,9 +34,11 @@ 'WaveAttenuationPOEnv', 'TrafficLightGridEnv', 'TrafficLightGridPOEnv', + 'TrafficLightGridBenchmarkEnv', 'BottleneckDesiredVelocityEnv', 'TestEnv', 'BayBridgeEnv', + 'SingleStraightRoad', # deprecated classes 'BottleNeckAccelEnv', 'DesiredVelocityEnv', diff --git a/flow/envs/base.py b/flow/envs/base.py index de50f2447..2efeecbb1 100644 --- a/flow/envs/base.py +++ b/flow/envs/base.py @@ -1,5 +1,6 @@ """Base environment class. This is the parent of all other environments.""" +from abc import ABCMeta, abstractmethod from copy import deepcopy import os import atexit @@ -24,8 +25,10 @@ from flow.core.kernel import Kernel from flow.utils.exceptions import FatalFlowError +from flow.data_pipeline.data_pipeline import get_extra_info -class Env(gym.Env): + +class Env(gym.Env, metaclass=ABCMeta): """Base environment class. Provides the interface for interacting with various aspects of a traffic @@ -152,6 +155,13 @@ def __init__(self, self.state = None self.obs_var_labels = [] + # number of training iterations (used by the rllib training procedure) + self._num_training_iters = 0 + + # track IDs that have ever been observed in the system + self._observed_ids = set() + self._observed_rl_ids = set() + # simulation step size self.sim_step = sim_params.sim_step @@ -326,6 +336,11 @@ def step(self, rl_actions): contains other diagnostic information from the previous action """ for _ in range(self.env_params.sims_per_step): + # This tracks vehicles that have appeared during warmup steps + if self.time_counter <= self.env_params.sims_per_step * self.env_params.warmup_steps: + self._observed_ids.update(self.k.vehicle.get_ids()) + self._observed_rl_ids.update(self.k.vehicle.get_rl_ids()) + self.time_counter += 1 self.step_counter += 1 @@ -400,8 +415,7 @@ def step(self, rl_actions): # test if the environment should terminate due to a collision or the # time horizon being met done = (self.time_counter >= self.env_params.sims_per_step * - (self.env_params.warmup_steps + self.env_params.horizon) - or crash) + (self.env_params.warmup_steps + self.env_params.horizon)) # compute the info for each agent infos = {} @@ -434,6 +448,10 @@ def reset(self): # reset the time counter self.time_counter = 0 + # reset the observed ids + self._observed_ids = set() + self._observed_rl_ids = set() + # Now that we've passed the possibly fake init steps some rl libraries # do, we can feel free to actually render things if self.should_render: @@ -568,6 +586,14 @@ def reset(self): # perform (optional) warm-up steps before training for _ in range(self.env_params.warmup_steps): observation, _, _, _ = self.step(rl_actions=None) + # collect data for pipeline during the warmup period + try: + extra_info, source_id, run_id = self.pipeline_params + veh_ids = self.k.vehicle.get_ids() + get_extra_info(self.k.vehicle, extra_info, veh_ids, source_id, run_id) + # In case the attribute `pipeline_params` if not added to this instance + except AttributeError: + pass # render a frame self.render(reset=True) @@ -629,9 +655,11 @@ def apply_rl_actions(self, rl_actions=None): rl_clipped = self.clip_actions(rl_actions) self._apply_rl_actions(rl_clipped) + @abstractmethod def _apply_rl_actions(self, rl_actions): - raise NotImplementedError + pass + @abstractmethod def get_state(self): """Return the state of the simulation as perceived by the RL agent. @@ -643,9 +671,10 @@ def get_state(self): information on the state of the vehicles, which is provided to the agent """ - raise NotImplementedError + pass @property + @abstractmethod def action_space(self): """Identify the dimensions and bounds of the action space. @@ -656,9 +685,10 @@ def action_space(self): gym Box or Tuple type a bounded box depicting the shape and bounds of the action space """ - raise NotImplementedError + pass @property + @abstractmethod def observation_space(self): """Identify the dimensions and bounds of the observation space. @@ -670,7 +700,7 @@ def observation_space(self): a bounded box depicting the shape and bounds of the observation space """ - raise NotImplementedError + pass def compute_reward(self, rl_actions, **kwargs): """Reward function for the RL agent(s). @@ -812,3 +842,7 @@ def pyglet_render(self): sight = self.renderer.get_sight( orientation, id) self.sights.append(sight) + + def set_iteration_num(self): + """Increment the number of training iterations.""" + self._num_training_iters += 1 diff --git a/flow/envs/bay_bridge.py b/flow/envs/bay_bridge.py index dcc5e991d..0c5570367 100644 --- a/flow/envs/bay_bridge.py +++ b/flow/envs/bay_bridge.py @@ -234,6 +234,22 @@ def compute_reward(self, rl_actions, **kwargs): # The below methods need to be updated by child classes. # ########################################################################### + @property + def action_space(self): + """See parent class. + + To be implemented by child classes. + """ + pass + + @property + def observation_space(self): + """See parent class. + + To be implemented by child classes. + """ + pass + def _apply_rl_actions(self, rl_actions): """See parent class. diff --git a/flow/envs/bottleneck.py b/flow/envs/bottleneck.py index c647e1d6e..a5f1508f9 100644 --- a/flow/envs/bottleneck.py +++ b/flow/envs/bottleneck.py @@ -472,6 +472,13 @@ def observation_space(self): shape=(1, ), dtype=np.float32) + def _apply_rl_actions(self, rl_actions): + """See parent class. + + To be implemented by child classes. + """ + pass + def compute_reward(self, rl_actions, **kwargs): """Outflow rate over last ten seconds normalized to max of 1.""" reward = self.k.vehicle.get_outflow_rate(10 * self.sim_step) / \ diff --git a/flow/envs/multiagent/__init__.py b/flow/envs/multiagent/__init__.py index f7889591d..8c5552580 100644 --- a/flow/envs/multiagent/__init__.py +++ b/flow/envs/multiagent/__init__.py @@ -10,7 +10,8 @@ from flow.envs.multiagent.traffic_light_grid import MultiTrafficLightGridPOEnv from flow.envs.multiagent.highway import MultiAgentHighwayPOEnv from flow.envs.multiagent.merge import MultiAgentMergePOEnv -from flow.envs.multiagent.i210 import I210MultiEnv +from flow.envs.multiagent.i210 import I210MultiEnv, MultiStraightRoad + __all__ = [ 'MultiEnv', @@ -21,5 +22,6 @@ 'MultiAgentAccelPOEnv', 'MultiAgentWaveAttenuationPOEnv', 'MultiAgentMergePOEnv', - 'I210MultiEnv' + 'I210MultiEnv', + 'MultiStraightRoad', ] diff --git a/flow/envs/multiagent/base.py b/flow/envs/multiagent/base.py index ec95474c6..8f3c0bd22 100644 --- a/flow/envs/multiagent/base.py +++ b/flow/envs/multiagent/base.py @@ -49,6 +49,10 @@ def step(self, rl_actions): contains other diagnostic information from the previous action """ for _ in range(self.env_params.sims_per_step): + if self.time_counter <= self.env_params.sims_per_step * self.env_params.warmup_steps: + self._observed_ids.update(self.k.vehicle.get_ids()) + self._observed_rl_ids.update(self.k.vehicle.get_rl_ids()) + self.time_counter += 1 self.step_counter += 1 @@ -103,6 +107,7 @@ def step(self, rl_actions): # stop collecting new simulation steps if there is a collision if crash: + print('A CRASH! A CRASH!!!!!! AAAAAAAAAH!!!!!') break states = self.get_state() @@ -122,11 +127,11 @@ def step(self, rl_actions): else: reward = self.compute_reward(rl_actions, fail=crash) - for rl_id in self.k.vehicle.get_arrived_rl_ids(): - done[rl_id] = True - reward[rl_id] = 0 - states[rl_id] = np.zeros(self.observation_space.shape[0]) - + if self.env_params.done_at_exit: + for rl_id in self.k.vehicle.get_arrived_rl_ids(self.env_params.sims_per_step): + done[rl_id] = True + reward[rl_id] = 0 + states[rl_id] = -1 * np.ones(self.observation_space.shape[0]) return states, reward, done, infos def reset(self, new_inflow_rate=None): @@ -148,12 +153,17 @@ def reset(self, new_inflow_rate=None): # reset the time counter self.time_counter = 0 + # reset the observed ids + self._observed_ids = set() + self._observed_rl_ids = set() + # Now that we've passed the possibly fake init steps some rl libraries # do, we can feel free to actually render things if self.should_render: self.sim_params.render = True # got to restart the simulation to make it actually display anything self.restart_simulation(self.sim_params) + self.should_render = False # warn about not using restart_instance when using inflows if len(self.net_params.inflows.get()) > 0 and \ diff --git a/flow/envs/multiagent/i210.py b/flow/envs/multiagent/i210.py index 409aeb14f..ea2ff640a 100644 --- a/flow/envs/multiagent/i210.py +++ b/flow/envs/multiagent/i210.py @@ -3,19 +3,27 @@ from gym.spaces import Box import numpy as np -from flow.core.rewards import average_velocity +from flow.core.rewards import instantaneous_mpg from flow.envs.multiagent.base import MultiEnv # largest number of lanes on any given edge in the network MAX_LANES = 6 +SPEED_SCALE = 50 +HEADWAY_SCALE = 1000 ADDITIONAL_ENV_PARAMS = { # maximum acceleration for autonomous vehicles, in m/s^2 "max_accel": 1, # maximum deceleration for autonomous vehicles, in m/s^2 "max_decel": 1, - # whether we use an obs space that contains adjacent lane info or just the lead obs + # whether we use an obs space that contains adjacent lane info or just the + # lead obs "lead_obs": True, + # whether the reward should come from local vehicles instead of global + # rewards + "local_reward": True, + # desired velocity + "target_velocity": 25 } @@ -59,6 +67,40 @@ class I210MultiEnv(MultiEnv): def __init__(self, env_params, sim_params, network, simulator='traci'): super().__init__(env_params, sim_params, network, simulator) self.lead_obs = env_params.additional_params.get("lead_obs") + self.reroute_on_exit = env_params.additional_params.get("reroute_on_exit") + self.max_lanes = MAX_LANES + self.num_enter_lanes = 5 + self.entrance_edge = "ghost0" + self.exit_edge = "119257908#3" + self.control_range = env_params.additional_params.get('control_range', None) + self.no_control_edges = env_params.additional_params.get('no_control_edges', []) + self.mpg_reward = env_params.additional_params["mpg_reward"] + self.look_back_length = env_params.additional_params["look_back_length"] + + # list of all RL vehicles (even out of network) after reroute_on_exit starts + self.reroute_rl_ids = set() + + # whether to add a slight reward for opening up a gap that will be annealed out N iterations in + self.headway_curriculum = env_params.additional_params["headway_curriculum"] + # how many timesteps to anneal the headway curriculum over + self.headway_curriculum_iters = env_params.additional_params["headway_curriculum_iters"] + self.headway_reward_gain = env_params.additional_params["headway_reward_gain"] + self.min_time_headway = env_params.additional_params["min_time_headway"] + + # whether to add a slight reward for opening up a gap that will be annealed out N iterations in + self.speed_curriculum = env_params.additional_params["speed_curriculum"] + # how many timesteps to anneal the headway curriculum over + self.speed_curriculum_iters = env_params.additional_params["speed_curriculum_iters"] + self.speed_reward_gain = env_params.additional_params["speed_reward_gain"] + self.leader = [] + + # penalize stops + self.penalize_stops = env_params.additional_params["penalize_stops"] + self.stop_penalty = env_params.additional_params["stop_penalty"] + + # penalize accel + self.penalize_accel = env_params.additional_params.get("penalize_accel", False) + self.accel_penalty = env_params.additional_params["accel_penalty"] @property def observation_space(self): @@ -74,8 +116,8 @@ def observation_space(self): # speed, dist to ego vehicle, binary value which is 1 if the vehicle is # an AV else: - leading_obs = 3 * MAX_LANES - follow_obs = 3 * MAX_LANES + leading_obs = 3 * self.max_lanes + follow_obs = 3 * self.max_lanes # speed and lane self_obs = 2 @@ -99,90 +141,208 @@ def action_space(self): def _apply_rl_actions(self, rl_actions): """See class definition.""" # in the warmup steps, rl_actions is None + id_list = [] + accel_list = [] if rl_actions: for rl_id, actions in rl_actions.items(): accel = actions[0] + id_list.append(rl_id) + accel_list.append(accel) + self.k.vehicle.apply_acceleration(id_list, accel_list) - # lane_change_softmax = np.exp(actions[1:4]) - # lane_change_softmax /= np.sum(lane_change_softmax) - # lane_change_action = np.random.choice([-1, 0, 1], - # p=lane_change_softmax) + def in_control_range(self, veh_id): + """Return if a veh_id is on an edge that is allowed to be controlled. - self.k.vehicle.apply_acceleration(rl_id, accel) - # self.k.vehicle.apply_lane_change(rl_id, lane_change_action) + If control range is defined it uses control range, otherwise it searches over a set of edges + """ + return (self.control_range and self.control_range[1] > + self.k.vehicle.get_x_by_id(veh_id) > self.control_range[0]) or \ + (len(self.no_control_edges) > 0 and self.k.vehicle.get_edge(veh_id) not in + self.no_control_edges) def get_state(self): """See class definition.""" + valid_ids = [rl_id for rl_id in self.k.vehicle.get_rl_ids() if self.in_control_range(rl_id)] if self.lead_obs: veh_info = {} - for rl_id in self.k.vehicle.get_rl_ids(): + for rl_id in valid_ids: speed = self.k.vehicle.get_speed(rl_id) - headway = self.k.vehicle.get_headway(rl_id) - lead_speed = self.k.vehicle.get_speed(self.k.vehicle.get_leader(rl_id)) - if lead_speed == -1001: - lead_speed = 0 - veh_info.update({rl_id: np.array([speed / 50.0, headway / 1000.0, lead_speed / 50.0])}) + lead_id = self.k.vehicle.get_leader(rl_id) + if lead_id in ["", None]: + # in case leader is not visible + lead_speed = SPEED_SCALE + headway = HEADWAY_SCALE + else: + lead_speed = self.k.vehicle.get_speed(lead_id) + headway = self.k.vehicle.get_headway(rl_id) + veh_info.update({rl_id: np.array([speed / SPEED_SCALE, headway / HEADWAY_SCALE, + lead_speed / SPEED_SCALE])}) else: veh_info = {rl_id: np.concatenate((self.state_util(rl_id), self.veh_statistics(rl_id))) - for rl_id in self.k.vehicle.get_rl_ids()} + for rl_id in valid_ids} return veh_info def compute_reward(self, rl_actions, **kwargs): - # TODO(@evinitsky) we need something way better than this. Something that adds - # in notions of local reward """See class definition.""" # in the warmup steps if rl_actions is None: return {} rewards = {} - for rl_id in self.k.vehicle.get_rl_ids(): - if self.env_params.evaluate: - # reward is speed of vehicle if we are in evaluation mode - reward = self.k.vehicle.get_speed(rl_id) - elif kwargs['fail']: - # reward is 0 if a collision occurred - reward = 0 - else: - # reward high system-level velocities - cost1 = average_velocity(self, fail=kwargs['fail']) - - # penalize small time headways - cost2 = 0 - t_min = 1 # smallest acceptable time headway + valid_ids = [rl_id for rl_id in self.k.vehicle.get_rl_ids() if self.in_control_range(rl_id)] + valid_human_ids = [veh_id for veh_id in self.k.vehicle.get_ids() if self.in_control_range(veh_id)] + + if self.env_params.additional_params["local_reward"]: + des_speed = self.env_params.additional_params["target_velocity"] + for rl_id in valid_ids: + rewards[rl_id] = 0 + if self.mpg_reward: + rewards[rl_id] = instantaneous_mpg(self, rl_id, gain=1.0) / 100.0 + follow_id = rl_id + for i in range(self.look_back_length): + follow_id = self.k.vehicle.get_follower(follow_id) + if follow_id not in ["", None]: + rewards[rl_id] += instantaneous_mpg(self, follow_id, gain=1.0) / 100.0 + else: + break + else: + follow_id = rl_id + for i in range(self.look_back_length + 1): + if follow_id not in ["", None]: + follow_speed = self.k.vehicle.get_speed(self.k.vehicle.get_follower(follow_id)) + reward = (des_speed - min(np.abs(follow_speed - des_speed), des_speed)) ** 2 + reward /= ((des_speed ** 2) * self.look_back_length) + rewards[rl_id] += reward + else: + break + follow_id = self.k.vehicle.get_follower(follow_id) - lead_id = self.k.vehicle.get_leader(rl_id) + else: + if self.mpg_reward: + reward = np.nan_to_num(instantaneous_mpg(self, valid_human_ids, gain=1.0)) / 100.0 + else: + speeds = self.k.vehicle.get_speed(valid_human_ids) + des_speed = self.env_params.additional_params["target_velocity"] + # rescale so the critic can estimate it quickly + if self.reroute_on_exit: + reward = np.nan_to_num(np.mean([(des_speed - np.abs(speed - des_speed)) + for speed in speeds]) / des_speed) + else: + reward = np.nan_to_num(np.mean([(des_speed - np.abs(speed - des_speed)) ** 2 + for speed in speeds]) / (des_speed ** 2)) + rewards = {rl_id: reward for rl_id in valid_ids} + + # curriculum over time-gaps + if self.headway_curriculum and self._num_training_iters <= self.headway_curriculum_iters: + t_min = self.min_time_headway # smallest acceptable time headway + for veh_id, rew in rewards.items(): + lead_id = self.k.vehicle.get_leader(veh_id) + penalty = 0 if lead_id not in ["", None] \ - and self.k.vehicle.get_speed(rl_id) > 0: + and self.k.vehicle.get_speed(veh_id) > 0: t_headway = max( - self.k.vehicle.get_headway(rl_id) / - self.k.vehicle.get_speed(rl_id), 0) - cost2 += min((t_headway - t_min) / t_min, 0) - - # weights for cost1, cost2, and cost3, respectively - eta1, eta2 = 1.00, 0.10 - - reward = max(eta1 * cost1 + eta2 * cost2, 0) - - rewards[rl_id] = reward + self.k.vehicle.get_headway(veh_id) / + self.k.vehicle.get_speed(veh_id), 0) + scaling_factor = max(0, 1 - self._num_training_iters / self.headway_curriculum_iters) + penalty += scaling_factor * self.headway_reward_gain * min((t_headway - t_min) / t_min, 0) + + rewards[veh_id] += penalty + + if self.speed_curriculum and self._num_training_iters <= self.speed_curriculum_iters: + des_speed = self.env_params.additional_params["target_velocity"] + + for veh_id, rew in rewards.items(): + speed = self.k.vehicle.get_speed(veh_id) + speed_reward = 0.0 + follow_id = veh_id + for i in range(self.look_back_length): + follow_id = self.k.vehicle.get_follower(follow_id) + if follow_id not in ["", None]: + if self.reroute_on_exit: + speed_reward += (des_speed - np.abs(speed - des_speed)) / des_speed + else: + speed_reward += ((des_speed - np.abs(speed - des_speed)) ** 2) / (des_speed ** 2) + else: + break + scaling_factor = max(0, 1 - self._num_training_iters / self.speed_curriculum_iters) + + rewards[veh_id] += speed_reward * scaling_factor * self.speed_reward_gain + + for veh_id in rewards.keys(): + speed = self.k.vehicle.get_speed(veh_id) + if self.penalize_stops: + if speed < 1.0: + rewards[veh_id] -= self.stop_penalty + if self.penalize_accel and veh_id in self.k.vehicle.previous_speeds: + prev_speed = self.k.vehicle.get_previous_speed(veh_id) + abs_accel = abs(speed - prev_speed) / self.sim_step + rewards[veh_id] -= abs_accel * self.accel_penalty + + # print('time to get reward is ', time() - t) return rewards def additional_command(self): """See parent class. - Define which vehicles are observed for visualization purposes. + Define which vehicles are observed for visualization purposes. Additionally, optionally reroute vehicles + back once they have exited. """ + super().additional_command() # specify observed vehicles for rl_id in self.k.vehicle.get_rl_ids(): # leader lead_id = self.k.vehicle.get_leader(rl_id) if lead_id: self.k.vehicle.set_observed(lead_id) - # follower - follow_id = self.k.vehicle.get_follower(rl_id) - if follow_id: - self.k.vehicle.set_observed(follow_id) + + if self.reroute_on_exit and self.time_counter >= self.env_params.sims_per_step * self.env_params.warmup_steps \ + and not self.env_params.evaluate: + veh_ids = list(self.k.vehicle.get_ids()) + edges = self.k.vehicle.get_edge(veh_ids) + valid_lanes = list(range(self.num_enter_lanes)) + for veh_id, edge in zip(veh_ids, edges): + if edge == "": + continue + if edge[0] == ":": # center edge + continue + # on the exit edge, near the end, and is the vehicle furthest along + if edge == self.exit_edge and \ + (self.k.vehicle.get_position(veh_id) > self.k.network.edge_length(self.exit_edge) - 100) \ + and self.k.vehicle.get_leader(veh_id) is None: + type_id = self.k.vehicle.get_type(veh_id) + # remove the vehicle + self.k.vehicle.remove(veh_id) + index = np.random.randint(low=0, high=len(valid_lanes)) + lane = valid_lanes[index] + del valid_lanes[index] + # reintroduce it at the start of the network + # Note, the position is 20 so you are not overlapping with the inflow car that is being removed. + # this allows the vehicle to be immediately inserted. + try: + self.k.vehicle.add( + veh_id=veh_id, + edge=self.entrance_edge, + type_id=str(type_id), + lane=str(lane), + pos="20.0", + speed="23.0") + except Exception as e: + print(e) + if len(valid_lanes) == 0: + break + + departed_ids = list(self.k.vehicle.get_departed_ids()) + if isinstance(departed_ids, tuple) and len(departed_ids) > 0: + for veh_id in departed_ids: + if veh_id not in self._observed_ids: + self.k.vehicle.remove(veh_id) + + # update set of all reroute RL vehicles + self.reroute_rl_ids = set(self.k.vehicle.get_rl_ids()) | self.reroute_rl_ids + else: + # reset + self.reroute_rl_ids = set() def state_util(self, rl_id): """Return an array of headway, tailway, leader speed, follower speed. @@ -223,3 +383,48 @@ def veh_statistics(self, rl_id): speed = self.k.vehicle.get_speed(rl_id) / 100.0 lane = (self.k.vehicle.get_lane(rl_id) + 1) / 10.0 return np.array([speed, lane]) + + def step(self, rl_actions): + """See parent class for more details; add option to reroute vehicles.""" + state, reward, done, info = super().step(rl_actions) + if done['__all__']: + # handle the edge case where a vehicle hasn't been put back when the rollout terminates + if self.reroute_on_exit: + for rl_id in self.reroute_rl_ids: + if rl_id not in state.keys(): + done[rl_id] = True + reward[rl_id] = 0 + state[rl_id] = -1 * np.ones(self.observation_space.shape[0]) + # you have to catch the vehicles on the exit edge, they have not yet + # recieved a done when the env terminates + on_exit_edge = [rl_id for rl_id in self.k.vehicle.get_rl_ids() + if self.k.vehicle.get_edge(rl_id) == self.exit_edge] + for rl_id in on_exit_edge: + done[rl_id] = True + reward[rl_id] = 0 + state[rl_id] = -1 * np.ones(self.observation_space.shape[0]) + + return state, reward, done, info + + +class MultiStraightRoad(I210MultiEnv): + """Partially observable multi-agent environment for a straight road. Look at superclass for more information.""" + + def __init__(self, env_params, sim_params, network, simulator): + super().__init__(env_params, sim_params, network, simulator) + self.num_enter_lanes = 1 + self.entrance_edge = self.network.routes['highway_0'][0][0][0] + self.exit_edge = self.network.routes['highway_0'][0][0][-1] + + def _apply_rl_actions(self, rl_actions): + """See class definition.""" + # in the warmup steps, rl_actions is None + if rl_actions: + rl_ids = [] + accels = [] + for rl_id, actions in rl_actions.items(): + accels.append(actions[0]) + rl_ids.append(rl_id) + + # prevent the AV from blocking the entrance + self.k.vehicle.apply_acceleration(rl_ids, accels) diff --git a/flow/envs/straightroad_env.py b/flow/envs/straightroad_env.py new file mode 100644 index 000000000..92fbb855b --- /dev/null +++ b/flow/envs/straightroad_env.py @@ -0,0 +1,231 @@ +"""Environment for training vehicles to reduce congestion in the I210.""" + +from gym.spaces import Box +import numpy as np + +from flow.envs.base import Env + +# largest number of lanes on any given edge in the network +MAX_LANES = 6 +MAX_NUM_VEHS = 8 +SPEED_SCALE = 50 +HEADWAY_SCALE = 1000 + +ADDITIONAL_ENV_PARAMS = { + # maximum acceleration for autonomous vehicles, in m/s^2 + "max_accel": 1, + # maximum deceleration for autonomous vehicles, in m/s^2 + "max_decel": 1, + # whether we use an obs space that contains adjacent lane info or just the lead obs + "lead_obs": True, + # whether the reward should come from local vehicles instead of global rewards + "local_reward": True, + # if the environment terminates once a wave has occurred + "terminate_on_wave": False, + # the environment is not allowed to terminate below this horizon length + 'wave_termination_horizon': 500, + # the speed below which we consider a wave to have occured + 'wave_termination_speed': 10.0 +} + + +class I210SingleEnv(Env): + """Partially observable single-agent environment for the I-210 subnetworks. + + The policy is shared among the agents, so there can be a non-constant + number of RL vehicles throughout the simulation. + Required from env_params: + * max_accel: maximum acceleration for autonomous vehicles, in m/s^2 + * max_decel: maximum deceleration for autonomous vehicles, in m/s^2 + The following states, actions and rewards are considered for one autonomous + vehicle only, as they will be computed in the same way for each of them. + States + The observation consists of the speeds and bumper-to-bumper headways of + the vehicles immediately preceding and following autonomous vehicles in + all of the preceding lanes as well, a binary value indicating which of + these vehicles is autonomous, and the speed of the autonomous vehicle. + Missing vehicles are padded with zeros. + Actions + The action consists of an acceleration, bound according to the + environment parameters, as well as three values that will be converted + into probabilities via softmax to decide of a lane change (left, none + or right). NOTE: lane changing is currently not enabled. It's a TODO. + Rewards + The reward function encourages proximity of the system-level velocity + to a desired velocity specified in the environment parameters, while + slightly penalizing small time headways among autonomous vehicles. + Termination + A rollout is terminated if the time horizon is reached or if two + vehicles collide into one another. + """ + + def __init__(self, env_params, sim_params, network, simulator='traci'): + super().__init__(env_params, sim_params, network, simulator) + self.lead_obs = env_params.additional_params.get("lead_obs") + self.max_lanes = MAX_LANES + self.total_reward = 0.0 + + @property + def observation_space(self): + """See class definition.""" + # speed, speed of leader, headway + if self.lead_obs: + return Box( + low=-float('inf'), + high=float('inf'), + shape=(3 * MAX_NUM_VEHS,), + dtype=np.float32 + ) + # speed, dist to ego vehicle, binary value which is 1 if the vehicle is + # an AV + else: + leading_obs = 3 * self.max_lanes + follow_obs = 3 * self.max_lanes + + # speed and lane + self_obs = 2 + + return Box( + low=-float('inf'), + high=float('inf'), + shape=(leading_obs + follow_obs + self_obs,), + dtype=np.float32 + ) + + @property + def action_space(self): + """See class definition.""" + return Box( + low=-np.abs(self.env_params.additional_params['max_decel']), + high=self.env_params.additional_params['max_accel'], + shape=(1 * MAX_NUM_VEHS,), # (4,), + dtype=np.float32) + + def _apply_rl_actions(self, rl_actions): + """See class definition.""" + # in the warmup steps, rl_actions is None + if rl_actions is not None: + accels = [] + veh_ids = [] + rl_ids = self.get_sorted_rl_ids() + + for i, rl_id in enumerate(self.rl_id_list): + accels.append(rl_actions[i]) + veh_ids.append(rl_id) + + # lane_change_softmax = np.exp(actions[1:4]) + # lane_change_softmax /= np.sum(lane_change_softmax) + # lane_change_action = np.random.choice([-1, 0, 1], + # p=lane_change_softmax) + + self.k.vehicle.apply_acceleration(rl_ids, accels) + + def get_state(self): + """See class definition.""" + rl_ids = self.get_sorted_rl_ids() + self.rl_id_list = rl_ids + veh_info = np.zeros(self.observation_space.shape[0]) + per_vehicle_obs = 3 + for i, rl_id in enumerate(rl_ids): + speed = self.k.vehicle.get_speed(rl_id) + lead_id = self.k.vehicle.get_leader(rl_id) + if lead_id in ["", None]: + # in case leader is not visible + lead_speed = SPEED_SCALE + headway = HEADWAY_SCALE + else: + lead_speed = self.k.vehicle.get_speed(lead_id) + headway = self.k.vehicle.get_headway(rl_id) + veh_info[i * per_vehicle_obs: (i + 1) * per_vehicle_obs] = [speed / SPEED_SCALE, + headway / HEADWAY_SCALE, + lead_speed / SPEED_SCALE] + return veh_info + + def compute_reward(self, rl_actions, **kwargs): + """See class definition.""" + # in the warmup steps + if rl_actions is None: + return {} + + rl_ids = self.get_sorted_rl_ids() + + des_speed = self.env_params.additional_params["target_velocity"] + rewards = np.nan_to_num(np.mean([(des_speed - np.abs(speed - des_speed)) ** 2 + for speed in self.k.vehicle.get_speed(rl_ids)])) / (des_speed ** 2) + return rewards + + def get_sorted_rl_ids(self): + """Return the MAX_NUM_VEHS closest to the exit.""" + rl_ids = self.k.vehicle.get_rl_ids() + rl_ids = sorted(rl_ids, key=lambda veh_id: self.k.vehicle.get_x_by_id(veh_id)) + rl_ids = rl_ids[-MAX_NUM_VEHS:] + return rl_ids + + def additional_command(self): + """Define which vehicles are observed for visualization purposes.""" + # specify observed vehicles + for rl_id in self.k.vehicle.get_rl_ids(): + # leader + lead_id = self.k.vehicle.get_leader(rl_id) + if lead_id: + self.k.vehicle.set_observed(lead_id) + + def state_util(self, rl_id): + """Return an array of headway, tailway, leader speed, follower speed. + + Also return a 1 if leader is rl 0 otherwise, a 1 if follower is rl 0 otherwise. + If there are fewer than MAX_LANES the extra + entries are filled with -1 to disambiguate from zeros. + """ + veh = self.k.vehicle + lane_headways = veh.get_lane_headways(rl_id).copy() + lane_tailways = veh.get_lane_tailways(rl_id).copy() + lane_leader_speed = veh.get_lane_leaders_speed(rl_id).copy() + lane_follower_speed = veh.get_lane_followers_speed(rl_id).copy() + leader_ids = veh.get_lane_leaders(rl_id).copy() + follower_ids = veh.get_lane_followers(rl_id).copy() + rl_ids = self.k.vehicle.get_rl_ids() + is_leader_rl = [1 if l_id in rl_ids else 0 for l_id in leader_ids] + is_follow_rl = [1 if f_id in rl_ids else 0 for f_id in follower_ids] + diff = MAX_LANES - len(is_leader_rl) + if diff > 0: + # the minus 1 disambiguates missing cars from missing lanes + lane_headways += diff * [-1] + lane_tailways += diff * [-1] + lane_leader_speed += diff * [-1] + lane_follower_speed += diff * [-1] + is_leader_rl += diff * [-1] + is_follow_rl += diff * [-1] + lane_headways = np.asarray(lane_headways) / 1000 + lane_tailways = np.asarray(lane_tailways) / 1000 + lane_leader_speed = np.asarray(lane_leader_speed) / 100 + lane_follower_speed = np.asarray(lane_follower_speed) / 100 + return np.concatenate((lane_headways, lane_tailways, lane_leader_speed, + lane_follower_speed, is_leader_rl, + is_follow_rl)) + + def veh_statistics(self, rl_id): + """Return speed, edge information, and x, y about the vehicle itself.""" + speed = self.k.vehicle.get_speed(rl_id) / 100.0 + lane = (self.k.vehicle.get_lane(rl_id) + 1) / 10.0 + return np.array([speed, lane]) + + +class SingleStraightRoad(I210SingleEnv): + """Partially observable multi-agent environment for a straight road. Look at superclass for more information.""" + + def __init__(self, env_params, sim_params, network, simulator): + super().__init__(env_params, sim_params, network, simulator) + self.max_lanes = 1 + + def step(self, rl_actions): + """See parent class.""" + obs, rew, done, info = super().step(rl_actions) + mean_speed = np.nan_to_num(np.mean(self.k.vehicle.get_speed(self.k.vehicle.get_ids()))) + if self.env_params.additional_params['terminate_on_wave'] and \ + mean_speed < self.env_params.additional_params['wave_termination_speed'] \ + and self.time_counter > self.env_params.additional_params['wave_termination_horizon'] \ + and len(self.k.vehicle.get_ids()) > 0: + done = True + + return obs, rew, done, info diff --git a/flow/envs/traffic_light_grid.py b/flow/envs/traffic_light_grid.py index 53391a329..8be0cb8a5 100644 --- a/flow/envs/traffic_light_grid.py +++ b/flow/envs/traffic_light_grid.py @@ -731,6 +731,17 @@ def additional_command(self): [self.k.vehicle.set_observed(veh_id) for veh_id in self.observed_ids] +class TrafficLightGridBenchmarkEnv(TrafficLightGridPOEnv): + """Class used for the benchmarks in `Benchmarks for reinforcement learning inmixed-autonomy traffic`.""" + + def compute_reward(self, rl_actions, **kwargs): + """See class definition.""" + if self.env_params.evaluate: + return - rewards.min_delay_unscaled(self) + else: + return rewards.desired_velocity(self) + + class TrafficLightGridTestEnv(TrafficLightGridEnv): """ Class for use in testing. diff --git a/flow/networks/highway.py b/flow/networks/highway.py index c63292067..e48331cf9 100644 --- a/flow/networks/highway.py +++ b/flow/networks/highway.py @@ -13,7 +13,14 @@ # speed limit for all edges "speed_limit": 30, # number of edges to divide the highway into - "num_edges": 1 + "num_edges": 1, + # whether to include a ghost edge. This edge is provided a different speed + # limit. + "use_ghost_edge": False, + # speed limit for the ghost edge + "ghost_speed_limit": 25, + # length of the downstream ghost edge with the reduced speed limit + "boundary_cell_length": 500 } @@ -29,6 +36,11 @@ class HighwayNetwork(Network): * **lanes** : number of lanes in the highway * **speed_limit** : max speed limit of the highway * **num_edges** : number of edges to divide the highway into + * **use_ghost_edge** : whether to include a ghost edge. This edge is + provided a different speed limit. + * **ghost_speed_limit** : speed limit for the ghost edge + * **boundary_cell_length** : length of the downstream ghost edge with the + reduced speed limit Usage ----- @@ -62,10 +74,6 @@ def __init__(self, if p not in net_params.additional_params: raise KeyError('Network parameter "{}" not supplied'.format(p)) - self.length = net_params.additional_params["length"] - self.lanes = net_params.additional_params["lanes"] - self.num_edges = net_params.additional_params.get("num_edges", 1) - super().__init__(name, vehicles, net_params, initial_config, traffic_lights) @@ -74,6 +82,7 @@ def specify_nodes(self, net_params): length = net_params.additional_params["length"] num_edges = net_params.additional_params.get("num_edges", 1) segment_lengths = np.linspace(0, length, num_edges+1) + end_length = net_params.additional_params["boundary_cell_length"] nodes = [] for i in range(num_edges+1): @@ -83,6 +92,13 @@ def specify_nodes(self, net_params): "y": 0 }] + if self.net_params.additional_params["use_ghost_edge"]: + nodes += [{ + "id": "edge_{}".format(num_edges + 1), + "x": length + end_length, + "y": 0 + }] + return nodes def specify_edges(self, net_params): @@ -90,6 +106,7 @@ def specify_edges(self, net_params): length = net_params.additional_params["length"] num_edges = net_params.additional_params.get("num_edges", 1) segment_length = length/float(num_edges) + end_length = net_params.additional_params["boundary_cell_length"] edges = [] for i in range(num_edges): @@ -101,12 +118,22 @@ def specify_edges(self, net_params): "length": segment_length }] + if self.net_params.additional_params["use_ghost_edge"]: + edges += [{ + "id": "highway_end", + "type": "highway_end", + "from": "edge_{}".format(num_edges), + "to": "edge_{}".format(num_edges + 1), + "length": end_length + }] + return edges def specify_types(self, net_params): """See parent class.""" lanes = net_params.additional_params["lanes"] speed_limit = net_params.additional_params["speed_limit"] + end_speed_limit = net_params.additional_params["ghost_speed_limit"] types = [{ "id": "highwayType", @@ -114,6 +141,13 @@ def specify_types(self, net_params): "speed": speed_limit }] + if self.net_params.additional_params["use_ghost_edge"]: + types += [{ + "id": "highway_end", + "numLanes": lanes, + "speed": end_speed_limit + }] + return types def specify_routes(self, net_params): @@ -123,14 +157,51 @@ def specify_routes(self, net_params): for i in range(num_edges): rts["highway_{}".format(i)] = ["highway_{}".format(j) for j in range(i, num_edges)] + if self.net_params.additional_params["use_ghost_edge"]: + rts["highway_{}".format(i)].append("highway_end") return rts def specify_edge_starts(self): """See parent class.""" - edgestarts = [("highway_{}".format(i), 0) - for i in range(self.num_edges)] - return edgestarts + junction_length = 0.1 + length = self.net_params.additional_params["length"] + num_edges = self.net_params.additional_params.get("num_edges", 1) + + # Add the main edges. + edge_starts = [ + ("highway_{}".format(i), + i * (length / num_edges + junction_length)) + for i in range(num_edges) + ] + + if self.net_params.additional_params["use_ghost_edge"]: + edge_starts += [ + ("highway_end", length + num_edges * junction_length) + ] + + return edge_starts + + def specify_internal_edge_starts(self): + """See parent class.""" + junction_length = 0.1 + length = self.net_params.additional_params["length"] + num_edges = self.net_params.additional_params.get("num_edges", 1) + + # Add the junctions. + edge_starts = [ + (":edge_{}".format(i + 1), + (i + 1) * length / num_edges + i * junction_length) + for i in range(num_edges - 1) + ] + + if self.net_params.additional_params["use_ghost_edge"]: + edge_starts += [ + (":edge_{}".format(num_edges), + length + (num_edges - 1) * junction_length) + ] + + return edge_starts @staticmethod def gen_custom_start_pos(cls, net_params, initial_config, num_vehicles): diff --git a/flow/networks/i210_subnetwork.py b/flow/networks/i210_subnetwork.py index d8e05efb5..f4315b07f 100644 --- a/flow/networks/i210_subnetwork.py +++ b/flow/networks/i210_subnetwork.py @@ -1,9 +1,18 @@ """Contains the I-210 sub-network class.""" - from flow.networks.base import Network +from flow.core.params import InitialConfig +from flow.core.params import TrafficLightParams + +ADDITIONAL_NET_PARAMS = { + # whether to include vehicle on the on-ramp + "on_ramp": False, + # whether to include the downstream slow-down edge in the network + "ghost_edge": False, +} EDGES_DISTRIBUTION = [ # Main highway + "ghost0", "119257914", "119257908#0", "119257908#1-AddedOnRampEdge", @@ -25,6 +34,12 @@ class I210SubNetwork(Network): """A network used to simulate the I-210 sub-network. + Requires from net_params: + + * **on_ramp** : whether to include vehicle on the on-ramp + * **ghost_edge** : whether to include the downstream slow-down edge in the + network + Usage ----- >>> from flow.core.params import NetParams @@ -39,103 +54,233 @@ class I210SubNetwork(Network): >>> ) """ - def specify_routes(self, net_params): - """See parent class. + def __init__(self, + name, + vehicles, + net_params, + initial_config=InitialConfig(), + traffic_lights=TrafficLightParams()): + """Initialize the I210 sub-network scenario.""" + for p in ADDITIONAL_NET_PARAMS.keys(): + if p not in net_params.additional_params: + raise KeyError('Network parameter "{}" not supplied'.format(p)) + + # The length of each edge and junction is a fixed term that can be + # found in the xml file. + self.length_with_ghost_edge = [ + ("ghost0", 573.08), + (":300944378_0", 0.30), + ("119257914", 61.28), + (":300944379_0", 0.31), + ("119257908#0", 696.97), + (":300944436_0", 2.87), + ("119257908#1-AddedOnRampEdge", 97.20), + (":119257908#1-AddedOnRampNode_0", 3.24), + ("119257908#1", 239.68), + (":119257908#1-AddedOffRampNode_0", 3.24), + ("119257908#1-AddedOffRampEdge", 98.50), + (":1686591010_1", 5.46), + ("119257908#2", 576.61), + (":1842086610_1", 4.53), + ("119257908#3", 17.49), + ] - Routes for vehicles moving through the bay bridge from Oakland to San - Francisco. - """ + super(I210SubNetwork, self).__init__( + name=name, + vehicles=vehicles, + net_params=net_params, + initial_config=initial_config, + traffic_lights=traffic_lights, + ) + + def specify_routes(self, net_params): + """See parent class.""" rts = { - # Main highway "119257914": [ - (["119257914", "119257908#0", "119257908#1-AddedOnRampEdge", - "119257908#1", "119257908#1-AddedOffRampEdge", "119257908#2", - "119257908#3"], - 1), # HOV: 1509 (on ramp: 57), Non HOV: 6869 (onramp: 16) - # (["119257914", "119257908#0", "119257908#1-AddedOnRampEdge", - # "119257908#1", "119257908#1-AddedOffRampEdge", "173381935"], - # 17 / 8378) - ], - # "119257908#0": [ - # (["119257908#0", "119257908#1-AddedOnRampEdge", "119257908#1", - # "119257908#1-AddedOffRampEdge", "119257908#2", - # "119257908#3"], - # 1.0), - # # (["119257908#0", "119257908#1-AddedOnRampEdge", "119257908#1", - # # "119257908#1-AddedOffRampEdge", "173381935"], - # # 0.5), - # ], - # "119257908#1-AddedOnRampEdge": [ - # (["119257908#1-AddedOnRampEdge", "119257908#1", - # "119257908#1-AddedOffRampEdge", "119257908#2", - # "119257908#3"], - # 1.0), - # # (["119257908#1-AddedOnRampEdge", "119257908#1", - # # "119257908#1-AddedOffRampEdge", "173381935"], - # # 0.5), - # ], - # "119257908#1": [ - # (["119257908#1", "119257908#1-AddedOffRampEdge", "119257908#2", - # "119257908#3"], - # 1.0), - # # (["119257908#1", "119257908#1-AddedOffRampEdge", "173381935"], - # # 0.5), - # ], - # "119257908#1-AddedOffRampEdge": [ - # (["119257908#1-AddedOffRampEdge", "119257908#2", - # "119257908#3"], - # 1.0), - # # (["119257908#1-AddedOffRampEdge", "173381935"], - # # 0.5), - # ], - # "119257908#2": [ - # (["119257908#2", "119257908#3"], 1), - # ], - # "119257908#3": [ - # (["119257908#3"], 1), - # ], - # - # # On-ramp - # "27414345": [ - # (["27414345", "27414342#1-AddedOnRampEdge", - # "27414342#1", - # "119257908#1-AddedOnRampEdge", "119257908#1", - # "119257908#1-AddedOffRampEdge", "119257908#2", - # "119257908#3"], - # 1 - 9 / 321), - # (["27414345", "27414342#1-AddedOnRampEdge", - # "27414342#1", - # "119257908#1-AddedOnRampEdge", "119257908#1", - # "119257908#1-AddedOffRampEdge", "173381935"], - # 9 / 321), - # ], - # "27414342#0": [ - # (["27414342#0", "27414342#1-AddedOnRampEdge", - # "27414342#1", - # "119257908#1-AddedOnRampEdge", "119257908#1", - # "119257908#1-AddedOffRampEdge", "119257908#2", - # "119257908#3"], - # 1 - 20 / 421), - # (["27414342#0", "27414342#1-AddedOnRampEdge", - # "27414342#1", - # "119257908#1-AddedOnRampEdge", "119257908#1", - # "119257908#1-AddedOffRampEdge", "173381935"], - # 20 / 421), - # ], - # "27414342#1-AddedOnRampEdge": [ - # (["27414342#1-AddedOnRampEdge", "27414342#1", "119257908#1-AddedOnRampEdge", - # "119257908#1", "119257908#1-AddedOffRampEdge", "119257908#2", - # "119257908#3"], - # 0.5), - # (["27414342#1-AddedOnRampEdge", "27414342#1", "119257908#1-AddedOnRampEdge", - # "119257908#1", "119257908#1-AddedOffRampEdge", "173381935"], - # 0.5), - # ], - # - # # Off-ramp - # "173381935": [ - # (["173381935"], 1), - # ], + (["119257914", + "119257908#0", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1.0), + ] } + if net_params.additional_params["ghost_edge"]: + rts.update({ + "ghost0": [ + (["ghost0", + "119257914", + "119257908#0", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1), + ], + }) + + if net_params.additional_params["on_ramp"]: + rts.update({ + # Main highway + "119257908#0": [ + (["119257908#0", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1.0), + ], + "119257908#1-AddedOnRampEdge": [ + (["119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1.0), + ], + "119257908#1": [ + (["119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1.0), + ], + "119257908#1-AddedOffRampEdge": [ + (["119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1.0), + ], + "119257908#2": [ + (["119257908#2", + "119257908#3"], 1), + ], + "119257908#3": [ + (["119257908#3"], 1), + ], + + # On-ramp + "27414345": [ + (["27414345", + "27414342#1-AddedOnRampEdge", + "27414342#1", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1 - 9 / 321), + (["27414345", + "27414342#1-AddedOnRampEdge", + "27414342#1", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "173381935"], 9 / 321), + ], + "27414342#0": [ + (["27414342#0", + "27414342#1-AddedOnRampEdge", + "27414342#1", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1 - 20 / 421), + (["27414342#0", + "27414342#1-AddedOnRampEdge", + "27414342#1", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "173381935"], 20 / 421), + ], + "27414342#1-AddedOnRampEdge": [ + (["27414342#1-AddedOnRampEdge", + "27414342#1", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 0.5), + (["27414342#1-AddedOnRampEdge", + "27414342#1", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "173381935"], 0.5), + ], + + # Off-ramp + "173381935": [ + (["173381935"], 1), + ], + }) + return rts + + def specify_edge_starts(self): + """See parent class.""" + if self.net_params.additional_params["ghost_edge"]: + # Collect the names of all the edges. + edge_names = [ + e[0] for e in self.length_with_ghost_edge + if not e[0].startswith(":") + ] + + edge_starts = [] + for edge in edge_names: + # Find the position of the edge in the list of tuples. + edge_pos = next( + i for i in range(len(self.length_with_ghost_edge)) + if self.length_with_ghost_edge[i][0] == edge + ) + + # Sum of lengths until the edge is reached to compute the + # starting position of the edge. + edge_starts.append(( + edge, + sum(e[1] for e in self.length_with_ghost_edge[:edge_pos]) + )) + + elif self.net_params.additional_params["on_ramp"]: + # TODO: this will incorporated in the future, if needed. + edge_starts = [] + + else: + # TODO: this will incorporated in the future, if needed. + edge_starts = [] + + return edge_starts + + def specify_internal_edge_starts(self): + """See parent class.""" + if self.net_params.additional_params["ghost_edge"]: + # Collect the names of all the junctions. + edge_names = [ + e[0] for e in self.length_with_ghost_edge + if e[0].startswith(":") + ] + + edge_starts = [] + for edge in edge_names: + # Find the position of the edge in the list of tuples. + edge_pos = next( + i for i in range(len(self.length_with_ghost_edge)) + if self.length_with_ghost_edge[i][0] == edge + ) + + # Sum of lengths until the edge is reached to compute the + # starting position of the edge. + edge_starts.append(( + edge, + sum(e[1] for e in self.length_with_ghost_edge[:edge_pos]) + )) + + elif self.net_params.additional_params["on_ramp"]: + # TODO: this will incorporated in the future, if needed. + edge_starts = [] + + else: + # TODO: this will incorporated in the future, if needed. + edge_starts = [] + + return edge_starts diff --git a/flow/networks/ring.py b/flow/networks/ring.py index de4d17503..ceef22a78 100755 --- a/flow/networks/ring.py +++ b/flow/networks/ring.py @@ -37,7 +37,7 @@ class RingNetwork(Network): >>> from flow.core.params import NetParams >>> from flow.core.params import VehicleParams >>> from flow.core.params import InitialConfig - >>> from flow.scenarios import RingNetwork + >>> from flow.networks import RingNetwork >>> >>> network = RingNetwork( >>> name='ring_road', diff --git a/flow/replay/hbaselines_replay.py b/flow/replay/hbaselines_replay.py new file mode 100644 index 000000000..a8f7dfc67 --- /dev/null +++ b/flow/replay/hbaselines_replay.py @@ -0,0 +1,354 @@ +"""Visualizer for rllib experiments. + +Attributes +---------- +EXAMPLE_USAGE : str + Example call to the function, which is + :: + + python ./visualizer_rllib.py /tmp/ray/result_dir 1 +""" +import tensorflow as tf +import numpy as np +import argparse +import os +import sys +import random +import json + +from flow.core.experiment import Experiment + +from hbaselines.fcnet.td3 import FeedForwardPolicy \ + as TD3FeedForwardPolicy +from hbaselines.fcnet.sac import FeedForwardPolicy \ + as SACFeedForwardPolicy +from hbaselines.goal_conditioned.td3 import GoalConditionedPolicy \ + as TD3GoalConditionedPolicy +from hbaselines.goal_conditioned.sac import GoalConditionedPolicy \ + as SACGoalConditionedPolicy +from hbaselines.envs.mixed_autonomy.params.highway \ + import get_flow_params as highway +from hbaselines.envs.mixed_autonomy.params.i210 \ + import get_flow_params as i210 + +EXAMPLE_USAGE = """ +example usage: + python ./visualizer_rllib.py /ray_results/experiment_dir/result_dir 1 + +Here the arguments are: +1 - the path to the simulation results +2 - the number of the checkpoint +""" + +# dictionary that maps policy names to policy objects +POLICY_DICT = { + "FeedForwardPolicy": { + "TD3": TD3FeedForwardPolicy, + "SAC": SACFeedForwardPolicy, + }, + "GoalConditionedPolicy": { + "TD3": TD3GoalConditionedPolicy, + "SAC": SACGoalConditionedPolicy, + }, +} + + +ENV_ATTRIBUTES = { + "highway-v0": lambda multiagent: highway( + fixed_boundary=True, + stopping_penalty=True, + acceleration_penalty=True, + multiagent=multiagent, + ), + "highway-v1": lambda multiagent: highway( + fixed_boundary=True, + stopping_penalty=False, + acceleration_penalty=True, + multiagent=multiagent, + ), + "highway-v2": lambda multiagent: highway( + fixed_boundary=True, + stopping_penalty=False, + acceleration_penalty=False, + multiagent=multiagent, + ), + "i210-v0": lambda multiagent: i210( + fixed_boundary=True, + stopping_penalty=True, + acceleration_penalty=True, + multiagent=multiagent, + ), + "i210-v1": lambda multiagent: i210( + fixed_boundary=True, + stopping_penalty=False, + acceleration_penalty=True, + multiagent=multiagent, + ), + "i210-v2": lambda multiagent: i210( + fixed_boundary=True, + stopping_penalty=False, + acceleration_penalty=False, + multiagent=multiagent, + ), +} + + +def parse_options(args): + """Parse training options user can specify in command line. + + Returns + ------- + argparse.Namespace + the output parser object + """ + parser = argparse.ArgumentParser( + description='Run evaluation episodes of a given checkpoint.', + epilog='python run_eval "/path/to/dir_name" ckpt_num') + + # required input parameters + parser.add_argument( + 'dir_name', type=str, help='the path to the checkpoints folder') + + # optional arguments + parser.add_argument( + '--ckpt_num', type=int, default=None, + help='the checkpoint number. If not specified, the last checkpoint is ' + 'used.') + parser.add_argument( + '--num_rollouts', type=int, default=1, + help='number of eval episodes') + parser.add_argument( + '--no_render', action='store_true', + help='shuts off rendering') + parser.add_argument( + '--random_seed', action='store_true', + help='whether to run the simulation on a random seed. If not added, ' + 'the original seed is used.') + parser.add_argument( + '--to_aws', action='store_true', + help='shuts off rendering') + + flags, _ = parser.parse_known_args(args) + + return flags + + +def get_hyperparameters_from_dir(ckpt_path): + """Collect the algorithm-specific hyperparameters from the checkpoint. + + Parameters + ---------- + ckpt_path : str + the path to the checkpoints folder + + Returns + ------- + str + environment name + hbaselines.goal_conditioned.* + policy object + dict + algorithm and policy hyperparaemters + int + the seed value + """ + # import the dictionary of hyperparameters + with open(os.path.join(ckpt_path, 'hyperparameters.json'), 'r') as f: + hp = json.load(f) + + # collect the policy object + policy_name = hp['policy_name'] + alg_name = hp['algorithm'] + policy = POLICY_DICT[policy_name][alg_name] + + # collect the environment name + env_name = hp['env_name'] + + # collect the seed value + seed = hp['seed'] + + # remove unnecessary features from hp dict + hp = hp.copy() + del hp['policy_name'], hp['env_name'], hp['seed'] + del hp['algorithm'], hp['date/time'] + + return env_name, policy, hp, seed + + +def get_flow_params(env_name): + """Read the provided result_dir and get config and flow_params. + + Parameters + ---------- + env_name : str + the name of the environment (in h-baselines) + + Returns + ------- + dict + the flow-params dict object + """ + # Handle multi-agent environments. + multiagent = env_name.startswith("multiagent") + if multiagent: + env_name = env_name[11:] + + flow_params = ENV_ATTRIBUTES[env_name](multiagent) + + dir_path = os.path.dirname(os.path.realpath(__file__)) + emission_path = '{0}/test_time_rollout/'.format(dir_path) + flow_params["sim"].emission_path = emission_path + + flow_params["sim"].render = True + + return flow_params + + +def get_rl_actions(policy_tf, ob_space): + """Define the method through which actions are assigned to the RL agent(s). + + Parameters + ---------- + policy_tf : hbaselines.base_policies.* + the policy object + ob_space : gym.space.* + the observation space + + Returns + ------- + method + the rl_actions method to use in the Experiment object + """ + def rl_actions(obs): + """Get the actions from a given observation. + + Parameters + ---------- + obs : array_like + the observation + + Returns + ------- + list of float + the action value + """ + # Reshape the observation to match the input structure of the policy. + if isinstance(obs, dict): + # In multi-agent environments, observations come in dict form + for key in obs.keys(): + # Shared policies with have one observation space, while + # independent policies have a different observation space based + # on their agent ID. + if isinstance(ob_space, dict): + ob_shape = ob_space[key].shape + else: + ob_shape = ob_space.shape + obs[key] = np.array(obs[key]).reshape((-1,) + ob_shape) + else: + obs = np.array(obs).reshape((-1,) + ob_space.shape) + + action = policy_tf.get_action( + obs, None, + apply_noise=False, + random_actions=False, + env_num=0, + ) + + # Flatten the actions. Dictionaries correspond to multi-agent policies. + if isinstance(action, dict): + action = {key: action[key].flatten() for key in action.keys()} + else: + action = action.flatten() + + return action + + return rl_actions + + +def main(args): + """Visualizer for RLlib experiments. + + This function takes args (see function create_parser below for + more detailed information on what information can be fed to this + visualizer), and renders the experiment associated with it. + """ + # ======================================================================= # + # Step 1: Import relevant data. # + # ======================================================================= # + + flags = parse_options(args) + + # get the hyperparameters + env_name, policy, hp, seed = get_hyperparameters_from_dir(flags.dir_name) + hp['num_envs'] = 1 + + # Get the flow-specific parameters. + flow_params = get_flow_params(env_name) + + # setup the seed value + if not flags.random_seed: + random.seed(seed) + np.random.seed(seed) + tf.compat.v1.set_random_seed(seed) + + # Create the experiment object. + exp = Experiment(flow_params) + + # ======================================================================= # + # Step 2: Setup the policy. # + # ======================================================================= # + + # Create a tensorflow session. + sess = tf.compat.v1.Session() + + # Get the checkpoint number. + if flags.ckpt_num is None: + filenames = os.listdir(os.path.join(flags.dir_name, "checkpoints")) + metafiles = [f[:-5] for f in filenames if f[-5:] == ".meta"] + metanum = [int(f.split("-")[-1]) for f in metafiles] + ckpt_num = max(metanum) + else: + ckpt_num = flags.ckpt_num + + # location to the checkpoint + ckpt = os.path.join(flags.dir_name, "checkpoints/itr-{}".format(ckpt_num)) + + # Create the policy. + policy_tf = policy( + sess=sess, + ob_space=exp.env.observation_space, + ac_space=exp.env.action_space, + co_space=None, + verbose=2, + layers=[256, 256], + act_fun=tf.nn.relu, + fingerprint_range=([0, 0], [5, 5]), + env_name=env_name, + **hp["policy_kwargs"] + ) + + trainable_vars = tf.compat.v1.get_collection( + tf.compat.v1.GraphKeys.TRAINABLE_VARIABLES) + + # Restore the previous checkpoint. + saver = tf.compat.v1.train.Saver(trainable_vars) + saver.restore(sess, ckpt) + + # Create a method to compute the RL actions. + rl_actions = get_rl_actions(policy_tf, exp.env.observation_space) + + # ======================================================================= # + # Step 3: Setup the and run the experiment. # + # ======================================================================= # + + exp.run( + num_runs=flags.num_rollouts, + convert_to_csv=True, + to_aws=flags.to_aws, + rl_actions=rl_actions, + multiagent=env_name.startswith("multiagent"), + ) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/flow/replay/rllib_replay.py b/flow/replay/rllib_replay.py new file mode 100644 index 000000000..fcadf75e7 --- /dev/null +++ b/flow/replay/rllib_replay.py @@ -0,0 +1,455 @@ +"""Replay script for rllib experiments. + +Attributes +---------- +EXAMPLE_USAGE : str + Example call to the function, which is + :: + + python ./rl_replay.py /tmp/ray/result_dir 1 +""" +import argparse +import gym +import numpy as np +from collections import defaultdict +import os +import sys + +import ray +try: + from ray.rllib.agents.agent import get_agent_class +except ImportError: + from ray.rllib.agents.registry import get_agent_class +from ray.tune.registry import register_env + +from flow.utils.rllib import get_flow_params +from flow.utils.rllib import get_rllib_config +from flow.utils.rllib import get_rllib_pkl + +from flow.core.experiment import Experiment + +EXAMPLE_USAGE = """ +example usage: + python ./rl_replay.py /ray_results/experiment_dir/result_dir 1 + +Here the arguments are: +1 - the path to the simulation results +2 - the number of the checkpoint +""" + + +def read_result_dir(result_dir_path): + """Read the provided result_dir and get config and flow_params. + + Parameters + ---------- + result_dir_path : str + Directory containing rllib results + + Returns + ------- + str + the path to the results directory + dict + TODO + bool + True if the agent is multi-agent, False otherwise + dict + flow-specific parameters from the results directory + """ + result_dir = result_dir_path if result_dir_path[-1] != '/' \ + else result_dir_path[:-1] + + config = get_rllib_config(result_dir) + + # check if we have a multiagent environment but in a + # backwards compatible way + if config.get('multiagent', {}).get('policies', None): + multiagent = True + pkl = get_rllib_pkl(result_dir) + config['multiagent'] = pkl['multiagent'] + else: + multiagent = False + + # Run on only one cpu for rendering purposes + config['num_workers'] = 0 + + flow_params = get_flow_params(config) + return result_dir, config, multiagent, flow_params + + +def set_sim_params(sim_params, render_mode, save_render, gen_emission, output_dir=''): + """Set up sim_params according to render mode. + + Parameters + ---------- + sim_params : flow.core.params.SimParams + simulation-specific parameters + render_mode : str + the render mode. Options include sumo_web3d, rgbd and sumo_gui + save_render : bool + Saves a rendered video to a file. NOTE: Overrides render_mode with + pyglet rendering. + gen_emission : bool + whether to generate the emission file + output_dir : str + directory to store the emission, optional + """ + # hack for old pkl files TODO(ev) remove eventually + setattr(sim_params, 'num_clients', 1) + if not hasattr(sim_params, 'use_ballistic'): + sim_params.use_ballistic = False + + # Set the emission path. + sim_params.restart_instance = True + if not output_dir: + dir_path = os.path.dirname(os.path.realpath(__file__)) + emission_path = '{0}/test_time_rollout/'.format(dir_path) + else: + emission_path = output_dir + sim_params.emission_path = emission_path if gen_emission else None + + # Pick your rendering mode. + if render_mode == 'sumo_web3d': + sim_params.num_clients = 2 + sim_params.render = False + elif render_mode == 'drgb': + sim_params.render = 'drgb' + sim_params.pxpm = 4 + elif render_mode == 'sumo_gui': + sim_params.render = False # will be set to True below + elif render_mode == 'no_render': + sim_params.render = False + + # Add option for saving rendered results. + if save_render: + if render_mode != 'sumo_gui': + sim_params.render = 'drgb' + sim_params.pxpm = 4 + sim_params.save_render = True + + +def set_env_params(env_params, evaluate, horizon, config=None): + """Set up env_params according to commandline arguments. + + Parameters + ---------- + env_params : flow.core.params.EnvParams + environment-specific parameters + evaluate : bool + flag indicating that the evaluation reward should be used so the + evaluation reward should be used rather than the normal reward + horizon : int + an update environment time horizon. If set to None, the original time + horizon is used. + config : dict + TODO + """ + # Start the environment with the gui turned on and a path for the + # emission file + env_params.restart_instance = False + if evaluate: + env_params.evaluate = True + + # lower the horizon if testing + if horizon: + if config: + config['horizon'] = horizon + env_params.horizon = horizon + + +def set_agents(config, result_dir, env_name, run=None, checkpoint_num=None): + """Determine and create agents that will be used to compute actions. + + Parameters + ---------- + config : dict + TODO + result_dir : str + Directory containing rllib results + env_name : str + the name of the environment + run : str + The algorithm or model to train. This may refer to the name of a + built-on algorithm (e.g. RLLib's DQN or PPO), or a user-defined + trainable function or class registered in the tune registry. Required + for results trained with flow-0.2.0 and before. + checkpoint_num : int + the checkpoint number. If set to None, the most recent checkpoint is + used. + + Returns + ------- + TODO + the trained agent/policy object + """ + # Determine agent and checkpoint + config_run = config['env_config']['run'] if 'run' in config['env_config'] \ + else None + if run and config_run: + if run != config_run: + print('rl_replay.py: error: run argument ' + + '\'{}\' passed in '.format(run) + + 'differs from the one stored in params.json ' + + '\'{}\''.format(config_run)) + sys.exit(1) + if run: + agent_cls = get_agent_class(run) + elif config['env_config']['run'] == \ + "": + from flow.algorithms.centralized_PPO import CCTrainer + from flow.algorithms.centralized_PPO import CentralizedCriticModel + from ray.rllib.models import ModelCatalog + agent_cls = CCTrainer + ModelCatalog.register_custom_model("cc_model", CentralizedCriticModel) + elif config['env_config']['run'] == \ + "": + from flow.algorithms.custom_ppo import CustomPPOTrainer + agent_cls = CustomPPOTrainer + elif config_run: + agent_cls = get_agent_class(config_run) + else: + print('rl_replay.py: error: could not find flow parameter ' + '\'run\' in params.json, ' + 'add argument --run to provide the algorithm or model used ' + 'to train the results\n e.g. ' + 'python ./rl_replay.py /tmp/ray/result_dir 1 --run PPO') + sys.exit(1) + + # get checkpoint number if not provided + if checkpoint_num is None: + dirs = os.listdir(result_dir) + cp_numbers = [int(name.split('_')[1]) for name in dirs if name.startswith('checkpoint')] + checkpoint_num = str(max(cp_numbers)) + + # create the agent that will be used to compute the actions + agent = agent_cls(env=env_name, config=config) + checkpoint = result_dir + '/checkpoint_' + checkpoint_num + checkpoint = checkpoint + '/checkpoint-' + checkpoint_num + agent.restore(checkpoint) + + return agent + + +def get_rl_action(config, agent, multiagent, multi_only=False, deterministic=False): + """Return a function that compute action based on a given state. + + Parameters + ---------- + config : dict + TODO + agent : TODO + the trained agent/policy object + multiagent : bool + whether the policy is a multi-agent policy + multi_only : bool + If this is set to true, the function will raise an error + if it is single agent use_lstm is true. + deterministic : bool + If this is set to true, the exploration noise in the policy + will be removed when computing actions. + + Returns + ------- + policy_map_fn : function + a mapping from agent to their respective policy + rl_action : method + the rl_actions method to use in the Experiment object + rets : dict + a pre-initialized dictionary to store rewards for multi-agent simulation + """ + policy_map_fn = None + if multiagent: + rets = {} + # map the agent id to its policy + policy_map_fn = config['multiagent']['policy_mapping_fn'] + for key in config['multiagent']['policies'].keys(): + rets[key] = [] + else: + rets = [] + + if config['model']['use_lstm']: + use_lstm = True + if multiagent: + state_init = {} + size = config['model']['lstm_cell_size'] + state_init = defaultdict(lambda: [np.zeros(size, np.float32), + np.zeros(size, np.float32)]) + else: + state_init = [ + np.zeros(config['model']['lstm_cell_size'], np.float32), + np.zeros(config['model']['lstm_cell_size'], np.float32) + ] + else: + use_lstm = False + + def rl_action(state): + if multiagent: + action = {} + for agent_id in state.keys(): + if use_lstm: + action[agent_id], state_init[agent_id], logits = \ + agent.compute_action( + state[agent_id], state=state_init[agent_id], + policy_id=policy_map_fn(agent_id), + explore=not deterministic) + else: + action[agent_id] = agent.compute_action( + state[agent_id], policy_id=policy_map_fn(agent_id), + explore=not deterministic) + else: + if use_lstm and multi_only: + raise NotImplementedError + action = agent.compute_action(state, explore=not deterministic) + return action + + return policy_map_fn, rl_action, rets + + +def replay_rllib(args): + """Replay for RLlib experiments. + + This function takes args (see function create_parser below for + more detailed information on what information can be fed to this + replay script), and renders the experiment associated with it. + """ + result_dir, config, multiagent, flow_params = read_result_dir( + args.result_dir) + + set_sim_params( + flow_params['sim'], + args.render_mode, + args.save_render, + args.gen_emission + ) + sim_params = flow_params['sim'] + + # Create and register a gym+rllib env + exp = Experiment(flow_params, register_with_ray=True) + register_env(exp.env_name, exp.create_env) + + set_env_params(flow_params['env'], args.evaluate, args.horizon, config) + + agent = set_agents( + config, + result_dir, + exp.env_name, + run=args.run, + checkpoint_num=args.checkpoint_num + ) + + if hasattr(agent, "local_evaluator") and \ + os.environ.get("TEST_FLAG") != 'True': + exp.env = agent.local_evaluator.env + else: + exp.env = gym.make(exp.env_name) + + # reroute on exit is a training hack, it should be turned off at test time. + if hasattr(exp.env, "reroute_on_exit"): + exp.env.reroute_on_exit = False + + # Set to True after initializing agent and env. + if args.render_mode == 'sumo_gui': + exp.env.sim_params.render = True + + policy_map_fn, rl_action, rets = get_rl_action(config, agent, multiagent, + deterministic=args.deterministic) + + # If restart_instance, don't restart here because env.reset will restart + # later. + if not sim_params.restart_instance: + exp.env.restart_simulation(sim_params, sim_params.render) + + exp.run( + num_runs=args.num_rollouts, + convert_to_csv=args.gen_emission, + to_aws=args.to_aws, + rl_actions=rl_action, + multiagent=multiagent, + rets=rets, + policy_map_fn=policy_map_fn + ) + + +def create_parser(): + """Create the parser to capture CLI arguments.""" + parser = argparse.ArgumentParser( + description='[Flow] Evaluates a reinforcement learning agent ' + 'given a checkpoint.', + epilog=EXAMPLE_USAGE) + + # required input parameters + parser.add_argument( + 'result_dir', type=str, help='Directory containing results') + parser.add_argument('checkpoint_num', type=str, help='Checkpoint number.') + + # optional input parameters + parser.add_argument( + '--run', + type=str, + help='The algorithm or model to train. This may refer to ' + 'the name of a built-on algorithm (e.g. RLLib\'s DQN ' + 'or PPO), or a user-defined trainable function or ' + 'class registered in the tune registry. ' + 'Required for results trained with flow-0.2.0 and before.') + parser.add_argument( + '--num_rollouts', + type=int, + default=1, + help='The number of rollouts to replay.') + parser.add_argument( + '--gen_emission', + action='store_true', + help='Specifies whether to generate an emission file from the ' + 'simulation') + parser.add_argument( + '--evaluate', + action='store_true', + help='Specifies whether to use the \'evaluate\' reward ' + 'for the environment.') + parser.add_argument( + '--render_mode', + type=str, + default='sumo_gui', + help='Pick the render mode. Options include sumo_web3d, ' + 'rgbd and sumo_gui') + parser.add_argument( + '--save_render', + action='store_true', + help='Saves a rendered video to a file. NOTE: Overrides render_mode ' + 'with pyglet rendering.') + parser.add_argument( + '--horizon', + type=int, + help='Specifies the horizon.') + parser.add_argument( + '--is_baseline', + action='store_true', + help='specifies whether this is a baseline run' + ) + parser.add_argument( + '--to_aws', + type=str, nargs='?', default=None, const="default", + help='Specifies the name of the partition to store the output' + 'file on S3. Putting not None value for this argument' + 'automatically set gen_emission to True.' + ) + parser.add_argument( + '--deterministic', + action='store_true', + default=False, + help='specifies whether to remove exploration noise in the policy' + ) + return parser + + +def main(): + """Run the rl_replay according to the commandline arguments.""" + parser = create_parser() + args = parser.parse_args() + ray.init(num_cpus=1) + replay_rllib(args) + + +if __name__ == '__main__': + main() diff --git a/flow/replay/transfer_tests.py b/flow/replay/transfer_tests.py new file mode 100644 index 000000000..24d21614b --- /dev/null +++ b/flow/replay/transfer_tests.py @@ -0,0 +1,388 @@ +"""Transfer and replay for i210 environment.""" +import argparse +from copy import deepcopy +import os + +from examples.exp_configs.rl.multiagent.multiagent_i210 import flow_params as I210_MA_DEFAULT_FLOW_PARAMS +from examples.exp_configs.rl.multiagent.multiagent_i210 import custom_callables +from flow.core.experiment import Experiment +from flow.utils.registry import make_create_env +from flow.visualize.transfer.util import inflows_range +from flow.replay.rllib_replay import read_result_dir +from flow.replay.rllib_replay import set_sim_params +from flow.replay.rllib_replay import set_env_params +from flow.replay.rllib_replay import set_agents +from flow.replay.rllib_replay import get_rl_action +import ray +from ray.tune.registry import register_env + +from flow.data_pipeline.data_pipeline import collect_metadata_from_config + +EXAMPLE_USAGE = """ +example usage: + python i210_replay.py -r /ray_results/experiment_dir/result_dir -c 1 + python i210_replay.py --controller idm + python i210_replay.py --controller idm --run_transfer + +Here the arguments are: +1 - the path to the simulation results +2 - the number of the checkpoint +""" + + +@ray.remote +def replay(args, + flow_params, + output_dir=None, + transfer_test=None, + rllib_config=None, + max_completed_trips=None, + v_des=12, + supplied_metadata=None): + """Replay or run transfer test (defined by transfer_fn) by modif. + + Parameters + ---------- + args : argparse.Namespace + input arguments passed via a parser. See create_parser. + flow_params : dict + flow-specific parameters + output_dir : str + Directory to save results. + transfer_test : TODO + TODO + rllib_config : str + Directory containing rllib results + max_completed_trips : int + Terminate rollout after max_completed_trips vehicles have started and + ended. + v_des : float + the desired speed for the FollowerStopper vehicles + supplied_metadata: dict (str : list) + the metadata associated with this simulation. + """ + assert bool(args.controller) ^ bool(rllib_config), \ + "Need to specify either controller or rllib_config, but not both" + + args.gen_emission = args.gen_emission or args.use_s3 + + if transfer_test is not None: + if type(transfer_test) == bytes: + transfer_test = ray.cloudpickle.loads(transfer_test) + flow_params = transfer_test.flow_params_modifier_fn(flow_params) + + if args.controller: + test_params = {} + if args.controller == 'idm': + from flow.controllers.car_following_models import IDMController + controller = IDMController + # An example of really obvious changes + test_params.update({'v0': 1, 'T': 1, 'a': 0.2, 'b': 0.2}) + elif args.controller == 'default_human': + controller = flow_params['veh'].type_parameters['human']['acceleration_controller'][0] + test_params.update(flow_params['veh'].type_parameters['human']['acceleration_controller'][1]) + elif args.controller == 'follower_stopper': + from flow.controllers.velocity_controllers import FollowerStopper + controller = FollowerStopper + test_params.update({'v_des': v_des}) + # flow_params['veh'].type_parameters['av']['car_following_params'] + elif args.controller == 'sumo': + from flow.controllers.car_following_models import SimCarFollowingController + controller = SimCarFollowingController + + flow_params['veh'].type_parameters['av']['acceleration_controller'] = (controller, test_params) + + for veh_param in flow_params['veh'].initial: + if veh_param['veh_id'] == 'av': + veh_param['acceleration_controller'] = (controller, test_params) + + set_sim_params(flow_params['sim'], args.render_mode, args.save_render, args.gen_emission, output_dir) + sim_params = flow_params['sim'] + + set_env_params(flow_params['env'], args.evaluate, args.horizon) + + # Create and register a gym+rllib env + exp = Experiment(flow_params, custom_callables=custom_callables) + + # set to True after initializing agent and env + if args.render_mode == 'sumo_gui': + exp.env.sim_params.render = True + + # if restart_instance, don't restart here because env.reset will restart later + if not sim_params.restart_instance: + exp.env.restart_simulation(sim_params=sim_params, render=sim_params.render) + + # reroute on exit is a training hack, it should be turned off at test time. + if hasattr(exp.env, "reroute_on_exit"): + exp.env.reroute_on_exit = False + + policy_map_fn, rets = None, None + if rllib_config: + result_dir, rllib_config, multiagent, rllib_flow_params = read_result_dir(rllib_config) + + # lower the horizon if testing + if args.horizon: + rllib_config['horizon'] = args.horizon + + agent_create_env, agent_env_name = make_create_env(params=rllib_flow_params, version=0) + register_env(agent_env_name, agent_create_env) + + assert 'run' in rllib_config['env_config'], \ + "Was this trained with the latest version of Flow?" + # Determine agent and checkpoint + agent = set_agents(rllib_config, result_dir, agent_env_name, checkpoint_num=args.checkpoint_num) + + policy_map_fn, rllib_rl_action, rets = get_rl_action(rllib_config, agent, multiagent) + + # reroute on exit is a training hack, it should be turned off at test time. + if hasattr(exp.env, "reroute_on_exit"): + exp.env.reroute_on_exit = False + + def rl_action(state): + if rllib_config: + action = rllib_rl_action(state) + else: + action = None + return action + + info_dict = exp.run( + num_runs=args.num_rollouts, + convert_to_csv=args.gen_emission, + to_aws=args.use_s3, + rl_actions=rl_action, + multiagent=True, + rets=rets, + policy_map_fn=policy_map_fn, + supplied_metadata=supplied_metadata + ) + + return info_dict + + +def create_parser(): + """Create the parser to capture CLI arguments.""" + parser = argparse.ArgumentParser( + description='[Flow] Evaluates a reinforcement learning agent ' + 'given a checkpoint.', + epilog=EXAMPLE_USAGE) + + parser.add_argument( + '--rllib_result_dir', '-r', + required=False, + type=str, + help='Directory containing results' + ) + parser.add_argument( + '--checkpoint_num', '-c', + required=False, + type=str, + default=None, + help='Checkpoint number.' + ) + parser.add_argument( + '--num_rollouts', + type=int, + default=1, + help='The number of rollouts to visualize.') + parser.add_argument( + '--gen_emission', + action='store_true', + help='Specifies whether to generate an emission file from the ' + 'simulation') + parser.add_argument( + '--evaluate', + action='store_true', + help='Specifies whether to use the \'evaluate\' reward ' + 'for the environment.') + parser.add_argument( + '--render_mode', '-rm', + type=str, + default=None, + help='Pick the render mode. Options include sumo_web3d, ' + 'rgbd and sumo_gui') + parser.add_argument( + '--save_render', + action='store_true', + help='Saves a rendered video to a file. NOTE: Overrides render_mode ' + 'with pyglet rendering.') + parser.add_argument( + '--horizon', + type=int, + help='Specifies the horizon.') + parser.add_argument( + '--local', + action='store_true', + help='Adjusts run settings to be compatible with limited ' + 'memory capacity' + ) + parser.add_argument( + '--controller', + type=str, + help='Which custom controller to use. Defaults to IDM' + ) + parser.add_argument( + '--run_transfer', + action='store_true', + help='Runs transfer tests if true' + ) + parser.add_argument( + '-pr', + '--penetration_rate', + type=float, + help='Specifies percentage of AVs.', + required=False) + parser.add_argument( + '-mct', + '--max_completed_trips', + type=int, + help='Terminate rollout after max_completed_trips vehicles have ' + 'started and ended.', + default=None) + parser.add_argument( + '--v_des_sweep', + action='store_true', + help='Runs a sweep over v_des params.', + default=None) + parser.add_argument( + '--output_dir', + type=str, + help='Directory to save results.', + default=None + ) + parser.add_argument( + '--use_s3', + action='store_true', + help='If true, upload results to s3' + ) + parser.add_argument( + '--num_cpus', + type=int, + default=1, + help='Number of cpus to run experiment with' + ) + parser.add_argument( + '--multi_node', + action='store_true', + help='Set to true if this will be run in cluster mode' + ) + parser.add_argument( + '--exp_title', + type=str, + required=False, + default=None, + help='Informative experiment title to help distinguish results' + ) + parser.add_argument( + '--only_query', + nargs='*', default="[\'all\']", + help='specify which query should be run by lambda for detail, see ' + 'upload_to_s3 in data_pipeline.py' + ) + parser.add_argument( + '--is_baseline', + action='store_true', + help='specifies whether this is a baseline run' + ) + parser.add_argument( + '--exp_config', type=str, + help='Name of the experiment configuration file, as located in ' + 'exp_configs/non_rl.') + parser.add_argument( + '--submitter_name', type=str, + help='Name displayed next to the submission on the leaderboard.') + parser.add_argument( + '--strategy_name', type=str, + help='Strategy displayed next to the submission on the leaderboard.') + return parser + + +def generate_graphs(args): + """Run the replay according to the commandline arguments.""" + supplied_metadata = None + if args.submitter_name and args.strategy_name: + supplied_metadata = {'name': args.submitter_name, + 'strategy': args.strategy_name} + if args.exp_config: + module = __import__("../../examples/exp_configs.non_rl", fromlist=[args.exp_config]) + flow_params = getattr(module, args.exp_config).flow_params + supplied_metadata = collect_metadata_from_config(getattr(module, args.exp_config)) + else: + flow_params = deepcopy(I210_MA_DEFAULT_FLOW_PARAMS) + + if ray.is_initialized(): + ray.shutdown() + if args.multi_node: + ray.init(redis_address='localhost:6379') + elif args.local: + ray.init(local_mode=True, object_store_memory=200 * 1024 * 1024) + else: + ray.init(num_cpus=args.num_cpus + 1, object_store_memory=200 * 1024 * 1024) + + if args.exp_title: + output_dir = os.path.join(args.output_dir, args.exp_title) + else: + output_dir = args.output_dir + + if args.run_transfer: + s = [ray.cloudpickle.dumps(transfer_test) for transfer_test in + inflows_range(penetration_rates=[0.0, 0.1, 0.2, 0.3])] + ray_output = [ + replay.remote( + args, + flow_params, + output_dir=output_dir, + transfer_test=transfer_test, + rllib_config=args.rllib_result_dir, + max_completed_trips=args.max_completed_trips, + supplied_metadata=supplied_metadata + ) + for transfer_test in s + ] + ray.get(ray_output) + + elif args.v_des_sweep: + assert args.controller == 'follower_stopper' + + ray_output = [ + replay.remote( + args, + flow_params, + output_dir="{}/{}".format(output_dir, v_des), + rllib_config=args.rllib_result_dir, + max_completed_trips=args.max_completed_trips, + v_des=v_des, + supplied_metadata=supplied_metadata + ) + for v_des in range(8, 13, 1) + ] + ray.get(ray_output) + + else: + if args.penetration_rate is not None: + pr = args.penetration_rate + single_transfer = next(inflows_range(penetration_rates=pr)) + ray.get(replay.remote( + args, + flow_params, + output_dir=output_dir, + transfer_test=single_transfer, + rllib_config=args.rllib_result_dir, + max_completed_trips=args.max_completed_trips, + supplied_metadata=supplied_metadata + )) + else: + ray.get(replay.remote( + args, + flow_params, + output_dir=output_dir, + rllib_config=args.rllib_result_dir, + max_completed_trips=args.max_completed_trips, + supplied_metadata=supplied_metadata + )) + + +if __name__ == "__main__": + parser = create_parser() + args = parser.parse_args() + + generate_graphs(args) diff --git a/flow/utils/aimsun/Aimsun_Flow.ang b/flow/utils/aimsun/Aimsun_Flow.ang new file mode 100644 index 000000000..17119f4e9 Binary files /dev/null and b/flow/utils/aimsun/Aimsun_Flow.ang differ diff --git a/flow/utils/registry.py b/flow/utils/registry.py index 3f6c9dad5..a1163849f 100644 --- a/flow/utils/registry.py +++ b/flow/utils/registry.py @@ -117,15 +117,19 @@ def create_env(*_): entry_point = params["env_name"].__module__ + ':' + params["env_name"].__name__ # register the environment with OpenAI gym - register( - id=env_name, - entry_point=entry_point, - kwargs={ - "env_params": env_params, - "sim_params": sim_params, - "network": network, - "simulator": params['simulator'] - }) + try: + register( + id=env_name, + entry_point=entry_point, + kwargs={ + "env_params": env_params, + "sim_params": sim_params, + "network": network, + "simulator": params['simulator'] + }) + except gym.error.Error: + print("WARNING: Environment {} already registered, ignoring." + .format(env_name)) return gym.envs.make(env_name) diff --git a/flow/utils/rllib.py b/flow/utils/rllib.py index b5abc9a23..db0e811b8 100644 --- a/flow/utils/rllib.py +++ b/flow/utils/rllib.py @@ -6,7 +6,9 @@ import json from copy import deepcopy import os +import sys +import flow.config import flow.envs from flow.core.params import SumoLaneChangeParams, SumoCarFollowingParams, \ SumoParams, InitialConfig, EnvParams, NetParams, InFlows @@ -95,6 +97,8 @@ def get_flow_params(config): flow_params = json.loads(config['env_config']['flow_params']) else: flow_params = json.load(open(config, 'r')) + if 'env_config' in flow_params: + flow_params = json.loads(flow_params['env_config']['flow_params']) # reinitialize the vehicles class from stored data veh = VehicleParams() @@ -145,6 +149,12 @@ def get_flow_params(config): if flow_params["net"]["inflows"]: net.inflows.__dict__ = flow_params["net"]["inflows"].copy() + if net.template is not None and len(net.template) > 0: + filename = os.path.join(flow.config.PROJECT_PATH, 'examples') + split = net.template.split('examples')[1][1:] + path = os.path.abspath(os.path.join(filename, split)) + net.template = path + env = EnvParams() env.__dict__ = flow_params["env"].copy() @@ -207,6 +217,9 @@ def get_rllib_config(path): def get_rllib_pkl(path): """Return the data from the specified rllib configuration file.""" + dirname = os.path.dirname(__file__) + filename = os.path.join(dirname, '../../examples/') + sys.path.append(filename) config_path = os.path.join(path, "params.pkl") if not os.path.exists(config_path): config_path = os.path.join(path, "../params.pkl") diff --git a/flow/visualize/plot_custom_callables.py b/flow/visualize/plot_custom_callables.py new file mode 100644 index 000000000..ee9a10c1d --- /dev/null +++ b/flow/visualize/plot_custom_callables.py @@ -0,0 +1,115 @@ +"""Generate charts from with .npy files containing custom callables through replay.""" + +import argparse +from datetime import datetime +import errno +import numpy as np + +try: + from matplotlib import pyplot as plt +except ImportError: + import matplotlib + + matplotlib.use('TkAgg') + from matplotlib import pyplot as plt +import os +import pytz +import sys + + +def make_bar_plot(vals, title): + """Make a bar plot.""" + print(len(vals)) + fig = plt.figure() + plt.hist(vals, 10, facecolor='blue', alpha=0.5) + plt.title(title) + plt.xlim(1000, 3000) + return fig + + +def plot_trip_distribution(all_trip_energy_distribution): + """Plot a distribution of trips.""" + non_av_vals = [] + figures = [] + figure_names = [] + for key in all_trip_energy_distribution: + if key != 'av': + non_av_vals.extend(all_trip_energy_distribution[key]) + figures.append(make_bar_plot(all_trip_energy_distribution[key], key)) + figure_names.append(key) + + figure_names.append('All Non-AV') + figures.append(make_bar_plot(non_av_vals, 'All Non-AV')) + + figure_names.append('All') + figures.append(make_bar_plot(non_av_vals + all_trip_energy_distribution['av'], 'All')) + + return figure_names, figures + + +def parse_flags(args): + """Parse training options user can specify in command line. + + Returns + ------- + argparse.Namespace + the output parser object + """ + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="Parse argument used when running a Flow simulation.", + epilog="python train.py EXP_CONFIG") + parser.add_argument("target_folder", type=str, + help='Folder containing results') + parser.add_argument("--output_folder", type=str, required=False, default=None, + help='Folder to save charts to.') + parser.add_argument("--show_images", action='store_true', + help='Whether to display charts.') + return parser.parse_args(args) + + +if __name__ == "__main__": + flags = parse_flags(sys.argv[1:]) + + date = datetime.now(tz=pytz.utc) + date = date.astimezone(pytz.timezone('US/Pacific')).strftime("%m-%d-%Y") + + if flags.output_folder: + if not os.path.exists(flags.output_folder): + try: + os.makedirs(flags.output_folder) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + + info_dicts = [] + custom_callable_names = set() + exp_names = [] + for (dirpath, dir_names, file_names) in os.walk(flags.target_folder): + for file_name in file_names: + if file_name[-4:] == ".npy": + exp_name = os.path.basename(dirpath) + info_dict = np.load(os.path.join(dirpath, file_name), allow_pickle=True).item() + + info_dicts.append(info_dict) + exp_names.append(exp_name) + custom_callable_names.update(info_dict.keys()) + + idxs = np.argsort(exp_names) + exp_names = [exp_names[i] for i in idxs] + info_dicts = [info_dicts[i] for i in idxs] + + for name in custom_callable_names: + y_vals = [np.mean(info_dict[name]) for info_dict in info_dicts] + y_stds = [np.std(info_dict[name]) for info_dict in info_dicts] + x_pos = np.arange(len(exp_names)) + + plt.bar(x_pos, y_vals, align='center', alpha=0.5) + plt.xticks(x_pos, [exp_name for exp_name in exp_names], rotation=60) + plt.xlabel('Experiment') + plt.title('I210 Replay Result: {}'.format(name)) + plt.tight_layout() + if flags.output_folder: + plt.savefig(os.path.join(flags.output_folder, '{}-plot.png'.format(name))) + + plt.show() diff --git a/flow/visualize/time_space_diagram.py b/flow/visualize/time_space_diagram.py index d8dad01e9..650824c4c 100644 --- a/flow/visualize/time_space_diagram.py +++ b/flow/visualize/time_space_diagram.py @@ -17,14 +17,21 @@ python time_space_diagram.py .csv .json """ from flow.utils.rllib import get_flow_params -from flow.networks import RingNetwork, FigureEightNetwork, MergeNetwork, I210SubNetwork +from flow.networks import RingNetwork, FigureEightNetwork, MergeNetwork, I210SubNetwork, HighwayNetwork import argparse -import csv -from matplotlib import pyplot as plt -from matplotlib.collections import LineCollection +from collections import defaultdict +try: + from matplotlib import pyplot as plt +except ImportError: + import matplotlib + matplotlib.use('TkAgg') + from matplotlib import pyplot as plt +from matplotlib.collections import LineCollection, PatchCollection +from matplotlib.patches import Rectangle import matplotlib.colors as colors import numpy as np +import pandas as pd # networks that can be plotted by this method @@ -32,88 +39,104 @@ RingNetwork, FigureEightNetwork, MergeNetwork, - I210SubNetwork + I210SubNetwork, + HighwayNetwork ] +# networks that use edgestarts +USE_EDGESTARTS = set([ + RingNetwork, + FigureEightNetwork, + MergeNetwork +]) + +GHOST_DICT = defaultdict(dict) +GHOST_DICT[I210SubNetwork] = {'ghost_edges': {'ghost0', '119257908#3'}} +GHOST_DICT[HighwayNetwork] = {'ghost_bounds': (500, 2300)} -def import_data_from_emission(fp): - r"""Import relevant data from the predefined emission (.csv) file. + +def import_data_from_trajectory(fp, params=dict()): + r"""Import and preprocess data from the Flow trajectory (.csv) file. Parameters ---------- fp : str file path (for the .csv formatted file) + params : dict + flow-specific parameters, including: + * "network" (str): name of the network that was used when generating + the emission file. Must be one of the network names mentioned in + ACCEPTABLE_NETWORKS, + * "net_params" (flow.core.params.NetParams): network-specific + parameters. This is used to collect the lengths of various network + links. Returns ------- - dict of dict - Key = "veh_id": name of the vehicle \n Elements: - - * "time": time step at every sample - * "edge": edge ID at every sample - * "pos": relative position at every sample - * "vel": speed at every sample + pd.DataFrame, float, float """ - # initialize all output variables - veh_id, t, edge, rel_pos, vel, lane = [], [], [], [], [], [] - - # import relevant data from emission file - for record in csv.DictReader(open(fp)): - veh_id.append(record['id']) - t.append(record['time']) - edge.append(record['edge_id']) - rel_pos.append(record['relative_position']) - vel.append(record['speed']) - lane.append(record['lane_number']) - - # we now want to separate data by vehicle ID - ret = {key: {'time': [], 'edge': [], 'pos': [], 'vel': [], 'lane': []} - for key in np.unique(veh_id)} - for i in range(len(veh_id)): - ret[veh_id[i]]['time'].append(float(t[i])) - ret[veh_id[i]]['edge'].append(edge[i]) - ret[veh_id[i]]['pos'].append(float(rel_pos[i])) - ret[veh_id[i]]['vel'].append(float(vel[i])) - ret[veh_id[i]]['lane'].append(float(lane[i])) + network = params['network'] - return ret + # Read trajectory csv into pandas dataframe + df = pd.read_csv(fp) + + # Convert column names for backwards compatibility using emissions csv + column_conversions = { + 'time': 'time_step', + 'lane_number': 'lane_id', + } + df = df.rename(columns=column_conversions) + if network in USE_EDGESTARTS: + df['distance'] = _get_abs_pos(df, params) + + start = params['env'].warmup_steps * params['env'].sims_per_step * params['sim'].sim_step + # produce upper and lower bounds for the non-greyed-out domain + ghost_edges = GHOST_DICT[network].get('ghost_edges') + ghost_bounds = GHOST_DICT[network].get('ghost_bounds') + if ghost_edges: + domain_lb = df[~df['edge_id'].isin(ghost_edges)]['distance'].min() + domain_ub = df[~df['edge_id'].isin(ghost_edges)]['distance'].max() + elif ghost_bounds: + domain_lb = ghost_bounds[0] + domain_ub = ghost_bounds[1] + else: + domain_lb = df['distance'].min() + domain_ub = df['distance'].max() + + df.loc[:, 'time_step'] = df['time_step'].apply(lambda x: x - start) + df.loc[:, 'distance'] = df['distance'].apply(lambda x: x - domain_lb) + domain_ub -= domain_lb + # Compute line segment ends by shifting dataframe by 1 row + df[['next_pos', 'next_time']] = df.groupby('id')[['distance', 'time_step']].shift(-1) -def get_time_space_data(data, params): + # Remove nans from data + df = df[df['next_time'].notna()] + + return df, domain_lb, domain_ub, start + + +def get_time_space_data(data, network): r"""Compute the unique inflows and subsequent outflow statistics. Parameters ---------- - data : dict of dict - Key = "veh_id": name of the vehicle \n Elements: - - * "time": time step at every sample - * "edge": edge ID at every sample - * "pos": relative position at every sample - * "vel": speed at every sample - params : dict - flow-specific parameters, including: - - * "network" (str): name of the network that was used when generating - the emission file. Must be one of the network names mentioned in - ACCEPTABLE_NETWORKS, - * "net_params" (flow.core.params.NetParams): network-specific - parameters. This is used to collect the lengths of various network - links. + data : pd.DataFrame + cleaned dataframe of the trajectory data + network : child class of Network() + network that was used when generating the emission file. + Must be one of the network names mentioned in + ACCEPTABLE_NETWORKS Returns ------- - as_array - n_steps x n_veh matrix specifying the absolute position of every - vehicle at every time step. Set to zero if the vehicle is not present - in the network at that time step. - as_array - n_steps x n_veh matrix specifying the speed of every vehicle at every - time step. Set to zero if the vehicle is not present in the network at - that time step. - as_array - a (n_steps,) vector representing the unique time steps in the - simulation + ndarray (or dict < str, np.ndarray >) + 3d array (n_segments x 2 x 2) containing segments to be plotted. + every inner 2d array is comprised of two 1d arrays representing + [start time, start distance] and [end time, end distance] pairs. + in the case of I210, the nested arrays are wrapped into a dict, + keyed on the lane number, so that each lane can be plotted + separately. Raises ------ @@ -121,256 +144,138 @@ def get_time_space_data(data, params): if the specified network is not supported by this method """ # check that the network is appropriate - assert params['network'] in ACCEPTABLE_NETWORKS, \ - 'Network must be one of: ' + ', '.join(ACCEPTABLE_NETWORKS) + assert network in ACCEPTABLE_NETWORKS, \ + 'Network must be one of: ' + ', '.join([network_.__name__ for network_ in ACCEPTABLE_NETWORKS]) # switcher used to compute the positions based on the type of network switcher = { RingNetwork: _ring_road, MergeNetwork: _merge, FigureEightNetwork: _figure_eight, - I210SubNetwork: _i210_subnetwork + I210SubNetwork: _i210_subnetwork, + HighwayNetwork: _highway, } - # Collect a list of all the unique times. - all_time = [] - for veh_id in data.keys(): - all_time.extend(data[veh_id]['time']) - all_time = np.sort(np.unique(all_time)) - # Get the function from switcher dictionary - func = switcher[params['network']] + func = switcher[network] # Execute the function - pos, speed, all_time = func(data, params, all_time) + segs, data = func(data) - return pos, speed, all_time + return segs, data -def _merge(data, params, all_time): - r"""Generate position and speed data for the merge. +def _merge(data): + r"""Generate time and position data for the merge. This only include vehicles on the main highway, and not on the adjacent on-ramp. Parameters ---------- - data : dict of dict - Key = "veh_id": name of the vehicle \n Elements: + data : pd.DataFrame + cleaned dataframe of the trajectory data - * "time": time step at every sample - * "edge": edge ID at every sample - * "pos": relative position at every sample - * "vel": speed at every sample - params : dict - flow-specific parameters - all_time : array_like - a (n_steps,) vector representing the unique time steps in the - simulation + Returns + ------- + ndarray + 3d array (n_segments x 2 x 2) containing segments to be plotted. + every inner 2d array is comprised of two 1d arrays representing + [start time, start distance] and [end time, end distance] pairs. + pd.DataFrame + modified trajectory dataframe + """ + # Omit ghost edges + keep_edges = {'inflow_merge', 'bottom', ':bottom_0'} + data = data[data['edge_id'].isin(keep_edges)] + + segs = data[['time_step', 'distance', 'next_time', 'next_pos']].values.reshape((len(data), 2, 2)) + + return segs, data + + +def _highway(data): + r"""Generate time and position data for the highway. + + Parameters + ---------- + data : pd.DataFrame + cleaned dataframe of the trajectory data Returns ------- - as_array - n_steps x n_veh matrix specifying the absolute position of every - vehicle at every time step. Set to zero if the vehicle is not present - in the network at that time step. - as_array - n_steps x n_veh matrix specifying the speed of every vehicle at every - time step. Set to zero if the vehicle is not present in the network at - that time step. + ndarray + 3d array (n_segments x 2 x 2) containing segments to be plotted. + every inner 2d array is comprised of two 1d arrays representing + [start time, start distance] and [end time, end distance] pairs. + pd.DataFrame + modified trajectory dataframe """ - # import network data from flow params - inflow_edge_len = 100 - premerge = params['net'].additional_params['pre_merge_length'] - postmerge = params['net'].additional_params['post_merge_length'] - - # generate edge starts - edgestarts = { - 'inflow_highway': 0, - 'left': inflow_edge_len + 0.1, - 'center': inflow_edge_len + premerge + 22.6, - 'inflow_merge': inflow_edge_len + premerge + postmerge + 22.6, - 'bottom': 2 * inflow_edge_len + premerge + postmerge + 22.7, - ':left_0': inflow_edge_len, - ':center_0': inflow_edge_len + premerge + 0.1, - ':center_1': inflow_edge_len + premerge + 0.1, - ':bottom_0': 2 * inflow_edge_len + premerge + postmerge + 22.6 - } + segs = data[['time_step', 'distance', 'next_time', 'next_pos']].values.reshape((len(data), 2, 2)) - # compute the absolute position - for veh_id in data.keys(): - data[veh_id]['abs_pos'] = _get_abs_pos(data[veh_id]['edge'], - data[veh_id]['pos'], edgestarts) - - # prepare the speed and absolute position in a way that is compatible with - # the space-time diagram, and compute the number of vehicles at each step - pos = np.zeros((all_time.shape[0], len(data.keys()))) - speed = np.zeros((all_time.shape[0], len(data.keys()))) - for i, veh_id in enumerate(sorted(data.keys())): - for spd, abs_pos, ti, edge in zip(data[veh_id]['vel'], - data[veh_id]['abs_pos'], - data[veh_id]['time'], - data[veh_id]['edge']): - # avoid vehicles outside the main highway - if edge in ['inflow_merge', 'bottom', ':bottom_0']: - continue - ind = np.where(ti == all_time)[0] - pos[ind, i] = abs_pos - speed[ind, i] = spd - - return pos, speed, all_time - - -def _ring_road(data, params, all_time): - r"""Generate position and speed data for the ring road. + return segs, data + + +def _ring_road(data): + r"""Generate time and position data for the ring road. Vehicles that reach the top of the plot simply return to the bottom and continue. Parameters ---------- - data : dict of dict - Key = "veh_id": name of the vehicle \n Elements: - - * "time": time step at every sample - * "edge": edge ID at every sample - * "pos": relative position at every sample - * "vel": speed at every sample - params : dict - flow-specific parameters - all_time : array_like - a (n_steps,) vector representing the unique time steps in the - simulation + data : pd.DataFrame + cleaned dataframe of the trajectory data Returns ------- - as_array - n_steps x n_veh matrix specifying the absolute position of every - vehicle at every time step. Set to zero if the vehicle is not present - in the network at that time step. - as_array - n_steps x n_veh matrix specifying the speed of every vehicle at every - time step. Set to zero if the vehicle is not present in the network at - that time step. + ndarray + 3d array (n_segments x 2 x 2) containing segments to be plotted. + every inner 2d array is comprised of two 1d arrays representing + [start time, start distance] and [end time, end distance] pairs. + pd.DataFrame + unmodified trajectory dataframe """ - # import network data from flow params - ring_length = params['net'].additional_params["length"] - junction_length = 0.1 # length of inter-edge junctions - - edgestarts = { - "bottom": 0, - ":right_0": 0.25 * ring_length, - "right": 0.25 * ring_length + junction_length, - ":top_0": 0.5 * ring_length + junction_length, - "top": 0.5 * ring_length + 2 * junction_length, - ":left_0": 0.75 * ring_length + 2 * junction_length, - "left": 0.75 * ring_length + 3 * junction_length, - ":bottom_0": ring_length + 3 * junction_length - } - - # compute the absolute position - for veh_id in data.keys(): - data[veh_id]['abs_pos'] = _get_abs_pos(data[veh_id]['edge'], - data[veh_id]['pos'], edgestarts) + segs = data[['time_step', 'distance', 'next_time', 'next_pos']].values.reshape((len(data), 2, 2)) - # create the output variables - pos = np.zeros((all_time.shape[0], len(data.keys()))) - speed = np.zeros((all_time.shape[0], len(data.keys()))) - for i, veh_id in enumerate(sorted(data.keys())): - for spd, abs_pos, ti in zip(data[veh_id]['vel'], - data[veh_id]['abs_pos'], - data[veh_id]['time']): - ind = np.where(ti == all_time)[0] - pos[ind, i] = abs_pos - speed[ind, i] = spd + return segs, data - return pos, speed, all_time +def _i210_subnetwork(data): + r"""Generate time and position data for the i210 subnetwork. -def _i210_subnetwork(data, params, all_time): - r"""Generate position and speed data for the i210 subnetwork. - - We only look at the second to last lane of edge 119257908#1-AddedOnRampEdge + We generate plots for all lanes, so the segments are wrapped in + a dictionary. Parameters ---------- - data : dict of dict - Key = "veh_id": name of the vehicle \n Elements: - - * "time": time step at every sample - * "edge": edge ID at every sample - * "pos": relative position at every sample - * "vel": speed at every sample - params : dict - flow-specific parameters - all_time : array_like - a (n_steps,) vector representing the unique time steps in the - simulation + data : pd.DataFrame + cleaned dataframe of the trajectory data Returns ------- - as_array - n_steps x n_veh matrix specifying the absolute position of every - vehicle at every time step. Set to zero if the vehicle is not present - in the network at that time step. - as_array - n_steps x n_veh matrix specifying the speed of every vehicle at every - time step. Set to zero if the vehicle is not present in the network at - that time step. + dict < str, np.ndarray > + dictionary of 3d array (n_segments x 2 x 2) containing segments + to be plotted. the dictionary is keyed on lane numbers, with the + values being the 3d array representing the segments. every inner + 2d array is comprised of two 1d arrays representing + [start time, start distance] and [end time, end distance] pairs. + pd.DataFrame + modified trajectory dataframe """ - # import network data from flow params - # - # edge_starts = {"119257908#0": 0, - # "119257908#1-AddedOnRampEdge": 686.98} - desired_lane = 1 - edge_starts = {"119257914": 0, - "119257908#0": 61.58, - "119257908#1-AddedOnRampEdge": 686.98 + 61.58} - # edge_starts = {"119257908#0": 0} - # edge_starts = {"119257908#1-AddedOnRampEdge": 0} - # desired_lane = 5 - - # compute the absolute position - for veh_id in data.keys(): - data[veh_id]['abs_pos'] = _get_abs_pos_1_edge(data[veh_id]['edge'], - data[veh_id]['pos'], - edge_starts) - - # create the output variables - # TODO(@ev) handle subsampling better than this - low_time = int(0 / params['sim'].sim_step) - high_time = int(1600 / params['sim'].sim_step) - all_time = all_time[low_time:high_time] - - # track only vehicles that were around during this time period - observed_row_list = [] - pos = np.zeros((all_time.shape[0], len(data.keys()))) - speed = np.zeros((all_time.shape[0], len(data.keys()))) - for i, veh_id in enumerate(sorted(data.keys())): - for spd, abs_pos, ti, edge, lane in zip(data[veh_id]['vel'], - data[veh_id]['abs_pos'], - data[veh_id]['time'], - data[veh_id]['edge'], - data[veh_id]['lane']): - # avoid vehicles not on the relevant edges. Also only check the second to - # last lane - if edge not in edge_starts.keys() or ti not in all_time or lane != desired_lane: - continue - else: - if i not in observed_row_list: - observed_row_list.append(i) - ind = np.where(ti == all_time)[0] - pos[ind, i] = abs_pos - speed[ind, i] = spd - - pos = pos[:, observed_row_list] - speed = speed[:, observed_row_list] - - return pos, speed, all_time - - -def _figure_eight(data, params, all_time): - r"""Generate position and speed data for the figure eight. + # Reset lane numbers that are offset by ramp lanes + offset_edges = set(data[data['lane_id'] == 5]['edge_id'].unique()) + data.loc[data['edge_id'].isin(offset_edges), 'lane_id'] = data[data['edge_id'].isin(offset_edges)]['lane_id'] - 1 + + segs = dict() + for lane, df in data.groupby('lane_id'): + segs[lane] = df[['time_step', 'distance', 'next_time', 'next_pos']].values.reshape((len(df), 2, 2)) + + return segs, data + + +def _figure_eight(data): + r"""Generate time and position data for the figure eight. The vehicles traveling towards the intersection from one side will be plotted from the top downward, while the vehicles from the other side will @@ -378,137 +283,266 @@ def _figure_eight(data, params, all_time): Parameters ---------- - data : dict of dict - Key = "veh_id": name of the vehicle \n Elements: - - * "time": time step at every sample - * "edge": edge ID at every sample - * "pos": relative position at every sample - * "vel": speed at every sample - params : dict - flow-specific parameters - all_time : array_like - a (n_steps,) vector representing the unique time steps in the - simulation + data : pd.DataFrame + cleaned dataframe of the trajectory data Returns ------- - as_array - n_steps x n_veh matrix specifying the absolute position of every - vehicle at every time step. Set to zero if the vehicle is not present - in the network at that time step. - as_array - n_steps x n_veh matrix specifying the speed of every vehicle at every - time step. Set to zero if the vehicle is not present in the network at - that time step. + ndarray + 3d array (n_segments x 2 x 2) containing segments to be plotted. + every inner 2d array is comprised of two 1d arrays representing + [start time, start distance] and [end time, end distance] pairs. + pd.DataFrame + unmodified trajectory dataframe """ - # import network data from flow params - net_params = params['net'] - ring_radius = net_params.additional_params['radius_ring'] - ring_edgelen = ring_radius * np.pi / 2. - intersection = 2 * ring_radius - junction = 2.9 + 3.3 * net_params.additional_params['lanes'] - inner = 0.28 - - # generate edge starts - edgestarts = { - 'bottom': inner, - 'top': intersection / 2 + junction + inner, - 'upper_ring': intersection + junction + 2 * inner, - 'right': intersection + 3 * ring_edgelen + junction + 3 * inner, - 'left': 1.5 * intersection + 3 * ring_edgelen + 2 * junction + 3 * inner, - 'lower_ring': 2 * intersection + 3 * ring_edgelen + 2 * junction + 4 * inner, - ':bottom_0': 0, - ':center_1': intersection / 2 + inner, - ':top_0': intersection + junction + inner, - ':right_0': intersection + 3 * ring_edgelen + junction + 2 * inner, - ':center_0': 1.5 * intersection + 3 * ring_edgelen + junction + 3 * inner, - ':left_0': 2 * intersection + 3 * ring_edgelen + 2 * junction + 3 * inner, - # for aimsun - 'bottom_to_top': intersection / 2 + inner, - 'right_to_left': junction + 3 * inner, - } + segs = data[['time_step', 'distance', 'next_time', 'next_pos']].values.reshape((len(data), 2, 2)) + + return segs, data + - # compute the absolute position - for veh_id in data.keys(): - data[veh_id]['abs_pos'] = _get_abs_pos(data[veh_id]['edge'], - data[veh_id]['pos'], edgestarts) - - # create the output variables - pos = np.zeros((all_time.shape[0], len(data.keys()))) - speed = np.zeros((all_time.shape[0], len(data.keys()))) - for i, veh_id in enumerate(sorted(data.keys())): - for spd, abs_pos, ti in zip(data[veh_id]['vel'], - data[veh_id]['abs_pos'], - data[veh_id]['time']): - ind = np.where(ti == all_time)[0] - pos[ind, i] = abs_pos - speed[ind, i] = spd - - # reorganize data for space-time plot - figure_eight_len = 6 * ring_edgelen + 2 * intersection + 2 * junction + 10 * inner - intersection_loc = [edgestarts[':center_1'] + intersection / 2, - edgestarts[':center_0'] + intersection / 2] - pos[pos < intersection_loc[0]] += figure_eight_len - pos[np.logical_and(pos > intersection_loc[0], pos < intersection_loc[1])] \ - += - intersection_loc[1] - pos[pos > intersection_loc[1]] = \ - - pos[pos > intersection_loc[1]] + figure_eight_len + intersection_loc[0] - - return pos, speed, all_time - - -def _get_abs_pos(edge, rel_pos, edgestarts): +def _get_abs_pos(df, params): """Compute the absolute positions from edges and relative positions. This is the variable we will ultimately use to plot individual vehicles. Parameters ---------- - edge : list of str - list of edges at every time step - rel_pos : list of float - list of relative positions at every time step - edgestarts : dict - the absolute starting position of every edge + df : pd.DataFrame + dataframe of trajectory data + params : dict + flow-specific parameters Returns ------- - list of float + pd.Series the absolute positive for every sample """ - ret = [] - for edge_i, pos_i in zip(edge, rel_pos): - ret.append(pos_i + edgestarts[edge_i]) + if params['network'] == MergeNetwork: + inflow_edge_len = 100 + premerge = params['net'].additional_params['pre_merge_length'] + postmerge = params['net'].additional_params['post_merge_length'] + + # generate edge starts + edgestarts = { + 'inflow_highway': 0, + 'left': inflow_edge_len + 0.1, + 'center': inflow_edge_len + premerge + 22.6, + 'inflow_merge': inflow_edge_len + premerge + postmerge + 22.6, + 'bottom': 2 * inflow_edge_len + premerge + postmerge + 22.7, + ':left_0': inflow_edge_len, + ':center_0': inflow_edge_len + premerge + 0.1, + ':center_1': inflow_edge_len + premerge + 0.1, + ':bottom_0': 2 * inflow_edge_len + premerge + postmerge + 22.6 + } + elif params['network'] == RingNetwork: + ring_length = params['net'].additional_params["length"] + junction_length = 0.1 # length of inter-edge junctions + + edgestarts = { + "bottom": 0, + ":right_0": 0.25 * ring_length, + "right": 0.25 * ring_length + junction_length, + ":top_0": 0.5 * ring_length + junction_length, + "top": 0.5 * ring_length + 2 * junction_length, + ":left_0": 0.75 * ring_length + 2 * junction_length, + "left": 0.75 * ring_length + 3 * junction_length, + ":bottom_0": ring_length + 3 * junction_length + } + elif params['network'] == FigureEightNetwork: + net_params = params['net'] + ring_radius = net_params.additional_params['radius_ring'] + ring_edgelen = ring_radius * np.pi / 2. + intersection = 2 * ring_radius + junction = 2.9 + 3.3 * net_params.additional_params['lanes'] + inner = 0.28 + + # generate edge starts + edgestarts = { + 'bottom': inner, + 'top': intersection / 2 + junction + inner, + 'upper_ring': intersection + junction + 2 * inner, + 'right': intersection + 3 * ring_edgelen + junction + 3 * inner, + 'left': 1.5 * intersection + 3 * ring_edgelen + 2 * junction + 3 * inner, + 'lower_ring': 2 * intersection + 3 * ring_edgelen + 2 * junction + 4 * inner, + ':bottom_0': 0, + ':center_1': intersection / 2 + inner, + ':top_0': intersection + junction + inner, + ':right_0': intersection + 3 * ring_edgelen + junction + 2 * inner, + ':center_0': 1.5 * intersection + 3 * ring_edgelen + junction + 3 * inner, + ':left_0': 2 * intersection + 3 * ring_edgelen + 2 * junction + 3 * inner, + # for aimsun + 'bottom_to_top': intersection / 2 + inner, + 'right_to_left': junction + 3 * inner, + } + elif params['network'] == HighwayNetwork: + return df['x'] + elif params['network'] == I210SubNetwork: + edgestarts = { + '119257914': -5.0999999999995795, + '119257908#0': 56.49000000018306, + ':300944379_0': 56.18000000000016, + ':300944436_0': 753.4599999999871, + '119257908#1-AddedOnRampEdge': 756.3299999991157, + ':119257908#1-AddedOnRampNode_0': 853.530000000022, + '119257908#1': 856.7699999997207, + ':119257908#1-AddedOffRampNode_0': 1096.4499999999707, + '119257908#1-AddedOffRampEdge': 1099.6899999995558, + ':1686591010_1': 1198.1899999999541, + '119257908#2': 1203.6499999994803, + ':1842086610_1': 1780.2599999999056, + '119257908#3': 1784.7899999996537, + } + else: + edgestarts = defaultdict(float) + + ret = df.apply(lambda x: x['relative_position'] + edgestarts[x['edge_id']], axis=1) + + if params['network'] == FigureEightNetwork: + # reorganize data for space-time plot + figure_eight_len = 6 * ring_edgelen + 2 * intersection + 2 * junction + 10 * inner + intersection_loc = [edgestarts[':center_1'] + intersection / 2, + edgestarts[':center_0'] + intersection / 2] + ret.loc[ret < intersection_loc[0]] += figure_eight_len + ret.loc[(ret > intersection_loc[0]) & (ret < intersection_loc[1])] += -intersection_loc[1] + ret.loc[ret > intersection_loc[1]] = \ + - ret.loc[ret > intersection_loc[1]] + figure_eight_len + intersection_loc[0] return ret -def _get_abs_pos_1_edge(edges, rel_pos, edge_starts): - """Compute the absolute positions from a subset of edges. +def plot_tsd(df, network, cmap, min_speed=0, max_speed=10, start=0, domain_bounds=None): + """Plot the time-space diagram. - This is the variable we will ultimately use to plot individual vehicles. + Take the pre-processed segments and other meta-data, then plot all the line + segments. Parameters ---------- - edges : list of str - list of edges at every time step - rel_pos : list of float - list of relative positions at every time step - edge_starts : dict - the absolute starting position of every edge - - Returns - ------- - list of float - the absolute positive for every sample + df : pd.DataFrame + data used for axes bounds and speed coloring + network : child class of Network() + network that was used when generating the emission file. + Must be one of the network names mentioned in + ACCEPTABLE_NETWORKS + cmap : colors.LinearSegmentedColormap + colormap for plotting speed + min_speed : int or float + minimum speed in colorbar + max_speed : int or float + maximum speed in colorbar + start : int or float + starting time_step not greyed out + domain_bounds : tuple + lower and upper bounds of domain, excluding ghost edges, default None """ - ret = [] - for edge_i, pos_i in zip(edges, rel_pos): - if edge_i in edge_starts.keys(): - ret.append(pos_i + edge_starts[edge_i]) + norm = plt.Normalize(min_speed, max_speed) + + xmin, xmax = df['time_step'].min(), df['time_step'].max() + xbuffer = (xmax - xmin) * 0.025 # 2.5% of range + ymin, ymax = df['distance'].min(), df['distance'].max() + ybuffer = (ymax - ymin) * 0.025 # 2.5% of range + + # Convert df data into segments for plotting + segs, df = get_time_space_data(df, network) + + nlanes = df['lane_id'].nunique() + plt.figure(figsize=(16, 9*nlanes)) + if nlanes == 1: + segs = [segs] + + for lane, lane_df in df.groupby('lane_id'): + ax = plt.subplot(nlanes, 1, lane+1) + + ax.set_xlim(xmin - xbuffer, xmax + xbuffer) + ax.set_ylim(ymin - ybuffer, ymax + ybuffer) + + lc = LineCollection(segs[lane], cmap=cmap, norm=norm) + lc.set_array(lane_df['speed'].values) + lc.set_linewidth(1) + ax.add_collection(lc) + ax.autoscale() + + rects = [] + # rectangle for warmup period, but not ghost edges + rects.append(Rectangle((xmin, 0), start, domain_bounds[1])) + # rectangle for lower ghost edge (including warmup period) + rects.append(Rectangle((xmin, ymin), xmax - xmin, domain_bounds[0])) + # rectangle for upper ghost edge (including warmup period) + rects.append(Rectangle((xmin, domain_bounds[1]), xmax - xmin, ymax - domain_bounds[1])) + + pc = PatchCollection(rects, facecolor='grey', alpha=0.5, edgecolor=None) + pc.set_zorder(20) + ax.add_collection(pc) + + if nlanes > 1: + ax.set_title('Time-Space Diagram: Lane {}'.format(lane), fontsize=25) else: - ret.append(-1) - return ret + ax.set_title('Time-Space Diagram', fontsize=25) + + ax.set_ylabel('Position (m)', fontsize=20) + if lane == nlanes - 1: + ax.set_xlabel('Time (s)', fontsize=20) + plt.xticks(fontsize=18) + plt.yticks(fontsize=18) + + cbar = plt.colorbar(lc, ax=ax, norm=norm) + cbar.set_label('Velocity (m/s)', fontsize=20) + cbar.ax.tick_params(labelsize=18) + + plt.tight_layout() + + +def tsd_main(trajectory_path, flow_params, min_speed=0, max_speed=10): + """Prepare and plot the time-space diagram. + + Parameters + ---------- + trajectory_path : str + file path (for the .csv formatted file) + flow_params : dict + flow-specific parameters, including: + * "network" (str): name of the network that was used when generating + the emission file. Must be one of the network names mentioned in + ACCEPTABLE_NETWORKS, + * "net_params" (flow.core.params.NetParams): network-specific + parameters. This is used to collect the lengths of various network + links. + min_speed : int or float + minimum speed in colorbar + max_speed : int or float + maximum speed in colorbar + """ + network = flow_params['network'] + + # some plotting parameters + cdict = { + 'red': ((0, 0, 0), (0.2, 1, 1), (0.6, 1, 1), (1, 0, 0)), + 'green': ((0, 0, 0), (0.2, 0, 0), (0.6, 1, 1), (1, 1, 1)), + 'blue': ((0, 0, 0), (0.2, 0, 0), (0.6, 0, 0), (1, 0, 0)) + } + my_cmap = colors.LinearSegmentedColormap('my_colormap', cdict, 1024) + + # Read trajectory csv into pandas dataframe + traj_df, domain_lb, domain_ub, start = import_data_from_trajectory(trajectory_path, flow_params) + + plot_tsd(df=traj_df, + network=network, + cmap=my_cmap, + min_speed=min_speed, + max_speed=max_speed, + start=start, + domain_bounds=(domain_lb, domain_ub)) + + ########################################################################### + # Note: For MergeNetwork only # + if network == MergeNetwork: # + plt.plot([traj_df['time_step'].min(), traj_df['time_step'].max()], + [0, 0], linewidth=3, color="white") # + plt.plot([traj_df['time_step'].min(), traj_df['time_step'].max()], + [-0.1, -0.1], linewidth=3, color="white") # + ########################################################################### + + outfile = trajectory_path.replace('csv', 'png') + plt.savefig(outfile) if __name__ == '__main__': @@ -520,8 +554,8 @@ def _get_abs_pos_1_edge(edges, rel_pos, edge_starts): '.json') # required arguments - parser.add_argument('emission_path', type=str, - help='path to the csv file.') + parser.add_argument('trajectory_path', type=str, + help='path to the Flow trajectory csv file.') parser.add_argument('flow_params', type=str, help='path to the flow_params json file.') @@ -529,16 +563,11 @@ def _get_abs_pos_1_edge(edges, rel_pos, edge_starts): parser.add_argument('--steps', type=int, default=1, help='rate at which steps are plotted.') parser.add_argument('--title', type=str, default='Time Space Diagram', - help='rate at which steps are plotted.') + help='Title for the time-space diagrams.') parser.add_argument('--max_speed', type=int, default=8, help='The maximum speed in the color range.') parser.add_argument('--min_speed', type=int, default=0, help='The minimum speed in the color range.') - parser.add_argument('--start', type=float, default=0, - help='initial time (in sec) in the plot.') - parser.add_argument('--stop', type=float, default=float('inf'), - help='final time (in sec) in the plot.') - args = parser.parse_args() # flow_params is imported as a dictionary @@ -548,78 +577,9 @@ def _get_abs_pos_1_edge(edges, rel_pos, edge_starts): module = __import__("examples.exp_configs.non_rl", fromlist=[args.flow_params]) flow_params = getattr(module, args.flow_params).flow_params - # import data from the emission.csv file - emission_data = import_data_from_emission(args.emission_path) - - # compute the position and speed for all vehicles at all times - pos, speed, time = get_time_space_data(emission_data, flow_params) - - # some plotting parameters - cdict = { - 'red': ((0, 0, 0), (0.2, 1, 1), (0.6, 1, 1), (1, 0, 0)), - 'green': ((0, 0, 0), (0.2, 0, 0), (0.6, 1, 1), (1, 1, 1)), - 'blue': ((0, 0, 0), (0.2, 0, 0), (0.6, 0, 0), (1, 0, 0)) - } - my_cmap = colors.LinearSegmentedColormap('my_colormap', cdict, 1024) - - # perform plotting operation - fig = plt.figure(figsize=(16, 9)) - ax = plt.axes() - norm = plt.Normalize(args.min_speed, args.max_speed) - cols = [] - - xmin = max(time[0], args.start) - xmax = min(time[-1], args.stop) - xbuffer = (xmax - xmin) * 0.025 # 2.5% of range - ymin, ymax = np.amin(pos), np.amax(pos) - ybuffer = (ymax - ymin) * 0.025 # 2.5% of range - - ax.set_xlim(xmin - xbuffer, xmax + xbuffer) - ax.set_ylim(ymin - ybuffer, ymax + ybuffer) - - for indx_car in range(pos.shape[1]): - unique_car_pos = pos[:, indx_car] - - if flow_params['network'] == I210SubNetwork: - indices = np.where(pos[:, indx_car] != 0)[0] - unique_car_speed = speed[indices, indx_car] - points = np.array([time[indices], pos[indices, indx_car]]).T.reshape(-1, 1, 2) - else: - - # discontinuity from wraparound - disc = np.where(np.abs(np.diff(unique_car_pos)) >= 10)[0] + 1 - unique_car_time = np.insert(time, disc, np.nan) - unique_car_pos = np.insert(unique_car_pos, disc, np.nan) - unique_car_speed = np.insert(speed[:, indx_car], disc, np.nan) - # - points = np.array( - [unique_car_time, unique_car_pos]).T.reshape(-1, 1, 2) - segments = np.concatenate([points[:-1], points[1:]], axis=1) - lc = LineCollection(segments, cmap=my_cmap, norm=norm) - - # Set the values used for color mapping - lc.set_array(unique_car_speed) - lc.set_linewidth(1.75) - cols.append(lc) - - plt.title(args.title, fontsize=25) - plt.ylabel('Position (m)', fontsize=20) - plt.xlabel('Time (s)', fontsize=20) - - for col in cols: - line = ax.add_collection(col) - cbar = plt.colorbar(line, ax=ax, norm=norm) - cbar.set_label('Velocity (m/s)', fontsize=20) - cbar.ax.tick_params(labelsize=18) - - plt.xticks(fontsize=18) - plt.yticks(fontsize=18) - - ########################################################################### - # Note: For MergeNetwork only # - if flow_params['network'] == 'MergeNetwork': # - plt.plot(time, [0] * pos.shape[0], linewidth=3, color="white") # - plt.plot(time, [-0.1] * pos.shape[0], linewidth=3, color="white") # - ########################################################################### - - plt.show() + tsd_main( + args.trajectory_path, + flow_params, + min_speed=args.min_speed, + max_speed=args.max_speed + ) diff --git a/flow/visualize/transfer/util.py b/flow/visualize/transfer/util.py new file mode 100644 index 000000000..8c933c5a3 --- /dev/null +++ b/flow/visualize/transfer/util.py @@ -0,0 +1,141 @@ +"""Definitions of transfer classes.""" +from copy import deepcopy + +from flow.core.params import InFlows +from examples.exp_configs.rl.multiagent.multiagent_i210 import INFLOW_RATE, ON_RAMP_INFLOW_RATE + + +def make_inflows(pr=0.1, fr_coef=1.0, departSpeed=20, on_ramp=False): + """Generate inflows object from parameters. Uses default inflows from multiagent_i210. + + Keyword Arguments: + ----------------- + pr {float} -- [AV Penetration Rate] (default: {0.1}) + fr_coef {float} -- [Scale flow rate by] (default: {1.0}) + departSpeed {int} -- [Initial speed of all flows] (default: {20}) + + Returns + ------- + [Inflows] -- [Inflows parameter object] + + """ + inflow = InFlows() + # main highway + assert pr < 1.0, "your penetration rate is over 100%" + + all_inflows = [] + + inflow_119257914 = dict(veh_type="human", + edge="ghost0", + vehs_per_hour=INFLOW_RATE * (1 - (pr)) * fr_coef, + # probability=1.0, + departLane="random", + departSpeed=departSpeed) + all_inflows.append(inflow_119257914) + + if pr > 0.0: + inflow_119257914_av = dict(veh_type="av", + edge="ghost0", + vehs_per_hour=int(INFLOW_RATE * pr * fr_coef), + # probability=1.0, + departLane="random", + departSpeed=departSpeed) + all_inflows.append(inflow_119257914_av) + + if on_ramp: + inflow_27414345 = dict(veh_type="human", + edge="27414345", + vehs_per_hour=ON_RAMP_INFLOW_RATE * (1 - (pr)) * fr_coef, + departLane="random", + departSpeed=departSpeed) + all_inflows.append(inflow_27414345) + if pr > 0.0: + inflow_27414342 = dict(veh_type="human", + edge="27414342#0", + vehs_per_hour=ON_RAMP_INFLOW_RATE * pr * fr_coef, + departLane="random", + departSpeed=departSpeed) + all_inflows.append(inflow_27414342) + + for inflow_def in all_inflows: + inflow.add(**inflow_def) + + return inflow + + +class BaseTransfer: + """Base Transfer class.""" + + def __init__(self): + self.transfer_str = "Base" + pass + + def flow_params_modifier_fn(self, flow_params, clone_params=True): + """Return modified flow_params. + + Arguments: + --------- + flow_params {[flow_params_dictionary]} -- [flow_params] + """ + if clone_params: + flow_params = deepcopy(flow_params) + + return flow_params + + def env_modifier_fn(self, env): + """Modify the env before rollouts are run. + + Arguments: + --------- + env {[I210MultiEnv]} -- [Env to modify] + """ + pass + + +class InflowTransfer(BaseTransfer): + """Modifies the inflow of i210 env.""" + + def __init__(self, penetration_rate=0.1, flow_rate_coef=1.0, departSpeed=20): + super(InflowTransfer, self).__init__() + self.penetration_rate = penetration_rate + self.flow_rate_coef = flow_rate_coef + self.departSpeed = departSpeed + + self.transfer_str = "{:0.2f}_pen_{:0.2f}_flow_rate_coef_{:0.2f}_depspeed".format( + penetration_rate, flow_rate_coef, departSpeed) + + def flow_params_modifier_fn(self, flow_params, clone_params=True): + """See Parent.""" + if clone_params: + flow_params = deepcopy(flow_params) + + flow_params['net'].inflows = make_inflows(self.penetration_rate, self.flow_rate_coef, self.departSpeed) + + return flow_params + + +def inflows_range(penetration_rates=0.1, flow_rate_coefs=1.0, departSpeeds=20.0): + """Generate inflow objects given penetration_rates, flow_rates, and depart speeds. + + Keyword Arguments: + ----------------- + penetration_rates {float | list of floats} -- [single, or multiple penetration rates] (default: {0.1}) + flow_rate_coefs {float | list of floats} -- [single, or multiple flow rate coefficient] (default: {1.0}) + departSpeeds {float | list of floats} -- [single, or multiple depart speeds] (default: {20.0}) + + Yields + ------ + [InflowTransfer] -- [Transfer object] + """ + if not hasattr(penetration_rates, '__iter__'): + penetration_rates = [penetration_rates] + if not hasattr(flow_rate_coefs, '__iter__'): + flow_rate_coefs = [flow_rate_coefs] + if not hasattr(departSpeeds, '__iter__'): + departSpeeds = [departSpeeds] + + for departSpeed in departSpeeds: + for penetration_rate in penetration_rates: + for flow_rate_coef in flow_rate_coefs: + yield InflowTransfer(penetration_rate=penetration_rate, flow_rate_coef=flow_rate_coef, + departSpeed=departSpeed) diff --git a/flow/visualize/visualizer_rllib.py b/flow/visualize/visualizer_rllib.py deleted file mode 100644 index 8c38a91c1..000000000 --- a/flow/visualize/visualizer_rllib.py +++ /dev/null @@ -1,386 +0,0 @@ -"""Visualizer for rllib experiments. - -Attributes ----------- -EXAMPLE_USAGE : str - Example call to the function, which is - :: - - python ./visualizer_rllib.py /tmp/ray/result_dir 1 - -parser : ArgumentParser - Command-line argument parser -""" - -import argparse -import gym -import numpy as np -import os -import sys -import time - -import ray -try: - from ray.rllib.agents.agent import get_agent_class -except ImportError: - from ray.rllib.agents.registry import get_agent_class -from ray.tune.registry import register_env - -from flow.core.util import emission_to_csv -from flow.utils.registry import make_create_env -from flow.utils.rllib import get_flow_params -from flow.utils.rllib import get_rllib_config -from flow.utils.rllib import get_rllib_pkl - - -EXAMPLE_USAGE = """ -example usage: - python ./visualizer_rllib.py /ray_results/experiment_dir/result_dir 1 - -Here the arguments are: -1 - the path to the simulation results -2 - the number of the checkpoint -""" - - -def visualizer_rllib(args): - """Visualizer for RLlib experiments. - - This function takes args (see function create_parser below for - more detailed information on what information can be fed to this - visualizer), and renders the experiment associated with it. - """ - result_dir = args.result_dir if args.result_dir[-1] != '/' \ - else args.result_dir[:-1] - - config = get_rllib_config(result_dir) - - # check if we have a multiagent environment but in a - # backwards compatible way - if config.get('multiagent', {}).get('policies', None): - multiagent = True - pkl = get_rllib_pkl(result_dir) - config['multiagent'] = pkl['multiagent'] - else: - multiagent = False - - # Run on only one cpu for rendering purposes - config['num_workers'] = 0 - - flow_params = get_flow_params(config) - - # hack for old pkl files - # TODO(ev) remove eventually - sim_params = flow_params['sim'] - setattr(sim_params, 'num_clients', 1) - - # for hacks for old pkl files TODO: remove eventually - if not hasattr(sim_params, 'use_ballistic'): - sim_params.use_ballistic = False - - # Determine agent and checkpoint - config_run = config['env_config']['run'] if 'run' in config['env_config'] \ - else None - if args.run and config_run: - if args.run != config_run: - print('visualizer_rllib.py: error: run argument ' - + '\'{}\' passed in '.format(args.run) - + 'differs from the one stored in params.json ' - + '\'{}\''.format(config_run)) - sys.exit(1) - if args.run: - agent_cls = get_agent_class(args.run) - elif config_run: - agent_cls = get_agent_class(config_run) - else: - print('visualizer_rllib.py: error: could not find flow parameter ' - '\'run\' in params.json, ' - 'add argument --run to provide the algorithm or model used ' - 'to train the results\n e.g. ' - 'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO') - sys.exit(1) - - sim_params.restart_instance = True - dir_path = os.path.dirname(os.path.realpath(__file__)) - emission_path = '{0}/test_time_rollout/'.format(dir_path) - sim_params.emission_path = emission_path if args.gen_emission else None - - # pick your rendering mode - if args.render_mode == 'sumo_web3d': - sim_params.num_clients = 2 - sim_params.render = False - elif args.render_mode == 'drgb': - sim_params.render = 'drgb' - sim_params.pxpm = 4 - elif args.render_mode == 'sumo_gui': - sim_params.render = False # will be set to True below - elif args.render_mode == 'no_render': - sim_params.render = False - if args.save_render: - if args.render_mode != 'sumo_gui': - sim_params.render = 'drgb' - sim_params.pxpm = 4 - sim_params.save_render = True - - # Create and register a gym+rllib env - create_env, env_name = make_create_env(params=flow_params, version=0) - register_env(env_name, create_env) - - # check if the environment is a single or multiagent environment, and - # get the right address accordingly - # single_agent_envs = [env for env in dir(flow.envs) - # if not env.startswith('__')] - - # if flow_params['env_name'] in single_agent_envs: - # env_loc = 'flow.envs' - # else: - # env_loc = 'flow.envs.multiagent' - - # Start the environment with the gui turned on and a path for the - # emission file - env_params = flow_params['env'] - env_params.restart_instance = False - if args.evaluate: - env_params.evaluate = True - - # lower the horizon if testing - if args.horizon: - config['horizon'] = args.horizon - env_params.horizon = args.horizon - - # create the agent that will be used to compute the actions - agent = agent_cls(env=env_name, config=config) - checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num - checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num - agent.restore(checkpoint) - - if hasattr(agent, "local_evaluator") and \ - os.environ.get("TEST_FLAG") != 'True': - env = agent.local_evaluator.env - else: - env = gym.make(env_name) - - if args.render_mode == 'sumo_gui': - env.sim_params.render = True # set to True after initializing agent and env - - if multiagent: - rets = {} - # map the agent id to its policy - policy_map_fn = config['multiagent']['policy_mapping_fn'].func - for key in config['multiagent']['policies'].keys(): - rets[key] = [] - else: - rets = [] - - if config['model']['use_lstm']: - use_lstm = True - if multiagent: - state_init = {} - # map the agent id to its policy - policy_map_fn = config['multiagent']['policy_mapping_fn'].func - size = config['model']['lstm_cell_size'] - for key in config['multiagent']['policies'].keys(): - state_init[key] = [np.zeros(size, np.float32), - np.zeros(size, np.float32)] - else: - state_init = [ - np.zeros(config['model']['lstm_cell_size'], np.float32), - np.zeros(config['model']['lstm_cell_size'], np.float32) - ] - else: - use_lstm = False - - # if restart_instance, don't restart here because env.reset will restart later - if not sim_params.restart_instance: - env.restart_simulation(sim_params=sim_params, render=sim_params.render) - - # Simulate and collect metrics - final_outflows = [] - final_inflows = [] - mean_speed = [] - std_speed = [] - for i in range(args.num_rollouts): - vel = [] - state = env.reset() - if multiagent: - ret = {key: [0] for key in rets.keys()} - else: - ret = 0 - for _ in range(env_params.horizon): - vehicles = env.unwrapped.k.vehicle - speeds = vehicles.get_speed(vehicles.get_ids()) - - # only include non-empty speeds - if speeds: - vel.append(np.mean(speeds)) - - if multiagent: - action = {} - for agent_id in state.keys(): - if use_lstm: - action[agent_id], state_init[agent_id], logits = \ - agent.compute_action( - state[agent_id], state=state_init[agent_id], - policy_id=policy_map_fn(agent_id)) - else: - action[agent_id] = agent.compute_action( - state[agent_id], policy_id=policy_map_fn(agent_id)) - else: - action = agent.compute_action(state) - state, reward, done, _ = env.step(action) - if multiagent: - for actor, rew in reward.items(): - ret[policy_map_fn(actor)][0] += rew - else: - ret += reward - if multiagent and done['__all__']: - break - if not multiagent and done: - break - - if multiagent: - for key in rets.keys(): - rets[key].append(ret[key]) - else: - rets.append(ret) - outflow = vehicles.get_outflow_rate(500) - final_outflows.append(outflow) - inflow = vehicles.get_inflow_rate(500) - final_inflows.append(inflow) - if np.all(np.array(final_inflows) > 1e-5): - throughput_efficiency = [x / y for x, y in - zip(final_outflows, final_inflows)] - else: - throughput_efficiency = [0] * len(final_inflows) - mean_speed.append(np.mean(vel)) - std_speed.append(np.std(vel)) - if multiagent: - for agent_id, rew in rets.items(): - print('Round {}, Return: {} for agent {}'.format( - i, ret, agent_id)) - else: - print('Round {}, Return: {}'.format(i, ret)) - - print('==== Summary of results ====') - print("Return:") - print(mean_speed) - if multiagent: - for agent_id, rew in rets.items(): - print('For agent', agent_id) - print(rew) - print('Average, std return: {}, {} for agent {}'.format( - np.mean(rew), np.std(rew), agent_id)) - else: - print(rets) - print('Average, std: {}, {}'.format( - np.mean(rets), np.std(rets))) - - print("\nSpeed, mean (m/s):") - print(mean_speed) - print('Average, std: {}, {}'.format(np.mean(mean_speed), np.std( - mean_speed))) - print("\nSpeed, std (m/s):") - print(std_speed) - print('Average, std: {}, {}'.format(np.mean(std_speed), np.std( - std_speed))) - - # Compute arrival rate of vehicles in the last 500 sec of the run - print("\nOutflows (veh/hr):") - print(final_outflows) - print('Average, std: {}, {}'.format(np.mean(final_outflows), - np.std(final_outflows))) - # Compute departure rate of vehicles in the last 500 sec of the run - print("Inflows (veh/hr):") - print(final_inflows) - print('Average, std: {}, {}'.format(np.mean(final_inflows), - np.std(final_inflows))) - # Compute throughput efficiency in the last 500 sec of the - print("Throughput efficiency (veh/hr):") - print(throughput_efficiency) - print('Average, std: {}, {}'.format(np.mean(throughput_efficiency), - np.std(throughput_efficiency))) - - # terminate the environment - env.unwrapped.terminate() - - # if prompted, convert the emission file into a csv file - if args.gen_emission: - time.sleep(0.1) - - dir_path = os.path.dirname(os.path.realpath(__file__)) - emission_filename = '{0}-emission.xml'.format(env.network.name) - - emission_path = \ - '{0}/test_time_rollout/{1}'.format(dir_path, emission_filename) - - # convert the emission file into a csv file - emission_to_csv(emission_path) - - # print the location of the emission csv file - emission_path_csv = emission_path[:-4] + ".csv" - print("\nGenerated emission file at " + emission_path_csv) - - # delete the .xml version of the emission file - os.remove(emission_path) - - -def create_parser(): - """Create the parser to capture CLI arguments.""" - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description='[Flow] Evaluates a reinforcement learning agent ' - 'given a checkpoint.', - epilog=EXAMPLE_USAGE) - - # required input parameters - parser.add_argument( - 'result_dir', type=str, help='Directory containing results') - parser.add_argument('checkpoint_num', type=str, help='Checkpoint number.') - - # optional input parameters - parser.add_argument( - '--run', - type=str, - help='The algorithm or model to train. This may refer to ' - 'the name of a built-on algorithm (e.g. RLLib\'s DQN ' - 'or PPO), or a user-defined trainable function or ' - 'class registered in the tune registry. ' - 'Required for results trained with flow-0.2.0 and before.') - parser.add_argument( - '--num_rollouts', - type=int, - default=1, - help='The number of rollouts to visualize.') - parser.add_argument( - '--gen_emission', - action='store_true', - help='Specifies whether to generate an emission file from the ' - 'simulation') - parser.add_argument( - '--evaluate', - action='store_true', - help='Specifies whether to use the \'evaluate\' reward ' - 'for the environment.') - parser.add_argument( - '--render_mode', - type=str, - default='sumo_gui', - help='Pick the render mode. Options include sumo_web3d, ' - 'rgbd and sumo_gui') - parser.add_argument( - '--save_render', - action='store_true', - help='Saves a rendered video to a file. NOTE: Overrides render_mode ' - 'with pyglet rendering.') - parser.add_argument( - '--horizon', - type=int, - help='Specifies the horizon.') - return parser - - -if __name__ == '__main__': - parser = create_parser() - args = parser.parse_args() - ray.init(num_cpus=1) - visualizer_rllib(args) diff --git a/requirements.txt b/requirements.txt index 546cb4e26..2d3c61093 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ gym==0.14.0 -numpy==1.16.0 +numpy==1.18.4 scipy==1.1.0 lxml==4.4.1 pyprind==2.11.2 @@ -9,19 +9,24 @@ path.py joblib==0.10.3 python-dateutil==2.7.3 cached_property -cloudpickle==1.2.0 pyglet==1.3.2 matplotlib==3.1.0 imutils==0.5.1 numpydoc -ray==0.7.3 +ray==0.8.4 opencv-python dill lz4 setproctitle psutil opencv-python -boto3==1.4.8 +boto3==1.10.45 redis~=2.10.6 pandas==0.24.2 plotly==2.4.0 +tabulate +tensorflow==1.15.2 +awscli==1.16.309 +torch==1.4.0 +pytz +tensorboardX diff --git a/scripts/ray_autoscale.yaml b/scripts/ray_autoscale.yaml index 5bf2a9c4a..197ed76e1 100644 --- a/scripts/ray_autoscale.yaml +++ b/scripts/ray_autoscale.yaml @@ -1,4 +1,4 @@ -# cluster.yaml ========================================= +# cluster.yaml ========================================= # An unique identifier for the head node and workers of this cluster. cluster_name: test # @@ -32,15 +32,14 @@ auth: # By default Ray creates a new private keypair, but you can also use your own. # If you do so, make sure to also set "KeyName" in the head and worker node # configurations below. -# ssh_private_key: /path/to/your/key.pem # Provider-specific config for the head node, e.g. instance type. By default # Ray will auto-configure unspecified fields such as SubnetId and KeyName. # For more documentation on available fields, see: # http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances head_node: - InstanceType: c4.4xlarge - ImageId: ami-09544298704576518 # Flow AMI (Ubuntu) + InstanceType: c4.8xlarge + ImageId: ami-0d24fb9ee1a709550 # Flow AMI (Ubuntu) InstanceMarketOptions: MarketType: spot #Additional options can be found in the boto docs, e.g. @@ -54,11 +53,11 @@ head_node: # For more documentation on available fields, see: # http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances worker_nodes: - InstanceType: c4.4xlarge - ImageId: ami-09544298704576518 # Flow AMI (Ubuntu) - - #Run workers on spot by default. Comment this out to use on-demand. + InstanceType: c4.8xlarge + ImageId: ami-0d24fb9ee1a709550 # Flow AMI (Ubuntu) + #Run workers on spot by default. Comment this out to use on-demand. InstanceMarketOptions: + MarketType: spot # Additional options can be found in the boto docs, e.g. SpotOptions: @@ -67,12 +66,21 @@ worker_nodes: # Additional options in the boto docs. setup_commands: - - cd flow && git fetch && git checkout origin/master - -head_setup_commands: + - cd flow && git fetch && git checkout origin/libsumo_new_binaries +# - flow/scripts/setup_sumo_ubuntu1604.sh + - pip install ray==0.8.0 + - pip install tabulate - pip install boto3==1.10.45 # 1.4.8 adds InstanceMarketOptions - pip install awscli==1.16.309 + - pip install stable-baselines - pip install pytz + - pip install torch==1.3.1 + - pip install tensorflow==2.0.0 + - pip install lz4 + - pip install dm-tree + - pip install numpy==1.18.4 + +head_setup_commands: [] # Custom commands that will be run on worker nodes after common setup. worker_setup_commands: [] diff --git a/scripts/setup_libsumo_ubuntu.sh b/scripts/setup_libsumo_ubuntu.sh old mode 100755 new mode 100644 index 09cee8c08..b69017961 --- a/scripts/setup_libsumo_ubuntu.sh +++ b/scripts/setup_libsumo_ubuntu.sh @@ -14,6 +14,6 @@ cmake . make -j$(nproc) popd -echo 'export PATH="$PATH:$HOME/sumo_binaries/sumo/bin"' >> ~/.bashrc +echo 'export PATH="$HOME/sumo_binaries/sumo/bin:$PATH"' >> ~/.bashrc echo 'export SUMO_HOME="$HOME/sumo_binaries/sumo"' >> ~/.bashrc -echo 'export PYTHONPATH="$PYTHONPATH:$HOME/sumo_binaries/sumo/tools"' >> ~/.bashrc +echo 'export PYTHONPATH="$HOME/sumo_binaries/sumo/tools:$PYTHONPATH"' >> ~/.bashrc diff --git a/scripts/setup_sumo_osx.sh b/scripts/setup_sumo_osx.sh index 581e4b880..d73507b30 100755 --- a/scripts/setup_sumo_osx.sh +++ b/scripts/setup_sumo_osx.sh @@ -8,13 +8,15 @@ brew install wget brew install swig sdl sdl_image sdl_mixer sdl_ttf portmidi # sumo dependencies brew install Caskroom/cask/xquartz autoconf automake pkg-config libtool gdal proj xerces-c fox +# link the libproj versions because the default install does not match what sumo wants +ln -s /usr/local/opt/proj/lib/libproj.19.dylib /usr/local/opt/proj/lib/libproj.15.dylib echo "Installing sumo binaries" mkdir -p $HOME/sumo_binaries/bin pushd $HOME/sumo_binaries/bin -wget https://akreidieh.s3.amazonaws.com/sumo/flow-0.4.0/binaries-mac.tar.xz -tar -xf binaries-mac.tar.xz -rm binaries-mac.tar.xz +wget https://flow-sumo.s3-us-west-1.amazonaws.com/libsumo/sumo_binaries_MacOS.zip +unzip sumo_binaries_MacOS.zip +rm sumo_binaries_MacOS.zip chmod +x * popd export SUMO_HOME="$HOME/sumo_binaries/bin" diff --git a/scripts/setup_sumo_ubuntu1404.sh b/scripts/setup_sumo_ubuntu1404.sh index 41d4cb113..dcb0b69c3 100755 --- a/scripts/setup_sumo_ubuntu1404.sh +++ b/scripts/setup_sumo_ubuntu1404.sh @@ -8,7 +8,7 @@ sudo apt-get install -y libxerces-c3.1 libxerces-c3-dev libproj-dev sudo apt-get install -y proj-bin proj-data libgdal1-dev libfox-1.6-0 sudo apt-get install -y libfox-1.6-dev -echo "Installing sumo binaries" +echo "Installing sumo binaries - Libsumo not available on Ubuntu 1404" mkdir -p $HOME/sumo_binaries/bin pushd $HOME/sumo_binaries/bin wget https://akreidieh.s3.amazonaws.com/sumo/flow-0.4.0/binaries-ubuntu1404.tar.xz @@ -16,5 +16,7 @@ tar -xf binaries-ubuntu1404.tar.xz rm binaries-ubuntu1404.tar.xz chmod +x * popd + +echo "# Added by Sumo installation" >> ~/.bashrc echo 'export PATH="$HOME/sumo_binaries/bin:$PATH"' >> ~/.bashrc echo 'export SUMO_HOME="$HOME/sumo_binaries/bin"' >> ~/.bashrc diff --git a/scripts/setup_sumo_ubuntu1604.sh b/scripts/setup_sumo_ubuntu1604.sh index 744a11929..d287d207f 100755 --- a/scripts/setup_sumo_ubuntu1604.sh +++ b/scripts/setup_sumo_ubuntu1604.sh @@ -1,4 +1,5 @@ #!/bin/bash + echo "Installing system dependencies for SUMO" sudo apt-get update sudo apt-get install -y cmake swig libgtest-dev python-pygame python-scipy @@ -9,12 +10,17 @@ sudo apt-get install -y python3-dev sudo pip3 install cmake cython echo "Installing sumo binaries" +rm -rf $HOME/sumo_binaries/bin mkdir -p $HOME/sumo_binaries/bin pushd $HOME/sumo_binaries/bin -wget https://akreidieh.s3.amazonaws.com/sumo/flow-0.4.0/binaries-ubuntu1604.tar.xz -tar -xf binaries-ubuntu1604.tar.xz -rm binaries-ubuntu1604.tar.xz -chmod +x * +wget https://flow-sumo.s3-us-west-1.amazonaws.com/libsumo_test/binaries-ubuntu1604.tar.gz +tar xvf binaries-ubuntu1604.tar.gz +rm binaries-ubuntu1604.tar.gz popd +cd sumo_binaries +chmod +x * + +echo '# Added by Sumo / Libsumo instalation' >> ~/.bashrc echo 'export PATH="$HOME/sumo_binaries/bin:$PATH"' >> ~/.bashrc echo 'export SUMO_HOME="$HOME/sumo_binaries/bin"' >> ~/.bashrc +echo 'export PYTHONPATH="$PYTHONPATH:$HOME/sumo_binaries/tools"' >> ~/.bashrc diff --git a/scripts/setup_sumo_ubuntu1804.sh b/scripts/setup_sumo_ubuntu1804.sh index c9e2a09af..3437cdf28 100755 --- a/scripts/setup_sumo_ubuntu1804.sh +++ b/scripts/setup_sumo_ubuntu1804.sh @@ -12,10 +12,14 @@ sudo pip3 install cmake cython echo "Installing sumo binaries" mkdir -p $HOME/sumo_binaries/bin pushd $HOME/sumo_binaries/bin -wget https://akreidieh.s3.amazonaws.com/sumo/flow-0.4.0/binaries-ubuntu1804.tar.xz -tar -xf binaries-ubuntu1804.tar.xz -rm binaries-ubuntu1804.tar.xz +wget https://flow-sumo.s3-us-west-1.amazonaws.com/libsumo/sumo_binaries_ubuntu1804.tar.gz +tar -zxvf sumo_binaries_ubuntu1804.tar.gz +rm sumo_binaries_ubuntu1804.tar.gz +cd sumo_binaries chmod +x * popd + +echo '# Added by Sumo / Libsumo instalation' >> ~/.bashrc echo 'export PATH="$HOME/sumo_binaries/bin:$PATH"' >> ~/.bashrc echo 'export SUMO_HOME="$HOME/sumo_binaries/bin"' >> ~/.bashrc +echo 'export PYTHONPATH="$HOME/sumo_binaries/tools:$PYTHONPATH"' >> ~/.bashrc diff --git a/setup.py b/setup.py index f9508507e..70b894da2 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ class build_ext(_build_ext.build_ext): def run(self): """Install traci wheels.""" subprocess.check_call( - ['pip', 'install', + ['python3','-m','pip', 'install', 'https://akreidieh.s3.amazonaws.com/sumo/flow-0.4.0/' 'sumotools-0.4.0-py3-none-any.whl']) diff --git a/tests/data/rllib_data/multi_agent/checkpoint_1/.is_checkpoint b/tests/data/rllib_data/multi_agent/checkpoint_1/.is_checkpoint new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1 b/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1 index 0693ed4b6..127111e67 100644 Binary files a/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1 and b/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1 differ diff --git a/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1.tune_metadata b/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1.tune_metadata index 7eef2ef15..268e5cfe5 100644 Binary files a/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1.tune_metadata and b/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1.tune_metadata differ diff --git a/tests/data/rllib_data/multi_agent/params.json b/tests/data/rllib_data/multi_agent/params.json index 01089f730..9451d356e 100644 --- a/tests/data/rllib_data/multi_agent/params.json +++ b/tests/data/rllib_data/multi_agent/params.json @@ -1,54 +1,64 @@ { "batch_mode": "truncate_episodes", "callbacks": { - "on_episode_end": null, - "on_episode_start": null, - "on_episode_step": null, - "on_postprocess_traj": null, - "on_sample_end": null, - "on_train_result": null + "on_episode_end": ".on_episode_end at 0x1460752f0>", + "on_episode_start": ".on_episode_start at 0x11d064048>", + "on_episode_step": ".on_episode_step at 0x146075268>", + "on_train_result": ".on_train_result at 0x146075378>" }, - "clip_actions": false, + "clip_actions": true, "clip_param": 0.3, "clip_rewards": null, "collect_metrics_timeout": 180, "compress_observations": false, + "custom_eval_function": null, "custom_resources_per_worker": {}, + "eager": false, + "eager_tracing": false, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, - "env": "MultiWaveAttenuationPOEnv-v0", + "env": "MultiStraightRoad-v1", "env_config": { - "flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 230,\n 230\n ],\n \"target_velocity\": 4\n },\n \"clip_actions\": true,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"MultiWaveAttenuationPOEnv\",\n \"exp_tag\": \"lord_of_numrings1\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 20.0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"custom\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 230,\n \"num_rings\": 1,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"MultiRingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human_0\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl_0\"\n }\n ]\n}", - "run": "PPO" + "flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"accel_penalty\": 0.05,\n \"control_range\": [\n 500,\n 2300\n ],\n \"headway_curriculum\": false,\n \"headway_curriculum_iters\": 100,\n \"headway_reward_gain\": 2.0,\n \"lead_obs\": true,\n \"local_reward\": true,\n \"look_back_length\": 3,\n \"max_accel\": 2.6,\n \"max_decel\": 4.5,\n \"max_num_agents\": 10,\n \"min_time_headway\": 2.0,\n \"mpg_reward\": false,\n \"mpj_reward\": false,\n \"penalize_accel\": true,\n \"penalize_stops\": true,\n \"reroute_on_exit\": true,\n \"sort_vehicles\": false,\n \"speed_curriculum\": true,\n \"speed_curriculum_iters\": 20,\n \"speed_reward_gain\": 1.0,\n \"stop_penalty\": 0.05,\n \"target_velocity\": 6.0\n },\n \"clip_actions\": true,\n \"done_at_exit\": true,\n \"evaluate\": false,\n \"horizon\": 1000,\n \"sims_per_step\": 3,\n \"warmup_steps\": 500\n },\n \"env_name\": \"flow.envs.multiagent.i210.MultiStraightRoad\",\n \"exp_tag\": \"multiagent_highway\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"boundary_cell_length\": 300,\n \"ghost_speed_limit\": 6.0,\n \"lanes\": 1,\n \"length\": 2500,\n \"num_edges\": 2,\n \"speed_limit\": 30,\n \"use_ghost_edge\": true\n },\n \"inflows\": {\n \"_InFlows__flows\": [\n {\n \"begin\": 1,\n \"departLane\": \"free\",\n \"departSpeed\": 24.1,\n \"edge\": \"highway_0\",\n \"end\": 86400,\n \"name\": \"idm_highway_inflow_0\",\n \"vehsPerHour\": 1993,\n \"vtype\": \"human\"\n },\n {\n \"begin\": 1,\n \"departLane\": \"free\",\n \"departSpeed\": 24.1,\n \"edge\": \"highway_0\",\n \"end\": 86400,\n \"name\": \"rl_highway_inflow_1\",\n \"vehsPerHour\": 221,\n \"vtype\": \"rl\"\n }\n ]\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.highway.HighwayNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"disable_collisions\": false,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": true,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.4,\n \"teleport_time\": -1,\n \"use_ballistic\": true\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"a\": 1.3,\n \"b\": 2.0,\n \"noise\": 0.3\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"SL2015\",\n \"lcAccelLat\": \"1.0\",\n \"lcAssertive\": \"1\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcLookaheadLeft\": \"2.0\",\n \"lcPushy\": \"0\",\n \"lcPushyGap\": \"0.6\",\n \"lcSpeedGain\": \"1.0\",\n \"lcSpeedGainRight\": \"1.0\",\n \"lcStrategic\": \"1.0\",\n \"lcSublane\": \"2.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"energy_model\": \"PDMCombustionEngine\",\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 0,\n \"routing_controller\": null,\n \"veh_id\": \"rl\"\n }\n ]\n}", + "run": "" }, "evaluation_config": {}, "evaluation_interval": null, "evaluation_num_episodes": 10, - "gamma": 0.999, + "evaluation_num_workers": 0, + "exploration_config": { + "type": "StochasticSampling" + }, + "explore": true, + "gamma": 0.995, "grad_clip": null, - "horizon": 3000, + "horizon": 1000, "ignore_worker_failures": false, + "in_evaluation": false, "input": "sampler", "input_evaluation": [ "is", "wis" ], "kl_coeff": 0.2, - "kl_target": 0.01, - "lambda": 1.0, + "kl_target": 0.02, + "lambda": 0.97, "local_tf_session_args": { "inter_op_parallelism_threads": 8, "intra_op_parallelism_threads": 8 }, - "log_level": "INFO", + "log_level": "WARN", "log_sys_usage": true, - "lr": 1e-05, + "lr": 5e-05, "lr_schedule": null, + "memory": 0, + "memory_per_worker": 0, "metrics_smoothing_episodes": 100, "min_iter_time_s": 0, "model": { "conv_activation": "relu", "conv_filters": null, + "custom_action_dist": null, "custom_model": null, "custom_options": {}, "custom_preprocessor": null, @@ -74,24 +84,27 @@ "multiagent": { "policies": { "av": [ - "", + null, "Box(3,)", "Box(1,)", {} ] }, - "policies_to_train": [ - "av" - ], - "policy_mapping_fn": "tune.function(.policy_mapping_fn at 0x7fda132e6c80>)" + "policies_to_train": null, + "policy_mapping_fn": "" }, + "no_done_at_end": false, + "no_eager_on_workers": false, + "normalize_actions": false, "num_cpus_for_driver": 1, "num_cpus_per_worker": 1, "num_envs_per_worker": 1, "num_gpus": 0, "num_gpus_per_worker": 0, - "num_sgd_iter": 30, - "num_workers": 2, + "num_sgd_iter": 10, + "num_workers": 1, + "object_store_memory": 0, + "object_store_memory_per_worker": 0, "observation_filter": "NoFilter", "optimizer": {}, "output": null, @@ -104,13 +117,14 @@ "preprocessor_pref": "deepmind", "remote_env_batch_wait_ms": 0, "remote_worker_envs": false, + "rollout_fragment_length": 200, "sample_async": false, - "sample_batch_size": 200, + "sample_batch_size": -1, "seed": null, "sgd_minibatch_size": 128, "shuffle_buffer_size": 0, "shuffle_sequences": true, - "simple_optimizer": true, + "simple_optimizer": false, "soft_horizon": false, "synchronize_filters": true, "tf_session_args": { @@ -126,8 +140,11 @@ "log_device_placement": false }, "timesteps_per_iteration": 0, - "train_batch_size": 60000, + "train_batch_size": 1000, + "use_critic": true, + "use_exec_api": false, "use_gae": true, + "use_pytorch": false, "vf_clip_param": 10.0, "vf_loss_coeff": 1.0, "vf_share_layers": false diff --git a/tests/data/rllib_data/multi_agent/params.pkl b/tests/data/rllib_data/multi_agent/params.pkl index cd832aa1c..d1260bcbb 100644 Binary files a/tests/data/rllib_data/multi_agent/params.pkl and b/tests/data/rllib_data/multi_agent/params.pkl differ diff --git a/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1 b/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1 index f8a7e8976..b7ae94640 100644 Binary files a/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1 and b/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1 differ diff --git a/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1.tune_metadata b/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1.tune_metadata index e83b72aea..55b72be28 100644 Binary files a/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1.tune_metadata and b/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1.tune_metadata differ diff --git a/tests/data/rllib_data/single_agent/params.json b/tests/data/rllib_data/single_agent/params.json index c5e605ef4..2f55a1eed 100644 --- a/tests/data/rllib_data/single_agent/params.json +++ b/tests/data/rllib_data/single_agent/params.json @@ -8,25 +8,27 @@ "on_sample_end": null, "on_train_result": null }, - "clip_actions": false, + "clip_actions": true, "clip_param": 0.3, "clip_rewards": null, "collect_metrics_timeout": 180, "compress_observations": false, "custom_resources_per_worker": {}, + "eager": false, + "eager_tracing": false, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, - "env": "WaveAttenuationPOEnv-v0", + "env": "AccelEnv-v0", "env_config": { - "flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", - "run": "PPO" + "run": "PPO", + "flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 3,\n \"max_decel\": 3,\n \"sort_vehicles\": false,\n \"target_velocity\": 20\n },\n \"clip_actions\": true,\n \"evaluate\": false,\n \"horizon\": 1500,\n \"sims_per_step\": 1,\n \"warmup_steps\": 0\n },\n \"env_name\": \"flow.envs.ring.accel.AccelEnv\",\n \"exp_tag\": \"singleagent_figure_eight\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"radius_ring\": 30,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.figure_eight.FigureEightNetwork\",\n \"sim\": {\n \"disable_collisions\": false,\n \"color_by_speed\": false,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1,\n \"use_ballistic\": false\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 1.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 13,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 1.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}" }, "evaluation_config": {}, "evaluation_interval": null, "evaluation_num_episodes": 10, "gamma": 0.999, "grad_clip": null, - "horizon": 3000, + "horizon": 1500, "ignore_worker_failures": false, "input": "sampler", "input_evaluation": [ @@ -40,23 +42,27 @@ "inter_op_parallelism_threads": 8, "intra_op_parallelism_threads": 8 }, - "log_level": "INFO", + "log_level": "WARN", "log_sys_usage": true, "lr": 5e-05, "lr_schedule": null, + "memory": 0, + "memory_per_worker": 0, "metrics_smoothing_episodes": 100, "min_iter_time_s": 0, "model": { "conv_activation": "relu", "conv_filters": null, + "custom_action_dist": null, "custom_model": null, "custom_options": {}, "custom_preprocessor": null, "dim": 84, "fcnet_activation": "tanh", "fcnet_hiddens": [ - 3, - 3 + 32, + 32, + 32 ], "framestack": true, "free_log_std": false, @@ -76,6 +82,8 @@ "policies_to_train": null, "policy_mapping_fn": null }, + "no_done_at_end": false, + "no_eager_on_workers": false, "num_cpus_for_driver": 1, "num_cpus_per_worker": 1, "num_envs_per_worker": 1, @@ -83,6 +91,8 @@ "num_gpus_per_worker": 0, "num_sgd_iter": 10, "num_workers": 2, + "object_store_memory": 0, + "object_store_memory_per_worker": 0, "observation_filter": "NoFilter", "optimizer": {}, "output": null, @@ -117,7 +127,7 @@ "log_device_placement": false }, "timesteps_per_iteration": 0, - "train_batch_size": 60000, + "train_batch_size": 30000, "use_gae": true, "vf_clip_param": 10.0, "vf_loss_coeff": 1.0, diff --git a/tests/data/rllib_data/single_agent/params.pkl b/tests/data/rllib_data/single_agent/params.pkl index 511d34343..e69753b7f 100644 Binary files a/tests/data/rllib_data/single_agent/params.pkl and b/tests/data/rllib_data/single_agent/params.pkl differ diff --git a/tests/fast_tests/test_controllers.py b/tests/fast_tests/test_controllers.py index 76146dbe6..bef765396 100644 --- a/tests/fast_tests/test_controllers.py +++ b/tests/fast_tests/test_controllers.py @@ -8,7 +8,7 @@ from flow.controllers.routing_controllers import ContinuousRouter from flow.controllers.car_following_models import IDMController, \ OVMController, BCMController, LinearOVM, CFMController, LACController, \ - GippsController + GippsController, BandoFTLController from flow.controllers import FollowerStopper, PISaturation, NonLocalFollowerStopper from tests.setup_scripts import ring_road_exp_setup import os @@ -405,6 +405,175 @@ def test_no_crash_LinearOVM(self): self.tearDown_failsafe() +class TestFeasibleAccelFailsafe(TestInstantaneousFailsafe): + """ + Tests that the feasible accel failsafe of the base acceleration controller + does not fail under extreme conditions. + """ + + def test_no_crash_OVM(self): + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(OVMController, { + "fail_safe": "feasible_accel" + }), + routing_controller=(ContinuousRouter, {}), + num_vehicles=10, + ) + + self.setUp_failsafe(vehicles=vehicles) + + # run the experiment, see if it fails + self.exp.run(1) + + self.tearDown_failsafe() + + def test_no_crash_LinearOVM(self): + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(LinearOVM, { + "fail_safe": "feasible_accel" + }), + routing_controller=(ContinuousRouter, {}), + num_vehicles=10, + ) + + self.setUp_failsafe(vehicles=vehicles) + + # run the experiment, see if it fails + self.exp.run(1) + + self.tearDown_failsafe() + + +class TestObeySpeedLimitFailsafe(TestInstantaneousFailsafe): + """ + Tests that the obey speed limit failsafe of the base acceleration controller + does not fail under extreme conditions. + """ + + def test_no_crash_OVM(self): + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(OVMController, { + "fail_safe": "obey_speed_limit" + }), + routing_controller=(ContinuousRouter, {}), + num_vehicles=10, + ) + + self.setUp_failsafe(vehicles=vehicles) + + # run the experiment, see if it fails + self.exp.run(1) + + self.tearDown_failsafe() + + def test_no_crash_LinearOVM(self): + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(LinearOVM, { + "fail_safe": "obey_speed_limit" + }), + routing_controller=(ContinuousRouter, {}), + num_vehicles=10, + ) + + self.setUp_failsafe(vehicles=vehicles) + + # run the experiment, see if it fails + self.exp.run(1) + + self.tearDown_failsafe() + + +class TestBrokenFailsafe(TestInstantaneousFailsafe): + """ + Tests that the failsafe logic triggers exceptions when instantiated + incorrectly. + """ + + def test_invalid_failsafe_string(self): + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(OVMController, { + "fail_safe": "default" + }), + routing_controller=(ContinuousRouter, {}), + num_vehicles=10, + ) + + additional_env_params = { + "target_velocity": 8, + "max_accel": 3, + "max_decel": 3, + "sort_vehicles": False + } + env_params = EnvParams(additional_params=additional_env_params) + + additional_net_params = { + "length": 100, + "lanes": 1, + "speed_limit": 30, + "resolution": 40 + } + net_params = NetParams(additional_params=additional_net_params) + + initial_config = InitialConfig(bunching=10) + + # create the environment and network classes, see that it raises ValueError + with self.assertRaises(ValueError): + ring_road_exp_setup(vehicles=vehicles, + env_params=env_params, + net_params=net_params, + initial_config=initial_config) + + self.tearDown_failsafe() + + def test_invalid_failsafe_type(self): + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(LinearOVM, { + "fail_safe": True + }), + routing_controller=(ContinuousRouter, {}), + num_vehicles=10, + ) + + additional_env_params = { + "target_velocity": 8, + "max_accel": 3, + "max_decel": 3, + "sort_vehicles": False + } + env_params = EnvParams(additional_params=additional_env_params) + + additional_net_params = { + "length": 100, + "lanes": 1, + "speed_limit": 30, + "resolution": 40 + } + net_params = NetParams(additional_params=additional_net_params) + + initial_config = InitialConfig(bunching=10) + + # create the environment and network classes, see that it raises ValueError + with self.assertRaises(ValueError): + ring_road_exp_setup(vehicles=vehicles, + env_params=env_params, + net_params=net_params, + initial_config=initial_config) + + self.tearDown_failsafe() + + class TestStaticLaneChanger(unittest.TestCase): """ Makes sure that vehicles with a static lane-changing controller do not @@ -709,7 +878,7 @@ def test_get_action(self): np.testing.assert_array_almost_equal(requested_accel, expected_accel) -class TestGippsontroller(unittest.TestCase): +class TestGippsController(unittest.TestCase): """ Tests that the Gipps Controller returning mathematically accurate values. """ @@ -765,5 +934,59 @@ def test_get_action(self): np.testing.assert_array_almost_equal(requested_accel, expected_accel) +class TestBandoFTLController(unittest.TestCase): + """ + Tests that the Bando Controller returning mathematically accurate values. + """ + + def setUp(self): + # add a few vehicles to the network using the requested model + # also make sure that the input params are what is expected + contr_params = { + "alpha": .5, + "beta": 20, + "h_st": 2, + "h_go": 10, + "v_max": 32, + "want_max_accel": False, + } + + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(BandoFTLController, contr_params), + routing_controller=(ContinuousRouter, {}), + car_following_params=SumoCarFollowingParams( + accel=15, decel=5), + num_vehicles=5) + + # create the environment and network classes for a ring road + self.env, _, _ = ring_road_exp_setup(vehicles=vehicles) + + def tearDown(self): + # terminate the traci instance + self.env.terminate() + + # free data used by the class + self.env = None + + def test_get_action(self): + self.env.reset() + ids = self.env.k.vehicle.get_ids() + + test_headways = [2, 4, 6, 8, 10] + for i, veh_id in enumerate(ids): + self.env.k.vehicle.set_headway(veh_id, test_headways[i]) + + requested_accel = [ + self.env.k.vehicle.get_acc_controller(veh_id).get_action(self.env) + for veh_id in ids + ] + + expected_accel = [1.649129, 7.853475, 14.057821, 15.70695, 15.959713] + + np.testing.assert_array_almost_equal(requested_accel, expected_accel) + + if __name__ == '__main__': unittest.main() diff --git a/tests/fast_tests/test_environment_base_class.py b/tests/fast_tests/test_environment_base_class.py index b5c6cbc17..ee815393c 100644 --- a/tests/fast_tests/test_environment_base_class.py +++ b/tests/fast_tests/test_environment_base_class.py @@ -13,8 +13,9 @@ from tests.setup_scripts import ring_road_exp_setup, highway_exp_setup import os -import numpy as np import gym.spaces as spaces +from gym.spaces.box import Box +import numpy as np os.environ["TEST_FLAG"] = "True" @@ -25,6 +26,41 @@ YELLOW = (255, 255, 0) +class TestFailRLActionsEnv(Env): + """Test environment designed to fail _apply_rl_actions not-implemented test.""" + + @property + def action_space(self): + """See parent class.""" + return Box(low=0, high=0, shape=(0,), dtype=np.float32) # pragma: no cover + + @property + def observation_space(self): + """See parent class.""" + return Box(low=0, high=0, shape=(0,), dtype=np.float32) # pragma: no cover + + def get_state(self, **kwargs): + """See class definition.""" + return np.array([]) # pragma: no cover + + +class TestFailGetStateEnv(Env): + """Test environment designed to fail get_state not-implemented test.""" + + @property + def action_space(self): + """See parent class.""" + return Box(low=0, high=0, shape=(0,), dtype=np.float32) # pragma: no cover + + @property + def observation_space(self): + """See parent class.""" + return Box(low=0, high=0, shape=(0,), dtype=np.float32) # pragma: no cover + + def _apply_rl_actions(self, rl_actions): + return # pragma: no cover + + class TestShuffle(unittest.TestCase): """ Tests that, at resets, the ordering of vehicles changes while the starting @@ -311,28 +347,34 @@ class TestAbstractMethods(unittest.TestCase): """ def setUp(self): - env, network, _ = ring_road_exp_setup() - sim_params = SumoParams() # FIXME: make ambiguous - env_params = EnvParams() - self.env = Env(sim_params=sim_params, - env_params=env_params, - network=network) + self.env, self.network, _ = ring_road_exp_setup() + self.sim_params = SumoParams() # FIXME: make ambiguous + self.env_params = EnvParams() - def tearDown(self): - self.env.terminate() - self.env = None + def test_abstract_base_class(self): + """Checks that instantiating abstract base class raises an error.""" + with self.assertRaises(TypeError): + Env(sim_params=self.sim_params, + env_params=self.env_params, + network=self.network) def test_get_state(self): - """Checks that get_state raises an error.""" - self.assertRaises(NotImplementedError, self.env.get_state) - - def test_compute_reward(self): - """Checks that compute_reward returns 0.""" - self.assertEqual(self.env.compute_reward([]), 0) + """Checks that instantiating without get_state implemented + raises an error. + """ + with self.assertRaises(TypeError): + TestFailGetStateEnv(sim_params=self.sim_params, + env_params=self.env_params, + network=self.network) def test__apply_rl_actions(self): - self.assertRaises(NotImplementedError, self.env._apply_rl_actions, - rl_actions=None) + """Checks that instantiating without _apply_rl_actions + implemented raises an error. + """ + with self.assertRaises(TypeError): + TestFailRLActionsEnv(sim_params=self.sim_params, + env_params=self.env_params, + network=self.network) class TestVehicleColoring(unittest.TestCase): diff --git a/tests/fast_tests/test_examples.py b/tests/fast_tests/test_examples.py index 0f0a1492c..6e5a6b132 100644 --- a/tests/fast_tests/test_examples.py +++ b/tests/fast_tests/test_examples.py @@ -26,8 +26,11 @@ flow_params as multiagent_traffic_light_grid from examples.exp_configs.rl.multiagent.multiagent_highway import flow_params as multiagent_highway +from examples.simulate import parse_args as parse_simulate_args +from examples.train import parse_args as parse_train_args from examples.train import run_model_stablebaseline as run_stable_baselines_model from examples.train import setup_exps_rllib as setup_rllib_exps +from examples.train import train_h_baselines from examples.exp_configs.non_rl.bay_bridge import flow_params as non_rl_bay_bridge from examples.exp_configs.non_rl.bay_bridge_toll import flow_params as non_rl_bay_bridge_toll @@ -40,6 +43,7 @@ from examples.exp_configs.non_rl.minicity import flow_params as non_rl_minicity from examples.exp_configs.non_rl.ring import flow_params as non_rl_ring from examples.exp_configs.non_rl.i210_subnetwork import flow_params as non_rl_i210 +from examples.exp_configs.non_rl.highway_single import flow_params as non_rl_highway_single os.environ['TEST_FLAG'] = 'True' os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' @@ -56,6 +60,40 @@ class TestNonRLExamples(unittest.TestCase): done to the functions within the experiment class. """ + def test_parse_args(self): + """Validate the functionality of the parse_args method in simulate.py.""" + # test the default case + args = parse_simulate_args(["exp_config"]) + + self.assertDictEqual(vars(args), { + 'aimsun': False, + 'exp_config': 'exp_config', + 'gen_emission': False, + 'is_baseline': False, + 'no_render': False, + 'num_runs': 1, + 'to_aws': None, + }) + + # test the case when optional args are specified + args = parse_simulate_args([ + "exp_config", + '--aimsun', + '--gen_emission', + '--no_render', + '--num_runs', '2' + ]) + + self.assertDictEqual(vars(args), { + 'aimsun': True, + 'exp_config': 'exp_config', + 'gen_emission': True, + 'is_baseline': False, + 'no_render': True, + 'num_runs': 2, + 'to_aws': None, + }) + def test_bottleneck(self): """Verify that examples/exp_configs/non_rl/bottleneck.py is working.""" self.run_simulation(non_rl_bottleneck) @@ -108,11 +146,18 @@ def test_i210(self): """Verify that examples/exp_configs/non_rl/i210_subnetwork.py is working.""" self.run_simulation(non_rl_i210) + def test_highway_single(self): + """Verify that examples/exp_configs/non_rl/highway_single.py is working.""" + self.run_simulation(non_rl_highway_single) + @staticmethod def run_simulation(flow_params): + flow_params = deepcopy(flow_params) + # make the horizon small and set render to False flow_params['sim'].render = False flow_params['env'].horizon = 5 + flow_params['env'].warmup_steps = 0 # create an experiment object exp = Experiment(flow_params) @@ -192,6 +237,65 @@ def run_simulation_libsumo(flow_params): exp.run(1) +class TestTrain(unittest.TestCase): + + def test_parse_args(self): + """Tests the parse_args method in train.py.""" + # test the default case + args = parse_train_args(["exp_config"]) + + self.assertDictEqual(vars(args), { + 'algorithm': 'PPO', + 'checkpoint_freq': 20, + 'exp_config': 'exp_config', + 'exp_title': None, + 'grid_search': False, + 'local_mode': False, + 'rl_trainer': 'rllib', + 'num_cpus': 1, + 'num_iterations': 200, + 'num_rollouts': 1, + 'num_steps': 5000, + 'render': False, + 'rollout_size': 1000, + 'checkpoint_path': None, + 'use_s3': False, + 'multi_node': False, + 'upload_graphs': None + }) + + # test the case when optional args are specified + args = parse_train_args([ + "exp_config", + "--rl_trainer", "h-baselines", + "--num_cpus" "2", + "--num_steps", "3", + "--rollout_size", "4", + "--checkpoint_path", "5", + "--upload_graphs", "name", "strategy" + ]) + + self.assertDictEqual(vars(args), { + 'algorithm': 'PPO', + 'checkpoint_freq': 20, + 'checkpoint_path': '5', + 'exp_config': 'exp_config', + 'exp_title': None, + 'grid_search': False, + 'local_mode': False, + 'num_cpus': 1, + 'num_iterations': 200, + 'num_rollouts': 1, + 'num_steps': 3, + 'render': False, + 'rl_trainer': 'h-baselines', + 'rollout_size': 4, + 'use_s3': False, + 'multi_node': False, + 'upload_graphs': ['name', 'strategy'] + }) + + class TestStableBaselineExamples(unittest.TestCase): """Tests the example scripts in examples/exp_configs/rl/singleagent for stable_baselines. @@ -200,6 +304,11 @@ class TestStableBaselineExamples(unittest.TestCase): """ @staticmethod def run_exp(flow_params): + # Reduce the number of warmup steps to speedup tests. + flow_params = deepcopy(flow_params) + flow_params['env'].warmup_steps = 0 + + # Run the example. train_model = run_stable_baselines_model(flow_params, 1, 4, 4) train_model.env.close() @@ -219,6 +328,31 @@ def test_singleagent_bottleneck(self): self.run_exp(singleagent_bottleneck) +class TestHBaselineExamples(unittest.TestCase): + """Tests the functionality of the h-baselines features in train.py. + + This is done by running a set of experiments for 10 time-steps and + confirming that it runs. + """ + @staticmethod + def run_exp(env_name, multiagent): + train_h_baselines( + env_name=env_name, + args=[ + env_name, + "--initial_exploration_steps", "1", + "--total_steps", "10" + ], + multiagent=multiagent, + ) + + def test_singleagent_ring(self): + self.run_exp("singleagent_ring", multiagent=False) + + def test_multiagent_ring(self): + self.run_exp("multiagent_ring", multiagent=True) + + class TestRllibExamples(unittest.TestCase): """Tests the example scripts in examples/exp_configs/rl/singleagent and examples/exp_configs/rl/multiagent for RLlib. @@ -330,7 +464,7 @@ def test_multiagent_i210(self): from examples.exp_configs.rl.multiagent.multiagent_i210 import POLICIES_TO_TRAIN as mi210pr from examples.exp_configs.rl.multiagent.multiagent_i210 import policy_mapping_fn as mi210mf - from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy + from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicy from ray.tune.registry import register_env from flow.utils.registry import make_create_env # test observation space 1 @@ -379,10 +513,16 @@ def test_multiagent_i210(self): @staticmethod def run_exp(flow_params, **kwargs): - alg_run, env_name, config = setup_rllib_exps(flow_params, 1, 1, **kwargs) + # Reduce the number of warmup steps to speedup tests. + flow_params = deepcopy(flow_params) + flow_params['env'].warmup_steps = 0 + + # Run the example. + alg_run, env_name, config = setup_rllib_exps( + flow_params, 1, 1, parse_train_args([""]), **kwargs) try: - ray.init(num_cpus=1) + ray.init(num_cpus=1, local_mode=True) except Exception as e: print("ERROR", e) config['train_batch_size'] = 50 diff --git a/tests/fast_tests/test_experiment_base_class.py b/tests/fast_tests/test_experiment_base_class.py index b3863a77c..8a7a9500c 100644 --- a/tests/fast_tests/test_experiment_base_class.py +++ b/tests/fast_tests/test_experiment_base_class.py @@ -1,6 +1,7 @@ import unittest import os import time +import csv from flow.core.experiment import Experiment from flow.core.params import VehicleParams @@ -168,15 +169,44 @@ def test_convert_to_csv(self): time.sleep(1.0) # check that both the csv file exists and the xml file doesn't. - self.assertFalse(os.path.isfile(dir_path + "/{}-emission.xml".format( + self.assertFalse(os.path.isfile(dir_path + "/{}-0_emission.xml".format( exp.env.network.name))) - self.assertTrue(os.path.isfile(dir_path + "/{}-emission.csv".format( + self.assertTrue(os.path.isfile(dir_path + "/{}-0_emission.csv".format( exp.env.network.name))) + # check that the keys within the emission file matches its expected + # values + with open(dir_path + "/{}-0_emission.csv".format( + exp.env.network.name), "r") as f: + reader = csv.reader(f) + header = next(reader) + + self.assertListEqual(header, [ + "time", + "id", + "x", + "y", + "speed", + "headway", + "leader_id", + "follower_id", + "leader_rel_speed", + "target_accel_with_noise_with_failsafe", + "target_accel_no_noise_no_failsafe", + "target_accel_with_noise_no_failsafe", + "target_accel_no_noise_with_failsafe", + "realized_accel", + "road_grade", + "edge_id", + "lane_number", + "distance", + "relative_position", + ]) + time.sleep(0.1) # delete the files - os.remove(os.path.expanduser(dir_path + "/{}-emission.csv".format( + os.remove(os.path.expanduser(dir_path + "/{}-0_emission.csv".format( exp.env.network.name))) diff --git a/tests/fast_tests/test_files/i210_emission.csv b/tests/fast_tests/test_files/i210_emission.csv index d43c115a4..ec63cf9cf 100644 --- a/tests/fast_tests/test_files/i210_emission.csv +++ b/tests/fast_tests/test_files/i210_emission.csv @@ -1,4 +1,4 @@ -x,time,edge_id,eclass,type,PMx,speed,angle,CO,CO2,electricity,noise,lane_number,NOx,relative_position,route,y,id,fuel,HC,waiting +x,time,edge_id,eclass,type,PMx,speed,angle,CO,CO2,electricity,noise,lane_number,NOx,distance,route,y,id,fuel,HC,waiting 485.04,0.8,119257914,HBEFA3/PC_G_EU4,human,0.05,23.0,119.74,3.32,3793.12,0.0,70.29,1,1.17,5.1,route119257914_0,1068.18,flow_00.0,1.63,0.11,0.0 500.91,1.6,119257914,HBEFA3/PC_G_EU4,human,0.0,22.84,119.74,0.0,0.0,0.0,69.9,1,0.0,23.37,route119257914_0,1059.12,flow_00.0,0.0,0.0,0.0 517.1,2.4,119257914,HBEFA3/PC_G_EU4,human,0.15,23.31,119.74,78.83,7435.5,0.0,71.61,1,2.88,42.02,route119257914_0,1049.87,flow_00.0,3.2,0.54,0.0 diff --git a/tests/fast_tests/test_files/ring_230_emission.csv b/tests/fast_tests/test_files/ring_230_emission.csv index 9051074c8..342c5c7f3 100644 --- a/tests/fast_tests/test_files/ring_230_emission.csv +++ b/tests/fast_tests/test_files/ring_230_emission.csv @@ -1,117 +1,25 @@ -speed,CO,electricity,x,NOx,id,fuel,angle,time,edge_id,eclass,route,waiting,CO2,lane_number,PMx,type,noise,relative_position,HC,y -0.0,164.78,0.0,36.64,1.2,idm_0,1.13,94.02,0.1,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2624.72,0,0.07,idm,55.94,0.0,0.81,-1.65 -0.08,163.5,0.0,36.65,1.21,idm_0,1.13,94.01,0.2,bottom,HBEFA3/PC_G_EU4,routebottom,0.1,2631.03,0,0.07,idm,59.48,0.01,0.81,-1.65 -0.16,162.24,0.0,36.66,1.21,idm_0,1.13,93.98,0.3,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2636.67,0,0.07,idm,59.44,0.02,0.8,-1.65 -0.23,161.0,0.0,36.69,1.21,idm_0,1.14,93.94,0.4,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2641.63,0,0.07,idm,59.4,0.05,0.79,-1.65 -0.31,159.78,0.0,36.72,1.21,idm_0,1.14,93.88,0.5,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2645.91,0,0.06,idm,59.36,0.08,0.79,-1.65 -0.41,158.73,0.0,36.76,1.22,idm_0,1.15,93.8,0.6,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2679.14,0,0.07,idm,60.47,0.12,0.79,-1.65 -0.0,164.78,0.0,46.49,1.2,idm_1,1.13,78.81,0.1,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2624.72,0,0.07,idm,55.94,9.55,0.81,-0.34 -0.08,163.5,0.0,46.5,1.21,idm_1,1.13,78.8,0.2,bottom,HBEFA3/PC_G_EU4,routebottom,0.1,2631.03,0,0.07,idm,59.48,9.55,0.81,-0.33 -0.16,162.24,0.0,46.51,1.21,idm_1,1.13,78.78,0.3,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2636.67,0,0.07,idm,59.44,9.57,0.8,-0.33 -0.23,161.0,0.0,46.54,1.21,idm_1,1.14,78.74,0.4,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2641.63,0,0.07,idm,59.4,9.59,0.79,-0.32 -0.31,159.78,0.0,46.57,1.21,idm_1,1.14,78.7,0.5,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2645.91,0,0.06,idm,59.36,9.62,0.79,-0.31 -0.41,158.73,0.0,46.61,1.22,idm_1,1.15,78.64,0.6,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2679.14,0,0.07,idm,60.47,9.66,0.79,-0.3 -0.0,164.78,0.0,56.08,1.2,idm_10,1.13,304.55,0.1,right,HBEFA3/PC_G_EU4,routeright,0.0,2624.72,0,0.07,idm,55.94,37.95,0.81,69.53 -0.08,163.5,0.0,56.08,1.21,idm_10,1.13,304.54,0.2,right,HBEFA3/PC_G_EU4,routeright,0.1,2631.03,0,0.07,idm,59.48,37.96,0.81,69.53 -0.16,162.24,0.0,56.06,1.21,idm_10,1.13,304.52,0.3,right,HBEFA3/PC_G_EU4,routeright,0.0,2636.67,0,0.07,idm,59.44,37.98,0.8,69.54 -0.23,161.0,0.0,56.04,1.21,idm_10,1.14,304.48,0.4,right,HBEFA3/PC_G_EU4,routeright,0.0,2641.63,0,0.07,idm,59.4,38.0,0.79,69.55 -0.31,159.78,0.0,56.01,1.21,idm_10,1.14,304.44,0.5,right,HBEFA3/PC_G_EU4,routeright,0.0,2645.91,0,0.06,idm,59.36,38.03,0.79,69.57 -0.41,158.73,0.0,55.98,1.22,idm_10,1.15,304.38,0.6,right,HBEFA3/PC_G_EU4,routeright,0.0,2679.14,0,0.07,idm,60.47,38.07,0.79,69.59 -0.0,164.78,0.0,46.95,1.2,idm_11,1.13,289.47,0.1,right,HBEFA3/PC_G_EU4,routeright,0.0,2624.72,0,0.07,idm,55.94,47.5,0.81,73.43 -0.08,163.5,0.0,46.94,1.21,idm_11,1.13,289.45,0.2,right,HBEFA3/PC_G_EU4,routeright,0.1,2631.03,0,0.07,idm,59.48,47.51,0.81,73.43 -0.16,162.24,0.0,46.92,1.21,idm_11,1.13,289.42,0.3,right,HBEFA3/PC_G_EU4,routeright,0.0,2636.67,0,0.07,idm,59.44,47.52,0.8,73.44 -0.23,161.0,0.0,46.9,1.21,idm_11,1.14,289.38,0.4,right,HBEFA3/PC_G_EU4,routeright,0.0,2641.63,0,0.07,idm,59.4,47.55,0.79,73.44 -0.31,159.78,0.0,46.87,1.21,idm_11,1.14,289.32,0.5,right,HBEFA3/PC_G_EU4,routeright,0.0,2645.91,0,0.06,idm,59.36,47.58,0.79,73.45 -0.41,158.73,0.0,46.83,1.22,idm_11,1.15,289.24,0.6,right,HBEFA3/PC_G_EU4,routeright,0.0,2679.14,0,0.07,idm,60.47,47.62,0.79,73.46 -0.0,164.78,0.0,37.11,1.2,idm_12,1.13,274.71,0.1,right,HBEFA3/PC_G_EU4,routeright,0.0,2624.72,0,0.07,idm,55.94,57.05,0.81,74.86 -0.08,163.5,0.0,37.11,1.21,idm_12,1.13,274.7,0.2,right,HBEFA3/PC_G_EU4,routeright,0.1,2631.03,0,0.07,idm,59.48,57.05,0.81,74.86 -0.16,162.24,0.0,37.09,1.21,idm_12,1.13,274.68,0.3,right,HBEFA3/PC_G_EU4,routeright,0.0,2636.67,0,0.07,idm,59.44,57.07,0.8,74.86 -0.23,161.0,0.0,37.07,1.21,idm_12,1.14,274.65,0.4,right,HBEFA3/PC_G_EU4,routeright,0.0,2641.63,0,0.07,idm,59.4,57.09,0.79,74.86 -0.31,159.78,0.0,37.03,1.21,idm_12,1.14,274.6,0.5,right,HBEFA3/PC_G_EU4,routeright,0.0,2645.91,0,0.06,idm,59.36,57.12,0.79,74.86 -0.41,158.73,0.0,36.99,1.22,idm_12,1.15,274.55,0.6,right,HBEFA3/PC_G_EU4,routeright,0.0,2679.14,0,0.07,idm,60.47,57.16,0.79,74.86 -0.0,164.78,0.0,27.19,1.2,idm_13,1.13,259.6,0.1,top,HBEFA3/PC_G_EU4,routetop,0.0,2624.72,0,0.07,idm,55.94,9.09,0.81,73.68 -0.08,163.5,0.0,27.18,1.21,idm_13,1.13,259.58,0.2,top,HBEFA3/PC_G_EU4,routetop,0.1,2631.03,0,0.07,idm,59.48,9.1,0.81,73.68 -0.16,162.24,0.0,27.17,1.21,idm_13,1.13,259.55,0.3,top,HBEFA3/PC_G_EU4,routetop,0.0,2636.67,0,0.07,idm,59.44,9.11,0.8,73.67 -0.23,161.0,0.0,27.14,1.21,idm_13,1.14,259.51,0.4,top,HBEFA3/PC_G_EU4,routetop,0.0,2641.63,0,0.07,idm,59.4,9.14,0.79,73.67 -0.31,159.78,0.0,27.11,1.21,idm_13,1.14,259.45,0.5,top,HBEFA3/PC_G_EU4,routetop,0.0,2645.91,0,0.06,idm,59.36,9.17,0.79,73.66 -0.41,158.73,0.0,27.07,1.22,idm_13,1.15,259.37,0.6,top,HBEFA3/PC_G_EU4,routetop,0.0,2679.14,0,0.07,idm,60.47,9.21,0.79,73.65 -0.0,164.78,0.0,17.96,1.2,idm_14,1.13,244.67,0.1,top,HBEFA3/PC_G_EU4,routetop,0.0,2624.72,0,0.07,idm,55.94,18.64,0.81,70.0 -0.08,163.5,0.0,17.95,1.21,idm_14,1.13,244.66,0.2,top,HBEFA3/PC_G_EU4,routetop,0.1,2631.03,0,0.07,idm,59.48,18.64,0.81,70.0 -0.16,162.24,0.0,17.94,1.21,idm_14,1.13,244.63,0.3,top,HBEFA3/PC_G_EU4,routetop,0.0,2636.67,0,0.07,idm,59.44,18.66,0.8,69.99 -0.23,161.0,0.0,17.92,1.21,idm_14,1.14,244.6,0.4,top,HBEFA3/PC_G_EU4,routetop,0.0,2641.63,0,0.07,idm,59.4,18.68,0.79,69.98 -0.31,159.78,0.0,17.89,1.21,idm_14,1.14,244.55,0.5,top,HBEFA3/PC_G_EU4,routetop,0.0,2645.91,0,0.06,idm,59.36,18.71,0.79,69.96 -0.0,164.78,0.0,9.98,1.2,idm_15,1.13,229.84,0.1,top,HBEFA3/PC_G_EU4,routetop,0.0,2624.72,0,0.07,idm,55.94,28.18,0.81,64.07 -0.08,163.5,0.0,9.98,1.21,idm_15,1.13,229.83,0.2,top,HBEFA3/PC_G_EU4,routetop,0.1,2631.03,0,0.07,idm,59.48,28.19,0.81,64.07 -0.16,162.24,0.0,9.97,1.21,idm_15,1.13,229.8,0.3,top,HBEFA3/PC_G_EU4,routetop,0.0,2636.67,0,0.07,idm,59.44,28.21,0.8,64.06 -0.23,161.0,0.0,9.95,1.21,idm_15,1.14,229.76,0.4,top,HBEFA3/PC_G_EU4,routetop,0.0,2641.63,0,0.07,idm,59.4,28.23,0.79,64.04 -0.31,159.78,0.0,9.93,1.21,idm_15,1.14,229.7,0.5,top,HBEFA3/PC_G_EU4,routetop,0.0,2645.91,0,0.06,idm,59.36,28.26,0.79,64.02 -0.0,164.78,0.0,3.81,1.2,idm_16,1.13,214.88,0.1,top,HBEFA3/PC_G_EU4,routetop,0.0,2624.72,0,0.07,idm,55.94,37.73,0.81,56.29 -0.08,163.5,0.0,3.81,1.21,idm_16,1.13,214.87,0.2,top,HBEFA3/PC_G_EU4,routetop,0.1,2631.03,0,0.07,idm,59.48,37.74,0.81,56.28 -0.16,162.24,0.0,3.8,1.21,idm_16,1.13,214.85,0.3,top,HBEFA3/PC_G_EU4,routetop,0.0,2636.67,0,0.07,idm,59.44,37.75,0.8,56.27 -0.23,161.0,0.0,3.79,1.21,idm_16,1.14,214.81,0.4,top,HBEFA3/PC_G_EU4,routetop,0.0,2641.63,0,0.07,idm,59.4,37.77,0.79,56.24 -0.31,159.78,0.0,3.77,1.21,idm_16,1.14,214.77,0.5,top,HBEFA3/PC_G_EU4,routetop,0.0,2645.91,0,0.06,idm,59.36,37.81,0.79,56.22 -0.0,164.78,0.0,-0.15,1.2,idm_17,1.13,199.9,0.1,top,HBEFA3/PC_G_EU4,routetop,0.0,2624.72,0,0.07,idm,55.94,47.27,0.81,47.18 -0.08,163.5,0.0,-0.15,1.21,idm_17,1.13,199.88,0.2,top,HBEFA3/PC_G_EU4,routetop,0.1,2631.03,0,0.07,idm,59.48,47.28,0.81,47.17 -0.16,162.24,0.0,-0.16,1.21,idm_17,1.13,199.85,0.3,top,HBEFA3/PC_G_EU4,routetop,0.0,2636.67,0,0.07,idm,59.44,47.3,0.8,47.15 -0.23,161.0,0.0,-0.16,1.21,idm_17,1.14,199.81,0.4,top,HBEFA3/PC_G_EU4,routetop,0.0,2641.63,0,0.07,idm,59.4,47.32,0.79,47.13 -0.31,159.78,0.0,-0.17,1.21,idm_17,1.14,199.75,0.5,top,HBEFA3/PC_G_EU4,routetop,0.0,2645.91,0,0.06,idm,59.36,47.35,0.79,47.1 -0.0,164.78,0.0,-1.64,1.2,idm_18,1.13,185.04,0.1,top,HBEFA3/PC_G_EU4,routetop,0.0,2624.72,0,0.07,idm,55.94,56.82,0.81,37.35 -0.08,163.5,0.0,-1.64,1.21,idm_18,1.13,185.03,0.2,top,HBEFA3/PC_G_EU4,routetop,0.1,2631.03,0,0.07,idm,59.48,56.83,0.81,37.34 -0.16,162.24,0.0,-1.64,1.21,idm_18,1.13,185.0,0.3,top,HBEFA3/PC_G_EU4,routetop,0.0,2636.67,0,0.07,idm,59.44,56.84,0.8,37.33 -0.23,161.0,0.0,-1.64,1.21,idm_18,1.14,184.97,0.4,top,HBEFA3/PC_G_EU4,routetop,0.0,2641.63,0,0.07,idm,59.4,56.87,0.79,37.3 -0.31,159.78,0.0,-1.64,1.21,idm_18,1.14,184.93,0.5,top,HBEFA3/PC_G_EU4,routetop,0.0,2645.91,0,0.06,idm,59.36,56.9,0.79,37.27 -0.0,164.78,0.0,-0.52,1.2,idm_19,1.13,170.03,0.1,left,HBEFA3/PC_G_EU4,routeleft,0.0,2624.72,0,0.07,idm,55.94,8.86,0.81,27.42 -0.08,163.5,0.0,-0.52,1.21,idm_19,1.13,170.01,0.2,left,HBEFA3/PC_G_EU4,routeleft,0.1,2631.03,0,0.07,idm,59.48,8.87,0.81,27.41 -0.16,162.24,0.0,-0.51,1.21,idm_19,1.13,169.98,0.3,left,HBEFA3/PC_G_EU4,routeleft,0.0,2636.67,0,0.07,idm,59.44,8.89,0.8,27.39 -0.23,161.0,0.0,-0.51,1.21,idm_19,1.14,169.94,0.4,left,HBEFA3/PC_G_EU4,routeleft,0.0,2641.63,0,0.07,idm,59.4,8.91,0.79,27.37 -0.31,159.78,0.0,-0.5,1.21,idm_19,1.14,169.88,0.5,left,HBEFA3/PC_G_EU4,routeleft,0.0,2645.91,0,0.06,idm,59.36,8.94,0.79,27.34 -0.0,164.78,0.0,55.68,1.2,idm_2,1.13,64.0,0.1,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2624.72,0,0.07,idm,55.94,19.09,0.81,3.45 -0.08,163.5,0.0,55.68,1.21,idm_2,1.13,63.99,0.2,bottom,HBEFA3/PC_G_EU4,routebottom,0.1,2631.03,0,0.07,idm,59.48,19.1,0.81,3.45 -0.16,162.24,0.0,55.7,1.21,idm_2,1.13,63.97,0.3,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2636.67,0,0.07,idm,59.44,19.11,0.8,3.46 -0.23,161.0,0.0,55.72,1.21,idm_2,1.14,63.93,0.4,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2641.63,0,0.07,idm,59.4,19.14,0.79,3.47 -0.31,159.78,0.0,55.75,1.21,idm_2,1.14,63.88,0.5,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2645.91,0,0.06,idm,59.36,19.17,0.79,3.49 -0.0,164.78,0.0,3.11,1.2,idm_20,1.13,155.0,0.1,left,HBEFA3/PC_G_EU4,routeleft,0.0,2624.72,0,0.07,idm,55.94,18.41,0.81,18.17 -0.08,163.5,0.0,3.11,1.21,idm_20,1.13,154.99,0.2,left,HBEFA3/PC_G_EU4,routeleft,0.1,2631.03,0,0.07,idm,59.48,18.42,0.81,18.16 -0.16,162.24,0.0,3.12,1.21,idm_20,1.13,154.96,0.3,left,HBEFA3/PC_G_EU4,routeleft,0.0,2636.68,0,0.07,idm,59.44,18.43,0.8,18.15 -0.23,161.0,0.0,3.13,1.21,idm_20,1.14,154.93,0.4,left,HBEFA3/PC_G_EU4,routeleft,0.0,2641.7,0,0.07,idm,59.41,18.46,0.79,18.12 -0.31,159.77,0.0,3.15,1.21,idm_20,1.14,154.89,0.5,left,HBEFA3/PC_G_EU4,routeleft,0.0,2646.14,0,0.06,idm,59.37,18.49,0.79,18.1 -0.0,164.78,0.0,8.98,1.2,idm_21,1.13,140.22,0.1,left,HBEFA3/PC_G_EU4,routeleft,0.0,2624.72,0,0.07,idm,55.94,27.95,0.81,10.15 -0.1,163.3,0.0,8.99,1.21,idm_21,1.13,140.21,0.2,left,HBEFA3/PC_G_EU4,routeleft,0.1,2637.25,0,0.07,idm,60.3,27.96,0.81,10.15 -0.2,161.84,0.0,9.0,1.21,idm_21,1.14,140.18,0.3,left,HBEFA3/PC_G_EU4,routeleft,0.0,2649.89,0,0.07,idm,60.34,27.98,0.8,10.13 -0.29,160.38,0.0,9.02,1.21,idm_21,1.14,140.14,0.4,left,HBEFA3/PC_G_EU4,routeleft,0.0,2662.63,0,0.07,idm,60.37,28.01,0.79,10.11 -0.39,158.94,0.0,9.05,1.22,idm_21,1.15,140.07,0.5,left,HBEFA3/PC_G_EU4,routeleft,0.0,2675.48,0,0.07,idm,60.41,28.05,0.79,10.08 -0.0,164.78,0.0,63.57,1.2,idm_3,1.13,49.05,0.1,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2624.72,0,0.07,idm,55.94,28.64,0.81,9.48 -0.08,163.5,0.0,63.58,1.21,idm_3,1.13,49.04,0.2,bottom,HBEFA3/PC_G_EU4,routebottom,0.1,2631.03,0,0.07,idm,59.48,28.64,0.81,9.49 -0.16,162.24,0.0,63.59,1.21,idm_3,1.13,49.02,0.3,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2636.67,0,0.07,idm,59.44,28.66,0.8,9.5 -0.23,161.0,0.0,63.61,1.21,idm_3,1.14,48.99,0.4,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2641.63,0,0.07,idm,59.4,28.68,0.79,9.52 -0.31,159.78,0.0,63.63,1.21,idm_3,1.14,48.94,0.5,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2645.91,0,0.06,idm,59.36,28.71,0.79,9.54 -0.0,164.78,0.0,69.65,1.2,idm_4,1.13,34.22,0.1,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2624.72,0,0.07,idm,55.94,38.18,0.81,17.34 -0.08,163.5,0.0,69.65,1.21,idm_4,1.13,34.21,0.2,bottom,HBEFA3/PC_G_EU4,routebottom,0.1,2631.03,0,0.07,idm,59.48,38.19,0.81,17.35 -0.16,162.24,0.0,69.66,1.21,idm_4,1.13,34.19,0.3,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2636.67,0,0.07,idm,59.44,38.21,0.8,17.36 -0.23,161.0,0.0,69.68,1.21,idm_4,1.14,34.15,0.4,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2641.63,0,0.07,idm,59.4,38.23,0.79,17.38 -0.31,159.78,0.0,69.69,1.21,idm_4,1.14,34.11,0.5,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2645.91,0,0.06,idm,59.36,38.26,0.79,17.41 -0.0,164.78,0.0,73.49,1.2,idm_5,1.13,19.04,0.1,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2624.72,0,0.07,idm,55.94,47.73,0.81,26.5 -0.08,163.5,0.0,73.5,1.21,idm_5,1.13,19.02,0.2,bottom,HBEFA3/PC_G_EU4,routebottom,0.1,2631.03,0,0.07,idm,59.48,47.74,0.81,26.51 -0.16,162.24,0.0,73.5,1.21,idm_5,1.13,18.99,0.3,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2636.67,0,0.07,idm,59.44,47.75,0.8,26.53 -0.23,161.0,0.0,73.51,1.21,idm_5,1.14,18.95,0.4,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2641.63,0,0.07,idm,59.4,47.77,0.79,26.55 -0.31,159.78,0.0,73.52,1.21,idm_5,1.14,18.91,0.5,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2645.91,0,0.06,idm,59.36,47.81,0.79,26.58 -0.0,164.78,0.0,74.87,1.2,idm_6,1.13,4.39,0.1,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2624.72,0,0.07,idm,55.94,57.27,0.81,36.34 -0.08,163.5,0.0,74.87,1.21,idm_6,1.13,4.38,0.2,bottom,HBEFA3/PC_G_EU4,routebottom,0.1,2631.03,0,0.07,idm,59.48,57.28,0.81,36.35 -0.16,162.24,0.0,74.87,1.21,idm_6,1.13,4.36,0.3,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2636.67,0,0.07,idm,59.44,57.3,0.8,36.37 -0.23,161.0,0.0,74.87,1.21,idm_6,1.14,4.32,0.4,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2641.63,0,0.07,idm,59.4,57.32,0.79,36.39 -0.31,159.78,0.0,74.87,1.21,idm_6,1.14,4.28,0.5,bottom,HBEFA3/PC_G_EU4,routebottom,0.0,2645.91,0,0.06,idm,59.36,57.35,0.79,36.42 -0.0,164.78,0.0,73.62,1.2,idm_7,1.13,349.16,0.1,right,HBEFA3/PC_G_EU4,routeright,0.0,2624.72,0,0.07,idm,55.94,9.32,0.81,46.26 -0.08,163.5,0.0,73.62,1.21,idm_7,1.13,349.15,0.2,right,HBEFA3/PC_G_EU4,routeright,0.1,2631.03,0,0.07,idm,59.48,9.33,0.81,46.27 -0.16,162.24,0.0,73.61,1.21,idm_7,1.13,349.12,0.3,right,HBEFA3/PC_G_EU4,routeright,0.0,2636.67,0,0.07,idm,59.44,9.34,0.8,46.28 -0.23,161.0,0.0,73.6,1.21,idm_7,1.14,349.07,0.4,right,HBEFA3/PC_G_EU4,routeright,0.0,2641.63,0,0.07,idm,59.4,9.37,0.79,46.31 -0.31,159.78,0.0,73.6,1.21,idm_7,1.14,349.01,0.5,right,HBEFA3/PC_G_EU4,routeright,0.0,2645.91,0,0.06,idm,59.36,9.4,0.79,46.34 -0.0,164.78,0.0,69.89,1.2,idm_8,1.13,334.33,0.1,right,HBEFA3/PC_G_EU4,routeright,0.0,2624.72,0,0.07,idm,55.94,18.86,0.81,55.47 -0.08,163.5,0.0,69.88,1.21,idm_8,1.13,334.32,0.2,right,HBEFA3/PC_G_EU4,routeright,0.1,2631.03,0,0.07,idm,59.48,18.87,0.81,55.47 -0.16,162.24,0.0,69.87,1.21,idm_8,1.13,334.3,0.3,right,HBEFA3/PC_G_EU4,routeright,0.0,2636.67,0,0.07,idm,59.44,18.89,0.8,55.49 -0.23,161.0,0.0,69.86,1.21,idm_8,1.14,334.27,0.4,right,HBEFA3/PC_G_EU4,routeright,0.0,2641.63,0,0.07,idm,59.4,18.91,0.79,55.51 -0.31,159.78,0.0,69.85,1.21,idm_8,1.14,334.22,0.5,right,HBEFA3/PC_G_EU4,routeright,0.0,2645.91,0,0.06,idm,59.36,18.94,0.79,55.54 -0.0,164.78,0.0,63.91,1.2,idm_9,1.13,319.44,0.1,right,HBEFA3/PC_G_EU4,routeright,0.0,2624.72,0,0.07,idm,55.94,28.41,0.81,63.4 -0.08,163.5,0.0,63.9,1.21,idm_9,1.13,319.42,0.2,right,HBEFA3/PC_G_EU4,routeright,0.1,2631.03,0,0.07,idm,59.48,28.42,0.81,63.41 -0.16,162.24,0.0,63.89,1.21,idm_9,1.13,319.39,0.3,right,HBEFA3/PC_G_EU4,routeright,0.0,2636.67,0,0.07,idm,59.44,28.43,0.8,63.42 -0.23,161.0,0.0,63.87,1.21,idm_9,1.14,319.35,0.4,right,HBEFA3/PC_G_EU4,routeright,0.0,2641.63,0,0.07,idm,59.4,28.46,0.79,63.44 -0.31,159.78,0.0,63.85,1.21,idm_9,1.14,319.3,0.5,right,HBEFA3/PC_G_EU4,routeright,0.0,2645.91,0,0.06,idm,59.36,28.49,0.79,63.46 +time,id,x,y,speed,headway,leader_id,follower_id,leader_rel_speed,target_accel_with_noise_with_failsafe,target_accel_no_noise_no_failsafe,target_accel_with_noise_no_failsafe,target_accel_no_noise_with_failsafe,realized_accel,road_grade,edge_id,lane_number,distance,relative_position +0.0,idm_0,36.64,-1.6,0.0,4.545454545454547,idm_1,idm_21,0.0,0.0,0.0,0.0,0.0,0.0,0,bottom,0,0.0,0.0 +0.1,idm_0,36.648322761506634,-1.599834647122385,0.07984158415841586,4.545454545454546,idm_1,idm_21,0.0,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.7984158415841586,0,bottom,0,0.007984158415841587,0.007984158415841587 +0.2,idm_0,36.66480556684144,-1.599507174168713,0.15812219156578355,4.545454545454545,idm_1,idm_21,0.0,0.7906341348144134,0.7906341348144134,0.7906341348144134,0.7906341348144134,0.7828060740736771,0,bottom,0,0.023796377572419945,0.023796377572419945 +0.3,idm_0,36.68928269645688,-1.599020873580327,0.23481302481051264,4.545454545454546,idm_1,idm_21,5.551115123125783e-17,0.7745774157717638,0.7745774157717638,0.7745774157717638,0.7745774157717638,0.7669083324472908,0,bottom,0,0.04727768005347121,0.04727768005347121 +0.0,idm_1,46.477059895666216,-0.2910450274933619,0.0,4.545454545454547,idm_2,idm_0,0.0,0.0,0.0,0.0,0.0,0.0,0,bottom,0,0.0,9.545454545454547 +0.1,idm_1,46.48510950976829,-0.2889238453988948,0.07984158415841586,4.545454545454547,idm_2,idm_0,0.0,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.7984158415841586,0,bottom,0,0.007984158415840879,9.553438703870388 +0.2,idm_1,46.5010513605782,-0.2847229522800698,0.15812219156578355,4.5454545454545485,idm_2,idm_0,0.0,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7828060740736771,0,bottom,0,0.023796377572418592,9.569250923026964 +0.3,idm_1,46.524725167351825,-0.2784845842789108,0.2348130248105127,4.5454545454545485,idm_2,idm_0,-5.551115123125783e-17,0.7745774157717642,0.7745774157717642,0.7745774157717642,0.7745774157717642,0.7669083324472914,0,bottom,0,0.04727768005347066,9.592732225508016 +0.0,idm_2,55.65270828548022,3.488595652781747,0.0,4.545454545454547,idm_3,idm_1,0.0,0.0,0.0,0.0,0.0,0.0,0,bottom,0,0.0,19.090909090909093 +0.1,idm_2,55.66000796138611,3.4925969566116453,0.07984158415841586,4.545454545454547,idm_3,idm_1,0.0,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.7984158415841586,0,bottom,0,0.007984158415840879,19.098893249324934 +0.2,idm_2,55.67446459778839,3.5005213350840068,0.15812219156578355,4.545454545454547,idm_3,idm_1,0.0,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7828060740736771,0,bottom,0,0.02379637757242037,19.114705468481514 +0.3,idm_2,55.69593284641682,3.5122891158136613,0.23481302481051264,4.545454545454547,idm_3,idm_1,5.551115123125783e-17,0.7745774157717641,0.7745774157717641,0.7745774157717641,0.7745774157717641,0.7669083324472908,0,bottom,0,0.04727768005347244,19.138186770962566 +0.0,idm_3,63.54122270574333,9.511222705743334,0.0,4.545454545454547,idm_4,idm_2,0.0,0.0,0.0,0.0,0.0,0.0,0,bottom,0,0.0,28.63636363636364 +0.1,idm_3,63.54710894820649,9.517108948206497,0.07984158415841586,4.545454545454549,idm_4,idm_2,0.0,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.7984158415841586,0,bottom,0,0.007984158415840879,28.64434779477948 +0.2,idm_3,63.558766351653254,9.528766351653257,0.15812219156578355,4.545454545454549,idm_4,idm_2,0.0,0.7906341348144138,0.7906341348144138,0.7906341348144138,0.7906341348144138,0.7828060740736771,0,bottom,0,0.02379637757242037,28.660160013936057 +0.3,idm_3,63.57607771154312,9.546077711543122,0.2348130248105127,4.545454545454549,idm_4,idm_2,-5.551115123125783e-17,0.7745774157717643,0.7745774157717643,0.7745774157717643,0.7745774157717643,0.7669083324472914,0,bottom,0,0.04727768005347244,28.683641316417113 +0.0,idm_4,69.61055207686752,17.363529025870548,0.0,4.545454545454547,idm_5,idm_3,0.0,0.0,0.0,0.0,0.0,0.0,0,bottom,0,0.0,38.18181818181819 +0.1,idm_4,69.61489064350748,17.370633428743492,0.07984158415841586,4.545454545454547,idm_5,idm_3,0.0,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.7984158415841586,0,bottom,0,0.007984158415844433,38.18980234023403 +0.2,idm_4,69.62348295380036,17.384703336848084,0.15812219156578355,4.545454545454547,idm_5,idm_3,0.0,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7828060740736771,0,bottom,0,0.023796377572423918,38.20561455939061 +0.3,idm_4,69.63624261991681,17.40559729011379,0.23481302481051264,4.545454545454547,idm_5,idm_3,0.0,0.7745774157717641,0.7745774157717641,0.7745774157717641,0.7745774157717641,0.7669083324472908,0,bottom,0,0.04727768005347599,38.229095861871656 +0.0,idm_5,73.45066460734851,26.51380415096358,0.0,4.545454545454547,idm_6,idm_4,0.0,0.0,0.0,0.0,0.0,0.0,0,bottom,0,0.0,47.72727272727274 +0.1,idm_5,73.45278578944298,26.521853765065657,0.07984158415841586,4.545454545454547,idm_6,idm_4,-1.3877787807814454e-17,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.8064000000000001,0.7984158415841586,0,bottom,0,0.007984158415844433,47.73525688568858 +0.2,idm_5,73.4569866825618,26.53779561587557,0.15812219156578355,4.545454545454547,idm_6,idm_4,-5.551115123125783e-17,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7906341348144135,0.7828060740736771,0,bottom,0,0.023796377572423918,47.75106910484515 +0.3,idm_5,73.46322505056297,26.561469422649196,0.23481302481051264,4.545454545454547,idm_6,idm_4,-1.1102230246251563e-16,0.7745774157717641,0.7745774157717641,0.7745774157717641,0.7745774157717641,0.7669083324472908,0,bottom,0,0.04727768005347599,47.77455040732621 diff --git a/tests/fast_tests/test_replays.py b/tests/fast_tests/test_replays.py new file mode 100644 index 000000000..bc151d2d1 --- /dev/null +++ b/tests/fast_tests/test_replays.py @@ -0,0 +1,53 @@ +from flow.replay import rllib_replay as rl_replay +from flow.replay.rllib_replay import replay_rllib + +import os +import unittest +import ray + +os.environ['TEST_FLAG'] = 'True' + + +class TestRLReplay(unittest.TestCase): + """Tests rllib_replay""" + + def test_rllib_replay_single(self): + """Test for single agent replay""" + try: + ray.init(num_cpus=1) + except Exception: + pass + # current path + current_path = os.path.realpath(__file__).rsplit('/', 1)[0] + + # run the experiment and check it doesn't crash + arg_str = '{}/../data/rllib_data/single_agent 1 --num_rollouts 1 ' \ + '--render_mode no_render ' \ + '--horizon 10'.format(current_path).split() + parser = rl_replay.create_parser() + pass_args = parser.parse_args(arg_str) + replay_rllib(pass_args) + + # FIXME(ev) set the horizon so that this runs faster + def test_rllib_replay_multi(self): + """Test for multi-agent replay""" + try: + ray.init(num_cpus=1) + except Exception: + pass + # current path + current_path = os.path.realpath(__file__).rsplit('/', 1)[0] + + # run the experiment and check it doesn't crash + arg_str = '{}/../data/rllib_data/multi_agent 1 --num_rollouts 1 ' \ + '--render_mode no_render ' \ + '--horizon 10'.format(current_path).split() + parser = rl_replay.create_parser() + pass_args = parser.parse_args(arg_str) + replay_rllib(pass_args) + + +if __name__ == '__main__': + ray.init(num_cpus=1) + unittest.main() + ray.shutdown() diff --git a/tests/fast_tests/test_rewards.py b/tests/fast_tests/test_rewards.py index 3f2e08cde..ac406b545 100644 --- a/tests/fast_tests/test_rewards.py +++ b/tests/fast_tests/test_rewards.py @@ -7,7 +7,6 @@ from flow.core.rewards import average_velocity, min_delay from flow.core.rewards import desired_velocity, boolean_action_penalty from flow.core.rewards import penalize_near_standstill, penalize_standstill -from flow.core.rewards import energy_consumption os.environ["TEST_FLAG"] = "True" @@ -152,31 +151,6 @@ def test_penalize_near_standstill(self): self.assertEqual(penalize_near_standstill(env, thresh=2), -10) self.assertEqual(penalize_near_standstill(env, thresh=0.5), -9) - def test_energy_consumption(self): - """Test the energy consumption method.""" - vehicles = VehicleParams() - vehicles.add("test", num_vehicles=10) - - env_params = EnvParams(additional_params={ - "target_velocity": 10, "max_accel": 1, "max_decel": 1, - "sort_vehicles": False}) - - env, _, _ = ring_road_exp_setup(vehicles=vehicles, - env_params=env_params) - - # check the penalty is zero at speed zero - self.assertEqual(energy_consumption(env, gain=1), 0) - - # change the speed of one vehicle - env.k.vehicle.test_set_speed("test_0", 1) - self.assertEqual(energy_consumption(env), -12.059337750000001) - - # check that stepping change the previous speeds and increases the energy consumption - env.step(rl_actions=None) - env.step(rl_actions=None) - self.assertGreater(env.k.vehicle.get_previous_speed("test_0"), 0.0) - self.assertLess(energy_consumption(env), -12.059337750000001) - def test_boolean_action_penalty(self): """Test the boolean_action_penalty method.""" actions = [False, False, False, False, False] diff --git a/tests/fast_tests/test_scenarios.py b/tests/fast_tests/test_scenarios.py index f9dd47c04..5fccdcb3b 100644 --- a/tests/fast_tests/test_scenarios.py +++ b/tests/fast_tests/test_scenarios.py @@ -5,6 +5,10 @@ from flow.networks import BottleneckNetwork, FigureEightNetwork, \ TrafficLightGridNetwork, HighwayNetwork, RingNetwork, MergeNetwork, \ MiniCityNetwork, MultiRingNetwork +from flow.networks import I210SubNetwork +from tests.setup_scripts import highway_exp_setup + +import flow.config as config __all__ = [ "MultiRingNetwork", "MiniCityNetwork" @@ -94,11 +98,101 @@ def test_additional_net_params(self): "length": 1000, "lanes": 4, "speed_limit": 30, - "num_edges": 1 + "num_edges": 1, + "use_ghost_edge": False, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, } ) ) + def test_ghost_edge(self): + """Validate the functionality of the ghost edge feature.""" + # =================================================================== # + # Without a ghost edge # + # =================================================================== # + + # create the network + env, _, _ = highway_exp_setup( + net_params=NetParams(additional_params={ + "length": 1000, + "lanes": 4, + "speed_limit": 30, + "num_edges": 1, + "use_ghost_edge": False, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, + }) + ) + env.reset() + + # check the network length + self.assertEqual(env.k.network.length(), 1000) + + # check the edge list + self.assertEqual(env.k.network.get_edge_list(), ["highway_0"]) + + # check the speed limits of the edges + self.assertEqual(env.k.network.speed_limit("highway_0"), 30) + + # =================================================================== # + # With a ghost edge (300m, 25m/s) # + # =================================================================== # + + # create the network + env, _, _ = highway_exp_setup( + net_params=NetParams(additional_params={ + "length": 1000, + "lanes": 4, + "speed_limit": 30, + "num_edges": 1, + "use_ghost_edge": True, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, + }) + ) + env.reset() + + # check the network length + self.assertEqual(env.k.network.length(), 1300.1) + + # check the edge list + self.assertEqual(env.k.network.get_edge_list(), + ["highway_0", "highway_end"]) + + # check the speed limits of the edges + self.assertEqual(env.k.network.speed_limit("highway_0"), 30) + self.assertEqual(env.k.network.speed_limit("highway_end"), 25) + + # =================================================================== # + # With a ghost edge (500m, 10m/s) # + # =================================================================== # + + # create the network + env, _, _ = highway_exp_setup( + net_params=NetParams(additional_params={ + "length": 1000, + "lanes": 4, + "speed_limit": 30, + "num_edges": 1, + "use_ghost_edge": True, + "ghost_speed_limit": 10, + "boundary_cell_length": 500, + }) + ) + env.reset() + + # check the network length + self.assertEqual(env.k.network.length(), 1500.1) + + # check the edge list + self.assertEqual(env.k.network.get_edge_list(), + ["highway_0", "highway_end"]) + + # check the speed limits of the edges + self.assertEqual(env.k.network.speed_limit("highway_0"), 30) + self.assertEqual(env.k.network.speed_limit("highway_end"), 10) + class TestRingNetwork(unittest.TestCase): @@ -160,6 +254,150 @@ def test_additional_net_params(self): ) +class TestI210SubNetwork(unittest.TestCase): + + """Tests I210SubNetwork in flow/networks/i210_subnetwork.py.""" + + def test_additional_net_params(self): + """Ensures that not returning the correct params leads to an error.""" + self.assertTrue( + test_additional_params( + network_class=I210SubNetwork, + additional_params={ + "on_ramp": False, + "ghost_edge": False, + } + ) + ) + + def test_specify_routes(self): + """Validates that the routes are properly specified for the network. + + This is done simply by checking the initial edges routes are specified + from, which alternates based on choice of network configuration. + + This method tests the routes for the following cases: + + 1. on_ramp = False, ghost_edge = False + 2. on_ramp = True, ghost_edge = False + 3. on_ramp = False, ghost_edge = True + 4. on_ramp = True, ghost_edge = True + """ + # test case 1 + network = I210SubNetwork( + name='test-3', + vehicles=VehicleParams(), + net_params=NetParams( + template=os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/test2.net.xml" + ), + additional_params={ + "on_ramp": False, + "ghost_edge": False, + }, + ), + ) + + self.assertEqual( + ['119257914'], + sorted(list(network.specify_routes(network.net_params).keys())) + ) + + del network + + # test case 2 + network = I210SubNetwork( + name='test-3', + vehicles=VehicleParams(), + net_params=NetParams( + template=os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/test2.net.xml" + ), + additional_params={ + "on_ramp": True, + "ghost_edge": True, + }, + ), + ) + + self.assertEqual( + ['119257908#0', + '119257908#1', + '119257908#1-AddedOffRampEdge', + '119257908#1-AddedOnRampEdge', + '119257908#2', + '119257908#3', + '119257914', + '173381935', + '27414342#0', + '27414342#1-AddedOnRampEdge', + '27414345', + 'ghost0'], + sorted(list(network.specify_routes(network.net_params).keys())) + ) + + del network + + # test case 3 + network = I210SubNetwork( + name='test-3', + vehicles=VehicleParams(), + net_params=NetParams( + template=os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/test2.net.xml" + ), + additional_params={ + "on_ramp": False, + "ghost_edge": True, + }, + ), + ) + + self.assertEqual( + ['119257914', 'ghost0'], + sorted(list(network.specify_routes(network.net_params).keys())) + ) + + del network + + # test case 4 + network = I210SubNetwork( + name='test-3', + vehicles=VehicleParams(), + net_params=NetParams( + template=os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/test2.net.xml" + ), + additional_params={ + "on_ramp": True, + "ghost_edge": True, + }, + ), + ) + + self.assertEqual( + ['119257908#0', + '119257908#1', + '119257908#1-AddedOffRampEdge', + '119257908#1-AddedOnRampEdge', + '119257908#2', + '119257908#3', + '119257914', + '173381935', + '27414342#0', + '27414342#1-AddedOnRampEdge', + '27414345', + 'ghost0'], + sorted(list(network.specify_routes(network.net_params).keys())) + ) + + del network + + ############################################################################### # Utility methods # ############################################################################### diff --git a/tests/fast_tests/test_util.py b/tests/fast_tests/test_util.py index 458fadcf4..67386cc77 100644 --- a/tests/fast_tests/test_util.py +++ b/tests/fast_tests/test_util.py @@ -14,7 +14,6 @@ from flow.core.util import emission_to_csv from flow.envs import MergePOEnv from flow.networks import MergeNetwork -from flow.utils.flow_warnings import deprecated_attribute from flow.utils.registry import make_create_env from flow.utils.rllib import FlowParamsEncoder, get_flow_params @@ -60,25 +59,6 @@ def test_emission_to_csv(self): self.assertEqual(len(dict1), 104) -class TestWarnings(unittest.TestCase): - """Tests warning functions located in flow.utils.warnings""" - - def test_deprecated_attribute(self): - # dummy class - class Foo(object): - pass - - # dummy attribute name - dep_from = "bar_deprecated" - dep_to = "bar_new" - - # check the deprecation warning is printing what is expected - self.assertWarnsRegex( - PendingDeprecationWarning, - "The attribute bar_deprecated in Foo is deprecated, use bar_new " - "instead.", deprecated_attribute, Foo(), dep_from, dep_to) - - class TestRegistry(unittest.TestCase): """Tests the methods located in flow/utils/registry.py""" diff --git a/tests/fast_tests/test_vehicles.py b/tests/fast_tests/test_vehicles.py index 485a6a072..7e1405007 100644 --- a/tests/fast_tests/test_vehicles.py +++ b/tests/fast_tests/test_vehicles.py @@ -33,7 +33,7 @@ def test_speed_lane_change_modes(self): speed_mode='obey_safe_speed', ), lane_change_params=SumoLaneChangeParams( - lane_change_mode="no_lat_collide", + lane_change_mode="no_lc_safe", ) ) @@ -56,7 +56,7 @@ def test_speed_lane_change_modes(self): self.assertEqual(vehicles.type_parameters["typeB"][ "car_following_params"].speed_mode, 0) self.assertEqual(vehicles.type_parameters["typeB"][ - "lane_change_params"].lane_change_mode, 1621) + "lane_change_params"].lane_change_mode, 512) vehicles.add( "typeC", @@ -89,7 +89,7 @@ def test_controlled_id_params(self): speed_mode="obey_safe_speed", ), lane_change_params=SumoLaneChangeParams( - lane_change_mode="no_lat_collide", + lane_change_mode="no_lc_safe", )) default_mingap = SumoCarFollowingParams().controller_params["minGap"] self.assertEqual(vehicles.types[0]["type_params"]["minGap"], @@ -258,7 +258,10 @@ def test_no_junctions_highway(self): "lanes": 3, "speed_limit": 30, "resolution": 40, - "num_edges": 1 + "num_edges": 1, + "use_ghost_edge": False, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, } net_params = NetParams(additional_params=additional_net_params) vehicles = VehicleParams() @@ -330,7 +333,10 @@ def test_no_junctions_highway(self): "lanes": 4, "speed_limit": 30, "resolution": 40, - "num_edges": 1 + "num_edges": 1, + "use_ghost_edge": False, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, } net_params = NetParams(additional_params=additional_net_params) vehicles = VehicleParams() @@ -398,7 +404,10 @@ def test_no_junctions_highway(self): "lanes": 3, "speed_limit": 30, "resolution": 40, - "num_edges": 3 + "num_edges": 3, + "use_ghost_edge": False, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, } net_params = NetParams(additional_params=additional_net_params) vehicles = VehicleParams() @@ -465,7 +474,10 @@ def test_no_junctions_highway(self): "lanes": 3, "speed_limit": 30, "resolution": 40, - "num_edges": 3 + "num_edges": 3, + "use_ghost_edge": False, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, } net_params = NetParams(additional_params=additional_net_params) vehicles = VehicleParams() diff --git a/tests/fast_tests/test_visualizers.py b/tests/fast_tests/test_visualizers.py index 7af413909..5fa6d649a 100644 --- a/tests/fast_tests/test_visualizers.py +++ b/tests/fast_tests/test_visualizers.py @@ -1,5 +1,3 @@ -from flow.visualize import visualizer_rllib as vs_rllib -from flow.visualize.visualizer_rllib import visualizer_rllib import flow.visualize.capacity_diagram_generator as cdg import flow.visualize.time_space_diagram as tsd import flow.visualize.plot_ray_results as prr @@ -14,45 +12,6 @@ os.environ['TEST_FLAG'] = 'True' -class TestVisualizerRLlib(unittest.TestCase): - """Tests visualizer_rllib""" - - def test_visualizer_single(self): - """Test for single agent""" - try: - ray.init(num_cpus=1) - except Exception: - pass - # current path - current_path = os.path.realpath(__file__).rsplit('/', 1)[0] - - # run the experiment and check it doesn't crash - arg_str = '{}/../data/rllib_data/single_agent 1 --num_rollouts 1 ' \ - '--render_mode no_render ' \ - '--horizon 10'.format(current_path).split() - parser = vs_rllib.create_parser() - pass_args = parser.parse_args(arg_str) - visualizer_rllib(pass_args) - - # FIXME(ev) set the horizon so that this runs faster - def test_visualizer_multi(self): - """Test for multi-agent visualization""" - try: - ray.init(num_cpus=1) - except Exception: - pass - # current path - current_path = os.path.realpath(__file__).rsplit('/', 1)[0] - - # run the experiment and check it doesn't crash - arg_str = '{}/../data/rllib_data/multi_agent 1 --num_rollouts 1 ' \ - '--render_mode no_render ' \ - '--horizon 10'.format(current_path).split() - parser = vs_rllib.create_parser() - pass_args = parser.parse_args(arg_str) - visualizer_rllib(pass_args) - - class TestPlotters(unittest.TestCase): def test_capacity_diagram_generator(self): @@ -91,236 +50,150 @@ def test_capacity_diagram_generator(self): np.testing.assert_array_almost_equal(std_outflows, expected_stds) def test_time_space_diagram_figure_eight(self): - # check that the exported data matches the expected emission file data - fig8_emission_data = { - 'idm_3': {'pos': [27.25, 28.25, 30.22, 33.17], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.95], - 'edge': ['upper_ring', 'upper_ring', 'upper_ring', - 'upper_ring'], - 'lane': [0.0, 0.0, 0.0, 0.0]}, - 'idm_4': {'pos': [56.02, 57.01, 58.99, 61.93], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.95], - 'edge': ['upper_ring', 'upper_ring', 'upper_ring', - 'upper_ring'], - 'lane': [0.0, 0.0, 0.0, 0.0]}, - 'idm_5': {'pos': [84.79, 85.78, 87.76, 90.7], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.95], - 'edge': ['upper_ring', 'upper_ring', 'upper_ring', - 'upper_ring'], - 'lane': [0.0, 0.0, 0.0, 0.0]}, - 'idm_2': {'pos': [28.77, 29.76, 1.63, 4.58], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.97, 2.95], - 'edge': ['top', 'top', 'upper_ring', 'upper_ring'], - 'lane': [0.0, 0.0, 0.0, 0.0]}, - 'idm_13': {'pos': [106.79, 107.79, 109.77, 112.74], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.96], - 'edge': ['lower_ring', 'lower_ring', 'lower_ring', - 'lower_ring'], - 'lane': [0.0, 0.0, 0.0, 0.0]}, - 'idm_9': {'pos': [22.01, 23.0, 24.97, 27.92], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.97, 2.95], - 'edge': ['left', 'left', 'left', 'left'], - 'lane': [0.0, 0.0, 0.0, 0.0]}, - 'idm_6': {'pos': [113.56, 114.55, 116.52, 119.47], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.97, 2.95], - 'edge': ['upper_ring', 'upper_ring', 'upper_ring', - 'upper_ring'], - 'lane': [0.0, 0.0, 0.0, 0.0]}, - 'idm_8': {'pos': [29.44, 0.28, 2.03, 4.78], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.84, 1.76, 2.75], - 'edge': ['right', ':center_0', ':center_0', - ':center_0'], - 'lane': [0.0, 0.0, 0.0, 0.0]}, - 'idm_12': {'pos': [78.03, 79.02, 80.99, 83.94], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.95], - 'edge': ['lower_ring', 'lower_ring', 'lower_ring', - 'lower_ring'], - 'lane': [0.0, 0.0, 0.0, 0.0]}, - 'idm_10': {'pos': [20.49, 21.48, 23.46, 26.41], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.95], - 'edge': ['lower_ring', 'lower_ring', 'lower_ring', - 'lower_ring'], - 'lane': [0.0, 0.0, 0.0, 0.0]}, - 'idm_11': {'pos': [49.26, 50.25, 52.23, 55.17], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.95], - 'edge': ['lower_ring', 'lower_ring', 'lower_ring', - 'lower_ring'], - 'lane': [0.0, 0.0, 0.0, 0.0]}, - 'idm_1': {'pos': [0.0, 0.99, 2.97, 5.91], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.95], - 'edge': ['top', 'top', 'top', 'top'], - 'lane': [0.0, 0.0, 0.0, 0.0]}, - 'idm_7': {'pos': [0.67, 1.66, 3.64, 6.58], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.97, 2.94], - 'edge': ['right', 'right', 'right', 'right'], - 'lane': [0.0, 0.0, 0.0, 0.0]}, - 'idm_0': {'pos': [0.0, 1.0, 2.98, 5.95], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 1.0, 1.99, 2.97], - 'edge': ['bottom', 'bottom', 'bottom', 'bottom'], - 'lane': [0.0, 0.0, 0.0, 0.0]} - } dir_path = os.path.dirname(os.path.realpath(__file__)) - actual_emission_data = tsd.import_data_from_emission( - os.path.join(dir_path, 'test_files/fig8_emission.csv')) - self.assertDictEqual(fig8_emission_data, actual_emission_data) - - # test get_time_space_data for figure eight networks flow_params = tsd.get_flow_params( os.path.join(dir_path, 'test_files/fig8.json')) - pos, speed, _ = tsd.get_time_space_data( - actual_emission_data, flow_params) - - expected_pos = np.array( - [[60, 23.8, 182.84166941, 154.07166941, 125.30166941, 96.54166941, - -203.16166941, -174.40166941, -145.63166941, -116.86166941, - -88.09166941, -59.33, -30.56, -1.79], - [59, 22.81, 181.85166941, 153.08166941, 124.31166941, 95.54166941, - -202.17166941, -173.40166941, -144.64166941, -115.87166941, - -87.10166941, -58.34, -29.72, -0.8], - [57.02, 20.83, 179.87166941, 151.10166941, 122.34166941, - 93.56166941, -200.02166941, -171.43166941, -142.66166941, - -113.89166941, -85.13166941, -56.36, -27.97, 208.64166941]] + emission_data, _, _, _ = tsd.import_data_from_trajectory( + os.path.join(dir_path, 'test_files/fig8_emission.csv'), flow_params) + + segs, _ = tsd.get_time_space_data(emission_data, flow_params['network']) + + expected_segs = np.array([ + [[1., 263.16166941], [2., 262.16166941]], + [[2., 262.16166941], [3., 260.18166941]], + [[3., 260.18166941], [4., 257.21166941]], + [[1., 226.96166941], [2., 225.97166941]], + [[2., 225.97166941], [3., 223.99166941]], + [[3., 223.99166941], [4., 221.05166941]], + [[1., 386.00333882], [2., 385.01333882]], + [[2., 385.01333882], [3., 383.03333882]], + [[3., 383.03333882], [4., 380.08333882]], + [[1., 357.23333882], [2., 356.24333882]], + [[2., 356.24333882], [3., 354.26333882]], + [[3., 354.26333882], [4., 351.32333882]], + [[1., 328.46333882], [2., 327.47333882]], + [[2., 327.47333882], [3., 325.50333882]], + [[3., 325.50333882], [4., 322.55333882]], + [[1., 299.70333882], [2., 298.70333882]], + [[2., 298.70333882], [3., 296.72333882]], + [[3., 296.72333882], [4., 293.75333882]], + [[1., 0.], [2., 0.99]], + [[2., 0.99], [3., 3.14]], + [[3., 3.14], [4., 6.09]], + [[1., 28.76], [2., 29.76]], + [[2., 29.76], [3., 31.73]], + [[3., 31.73], [4., 34.68]], + [[1., 57.53], [2., 58.52]], + [[2., 58.52], [3., 60.5]], + [[3., 60.5], [4., 63.44]], + [[1., 86.3], [2., 87.29]], + [[2., 87.29], [3., 89.27]], + [[3., 89.27], [4., 92.21]], + [[1., 115.07], [2., 116.06]], + [[2., 116.06], [3., 118.03]], + [[3., 118.03], [4., 120.98]], + [[1., 143.83166941], [2., 144.82166941]], + [[2., 144.82166941], [3., 146.80166941]], + [[3., 146.80166941], [4., 149.74166941]], + [[1., 172.60166941], [2., 173.44166941]], + [[2., 173.44166941], [3., 175.19166941]], + [[3., 175.19166941], [4., 177.94166941]], + [[1., 201.37166941], [2., 202.36166941]], + [[2., 202.36166941], [3., 411.80333882]], + [[3., 411.80333882], [4., 408.85333882]]] ) - expected_speed = np.array([ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, - 0.99, 0.84, 0.99], - [1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.98, 1.98, 1.98, 1.97, - 1.97, 1.76, 1.97] - ]) - np.testing.assert_array_almost_equal(pos[:-1, :], expected_pos) - np.testing.assert_array_almost_equal(speed[:-1, :], expected_speed) + np.testing.assert_array_almost_equal(segs, expected_segs) def test_time_space_diagram_merge(self): dir_path = os.path.dirname(os.path.realpath(__file__)) - emission_data = tsd.import_data_from_emission( - os.path.join(dir_path, 'test_files/merge_emission.csv')) - flow_params = tsd.get_flow_params( os.path.join(dir_path, 'test_files/merge.json')) - pos, speed, _ = tsd.get_time_space_data(emission_data, flow_params) - - expected_pos = np.array( - [[4.86, 180.32, 361.32, 547.77, 0], - [4.88, 180.36, 361.36, 547.8, 0], - [4.95, 180.43, 361.44, 547.87, 0], - [5.06, 180.54, 361.56, 547.98, 0], - [5.21, 180.68, 361.72, 548.12, 0], - [5.4, 180.86, 0, 0, 0]] - ) - expected_speed = np.array( - [[0, 0, 0, 0, 0], - [0.15, 0.17, 0.19, 0.14, 0], - [0.35, 0.37, 0.39, 0.34, 0], - [0.54, 0.57, 0.59, 0.54, 0], - [0.74, 0.7, 0.79, 0.71, 0], - [0.94, 0.9, 0, 0, 0]] + emission_data, _, _, _ = tsd.import_data_from_trajectory( + os.path.join(dir_path, 'test_files/merge_emission.csv'), flow_params) + + segs, _ = tsd.get_time_space_data(emission_data, flow_params['network']) + + expected_segs = np.array([ + [[2.0000e-01, 7.2463e+02], [4.0000e-01, 7.2467e+02]], + [[4.0000e-01, 7.2467e+02], [6.0000e-01, 7.2475e+02]], + [[6.0000e-01, 7.2475e+02], [8.0000e-01, 7.2487e+02]], + [[8.0000e-01, 7.2487e+02], [1.0000e+00, 7.2502e+02]]] ) - np.testing.assert_array_almost_equal(pos, expected_pos) - np.testing.assert_array_almost_equal(speed, expected_speed) + np.testing.assert_array_almost_equal(segs, expected_segs) def test_time_space_diagram_I210(self): dir_path = os.path.dirname(os.path.realpath(__file__)) - emission_data = tsd.import_data_from_emission( - os.path.join(dir_path, 'test_files/i210_emission.csv')) - module = __import__("examples.exp_configs.non_rl", fromlist=["i210_subnetwork"]) flow_params = getattr(module, "i210_subnetwork").flow_params - pos, speed, _ = tsd.get_time_space_data(emission_data, flow_params) - - expected_pos = np.array( - [[5.1, 0., 0.], - [23.37, 0., 0.], - [42.02, 5.1, 0.], - [61.21, 22.97, 0.], - [80.45, 40.73, 5.1], - [101.51, 0., 0.]] - ) - expected_speed = np.array( - [[23., 0., 0.], - [22.84, 0., 0.], - [23.31, 23., 0.], - [23.98, 22.33, 0.], - [24.25, 22.21, 23.], - [26.33, 0., 0.]] - ) - - np.testing.assert_array_almost_equal(pos, expected_pos) - np.testing.assert_array_almost_equal(speed, expected_speed) + emission_data, _, _, _ = tsd.import_data_from_trajectory( + os.path.join(dir_path, 'test_files/i210_emission.csv'), flow_params) + + segs, _ = tsd.get_time_space_data(emission_data, flow_params['network']) + + expected_segs = { + 1: np.array([ + [[-719.2, 3.77], [-718.4, 22.04]], + [[-718.4, 22.04], [-717.6, 40.69]], + [[-717.6, 40.69], [-716.8, 59.88]], + [[-716.8, 59.88], [-716., 17.54]], + [[-716., 17.54], [-715.2, 38.6]], + [[-717.6, 3.77], [-716.8, 21.64]], + [[-716.8, 21.64], [-716., 39.4]]] + ), + 2: np.array([ + [[-717.6, 3.77], [-716.8, 22.65]], + [[-716.8, 22.65], [-716., 41.85]]] + ), + 3: np.array([ + [[-719.2, 3.77], [-718.4, 22.39]], + [[-718.4, 22.39], [-717.6, 41.73]], + [[-717.6, 41.73], [-716.8, 0.]], + [[-716.8, 0.], [-716., 20.32]], + [[-716., 20.32], [-715.2, 42.13]], + [[-717.6, 3.77], [-716.8, 22.41]], + [[-716.8, 22.41], [-716., 41.05]]] + ), + 4: np.array([ + [[-717.6, 3.77], [-716.8, 22.27]], + [[-716.8, 22.27], [-716., 41.13]]] + )} + + for lane, expected_seg in expected_segs.items(): + np.testing.assert_array_almost_equal(segs[lane], expected_seg) def test_time_space_diagram_ring_road(self): dir_path = os.path.dirname(os.path.realpath(__file__)) - emission_data = tsd.import_data_from_emission( - os.path.join(dir_path, 'test_files/ring_230_emission.csv')) - flow_params = tsd.get_flow_params( os.path.join(dir_path, 'test_files/ring_230.json')) - pos, speed, _ = tsd.get_time_space_data(emission_data, flow_params) - - expected_pos = np.array( - [[0.0000e+00, 9.5500e+00, 9.5550e+01, 1.0510e+02, 1.1465e+02, - 1.2429e+02, 1.3384e+02, 1.4338e+02, 1.5293e+02, 1.6247e+02, - 1.7202e+02, 1.8166e+02, 1.9090e+01, 1.9121e+02, 2.0075e+02, - 2.8640e+01, 3.8180e+01, 4.7730e+01, 5.7270e+01, 6.6920e+01, - 7.6460e+01, 8.6010e+01], - [1.0000e-02, 9.5500e+00, 9.5560e+01, 1.0511e+02, 1.1465e+02, - 1.2430e+02, 1.3384e+02, 1.4339e+02, 1.5294e+02, 1.6248e+02, - 1.7203e+02, 1.8167e+02, 1.9100e+01, 1.9122e+02, 2.0076e+02, - 2.8640e+01, 3.8190e+01, 4.7740e+01, 5.7280e+01, 6.6930e+01, - 7.6470e+01, 8.6020e+01], - [2.0000e-02, 9.5700e+00, 9.5580e+01, 1.0512e+02, 1.1467e+02, - 1.2431e+02, 1.3386e+02, 1.4341e+02, 1.5295e+02, 1.6250e+02, - 1.7204e+02, 1.8169e+02, 1.9110e+01, 1.9123e+02, 2.0078e+02, - 2.8660e+01, 3.8210e+01, 4.7750e+01, 5.7300e+01, 6.6940e+01, - 7.6490e+01, 8.6030e+01], - [5.0000e-02, 9.5900e+00, 9.5600e+01, 1.0515e+02, 1.1469e+02, - 1.2434e+02, 1.3388e+02, 1.4343e+02, 1.5297e+02, 1.6252e+02, - 1.7207e+02, 1.8171e+02, 1.9140e+01, 1.9126e+02, 2.0081e+02, - 2.8680e+01, 3.8230e+01, 4.7770e+01, 5.7320e+01, 6.6970e+01, - 7.6510e+01, 8.6060e+01], - [8.0000e-02, 9.6200e+00, 9.5630e+01, 1.0518e+02, 1.1472e+02, - 1.2437e+02, 1.3391e+02, 1.4346e+02, 1.5301e+02, 1.6255e+02, - 1.7210e+02, 1.8174e+02, 1.9170e+01, 1.9129e+02, 2.0085e+02, - 2.8710e+01, 3.8260e+01, 4.7810e+01, 5.7350e+01, 6.7000e+01, - 7.6540e+01, 8.6090e+01], - [1.2000e-01, 9.6600e+00, 9.5670e+01, 1.0522e+02, 1.1476e+02, - 1.2441e+02, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, - 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, - 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, - 0.0000e+00, 0.0000e+00]] + emission_data, _, _, _ = tsd.import_data_from_trajectory( + os.path.join(dir_path, 'test_files/ring_230_emission.csv'), flow_params) + + segs, _ = tsd.get_time_space_data(emission_data, flow_params['network']) + + expected_segs = np.array([ + [[-7.50000000e+01, 0.00000000e+00], [-7.49000000e+01, 7.98415842e-03]], + [[-7.49000000e+01, 7.98415842e-03], [-7.48000000e+01, 2.37963776e-02]], + [[-7.48000000e+01, 2.37963776e-02], [-7.47000000e+01, 4.72776801e-02]], + [[-7.50000000e+01, 9.54545455e+00], [-7.49000000e+01, 9.55343870e+00]], + [[-7.49000000e+01, 9.55343870e+00], [-7.48000000e+01, 9.56925092e+00]], + [[-7.48000000e+01, 9.56925092e+00], [-7.47000000e+01, 9.59273223e+00]], + [[-7.50000000e+01, 1.90909091e+01], [-7.49000000e+01, 1.90988932e+01]], + [[-7.49000000e+01, 1.90988932e+01], [-7.48000000e+01, 1.91147055e+01]], + [[-7.48000000e+01, 1.91147055e+01], [-7.47000000e+01, 1.91381868e+01]], + [[-7.50000000e+01, 2.86363636e+01], [-7.49000000e+01, 2.86443478e+01]], + [[-7.49000000e+01, 2.86443478e+01], [-7.48000000e+01, 2.86601600e+01]], + [[-7.48000000e+01, 2.86601600e+01], [-7.47000000e+01, 2.86836413e+01]], + [[-7.50000000e+01, 3.81818182e+01], [-7.49000000e+01, 3.81898023e+01]], + [[-7.49000000e+01, 3.81898023e+01], [-7.48000000e+01, 3.82056146e+01]], + [[-7.48000000e+01, 3.82056146e+01], [-7.47000000e+01, 3.82290959e+01]], + [[-7.50000000e+01, 4.77272727e+01], [-7.49000000e+01, 4.77352569e+01]], + [[-7.49000000e+01, 4.77352569e+01], [-7.48000000e+01, 4.77510691e+01]], + [[-7.48000000e+01, 4.77510691e+01], [-7.47000000e+01, 4.77745504e+01]]] ) - expected_speed = np.array([ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0.08, 0.08, 0.08, 0.08, 0.08, 0.08, 0.08, 0.08, 0.08, 0.08, 0.08, - 0.08, 0.08, 0.08, 0.1, 0.08, 0.08, 0.08, 0.08, 0.08, 0.08, 0.08], - [0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, - 0.16, 0.16, 0.16, 0.2, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16], - [0.23, 0.23, 0.23, 0.23, 0.23, 0.23, 0.23, 0.23, 0.23, 0.23, 0.23, - 0.23, 0.23, 0.23, 0.29, 0.23, 0.23, 0.23, 0.23, 0.23, 0.23, 0.23], - [0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, - 0.31, 0.31, 0.31, 0.39, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31], - [0.41, 0.41, 0.41, 0.41, 0.41, 0.41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0] - ]) - - np.testing.assert_array_almost_equal(pos, expected_pos) - np.testing.assert_array_almost_equal(speed, expected_speed) + + np.testing.assert_array_almost_equal(segs, expected_segs) def test_plot_ray_results(self): dir_path = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/setup_scripts.py b/tests/setup_scripts.py index 08d5b2c1e..343bad906 100644 --- a/tests/setup_scripts.py +++ b/tests/setup_scripts.py @@ -343,7 +343,10 @@ def highway_exp_setup(sim_params=None, "lanes": 1, "speed_limit": 30, "resolution": 40, - "num_edges": 1 + "num_edges": 1, + "use_ghost_edge": False, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, } net_params = NetParams(additional_params=additional_net_params) diff --git a/tests/slow_tests/test_benchmarks.py b/tests/slow_tests/test_benchmarks.py index c119d4bd8..4a50e84cb 100644 --- a/tests/slow_tests/test_benchmarks.py +++ b/tests/slow_tests/test_benchmarks.py @@ -84,8 +84,7 @@ def ray_runner(self, num_runs, flow_params, version): # Register as rllib env register_env(env_name, create_env) - - alg = ppo.PPOAgent( + alg = ppo.ppo.PPOTrainer( env=env_name, config=config) for i in range(num_runs):