Update workflow with AWS config and commands

Author: rachelekm

.github/workflows/build-graph.yml

@@ -10,19 +10,25 @@ on:
 jobs:
   docker-compose:
     runs-on: ubuntu-latest
-
+    env:
+      CAC_OTP_DATA_BUCKET: 'cleanair-otp-data'
     steps:
       - name: Checkout GitHub Repo
         uses: actions/checkout@v4
 
-      - name: Run Docker Compose
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      
+      - name: Download latest GTFS, OSM, and elevation files from S3
+        run: aws s3 sync --exclude "Graph.obj" "s3://$CAC_OTP_DATA_BUCKET/" "otp_data"
+
+      - name: Build graph
         run: |
           cd deployment/graph
           docker compose run --rm otp otp --build /var/otp
 
-      - name: Upload graph object
-        uses: actions/upload-artifact@v4
-        with:
-          name: graph-artifact
-          path: otp_data/Graph.obj
-          retention-days: 30
+      - name: Upload graph object to S3
+        run: aws s3 cp "otp_data/Graph.obj" "s3://$CAC_OTP_DATA_BUCKET/Graph.obj"

README.md

@@ -13,16 +13,17 @@ Development Installation
 ------------------------
 
 1. Make sure you have the development dependencies installed
-2. Place GTFS .zip files, OSM files, and elevation .tif files (optional) in the root of the otp_data folder
-3. (Optional) Generate a graph file with (takes approx 3 hours) `docker-compose run --rm otp otp --build /var/otp` in the deployment/graph directory.
-4. Copy `deployment/ansible/group_vars/development_template` to `deployment/ansible/group_vars/development`
-5. Change into the `src/` folder and run `npm install` to install the node modules on the host machine
-6. Run `vagrant up`. You can choose to change the Virtualbox shared folder type for the `app` VM from its default VirtualBox by:
+2. Download the latest Graph.obj for OTP: `scripts/download-latest-graph.sh`
+     - This will take ~10 minutes to download
+     - If you already have a local graph file but want the latest from S3, run `scripts/download-latest-graph.sh --force`
+3. Copy `deployment/ansible/group_vars/development_template` to `deployment/ansible/group_vars/development`
+4. Change into the `src/` folder and run `npm install` to install the node modules on the host machine
+5. Run `vagrant up`. You can choose to change the Virtualbox shared folder type for the `app` VM from its default VirtualBox by:
 ```
 CAC_APP_SHARED_FOLDER_TYPE=nfs vagrant up
 ```
-7. See the app at http://localhost:8024! See OpenTripPlanner at http://localhost:9090.
-8. Running `./scripts/serve-js-dev.sh` on the host will rebuild the front-end app on file change (the browser must be reloaded manually to pick up the change). Alternatively, `cd /opt/app/src && npm run gulp-development` can be run manually in the VM to pick up changes to the static files.
+6. See the app at http://localhost:8024! See OpenTripPlanner at http://localhost:9090.
+7. Running `./scripts/serve-js-dev.sh` on the host will rebuild the front-end app on file change (the browser must be reloaded manually to pick up the change). Alternatively, `cd /opt/app/src && npm run gulp-development` can be run manually in the VM to pick up changes to the static files.
 
 Note that if there is an existing build Graph.obj in `otp_data`, vagrant provisioning in development mode will not attempt to rebuild the graph, but will use the one already present.
 

scripts/download-latest-graph.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+graph_path="otp_data/Graph.obj"
+
+if [[ ! -f "$graph_path" ]] || [[ "$*" == *"force"* ]]; then
+    echo "Pulling latest Graph.obj from S3..."
+    aws --profile=gophillygo s3 cp "s3://cleanair-otp-data/Graph.obj" "$graph_path"
+else
+    echo "Built graph already in otp_data directory, skip pulling latest from S3"
+fi