diff --git a/README.md b/README.md index 3e614996ec..5fa4255c86 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ RR_DATABASE="" 3. Start the DB + GraphQL API. You'll need to be on City of Austin VPN _or_ your IP address will need to be whitelisted to reach our read-replica host. ```shell -./vision-zero replicate-db +./vision-zero replicate ``` This command will: diff --git a/database/metadata/databases/default/tables/public_locations.yaml b/database/metadata/databases/default/tables/public_locations.yaml index b39045da1f..a72b1baec3 100644 --- a/database/metadata/databases/default/tables/public_locations.yaml +++ b/database/metadata/databases/default/tables/public_locations.yaml @@ -42,9 +42,6 @@ select_permissions: - is_service_road - is_signalized - location_id - - street_level - - latitude - - longitude - council_district - signal_id - location_group @@ -74,9 +71,6 @@ select_permissions: - is_service_road - is_signalized - location_id - - street_level - - latitude - - longitude - council_district - signal_id - location_group @@ -106,9 +100,6 @@ select_permissions: - is_service_road - is_signalized - location_id - - street_level - - latitude - - longitude - council_district - signal_id - location_group diff --git a/database/metadata/databases/default/tables/tables.yaml b/database/metadata/databases/default/tables/tables.yaml index 03332c89cb..10cc1d50ac 100644 --- a/database/metadata/databases/default/tables/tables.yaml +++ b/database/metadata/databases/default/tables/tables.yaml @@ -57,7 +57,6 @@ - "!include public_afd__incidents.yaml" - "!include public_atd__recommendation_status_lkp.yaml" - "!include public_atd_apd_blueform.yaml" -- "!include public_locations.yaml" - "!include public_change_log_crashes.yaml" - "!include public_change_log_crashes_cris.yaml" - "!include public_change_log_ems__incidents.yaml" @@ -76,6 +75,7 @@ - "!include public_fatalities_view.yaml" - "!include public_location_crashes_view.yaml" - "!include public_location_notes.yaml" +- "!include public_locations.yaml" - "!include public_locations_list_view.yaml" - "!include public_moped_component_crashes.yaml" - "!include public_people.yaml" diff --git a/database/migrations/default/1767979945377_drop_street_level/down.sql b/database/migrations/default/1767979945377_drop_street_level/down.sql new file mode 100644 index 0000000000..72bd19a49a --- /dev/null +++ b/database/migrations/default/1767979945377_drop_street_level/down.sql @@ -0,0 +1,3 @@ +alter table locations add column street_level text; +alter table locations add column latitude numeric; +alter table locations add column longitude numeric; diff --git a/database/migrations/default/1767979945377_drop_street_level/up.sql b/database/migrations/default/1767979945377_drop_street_level/up.sql new file mode 100644 index 0000000000..48772b61e5 --- /dev/null +++ b/database/migrations/default/1767979945377_drop_street_level/up.sql @@ -0,0 +1,4 @@ +alter table locations + drop column street_level, + drop column latitude, + drop column longitude; diff --git a/docker-compose.yml b/docker-compose.yml index e6555b6357..06e79759d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,6 @@ services: graphql-engine: image: hasura/graphql-engine:v2.48.9 - volumes: - - ./viewer/graphql-engine-metadata:/metadata ports: - 8084:8080 environment: @@ -19,6 +17,7 @@ services: env_file: - .env volumes: + - postgres_data:/var/lib/postgresql/data - ./database/snapshots:/snapshots ports: - 5431:5432 @@ -43,3 +42,6 @@ services: networks: default: name: vision-zero_default + +volumes: + postgres_data: diff --git a/docs/local_dev.md b/docs/local_dev.md index a4996ba21d..2f6c1df75b 100644 --- a/docs/local_dev.md +++ b/docs/local_dev.md @@ -1,6 +1,6 @@ ## Local Development -The suite has a python script in the root of the respository, named `vision-zero`, which can be used to replicate and run the Vision Zero database and graphQL API locally. +Use the [`vision-zero`](./vision-zero) helper script to replicate and run the Vision Zero database and graphQL API locally. See the repo [README](/README.md) for quick start instructions for using this tool. @@ -20,12 +20,6 @@ source ./venv/bin/active; ### Examples of `vision-zero` commands -Note: There is a flag which ends up being observed for any of the following commands which start the postgres database: - -`-r / --ram-disk` will cause the database to back its "storage" on a RAM disk instead of non-volatile storage. This has the upside of being much faster as there is essentially no limit to the IOPS available to the database, but the data won't be able to survive a restart and will require being `replicate-db`'d back into place. - -The default is to use the disk in the host to back the database, which is the operation our team is most familiar with, so if you don't need or want the RAM disk configuration, you can ignore this option. - #### `vision-zero build` Rebuild the stack's images based on the Dockerfiles found in the repository. They are built with the `--no-cache` flag which will make the build process slower, but avoid any stale image layers that have inadvertently cached out-of-date apt resource lists. @@ -42,7 +36,7 @@ Start and stop the Hasura graphql-engine software as well as the database Start and stop the Vision Zero CR3 / User API -### `vision-zero local-stack-up` & `vision-zero local-stack-down` +### `vision-zero up` & `vision-zero down` Start and stop the DB, `graphql-engine`, and the API at once @@ -58,7 +52,7 @@ Start a `bash` shell on a machine with supporting tooling Stop the stack -#### `vision-zero replicate-db` +#### `vision-zero replicate` - Download a snapshot of the production database - Store the file in `./database/snapshots/visionzero-{date}-{with|without}-change-log.sql` @@ -68,8 +62,8 @@ Stop the stack Note: the `-c / --include-change-log-data` flag can be used to opt to include the data of past change log events. The schema is created either way. Note: the `-f / --filename` flag can be optionally used to point to a specific data dump .sql file to use to restore. -The way the snapshots are dated means that one will only end up downloading -one copy of the data per-day, both in the with and without change log data. +The way the snapshots are dated means that one will only end up downloading one copy of the data per-day, both in the with and without change log data. + #### `vision-zero dump-local-db` diff --git a/editor/configs/locationColumns.ts b/editor/configs/locationColumns.ts index 18a55ad799..10835b2b3c 100644 --- a/editor/configs/locationColumns.ts +++ b/editor/configs/locationColumns.ts @@ -1,6 +1,7 @@ import { ColDataCardDef } from "@/types/types"; import { Location } from "@/types/locations"; -import { formatDollars } from "@/utils/formatters"; +import { formatArrayToString, formatDollars } from "@/utils/formatters"; + export const locationColumns = { location_id: { path: "location_id", @@ -20,7 +21,8 @@ export const locationColumns = { valueFormatter: formatDollars, }, street_level: { - path: "street_level", - label: "ASMP Street level", + path: "street_levels", + label: "ASMP Street level(s)", + valueFormatter: formatArrayToString, }, } satisfies Record>; diff --git a/editor/queries/location.ts b/editor/queries/location.ts index 14ffc58391..9afac837ae 100644 --- a/editor/queries/location.ts +++ b/editor/queries/location.ts @@ -6,11 +6,9 @@ export const GET_LOCATION = gql` where: { location_id: { _eq: $locationId }, is_deleted: { _eq: false } } ) { location_id - street_level + street_levels location_name geometry - latitude - longitude locations_list_view { cr3_crash_count non_cr3_crash_count diff --git a/editor/types/locations.ts b/editor/types/locations.ts index db3ebd82da..2453ab0520 100644 --- a/editor/types/locations.ts +++ b/editor/types/locations.ts @@ -5,10 +5,8 @@ import { LocationNote } from "@/types/locationNote"; export type Location = { location_id: string; location_name: string | null; - latitude: number | null; - longitude: number | null; geometry: MultiPolygon | null; - street_level: string | null; + street_levels: string[] | null; locations_list_view: LocationsListRow | null; location_notes: LocationNote[] }; diff --git a/editor/utils/formatters.ts b/editor/utils/formatters.ts index 3475e0fb14..d7c564af64 100644 --- a/editor/utils/formatters.ts +++ b/editor/utils/formatters.ts @@ -56,3 +56,17 @@ export const formatTime = (value: unknown): string => { export const formatFileTimestamp = (date: Date): string => { return format(date, "yyyy-MM-dd h.mm.ss a"); }; + +/** + * Format an array of values to a comma-separated-string, removing + * null, undefined, and empty strings + */ +export const formatArrayToString = (value: unknown): string => { + if (value && Array.isArray(value)) { + return value + .filter((val) => val !== undefined && val !== null && val !== "") + .map((val) => String(val)) + .join(", "); + } + return ""; +}; diff --git a/requirements.txt b/requirements.txt index 6eb74bd684..85ce8e49fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ shtab==1.* python-dotenv==0.* +requests==2.* diff --git a/vision-zero b/vision-zero index cb708deddc..24e6ff9545 100755 --- a/vision-zero +++ b/vision-zero @@ -1,14 +1,16 @@ #!/usr/bin/env python - -import time import argparse -import subprocess +from datetime import date, datetime import os -import shtab import shutil -from datetime import date, datetime -import pprint +import subprocess +import time + from dotenv import load_dotenv +import requests +import shtab + +GRAPHQL_ENGINE_PORT = 8084 # check if the .env file exists if not os.path.exists(".env"): @@ -54,14 +56,34 @@ def checkDockerComposeAvailability(): exit() +def wait_for_hasura(max_attempts=5, delay=2): + """Wait for Hasura to be ready by polling health endpoint""" + for attempt in range(max_attempts): + print("⏳ Waiting for GraphQL Engine to be ready...") + try: + response = requests.get( + f"http://localhost:{GRAPHQL_ENGINE_PORT}/healthz", timeout=1 + ) + if response.status_code == 200: + print("✅ GraphQL Engine is ready!") + return True + except requests.exceptions.RequestException: + pass + time.sleep(delay) + + print("❌ GraphQL Engine failed to become ready") + return False + + def doCommand(args): + apply_migrations = args.apply if args.command == "build": buildTools() - elif args.command == "local-stack-up": + elif args.command == "up": dbUp() - graphqlEngineUp() + graphqlEngineUp(apply_migrations) cr3UserApiUp() - elif args.command == "local-stack-down": + elif args.command == "down": cr3UserApiDown() graphqlEngineDown() dbDown() @@ -71,7 +93,7 @@ def doCommand(args): dbDown() elif args.command == "graphql-engine-up": dbUp() - graphqlEngineUp() + graphqlEngineUp(apply_migrations) elif args.command == "graphql-engine-down": graphqlEngineDown() elif args.command == "api-up": @@ -87,7 +109,7 @@ def doCommand(args): toolsShell() elif args.command == "stop": stop() - elif args.command == "replicate-db": + elif args.command == "replicate": dbUp() replicateDb(args) elif args.command == "dump-local-db": @@ -232,7 +254,7 @@ def replicateDb(args): print(f"Error output: {result.stderr}") exit(1) - graphqlEngineUp() + graphqlEngineUp(args.apply) def dumpLocalDb(): @@ -317,7 +339,6 @@ def dbUp(): print(f"Error output: {result.stderr}") exit(1) subprocess.run(["docker", "ps"]) - time.sleep(5) def dbDown(): @@ -333,10 +354,9 @@ def dbDown(): ] ) subprocess.run(["docker", "ps"]) - time.sleep(5) -def graphqlEngineUp(): +def graphqlEngineUp(apply_migrations=False): print("🚀 Starting GraphQL Engine") subprocess.run( docker_compose_invocation @@ -348,6 +368,24 @@ def graphqlEngineUp(): "graphql-engine", ] ) + + if apply_migrations: + if wait_for_hasura(): + subprocess.call( + [ + "hasura", + "migrate", + "apply", + "--all-databases", + "--disable-interactive", + ], + cwd="./database", + ) + subprocess.call( + ["hasura", "metadata", "apply"], + cwd="./database", + ) + subprocess.run(["docker", "ps"]) print("🚀 GraphQL Engine started.") print("You can run './vision-zero hasura-console' to open the console.") @@ -449,8 +487,8 @@ def get_main_parser(): "command", choices=[ "build", - "local-stack-up", - "local-stack-down", + "up", + "down", "db-up", "db-down", "api-up", @@ -460,13 +498,18 @@ def get_main_parser(): "stop", "psql", "tools-shell", - "replicate-db", + "replicate", "dump-local-db", "remove-snapshots", "hasura-console", ], ) parser.add_argument("-f", "--filename", required=False) + parser.add_argument( + "--apply", + action="store_true", + help="Apply migrations and metadata after starting", + ) return parser