Skip to content

Commit eec4fe5

Browse files
frankherefordfrankhereford
authored
Merge pull request #1951 from cityofaustin/frank/view-diff-ci
Add Database View Bot 🤖, now powered by `sqruff`
2 parents 29dfcce + e74d061 commit eec4fe5

21 files changed

Lines changed: 1229 additions & 44 deletions

‎.github/workflows/dbdocs.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,9 @@ jobs:
4646
echo "POSTGRES_DB=vision_zero" >> .env;
4747
echo "POSTGRES_HOST_AUTH_METHOD=trust" >> .env;
4848
49-
docker compose -f docker-compose-github-dbdocs-action.yml up -d postgis;
49+
docker compose -f docker-compose-github-actions.yml up -d postgis;
5050
sleep 10;
51-
docker compose -f docker-compose-github-dbdocs-action.yml up -d graphql-engine;
51+
docker compose -f docker-compose-github-actions.yml up -d graphql-engine;
5252
sleep 10;
5353
cd database;
5454
hasura --skip-update-check --database-name=default migrate apply;

‎.github/workflows/export_database_views.sh‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
#!/usr/bin/env bash
2+
3+
USE_GITHUB_ACTION=false
4+
for arg in "$@"; do
5+
if [[ "$arg" == "--github-action" ]]; then
6+
USE_GITHUB_ACTION=true
7+
break
8+
fi
9+
done
10+
11+
export USE_GITHUB_ACTION
12+
13+
if ! $USE_GITHUB_ACTION; then
14+
SCRIPT_DIR=$(cd -- "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
15+
REPO_ROOT=$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel 2>/dev/null)
16+
CURRENT_DIR=$(pwd -P)
17+
if [[ "$CURRENT_DIR" != "$REPO_ROOT" ]]; then
18+
echo "Error: This script must be run from the repo root when not in GitHub Action mode." >&2
19+
echo " Repo root: $REPO_ROOT" >&2
20+
echo " Current dir: $CURRENT_DIR" >&2
21+
exit 1
22+
fi
23+
# Load DB credentials for local psql (e.g. from .env)
24+
if [[ -f "$REPO_ROOT/.env" ]]; then
25+
set -a
26+
# shellcheck source=/dev/null
27+
source "$REPO_ROOT/.env"
28+
set +a
29+
fi
30+
# Defaults matching docker-compose-github-actions.yml if .env not present
31+
: "${POSTGRES_USER:=visionzero}"
32+
: "${POSTGRES_DB:=vision_zero}"
33+
fi
34+
35+
function run_psql() {
36+
if $USE_GITHUB_ACTION; then
37+
psql "$@"
38+
else
39+
docker compose -f ./docker-compose-github-actions.yml exec postgis psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"
40+
fi
41+
}
42+
43+
function create_view_file() {
44+
local VIEW_NAME=$1
45+
46+
echo "View: $VIEW_NAME"
47+
MOST_RECENT_MIGRATION=$(grep -rl --include=up.sql -E "CREATE (OR REPLACE )?VIEW (\"?public\"?.)?\"?$VIEW_NAME\"?" database/migrations/default | sort -V | tail -n 1)
48+
echo "MOST_RECENT_MIGRATION: $MOST_RECENT_MIGRATION"
49+
50+
# Create the view file with header
51+
echo "-- Most recent migration: $MOST_RECENT_MIGRATION" > database/views/$VIEW_NAME.sql
52+
echo "" >> database/views/$VIEW_NAME.sql
53+
54+
# Query the view definition and append to the file
55+
run_psql -v ON_ERROR_STOP=1 -A -t -c "SELECT 'CREATE OR REPLACE VIEW ' || '$VIEW_NAME' || ' AS ' || pg_get_viewdef('$VIEW_NAME'::regclass, true);" >> database/views/$VIEW_NAME.sql
56+
}
57+
58+
# Materialized views do not support CREATE OR REPLACE, so we emit
59+
# DROP MATERIALIZED VIEW IF EXISTS ...
60+
# CREATE MATERIALIZED VIEW ... AS ...
61+
function create_materialized_view_file() {
62+
local VIEW_NAME=$1
63+
64+
echo "Materialized view: $VIEW_NAME"
65+
MOST_RECENT_MIGRATION=$(grep -rl --include=up.sql -E "CREATE (OR REPLACE )?MATERIALIZED VIEW( IF NOT EXISTS)? (\"?public\"?.)?\"?$VIEW_NAME\"?" database/migrations/default | sort -V | tail -n 1)
66+
echo "MOST_RECENT_MIGRATION: $MOST_RECENT_MIGRATION"
67+
68+
mkdir -p database/views/materialized
69+
70+
# Create the materialized view file with header
71+
echo "-- Most recent migration: $MOST_RECENT_MIGRATION" > database/views/materialized/$VIEW_NAME.sql
72+
echo "" >> database/views/materialized/$VIEW_NAME.sql
73+
echo "DROP MATERIALIZED VIEW IF EXISTS $VIEW_NAME;" >> database/views/materialized/$VIEW_NAME.sql
74+
echo "" >> database/views/materialized/$VIEW_NAME.sql
75+
76+
# Query the materialized view definition and append to the file
77+
run_psql -v ON_ERROR_STOP=1 -A -t -c "SELECT 'CREATE MATERIALIZED VIEW ' || '$VIEW_NAME' || ' AS ' || pg_get_viewdef('$VIEW_NAME'::regclass, true);" >> database/views/materialized/$VIEW_NAME.sql
78+
}
79+
80+
# Export the function
81+
export -f run_psql
82+
export -f create_view_file
83+
export -f create_materialized_view_file
84+
85+
function populate_views() {
86+
mkdir -p database/views
87+
run_psql -v ON_ERROR_STOP=1 -A -t -c "SELECT table_name FROM information_schema.views WHERE table_schema = 'public' ORDER BY table_name;" | \
88+
grep -v -E '^geo[a-zA-Z]+y_columns$' | \
89+
xargs -I {} bash -c "create_view_file '{}'"
90+
}
91+
92+
function populate_materialized_views() {
93+
mkdir -p database/views/materialized
94+
run_psql -v ON_ERROR_STOP=1 -A -t -c "SELECT matviewname FROM pg_matviews WHERE schemaname = 'public' ORDER BY matviewname;" | \
95+
grep -v -E '^geo[a-zA-Z]+y_columns$' | \
96+
xargs -I {} bash -c "create_materialized_view_file '{}'"
97+
}
98+
99+
populate_views
100+
populate_materialized_views

‎.github/workflows/export_database_views.yml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: Export Vision Zero Views to SQL
2+
3+
on:
4+
push:
5+
paths:
6+
- database/migrations/default/**
7+
- .github/workflows/export_database_views.yml
8+
- .github/workflows/export_database_views.sh
9+
- .sqruff
10+
branches-ignore:
11+
- "main"
12+
- "production"
13+
14+
jobs:
15+
publish-views:
16+
runs-on: ubuntu-latest
17+
permissions:
18+
contents: write
19+
20+
steps:
21+
- name: Checkout code
22+
uses: actions/checkout@v4
23+
with:
24+
ref: ${{ github.ref }}
25+
26+
- name: Install PostgreSQL client
27+
run: sudo apt-get update && sudo apt-get install -y postgresql-client
28+
29+
- name: Set PostgreSQL environment variables
30+
run: |
31+
echo "PGHOST=localhost" >> $GITHUB_ENV
32+
echo "PGPORT=5432" >> $GITHUB_ENV
33+
echo "PGDATABASE=vision_zero" >> $GITHUB_ENV
34+
echo "PGUSER=visionzero" >> $GITHUB_ENV
35+
echo "PGPASSWORD=visionzero" >> $GITHUB_ENV
36+
37+
- name: Install Hasura CLI
38+
run: |
39+
curl -L https://github.com/hasura/graphql-engine/raw/stable/cli/get.sh | bash
40+
41+
- name: "Spin-up the database and graphql-engine containers"
42+
run: |
43+
export BRANCH_NAME="${GITHUB_REF_NAME}"
44+
echo "SHA: ${GITHUB_SHA}"
45+
echo "ACTION/BRANCH_NAME: ${BRANCH_NAME}"
46+
echo "GR: ${GITHUB_REF}"
47+
echo "PWD: $(pwd)"
48+
docker compose -f docker-compose-github-actions.yml up -d postgis;
49+
sleep 10;
50+
docker compose -f docker-compose-github-actions.yml up -d graphql-engine;
51+
sleep 10;
52+
cd database;
53+
hasura --skip-update-check --database-name default migrate apply;
54+
hasura metadata apply --skip-update-check;
55+
56+
- name: Generate CREATE VIEW + MATERIALIZED VIEW statements
57+
run: $(pwd)/.github/workflows/export_database_views.sh --github-action
58+
59+
- name: Install sqruff
60+
uses: quarylabs/install-sqruff-cli-action@7805329baf7cf340e849e254cddc782f08f2d36c
61+
- name: Format SQL with sqruff
62+
run: |
63+
sqruff fix --format human database/views
64+
continue-on-error: true
65+
66+
- name: Set Git Config
67+
run: |
68+
git config --local user.email "visionzero@view.bot"
69+
git config --local user.name "Vision Zero View Bot"
70+
71+
- name: Commit and push changes
72+
run: |
73+
export BRANCH_NAME="${GITHUB_REF_NAME}"
74+
git add database/views
75+
git commit -m "🤖 Export database views for $BRANCH_NAME" || echo "No changes to commit"
76+
git push origin HEAD || echo "No changes to push"

‎.gitignore‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,8 +88,8 @@ etl/app/**/*.json
8888
env
8989

9090
# vim
91-
[._]*.s[a-v][a-z]
92-
[._]*.sw[a-p]
91+
.*.s[a-v][a-z]
92+
.*.sw[a-p]
9393

9494
# Zappa API deployment secrets
9595
zappa_settings.json

‎.sqruff‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[sqruff]
2+
dialect = postgres
3+
max_line_length = 100
4+
5+
# NB: The spelling used for capitalisation is using UK English. An 'es' not a 'zee'.
6+
rules = ambiguous,capitalisation,convention,layout,references
7+
8+
# This causes all the AS aliases to line up.
9+
[sqruff:layout:type:alias_expression]
10+
spacing_before = align
11+
align_within = select_clause
12+
align_scope = bracketed

0 commit comments

Comments
 (0)