Merge pull request #3 from mggg/dev

Dev -> Main import to allow testing of client before schema change

.gitignore

@@ -130,3 +130,7 @@ dmypy.json
 
 .DS_Store
 .gerryrc
+
+
+# Ignore all of the backup files that might be floating around
+*.tar

README.md

@@ -3,12 +3,44 @@
 This repository contains the code and deployment configuration for GerryDB's metadata server, which is a simple FastAPI application in front of a PostGIS database. PostGIS is the server's sole external dependency.
 
 ## Running the server
-1. Install dependencies with `poetry install`.
+1. In the root of this repository, install dependencies with `poetry install`.
 2. In the root of this repository, launch a PostGIS server with `docker-compose up -d`.
 3. Connect to the PostGIS server with a Postgres client (port `54320`, username `postgres`, password `dev`).
-4. Initialize a PostGIS database by executing the SQL statement `CREATE DATABASE gerrydb; CREATE EXTENSION postgis;`
-5. Initialize the application schema by running `GERRYDB_DATABASE_URI=postgresql://postgres:dev@localhost:54320/gerrydb python init.py`. Save the generated API key.
-6. Run the application server with `uvicorn gerrydb_meta.main:app --reload`.
+4. Initialize a PostGIS database by executing the SQL statement `CREATE DATABASE gerrydb`. Close the connection, make `gerrydb` the defualt database. Reopen the connection, and add geography database objects via `CREATE EXTENSION postgis`.
+```
+export GERRYDB_DATABASE_URI="postgresql://postgres:dev@localhost:54320/gerrydb"
+export GERRYDB_TEST_DATABASE_URI="postgresql://postgres:test@localhost:54321"
+source <wherever you keep your code>/gerrydb-meta/.gerryrc
+```
+Be sure to source the profile again.
+
+6. You need a `$HOME/.gerrydb/config` file that reads something like this (if you want to connect locally
+and not to production). 
+```
+[default]
+host = "localhost:8000"
+key = <API_key>
+```
+If this file does not exist yet, initializing the database should have created it in your home directory. 
+If it already exists, you will be given the option to overwrite it, or leave it.
+If you leave it, you may need to manually edit it.
+
+7. Run the application server with `uvicorn gerrydb_meta.main:app --reload`.
+
+## Saving the database
+To save the current database into a file called `dump.tar`, run the following.
+`pg_dump -U postgres -h localhost -p 54320 -Ft gerrydb > dump.tar`.
+Chris has set up a `pgd` alias.
+
+## Reloading the database
+To load a previous version of the database from a file called `dump.tar`, run the following.
+`pg_restore -U postgres -h localhost -p 54320 -d gerrydb -c -Ft dump.tar`
+Chris has set up a `pgr` alias.
+
+## Deleting the database
+If you want to clear your database and start over, delete the relevant docker containers and volumes, then follow the steps above.
+Note that your profile will still have your old API keys stored in it, but your most recent one
+will be appended to the end.
 
 ## Running tests
 The Docker Compose manifest for this project spins up two PostGIS instances: one for persisting data for long-term local development, and one for ephemeral use by unit tests. The test server is exposed on port 54321; the username is `postgres`, and the password is `test`. To run the test suite, set up the app server as described above, initialize the test database by repeating step 4 (`CREATE DATABASE gerrydb; CREATE EXTENSION postgis;`), and execute the test suite with

TODO.md

@@ -4,13 +4,16 @@ Basic CRUD tests =
  * simple write/read
  * simple error handling
 
-[ ] Basic CRUD tests
-    [ ] Columns
-    [ ] Column sets
-    [ ] Geographic layers
-    [ ] ...add more here...
 
 # Features (ordered, short-term)
+
+
+# Small improvements 
+[ ] Find locations where path errors can occur and raise the GerryPathError rather than
+    the current ValueError
+[ ] Fix the logging on PUT calls
+
+
 Expose:
 [x] GeoImport / GeoLayer
 [ ] Geography (with versioning)

alembic/versions/3e14966c308e_add_view_render_table.py

@@ -5,6 +5,7 @@
 Create Date: 2023-04-17 20:40:21.434928
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

alembic/versions/6898afa765ca_create_view_schema.py

@@ -5,6 +5,7 @@
 Create Date: 2023-03-21 16:54:56.498038
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

alembic/versions/7367a058533d_create_authn_authz.py

@@ -5,6 +5,7 @@
 Create Date: 2023-03-21 17:25:22.277920
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

alembic/versions/7e83d298e241_create_ensembles.py

@@ -5,6 +5,7 @@
 Create Date: 2023-03-23 17:02:22.588999
 
 """
+
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 

alembic/versions/88f266906828_create_plans.py

@@ -5,6 +5,7 @@
 Create Date: 2023-03-23 16:57:10.921907
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

alembic/versions/8dd630f55d05_create_tabular_schema.py

@@ -5,6 +5,7 @@
 Create Date: 2023-03-22 14:47:10.447269
 
 """
+
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
@@ -172,7 +173,7 @@ def upgrade() -> None:
     )
     op.create_table(
         "column_value",
-        sa.Column("val_id", sa.Integer(), nullable=False),
+        sa.Column("val_id", sa.BigInteger(), nullable=False),
         sa.Column("col_id", sa.Integer(), nullable=False),
         sa.Column("geo_id", sa.Integer(), nullable=False),
         sa.Column("meta_id", sa.Integer(), nullable=False),

alembic/versions/96d3750366ff_add_set_version_id_to_view.py

@@ -5,6 +5,7 @@
 Create Date: 2023-04-10 15:06:09.833837
 
 """
+
 from sqlalchemy import Column, ForeignKey, Integer
 
 from alembic import op

alembic/versions/a833eac3e58d_add_num_geos_to_view.py

@@ -5,6 +5,7 @@
 Create Date: 2023-04-11 10:28:48.058441
 
 """
+
 from sqlalchemy import Column, ForeignKey, Integer
 
 from alembic import op

alembic/versions/abd59c616667_create_graphs.py

@@ -5,6 +5,7 @@
 Create Date: 2023-03-23 16:59:16.924503
 
 """
+
 import sqlalchemy as sa
 from sqlalchemy import Column, ForeignKey, Integer
 from sqlalchemy.dialects import postgresql

alembic/versions/d3b6ebaf041f_create_localities.py

@@ -5,6 +5,7 @@
 Create Date: 2023-03-22 14:36:48.946268
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op

alembic/versions/f87a37bcd66a_add_index_to_geoversion.py

@@ -5,6 +5,7 @@
 Create Date: 2023-04-05 21:29:46.680630
 
 """
+
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/f92609b1a9bd_create_geographic_schema.py

@@ -5,6 +5,7 @@
 Create Date: 2023-03-22 14:05:07.271553
 
 """
+
 import geoalchemy2
 import sqlalchemy as sa
 

docker-compose.yml

@@ -7,13 +7,36 @@ services:
     environment:
       POSTGRES_PASSWORD: dev
     volumes:
-      - db:/var/lib/postgresql
+      - db_data:/var/lib/postgresql/data
+    deploy:
+      # Do not touch the memory limits or TX will destroy you
+      resources:
+        limits:
+          memory: 16g
+        reservations:
+          memory: 8g
+    shm_size: '16gb'
+
   test-db:
     image: postgis/postgis
     ports:
       - "54321:5432"
     environment:
       POSTGRES_PASSWORD: test
+    volumes:
+      - test_db_data:/var/lib/postgresql/data
+    deploy:
+      resources:
+        limits:
+          memory: 16g
+        reservations:
+          memory: 8g
+    shm_size: '16gb'
+
 volumes:
-  db:
+  db_data:
+    external: true
+  test_db_data:
+    external: false
+
 

gerrydb_meta/admin.py

@@ -1,4 +1,5 @@
 """Administration tools for GerryDB."""
+
 import csv
 import logging
 import os
@@ -69,32 +70,35 @@ class GerryAdmin:
 
     session: SessionType
 
-    def user_create(self, email: str, name: str) -> User:
-        """Returns a new user."""
+    def user_create(self, email: str, name: str, super_user: bool = False) -> User:
+        """Returns a new user.
+        If `super_user`, grants user global privileges."""
         user = User(email=email, name=name)
         log.info("Created new user: %s", user)
         self.session.add(user)
         self.session.flush()
         self.session.refresh(user)
 
-        # TODO: don't grant broad privileges by default!
-        # grant_scope(self.session, user, ScopeType.ALL, namespace_group=None)
-        # grant_scope(
-        #    self.session, user, ScopeType.ALL, namespace_group=NamespaceGroup.ALL
-        # )
-
-        grant_scope(
-            self.session,
-            user,
-            ScopeType.NAMESPACE_READ,
-            namespace_group=NamespaceGroup.PUBLIC,
-        )
-        grant_scope(
-            self.session,
-            user,
-            ScopeType.NAMESPACE_WRITE_DERIVED,
-            namespace_group=NamespaceGroup.PUBLIC,
-        )
+        if super_user:
+            # global privileges
+            grant_scope(self.session, user, ScopeType.ALL, namespace_group=None)
+            grant_scope(
+                self.session, user, ScopeType.ALL, namespace_group=NamespaceGroup.ALL
+            )
+
+        else:
+            grant_scope(
+                self.session,
+                user,
+                ScopeType.NAMESPACE_READ,
+                namespace_group=NamespaceGroup.PUBLIC,
+            )
+            grant_scope(
+                self.session,
+                user,
+                ScopeType.NAMESPACE_WRITE_DERIVED,
+                namespace_group=NamespaceGroup.PUBLIC,
+            )
 
         return user
 

gerrydb_meta/api/__init__.py

@@ -1,4 +1,5 @@
 """Base configuration and routing for Gerry API endpoints."""
+
 from fastapi import APIRouter
 
 from gerrydb_meta import crud, schemas
@@ -8,6 +9,7 @@
     geo_set,
     geography,
     graph,
+    list_geo,
     locality,
     namespace,
     obj_meta,
@@ -22,6 +24,7 @@
 api_router.include_router(namespace.router, prefix="/namespaces", tags=["namespaces"])
 api_router.include_router(obj_meta.router, prefix="/meta", tags=["meta"])
 api_router.include_router(geography.router, prefix="/geographies", tags=["geographies"])
+api_router.include_router(list_geo.router, prefix="/__list_geo", tags=["list_geo"])
 api_router.include_router(
     geo_import.router, prefix="/geo-imports", tags=["geo-imports"]
 )

gerrydb_meta/api/base.py

@@ -1,4 +1,5 @@
 """Generic CR(UD) views and utilities."""
+
 import inspect
 import logging
 from collections import defaultdict
@@ -143,36 +144,47 @@ def from_resource_paths(
         HTTPException: On parsing failure, authorization failure, or lookup failure.
     """
     # Break paths into (object, namespace, path) form.
+
     parsed_paths = []
     for path in paths:
-        parts = path.strip().lower().split("/")
-        if len(parts) < 4:
+        path = normalize_path(path, case_sensitive_uid="geographies" in path)
+        parts = path.strip().split("/")
+        if len(parts) < 3:
             raise HTTPException(
                 status_code=HTTPStatus.BAD_REQUEST,
                 detail=(
                     f'Bad resource path "{path}": must have form '
                     "/<resource>/<namespace>/<path>"
                 ),
             )
-        if parts[1] not in ENDPOINT_TO_CRUD:
+        if parts[0] not in ENDPOINT_TO_CRUD:
             raise HTTPException(
                 status_code=HTTPStatus.NOT_FOUND,
-                detail=f'Unknown resource "{parts[1]}".',
+                detail=f'Unknown resource "{parts[0]}".',
             )
 
-        parsed_paths.append((parts[1], parts[2], normalize_path("/".join(parts[3:]))))
+        parsed_paths.append(tuple(parts))
 
     # Check for duplicates, which usually violate uniqueness constraints
     # somewhere down the line.
     if len(set(parsed_paths)) < len(parsed_paths):
+        dup_paths = [path for path in parsed_paths if parsed_paths.count(path) > 1]
         raise HTTPException(
             status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
-            detail="Duplicate resource paths found.",
+            detail=f"Duplicate resource paths found {dup_paths}",
         )
 
     # Verify that the user has read access in all namespaces
     # the objects are in.
-    namespaces = {namespace for _, namespace, _ in parsed_paths}
+    try:
+        namespaces = {namespace for _, namespace, _ in parsed_paths}
+    except Exception as e:
+        bad_paths = [path for path in parsed_paths if len(path) != 3]
+        raise ValueError(
+            f"Failed to parse paths: {['/'.join(path) for path in bad_paths]}. "
+            "Paths must verify the form '/<resource>/<namespace>/<path>'."
+        ) from e
+
     namespace_objs = {}
     for namespace in namespaces:
         namespace_obj = crud.namespace.get(db=db, path=namespace)
@@ -292,9 +304,11 @@ def geos_from_paths(
     """
     return from_resource_paths(
         paths=[
-            f"/geographies{path}"
-            if path.startswith("/")
-            else f"/geographies/{namespace}/{path}"
+            (
+                f"/geographies{path}"
+                if path.startswith("/")
+                else f"/geographies/{namespace}/{path}"
+            )
             for path in paths
         ],
         db=db,

gerrydb_meta/api/column_value.py

@@ -1,4 +1,5 @@
 """API operations for manipulating column values."""
+
 from http import HTTPStatus
 
 from fastapi import APIRouter, Depends, HTTPException

gerrydb_meta/api/deps.py

@@ -1,4 +1,5 @@
 """Database and authorization dependencies for GerryDB endpoints."""
+
 import os
 import re
 import threading