feat: run global test in CI

Author: garobrik

.github/workflows/build.yml

@@ -81,5 +81,13 @@ jobs:
         if: ${{ hashFiles('Data/*') == '' }}
         run: make download-soil-data
 
+      - name: Start soil id DB
+        run: docker compose up -d
+
       - name: Run tests
+        env:
+          DB_NAME: soil_id
+          DB_HOST: localhost
+          DB_USERNAME: postgres
+          DB_PASSWORD: postgres
         run: make test

Dockerfile

@@ -0,0 +1,44 @@
+# Stage 1: Build database with preloaded data
+FROM postgis/postgis:16-3.5 as builder
+
+ENV POSTGRES_DB=soil_id
+ENV POSTGRES_USER=postgres
+ENV POSTGRES_PASSWORD=postgres
+
+# Copy dump into builder stage
+COPY Data/soil_id_db.dump /tmp/soil_id_db.dump
+
+# Initialize database and preload it
+RUN mkdir -p /var/lib/postgresql/data \
+ && chown -R postgres:postgres /var/lib/postgresql
+
+USER postgres
+
+# Initialize database system
+RUN /usr/lib/postgresql/16/bin/initdb -D /var/lib/postgresql/data
+
+# Start database in background, preload dump, shut it down
+RUN pg_ctl -D /var/lib/postgresql/data -o "-c listen_addresses=''" -w start \
+ && createdb -U postgres soil_id \
+ && psql -U postgres -d $POSTGRES_DB -c "CREATE EXTENSION postgis;" \
+ && pg_restore -U postgres -d soil_id /tmp/soil_id_db.dump \
+ && pg_ctl -D /var/lib/postgresql/data -m fast -w stop
+
+# Stage 2: Final image with loaded data
+FROM postgis/postgis:16-3.5
+
+COPY --from=builder /var/lib/postgresql/data /var/lib/postgresql/data
+
+# Create a Docker-friendly pg_hba.conf
+USER root
+RUN echo "local   all             all                                     trust" > /var/lib/postgresql/data/pg_hba.conf \
+ && echo "host    all             all             127.0.0.1/32            trust" >> /var/lib/postgresql/data/pg_hba.conf \
+ && echo "host    all             all             ::1/128                 trust" >> /var/lib/postgresql/data/pg_hba.conf \
+ && echo "host    all             all             0.0.0.0/0               trust" >> /var/lib/postgresql/data/pg_hba.conf \
+ && chown postgres:postgres /var/lib/postgresql/data/pg_hba.conf
+
+ENV POSTGRES_DB=soil_id
+ENV POSTGRES_USER=postgres
+ENV POSTGRES_PASSWORD=postgres
+
+USER postgres

docker-compose.yml

@@ -1,16 +1,11 @@
-name: soil_data
+name: soil_id
 
 services:
   db:
-    image: postgis/postgis:16-3.5
+    image: ghcr.io/techmatters/soil-id-db:0.2
     ports: 
       - "5432:5432"
     environment:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
-      POSTGRES_DB: soil_data # default database
-    volumes:
-      - pg_data:/var/lib/postgresql/data
-
-volumes:
-  pg_data:
+      POSTGRES_DB: soil_id # default database

soil_id/global_soil.py

@@ -1260,7 +1260,7 @@ def normalize(arr):
 ##################################################################################################
 
 
-def sg_list(lon, lat):
+def sg_list(connection, lon, lat):
     """
     Query the SoilGrids API (via get_soilgrids_property_data) and post-process
     the returned JSON into a structured dictionary that includes:
@@ -1395,7 +1395,7 @@ def sg_list(lon, lat):
             sg_tax_prob.sort_values("Prob", ascending=False, inplace=True)
 
             # Merge with descriptive info
-            WRB_Comp_Desc = getSG_descriptions(sg_tax_prob["WRB_tax"].tolist())
+            WRB_Comp_Desc = getSG_descriptions(connection, sg_tax_prob["WRB_tax"].tolist())
             TAXNWRB_pd = pd.merge(sg_tax_prob, WRB_Comp_Desc, on="WRB_tax", how="left")
 
             # Only handle top 3 entries (or fewer if less are returned)

soil_id/tests/global/test_global.py

@@ -18,7 +18,7 @@
 import pytest
 
 from soil_id.db import get_datastore_connection
-from soil_id.global_soil import list_soils_global, rank_soils_global, sg_list
+from soil_id.global_soil import list_soils_global, rank_soils_global
 
 test_locations = [
     {"lon": -1.4631, "lat": 7.3318},
@@ -27,7 +27,6 @@
     {"lat": -10.07856, "lon": 15.107436},
 ]
 
-@pytest.mark.skip
 @pytest.mark.parametrize("location", test_locations)
 def test_soil_location(location):
     with get_datastore_connection() as connection:
@@ -41,10 +40,10 @@ def test_soil_location(location):
             location["lat"],
             list_output_data=list_soils_result,
             soilHorizon=["Loam"],
-            horizonDepth=[15],
+            topDepth=[0],
+            bottomDepth=[15],
             rfvDepth=[20],
             lab_Color=[[41.23035939, 3.623018224, 13.27654356]],
             bedrock=None,
             cracks=None,
         )
-        sg_list(location["lon"], location["lat"])