feat: integrate global with terraso

.github/workflows/build.yml

@@ -6,8 +6,6 @@ on:
     branches:
       - main
   pull_request:
-    branches:
-      - main
     types:
       - opened
       - edited
@@ -83,5 +81,13 @@ jobs:
         if: ${{ hashFiles('Data/*') == '' }}
         run: make download-soil-data
 
+      - name: Start soil id DB
+        run: docker compose up -d
+
       - name: Run tests
+        env:
+          DB_NAME: soil_id
+          DB_HOST: localhost
+          DB_USERNAME: postgres
+          DB_PASSWORD: postgres
         run: make test

Dockerfile

@@ -0,0 +1,44 @@
+# Stage 1: Build database with preloaded data
+FROM postgis/postgis:16-3.5 as builder
+
+ENV POSTGRES_DB=soil_id
+ENV POSTGRES_USER=postgres
+ENV POSTGRES_PASSWORD=postgres
+
+# Copy dump into builder stage
+COPY Data/soil_id_db.dump /tmp/soil_id_db.dump
+
+# Initialize database and preload it
+RUN mkdir -p /var/lib/postgresql/data \
+ && chown -R postgres:postgres /var/lib/postgresql
+
+USER postgres
+
+# Initialize database system
+RUN /usr/lib/postgresql/16/bin/initdb -D /var/lib/postgresql/data
+
+# Start database in background, preload dump, shut it down
+RUN pg_ctl -D /var/lib/postgresql/data -o "-c listen_addresses=''" -w start \
+ && createdb -U postgres soil_id \
+ && psql -U postgres -d $POSTGRES_DB -c "CREATE EXTENSION postgis;" \
+ && pg_restore -U postgres -d soil_id /tmp/soil_id_db.dump \
+ && pg_ctl -D /var/lib/postgresql/data -m fast -w stop
+
+# Stage 2: Final image with loaded data
+FROM postgis/postgis:16-3.5
+
+COPY --from=builder /var/lib/postgresql/data /var/lib/postgresql/data
+
+# Create a Docker-friendly pg_hba.conf
+USER root
+RUN echo "local   all             all                                     trust" > /var/lib/postgresql/data/pg_hba.conf \
+ && echo "host    all             all             127.0.0.1/32            trust" >> /var/lib/postgresql/data/pg_hba.conf \
+ && echo "host    all             all             ::1/128                 trust" >> /var/lib/postgresql/data/pg_hba.conf \
+ && echo "host    all             all             0.0.0.0/0               trust" >> /var/lib/postgresql/data/pg_hba.conf \
+ && chown postgres:postgres /var/lib/postgresql/data/pg_hba.conf
+
+ENV POSTGRES_DB=soil_id
+ENV POSTGRES_USER=postgres
+ENV POSTGRES_PASSWORD=postgres
+
+USER postgres

Makefile

@@ -73,6 +73,14 @@ process_bulk_test_results_legacy:
 	python -m soil_id.tests.legacy.process_bulk_test_results $(RESULTS_FILE)
 
 # Donwload Munsell CSV, SHX, SHP, SBX, SBN, PRJ, DBF
+# 1tN23iVe6X1fcomcfveVp4w3Pwd0HJuTe: LandPKS_munsell_rgb_lab.csv
+# 1WUa9e3vTWPi6G8h4OI3CBUZP5y7tf1Li: gsmsoilmu_a_us.shx
+# 1l9MxC0xENGmI_NmGlBY74EtlD6SZid_a: gsmsoilmu_a_us.shp
+# 1asGnnqe0zI2v8xuOszlsNmZkOSl7cJ2n: gsmsoilmu_a_us.sbx
+# 185Qjb9pJJn4AzOissiTz283tINrDqgI0: gsmsoilmu_a_us.sbn
+# 1P3xl1YRlfcMjfO_4PM39tkrrlL3hoLzv: gsmsoilmu_a_us.prj
+# 1K0GkqxhZiVUND6yfFmaI7tYanLktekyp: gsmsoilmu_a_us.dbf
+# 1z7foFFHv_mTsuxMYnfOQRvXT5LKYlYFN: SoilID_US_Areas.shz
 download-soil-data:
 	mkdir -p Data
 	cd Data; \
@@ -82,4 +90,14 @@ download-soil-data:
 	gdown 1asGnnqe0zI2v8xuOszlsNmZkOSl7cJ2n; \
 	gdown 185Qjb9pJJn4AzOissiTz283tINrDqgI0; \
 	gdown 1P3xl1YRlfcMjfO_4PM39tkrrlL3hoLzv; \
-	gdown 1K0GkqxhZiVUND6yfFmaI7tYanLktekyp \
+	gdown 1K0GkqxhZiVUND6yfFmaI7tYanLktekyp; \
+	gdown 1z7foFFHv_mTsuxMYnfOQRvXT5LKYlYFN \
+
+DOCKER_IMAGE_TAG ?= ghcr.io/techmatters/soil-id-db:latest
+build_docker_image:
+	@echo "Building to tag $(DOCKER_IMAGE_TAG)"
+	docker build -t $(DOCKER_IMAGE_TAG) .
+
+push_docker_image:
+	@echo "Pushing tag $(DOCKER_IMAGE_TAG). Make sure to provide a versioned tag in addition to updating latest!"
+	docker push $(DOCKER_IMAGE_TAG)

docker-compose.yml

@@ -1,16 +1,11 @@
-name: soil_data
+name: soil_id
 
 services:
   db:
-    image: postgis/postgis:16-3.5
+    image: ghcr.io/techmatters/soil-id-db:0.2
     ports: 
       - "5432:5432"
     environment:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
-      POSTGRES_DB: soil_data # default database
-    volumes:
-      - pg_data:/var/lib/postgresql/data
-
-volumes:
-  pg_data:
+      POSTGRES_DB: soil_id # default database

soil_id/config.py

@@ -32,16 +32,12 @@
 REQUESTS_CACHE_PATH = f"{CACHE_DIR}/requests_cache"
 
 # Determines if in/out of US
-US_AREA_PATH = f"{DATA_PATH}/SoilID_US_Areas.shp"
+US_AREA_PATH = f"{DATA_PATH}/SoilID_US_Areas.shz"
 
 # US Soil ID
 STATSGO_PATH = f"{DATA_PATH}/gsmsoilmu_a_us.shp"
 MUNSELL_RGB_LAB_PATH = f"{DATA_PATH}/LandPKS_munsell_rgb_lab.csv"
 
-# Global Soil ID
-HWSD_PATH = f"{DATA_PATH}/HWSD_global_noWater_no_country.shp"
-WISE_PATH = f"{DATA_PATH}/wise30sec_poly_simp_soil.shp"
-
 # Database
 DB_NAME = os.environ.get("DB_NAME", "terraso_backend")
 DB_HOST = os.environ.get("DB_HOST")