feat: final aggregation of global code

.github/workflows/build.yml

@@ -6,8 +6,6 @@ on:
     branches:
       - main
   pull_request:
-    branches:
-      - main
     types:
       - opened
       - edited
@@ -83,5 +81,13 @@ jobs:
         if: ${{ hashFiles('Data/*') == '' }}
         run: make download-soil-data
 
+      - name: Start soil id DB
+        run: docker compose up -d
+
       - name: Run tests
+        env:
+          DB_NAME: soil_id
+          DB_HOST: localhost
+          DB_USERNAME: postgres
+          DB_PASSWORD: postgres
         run: make test

.gitignore

@@ -27,5 +27,6 @@ prof/
 
 # reference and output files
 Data/
+bulk_test_result*
 
 get-pip.py*

Dockerfile

@@ -0,0 +1,51 @@
+# Stage 1: Build database with preloaded data
+FROM postgis/postgis:16-3.5 as builder
+
+ARG POSTGRES_DB=soil_id
+ARG POSTGRES_USER=postgres
+ARG POSTGRES_PASSWORD=postgres
+ARG DATABASE_DUMP_FILE=Data/soil_id_db.dump
+
+ENV POSTGRES_DB=${POSTGRES_DB}
+ENV POSTGRES_USER=${POSTGRES_USER}
+ENV POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+
+# Copy dump into builder stage
+COPY ${DATABASE_DUMP_FILE} /tmp/soil_id_db.dump
+
+# Initialize database and preload it
+RUN mkdir -p /var/lib/postgresql/data \
+ && chown -R ${POSTGRES_USER}:${POSTGRES_USER} /var/lib/postgresql
+
+USER ${POSTGRES_USER}
+
+# Initialize database system
+RUN /usr/lib/postgresql/16/bin/initdb -D /var/lib/postgresql/data
+
+# Start database in background, preload dump, shut it down
+RUN pg_ctl -D /var/lib/postgresql/data -o "-c listen_addresses=''" -w start \
+  && createdb -U ${POSTGRES_USER} ${POSTGRES_DB} \
+  && psql -U ${POSTGRES_USER} -d ${POSTGRES_DB} -c "CREATE EXTENSION postgis;" \
+  && pg_restore -U ${POSTGRES_USER} -d ${POSTGRES_DB} /tmp/soil_id_db.dump \
+  && psql -U ${POSTGRES_USER} -d ${POSTGRES_DB} -c "CLUSTER hwsd2_segment USING hwsd2_segment_shape_idx;" \
+  && psql -U ${POSTGRES_USER} -d ${POSTGRES_DB} -c "ANALYZE;" \
+  && pg_ctl -D /var/lib/postgresql/data -m fast -w stop
+
+# Stage 2: Final image with loaded data
+FROM postgis/postgis:16-3.5
+
+COPY --from=builder /var/lib/postgresql/data /var/lib/postgresql/data
+
+# Create a Docker-friendly pg_hba.conf
+USER root
+RUN echo "local   all             all                                     trust" > /var/lib/postgresql/data/pg_hba.conf \
+ && echo "host    all             all             127.0.0.1/32            trust" >> /var/lib/postgresql/data/pg_hba.conf \
+ && echo "host    all             all             ::1/128                 trust" >> /var/lib/postgresql/data/pg_hba.conf \
+ && echo "host    all             all             0.0.0.0/0               trust" >> /var/lib/postgresql/data/pg_hba.conf \
+ && chown ${POSTGRES_USER}:${POSTGRES_USER} /var/lib/postgresql/data/pg_hba.conf
+
+ENV POSTGRES_DB=${POSTGRES_DB}
+ENV POSTGRES_USER={POSTGRES_USER}
+ENV POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+
+USER ${POSTGRES_USER}

Makefile

@@ -54,13 +54,33 @@ graphs:
 	gprof2dot -f pstats  prof/test_soil_location.prof | dot -Tsvg -o prof/test_soil_location.svg
 	flameprof prof/test_soil_location.prof > prof/test_soil_location_flame.svg
 
-generate_bulk_test_results:
+generate_bulk_test_results_us:
 	python -m soil_id.tests.us.generate_bulk_test_results
 
-process_bulk_test_results:
+process_bulk_test_results_us:
 	python -m soil_id.tests.us.process_bulk_test_results $(RESULTS_FILE)
 
+generate_bulk_test_results_global:
+	python -m soil_id.tests.global.generate_bulk_test_results
+
+process_bulk_test_results_global:
+	python -m soil_id.tests.global.process_bulk_test_results $(RESULTS_FILE)
+
+generate_bulk_test_results_legacy:
+	python -m soil_id.tests.legacy.generate_bulk_test_results
+
+process_bulk_test_results_legacy:
+	python -m soil_id.tests.legacy.process_bulk_test_results $(RESULTS_FILE)
+
 # Donwload Munsell CSV, SHX, SHP, SBX, SBN, PRJ, DBF
+# 1tN23iVe6X1fcomcfveVp4w3Pwd0HJuTe: LandPKS_munsell_rgb_lab.csv
+# 1WUa9e3vTWPi6G8h4OI3CBUZP5y7tf1Li: gsmsoilmu_a_us.shx
+# 1l9MxC0xENGmI_NmGlBY74EtlD6SZid_a: gsmsoilmu_a_us.shp
+# 1asGnnqe0zI2v8xuOszlsNmZkOSl7cJ2n: gsmsoilmu_a_us.sbx
+# 185Qjb9pJJn4AzOissiTz283tINrDqgI0: gsmsoilmu_a_us.sbn
+# 1P3xl1YRlfcMjfO_4PM39tkrrlL3hoLzv: gsmsoilmu_a_us.prj
+# 1K0GkqxhZiVUND6yfFmaI7tYanLktekyp: gsmsoilmu_a_us.dbf
+# 1z7foFFHv_mTsuxMYnfOQRvXT5LKYlYFN: SoilID_US_Areas.shz
 download-soil-data:
 	mkdir -p Data
 	cd Data; \
@@ -70,4 +90,35 @@ download-soil-data:
 	gdown 1asGnnqe0zI2v8xuOszlsNmZkOSl7cJ2n; \
 	gdown 185Qjb9pJJn4AzOissiTz283tINrDqgI0; \
 	gdown 1P3xl1YRlfcMjfO_4PM39tkrrlL3hoLzv; \
-	gdown 1K0GkqxhZiVUND6yfFmaI7tYanLktekyp \
+	gdown 1K0GkqxhZiVUND6yfFmaI7tYanLktekyp; \
+	gdown 1z7foFFHv_mTsuxMYnfOQRvXT5LKYlYFN \
+
+DATABASE_DUMP_FILE ?= Data/soil_id_db.dump
+DOCKER_IMAGE_TAG ?= ghcr.io/techmatters/soil-id-db:latest
+build_docker_image:
+	@echo "Building to tag $(DOCKER_IMAGE_TAG)"
+	docker build \
+	  --build-arg DATABASE_DUMP_FILE=$(DATABASE_DUMP_FILE) \
+	  -t $(DOCKER_IMAGE_TAG) \
+	  .
+
+push_docker_image:
+	@echo "Pushing tag $(DOCKER_IMAGE_TAG). Make sure to provide a versioned tag in addition to updating latest!"
+	docker push $(DOCKER_IMAGE_TAG)
+
+start_db:
+	docker compose up -d
+
+stop_db:
+	docker compose down
+
+connect_db:
+	docker compose exec db psql -U postgres -d soil_id
+
+dump_soil_id_db:
+	pg_dump --format=custom $(DATABASE_URL)  -t hwsd2_segment -t hwsd2_data -t landpks_munsell_rgb_lab -t normdist1 -t normdist2 -t wise_soil_data -t wrb2006_to_fao90 -t wrb_fao90_desc -f $(DATABASE_DUMP_FILE)
+
+restore_soil_id_db:
+	pg_restore --dbname=$(DATABASE_URL) --single-transaction --clean --if-exists --no-owner $(DATABASE_DUMP_FILE)
+	psql $(DATABASE_URL) -c "CLUSTER hwsd2_segment USING hwsd2_segment_shape_idx;"
+	psql $(DATABASE_URL) -c "ANALYZE;"

docker-compose.yml

@@ -0,0 +1,11 @@
+name: soil_id
+
+services:
+  db:
+    image: ghcr.io/techmatters/soil-id-db:0.2
+    ports: 
+      - "5432:5432"
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: soil_id # default database

soil_id/config.py

@@ -12,42 +12,32 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see https://www.gnu.org/licenses/.
-
 import os
 import tempfile
 
-from dotenv import load_dotenv
 from platformdirs import user_cache_dir
 
-load_dotenv()
-
 DATA_PATH = os.environ.get("DATA_PATH", "Data")
 
 # Numpy seeding
 RANDOM_SEED = os.environ.get("RANDOM_SEED", 19)
 
 # Output
 APP_NAME = os.environ.get("APP_NAME", "org.terraso.soilid")
-TEMP_DIR = tempfile.TemporaryDirectory(delete=False)
+TEMP_DIR = tempfile.TemporaryDirectory()
 CACHE_DIR = user_cache_dir(APP_NAME)
 OUTPUT_PATH = TEMP_DIR.name
 SOIL_ID_RANK_PATH = f"{OUTPUT_PATH}/soil_id_rank.csv"
 SOIL_ID_PROB_PATH = f"{OUTPUT_PATH}/soil_id_cond_prob.csv"
 REQUESTS_CACHE_PATH = f"{CACHE_DIR}/requests_cache"
 
 # Determines if in/out of US
-US_AREA_PATH = f"{DATA_PATH}/SoilID_US_Areas.shp"
+US_AREA_PATH = f"{DATA_PATH}/SoilID_US_Areas.shz"
 
 # US Soil ID
 STATSGO_PATH = f"{DATA_PATH}/gsmsoilmu_a_us.shp"
 MUNSELL_RGB_LAB_PATH = f"{DATA_PATH}/LandPKS_munsell_rgb_lab.csv"
 
-# Global Soil ID
-HWSD_PATH = f"{DATA_PATH}/HWSD_global_noWater_no_country.shp"
-WISE_PATH = f"{DATA_PATH}/wise30sec_poly_simp_soil.shp"
-NORM_DIST_1_PATH = f"{DATA_PATH}/NormDist1.csv"
-NORM_DIST_2_PATH = f"{DATA_PATH}/NormDist2.csv"
-
 # Database
 DB_NAME = os.environ.get("DB_NAME", "terraso_backend")
 DB_HOST = os.environ.get("DB_HOST")