Skip to content

chore: post global cleanup #270

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

chore: post global cleanup #270

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f685b95
feat: use RGB_LAB table from DB
garobrik Jul 23, 2025
d5e711e
feat: remove unused code, format, run format in CI
garobrik Jul 24, 2025
025f6f5
feat: improve DB restore script
garobrik Jul 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,4 @@ prof/
Data/
bulk_test_result*

get-pip.py*
get-pip.py*
6 changes: 3 additions & 3 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ setup-git-hooks:

lint:
ruff check soil_id
ruff format soil_id --diff

format:
ruff format soil_id
Expand Down Expand Up @@ -73,7 +74,6 @@ process_bulk_test_results_legacy:
python -m soil_id.tests.legacy.process_bulk_test_results $(RESULTS_FILE)

# Donwload Munsell CSV, SHX, SHP, SBX, SBN, PRJ, DBF
# 1tN23iVe6X1fcomcfveVp4w3Pwd0HJuTe: LandPKS_munsell_rgb_lab.csv
# 1WUa9e3vTWPi6G8h4OI3CBUZP5y7tf1Li: gsmsoilmu_a_us.shx
# 1l9MxC0xENGmI_NmGlBY74EtlD6SZid_a: gsmsoilmu_a_us.shp
# 1asGnnqe0zI2v8xuOszlsNmZkOSl7cJ2n: gsmsoilmu_a_us.sbx
Expand All @@ -84,7 +84,6 @@ process_bulk_test_results_legacy:
download-soil-data:
mkdir -p Data
cd Data; \
gdown 1tN23iVe6X1fcomcfveVp4w3Pwd0HJuTe; \
gdown 1WUa9e3vTWPi6G8h4OI3CBUZP5y7tf1Li; \
gdown 1l9MxC0xENGmI_NmGlBY74EtlD6SZid_a; \
gdown 1asGnnqe0zI2v8xuOszlsNmZkOSl7cJ2n; \
Expand Down Expand Up @@ -119,6 +118,7 @@ dump_soil_id_db:
pg_dump --format=custom $(DATABASE_URL) -t hwsd2_segment -t hwsd2_data -t landpks_munsell_rgb_lab -t normdist1 -t normdist2 -t wise_soil_data -t wrb2006_to_fao90 -t wrb_fao90_desc -f $(DATABASE_DUMP_FILE)

restore_soil_id_db:
pg_restore --dbname=$(DATABASE_URL) --single-transaction --clean --if-exists --no-owner $(DATABASE_DUMP_FILE)
psql $(DATABASE_URL) -c "CREATE EXTENSION IF NOT EXISTS postgis;"
pg_restore --dbname=$(DATABASE_URL) --clean --if-exists --no-owner --verbose $(DATABASE_DUMP_FILE)
psql $(DATABASE_URL) -c "CLUSTER hwsd2_segment USING hwsd2_segment_shape_idx;"
psql $(DATABASE_URL) -c "ANALYZE;"
15 changes: 1 addition & 14 deletions soil_id/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,33 +13,20 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see https://www.gnu.org/licenses/.
import os
import tempfile

from platformdirs import user_cache_dir

DATA_PATH = os.environ.get("DATA_PATH", "Data")

# Numpy seeding
RANDOM_SEED = os.environ.get("RANDOM_SEED", 19)

# Output
APP_NAME = os.environ.get("APP_NAME", "org.terraso.soilid")
TEMP_DIR = tempfile.TemporaryDirectory()
CACHE_DIR = user_cache_dir(APP_NAME)
OUTPUT_PATH = TEMP_DIR.name
SOIL_ID_RANK_PATH = f"{OUTPUT_PATH}/soil_id_rank.csv"
SOIL_ID_PROB_PATH = f"{OUTPUT_PATH}/soil_id_cond_prob.csv"
REQUESTS_CACHE_PATH = f"{CACHE_DIR}/requests_cache"

# Determines if in/out of US
US_AREA_PATH = f"{DATA_PATH}/SoilID_US_Areas.shz"

# US Soil ID
STATSGO_PATH = f"{DATA_PATH}/gsmsoilmu_a_us.shp"
MUNSELL_RGB_LAB_PATH = f"{DATA_PATH}/LandPKS_munsell_rgb_lab.csv"

# Database
DB_NAME = os.environ.get("DB_NAME", "terraso_backend")
DB_NAME = os.environ.get("DB_NAME", "soil_id")
DB_HOST = os.environ.get("DB_HOST")
DB_USERNAME = os.environ.get("DB_USERNAME")
DB_PASSWORD = os.environ.get("DB_PASSWORD")
Loading