Merge remote-tracking branch 'upstream/main' into main2

.github/workflows/db_export.yaml

@@ -44,8 +44,8 @@ jobs:
         env:
           HDX_SITE: ${{ vars.HDX_SITE }}
           HDX_KEY: ${{ secrets.HDX_BOT_SCRAPERS_API_TOKEN }}
-          PREPREFIX: ${{ secrets.HDX_PIPELINE_PREPREFIX }}
-          USER_AGENT: ${{ secrets.USER_AGENT }}
+          PREPREFIX: ${{ vars.HDX_USER_AGENT_PREPREFIX }}
+          USER_AGENT: ${{ vars.USER_AGENT }}
           BASIC_AUTHS: ${{ secrets.BASIC_AUTHS }}
           ERR_TO_HDX: ${{ secrets.ERR_TO_HDX }}
         run: python3.11 -m hapi.pipelines.app -db "postgresql+psycopg://postgres:postgres@localhost:5432/hapi"

CHANGELOG.md

@@ -4,6 +4,31 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.10.26] - 2024-12-06
+
+### Changed
+
+- Updated 3W data for NGA
+
+## [0.10.25] - 2024-12-05
+
+### Fixed
+
+- Update requirements to use latest `hapi-schema`
+
+## [0.10.24] - 2024-12-04
+
+### Changed
+
+- Updated 3W data for COD, ETH, SDN, SOM, YEM
+- Added 3W data for LBY, MWI, ZWE
+
+## [0.10.23] - 2024-11-22
+
+### Changed
+
+- Add missing poverty rate trends data
+
 ## [0.10.22] - 2024-11-21
 
 ### Changed

pyproject.toml

@@ -34,9 +34,9 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hapi-schema>= 0.9.2",
+    "hapi-schema>= 0.9.5",
     "hdx-python-api>= 6.3.5",
-    "hdx-python-country>= 3.8.4",
+    "hdx-python-country>= 3.8.5",
     "hdx-python-database[postgresql]>= 1.3.4",
     "hdx-python-scraper>= 2.5.0",
     "hdx-python-utilities>= 3.7.4",

requirements.txt

@@ -22,7 +22,7 @@ ckanapi==4.8
     # via hdx-python-api
 click==8.1.7
     # via typer
-coverage==7.6.7
+coverage==7.6.9
     # via pytest-cov
 defopt==6.4.0
     # via hdx-python-api
@@ -54,7 +54,7 @@ greenlet==3.1.1
     # via sqlalchemy
 gspread==6.1.4
     # via hdx-python-scraper
-hapi-schema==0.9.3
+hapi-schema==0.9.5
     # via hapi-pipelines (pyproject.toml)
 hdx-python-api==6.3.5
     # via
@@ -77,7 +77,7 @@ hdx-python-utilities==3.7.4
     #   hdx-python-scraper
 humanize==4.11.0
     # via frictionless
-identify==2.6.2
+identify==2.6.3
     # via pre-commit
 idna==3.10
     # via
@@ -108,7 +108,7 @@ libhxl==5.2.2
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
     #   hdx-python-country
-loguru==0.7.2
+loguru==0.7.3
     # via hdx-python-utilities
 makefun==1.15.6
     # via hdx-python-api
@@ -156,15 +156,15 @@ pyasn1==0.6.1
     #   rsa
 pyasn1-modules==0.4.1
     # via google-auth
-pydantic==2.10.0
+pydantic==2.10.3
     # via frictionless
-pydantic-core==2.27.0
+pydantic-core==2.27.1
     # via pydantic
 pygments==2.18.0
     # via rich
 pyphonetics==0.5.3
     # via hdx-python-country
-pytest==8.3.3
+pytest==8.3.4
     # via
     #   hapi-pipelines (pyproject.toml)
     #   pytest-check
@@ -218,7 +218,7 @@ rfc3986==2.0.0
     # via frictionless
 rich==13.9.4
     # via typer
-rpds-py==0.21.0
+rpds-py==0.22.3
     # via
     #   jsonschema
     #   referencing
@@ -263,7 +263,7 @@ text-unidecode==1.3
     # via python-slugify
 typeguard==4.4.1
     # via inflect
-typer==0.13.1
+typer==0.15.1
     # via frictionless
 typing-extensions==4.12.2
     # via
@@ -284,9 +284,9 @@ urllib3==2.2.3
     #   requests
 validators==0.34.0
     # via frictionless
-virtualenv==20.27.1
+virtualenv==20.28.0
     # via pre-commit
-wheel==0.45.0
+wheel==0.45.1
     # via libhxl
 xlrd==2.0.1
     # via hdx-python-utilities

src/hapi/pipelines/configs/core.yaml

@@ -314,6 +314,7 @@ org_type:
 org_type_map:
   academy: "431"
   agence un: "447"
+  agence des nations unies: "447"
   agence du systeme des nations unies: "447"
   autre type: "443"
   civile society: "501"
@@ -323,18 +324,21 @@ org_type_map:
   entite gouvernementale: "435"
   gouvernement: "435"
   gov: "435"
+  government agency: "435"
   govt: "435"
   iglesia grupo religioso: "446"
   ingo: "437"
   institution etatique: "435"
   institución publica: "435"
+  international government: "435"
   international institution: "438"
   international ngos: "437"
   local ngo: "504"
   mouv cr: "445"
   mouvement croix rouge: "445"
   mouvement de la croix rouge: "445"
   movimiento de la cruz roja media luna roja: "445"
+  national local ngo: "441"
   nations unies: "447"
   naciones unidas: "447"
   nngo: "441"
@@ -356,6 +360,7 @@ org_type_map:
   red cross mvt: "445"
   red cross and red crescent movement: "445"
   religious group: "446"
+  religious organization: "446"
   service etatique: "435"
   sociedad civil: "501"
   societe civile: "501"
@@ -395,17 +400,20 @@ sector_map:
   coordinacion informacion: "CCM"
   coordination: "CCM"
   coordination et gestion des camps: "CCM"
+  cp: "PRO-CPN"
   eah: "WSH"
   eau: "WSH"
   eau assainissement et hygiene: "WSH"
   eau hygiene: "WSH"
   eau hygiene assainissement: "WSH"
   eau hygiene et assainissement: "WSH"
+  educ: "EDU"
   educacion: "EDU"
   educacion en emergencias: "EDU"
   education: "EDU"
   eha: "WSH"
   emergency shelter and non food items: "SHL"
+  employment and labour force protection: "HEA"  # this relates to COVID-19
   epah: "WSH"
   erl: "ERY"
   esnfi: "SHL"
@@ -415,17 +423,22 @@ sector_map:
   food security and livelihoods: "FSC"
   food security and nutrition: "FSC"
   food security livelihood: "FSC"
+  food security and livelihoods fsl: "FSC"
   fsl: "FSC"
+  fss: "FSC"
   gestion des sites daccueil temporaires: "SHL"
   gbv: "PRO-GBV"
   hlp: "PRO-HLP"
+  housing land property: "PRO-HLP"
   humanitaire: "Hum"
   hygiene: "WSH"
   hygiene assainissement: "WSH"
   intercluster: "Multi"  # From Somalia 3W, hopefully not to be confused with intersectoral
   logement terre et biens: "PRO-HLP"
   logistica: "LOG"
   logistique: "LOG"
+  logs: "LOG"
+  ma: "PRO-MIN"
   manejo y gestion de campamentos: "CCM"
   mpca: "Cash"
   ms: "Multi"
@@ -458,6 +471,7 @@ sector_map:
   proteccion: "PRO"
   provbg: "PRO-GBV"
   psea: "PRO-GBV"
+  public communication: "HEA"  # this relates to COVID-19
   rapid response mechanism: "ERY"
   rcf: "CCM"
   rcf education: "EDU"
@@ -488,11 +502,14 @@ sector_map:
   shelter and nfis: "SHL"
   shelter and non food items: "SHL"
   site management: "CCM"
+  site mngt: "CCM"
   snfi: "SHL"
   telecommunications: "TEL"
   telecommunications durgence: "TEL"
   telecomunicaciones de emergencia: "TEL"
+  transport and logistics: "LOG"
   vbg: "PRO-GBV"
+  vgb pe: "PRO-GBV"
   violences basees sur le genre: "PRO-GBV"
   violence basee sur le genre: "PRO-GBV"
   violencia basada en genero: "PRO-GBV"