Merge pull request OCHA-DAP#109 from OCHA-DAP/bugfix/pipeline-updates

Bugfix/pipeline updates

CHANGELOG.md

@@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.9.14] - 2024-06-03
+
+### Added
+
+- Non-standard age categories to population data
+- Admin mappings for food prices data
+
+### Changed
+
+- Removed filter on refugee data
+
+### Fixed
+
+- Added missing population data for SLV
+
 ## [0.9.13] - 2024-05-30
 
 ### Added

requirements.txt

@@ -10,7 +10,7 @@ attrs==23.2.0
     #   referencing
 cachetools==5.3.3
     # via google-auth
-certifi==2024.2.2
+certifi==2024.6.2
     # via requests
 cffi==1.16.0
     # via cryptography
@@ -166,9 +166,9 @@ pyasn1-modules==0.4.0
     # via google-auth
 pycparser==2.22
     # via cffi
-pydantic==2.7.2
+pydantic==2.7.3
     # via frictionless
-pydantic-core==2.18.3
+pydantic-core==2.18.4
     # via pydantic
 pygments==2.18.0
     # via rich
@@ -278,7 +278,7 @@ typeguard==4.3.0
     # via inflect
 typer==0.12.3
     # via frictionless
-typing-extensions==4.12.0
+typing-extensions==4.12.1
     # via
     #   frictionless
     #   inflect

src/hapi/pipelines/configs/core.yaml

@@ -38,12 +38,16 @@ country_name_mappings:
 admin1:
   admin_name_mappings:
     "ETH|B. Gumuz": "ET06"
+    "NGA|Abuja": "NG015"
     "SDN|Eastern Darfur": "SD05"
+    "SDN|Nile": "SD16"
     "SOM|Juba Dhexe": "SO27"
     "SOM|Juba Hoose": "SO28"
     "SOM|Shabelle Dhexe": "SO21"
     "SOM|Shabelle Hoose": "SO23"
+    "TCD|Mayo Kebbi Ouest": "TD12"
     "UKR|Avtonomna Respublika Krym": "UA01"
+    "UKR|Kyivska": "UA80"
     "YEM|Amanat Al Asimah": "YE13"
 
 admin2:
@@ -53,13 +57,146 @@ admin2:
     "AF23|Chaghcharan": "AF2301"
     "CO08|Distrito Especial, Industrial Y Portuario De Barranquilla": "CO08001"
     "ET01|C. TIGRAY": "ET0102"
-    "ET01|E. TIGRAY": "ET0103"
     "ET01|NW. TIGRAY": "ET0101"
-    "ET01|S. TIGRAY": "ET0104"
-    "ET01|W. TIGRAY": "ET0105"
+    "ET02|ZONE1": "ET0201"
+    "ET02|ZONE3": "ET0203"
+    "ET02|ZONE4": "ET0204"
+    "ET02|ZONE5": "ET0205"
+    "ET05|DEGEHABUR": "ET0503"
+    "ET05|FIK": "ET0504"
+    "ET05|GODE": "ET0506"
+    "ET05|JIJIGA": "ET0502"
+    "ET05|SHINILE": "ET0501"
+    "ET05|WARDER": "ET0507"
+    "ET07|SIDAMA": "ET1600"
+    "ET12|GODERE": "ET1203"
+    "ET12|ZONE 1": "ET1204"
+    "ET12|ZONE 2": "ET1202"
+    "ET12|ZONE 3": "ET1201"
+    "ET13|HARAR/HUNDENE": "ET1301"
+    "ET14|AA ZONE1": "ET1401"
+    "GT13|San Idelfonso Ixtahuacán": "GT1309"
+    "ML07|Menaka": "ML1001"
+    "NE006|Tagazar": "NE006003"
+    "PS01|Ramallah and Albireh": "PS0130"
+    "SD01|Sharq El Nile": "SD01004"
+    "SD10|Al Gonnob & Al Oleb": "SD10069"
+    "SD11|Wad Al Helew": "SD11061"
+    "SD12|Al Galabat Eastern": "SD12083"
+    "SD12|Al Gedaref Rural": "SD12080"
+    "SD15|Wad Madani El Kobra": "SD15030"
+    "SD18|El Salam": "SD18086"
+    "TD17|Iriba": "TD1703"
+    "UA12|Apostolivskyi": "UA1206"
+    "UA12|Marhanetska": "UA1208"
+    "UA12|Ordzhonikidzevska": "UA1208"
+    "UA12|Pokrovskyi": "UA1214"
+    "UA12|Sofiivskyi": "UA1206"
+    "UA12|Tomakivskyi": "UA1208"
+    "UA14|Amvrosiivskyi": "UA1408"
+    "UA14|Avdiivska": "UA1416"
+    "UA14|Debaltsevcka": "UA1406"
+    "UA14|Dobropilska": "UA1416"
+    "UA14|Dobropilskyi": "UA1416"
+    "UA14|Dokuchaievska": "UA1410"
+    "UA14|Dymytrivska": "UA1416"
+    "UA14|Khartsyzka": "UA1408"
+    "UA14|Kirovska": "UA1406"
+    "UA14|Kostiantynivska": "UA1412"
+    "UA14|Kostiantynivskyi": "UA1412"
+    "UA14|Lymanska": "UA1412"
+    "UA14|Marinskyi": "UA1416"
+    "UA14|Novoazovskyi": "UA1410"
+    "UA14|Novohrodivska": "UA1416"
+    "UA14|Selydivska": "UA1416"
+    "UA14|Shakhtarskyi": "UA1406"
+    "UA14|Slovianska": "UA1412"
+    "UA14|Snizhnianska": "UA1406"
+    "UA14|Toretska": "UA1402"
+    "UA14|Torezka": "UA1406"
+    "UA14|Velykonovosilkivskyi": "UA1404"
+    "UA14|Vuhledarska": "UA1404"
+    "UA14|Yasynuvatska": "UA1408"
+    "UA14|Yasynuvatskyi": "UA1416"
+    "UA14|Yenakiivska": "UA1406"
+    "UA63|Balakliiskyi": "UA6304"
+    "UA63|Blyzniukivskyi": "UA6310"
+    "UA63|Borivskyi": "UA6304"
+    "UA63|Derhachivskyi": "UA6312"
+    "UA63|Dvorichanskyi": "UA6308"
+    "UA63|Liubotynska": "UA6312"
+    "UA63|Novovodolazkyi": "UA6312"
+    "UA63|Pervomaiska": "UA6310"
+    "UA63|Shevchenkivskyi": "UA6308"
+    "UA63|Velykoburlutskyi": "UA6308"
+    "UA63|Vovchanskyi": "UA6314"
+    "UA63|Zolochivskyi": "UA6302"
+    "UA65|Bilozerskyi": "UA6510"
+    "UA65|Chaplynskyi": "UA6506"
+    "UA65|Holoprystanska": "UA6508"
+    "UA65|Hornostaivskyi": "UA6506"
+    "UA65|Kalanchatskyi": "UA6508"
+    "UA65|Novokakhovska": "UA6506"
+    "UA65|Novovorontsovskyi": "UA6502"
+    "UA65|Oleshkivskyi": "UA6510"
+    "UA65|Velykolepetyskyi": "UA6506"
+    "UA65|Velykooleksandrivskyi": "UA6502"
+    "UA65|Verkhnorohachytskyi": "UA6506"
+    "UA65|Vysokopilskyi": "UA6502"
+    "UA35|Kirovohradska": "UA3504"
+    "UA35|Malovyskivskyi": "UA3506"
+    "UA44|Troitskyi": "UA4410"
+    "UA48|Arbuzynskyi": "UA4808"
+    "UA48|Berezanskyi": "UA4806"
+    "UA48|Bereznehuvatskyi": "UA4802"
+    "UA48|Kazankivskyi": "UA4802"
+    "UA48|Novobuzkyi": "UA4802"
+    "UA48|Novoodeskyi": "UA4806"
+    "UA48|Ochakivska": "UA4806"
+    "UA48|Snihurivskyi": "UA4802"
+    "UA48|Veselynivskyi": "UA4804"
+    "UA48|Vitovskyi": "UA4806"
+    "UA48|Vradiivskyi": "UA4808"
+    "UA48|Yelanetskyi": "UA4804"
+    "UA48|Yuzhnoukrainska": "UA4804"
+    "UA59|Bilopilskyi": "UA5908"
+    "UA59|Burynskyi": "UA5902"
+    "UA59|Hlukhivska": "UA5910"
+    "UA59|Hlukhivskyi": "UA5910"
+    "UA59|Krasnopilskyi": "UA5908"
+    "UA59|Krolevetskyi": "UA5902"
+    "UA59|Lebedynska": "UA5908"
+    "UA59|Lypovodolynskyi": "UA5906"
+    "UA59|Nedryhailivskyi": "UA5906"
+    "UA59|Putyvlskyi": "UA5902"
+    "UA59|Seredyno-Budskyi": "UA5910"
+    "UA59|Trostianetskyi": "UA5904"
+    "UA59|Velykopysarivskyi": "UA5904"
+    "UA59|Yampilskyi": "UA5910"
+    "UA23|Bilmatskyi": "UA2310"
+    "UA23|Chernihivskyi": "UA2302"
+    "UA23|Enerhodarska": "UA2304"
+    "UA23|Huliaipilskyi": "UA2310"
+    "UA23|Kamiansko-Dniprovskyi": "UA2304"
+    "UA23|Novomykolaivskyi": "UA2306"
+    "UA23|Pryazovskyi": "UA2308"
+    "UA23|Prymorskyi": "UA2302"
+    "UA23|Rozivskyi": "UA2310"
+    "UA23|Tokmatska": "UA2310"
+    "UA23|Tokmatskyi": "UA2310"
+    "UA23|Vilnianskyi": "UA2306"
+    "UA23|Yakymivskyi": "UA2308"
+    "YE14|Radman Al Awad": "YE1412"
+    "YE24|Craiter": "YE2407"
+    "YE26|Medghal": "YE2603"
 
   admin_name_replacements:
     "COD| city": ""
+    "ETH|c.": "central"
+    "ETH|e.": "east"
+    "ETH|n.": "north"
+    "ETH|s.": "south"
+    "ETH|w.": "west"
 
 orphan_admin2s:
   SS0001: "SSD"