Merge pull request #37 from OCHA-DAP/hapi-pipelines-0.10.9-schema-0.9.0

PR Hapi pipelines 0.10.9 schema 0.9.0 into main3

CHANGELOG.md

@@ -4,6 +4,31 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.10.10] - 2024-10-16
+
+### Fixed
+
+- Output error for blank appeal code
+
+## [0.10.9] - 2024-10-15
+
+### Added
+
+- Lebanon operational presence
+
+## [0.10.8] - 2024-10-10
+
+### Fixed
+
+- Added check for null provider name
+
+## [0.10.7] - 2024-10-10
+
+### Changed
+
+- Update subnational data to include provider admin 1 and 2 names
+- Include global data for food prices, IDPs, conflict event, and food security
+
 ## [0.10.6] - 2024-10-09
 
 ### Fixed

pyproject.toml

@@ -34,7 +34,7 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hapi-schema==0.8.17",
+    "hapi-schema==0.9.0",
     "hdx-python-api>= 6.3.4",
     "hdx-python-country>= 3.8.1",
     "hdx-python-database[postgresql]>= 1.3.1",

requirements.txt

@@ -16,17 +16,17 @@ cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
     # via frictionless
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
     # via requests
 ckanapi==4.8
     # via hdx-python-api
 click==8.1.7
     # via typer
-coverage==7.6.1
+coverage==7.6.3
     # via pytest-cov
 defopt==6.4.0
     # via hdx-python-api
-distlib==0.3.8
+distlib==0.3.9
     # via virtualenv
 dnspython==2.7.0
     # via email-validator
@@ -54,7 +54,7 @@ greenlet==3.1.1
     # via sqlalchemy
 gspread==6.1.3
     # via hdx-python-scraper
-hapi-schema==0.8.17
+hapi-schema==0.9.0
     # via hapi-pipelines (pyproject.toml)
 hdx-python-api==6.3.4
     # via
@@ -95,7 +95,7 @@ jinja2==3.1.4
     # via frictionless
 jsonlines==4.0.0
     # via hdx-python-utilities
-jsonpath-ng==1.6.1
+jsonpath-ng==1.7.0
     # via libhxl
 jsonschema==4.23.0
     # via

src/hapi/pipelines/app/__main__.py

@@ -201,7 +201,6 @@ def main(
         "conflict_event.yaml",
         "core.yaml",
         "food_security.yaml",
-        "funding.yaml",
         "idps.yaml",
         "national_risk.yaml",
         "operational_presence.yaml",

src/hapi/pipelines/app/pipelines.py

@@ -45,6 +45,7 @@ def __init__(
         scrapers_to_run: Optional[ListTuple[str]] = None,
         errors_on_exit: Optional[ErrorsOnExit] = None,
         use_live: bool = True,
+        countries_to_run: Optional[ListTuple[str]] = None,
     ):
         self.configuration = configuration
         self.session = session
@@ -53,8 +54,9 @@ def __init__(
             configuration=configuration,
             session=session,
             use_live=use_live,
+            countries=countries_to_run,
         )
-        self.countries = configuration["HAPI_countries"]
+        self.countries = self.locations.hapi_countries
         reader = Read.get_reader("hdx")
         libhxl_dataset = AdminLevel.get_libhxl_dataset(
             retriever=reader
@@ -69,11 +71,11 @@ def __init__(
         self.adminone = AdminLevel(admin_config=admin1_config, admin_level=1)
         admin2_config = configuration["admin2"]
         self.admintwo = AdminLevel(admin_config=admin2_config, admin_level=2)
-        self.adminone.setup_from_libhxl_dataset(libhxl_dataset, self.countries)
+        self.adminone.setup_from_libhxl_dataset(libhxl_dataset)
         self.adminone.load_pcode_formats_from_libhxl_dataset(
             libhxl_format_dataset
         )
-        self.admintwo.setup_from_libhxl_dataset(libhxl_dataset, self.countries)
+        self.admintwo.setup_from_libhxl_dataset(libhxl_dataset)
         self.admintwo.load_pcode_formats_from_libhxl_dataset(
             libhxl_format_dataset
         )
@@ -184,7 +186,6 @@ def _create_configurable_scrapers(
         )
         _create_configurable_scrapers("operational_presence", "national")
         _create_configurable_scrapers("national_risk", "national")
-        _create_configurable_scrapers("funding", "national")
         _create_configurable_scrapers("refugees_and_returnees", "national")
         _create_configurable_scrapers("idps", "national")
         _create_configurable_scrapers(
@@ -307,14 +308,11 @@ def output_idps(self):
 
     def output_funding(self):
         if not self.themes_to_run or "funding" in self.themes_to_run:
-            results = self.runner.get_hapi_results(
-                self.configurable_scrapers["funding"]
-            )
             funding = Funding(
                 session=self.session,
                 metadata=self.metadata,
                 locations=self.locations,
-                results=results,
+                configuration=self.configuration,
             )
             funding.populate()
 
@@ -360,6 +358,7 @@ def output_food_prices(self):
                 admins=self.admins,
                 adminone=self.adminone,
                 admintwo=self.admintwo,
+                configuration=self.configuration,
             )
             wfp_market.populate()
             food_price = FoodPrice(