Merge pull request #6 from OCHA-DAP/main

Hotfix for funding
OCHA-DAP · Jun 29, 2024 · c69a42b · c69a42b
2 parents 2584079 + 83d37ff
commit c69a42b
Show file tree

Hide file tree

Showing 6 changed files with 42 additions and 20 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+### [0.9.20] - 2024-06-29
+
+### Fixed
+
+- Add check for funding requirements value due to missing data for UKR
+
 ## [0.9.19] - 2024-06-25
 
 ### Fixed

diff --git a/requirements.txt b/requirements.txt
@@ -134,7 +134,7 @@ num2words==0.5.13
     # via quantulum3
 oauthlib==3.2.2
     # via requests-oauthlib
-openpyxl==3.1.4
+openpyxl==3.1.5
     # via hdx-python-utilities
 packaging==24.1
     # via pytest

diff --git a/src/hapi/pipelines/database/funding.py b/src/hapi/pipelines/database/funding.py
@@ -7,6 +7,7 @@
 from hdx.utilities.dateparse import parse_date
 from sqlalchemy.orm import Session
 
+from ..utilities.logging_helpers import add_message, add_missing_value_message
 from . import locations
 from .base_uploader import BaseUploader
 from .metadata import Metadata
@@ -29,7 +30,9 @@ def __init__(
 
     def populate(self):
         logger.info("Populating funding table")
+        errors = set()
         for dataset in self._results.values():
+            dataset_name = dataset["hdx_stub"]
             for admin_level, admin_results in dataset["results"].items():
                 resource_id = admin_results["hapi_resource_metadata"]["hdx_id"]
                 hxl_tags = admin_results["headers"][1]
@@ -55,8 +58,21 @@ def populate(self):
                             values[reference_period_end_i][admin_code][irow]
                         )
                         if reference_period_start > reference_period_end:
-                            logger.error(
-                                f"Date misalignment in funding data for {appeal_code} in {admin_code}"
+                            add_message(
+                                errors,
+                                dataset_name,
+                                f"Date misalignment in funding data for {appeal_code} in {admin_code}",
+                            )
+                            continue
+                        # This check for a missing funding line has been added due to
+                        # an error in the UKR funding requirements data
+                        funding_usd = values[funding_usd_i][admin_code][irow]
+                        if funding_usd is None:
+                            add_missing_value_message(
+                                errors,
+                                dataset_name,
+                                "funding_usd",
+                                appeal_code,
                             )
                             continue
                         funding_row = DBFunding(
@@ -72,9 +88,7 @@ def populate(self):
                             requirements_usd=values[requirements_usd_i][
                                 admin_code
                             ][irow],
-                            funding_usd=values[funding_usd_i][admin_code][
-                                irow
-                            ],
+                            funding_usd=funding_usd,
                             funding_pct=values[funding_pct_i][admin_code][
                                 irow
                             ],
@@ -84,3 +98,5 @@ def populate(self):
 
                         self._session.add(funding_row)
         self._session.commit()
+        for error in sorted(errors):
+            logger.error(error)
diff --git a/tests/fixtures/input/fts-requirements-and-funding-data-for-ukraine.json b/tests/fixtures/input/fts-requirements-and-funding-data-for-ukraine.json
diff --git a/tests/fixtures/input/funding_ukr_fts_requirements_funding_ukr.csv b/tests/fixtures/input/funding_ukr_fts_requirements_funding_ukr.csv
@@ -1,24 +1,24 @@
 countryCode,id,name,code,typeId,typeName,startDate,endDate,year,requirements,funding,percentFunded
 #country+code,#activity+appeal+id+fts_internal,#activity+appeal+name,#activity+appeal+id+external,#activity+appeal+type+id+fts_internal,#activity+appeal+type+name,#date+start,#date+end,#date+year,#value+funding+required+usd,#value+funding+total+usd,#value+funding+pct
-UKR,,Not specified,,,,,,2027,,2994012,
-UKR,,Not specified,,,,,,2026,,7592229,
-UKR,,Not specified,,,,,,2025,,77427247,
-UKR,1177,Ukraine Humanitarian Response Plan 2024,HUKR24,4,Humanitarian response plan,2024-01-01,2024-12-31,2024,3107702931,668516837,22
-UKR,,Not specified,,,,,,2024,,34392644,
-UKR,1124,Ukraine Humanitarian Response Plan 2023,HUKR23,4,Humanitarian response plan,2023-01-01,2023-12-31,2023,3945656687,2847032800,72
-UKR,,Not specified,,,,,,2023,,634487080,
-UKR,1081,Ukraine Humanitarian Response Plan 2022,HUKR22,4,Humanitarian response plan,2022-01-01,2022-12-31,2022,0,8476145,0
-UKR,1102,Ukraine Flash Appeal 2022,FUKR22,5,Flash appeal,2022-03-01,2022-12-31,2022,4292463880,3778429100,88
-UKR,,Not specified,,,,,,2022,,752433111,
+UKR,,Not specified,,,,,,2027,,29333446,
+UKR,,Not specified,,,,,,2026,,33336012,
+UKR,,Not specified,,,,,,2025,,97654028,
+UKR,1177,Ukraine Humanitarian Response Plan 2024,HUKR24,4,Humanitarian response plan,2024-01-01,2024-12-31,2024,3107702931,879188121,28
+UKR,,Not specified,,,,,,2024,,53181561,
+UKR,1124,Ukraine Humanitarian Response Plan 2023,HUKR23,4,Humanitarian response plan,2023-01-01,2023-12-31,2023,3945656687,2874078619,73
+UKR,,Not specified,,,,,,2023,,635875335,
+UKR,1081,Ukraine Humanitarian Response Plan 2022,HUKR22,4,Humanitarian response plan,2022-01-01,2022-12-31,2022,0,,
+UKR,1102,Ukraine Flash Appeal 2022,FUKR22,5,Flash appeal,2022-03-01,2022-12-31,2022,4292463880,3808701203,89
+UKR,,Not specified,,,,,,2022,,752189309,
 UKR,1015,Ukraine Humanitarian Response Plan 2021,HUKR21,4,Humanitarian response plan,2021-01-01,2021-12-31,2021,167982922,106445580,63
 UKR,,Not specified,,,,,,2021,,68087180,
 UKR,830,Ukraine 2020,HUKR20,4,Humanitarian response plan,2020-01-01,2020-12-31,2020,204653234,123030027,60
 UKR,,Not specified,,,,,,2020,,92287509,
 UKR,662,Ukraine Humanitarian Response Plan (HRP) 2019,HUKR19,4,Humanitarian response plan,2019-01-01,2019-12-31,2019,163640038,85630793,52
 UKR,,Not specified,,,,,,2019,,69459095,
-UKR,641,Ukraine Humanitarian Response Plan 2018,HUKR18,4,Humanitarian response plan,2018-01-01,2018-12-31,2018,186909122,70898032,38
+UKR,641,Ukraine Humanitarian Response Plan 2018,HUKR18,4,Humanitarian response plan,2018-01-01,2018-12-31,2018,186909122,71016515,38
 UKR,,Not specified,,,,,,2018,,83664720,
-UKR,531,Ukraine Humanitarian Response Plan 2017,HUKR17,4,Humanitarian response plan,2017-01-01,2017-12-31,2017,203608611,74606618,37
+UKR,531,Ukraine Humanitarian Response Plan 2017,HUKR17,4,Humanitarian response plan,2017-01-01,2017-12-31,2017,203608611,75992269,37
 UKR,,Not specified,,,,,,2017,,84949488,
 UKR,507,Ukraine 2016,HUKR1615,4,Humanitarian response plan,2016-01-01,2015-12-31,2016,297907511,105289272,35
 UKR,,Not specified,,,,,,2016,,107651499,

diff --git a/tests/test_main.py b/tests/test_main.py
@@ -182,7 +182,7 @@ def test_pipelines(self, configuration, folder):
                     count = session.scalar(
                         select(func.count(DBFunding.resource_hdx_id))
                     )
-                    check.equal(count, 57)
+                    check.equal(count, 56)
                     count = session.scalar(
                         select(func.count(DBConflictEvent.resource_hdx_id))
                     )