working on converting into api calls instead of time diff

Author: aysim319

nhsn/delphi_nhsn/constants.py

@@ -65,4 +65,4 @@
     f"{NUM_HOSP_REPORTING_RSV}_prelim": float,
 }
 
-RECENTLY_UPDATED_DIFF = timedelta(hours=36)
+RECENTLY_UPDATED_DIFF = timedelta(days=2)

nhsn/delphi_nhsn/pull.py

@@ -4,14 +4,16 @@
 import logging
 import random
 import time
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Optional
 from urllib.error import HTTPError
 
 import pandas as pd
 from delphi_utils import create_backup_csv
 from sodapy import Socrata
+from delphi_epidata import Epidata
+from epiweeks import Week
 
 from .constants import (
     MAIN_DATASET_ID,
@@ -47,10 +49,17 @@ def check_last_updated(socrata_token, dataset_id, logger):
         response = client.get_metadata(dataset_id)
 
         updated_timestamp = datetime.utcfromtimestamp(int(response["rowsUpdatedAt"]))
-        now = datetime.utcnow()
-        # currently set to run twice a week, RECENTLY_UPDATED_DIFF may need adjusting based on the cadence
-        recently_updated_source = (now - updated_timestamp) < RECENTLY_UPDATED_DIFF
+        cdc_updated_epiweek = Week.fromdate(updated_timestamp)
+
+        meta_df = pd.DataFrame(Epidata.covidcast_meta()["epidata"])
+        signal_suffix = 'prelim' if dataset_id == PRELIM_DATASET_ID else "ew"
+        nhsn_meta_df = meta_df[(meta_df["data_source"] == "nhsn") & (meta_df["signal"].str.endswith(signal_suffix))]
+        last_updated = datetime.utcfromtimestamp(nhsn_meta_df["last_update"].min())
+        covidcast_updated_epiweek = Week.fromdate(last_updated)
 
+        # currently set to run twice a week, RECENTLY_UPDATED_DIFF may need adjusting based on the cadence
+        recently_updated_source = (last_updated - updated_timestamp) > RECENTLY_UPDATED_DIFF
+        print("non")
         prelim_prefix = "Preliminary " if dataset_id == PRELIM_DATASET_ID else ""
         if recently_updated_source:
             logger.info(

nhsn/tests/conftest.py

@@ -23,6 +23,11 @@
 with open(f"{TEST_DIR}/test_data/prelim_page.json", "r") as f:
     PRELIM_TEST_DATA = json.load(f)
 
+
+covidcast_metadata = pd.read_csv(f"{TEST_DIR}/test_data/covid_metadata.csv",
+                                     parse_dates=["max_time", "min_time", "max_issue", "last_update"])
+
+
 @pytest.fixture(scope="session")
 def params():
     params = {

nhsn/tests/test_pull.py

@@ -161,6 +161,7 @@ def test_pull_nhsn_data_backup(self, mock_socrata, dataset, caplog, params):
     @pytest.mark.parametrize('dataset', DATASETS, ids=["data", "prelim_data"])
     @pytest.mark.parametrize("updatedAt", [time.time(), time.time() - 172800, time.time() - 108000], ids=["updated", "stale", "updated_late"])
     @patch("delphi_nhsn.pull.Socrata")
+    @patch("delphi_nhsn.pull.Epidata.covidcast_meta", return_value=covidcast_metadata):
     def test_check_last_updated(self, mock_socrata, dataset, updatedAt, caplog):
         mock_client = MagicMock()
         mock_socrata.return_value = mock_client