finished converting raw time diff to using api

Author: aysim319

nhsn/delphi_nhsn/constants.py

@@ -65,4 +65,4 @@
     f"{NUM_HOSP_REPORTING_RSV}_prelim": float,
 }
 
-RECENTLY_UPDATED_DIFF = timedelta(days=2)
+RECENTLY_UPDATED_DIFF = timedelta(days=1)

nhsn/delphi_nhsn/pull.py

@@ -4,16 +4,15 @@
 import logging
 import random
 import time
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Optional
 from urllib.error import HTTPError
 
 import pandas as pd
+from delphi_epidata import Epidata
 from delphi_utils import create_backup_csv
 from sodapy import Socrata
-from delphi_epidata import Epidata
-from epiweeks import Week
 
 from .constants import (
     MAIN_DATASET_ID,
@@ -48,25 +47,31 @@ def check_last_updated(socrata_token, dataset_id, logger):
         client = Socrata("data.cdc.gov", socrata_token)
         response = client.get_metadata(dataset_id)
 
-        updated_timestamp = datetime.utcfromtimestamp(int(response["rowsUpdatedAt"]))
-        cdc_updated_epiweek = Week.fromdate(updated_timestamp)
+        updated_timestamp = datetime.fromtimestamp(int(response["rowsUpdatedAt"]), tz=timezone.utc)
 
+        # pulling last updated from the api
         meta_df = pd.DataFrame(Epidata.covidcast_meta()["epidata"])
-        signal_suffix = 'prelim' if dataset_id == PRELIM_DATASET_ID else "ew"
+        signal_suffix = "prelim" if dataset_id == PRELIM_DATASET_ID else "ew"
         nhsn_meta_df = meta_df[(meta_df["data_source"] == "nhsn") & (meta_df["signal"].str.endswith(signal_suffix))]
-        last_updated = datetime.utcfromtimestamp(nhsn_meta_df["last_update"].min())
-        covidcast_updated_epiweek = Week.fromdate(last_updated)
+        est = timezone(timedelta(hours=-5))
+        last_updated = datetime.fromtimestamp(nhsn_meta_df["last_update"].min(), tz=est)
 
         # currently set to run twice a week, RECENTLY_UPDATED_DIFF may need adjusting based on the cadence
-        recently_updated_source = (last_updated - updated_timestamp) > RECENTLY_UPDATED_DIFF
-        print("non")
+        recently_updated_source = (updated_timestamp - last_updated) > RECENTLY_UPDATED_DIFF
+
         prelim_prefix = "Preliminary " if dataset_id == PRELIM_DATASET_ID else ""
         if recently_updated_source:
             logger.info(
-                f"{prelim_prefix}NHSN data was recently updated; Pulling data", updated_timestamp=updated_timestamp
+                f"{prelim_prefix}NHSN data was recently updated; Pulling data",
+                updated_timestamp=updated_timestamp,
+                metadata_timestamp=last_updated,
             )
         else:
-            logger.info(f"{prelim_prefix}NHSN data is stale; Skipping", updated_timestamp=updated_timestamp)
+            logger.info(
+                f"{prelim_prefix}NHSN data is stale; Skipping",
+                updated_timestamp=updated_timestamp,
+                metadata_timestamp=last_updated,
+            )
     # pylint: disable=W0703
     except Exception as e:
         logger.info("error while processing socrata metadata; treating data as stale", error=str(e))

nhsn/tests/conftest.py

@@ -23,9 +23,9 @@
 with open(f"{TEST_DIR}/test_data/prelim_page.json", "r") as f:
     PRELIM_TEST_DATA = json.load(f)
 
-
-covidcast_metadata = pd.read_csv(f"{TEST_DIR}/test_data/covid_metadata.csv",
-                                     parse_dates=["max_time", "min_time", "max_issue", "last_update"])
+# filtered metadata (just includes nhsn meta)
+with open(f"{TEST_DIR}/test_data/covidcast_meta.json", "r") as f:
+    COVID_META_DATA = json.load(f)
 
 
 @pytest.fixture(scope="session")