Add as_of support to covid_hosp

Author: chinandrew

integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py

@@ -89,3 +89,22 @@ def test_acquire_dataset(self):
       response = Epidata.covid_hosp('WY', Epidata.range(20200101, 20210101))
       self.assertEqual(response['result'], 1)
       self.assertEqual(len(response['epidata']), 1)
+
+    with self.subTest(name='as_of checks'):
+      # acquire new data with 3/16 issue date
+      mock_network.fetch_metadata.return_value = \
+        self.test_utils.load_sample_metadata("metadata2.csv")
+      mock_network.fetch_dataset.return_value = \
+        self.test_utils.load_sample_dataset("dataset2.csv")
+      response = Epidata.covid_hosp('WY', Epidata.range(20200101, 20210101))
+      row = response['epidata'][0]
+      self.assertEqual(row['date'], 20200827)
+
+      # previous data should have 3/15 issue date
+      response = Epidata.covid_hosp('WY', Epidata.range(20200101, 20210101), as_of=20210315)
+      row = response['epidata'][0]
+      self.assertEqual(row['date'], 20200826)
+
+      # no data before 3/15
+      response = Epidata.covid_hosp('WY', Epidata.range(20200101, 20210101), as_of=20210314)
+      self.assertEqual(response['result'], -2)

src/acquisition/covid_hosp/common/test_utils.py

@@ -50,8 +50,8 @@ def __init__(self, abs_path_to_caller):
         path_to_repo / UnitTestUtils.PATH_TO_TESTDATA / dataset_name
     ).resolve()
 
-  def load_sample_metadata(self):
-    df = pandas.read_csv(self.data_dir / 'metadata.csv', dtype=str)
+  def load_sample_metadata(self, metadata_name='metadata.csv'):
+    df = pandas.read_csv(self.data_dir / metadata_name, dtype=str)
     df["Update Date"] = pandas.to_datetime(df["Update Date"])
     df.sort_values("Update Date", inplace=True)
     df.set_index("Update Date", inplace=True)

src/client/delphi_epidata.py

@@ -612,7 +612,7 @@ def covidcast_meta():
 
   # Fetch COVID hospitalization data
   @staticmethod
-  def covid_hosp(states, dates, issues=None):
+  def covid_hosp(states, dates, issues=None, as_of=None):
     """Fetch COVID hospitalization data."""
     # Check parameters
     if states is None or dates is None:
@@ -625,6 +625,8 @@ def covid_hosp(states, dates, issues=None):
     }
     if issues is not None:
       params['issues'] = Epidata._list(issues)
+    if as_of is not None:
+      params['as_of'] = as_of
     # Make the API call
     return Epidata._request(params)
 

src/server/endpoints/covid_hosp_state_timeseries.py

@@ -1,7 +1,7 @@
 from flask import Blueprint
 
 from .._query import execute_query, QueryBuilder
-from .._validate import extract_integers, extract_strings, require_all
+from .._validate import extract_integers, extract_strings, extract_date, require_all
 
 # first argument is the endpoint name
 bp = Blueprint("covid_hosp_state_timeseries", __name__)
@@ -14,6 +14,7 @@ def handle():
     states = extract_strings("states")
     dates = extract_integers("dates")
     issues = extract_integers("issues")
+    as_of = extract_date("as_of")
 
     # build query
     q = QueryBuilder("covid_hosp_state_timeseries", "c")
@@ -95,6 +96,10 @@ def handle():
         q.where_integers("issue", issues)
         # final query using specific issues
         query = f"WITH c as (SELECT {q.fields_clause}, ROW_NUMBER() OVER (PARTITION BY date, state, issue ORDER BY record_type) row FROM {q.table} WHERE {q.conditions_clause}) SELECT {q.fields_clause} FROM {q.alias} where row = 1 ORDER BY {q.order_clause}"
+    elif as_of is not None:
+        sub_condition_asof = "(issue <= :as_of)"
+        q.params["as_of"] = as_of
+        query = f"WITH c as (SELECT {q.fields_clause}, ROW_NUMBER() OVER (PARTITION BY date, state, issue ORDER BY record_type) row FROM {q.table} WHERE {q.conditions_clause} and {sub_condition_asof}) SELECT {q.fields_clause} FROM {q.alias} where row = 1 ORDER BY {q.order_clause}"
     else:
         # final query using most recent issues
         subquery = f"(SELECT max(`issue`) `max_issue`, `date`, `state` FROM {q.table} WHERE {q.conditions_clause} GROUP BY `date`, `state`) x"

testdata/acquisition/covid_hosp/state_timeseries/dataset2.csv

@@ -0,0 +1,2 @@
+state,date,critical_staffing_shortage_today_yes,critical_staffing_shortage_today_no,critical_staffing_shortage_today_not_reported,critical_staffing_shortage_anticipated_within_week_yes,critical_staffing_shortage_anticipated_within_week_no,critical_staffing_shortage_anticipated_within_week_not_reported,hospital_onset_covid,hospital_onset_covid_coverage,inpatient_beds,inpatient_beds_coverage,inpatient_beds_used,inpatient_beds_used_coverage,inpatient_beds_used_covid,inpatient_beds_used_covid_coverage,previous_day_admission_adult_covid_confirmed,previous_day_admission_adult_covid_confirmed_coverage,previous_day_admission_adult_covid_suspected,previous_day_admission_adult_covid_suspected_coverage,previous_day_admission_pediatric_covid_confirmed,previous_day_admission_pediatric_covid_confirmed_coverage,previous_day_admission_pediatric_covid_suspected,previous_day_admission_pediatric_covid_suspected_coverage,staffed_adult_icu_bed_occupancy,staffed_adult_icu_bed_occupancy_coverage,staffed_icu_adult_patients_confirmed_and_suspected_covid,staffed_icu_adult_patients_confirmed_and_suspected_covid_coverage,staffed_icu_adult_patients_confirmed_covid,staffed_icu_adult_patients_confirmed_covid_coverage,total_adult_patients_hospitalized_confirmed_and_suspected_covid,total_adult_patients_hospitalized_confirmed_and_suspected_covid_coverage,total_adult_patients_hospitalized_confirmed_covid,total_adult_patients_hospitalized_confirmed_covid_coverage,total_pediatric_patients_hospitalized_confirmed_and_suspected_covid,total_pediatric_patients_hospitalized_confirmed_and_suspected_covid_coverage,total_pediatric_patients_hospitalized_confirmed_covid,total_pediatric_patients_hospitalized_confirmed_covid_coverage,total_staffed_adult_icu_beds,total_staffed_adult_icu_beds_coverage,inpatient_beds_utilization,inpatient_beds_utilization_coverage,inpatient_beds_utilization_numerator,inpatient_beds_utilization_denominator,percent_of_inpatients_with_covid,percent_of_inpatients_with_covid_coverage,percent_of_inpatients_with_covid_numerator,percent_of_inpatients_with_covid_denominator,inpatient_bed_covid_utilization,inpatient_bed_covid_utilization_coverage,inpatient_bed_covid_utilization_numerator,inpatient_bed_covid_utilization_denominator,adult_icu_bed_covid_utilization,adult_icu_bed_covid_utilization_coverage,adult_icu_bed_covid_utilization_numerator,adult_icu_bed_covid_utilization_denominator,adult_icu_bed_utilization,adult_icu_bed_utilization_coverage,adult_icu_bed_utilization_numerator,adult_icu_bed_utilization_denominator
+WY,2020/08/27,2,,4,2,19,7,0,26,1464,28,629,28,17,26,2,28,13,26,0,21,0,22,49,28,10,26,7,28,17,26,14,28,0,26,0,26,114,28,0.4296448087431694,28,629,1464,0.027597402597402596,26,17,616,0.011946591707659873,26,17,1423,0.09345794392523364,26,10,107,0.4298245614035088,28,49,114