Merge pull request #1309 from cmu-delphi/release/delphi-epidata-4.1.11

Release Delphi Epidata 4.1.11

Author: melange396

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.1.10
+current_version = 4.1.11
 commit = False
 tag = False
 

dev/local/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Delphi Development
-version = 4.1.10
+version = 4.1.11
 
 [options]
 packages =

docs/new_endpoint_tutorial.md

@@ -123,11 +123,9 @@ Here's what we add to each client:
     def fluview_meta():
       """Fetch FluView metadata."""
       # Set up request
-      params = {
-        'endpoint': 'fluview_meta',
-      }
+      params = {}
       # Make the API call
-      return Epidata._request(params)
+      return Epidata._request("fluview_meta", params)
     ```
 
 - [`delphi_epidata.R`](https://github.com/cmu-delphi/delphi-epidata/blob/dev/src/client/delphi_epidata.R)

docs/symptom-survey/publications.md

@@ -26,6 +26,10 @@ Pandemic"](https://www.pnas.org/topic/548) in *PNAS*:
 
 Research publications using the survey data include:
 
+- Ma, M.Z., Chen, S.X. (2023). [Beyond the surface: accounting for confounders
+  in understanding the link between collectivism and COVID-19 pandemic in the
+  United States](https://doi.org/10.1186/s12889-023-16384-2). *BMC Public
+  Health* 23, 1513.
 - C.K. Ettman, E. Badillo Goicoechea, and E.A. Stuart (2023). [Evolution of
   depression and anxiety over the COVID-19 pandemic and across demographic
   groups in a large sample of U.S. adults](https://doi.org/10.1016/j.focus.2023.100140).
@@ -62,7 +66,7 @@ Research publications using the survey data include:
 - Rönn MM, Menzies NA, Salomon JA (2023). [Vaccination and voting patterns in
   the United States: analysis of COVID-19 and flu surveys from 2010 to
   2022](https://doi.org/10.1016/j.amepre.2023.03.001). *American Journal of
-  Preventive Medicine.*
+  Preventive Medicine* 65 (3), 458-466.
 - Taube JC, Susswein Z, Bansal S (2023). [Spatiotemporal Trends in Self-Reported
   Mask-Wearing Behavior in the United States: Analysis of a Large
   Cross-sectional Survey](https://doi.org/10.2196/42128). *JMIR Public Health

integrations/acquisition/covid_hosp/facility/test_scenarios.py

@@ -28,7 +28,7 @@ def setUp(self):
     self.test_utils = UnitTestUtils(__file__)
 
     # use the local instance of the Epidata API
-    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata'
     Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database

integrations/acquisition/covid_hosp/state_daily/test_scenarios.py

@@ -32,7 +32,7 @@ def setUp(self):
     self.test_utils = UnitTestUtils(__file__)
 
     # use the local instance of the Epidata API
-    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata'
     Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database

integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py

@@ -28,7 +28,7 @@ def setUp(self):
     self.test_utils = UnitTestUtils(__file__)
 
     # use the local instance of the Epidata API
-    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata'
     Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database

integrations/acquisition/covidcast/test_covidcast_meta_caching.py

@@ -22,7 +22,7 @@
 )
 
 # use the local instance of the Epidata API
-BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+BASE_URL = 'http://delphi_web_epidata/epidata'
 
 
 class CovidcastMetaCacheTests(unittest.TestCase):
@@ -82,8 +82,8 @@ def tearDown(self):
 
   @staticmethod
   def _make_request():
-    params = {'endpoint': 'covidcast_meta', 'cached': 'true'}
-    response = requests.get(Epidata.BASE_URL, params=params, auth=Epidata.auth)
+    params = {'cached': 'true'}
+    response = requests.get(f"{Epidata.BASE_URL}/covidcast_meta", params=params, auth=Epidata.auth)
     response.raise_for_status()
     return response.json()
 

integrations/acquisition/covidcast/test_csv_uploading.py

@@ -69,7 +69,7 @@ def setUp(self):
     epidata_cnx.close()
 
     # use the local instance of the Epidata API
-    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata'
     Epidata.auth = ('epidata', 'key')
 
   def tearDown(self):
@@ -132,7 +132,7 @@ def test_uploading(self):
       main(args)
       response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*')
 
-      expected_values = pd.concat([values, pd.DataFrame({ "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records")
+      expected_values = pd.concat([values, pd.DataFrame({ "geo_type": "state", "source": "src-name", "time_type": "day", "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records")
       expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'}
 
       self.assertEqual(response, expected_response)
@@ -161,6 +161,9 @@ def test_uploading(self):
       response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*')
 
       expected_values = pd.concat([values, pd.DataFrame({
+        "geo_type": "state",
+        "source": "src-name",
+        "time_type": "day",
         "time_value": [20200419] * 3,
         "signal": [signal_name] * 3,
         "direction": [None] * 3,
@@ -194,7 +197,7 @@ def test_uploading(self):
       main(args)
       response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*')
 
-      expected_response = {'result': -2, 'message': 'no results'}
+      expected_response = {'epidata': [], 'result': -2, 'message': 'no results'}
 
       self.assertEqual(response, expected_response)
       self.verify_timestamps_and_defaults()
@@ -220,6 +223,9 @@ def test_uploading(self):
       response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*')
 
       expected_values_df = pd.concat([values, pd.DataFrame({
+        "geo_type": "state",
+        "source": "src-name",
+        "time_type": "day", 
         "time_value": [20200419],
         "signal": [signal_name],
         "direction": [None]})], axis=1).rename(columns=uploader_column_rename)
@@ -253,6 +259,9 @@ def test_uploading(self):
       response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*')
 
       expected_values = pd.concat([values, pd.DataFrame({
+        "geo_type": "state",
+        "source": "src-name",
+        "time_type": "day", 
         "time_value": [20200419],
         "signal": [signal_name],
         "direction": [None]
@@ -283,7 +292,7 @@ def test_uploading(self):
       main(args)
       response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*')
 
-      expected_response = {'result': -2, 'message': 'no results'}
+      expected_response = {'epidata': [], 'result': -2, 'message': 'no results'}
 
       self.assertEqual(response, expected_response)
       self.verify_timestamps_and_defaults()

integrations/acquisition/covidcast_nowcast/test_csv_uploading.py

@@ -55,7 +55,7 @@ def setUp(self):
     secrets.db.epi = ('user', 'pass')
 
     # use the local instance of the Epidata API
-    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata'
     Epidata.auth = ('epidata', 'key')
 
   def tearDown(self):

integrations/client/test_delphi_epidata.py

@@ -24,9 +24,9 @@
 def fake_epidata_endpoint(func):
   """This can be used as a decorator to enable a bogus Epidata endpoint to return 404 responses."""
   def wrapper(*args):
-    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/fake_api.php'
+    Epidata.BASE_URL = 'http://delphi_web_epidata/fake_epidata'
     func(*args)
-    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata'
   return wrapper
 
 class DelphiEpidataPythonClientTests(CovidcastBase):
@@ -39,7 +39,7 @@ def localSetUp(self):
     self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"')
 
     # use the local instance of the Epidata API
-    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata'
     Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
@@ -65,8 +65,8 @@ def test_covidcast(self):
       )
 
       expected = [
-        row_latest_issue.as_api_compatibility_row_dict(),
-        rows[-1].as_api_compatibility_row_dict()
+        row_latest_issue.as_api_row_dict(),
+        rows[-1].as_api_row_dict()
       ]
 
       self.assertEqual(response['epidata'], expected)
@@ -85,10 +85,10 @@ def test_covidcast(self):
 
       expected = [{
         rows[0].signal: [
-          row_latest_issue.as_api_compatibility_row_dict(ignore_fields=['signal']),
+          row_latest_issue.as_api_row_dict(ignore_fields=['signal']),
         ],
         rows[-1].signal: [
-          rows[-1].as_api_compatibility_row_dict(ignore_fields=['signal']),
+          rows[-1].as_api_row_dict(ignore_fields=['signal']),
         ],
       }]
 
@@ -105,7 +105,7 @@ def test_covidcast(self):
         **self.params_from_row(rows[0])
       )
 
-      expected = [row_latest_issue.as_api_compatibility_row_dict()]
+      expected = [row_latest_issue.as_api_row_dict()]
 
       # check result
       self.assertEqual(response_1, {
@@ -120,7 +120,7 @@ def test_covidcast(self):
         **self.params_from_row(rows[0], as_of=rows[1].issue)
       )
 
-      expected = [rows[1].as_api_compatibility_row_dict()]
+      expected = [rows[1].as_api_row_dict()]
 
       # check result
       self.maxDiff=None
@@ -130,15 +130,23 @@ def test_covidcast(self):
         'message': 'success',
       })
 
+    with self.subTest(name='bad as-of date'):
+      # fetch data, specifying as_of
+      as_of_response = Epidata.covidcast(
+        **self.params_from_row(rows[0], as_of="20230101-20230102")
+      )
+      self.assertEqual(as_of_response, {"epidata": [], "message": "not a valid date: 20230101-20230102", "result": -1})
+
+
     with self.subTest(name='request a range of issues'):
       # fetch data, specifying issue range, not lag
       response_2 = Epidata.covidcast(
         **self.params_from_row(rows[0], issues=Epidata.range(rows[0].issue, rows[1].issue))
       )
 
       expected = [
-        rows[0].as_api_compatibility_row_dict(),
-        rows[1].as_api_compatibility_row_dict()
+        rows[0].as_api_row_dict(),
+        rows[1].as_api_row_dict()
       ]
 
       # check result
@@ -154,7 +162,7 @@ def test_covidcast(self):
         **self.params_from_row(rows[0], lag=2)
       )
 
-      expected = [row_latest_issue.as_api_compatibility_row_dict()]
+      expected = [row_latest_issue.as_api_row_dict()]
 
       # check result
       self.assertDictEqual(response_3, {
@@ -170,7 +178,7 @@ def test_covidcast(self):
       )
 
       # check result
-      self.assertEqual(response_1, {'message': 'no results', 'result': -2})
+      self.assertEqual(response_1, {'epidata': [], 'message': 'no results', 'result': -2})
 
   @patch('requests.post')
   @patch('requests.get')
@@ -196,7 +204,7 @@ def test_retry_request(self, get):
       mock_response = MagicMock()
       mock_response.status_code = 200
       get.side_effect = [JSONDecodeError('Expecting value', "",  0), mock_response]
-      response = Epidata._request(None)
+      response = Epidata._request("")
       self.assertEqual(get.call_count, 2)
       self.assertEqual(response, mock_response.json())
 
@@ -207,7 +215,7 @@ def test_retry_request(self, get):
       get.side_effect = [JSONDecodeError('Expecting value', "",  0),
                          JSONDecodeError('Expecting value', "",  0),
                          mock_response]
-      response = Epidata._request(None)
+      response = Epidata._request("")
       self.assertEqual(get.call_count, 2)  # 2 from previous test + 2 from this one
       self.assertEqual(response,
                        {'result': 0, 'message': 'error: Expecting value: line 1 column 1 (char 0)'}
@@ -228,7 +236,7 @@ def test_geo_value(self):
     self._insert_rows(rows)
 
     counties = [
-      rows[i].as_api_compatibility_row_dict() for i in range(N)
+      rows[i].as_api_row_dict() for i in range(N)
     ]
 
     def fetch(geo):
@@ -339,13 +347,15 @@ def test_async_epidata(self):
       self.params_from_row(rows[0], source='covidcast'),
       self.params_from_row(rows[1], source='covidcast')
     ]*12, batch_size=10)
-    responses = [i[0] for i in test_output]
-    # check response is same as standard covidcast call, using 24 calls to test batch sizing
+    responses = [i[0]["epidata"] for i in test_output]
+    # check response is same as standard covidcast call (minus fields omitted by the api.php endpoint),
+    # using 24 calls to test batch sizing
+    ignore_fields = CovidcastTestRow._api_row_compatibility_ignore_fields
     self.assertEqual(
       responses,
       [
-        Epidata.covidcast(**self.params_from_row(rows[0])),
-        Epidata.covidcast(**self.params_from_row(rows[1])),
+        [{k: row[k] for k in row.keys() - ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[0]))["epidata"]],
+        [{k: row[k] for k in row.keys() - ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[1]))["epidata"]],
       ]*12
     )
 

integrations/client/test_nowcast.py

@@ -39,7 +39,7 @@ def setUp(self):
     self.cur = cnx.cursor()
 
     # use the local instance of the Epidata API
-    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
+    Epidata.BASE_URL = 'http://delphi_web_epidata/epidata'
     Epidata.auth = ('epidata', 'key')
 
     # use the local instance of the epidata database
@@ -133,4 +133,4 @@ def test_covidcast_nowcast(self):
     response = Epidata.covidcast_nowcast(
       'src', 'sig1', 'sensor', 'day', 'county', 22222222, '01001')
 
-    self.assertEqual(response, {'result': -2, 'message': 'no results'})
+    self.assertEqual(response, {'epidata': [], 'result': -2, 'message': 'no results'})