Server: update test_covidcast to use CovidcastRow

Author: dshemetov

integrations/server/test_covidcast.py

@@ -1,7 +1,7 @@
 """Integration tests for the `covidcast` endpoint."""
 
 # standard library
-import json
+from typing import Callable
 import unittest
 
 # third party
@@ -10,12 +10,12 @@
 
 # first party
 from delphi_utils import Nans
+from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow
 from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase
 
 # use the local instance of the Epidata API
 BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
-
-
+IGNORE_FIELDS = ["id", "direction_updated_timestamp", "value_updated_timestamp", "source", "time_type", "geo_type"]
 
 class CovidcastTests(CovidcastBase):
   """Tests the `covidcast` endpoint."""
@@ -24,64 +24,62 @@ def localSetUp(self):
     """Perform per-test setup."""
     self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"')
 
-  def request_based_on_row(self, row, extract_response=lambda x: x.json(), **kwargs):
+  def request_based_on_row(self, row: CovidcastRow, extract_response: Callable = lambda x: x.json(), **kwargs):
     params = self.params_from_row(row, endpoint='covidcast', **kwargs)
     response = requests.get(BASE_URL, params=params)
     response.raise_for_status()
     response = extract_response(response)
 
-    expected = self.expected_from_row(row)
-
-    return response, expected
+    return response
 
   def _insert_placeholder_set_one(self):
-    row, settings = self._make_placeholder_row()
+    row = CovidcastRow.make_default_row()
     self._insert_rows([row])
     return row
 
   def _insert_placeholder_set_two(self):
     rows = [
-      self._make_placeholder_row(geo_type='county', geo_value=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.)[0]
+      CovidcastRow.make_default_row(geo_type='county', geo_value=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.)
       for i in [1, 2, 3]
     ] + [
       # geo value intended to overlap with counties above
-      self._make_placeholder_row(geo_type='msa', geo_value=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.)[0]
+      CovidcastRow.make_default_row(geo_type='msa', geo_value=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.)
       for i in [4, 5, 6]
     ]
     self._insert_rows(rows)
     return rows
 
   def _insert_placeholder_set_three(self):
     rows = [
-      self._make_placeholder_row(geo_type='county', geo_value='11111', time_value=2000_01_01+i, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=2-i)[0]
+      CovidcastRow.make_default_row(geo_type='county', geo_value='11111', time_value=2000_01_01+i, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=2-i)
       for i in [1, 2, 3]
     ] + [
       # time value intended to overlap with 11111 above, with disjoint geo values
-      self._make_placeholder_row(geo_type='county', geo_value=str(i)*5, time_value=2000_01_01+i-3, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=5-i)[0]
+      CovidcastRow.make_default_row(geo_type='county', geo_value=str(i)*5, time_value=2000_01_01+i-3, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=5-i)
       for i in [4, 5, 6]
     ]
     self._insert_rows(rows)
     return rows
 
   def _insert_placeholder_set_four(self):
     rows = [
-      self._make_placeholder_row(source='src1', signal=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.)[0]
+      CovidcastRow.make_default_row(source='src1', signal=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.)
       for i in [1, 2, 3]
     ] + [
       # signal intended to overlap with the signal above
-      self._make_placeholder_row(source='src2', signal=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.)[0]
+      CovidcastRow.make_default_row(source='src2', signal=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.)
       for i in [4, 5, 6]
     ]
     self._insert_rows(rows)
     return rows
 
   def _insert_placeholder_set_five(self):
     rows = [
-      CovidcastRow(time_value=2000_01_01, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03+i)
+      CovidcastRow.make_default_row(time_value=2000_01_01, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03+i)
       for i in [1, 2, 3]
     ] + [
       # different time_values, same issues
-      CovidcastRow(time_value=2000_01_01+i-3, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03+i-3)
+      CovidcastRow.make_default_row(time_value=2000_01_01+i-3, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03+i-3)
       for i in [4, 5, 6]
     ]
     self._insert_rows(rows)
@@ -94,10 +92,13 @@ def test_round_trip(self):
     row = self._insert_placeholder_set_one()
 
     # make the request
-    response, expected = self.request_based_on_row(row)
+    response = self.request_based_on_row(row)
+
+    expected = [row.as_dict(ignore_fields=IGNORE_FIELDS)]
+
     self.assertEqual(response, {
       'result': 1,
-      'epidata': [expected],
+      'epidata': expected,
       'message': 'success',
     })
 
@@ -154,32 +155,25 @@ def test_csv_format(self):
 
     # make the request
     # NB 'format' is a Python reserved word
-    response, _ = self.request_based_on_row(
+    response = self.request_based_on_row(
       row,
       extract_response=lambda resp: resp.text,
       **{'format':'csv'}
     )
-    expected_response = (
-      "geo_value,signal,time_value,direction,issue,lag,missing_value," +
-      "missing_stderr,missing_sample_size,value,stderr,sample_size\n" +
-      ",".join("" if x is None else str(x) for x in [
-        row.geo_value,
-        row.signal,
-        row.time_value,
-        row.direction,
-        row.issue,
-        row.lag,
-        row.missing_value,
-        row.missing_stderr,
-        row.missing_sample_size,
-        row.value,
-        row.stderr,
-        row.sample_size
-      ]) + "\n"
+
+    # TODO: This is a mess because of api.php.
+    column_order = [
+      "geo_value", "signal", "time_value", "direction", "issue", "lag", "missing_value",
+      "missing_stderr", "missing_sample_size", "value", "stderr", "sample_size"
+    ]
+    expected = (
+      row.api_compatibility_row_df
+         .assign(direction = None)
+         .to_csv(columns=column_order, index=False)
     )
 
     # assert that the right data came back
-    self.assertEqual(response, expected_response)
+    self.assertEqual(response, expected)
 
   def test_raw_json_format(self):
     """Test generate raw json data."""
@@ -188,10 +182,12 @@ def test_raw_json_format(self):
     row = self._insert_placeholder_set_one()
 
     # make the request
-    response, expected = self.request_based_on_row(row, **{'format':'json'})
+    response = self.request_based_on_row(row, **{'format':'json'})
+
+    expected = [row.as_dict(ignore_fields=IGNORE_FIELDS)]
 
     # assert that the right data came back
-    self.assertEqual(response, [expected])
+    self.assertEqual(response, expected)
 
   def test_fields(self):
     """Test fields parameter"""
@@ -200,7 +196,9 @@ def test_fields(self):
     row = self._insert_placeholder_set_one()
 
     # limit fields
-    response, expected = self.request_based_on_row(row, fields='time_value,geo_value')
+    response = self.request_based_on_row(row, fields='time_value,geo_value')
+
+    expected = row.as_dict(ignore_fields=IGNORE_FIELDS)
     expected_all = {
       'result': 1,
       'epidata': [{
@@ -213,15 +211,14 @@ def test_fields(self):
     self.assertEqual(response, expected_all)
 
     # limit using invalid fields
-    response, _ = self.request_based_on_row(row, fields='time_value,geo_value,doesnt_exist')
+    response = self.request_based_on_row(row, fields='time_value,geo_value,doesnt_exist')
 
     # assert that the right data came back (only valid fields)
     self.assertEqual(response, expected_all)
 
 
     # limit exclude fields: exclude all except time_value and geo_value
-
-    response, _ = self.request_based_on_row(row, fields=(
+    response = self.request_based_on_row(row, fields=(
         '-value,-stderr,-sample_size,-direction,-issue,-lag,-signal,' +
         '-missing_value,-missing_stderr,-missing_sample_size'
     ))
@@ -234,18 +231,15 @@ def test_location_wildcard(self):
 
     # insert placeholder data
     rows = self._insert_placeholder_set_two()
-    expected_counties = [
-      self.expected_from_row(r) for r in rows[:3]
-    ]
-
+    expected = [row.as_dict(ignore_fields=IGNORE_FIELDS) for row in rows[:3]]
     # make the request
-    response, _ = self.request_based_on_row(rows[0], geo_value="*")
+    response = self.request_based_on_row(rows[0], geo_value="*")
 
     self.maxDiff = None
     # assert that the right data came back
     self.assertEqual(response, {
       'result': 1,
-      'epidata': expected_counties,
+      'epidata': expected,
       'message': 'success',
     })
 
@@ -290,12 +284,10 @@ def test_signal_wildcard(self):
 
     # insert placeholder data
     rows = self._insert_placeholder_set_four()
-    expected_signals = [
-      self.expected_from_row(r) for r in rows[:3]
-    ]
+    expected_signals = [row.as_dict(ignore_fields=IGNORE_FIELDS) for row in rows[:3]]
 
     # make the request
-    response, _ = self.request_based_on_row(rows[0], signals="*")
+    response = self.request_based_on_row(rows[0], signals="*")
 
     self.maxDiff = None
     # assert that the right data came back
@@ -310,35 +302,33 @@ def test_geo_value(self):
 
     # insert placeholder data
     rows = self._insert_placeholder_set_two()
-    expected_counties = [
-      self.expected_from_row(r) for r in rows[:3]
-    ]
+    expected = [row.as_dict(ignore_fields=IGNORE_FIELDS) for row in rows[:3]]
 
     def fetch(geo_value):
       # make the request
-      response, _ = self.request_based_on_row(rows[0], geo_value=geo_value)
+      response = self.request_based_on_row(rows[0], geo_value=geo_value)
 
       return response
 
     # test fetch a specific region
     r = fetch('11111')
     self.assertEqual(r['message'], 'success')
-    self.assertEqual(r['epidata'], [expected_counties[0]])
+    self.assertEqual(r['epidata'], expected[0:1])
     # test fetch a specific yet not existing region
     r = fetch('55555')
     self.assertEqual(r['message'], 'no results')
     # test fetch multiple regions
     r = fetch('11111,22222')
     self.assertEqual(r['message'], 'success')
-    self.assertEqual(r['epidata'], [expected_counties[0], expected_counties[1]])
+    self.assertEqual(r['epidata'], expected[0:2])
     # test fetch multiple noncontiguous regions
     r = fetch('11111,33333')
     self.assertEqual(r['message'], 'success')
-    self.assertEqual(r['epidata'], [expected_counties[0], expected_counties[2]])
+    self.assertEqual(r['epidata'], [expected[0], expected[2]])
     # test fetch multiple regions but one is not existing
     r = fetch('11111,55555')
     self.assertEqual(r['message'], 'success')
-    self.assertEqual(r['epidata'], [expected_counties[0]])
+    self.assertEqual(r['epidata'], expected[0:1])
     # test fetch empty region
     r = fetch('')
     self.assertEqual(r['message'], 'no results')
@@ -348,12 +338,10 @@ def test_location_timeline(self):
 
     # insert placeholder data
     rows = self._insert_placeholder_set_three()
-    expected_timeseries = [
-      self.expected_from_row(r) for r in rows[:3]
-    ]
+    expected_timeseries = [row.as_dict(ignore_fields=IGNORE_FIELDS) for row in rows[:3]]
 
     # make the request
-    response, _ = self.request_based_on_row(rows[0], time_values='20000101-20000105')
+    response = self.request_based_on_row(rows[0], time_values='20000101-20000105')
 
     # assert that the right data came back
     self.assertEqual(response, {
@@ -379,15 +367,16 @@ def test_unique_key_constraint(self):
   def test_nullable_columns(self):
     """Missing values should be surfaced as null."""
 
-    row, _ = self._make_placeholder_row(
+    row = CovidcastRow.make_default_row(
       stderr=None, sample_size=None,
       missing_stderr=Nans.OTHER.value, missing_sample_size=Nans.OTHER.value
     )
     self._insert_rows([row])
 
     # make the request
-    response, expected = self.request_based_on_row(row)
-    expected.update(stderr=None, sample_size=None)
+    response = self.request_based_on_row(row)
+    expected = row.as_dict(ignore_fields=IGNORE_FIELDS)
+    # expected.update(stderr=None, sample_size=None)
 
     # assert that the right data came back
     self.assertEqual(response, {
@@ -401,18 +390,19 @@ def test_temporal_partitioning(self):
 
     # insert placeholder data
     rows = [
-      self._make_placeholder_row(time_type=tt)[0]
+      CovidcastRow.make_default_row(time_type=tt)
       for tt in "hour day week month year".split()
     ]
     self._insert_rows(rows)
 
     # make the request
-    response, expected = self.request_based_on_row(rows[1], time_values="0-99999999")
+    response = self.request_based_on_row(rows[1], time_values="20000101-30010201")
+    expected = [rows[1].as_dict(ignore_fields=IGNORE_FIELDS)]
 
     # assert that the right data came back
     self.assertEqual(response, {
       'result': 1,
-      'epidata': [expected],
+      'epidata': expected,
       'message': 'success',
     })
 
@@ -423,37 +413,37 @@ def test_date_formats(self):
     rows = self._insert_placeholder_set_three()
 
     # make the request
-    response, expected = self.request_based_on_row(rows[0], time_values="20000102", geo_value="*")
+    response = self.request_based_on_row(rows[0], time_values="20000102", geo_value="*")
 
     # assert that the right data came back
     self.assertEqual(len(response['epidata']), 2)
 
     # make the request
-    response, expected = self.request_based_on_row(rows[0], time_values="2000-01-02", geo_value="*")
+    response = self.request_based_on_row(rows[0], time_values="2000-01-02", geo_value="*")
 
     # assert that the right data came back
     self.assertEqual(len(response['epidata']), 2)
 
     # make the request
-    response, expected = self.request_based_on_row(rows[0], time_values="20000102,20000103", geo_value="*")
+    response = self.request_based_on_row(rows[0], time_values="20000102,20000103", geo_value="*")
 
     # assert that the right data came back
-    self.assertEqual(len(response['epidata']), 4)
+    self.assertEqual(len(response['epidata']), 2 * 2)
 
     # make the request
-    response, expected = self.request_based_on_row(rows[0], time_values="2000-01-02,2000-01-03", geo_value="*")
+    response = self.request_based_on_row(rows[0], time_values="2000-01-02,2000-01-03", geo_value="*")
 
     # assert that the right data came back
-    self.assertEqual(len(response['epidata']), 4)
+    self.assertEqual(len(response['epidata']), 2 * 2)
 
     # make the request
-    response, expected = self.request_based_on_row(rows[0], time_values="20000102-20000104", geo_value="*")
+    response = self.request_based_on_row(rows[0], time_values="20000102-20000104", geo_value="*")
 
     # assert that the right data came back
-    self.assertEqual(len(response['epidata']), 6)
+    self.assertEqual(len(response['epidata']), 2 * 3)
 
     # make the request
-    response, expected = self.request_based_on_row(rows[0], time_values="2000-01-02:2000-01-04", geo_value="*")
+    response = self.request_based_on_row(rows[0], time_values="2000-01-02:2000-01-04", geo_value="*")
 
     # assert that the right data came back
-    self.assertEqual(len(response['epidata']), 6)
+    self.assertEqual(len(response['epidata']), 2 * 3)

tests/server/test_pandas.py

@@ -9,7 +9,6 @@
 from delphi.epidata.server._pandas import as_pandas
 from delphi.epidata.server._config import MAX_RESULTS
 
-
 # py3tester coverage target
 __test_target__ = "delphi.epidata.server._pandas"