CovidcastRow: address code review

Co-authored-by: Katie Mazaitis <[email protected]>
Co-authored-by: melange396 <[email protected]>

Author: 3 people

integrations/acquisition/covidcast/test_db.py

@@ -1,8 +1,7 @@
 from delphi_utils import Nans
 
 from delphi.epidata.acquisition.covidcast.database import DBLoadStateException
-from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow
-from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase
+from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow
 
 
 # all the Nans we use here are just one value, so this is a shortcut to it:
@@ -11,7 +10,7 @@
 class TestTest(CovidcastBase):
 
     def _find_matches_for_row(self, row):
-        # finds (if existing) row from both history and latest views that matches long-key of provided CovidcastRow
+        # finds (if existing) row from both history and latest views that matches long-key of provided CovidcastTestRow
         cols = "source signal time_type time_value geo_type geo_value issue".split()
         results = {}
         cur = self._db._cursor
@@ -31,8 +30,8 @@ def _find_matches_for_row(self, row):
 
     def test_insert_or_update_with_nonempty_load_table(self):
         # make rows
-        a_row = CovidcastRow.make_default_row(time_value=2020_02_02)
-        another_row = CovidcastRow.make_default_row(time_value=2020_02_03, issue=2020_02_03)
+        a_row = CovidcastTestRow.make_default_row(time_value=2020_02_02)
+        another_row = CovidcastTestRow.make_default_row(time_value=2020_02_03, issue=2020_02_03)
         # insert one
         self._db.insert_or_update_bulk([a_row])
         # put something into the load table
@@ -61,7 +60,7 @@ def test_id_sync(self):
         latest_view = 'epimetric_latest_v'
 
         # add a data point
-        base_row = CovidcastRow.make_default_row()
+        base_row = CovidcastTestRow.make_default_row()
         self._insert_rows([base_row])
         # ensure the primary keys match in the latest and history tables
         matches = self._find_matches_for_row(base_row)
@@ -71,7 +70,7 @@ def test_id_sync(self):
         old_pk_id = matches[latest_view][pk_column]
 
         # add a reissue for said data point
-        next_row = CovidcastRow.make_default_row()
+        next_row = CovidcastTestRow.make_default_row()
         next_row.issue += 1
         self._insert_rows([next_row])
         # ensure the new keys also match

integrations/acquisition/covidcast/test_delete_batch.py

@@ -8,7 +8,7 @@
 # first party
 import delphi.operations.secrets as secrets
 from delphi.epidata.acquisition.covidcast.database import Database
-from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow, covidcast_rows_from_args
+from delphi.epidata.acquisition.covidcast.test_utils import covidcast_rows_from_args
 
 # py3tester coverage target (equivalent to `import *`)
 __test_target__ = 'delphi.epidata.acquisition.covidcast.database'
@@ -57,6 +57,7 @@ def _test_delete_batch(self, cc_deletions):
             time_value = [0] * 5 + [1] * 5 + [0],
             geo_value = ["d_nonlatest"] * 2 + ["d_latest"] * 3 + ["d_nonlatest"] * 2 + ["d_latest"] * 3 + ["d_justone"],
             issue = [1, 2] + [1, 2, 3] + [1, 2] + [1, 2, 3] + [1],
+            sanitize_fields = True
         )
 
         self._db.insert_or_update_bulk(rows)

integrations/client/test_delphi_epidata.py

@@ -12,8 +12,7 @@
 # third party
 import delphi.operations.secrets as secrets
 from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import main as update_covidcast_meta_cache
-from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow
-from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase
+from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow
 from delphi.epidata.client.delphi_epidata import Epidata
 from delphi_utils import Nans
 
@@ -52,13 +51,11 @@ def test_covidcast(self):
 
     # insert placeholder data: three issues of one signal, one issue of another
     rows = [
-      CovidcastRow.make_default_row(issue=2020_02_02 + i, value=i, lag=i)
+      CovidcastTestRow.make_default_row(issue=2020_02_02 + i, value=i, lag=i)
       for i in range(3)
     ]
     row_latest_issue = rows[-1]
-    rows.append(
-      CovidcastRow.make_default_row(signal="sig2")
-    )
+    rows.append(CovidcastTestRow.make_default_row(signal="sig2"))
     self._insert_rows(rows)
 
     with self.subTest(name='request two signals'):
@@ -222,10 +219,10 @@ def test_geo_value(self):
     # insert placeholder data: three counties, three MSAs
     N = 3
     rows = [
-      CovidcastRow.make_default_row(geo_type="county", geo_value=str(i)*5, value=i)
+      CovidcastTestRow.make_default_row(geo_type="county", geo_value=str(i)*5, value=i)
       for i in range(N)
     ] + [
-      CovidcastRow.make_default_row(geo_type="msa", geo_value=str(i)*5, value=i*10)
+      CovidcastTestRow.make_default_row(geo_type="msa", geo_value=str(i)*5, value=i*10)
       for i in range(N)
     ]
     self._insert_rows(rows)
@@ -277,7 +274,11 @@ def test_covidcast_meta(self):
     # 2nd issue: 1 11 21
     # 3rd issue: 2 12 22
     rows = [
-      CovidcastRow.make_default_row(time_value=DEFAULT_TIME_VALUE + t, issue=DEFAULT_ISSUE + i, value=t*10 + i)
+      CovidcastTestRow.make_default_row(
+        time_value=DEFAULT_TIME_VALUE + t,
+        issue=DEFAULT_ISSUE + i,
+        value=t*10 + i
+      )
       for i in range(3) for t in range(3)
     ]
     self._insert_rows(rows)
@@ -324,10 +325,10 @@ def test_async_epidata(self):
     # insert placeholder data: three counties, three MSAs
     N = 3
     rows = [
-      CovidcastRow.make_default_row(geo_type="county", geo_value=str(i)*5, value=i)
+      CovidcastTestRow.make_default_row(geo_type="county", geo_value=str(i)*5, value=i)
       for i in range(N)
     ] + [
-      CovidcastRow.make_default_row(geo_type="msa", geo_value=str(i)*5, value=i*10)
+      CovidcastTestRow.make_default_row(geo_type="msa", geo_value=str(i)*5, value=i*10)
       for i in range(N)
     ]
     self._insert_rows(rows)

integrations/server/test_covidcast.py

@@ -10,8 +10,7 @@
 
 # first party
 from delphi_utils import Nans
-from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow
-from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase
+from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow
 
 # use the local instance of the Epidata API
 BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
@@ -23,7 +22,7 @@ def localSetUp(self):
     """Perform per-test setup."""
     self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"')
 
-  def request_based_on_row(self, row: CovidcastRow, extract_response: Callable = lambda x: x.json(), **kwargs):
+  def request_based_on_row(self, row: CovidcastTestRow, extract_response: Callable = lambda x: x.json(), **kwargs):
     params = self.params_from_row(row, endpoint='covidcast', **kwargs)
     response = requests.get(BASE_URL, params=params)
     response.raise_for_status()
@@ -32,53 +31,53 @@ def request_based_on_row(self, row: CovidcastRow, extract_response: Callable = l
     return response
 
   def _insert_placeholder_set_one(self):
-    row = CovidcastRow.make_default_row()
+    row = CovidcastTestRow.make_default_row()
     self._insert_rows([row])
     return row
 
   def _insert_placeholder_set_two(self):
     rows = [
-      CovidcastRow.make_default_row(geo_type='county', geo_value=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.)
+      CovidcastTestRow.make_default_row(geo_type='county', geo_value=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.)
       for i in [1, 2, 3]
     ] + [
       # geo value intended to overlap with counties above
-      CovidcastRow.make_default_row(geo_type='msa', geo_value=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.)
+      CovidcastTestRow.make_default_row(geo_type='msa', geo_value=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.)
       for i in [4, 5, 6]
     ]
     self._insert_rows(rows)
     return rows
 
   def _insert_placeholder_set_three(self):
     rows = [
-      CovidcastRow.make_default_row(geo_type='county', geo_value='11111', time_value=2000_01_01+i, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=2-i)
+      CovidcastTestRow.make_default_row(geo_type='county', geo_value='11111', time_value=2000_01_01+i, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=2-i)
       for i in [1, 2, 3]
     ] + [
       # time value intended to overlap with 11111 above, with disjoint geo values
-      CovidcastRow.make_default_row(geo_type='county', geo_value=str(i)*5, time_value=2000_01_01+i-3, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=5-i)
+      CovidcastTestRow.make_default_row(geo_type='county', geo_value=str(i)*5, time_value=2000_01_01+i-3, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=5-i)
       for i in [4, 5, 6]
     ]
     self._insert_rows(rows)
     return rows
 
   def _insert_placeholder_set_four(self):
     rows = [
-      CovidcastRow.make_default_row(source='src1', signal=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.)
+      CovidcastTestRow.make_default_row(source='src1', signal=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.)
       for i in [1, 2, 3]
     ] + [
       # signal intended to overlap with the signal above
-      CovidcastRow.make_default_row(source='src2', signal=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.)
+      CovidcastTestRow.make_default_row(source='src2', signal=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.)
       for i in [4, 5, 6]
     ]
     self._insert_rows(rows)
     return rows
 
   def _insert_placeholder_set_five(self):
     rows = [
-      CovidcastRow.make_default_row(time_value=2000_01_01, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03+i)
+      CovidcastTestRow.make_default_row(time_value=2000_01_01, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03+i)
       for i in [1, 2, 3]
     ] + [
       # different time_values, same issues
-      CovidcastRow.make_default_row(time_value=2000_01_01+i-3, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03+i-3)
+      CovidcastTestRow.make_default_row(time_value=2000_01_01+i-3, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03+i-3)
       for i in [4, 5, 6]
     ]
     self._insert_rows(rows)
@@ -366,7 +365,7 @@ def test_unique_key_constraint(self):
   def test_nullable_columns(self):
     """Missing values should be surfaced as null."""
 
-    row = CovidcastRow.make_default_row(
+    row = CovidcastTestRow.make_default_row(
       stderr=None, sample_size=None,
       missing_stderr=Nans.OTHER.value, missing_sample_size=Nans.OTHER.value
     )
@@ -388,7 +387,7 @@ def test_temporal_partitioning(self):
 
     # insert placeholder data
     rows = [
-      CovidcastRow.make_default_row(time_type=tt)
+      CovidcastTestRow.make_default_row(time_type=tt)
       for tt in "hour day week month year".split()
     ]
     self._insert_rows(rows)

integrations/server/test_covidcast_endpoints.py

@@ -1,26 +1,16 @@
 """Integration tests for the custom `covidcast/*` endpoints."""
 
 # standard library
-from copy import copy
-from itertools import accumulate, chain
-from typing import Iterable, Dict, Any, List, Sequence
-import unittest
 from io import StringIO
-
-# from typing import Optional
-from dataclasses import dataclass
+from typing import Sequence
 
 # third party
-import mysql.connector
-from more_itertools import interleave_longest, windowed
+from more_itertools import windowed
 import requests
 import pandas as pd
-import numpy as np
-from delphi_utils import Nans
 
 from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import main as update_cache
-from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow
-from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase
+from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow
 
 # use the local instance of the Epidata API
 BASE_URL = "http://delphi_web_epidata/epidata/covidcast"
@@ -64,7 +54,7 @@ def _smooth_rows(self, rows: Sequence[float]):
 
     def test_basic(self):
         """Request a signal from the / endpoint."""
-        rows = [CovidcastRow.make_default_row(time_value=2020_04_01 + i, value=i) for i in range(10)]
+        rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i) for i in range(10)]
         first = rows[0]
         self._insert_rows(rows)
 
@@ -78,7 +68,7 @@ def test_basic(self):
 
     def test_compatibility(self):
         """Request at the /api.php endpoint."""
-        rows = [CovidcastRow.make_default_row(source="src", signal="sig", time_value=2020_04_01 + i, value=i) for i in range(10)]
+        rows = [CovidcastTestRow.make_default_row(source="src", signal="sig", time_value=2020_04_01 + i, value=i) for i in range(10)]
         first = rows[0]
         self._insert_rows(rows)
 
@@ -94,7 +84,7 @@ def test_trend(self):
         """Request a signal from the /trend endpoint."""
 
         num_rows = 30
-        rows = [CovidcastRow.make_default_row(time_value=2020_04_01 + i, value=i) for i in range(num_rows)]
+        rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i) for i in range(num_rows)]
         first = rows[0]
         last = rows[-1]
         ref = rows[num_rows // 2]
@@ -130,7 +120,7 @@ def test_trendseries(self):
         """Request a signal from the /trendseries endpoint."""
 
         num_rows = 3
-        rows = [CovidcastRow.make_default_row(time_value=2020_04_01 + i, value=num_rows - i) for i in range(num_rows)]
+        rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=num_rows - i) for i in range(num_rows)]
         first = rows[0]
         last = rows[-1]
         self._insert_rows(rows)
@@ -196,10 +186,10 @@ def test_correlation(self):
         """Request a signal from the /correlation endpoint."""
 
         num_rows = 30
-        reference_rows = [CovidcastRow.make_default_row(signal="ref", time_value=20200401 + i, value=i) for i in range(num_rows)]
+        reference_rows = [CovidcastTestRow.make_default_row(signal="ref", time_value=20200401 + i, value=i) for i in range(num_rows)]
         first = reference_rows[0]
         self._insert_rows(reference_rows)
-        other_rows = [CovidcastRow.make_default_row(signal="other", time_value=20200401 + i, value=i) for i in range(num_rows)]
+        other_rows = [CovidcastTestRow.make_default_row(signal="other", time_value=20200401 + i, value=i) for i in range(num_rows)]
         other = other_rows[0]
         self._insert_rows(other_rows)
         max_lag = 3
@@ -222,23 +212,29 @@ def test_correlation(self):
     def test_csv(self):
         """Request a signal from the /csv endpoint."""
 
-        rows = [CovidcastRow.make_default_row(time_value=2020_04_01 + i, value=i) for i in range(10)]
+        rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i) for i in range(10)]
         first = rows[0]
         self._insert_rows(rows)
 
         response = requests.get(
             f"{BASE_URL}/csv",
             params=dict(signal=first.signal_pair(), start_day="2020-04-01", end_day="2020-12-12", geo_type=first.geo_type),
         )
+        response.raise_for_status()
+        out = response.text
+        df = pd.read_csv(StringIO(out), index_col=0)
+        self.assertEqual(df.shape, (len(rows), 10))
+        self.assertEqual(list(df.columns), ["geo_value", "signal", "time_value", "issue", "lag", "value", "stderr", "sample_size", "geo_type", "data_source"])
+
 
     def test_backfill(self):
         """Request a signal from the /backfill endpoint."""
 
         TEST_DATE_VALUE = 2020_04_01
         num_rows = 10
-        issue_0 = [CovidcastRow.make_default_row(time_value=TEST_DATE_VALUE + i, value=i, sample_size=1, lag=0, issue=TEST_DATE_VALUE + i) for i in range(num_rows)]
-        issue_1 = [CovidcastRow.make_default_row(time_value=TEST_DATE_VALUE + i, value=i + 1, sample_size=2, lag=1, issue=TEST_DATE_VALUE + i + 1) for i in range(num_rows)]
-        last_issue = [CovidcastRow.make_default_row(time_value=TEST_DATE_VALUE + i, value=i + 2, sample_size=3, lag=2, issue=TEST_DATE_VALUE + i + 2) for i in range(num_rows)] # <-- the latest issues
+        issue_0 = [CovidcastTestRow.make_default_row(time_value=TEST_DATE_VALUE + i, value=i, sample_size=1, lag=0, issue=TEST_DATE_VALUE + i) for i in range(num_rows)]
+        issue_1 = [CovidcastTestRow.make_default_row(time_value=TEST_DATE_VALUE + i, value=i + 1, sample_size=2, lag=1, issue=TEST_DATE_VALUE + i + 1) for i in range(num_rows)]
+        last_issue = [CovidcastTestRow.make_default_row(time_value=TEST_DATE_VALUE + i, value=i + 2, sample_size=3, lag=2, issue=TEST_DATE_VALUE + i + 2) for i in range(num_rows)] # <-- the latest issues
         self._insert_rows([*issue_0, *issue_1, *last_issue])
         first = issue_0[0]
 
@@ -264,7 +260,7 @@ def test_meta(self):
         """Request a signal from the /meta endpoint."""
 
         num_rows = 10
-        rows = [CovidcastRow.make_default_row(time_value=2020_04_01 + i, value=i, source="fb-survey", signal="smoothed_cli") for i in range(num_rows)]
+        rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="fb-survey", signal="smoothed_cli") for i in range(num_rows)]
         self._insert_rows(rows)
         first = rows[0]
         last = rows[-1]
@@ -305,7 +301,7 @@ def test_coverage(self):
 
         num_geos_per_date = [10, 20, 30, 40, 44]
         dates = [2020_04_01 + i for i in range(len(num_geos_per_date))]
-        rows = [CovidcastRow.make_default_row(time_value=dates[i], value=i, geo_value=str(geo_value)) for i, num_geo in enumerate(num_geos_per_date) for geo_value in range(num_geo)]
+        rows = [CovidcastTestRow.make_default_row(time_value=dates[i], value=i, geo_value=str(geo_value)) for i, num_geo in enumerate(num_geos_per_date) for geo_value in range(num_geo)]
         self._insert_rows(rows)
         first = rows[0]