Merge pull request #1091 from cmu-delphi/Issue67

Use epidata client in covidcast integration tests + fix csv format bug + fix fields param bug

Author: minhkhul

integrations/server/test_covidcast.py

@@ -11,6 +11,7 @@
 # first party
 from delphi_utils import Nans
 from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase, CovidcastTestRow
+from delphi.epidata.client.delphi_epidata import Epidata
 
 # use the local instance of the Epidata API
 BASE_URL = 'http://delphi_web_epidata/epidata/api.php'
@@ -22,11 +23,10 @@ def localSetUp(self):
     """Perform per-test setup."""
     self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"')
 
-  def request_based_on_row(self, row: CovidcastTestRow, extract_response: Callable = lambda x: x.json(), **kwargs):
+  def request_based_on_row(self, row: CovidcastTestRow, **kwargs):
     params = self.params_from_row(row, endpoint='covidcast', **kwargs)
-    response = requests.get(BASE_URL, params=params)
-    response.raise_for_status()
-    response = extract_response(response)
+    Epidata.BASE_URL = BASE_URL
+    response = Epidata.covidcast(**params) 
 
     return response
 
@@ -155,7 +155,6 @@ def test_csv_format(self):
     # NB 'format' is a Python reserved word
     response = self.request_based_on_row(
       row,
-      extract_response=lambda resp: resp.text,
       **{'format':'csv'}
     )
 
@@ -194,7 +193,7 @@ def test_fields(self):
     row = self._insert_placeholder_set_one()
 
     # limit fields
-    response = self.request_based_on_row(row, fields='time_value,geo_value')
+    response = self.request_based_on_row(row, **{"fields":"time_value,geo_value"})
 
     expected = row.as_api_compatibility_row_dict()
     expected_all = {

src/client/delphi_epidata.py

@@ -72,7 +72,11 @@ def _request(params):
     long and returns a 414.
     """
     try:
-      return Epidata._request_with_retry(params).json()
+      result = Epidata._request_with_retry(params)
+      if params is not None and "format" in params and params["format"]=="csv":
+        return result.text
+      else:
+        return result.json()
     except Exception as e:
       return {'result': 0, 'message': 'error: ' + str(e)}
 
@@ -606,6 +610,9 @@ def covidcast(
     if 'format' in kwargs:
       params['format'] = kwargs['format']
 
+    if 'fields' in kwargs:
+      params['fields'] = kwargs['fields']
+
     # Make the API call
     return Epidata._request(params)