From 6e86a08d3f9062e014521cae7bedb4a9f36f2cc2 Mon Sep 17 00:00:00 2001 From: elmotec Date: Sun, 18 Oct 2015 09:42:58 -0400 Subject: [PATCH 1/3] Fixed test_search Removed usage of sys.stderr in mock exception which caused the stream to be closed. This in turn failed the next test. Simplified test_search to avoid the zip and the loop. --- fredapi/tests/test_fred.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/fredapi/tests/test_fred.py b/fredapi/tests/test_fred.py index a88c4e8..1f616d2 100644 --- a/fredapi/tests/test_fred.py +++ b/fredapi/tests/test_fred.py @@ -79,26 +79,32 @@ def __init__(self, rel_url, response=None, side_effect=None): - - - @@ -150,7 +156,6 @@ def setUp(self): self.fake_fred_call = fake_fred_call self.__original_urlopen = fredapi.fred.urlopen - def tearDown(self): """Cleanup.""" pass @@ -230,9 +235,9 @@ def test_invalid_kwarg_in_get_series(self, urlopen): """Test invalid keyword argument in call to get_series.""" url = '{}/series?series_id=invalid&api_key={}'.format(self.root_url, fred_api_key) - side_effect = fredapi.fred.HTTPError(url, 400, '', '', sys.stderr) + side_effect = fredapi.fred.HTTPError(url, 400, '', '', io.StringIO()) self.prepare_urlopen(urlopen, side_effect=side_effect) - with self.assertRaises(ValueError) as context: + with self.assertRaises(ValueError): self.fred.get_series('SP500', observation_start='invalid-datetime-str') self.assertFalse(urlopen.called) @@ -249,12 +254,11 @@ def test_search(self, urlopen): 'seasonal_adjustment_short']]) expected = textwrap.dedent('''\ popularity observation_start seasonal_adjustment_short - series id + series id PCPI01001 0 1969-01-01 NSA PCPI01003 0 1969-01-01 NSA PCPI01005 0 1969-01-01 NSA''') - for aline, eline in zip(actual.split('\n'), expected.split('\n')): - self.assertEqual(aline.strip(), eline.strip()) + self.assertEqual(actual.split('\n'), expected.split('\n')) if __name__ == '__main__': From 095b9f66691e0a9ec01d3c0f13c5b79f7ba482c2 Mon Sep 17 00:00:00 2001 From: elmotec Date: Sat, 25 Jul 2015 22:58:18 -0400 Subject: [PATCH 2/3] Improved handling of realtime parameters in get_series(). Arguments realtime_start and realtime_end in get_series() now cause a pandas.DataFrame to be returned with pandas.MultiIndex for realtime data. Added simple test for the new feature and documentation. Added __init__.py in fredapi.tests so it's correctly interpreted as a package. Now we could revert to python setup.py test in .travis.yml. Fixed test_invalid_kwarg_in_get_series() as we sometimes get a TypeError and sometimes a ValueError. Seems that pandas passes through whatever exception it gets, might be a good reason for this so we follow the same policy. Simplified comparison of dataframe output in tests. --- README.md | 13 ++++++++ fredapi/fred.py | 51 ++++++++++++++++++++++++------ fredapi/tests/test_fred.py | 65 +++++++++++++++++++++++++++++++++++--- 3 files changed, 114 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index d3324f7..e24b548 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,18 @@ For instance, there has been three observations (data points) for the GDP of 201 This means the GDP value for Q1 2014 has been released three times. First release was on 4/30/2014 for a value of 17149.6, and then there have been two revisions on 5/29/2014 and 6/25/2014 for revised values of 17101.3 and 17016.0, respectively. +If you pass realtime_start and/or realtime_end to `get_series`, you will get a pandas.DataFrame with a pandas.MultiIndex instead of a pandas.Series. + +For instance, with observation_start and observation_end set to 2015-01-01 and +realtime_start set to 2015-01-01, one will get: +``` + GDP +obs_date rt_start rt_end +2015-01-01 2015-04-29 2015-05-28 00:00:00 17710.0 + 2015-05-29 2015-06-23 00:00:00 17665.0 + 2015-06-24 9999-12-31 17693.3 +``` + ### Get first data release only (i.e. ignore revisions) ```python @@ -83,6 +95,7 @@ this outputs: 2014-04-01 17294.7 dtype: float64 ``` + ### Get latest data known on a given date ```python diff --git a/fredapi/fred.py b/fredapi/fred.py index f4a27b1..bdae0bb 100644 --- a/fredapi/fred.py +++ b/fredapi/fred.py @@ -98,7 +98,9 @@ def get_series_info(self, series_id): info = pd.Series(root.getchildren()[0].attrib) return info - def get_series(self, series_id, observation_start=None, observation_end=None, **kwargs): + def get_series(self, series_id, observation_start=None, + observation_end=None, realtime_start=None, + realtime_end=None, **kwargs): """ Get data for a Fred series id. This fetches the latest known data, and is equivalent to get_series_latest_release() @@ -106,17 +108,25 @@ def get_series(self, series_id, observation_start=None, observation_end=None, ** ---------- series_id : str Fred series id such as 'CPIAUCSL' - observation_start : datetime or datetime-like str such as '7/1/2014', optional - earliest observation date - observation_end : datetime or datetime-like str such as '7/1/2014', optional - latest observation date + + observation_start : datetime or datetime-like str such as '7/1/2014' + earliest observation date (optional) + observation_end : datetime or datetime-like str such as '7/1/2014' + latest observation date (optional) + realtime_start : datetime or datetime-like str such as '7/1/2014' + earliest as-of date (optional) + realtime_end : datetime or datetime-like str such as '7/1/2014' + latest as-of date (optional) kwargs : additional parameters - Any additional parameters supported by FRED. You can see https://api.stlouisfed.org/docs/fred/series_observations.html for the full list + Any additional parameters supported by FRED. You can see + https://api.stlouisfed.org/docs/fred/series_observations.html + for the full list Returns ------- data : Series - a Series where each index is the observation date and the value is the data for the Fred series + a pandas Series where each index is the observation date and the + value is the data for the Fred series """ url = "%s/series/observations?series_id=%s" % (self.root_url, series_id) if observation_start is not None: @@ -126,20 +136,41 @@ def get_series(self, series_id, observation_start=None, observation_end=None, ** if observation_end is not None: observation_end = pd.to_datetime(observation_end, errors='raise') url += '&observation_end=' + observation_end.strftime('%Y-%m-%d') + if realtime_start is not None: + realtime_start = pd.to_datetime(realtime_start, errors='raise') + url += '&realtime_start=' + realtime_start.strftime('%Y-%m-%d') + if realtime_end is not None: + realtime_end = pd.to_datetime(realtime_end, errors='raise') + url += '&realtime_end=' + realtime_end.strftime('%Y-%m-%d') if kwargs.keys(): url += '&' + urlencode(kwargs) root = self.__fetch_data(url) if root is None: raise ValueError('No data exists for series id: ' + series_id) - data = {} + realtime = (realtime_start or realtime_end) + values = [] + obsdates = [] + rtstarts = [] + rtends = [] for child in root.getchildren(): val = child.get('value') if val == self.nan_char: val = float('NaN') else: val = float(val) - data[self._parse(child.get('date'))] = val - return pd.Series(data) + values.append(val) + obsdates.append(self._parse(child.get('date'))) + if realtime: + rtstarts.append(self._parse(child.get('realtime_start'))) + rtends.append(self._parse(child.get('realtime_end'))) + if realtime: + names = ['obs_date', 'rt_start', 'rt_end'] + index = pd.MultiIndex.from_arrays([obsdates, rtstarts, rtends], + names=names) + return pd.DataFrame(values, index=index, columns=[series_id]) + else: + return pd.Series(values, index=obsdates) + def get_series_latest_release(self, series_id): """ diff --git a/fredapi/tests/test_fred.py b/fredapi/tests/test_fred.py index 1f616d2..b8eb526 100644 --- a/fredapi/tests/test_fred.py +++ b/fredapi/tests/test_fred.py @@ -16,8 +16,6 @@ import textwrap import contextlib -import pandas as pd - import fredapi import fredapi.fred @@ -126,6 +124,39 @@ def __init__(self, rel_url, response=None, side_effect=None): last_updated="2015-06-05 08:47:20-05" popularity="86" notes="..." /> ''')) +gdp_obs_rt_call = HTTPCall('series/observations?{}&{}&{}&{}'. + format('series_id=GDP', + 'observation_start=2014-07-01', + 'observation_end=2015-01-01', + 'realtime_start=2014-07-01'), + response=textwrap.dedent('''\ + + + + + + + + + + + + +''')) + class TestFred(unittest.TestCase): @@ -237,9 +268,9 @@ def test_invalid_kwarg_in_get_series(self, urlopen): fred_api_key) side_effect = fredapi.fred.HTTPError(url, 400, '', '', io.StringIO()) self.prepare_urlopen(urlopen, side_effect=side_effect) - with self.assertRaises(ValueError): - self.fred.get_series('SP500', - observation_start='invalid-datetime-str') + # FIXME: different environment throw ValueError or TypeError. + with self.assertRaises(Exception): + self.fred.get_series('SP500', observation_start='invalid') self.assertFalse(urlopen.called) @mock.patch('fredapi.fred.urlopen') @@ -260,6 +291,30 @@ def test_search(self, urlopen): PCPI01005 0 1969-01-01 NSA''') self.assertEqual(actual.split('\n'), expected.split('\n')) + @mock.patch('fredapi.fred.urlopen') + def test_get_series_with_realtime(self, urlopen): + """Test get_series with realtime argument.""" + side_effects = [gdp_obs_rt_call.response] + self.prepare_urlopen(urlopen, side_effect=side_effects) + df = self.fred.get_series('GDP', observation_start='7/1/2014', + observation_end='1/1/2015', + realtime_start='7/1/2014') + urlopen.assert_called_with(gdp_obs_rt_call.url) + actual = str(df) + expected = textwrap.dedent('''\ + GDP + obs_date rt_start rt_end + 2014-07-01 2014-10-30 2014-11-24 00:00:00 17535.4 + 2014-11-25 2014-12-22 00:00:00 17555.2 + 2014-12-23 9999-12-31 17599.8 + 2014-10-01 2015-01-30 2015-02-26 00:00:00 17710.7 + 2015-02-27 2015-03-26 00:00:00 17701.3 + 2015-03-27 9999-12-31 17703.7 + 2015-01-01 2015-04-29 2015-05-28 00:00:00 17710.0 + 2015-05-29 2015-06-23 00:00:00 17665.0 + 2015-06-24 9999-12-31 17693.3''') + self.assertEqual(actual.split('\n'), expected.split('\n')) + if __name__ == '__main__': unittest.main() From 65c01b5d4e4d63ff00e6e2a0cae4ba10501b2570 Mon Sep 17 00:00:00 2001 From: elmotec Date: Sat, 24 Oct 2015 21:46:16 -0400 Subject: [PATCH 3/3] Changed _parse to return None for 9999-12-31 9999-12-31 cannot be converted to pandas.Timestamp because it's too big. Reason it's prefereable to use pandas.Timestamp than datetime.datetime is that the former can be used as an index whereas the second cannot. --- fredapi/fred.py | 2 ++ fredapi/tests/test_fred.py | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/fredapi/fred.py b/fredapi/fred.py index bdae0bb..bb59958 100644 --- a/fredapi/fred.py +++ b/fredapi/fred.py @@ -72,6 +72,8 @@ def _parse(self, date_str, format='%Y-%m-%d'): """ helper function for parsing FRED date string into datetime """ + if date_str == self.latest_realtime_end: + return None rv = pd.to_datetime(date_str, format=format) if hasattr(rv, 'to_datetime'): rv = rv.to_datetime() diff --git a/fredapi/tests/test_fred.py b/fredapi/tests/test_fred.py index b8eb526..2462dba 100644 --- a/fredapi/tests/test_fred.py +++ b/fredapi/tests/test_fred.py @@ -302,17 +302,17 @@ def test_get_series_with_realtime(self, urlopen): urlopen.assert_called_with(gdp_obs_rt_call.url) actual = str(df) expected = textwrap.dedent('''\ - GDP - obs_date rt_start rt_end - 2014-07-01 2014-10-30 2014-11-24 00:00:00 17535.4 - 2014-11-25 2014-12-22 00:00:00 17555.2 - 2014-12-23 9999-12-31 17599.8 - 2014-10-01 2015-01-30 2015-02-26 00:00:00 17710.7 - 2015-02-27 2015-03-26 00:00:00 17701.3 - 2015-03-27 9999-12-31 17703.7 - 2015-01-01 2015-04-29 2015-05-28 00:00:00 17710.0 - 2015-05-29 2015-06-23 00:00:00 17665.0 - 2015-06-24 9999-12-31 17693.3''') + GDP + obs_date rt_start rt_end + 2014-07-01 2014-10-30 2014-11-24 17535.4 + 2014-11-25 2014-12-22 17555.2 + 2014-12-23 NaT 17599.8 + 2014-10-01 2015-01-30 2015-02-26 17710.7 + 2015-02-27 2015-03-26 17701.3 + 2015-03-27 NaT 17703.7 + 2015-01-01 2015-04-29 2015-05-28 17710.0 + 2015-05-29 2015-06-23 17665.0 + 2015-06-24 NaT 17693.3''') self.assertEqual(actual.split('\n'), expected.split('\n'))