Merge pull request #525 from davidastephens/data

Reinstate Yahoo API

Author: davidastephens

docs/source/whatsnew/v0.7.0.txt

@@ -42,6 +42,10 @@ Enhancements
 
 Backwards incompatible API changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- When requesting multiple symbols from a DailyReader (ex: google, yahoo, IEX)
+  a MultiIndex DataFrame is now returned.  Previously Panel or dict of DataFrames
+  were returned. (:issue:`297`).
+
 
 .. _whatsnew_070.bug_fixes:
 

pandas_datareader/base.py

@@ -5,8 +5,8 @@
 import requests
 
 import pandas.compat as compat
-from pandas import Panel, DataFrame
-from pandas import read_csv
+from pandas import DataFrame
+from pandas import read_csv, concat
 from pandas.io.common import urlencode
 from pandas.compat import StringIO, bytes_to_str
 
@@ -239,7 +239,9 @@ def _dl_mult_symbols(self, symbols):
                 df_na[:] = np.nan
                 for sym in failed:
                     stocks[sym] = df_na
-            return Panel(stocks).swapaxes('items', 'minor')
+            result = concat(stocks).unstack(level=0)
+            result.columns.names = ['Attributes', 'Symbols']
+            return result
         except AttributeError:
             # cannot construct a panel with just 1D nans indicating no data
             msg = "No data fetched using {0!r}"

pandas_datareader/data.py

@@ -32,6 +32,7 @@
     RobinhoodQuoteReader
 from pandas_datareader.stooq import StooqDailyReader
 from pandas_datareader.tiingo import TiingoDailyReader, TiingoQuoteReader
+from pandas_datareader.yahoo.actions import (YahooActionReader, YahooDivReader)
 from pandas_datareader.yahoo.components import _get_data as \
     get_components_yahoo
 from pandas_datareader.yahoo.daily import YahooDailyReader
@@ -79,11 +80,11 @@ def get_quote_av(*args, **kwargs):
 
 
 def get_data_yahoo_actions(*args, **kwargs):
-    raise ImmediateDeprecationError(DEP_ERROR_MSG.format('Yahoo Actions'))
+    return YahooActionReader(*args, **kwargs).read()
 
 
 def get_quote_yahoo(*args, **kwargs):
-    raise ImmediateDeprecationError(DEP_ERROR_MSG.format('Yahoo Actions'))
+    raise ImmediateDeprecationError(DEP_ERROR_MSG.format('Yahoo Quotes'))
     return YahooQuotesReader(*args, **kwargs).read()
 
 
@@ -305,7 +306,6 @@ def DataReader(name, data_source=None, start=None, end=None,
     ff = DataReader("F-F_ST_Reversal_Factor", "famafrench")
     """
     if data_source == "yahoo":
-        raise ImmediateDeprecationError(DEP_ERROR_MSG.format('Yahoo Daily'))
         return YahooDailyReader(symbols=name, start=start, end=end,
                                 adjust_price=False, chunksize=25,
                                 retry_count=retry_count, pause=pause,
@@ -401,6 +401,18 @@ def DataReader(name, data_source=None, start=None, end=None,
                                  retry_count=retry_count, pause=pause,
                                  session=session,
                                  api_key=access_key).read()
+
+    elif data_source == "yahoo-actions":
+        return YahooActionReader(symbols=name, start=start, end=end,
+                                 retry_count=retry_count, pause=pause,
+                                 session=session).read()
+
+    elif data_source == "yahoo-dividends":
+        return YahooDivReader(symbols=name, start=start, end=end,
+                              adjust_price=False, chunksize=25,
+                              retry_count=retry_count, pause=pause,
+                              session=session, interval='d').read()
+
     else:
         msg = "data_source=%r is not implemented" % data_source
         raise NotImplementedError(msg)

pandas_datareader/google/daily.py

@@ -9,9 +9,10 @@
 
 class GoogleDailyReader(_DailyBaseReader):
     """
-    Returns DataFrame/Panel of historical stock prices from symbols, over date
-    range, start to end. To avoid being penalized by Google Finance servers,
-    pauses between downloading 'chunks' of symbols can be specified.
+    Returns DataFrame of historical stock prices from
+    symbols, over date range, start to end. To avoid being penalized by Google
+    Finance servers, pauses between downloading 'chunks' of symbols can be
+    specified.
 
     Parameters
     ----------

pandas_datareader/iex/daily.py

@@ -16,9 +16,10 @@
 class IEXDailyReader(_DailyBaseReader):
 
     """
-    Returns DataFrame/Panel of historical stock prices from symbols, over date
-    range, start to end. To avoid being penalized by Google Finance servers,
-    pauses between downloading 'chunks' of symbols can be specified.
+    Returns DataFrame of historical stock prices
+    from symbols, over date range, start to end. To avoid being penalized by
+    IEX servers, pauses between downloading 'chunks' of symbols can be
+    specified.
 
     Parameters
     ----------
@@ -110,5 +111,7 @@ def _read_lines(self, out):
             df = df.loc[sstart:send]
             result.update({symbol: df})
         if len(result) > 1:
+            result = pd.concat(result).unstack(level=0)
+            result.columns.names = ['Attributes', 'Symbols']
             return result
         return result[self.symbols]

pandas_datareader/stooq.py

@@ -4,8 +4,8 @@
 class StooqDailyReader(_DailyBaseReader):
 
     """
-    Returns DataFrame/Panel of historical stock prices from symbols, over date
-    range, start to end.
+    Returns DataFrame/dict of Dataframes of historical stock prices from
+    symbols, over date range, start to end.
 
     Parameters
     ----------
@@ -18,7 +18,7 @@ class StooqDailyReader(_DailyBaseReader):
         Time, in seconds, to pause between consecutive queries of chunks. If
         single value given for symbol, represents the pause between retries.
     chunksize : int, default 25
-        Number of symbols to download consecutively before intiating pause.
+        Number of symbols to download consecutively before initiating pause.
     session : Session, default None
         requests.sessions.Session instance to be used
 

pandas_datareader/tests/google/test_google.py

@@ -126,8 +126,8 @@ def test_get_multi1(self):
     @skip_on_exception(RemoteDataError)
     def test_get_multi_invalid(self):
         sl = ['AAPL', 'AMZN', 'INVALID']
-        pan = web.get_data_google(sl, '2012')
-        assert 'INVALID' in pan.minor_axis
+        data = web.get_data_google(sl, '2012')
+        assert 'INVALID' in data.columns.levels[1]
 
     def test_get_multi_all_invalid(self):
         sl = ['INVALID', 'INVALID2', 'INVALID3']

pandas_datareader/tests/test_iex_daily.py

@@ -1,9 +1,10 @@
 from datetime import datetime
 
+from pandas import DataFrame, MultiIndex
+
 import pytest
 
 import pandas_datareader.data as web
-import pandas.util.testing as tm
 
 
 class TestIEXDaily(object):
@@ -44,37 +45,26 @@ def test_single_symbol(self):
     def test_multiple_symbols(self):
         syms = ["AAPL", "MSFT", "TSLA"]
         df = web.DataReader(syms, "iex", self.start, self.end)
-        assert sorted(list(df)) == syms
+        assert sorted(list(df.columns.levels[1])) == syms
         for sym in syms:
-            assert len(df[sym] == 578)
+            assert len(df.xs(sym, level='Symbols', axis=1) == 578)
 
     def test_multiple_symbols_2(self):
         syms = ["AAPL", "MSFT", "TSLA"]
         good_start = datetime(2017, 2, 9)
         good_end = datetime(2017, 5, 24)
         df = web.DataReader(syms, "iex", good_start, good_end)
-        assert isinstance(df, dict)
-        assert len(df) == 3
-        assert sorted(list(df)) == syms
+        assert isinstance(df, DataFrame)
+        assert isinstance(df.columns, MultiIndex)
+        assert len(df.columns.levels[1]) == 3
+        assert sorted(list(df.columns.levels[1])) == syms
 
-        a = df["AAPL"]
-        t = df["TSLA"]
+        a = df.xs("AAPL", axis=1, level='Symbols')
+        t = df.xs("TSLA", axis=1, level='Symbols')
 
         assert len(a) == 73
         assert len(t) == 73
 
-        expected1 = a.loc["2017-02-09"]
-        tm.assert_almost_equal(expected1["close"], 130.36,
-                               check_less_precise=True)
-        tm.assert_almost_equal(expected1["high"], 130.38,
-                               check_less_precise=True)
-
-        expected2 = a.loc["2017-05-24"]
-        tm.assert_almost_equal(expected2["close"], 151.57,
-                               check_less_precise=True)
-        tm.assert_almost_equal(expected2["high"], 152.40,
-                               check_less_precise=True)
-
         expected3 = t.loc["2017-02-09"]
         assert expected3["close"] == 269.20
         assert expected3["high"] == 271.18

pandas_datareader/tests/test_wb.py

@@ -9,6 +9,8 @@
 from pandas_datareader.compat import assert_raises_regex
 from pandas_datareader.wb import (search, download, get_countries,
                                   get_indicators, WorldBankReader)
+from pandas_datareader._utils import RemoteDataError
+from pandas_datareader._testing import skip_on_exception
 
 
 class TestWB(object):
@@ -215,6 +217,7 @@ def test_wdi_get_indicators(self):
             assert result.columns.equals(exp_col)
             assert len(result) > 10000
 
+    @skip_on_exception(RemoteDataError)
     def test_wdi_download_monthly(self):
         expected = {'COPPER': {('World', '2012M01'): 8040.47,
                                ('World', '2011M12'): 7565.48,

pandas_datareader/tests/yahoo/test_yahoo.py

@@ -21,7 +21,6 @@ class TestYahoo(object):
 
     @classmethod
     def setup_class(cls):
-        pytest.skip('Skip all Yahoo! tests.')
         pytest.importorskip("lxml")
 
     @skip_on_exception(RemoteDataError)
@@ -30,7 +29,8 @@ def test_yahoo(self):
         start = datetime(2010, 1, 1)
         end = datetime(2013, 1, 25)
 
-        assert web.DataReader('F', 'yahoo', start, end)['Close'][-1] == 13.68
+        assert round(web.DataReader('F', 'yahoo', start, end)['Close'][-1],
+                     2) == 13.68
 
     def test_yahoo_fails(self):
         start = datetime(2010, 1, 1)
@@ -39,13 +39,15 @@ def test_yahoo_fails(self):
         with pytest.raises(Exception):
             web.DataReader('NON EXISTENT TICKER', 'yahoo', start, end)
 
+    @pytest.mark.skip('Yahoo quotes deprecated')
     def test_get_quote_series(self):
         try:
             df = web.get_quote_yahoo(pd.Series(['GOOG', 'AAPL', 'GOOG']))
         except ConnectionError:
             pytest.xfail(reason=XFAIL_REASON)
         tm.assert_series_equal(df.iloc[0], df.iloc[2])
 
+    @pytest.mark.skip('Yahoo quotes deprecated')
     def test_get_quote_string(self):
         _yahoo_codes.update({'MarketCap': 'j1'})
         try:
@@ -55,13 +57,15 @@ def test_get_quote_string(self):
 
         assert not pd.isnull(df['MarketCap'][0])
 
+    @pytest.mark.skip('Yahoo quotes deprecated')
     def test_get_quote_stringlist(self):
         try:
             df = web.get_quote_yahoo(['GOOG', 'AAPL', 'GOOG'])
         except ConnectionError:
             pytest.xfail(reason=XFAIL_REASON)
         tm.assert_series_equal(df.iloc[0], df.iloc[2])
 
+    @pytest.mark.skip('Yahoo quotes deprecated')
     def test_get_quote_comma_name(self):
         _yahoo_codes.update({'name': 'n'})
         try:
@@ -113,6 +117,10 @@ def test_get_data_single_symbol(self):
         # just test that we succeed
         web.get_data_yahoo('GOOG')
 
+    @skip_on_exception(RemoteDataError)
+    def test_data_with_no_actions(self):
+        web.get_data_yahoo('TSLA')
+
     @skip_on_exception(RemoteDataError)
     def test_get_data_adjust_price(self):
         goog = web.get_data_yahoo('GOOG')
@@ -163,23 +171,22 @@ def test_get_data_null_as_missing_data(self, adj_pr):
 
     @skip_on_exception(RemoteDataError)
     def test_get_data_multiple_symbols_two_dates(self):
-        pan = web.get_data_yahoo(['GE', 'MSFT', 'INTC'], 'JAN-01-12',
-                                 'JAN-31-12')
-        result = pan.Close['01-18-12'].T
+        data = web.get_data_yahoo(['GE', 'MSFT', 'INTC'], 'JAN-01-12',
+                                  'JAN-31-12')
+        result = data.Close.loc['01-18-12'].T
         assert result.size == 3
 
         # sanity checking
-        assert result.dtypes.all() == np.floating
+        assert result.dtypes == np.floating
 
         expected = np.array([[18.99, 28.4, 25.18],
                              [18.58, 28.31, 25.13],
                              [19.03, 28.16, 25.52],
                              [18.81, 28.82, 25.87]])
-        df = pan.Open
+        df = data.Open
         result = df[(df.index >= 'Jan-15-12') & (df.index <= 'Jan-20-12')]
         assert expected.shape == result.shape
 
-    @pytest.mark.xfail(reason="failing after #355")
     def test_get_date_ret_index(self):
         pan = web.get_data_yahoo(['GE', 'INTC', 'IBM'], '1977', '1987',
                                  ret_index=True)
@@ -193,7 +200,6 @@ def test_get_date_ret_index(self):
         # sanity checking
         assert np.issubdtype(pan.values.dtype, np.floating)
 
-    @pytest.mark.xfail(reason="failing after #355")
     def test_get_data_yahoo_actions(self):
         start = datetime(1990, 1, 1)
         end = datetime(2000, 4, 5)
@@ -228,7 +234,6 @@ def test_yahoo_reader_class(self):
         r = YahooDailyReader('GOOG', session=session)
         assert r.session is session
 
-    @pytest.mark.xfail(reason="failing after #355")
     def test_yahoo_DataReader(self):
         start = datetime(2010, 1, 1)
         end = datetime(2015, 5, 9)
@@ -241,6 +246,22 @@ def test_yahoo_DataReader(self):
                                     '2013-08-08', '2013-05-09',
                                     '2013-02-07', '2012-11-07',
                                     '2012-08-09'])
+
+        exp = pd.DataFrame({'action': ['DIVIDEND', 'DIVIDEND', 'DIVIDEND',
+                                       'DIVIDEND', 'SPLIT', 'DIVIDEND',
+                                       'DIVIDEND', 'DIVIDEND',
+                                       'DIVIDEND', 'DIVIDEND', 'DIVIDEND',
+                                       'DIVIDEND', 'DIVIDEND'],
+                            'value': [0.52, 0.47, 0.47, 0.47, 0.14285714,
+                                      3.29, 3.05, 3.05, 3.05,
+                                      3.05, 2.65, 2.65, 2.65]},
+                           index=exp_idx)
+        exp.index.name = 'Date'
+        tm.assert_frame_equal(result.reindex_like(exp).round(5), exp.round(5))
+
+        result = web.get_data_yahoo_actions('AAPL', start, end,
+                                            adjust_dividends=True)
+
         exp = pd.DataFrame({'action': ['DIVIDEND', 'DIVIDEND', 'DIVIDEND',
                                        'DIVIDEND', 'SPLIT', 'DIVIDEND',
                                        'DIVIDEND', 'DIVIDEND',
@@ -251,13 +272,11 @@ def test_yahoo_DataReader(self):
                                       0.43571, 0.37857, 0.37857, 0.37857]},
                            index=exp_idx)
         exp.index.name = 'Date'
-
-        tm.assert_frame_equal(result.reindex_like(exp), exp)
+        tm.assert_frame_equal(result.reindex_like(exp).round(5), exp.round(5))
 
     @skip_on_exception(RemoteDataError)
     def test_yahoo_DataReader_multi(self):
         start = datetime(2010, 1, 1)
         end = datetime(2015, 5, 9)
-
         result = web.DataReader(['AAPL', 'F'], 'yahoo-actions', start, end)
-        assert isinstance(result, pd.Panel)
+        assert isinstance(result, dict)