Merge pull request #540 from davidastephens/quotes

BUG: Fix yahoo quotes API

Author: davidastephens

docs/source/remote_data.rst

@@ -17,17 +17,14 @@
 Remote Data Access
 ******************
 
-.. warning::
-
-  Yahoo! Finance has been immediately deprecated.  Yahoo! substantially
-  altered their API in late 2017 and the csv endpoint was retired.
 
 .. _remote_data.data_reader:
 
 Functions from :mod:`pandas_datareader.data` and :mod:`pandas_datareader.wb`
 extract data from various Internet sources into a pandas DataFrame.
 Currently the following sources are supported:
 
+    - :ref:`Yahoo! Finance<remote_data.yahoo>`
     - :ref:`Google Finance<remote_data.google>`
     - :ref:`Tiingo<remote_data.tiingo>`
     - :ref:`Morningstar<remote_data.morningstar>`
@@ -48,8 +45,31 @@ Currently the following sources are supported:
 
 It should be noted, that various sources support different kinds of data, so not all sources implement the same methods and the data elements returned might also differ.
 
+.. _remote_data.yahoo:
+
+Yahoo! Finance
+==============
+
+.. ipython:: python
+
+    import pandas_datareader.data as web
+    import datetime
+    start = datetime.datetime(2010, 1, 1)
+    end = datetime.datetime(2013, 1, 27)
+    f = web.DataReader('F', 'yahoo', start, end)
+    f.ix['2010-01-04']
+
+
+.. ipython:: python
+
+    import pandas_datareader.data as web
+    aapl = web.get_quote_yahoo('AAPL')
+    aapl.loc['AAPL']
+
+
 .. _remote_data.google:
 
+
 Google Finance
 ==============
 

docs/source/whatsnew/v0.7.0.txt

@@ -62,4 +62,4 @@ Bug Fixes
 - Added back support for Yahoo! price, dividends, and splits data for stocks
 	and currency pairs (:issue:`487`).
 - Add `is_list_like` to compatibility layer to avoid failure on pandas >= 0.23 (:issue:`520`)
-
+- Fix Yahoo! quote reader (:issue: `540`)

pandas_datareader/data.py

@@ -84,7 +84,6 @@ def get_data_yahoo_actions(*args, **kwargs):
 
 
 def get_quote_yahoo(*args, **kwargs):
-    raise ImmediateDeprecationError(DEP_ERROR_MSG.format('Yahoo Quotes'))
     return YahooQuotesReader(*args, **kwargs).read()
 
 

pandas_datareader/tests/yahoo/test_yahoo.py

@@ -10,7 +10,6 @@
 
 import pandas_datareader.data as web
 from pandas_datareader.data import YahooDailyReader
-from pandas_datareader.yahoo.quotes import _yahoo_codes
 from pandas_datareader._utils import RemoteDataError
 from pandas_datareader._testing import skip_on_exception
 
@@ -39,41 +38,39 @@ def test_yahoo_fails(self):
         with pytest.raises(Exception):
             web.DataReader('NON EXISTENT TICKER', 'yahoo', start, end)
 
-    @pytest.mark.skip('Yahoo quotes deprecated')
     def test_get_quote_series(self):
+        stringlist = ['GOOG', 'AAPL']
+        fields = ['exchange', 'sharesOutstanding', 'epsForward']
         try:
-            df = web.get_quote_yahoo(pd.Series(['GOOG', 'AAPL', 'GOOG']))
+            AAPL = web.get_quote_yahoo('AAPL')
+            df = web.get_quote_yahoo(pd.Series(stringlist))
         except ConnectionError:
             pytest.xfail(reason=XFAIL_REASON)
-        tm.assert_series_equal(df.iloc[0], df.iloc[2])
+        tm.assert_series_equal(AAPL.iloc[0][fields], df.loc['AAPL'][fields])
+        assert sorted(stringlist) == sorted(list(df.index.values))
 
-    @pytest.mark.skip('Yahoo quotes deprecated')
     def test_get_quote_string(self):
-        _yahoo_codes.update({'MarketCap': 'j1'})
         try:
             df = web.get_quote_yahoo('GOOG')
         except ConnectionError:
             pytest.xfail(reason=XFAIL_REASON)
 
-        assert not pd.isnull(df['MarketCap'][0])
+        assert not pd.isnull(df['marketCap'][0])
 
-    @pytest.mark.skip('Yahoo quotes deprecated')
     def test_get_quote_stringlist(self):
+        stringlist = ['GOOG', 'AAPL']
         try:
-            df = web.get_quote_yahoo(['GOOG', 'AAPL', 'GOOG'])
+            df = web.get_quote_yahoo(stringlist)
         except ConnectionError:
             pytest.xfail(reason=XFAIL_REASON)
-        tm.assert_series_equal(df.iloc[0], df.iloc[2])
+        assert sorted(stringlist) == sorted(list(df.index.values))
 
-    @pytest.mark.skip('Yahoo quotes deprecated')
     def test_get_quote_comma_name(self):
-        _yahoo_codes.update({'name': 'n'})
         try:
             df = web.get_quote_yahoo(['RGLD'])
         except ConnectionError:
             pytest.xfail(reason=XFAIL_REASON)
-        del _yahoo_codes['name']
-        assert df['name'][0] == 'Royal Gold, Inc.'
+        assert df['longName'][0] == 'Royal Gold, Inc.'
 
     @pytest.mark.skip('Unreliable test, receive partial '
                       'components back for dow_jones')

pandas_datareader/yahoo/quotes.py

@@ -1,13 +1,18 @@
-from collections import defaultdict
-import csv
+import json
+from collections import OrderedDict
 
 import pandas.compat as compat
 from pandas import DataFrame
 
+
 from pandas_datareader.base import _BaseReader
 
-_yahoo_codes = {'symbol': 's', 'last': 'l1', 'change_pct': 'p2', 'PE': 'r',
-                'time': 't1', 'short_ratio': 's7'}
+
+_DEFAULT_PARAMS = {
+    'lang': 'en-US',
+    'corsDomain': 'finance.yahoo.com',
+    '.tsrc': 'finance',
+}
 
 
 class YahooQuotesReader(_BaseReader):
@@ -16,39 +21,28 @@ class YahooQuotesReader(_BaseReader):
 
     @property
     def url(self):
-        return 'http://finance.yahoo.com/d/quotes.csv'
+        return 'https://query1.finance.yahoo.com/v7/finance/quote'
 
-    @property
-    def params(self):
-        """Parameters to use in API calls"""
+    def read(self):
         if isinstance(self.symbols, compat.string_types):
-            sym_list = self.symbols
+            return self._read_one_data(self.url, self.params(self.symbols))
         else:
-            sym_list = '+'.join(self.symbols)
-
-        # For codes see: http://www.gummy-stuff.org/Yahoo-data.htm
-        #
+            data = OrderedDict()
+            for symbol in self.symbols:
+                data[symbol] = \
+                    self._read_one_data(
+                        self.url, self.params(symbol)).loc[symbol]
+            return DataFrame.from_dict(data, orient='index')
+
+    def params(self, symbol):
+        """Parameters to use in API calls"""
         # Construct the code request string.
-        request = ''.join(compat.itervalues(_yahoo_codes))
-        params = {'s': sym_list, 'f': request}
+        params = {'symbols': symbol}
+        params.update(_DEFAULT_PARAMS)
         return params
 
     def _read_lines(self, out):
-        data = defaultdict(list)
-        header = list(_yahoo_codes.keys())
-
-        for line in csv.reader(out.readlines()):
-            for i, field in enumerate(line):
-                if field[-2:] == '%"':
-                    v = float(field.strip('"%'))
-                elif field[0] == '"':
-                    v = field.strip('"')
-                else:
-                    try:
-                        v = float(field)
-                    except ValueError:
-                        v = field
-                data[header[i]].append(v)
-
+        data = json.loads(out.read())['quoteResponse']['result'][0]
         idx = data.pop('symbol')
-        return DataFrame(data, index=idx)
+        data['price'] = data['regularMarketPrice']
+        return DataFrame(data, index=[idx])