Merge branch 'master' into cln_080

Author: bashtage

docs/source/readers/econdb.rst

@@ -0,0 +1,8 @@
+Econdb
+--------
+
+.. py:module:: pandas_datareader.econdb
+
+.. autoclass:: EcondbReader
+   :members:
+   :inherited-members:

docs/source/readers/enigma.rst

@@ -1,4 +1,4 @@
-Engima
+Enigma
 ------
 
 .. py:module:: pandas_datareader.enigma

docs/source/readers/index.rst

@@ -8,14 +8,14 @@ Data Readers
    fred
    famafrench
    bank-of-canada
+   econdb
    enigma
    eurostat
    iex
    moex
    nasdaq-trader
    oecd
    quandl
-   robinhood
    stooq
    tiingo
    tsp

docs/source/remote_data.rst

@@ -18,6 +18,11 @@ Remote Data Access
 ******************
 
 
+.. warning::
+
+    Robinhood has been immediately deprecated. Endpoints from this provider
+    have been retired.
+
 .. _remote_data.data_reader:
 
 Functions from :mod:`pandas_datareader.data` and :mod:`pandas_datareader.wb`
@@ -26,7 +31,6 @@ Currently the following sources are supported:
 
     - :ref:`Tiingo<remote_data.tiingo>`
     - :ref:`IEX<remote_data.iex>`
-    - :ref:`Robinhood<remote_data.robinhood>`
     - :ref:`Alpha Vantage<remote_data.alphavantage>`
     - :ref:`Enigma<remote_data.enigma>`
     - :ref:`Quandl<remote_data.quandl>`
@@ -91,21 +95,6 @@ A third interface to the deep API is exposed through
     f[:10]
 
 
-.. _remote_data.robinhood:
-
-Robinhood
-=========
-`Robinhood <https://www.robinhood.com>`__ is a stock trading platform with an
-API that provides a limited set of data. Historical daily data is limited to 1
-year relative to today.
-
-.. ipython:: python
-
-    import pandas_datareader.data as web
-    from datetime import datetime
-    f = web.DataReader('F', 'robinhood')
-    f.head()
-
 .. _remote_data.alphavantage:
 
 Alpha Vantage
@@ -208,6 +197,23 @@ performances through the top-level function ``get_sector_performance_av``.
 
 .. _remote_data.enigma:
 
+Econdb
+======
+
+`Econdb <https://www.econdb.com>`__ provides economic data from 90+
+official statistical agencies. Free API allows access to the complete
+Econdb database of time series aggregated into datasets.
+
+.. ipython:: python
+
+    import os
+    import pandas_datareader.data as web
+
+    f = web.DataReader('ticker=RGDPQNO', 'econdb')
+    f.head()
+
+.. _remote_data.econdb:
+
 Enigma
 ======
 

docs/source/whatsnew/v0.8.0.txt

@@ -5,8 +5,13 @@ v0.8.0 (TBD)
 
 Highlights include:
 
-
+- A new connector for Econdb was introduced.  Econdb provides
+  aggregated economic data from 90+ official statistical agencies
+  (:issue:`615`)
 - Removal of Google finance and Morningstar, which were deprecated in 0.7.0.
+- Immediate deprecation of Robinhood for quotes and historical data. Robinhood
+  ended support for these endpoints in 1/2019
+
 
 .. contents:: What's new in v0.8.0
     :local:
@@ -24,6 +29,9 @@ Enhancements
 Backwards incompatible API changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+- Immediate deprecation of Robinhood for quotes and historical data. Robinhood
+  ended support for these endpoints in 1/2019. The Robinhood quotes and daily
+  readers will raise an ``ImmediateDeprecationError`` when called.
 
 
 .. _whatsnew_080.bug_fixes:

pandas_datareader/av/__init__.py

@@ -28,7 +28,7 @@ def __init__(self, symbols=None, start=None, end=None, retry_count=3,
         if not api_key or not isinstance(api_key, str):
             raise ValueError('The AlphaVantage API key must be provided '
                              'either through the api_key variable or '
-                             'through the environment varaible '
+                             'through the environment variable '
                              'ALPHAVANTAGE_API_KEY')
         self.api_key = api_key
 

pandas_datareader/data.py

@@ -9,6 +9,7 @@
 from pandas_datareader.av.sector import AVSectorPerformanceReader
 from pandas_datareader.av.time_series import AVTimeSeriesReader
 from pandas_datareader.bankofcanada import BankOfCanadaReader
+from pandas_datareader.econdb import EcondbReader
 from pandas_datareader.enigma import EnigmaReader
 from pandas_datareader.eurostat import EurostatReader
 from pandas_datareader.exceptions import DEP_ERROR_MSG, \
@@ -292,7 +293,7 @@ def DataReader(name, data_source=None, start=None, end=None,
                        "tiingo", "yahoo-actions", "yahoo-dividends",
                        "av-forex", "av-daily", "av-daily-adjusted",
                        "av-weekly", "av-weekly-adjusted", "av-monthly",
-                       "av-monthly-adjusted"]
+                       "av-monthly-adjusted", "econdb"]
 
     if data_source not in expected_source:
         msg = "data_source=%r is not implemented" % data_source
@@ -438,6 +439,11 @@ def DataReader(name, data_source=None, start=None, end=None,
                                   retry_count=retry_count, pause=pause,
                                   session=session, api_key=access_key).read()
 
+    elif data_source == "econdb":
+        return EcondbReader(symbols=name, start=start, end=end,
+                            retry_count=retry_count, pause=pause,
+                            session=session).read()
+
     else:
         msg = "data_source=%r is not implemented" % data_source
         raise NotImplementedError(msg)

pandas_datareader/econdb.py

@@ -0,0 +1,49 @@
+import requests
+import pandas as pd
+import pandas.compat as compat
+
+from pandas_datareader.base import _BaseReader
+
+
+class EcondbReader(_BaseReader):
+    """Get data for the given name from Econdb."""
+
+    _URL = 'https://www.econdb.com/api/series/'
+    _format = None
+    _show = 'labels'
+
+    @property
+    def url(self):
+        """API URL"""
+        if not isinstance(self.symbols, compat.string_types):
+            raise ValueError('data name must be string')
+
+        return ('{0}?{1}&format=json&page_size=500&expand=meta'
+                .format(self._URL, self.symbols))
+
+    def read(self):
+        """ read one data from specified URL """
+        results = requests.get(self.url).json()['results']
+        df = pd.DataFrame({'dates': []}).set_index('dates')
+
+        for entry in results:
+            head = entry['additional_metadata']
+            series = (pd.DataFrame(entry['data'])[['dates', 'values']]
+                      .set_index('dates'))
+            if self._show == 'labels':
+                def show_func(x): return x.split(':')[1]
+            elif self._show == 'codes':
+                def show_func(x): return x.split(':')[0]
+
+            series.columns = pd.MultiIndex.from_tuples(
+                [[show_func(x) for x in head.values()]],
+                names=[show_func(x) for x in head.keys()])
+
+            if not df.empty:
+                df = df.join(series, how='outer')
+            else:
+                df = series
+        df.index = pd.to_datetime(df.index, errors='ignore')
+        df.index.name = 'TIME_PERIOD'
+        df = df.truncate(self.start, self.end)
+        return df

pandas_datareader/robinhood.py

@@ -1,12 +1,17 @@
 import pandas as pd
 
 from pandas_datareader.base import _BaseReader
+from pandas_datareader.exceptions import (ImmediateDeprecationError,
+                                          DEP_ERROR_MSG)
 
 
 class RobinhoodQuoteReader(_BaseReader):
     """
     Read quotes from Robinhood
 
+    DEPRECATED 1/2019 - Robinhood ended support for the endpoints used by this
+    reader
+
     Parameters
     ----------
     symbols : {str, List[str]}
@@ -28,6 +33,7 @@ class RobinhoodQuoteReader(_BaseReader):
 
     def __init__(self, symbols, start=None, end=None, retry_count=3, pause=.1,
                  timeout=30, session=None, freq=None):
+        raise ImmediateDeprecationError(DEP_ERROR_MSG.format("Robinhood"))
         super(RobinhoodQuoteReader, self).__init__(symbols, start, end,
                                                    retry_count, pause,
                                                    timeout, session, freq)
@@ -72,6 +78,9 @@ class RobinhoodHistoricalReader(RobinhoodQuoteReader):
     """
     Read historical values from Robinhood
 
+    DEPRECATED 1/2019 - Robinhood ended support for the endpoints used by this
+    reader
+
     Parameters
     ----------
     symbols : {str, List[str]}
@@ -110,6 +119,7 @@ class RobinhoodHistoricalReader(RobinhoodQuoteReader):
     def __init__(self, symbols, start=None, end=None, retry_count=3, pause=.1,
                  timeout=30, session=None, freq=None, interval='day',
                  span='year'):
+        raise ImmediateDeprecationError(DEP_ERROR_MSG.format("Robinhood"))
         super(RobinhoodHistoricalReader, self).__init__(symbols, start, end,
                                                         retry_count, pause,
                                                         timeout, session, freq)

pandas_datareader/tests/test_econdb.py

@@ -0,0 +1,65 @@
+import numpy as np
+import pandas as pd
+import pandas.util.testing as tm
+import pandas_datareader.data as web
+
+
+class TestEcondb(object):
+
+    def test_get_cdh_e_fos(self):
+        # EUROSTAT
+        # Employed doctorate holders in non managerial and non professional
+        # occupations by fields of science (%)
+        df = web.DataReader(
+            'dataset=CDH_E_FOS&GEO=NO,PL,PT,RU&FOS07=FOS1&Y_GRAD=TOTAL',
+            'econdb',
+            start=pd.Timestamp('2005-01-01'),
+            end=pd.Timestamp('2010-01-01'))
+        assert isinstance(df, pd.DataFrame)
+        assert df.shape == (2, 4)
+
+        df = df['Natural sciences']['Annual'][
+                ['Norway', 'Poland', 'Portugal', 'Russia']]
+
+        exp_col = pd.MultiIndex.from_product(
+            [['Norway', 'Poland', 'Portugal', 'Russia'],
+             ['Percentage'], ['Total']],
+            names=['Geopolitical entity (reporting)', 'Unit of measure',
+                   'Year of graduation'])
+        exp_idx = pd.DatetimeIndex(['2006-01-01', '2009-01-01'],
+                                   name='TIME_PERIOD')
+
+        values = np.array([[25.49, np.nan, 39.05, np.nan],
+                           [20.38, 25.1, 27.77, 38.1]])
+        expected = pd.DataFrame(values, index=exp_idx, columns=exp_col)
+        tm.assert_frame_equal(df, expected)
+
+    def test_get_tourism(self):
+        # OECD
+        # TOURISM_INBOUND
+
+        df = web.DataReader(
+            'dataset=OE_TOURISM_INBOUND&COUNTRY=JPN,USA&'
+            'VARIABLE=INB_ARRIVALS_TOTAL', 'econdb',
+            start=pd.Timestamp('2008-01-01'), end=pd.Timestamp('2012-01-01'))
+        df = df.astype(np.float)
+        jp = np.array([8351000, 6790000, 8611000, 6219000,
+                       8368000], dtype=float)
+        us = np.array([175702309, 160507417, 164079732, 167600277,
+                       171320408], dtype=float)
+        index = pd.date_range('2008-01-01', '2012-01-01', freq='AS',
+                              name='TIME_PERIOD')
+        for label, values in [('Japan', jp), ('United States', us)]:
+            expected = pd.Series(values, index=index,
+                                 name='Total international arrivals')
+            tm.assert_series_equal(df[label]['Total international arrivals'],
+                                   expected)
+
+    def test_bls(self):
+        # BLS
+        # CPI
+        df = web.DataReader(
+            'ticker=BLS_CU.CUSR0000SA0.M.US', 'econdb',
+            start=pd.Timestamp('2010-01-01'), end=pd.Timestamp('2013-01-27'))
+
+        assert df.loc['2010-05-01'][0] == 217.3