Merge pull request #3 from pydata/master

getting up to date

Author: simongarisch

.travis.yml

@@ -10,8 +10,6 @@ env:
 matrix:
   fast_finish: true
   include:
-  - python: 2.7
-    env: PANDAS=0.24 NUMPY=1.16
   - python: 3.5
     env: PANDAS=0.21 NUMPY=1.13
   - python: 3.5
@@ -24,6 +22,8 @@ matrix:
     env: PANDAS=0.25 NUMPY=1.17
   - python: 3.7
     env: TEST_TYPE="quandl" PANDAS=0.25 NUMPY=1.17
+  - python: 3.8
+    env: PANDAS=0.25 NUMPY=1.18
     # In allow failures
   - python: 3.7
     env: TEST_TYPE="alpha_vantage" PANDAS=0.25 NUMPY=1.17

README.rst

@@ -24,7 +24,8 @@ Up to date remote data access for pandas, works for multiple versions of pandas.
 
 .. warning::
 
-  v0.8.0 is the last version which officially supports Python 2.7. Future versions of ``pandas_datareader`` will end support for Python 2.x.
+  v0.8.0 is the last version which officially supports Python 2.7. Future versions of
+  ``pandas_datareader`` will end support for Python 2.x.
 
 .. warning::
 

docs/source/remote_data.rst

@@ -621,10 +621,10 @@ You can specify dataset ID 'tran_sf_railac' to get corresponding data via ``Data
 
 .. _remote_data.tsp:
 
-TSP Fund Data
-=============
+Thrift Savings Plan (TSP) Fund Data
+===================================
 
-Download mutual fund index prices for the TSP.
+Download mutual fund index prices for the Thrift Savings Plan (TSP).
 
 .. ipython:: python
 

docs/source/whatsnew.rst

@@ -18,6 +18,7 @@ What's New
 
 These are new features and improvements of note in each release.
 
+.. include:: whatsnew/v0.9.0.txt
 .. include:: whatsnew/v0.8.0.txt
 .. include:: whatsnew/v0.7.0.txt
 .. include:: whatsnew/v0.6.0.txt

docs/source/whatsnew/v0.9.0.txt

@@ -0,0 +1,36 @@
+.. _whatsnew_090:
+
+v0.9.0 (TBD)
+---------------------------
+
+Highlights include:
+
+
+.. contents:: What's new in v0.9.0
+    :local:
+    :backlinks: none
+
+
+.. _whatsnew_090.enhancements:
+
+Enhancements
+~~~~~~~~~~~~
+
+
+.. _whatsnew_090.api_breaking:
+
+Backwards incompatible API changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+.. _whatsnew_090.bug_fixes:
+
+Bug Fixes
+~~~~~~~~~
+
+- Fix Yahoo! actions bug due to change in split format. (:issue: `755`)
+- Fix FutureWarning from pandas import. (:issue: `762`)
+
+Contributors
+~~~~~~~~~~~~
+- David Stephens

pandas_datareader/av/__init__.py

@@ -78,7 +78,10 @@ def _read_lines(self, out):
                     ".".format(self.symbols)
                 )
             else:
-                raise RemoteDataError()
+                raise RemoteDataError(
+                    " Their was an issue from the data vendor "
+                    "side, here is their response: {}".format(out)
+                )
         df = df[sorted(df.columns)]
         df.columns = [id[3:] for id in df.columns]
         return df

pandas_datareader/compat/__init__.py

@@ -1,12 +1,12 @@
 from distutils.version import LooseVersion
-import sys
+from functools import reduce
+from io import StringIO
+from urllib.error import HTTPError
 
 import pandas as pd
 from pandas.api.types import is_list_like, is_number
 import pandas.io.common as com
-from pandas.util.testing import assert_frame_equal
-
-PY3 = sys.version_info >= (3, 0)
+from pandas.testing import assert_frame_equal
 
 PANDAS_VERSION = LooseVersion(pd.__version__)
 
@@ -30,6 +30,7 @@
     "lmap",
     "lrange",
     "concat",
+    "reduce",
 ]
 
 
@@ -45,34 +46,16 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None, compression=None):
     )
 
 
-if PY3:
-    from urllib.error import HTTPError
-    from functools import reduce
-
-    string_types = (str,)
-    binary_type = bytes
-    from io import StringIO
-
-    def str_to_bytes(s, encoding=None):
-        return s.encode(encoding or "ascii")
-
-    def bytes_to_str(b, encoding=None):
-        return b.decode(encoding or "utf-8")
-
+string_types = (str,)
+binary_type = bytes
 
-else:
-    from urllib2 import HTTPError
-    from cStringIO import StringIO
 
-    reduce = reduce
-    binary_type = str
-    string_types = (basestring,)  # noqa: F821
+def str_to_bytes(s, encoding=None):
+    return s.encode(encoding or "ascii")
 
-    def bytes_to_str(b, encoding=None):
-        return b
 
-    def str_to_bytes(s, encoding=None):
-        return s
+def bytes_to_str(b, encoding=None):
+    return b.decode(encoding or "utf-8")
 
 
 def lmap(*args, **kwargs):

pandas_datareader/econdb.py

@@ -23,7 +23,7 @@ def url(self):
 
     def read(self):
         """ read one data from specified URL """
-        results = requests.get(self.url).json()["results"]
+        results = self.session.get(self.url).json()["results"]
         df = pd.DataFrame({"dates": []}).set_index("dates")
 
         if self._show == "labels":

pandas_datareader/moex.py

@@ -48,7 +48,7 @@ def __init__(self, *args, **kwargs):
         elif not is_list_like(self.symbols):
             self.symbols = [self.symbols]
 
-        self.__engines, self.__markets = {}, {}  # dicts for engines and markets
+        self.__markets_n_engines = {}  # dicts for tuples of engines and markets
 
     __url_metadata = "https://iss.moex.com/iss/securities/{symbol}.csv"
     __url_data = (
@@ -60,17 +60,17 @@ def __init__(self, *args, **kwargs):
     def url(self):
         """Return a list of API URLs per symbol"""
 
-        if not self.__engines or not self.__markets:
+        if not self.__markets_n_engines:
             raise Exception(
                 "Accessing url property before invocation "
                 "of read() or _get_metadata() methods"
             )
 
         return [
-            self.__url_data.format(
-                engine=self.__engines[s], market=self.__markets[s], symbol=s
-            )
+            self.__url_data.format(engine=engine, market=market, symbol=s)
             for s in self.symbols
+            if s in self.__markets_n_engines
+            for market, engine in self.__markets_n_engines[s]
         ]
 
     def _get_params(self, start):
@@ -81,13 +81,13 @@ def _get_params(self, start):
             "iss.dp": "point",
             "iss.df": "%Y-%m-%d",
             "iss.tf": "%H:%M:%S",
-            "iss.dft": "%Y-%m-%d %H:%M:%S",
+            "iss.dtf": "%Y-%m-%d %H:%M:%S",
             "iss.json": "extended",
             "callback": "JSON_CALLBACK",
             "from": start,
             "till": self.end_dt.strftime("%Y-%m-%d"),
             "limit": 100,
-            "start": 1,
+            "start": 0,
             "sort_order": "TRADEDATE",
             "sort_order_desc": "asc",
         }
@@ -96,7 +96,7 @@ def _get_params(self, start):
     def _get_metadata(self):
         """Get markets and engines for the given symbols"""
 
-        markets, engines = {}, {}
+        markets_n_engines = {}
 
         for symbol in self.symbols:
             response = self._get_response(self.__url_metadata.format(symbol=symbol))
@@ -118,9 +118,14 @@ def _get_metadata(self):
                     continue
                 if get_data and s != "":
                     fields = s.split(";")
-                    markets[symbol], engines[symbol] = fields[5], fields[7]
-                    break
-            if symbol not in markets or symbol not in engines:
+
+                    if symbol not in markets_n_engines:
+                        markets_n_engines[symbol] = list()
+
+                    markets_n_engines[symbol].append(
+                        (fields[5], fields[7])
+                    )  # market and engine
+            if symbol not in markets_n_engines:
                 raise IOError(
                     "{} request returned no metadata: {}\n"
                     "Typo in the security symbol `{}`?".format(
@@ -129,17 +134,20 @@ def _get_metadata(self):
                         symbol,
                     )
                 )
-        return markets, engines
+            if symbol in markets_n_engines:
+                markets_n_engines[symbol] = list(set(markets_n_engines[symbol]))
+        return markets_n_engines
 
     def read(self):
         """Read data"""
 
         try:
-            self.__markets, self.__engines = self._get_metadata()
+            self.__markets_n_engines = self._get_metadata()
+
             urls = self.url  # generate urls per symbols
             dfs = []  # an array of pandas dataframes per symbol to concatenate
 
-            for i in range(len(self.symbols)):
+            for i in range(len(urls)):
                 out_list = []
                 date_column = None
 
@@ -155,7 +163,7 @@ def read(self):
                         start_str = self.start.strftime("%Y-%m-%d")
                         start = self.start
 
-                    if start >= self.end or start >= dt.date.today():
+                    if start > self.end or start > dt.date.today():
                         break
 
                     params = self._get_params(start_str)
@@ -172,12 +180,19 @@ def read(self):
                         out_list += strings_out[1:]  # remove a CSV head line
                         if len(strings_out) < 100:  # all data recevied - break
                             break
-                str_io = StringIO("\r\n".join(out_list))
-                dfs.append(self._read_lines(str_io))  # add a new DataFrame
+
+                if len(out_list) > 0:
+                    str_io = StringIO("\r\n".join(out_list))
+                    dfs.append(self._read_lines(str_io))  # add a new DataFrame
         finally:
             self.close()
 
-        if len(dfs) > 1:
+        if len(dfs) == 0:
+            raise IOError(
+                "{} returned no data; "
+                "check URL or correct a date".format(self.__class__.__name__)
+            )
+        elif len(dfs) > 1:
             return concat(dfs, axis=0, join="outer", sort=True)
         else:
             return dfs[0]

pandas_datareader/tests/test_moex.py

@@ -15,3 +15,12 @@ def test_moex_datareader(self):
             assert "SECID" in df.columns
         except HTTPError as e:
             pytest.skip(e)
+
+    def test_moex_stock_datareader(self):
+        try:
+            df = web.DataReader(
+                ["GAZP", "SIBN"], "moex", start="2019-12-26", end="2019-12-26"
+            )
+            assert df.size == 720
+        except HTTPError as e:
+            pytest.skip(e)