Rebuild sanitize_dates and refactor AlphaVantage date handling

Author: addisonlynch

docs/source/remote_data.rst

@@ -140,7 +140,9 @@ Historical Time Series Data
 Through the
 `Alpha Vantage <https://www.alphavantage.co/documentation>`__ Time Series
 endpoints, it is possible to obtain historical equities data for individual
-symbols. The following endpoints are available:
+symbols. For daily, weekly, and monthly frequencies, 20+ years of historical data is available. The past 3-5 days of intraday data is also available.
+
+The following endpoints are available:
 
 * ``av-intraday`` - Intraday Time Series
 * ``av-daily`` - Daily Time Series

pandas_datareader/_utils.py

@@ -16,9 +16,16 @@ class RemoteDataError(IOError):
 
 def _sanitize_dates(start, end):
     """
-    Return (datetime_start, datetime_end) tuple
-    if start is None - default is 2010/01/01
+    Return (timestamp_start, timestamp_end) tuple
+    if start is None - default is 5 years before the current date
     if end is None - default is today
+
+    Parameters
+    ----------
+    start: str, int, date, datetime, timestamp
+        Desired start date
+    end: str, int, date, datetime, timestamp
+        Desired end date
     """
     if is_number(start):
         # regard int as year
@@ -30,11 +37,20 @@ def _sanitize_dates(start, end):
     end = to_datetime(end)
 
     if start is None:
-        start = dt.datetime(2010, 1, 1)
+        # default to 5 years before today
+        today = dt.date.today()
+        start = today - dt.timedelta(days=365 * 5)
     if end is None:
-        end = dt.datetime.today()
+        # default to today
+        end = dt.date.today()
     if start > end:
         raise ValueError("start must be an earlier date than end")
+
+    try:
+        start = to_datetime(start)
+        end = to_datetime(end)
+    except (TypeError, ValueError):
+        raise ValueError("Invalid date format.")
     return start, end
 
 

pandas_datareader/av/time_series.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+import datetime as dt
 
 from pandas_datareader.av import AlphaVantage
 
@@ -13,11 +13,11 @@ class AVTimeSeriesReader(AlphaVantage):
     ----------
     symbols : string
         Single stock symbol (ticker)
-    start : string, (defaults to '1/1/2010')
-        Starting date, timestamp. Parses many different kind of date
+    start : string, int, date, datetime, timestamp
+        Starting date. Parses many different kind of date
         representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980')
-    end : string, (defaults to today)
-        Ending date, timestamp. Same format as starting date.
+    end : string, int, date, datetime, timestamp
+        Ending date
     retry_count : int, default 3
         Number of times to retry query request.
     pause : int, default 0.1
@@ -52,6 +52,7 @@ def __init__(
         chunksize=25,
         api_key=None,
     ):
+        self._func = function
         super(AVTimeSeriesReader, self).__init__(
             symbols=symbols,
             start=start,
@@ -62,19 +63,26 @@ def __init__(
             api_key=api_key,
         )
 
-        self._func = function
+    @property
+    def default_start_date(self):
+        d_days = 3 if self.intraday else 365 * 20
+        return dt.datetime.today() - dt.timedelta(days=d_days)
 
     @property
     def function(self):
         return self._func
 
+    @property
+    def intraday(self):
+        return True if self.function == "TIME_SERIES_INTRADAY" else False
+
     @property
     def output_size(self):
         """ Used to limit the size of the Alpha Vantage query when
         possible.
         """
-        delta = datetime.now() - self.start
-        return "full" if delta.days > 80 else "compact"
+        delta = dt.datetime.now() - self.start
+        return "compact" if delta.days < 80 and not self.intraday else "full"
 
     @property
     def data_key(self):
@@ -88,7 +96,7 @@ def params(self):
             "apikey": self.api_key,
             "outputsize": self.output_size,
         }
-        if self.function == "TIME_SERIES_INTRADAY":
+        if self.intraday:
             p.update({"interval": "1min"})
         return p
 

pandas_datareader/base.py

@@ -1,3 +1,4 @@
+import datetime
 import time
 import warnings
 
@@ -27,7 +28,7 @@ class _BaseReader(object):
     ----------
     symbols : {str, List[str]}
         String symbol of like of symbols
-    start : string, (defaults to '1/1/2010')
+    start : string, (defaults to 5 years before current date)
         Starting date, timestamp. Parses many different kind of date
         representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980')
     end : string, (defaults to today)
@@ -59,7 +60,7 @@ def __init__(
 
         self.symbols = symbols
 
-        start, end = _sanitize_dates(start, end)
+        start, end = _sanitize_dates(start or self.default_start_date, end)
         self.start = start
         self.end = end
 
@@ -76,6 +77,12 @@ def close(self):
         """Close network session"""
         self.session.close()
 
+    @property
+    def default_start_date(self):
+        """Default start date for reader. Defaults to 5 years before current date"""
+        today = datetime.date.today()
+        return today - datetime.timedelta(days=365 * 5)
+
     @property
     def url(self):
         """API URL"""

pandas_datareader/tests/test_base.py

@@ -1,3 +1,4 @@
+import datetime as dt
 import pytest
 import requests
 
@@ -26,6 +27,10 @@ def test_invalid_format(self):
             b._format = "IM_NOT_AN_IMPLEMENTED_TYPE"
             b._read_one_data("a", None)
 
+    def test_default_start_date(self):
+        b = base._BaseReader([])
+        assert b.default_start_date == dt.date.today() - dt.timedelta(days=365 * 5)
+
 
 class TestDailyBaseReader(object):
     def test_get_params(self):

pandas_datareader/tests/test_utils.py

@@ -0,0 +1,39 @@
+import datetime as dt
+import pandas as pd
+import pytest
+
+from pandas_datareader._utils import _sanitize_dates
+
+
+class TestUtils(object):
+    @pytest.mark.parametrize(
+        "input_date", ["2019-01-01", dt.datetime(2019, 1, 1), dt.date(2019, 1, 1),
+                       pd.Timestamp(2019, 1, 1)]
+    )
+    def test_sanitize_dates(self, input_date):
+        expected_start = pd.to_datetime(input_date)
+        expected_end = pd.to_datetime(dt.date.today())
+        result = _sanitize_dates(input_date, None)
+        assert result == (expected_start, expected_end)
+
+    def test_sanitize_dates_int(self):
+        start_int = 2018
+        end_int = 2019
+        expected_start = pd.to_datetime(dt.datetime(start_int, 1, 1))
+        expected_end = pd.to_datetime(dt.datetime(end_int, 1, 1))
+        assert _sanitize_dates(start_int, end_int) == (expected_start, expected_end)
+
+    def test_sanitize_invalid_dates(self):
+        with pytest.raises(ValueError):
+            _sanitize_dates(2019, 2018)
+
+        with pytest.raises(ValueError):
+            _sanitize_dates("2019-01-01", "2018-01-01")
+
+        with pytest.raises(ValueError):
+            _sanitize_dates("20199", None)
+
+    def test_sanitize_dates_defaults(self):
+        default_start = pd.to_datetime(dt.date.today() - dt.timedelta(days=365 * 5))
+        default_end = pd.to_datetime(dt.date.today())
+        assert _sanitize_dates(None, None) == (default_start, default_end)