|
| 1 | +import time |
| 2 | +from datetime import datetime, timedelta |
| 3 | +from warnings import warn |
| 4 | + |
| 5 | +import requests |
| 6 | +from pandas import DataFrame |
| 7 | + |
| 8 | +from pandas_datareader._utils import SymbolWarning |
| 9 | +from pandas_datareader.base import _BaseReader |
| 10 | + |
| 11 | + |
| 12 | +class MorningstarDailyReader(_BaseReader): |
| 13 | + """ |
| 14 | + Read daily data from Morningstar |
| 15 | +
|
| 16 | + Parameters |
| 17 | + ---------- |
| 18 | + symbols : {str, List[str]} |
| 19 | + String symbol of like of symbols |
| 20 | + start : string, (defaults to '1/1/2010') |
| 21 | + Starting date, timestamp. Parses many different kind of date |
| 22 | + representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980') |
| 23 | + end : string, (defaults to today) |
| 24 | + Ending date, timestamp. Same format as starting date. |
| 25 | + retry_count : int, default 3 |
| 26 | + Number of times to retry query request. |
| 27 | + pause : float, default 0.1 |
| 28 | + Time, in seconds, of the pause between retries. |
| 29 | + session : Session, default None |
| 30 | + requests.sessions.Session instance to be used |
| 31 | + freq : {str, None} |
| 32 | + Frequency to use in select readers |
| 33 | + incl_splits : bool, optional |
| 34 | + Include splits in data |
| 35 | + incl_dividends : bool,, optional |
| 36 | + Include divdends in data |
| 37 | + incl_volume : bool, optional |
| 38 | + Include volume in data |
| 39 | + currency : str, optional |
| 40 | + Currency to use for data |
| 41 | + interval : str, optional |
| 42 | + Sampling interval to use for downloaded data |
| 43 | + """ |
| 44 | + |
| 45 | + def __init__(self, symbols, start=None, end=None, retry_count=3, |
| 46 | + pause=0.1, timeout=30, session=None, freq=None, |
| 47 | + incl_splits=False, incl_dividends=False, incl_volume=True, |
| 48 | + currency='usd', interval='d'): |
| 49 | + super(MorningstarDailyReader, self).__init__(symbols, start, end, |
| 50 | + retry_count, pause, |
| 51 | + timeout, session, freq) |
| 52 | + |
| 53 | + self.incl_splits = incl_splits |
| 54 | + self.incl_dividends = incl_dividends |
| 55 | + self.incl_vol = incl_volume |
| 56 | + self.currency = currency |
| 57 | + self.interval = interval |
| 58 | + |
| 59 | + self._symbol_data_cache = [] |
| 60 | + |
| 61 | + def _url_params(self): |
| 62 | + if self.interval not in ['d', 'wk', 'mo', 'm', 'w']: |
| 63 | + raise ValueError("Invalid interval: valid values are 'd', 'wk' " |
| 64 | + "and 'mo'. 'm' and 'w' have been implemented for " |
| 65 | + "backward compatibility") |
| 66 | + elif self.interval in ['m', 'mo']: |
| 67 | + self.interval = 'm' |
| 68 | + elif self.interval in ['w', 'wk']: |
| 69 | + self.interval = 'w' |
| 70 | + |
| 71 | + if self.currency != "usd": |
| 72 | + warn("Caution! There is no explicit check for a valid currency " |
| 73 | + "acronym\nIf an error is encountered consider changing this " |
| 74 | + "value.") |
| 75 | + |
| 76 | + p = {"range": "|".join( |
| 77 | + [self.start.strftime("%Y-%m-%d"), self.end.strftime("%Y-%m-%d")]), |
| 78 | + "f": self.interval, "curry": self.currency, |
| 79 | + "dtype": "his", "showVol": "true", |
| 80 | + "hasF": "true", "isD": "true", "isS": "true", |
| 81 | + "ProdCode": "DIRECT"} |
| 82 | + |
| 83 | + return p |
| 84 | + |
| 85 | + @property |
| 86 | + def url(self): |
| 87 | + """API URL""" |
| 88 | + return "http://globalquote.morningstar.com/globalcomponent/" \ |
| 89 | + "RealtimeHistoricalStockData.ashx" |
| 90 | + |
| 91 | + def _get_crumb(self, *args): |
| 92 | + """Not required """ |
| 93 | + pass |
| 94 | + |
| 95 | + def _dl_mult_symbols(self, symbols): |
| 96 | + failed = [] |
| 97 | + symbol_data = [] |
| 98 | + for symbol in symbols: |
| 99 | + |
| 100 | + params = self._url_params() |
| 101 | + params.update({"ticker": symbol}) |
| 102 | + |
| 103 | + try: |
| 104 | + resp = requests.get(self.url, params=params) |
| 105 | + except Exception: |
| 106 | + if symbol not in failed: |
| 107 | + if self.retry_count == 0: |
| 108 | + warn("skipping symbol %s: number of retries " |
| 109 | + "exceeded." % symbol) |
| 110 | + pass |
| 111 | + else: |
| 112 | + print("adding %s to retry list" % symbol) |
| 113 | + failed.append(symbol) |
| 114 | + else: |
| 115 | + if resp.status_code == requests.codes.ok: |
| 116 | + jsondata = resp.json() |
| 117 | + if jsondata is None: |
| 118 | + failed.append(symbol) |
| 119 | + continue |
| 120 | + jsdata = self._restruct_json(symbol=symbol, |
| 121 | + jsondata=jsondata) |
| 122 | + symbol_data.extend(jsdata) |
| 123 | + else: |
| 124 | + raise Exception("Request Error!: %s : %s" % ( |
| 125 | + resp.status_code, resp.reason)) |
| 126 | + |
| 127 | + time.sleep(self.pause) |
| 128 | + |
| 129 | + if len(failed) > 0 and self.retry_count > 0: |
| 130 | + # TODO: This appears to do nothing since |
| 131 | + # TODO: successful symbols are not added to |
| 132 | + self._dl_mult_symbols(symbols=failed) |
| 133 | + self.retry_count -= 1 |
| 134 | + else: |
| 135 | + self.retry_count = 0 |
| 136 | + |
| 137 | + if not symbol_data: |
| 138 | + raise ValueError('All symbols were invalid') |
| 139 | + elif self.retry_count == 0 and len(failed) > 0: |
| 140 | + warn("The following symbols were excluded do to http " |
| 141 | + "request errors: \n %s" % failed, SymbolWarning) |
| 142 | + |
| 143 | + symbols_df = DataFrame(data=symbol_data) |
| 144 | + dfx = symbols_df.set_index(["Symbol", "Date"]) |
| 145 | + return dfx |
| 146 | + |
| 147 | + @staticmethod |
| 148 | + def _convert_index2date(enddate, indexvals): |
| 149 | + i = 0 |
| 150 | + while i < len(indexvals): |
| 151 | + days = indexvals[len(indexvals) - 1] - indexvals[i] |
| 152 | + d = enddate - timedelta(days=days) |
| 153 | + i += 1 |
| 154 | + yield d.strftime("%Y-%m-%d") |
| 155 | + |
| 156 | + def _restruct_json(self, symbol, jsondata): |
| 157 | + if jsondata is None: |
| 158 | + return |
| 159 | + divdata = jsondata["DividendData"] |
| 160 | + |
| 161 | + pricedata = jsondata["PriceDataList"][0]["Datapoints"] |
| 162 | + dateidx = jsondata["PriceDataList"][0]["DateIndexs"] |
| 163 | + volumes = jsondata["VolumeList"]["Datapoints"] |
| 164 | + |
| 165 | + date_ = self._convert_index2date(enddate=self.end, indexvals=dateidx) |
| 166 | + barss = [] |
| 167 | + for p in range(len(pricedata)): |
| 168 | + bar = pricedata[p] |
| 169 | + d = next(date_) |
| 170 | + bardict = { |
| 171 | + "Symbol": symbol, "Date": d, "Open": bar[0], "High": bar[1], |
| 172 | + "Low": bar[2], |
| 173 | + "Close": bar[3] |
| 174 | + } |
| 175 | + if len(divdata) == 0: |
| 176 | + pass |
| 177 | + else: |
| 178 | + events = [] |
| 179 | + for x in divdata: |
| 180 | + delta = (datetime.strptime(x["Date"], "%Y-%m-%d") - |
| 181 | + datetime.strptime(d, "%Y-%m-%d")) |
| 182 | + if delta.days == 0: |
| 183 | + events.append(x) |
| 184 | + for e in events: |
| 185 | + if self.incl_dividends and e["Type"].find("Div") > -1: |
| 186 | + val = e["Desc"].replace(e["Type"], "") |
| 187 | + bardict.update({"isDividend": val}) |
| 188 | + elif (self.incl_splits is True and |
| 189 | + e["Type"].find("Split") > -1): |
| 190 | + val = e["Desc"].replace(e["Type"], "") |
| 191 | + bardict.update({"isSplit": val}) |
| 192 | + else: |
| 193 | + pass |
| 194 | + if self.incl_vol is True: |
| 195 | + bardict.update({"Volume": int(volumes[p] * 1000000)}) |
| 196 | + else: |
| 197 | + pass |
| 198 | + |
| 199 | + barss.append(bardict) |
| 200 | + return barss |
| 201 | + |
| 202 | + def read(self): |
| 203 | + """Read data""" |
| 204 | + if isinstance(self.symbols, str): |
| 205 | + symbols = [self.symbols] |
| 206 | + else: |
| 207 | + symbols = self.symbols |
| 208 | + |
| 209 | + is_str = False |
| 210 | + try: |
| 211 | + is_str = all(map(lambda v: isinstance(v, str), symbols)) |
| 212 | + except Exception: |
| 213 | + pass |
| 214 | + |
| 215 | + if not is_str: |
| 216 | + raise TypeError("symbols must be iterable or string and not " |
| 217 | + "type %s" % type(self.symbols)) |
| 218 | + |
| 219 | + df = self._dl_mult_symbols(symbols=symbols) |
| 220 | + if len(df.index.levels[0]) == 0: |
| 221 | + raise ValueError("None of the provided symbols were valid") |
| 222 | + else: |
| 223 | + return df |
0 commit comments