|
| 1 | +""" |
| 2 | +Fetch S&P 500 and FTSE All-World index constituent tickers. |
| 3 | +
|
| 4 | + S&P 500 — Wikipedia constituent table (stable, maintained by community) |
| 5 | + FTSE AW — Vanguard VT ETF portfolio API (tracks FTSE Global All Cap Index, |
| 6 | + which is FTSE All-World + small-cap; ~10 000 stocks) |
| 7 | +
|
| 8 | +Output format matches all_stocks.json: |
| 9 | + { "AAPL": ["AAPL", "Apple Inc.", "https://finance.yahoo.com/quote/AAPL/"] } |
| 10 | +
|
| 11 | +Usage: |
| 12 | + python fetch_index_stocks.py # both indices |
| 13 | + python fetch_index_stocks.py --sp500 |
| 14 | + python fetch_index_stocks.py --ftse |
| 15 | +""" |
| 16 | + |
| 17 | +import argparse |
| 18 | +import json |
| 19 | +import logging |
| 20 | +from pathlib import Path |
| 21 | + |
| 22 | +import pandas as pd |
| 23 | +import requests |
| 24 | + |
| 25 | +SCRIPT_DIR = Path(__file__).parent |
| 26 | + |
| 27 | +SP500_FILE = SCRIPT_DIR / "sp500_stocks.json" |
| 28 | +FTSE_AW_FILE = SCRIPT_DIR / "ftse_allworld_stocks.json" |
| 29 | + |
| 30 | +SP500_WIKI_URL = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies" |
| 31 | + |
| 32 | +# Vanguard VT ETF — tracks FTSE Global All Cap (= FTSE All-World + small-cap) |
| 33 | +VT_API_URL = ( |
| 34 | + "https://investor.vanguard.com/investment-products/etfs/profile/api/" |
| 35 | + "VT/portfolio-holding/stock" |
| 36 | +) |
| 37 | +VT_PAGE_SIZE = 500 |
| 38 | + |
| 39 | +HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; Python/requests)"} |
| 40 | + |
| 41 | +logging.basicConfig(level=logging.INFO, format="%(message)s") |
| 42 | + |
| 43 | +# ISIN country code → yfinance exchange suffix. |
| 44 | +# Countries absent from this map are assumed to already be in yfinance format (US) |
| 45 | +# or have no reliable single exchange. |
| 46 | +_ISIN_SUFFIX: dict[str, str] = { |
| 47 | + "AT": ".VI", |
| 48 | + "AU": ".AX", |
| 49 | + "BE": ".BR", |
| 50 | + "BR": ".SA", |
| 51 | + "CA": ".TO", |
| 52 | + "CH": ".SW", |
| 53 | + "CL": ".SN", |
| 54 | + "DE": ".DE", |
| 55 | + "DK": ".CO", |
| 56 | + "EG": ".CA", |
| 57 | + "ES": ".MC", |
| 58 | + "FI": ".HE", |
| 59 | + "FR": ".PA", |
| 60 | + "GB": ".L", |
| 61 | + "GR": ".AT", |
| 62 | + "HK": ".HK", # also zero-padded to 4 digits; handled in _to_yf_symbol |
| 63 | + "ID": ".JK", |
| 64 | + "IE": ".I", |
| 65 | + "IL": ".TA", |
| 66 | + "IN": ".NS", |
| 67 | + "IT": ".MI", |
| 68 | + "JP": ".T", |
| 69 | + "KW": ".KW", |
| 70 | + "MX": ".MX", |
| 71 | + "MY": ".KL", |
| 72 | + "NL": ".AS", |
| 73 | + "NO": ".OL", |
| 74 | + "NZ": ".NZ", |
| 75 | + "PH": ".PS", |
| 76 | + "PL": ".WA", |
| 77 | + "PT": ".LS", |
| 78 | + "QA": ".QA", |
| 79 | + "SA": ".SR", |
| 80 | + "SE": ".ST", |
| 81 | + "SG": ".SI", |
| 82 | + "TH": ".BK", # -F/-R/-W exchange suffixes stripped; handled in _to_yf_symbol |
| 83 | + "TR": ".IS", # -E exchange suffix stripped; handled in _to_yf_symbol |
| 84 | + "TW": ".TW", |
| 85 | + "ZA": ".JO", |
| 86 | +} |
| 87 | + |
| 88 | +# Vanguard-specific trailing qualifiers that are NOT part of the yfinance ticker |
| 89 | +_TH_STRIP = ("-F", "-R", "-W", "-P") # Thai foreign/NVDR/warrant shares |
| 90 | + |
| 91 | + |
| 92 | +def _to_yf_symbol(ticker: str, isin: str) -> str: |
| 93 | + """Convert a Vanguard-format ticker + ISIN to the correct yfinance symbol.""" |
| 94 | + if not ticker: |
| 95 | + return ticker |
| 96 | + |
| 97 | + country = isin[:2] if isin and len(isin) >= 2 else "" |
| 98 | + |
| 99 | + if not country or country == "US": |
| 100 | + return ticker |
| 101 | + |
| 102 | + t = ticker |
| 103 | + |
| 104 | + # Strip exchange-qualifier suffixes that Vanguard appends but yfinance doesn't use |
| 105 | + if country == "TH": |
| 106 | + for sfx in _TH_STRIP: |
| 107 | + if t.endswith(sfx): |
| 108 | + t = t[: -len(sfx)] |
| 109 | + break |
| 110 | + elif country == "TR" and t.endswith("-E"): |
| 111 | + t = t[:-2] |
| 112 | + |
| 113 | + # Country-specific suffix logic |
| 114 | + if country == "CN": |
| 115 | + # Shanghai tickers start with 6; everything else is Shenzhen |
| 116 | + suffix = ".SS" if t and t[0] == "6" else ".SZ" |
| 117 | + elif country == "KR": |
| 118 | + # ISIN position 2: '7' = KOSPI (.KS), '8' = KOSDAQ (.KQ) |
| 119 | + suffix = ".KS" if len(isin) > 2 and isin[2] == "7" else ".KQ" |
| 120 | + elif country in ("HK", "KY") and t.isdigit(): |
| 121 | + # HK-listed: direct HK ISIN, or Cayman-registered Chinese companies (KY ISIN) |
| 122 | + # KY + alphabetic ticker = US-listed ADR → handled below by the empty suffix |
| 123 | + t = t.zfill(4) # '700' → '0700' |
| 124 | + suffix = ".HK" |
| 125 | + else: |
| 126 | + suffix = _ISIN_SUFFIX.get(country, "") |
| 127 | + |
| 128 | + return f"{t}{suffix}" if suffix else t |
| 129 | + |
| 130 | + |
| 131 | +def fetch_sp500() -> dict[str, list]: |
| 132 | + """Scrape current S&P 500 constituents from Wikipedia.""" |
| 133 | + print("Fetching S&P 500 from Wikipedia ...") |
| 134 | + df = pd.read_html(SP500_WIKI_URL, attrs={"id": "constituents"})[0] |
| 135 | + stocks = {} |
| 136 | + for _, row in df.iterrows(): |
| 137 | + symbol = str(row["Symbol"]).strip().replace(".", "-") # BRK.B → BRK-B |
| 138 | + name = str(row["Security"]) |
| 139 | + stocks[symbol] = [symbol, name, f"https://finance.yahoo.com/quote/{symbol}/"] |
| 140 | + print(f" {len(stocks)} tickers") |
| 141 | + return dict(sorted(stocks.items())) |
| 142 | + |
| 143 | + |
| 144 | +def fetch_ftse_all_world() -> dict[str, list]: |
| 145 | + """Fetch FTSE All-World constituents via Vanguard's VT ETF portfolio API. |
| 146 | +
|
| 147 | + VT tracks the FTSE Global All Cap Index (FTSE All-World + small-cap). |
| 148 | + ISINs from the API are used to derive the correct yfinance exchange suffix |
| 149 | + for each stock (e.g. 600020 → 600020.SS, 7203 → 7203.T). |
| 150 | + """ |
| 151 | + print("Fetching FTSE All-World (Vanguard VT portfolio API) ...") |
| 152 | + stocks = {} |
| 153 | + start = 1 |
| 154 | + total = None |
| 155 | + |
| 156 | + while True: |
| 157 | + resp = requests.get( |
| 158 | + VT_API_URL, |
| 159 | + params={"start": start, "count": VT_PAGE_SIZE}, |
| 160 | + headers=HEADERS, |
| 161 | + timeout=30, |
| 162 | + ) |
| 163 | + resp.raise_for_status() |
| 164 | + data = resp.json() |
| 165 | + |
| 166 | + if total is None: |
| 167 | + total = data.get("size", "?") |
| 168 | + |
| 169 | + for entity in data.get("fund", {}).get("entity", []): |
| 170 | + raw_ticker = str(entity.get("ticker", "")).strip() |
| 171 | + isin = str(entity.get("isin", "")).strip() |
| 172 | + name = str(entity.get("longName", "") or entity.get("shortName", "")).strip() |
| 173 | + if not raw_ticker or raw_ticker == "nan": |
| 174 | + continue |
| 175 | + symbol = _to_yf_symbol(raw_ticker, isin) |
| 176 | + stocks[symbol] = [symbol, name, f"https://finance.yahoo.com/quote/{symbol}/"] |
| 177 | + |
| 178 | + fetched = start + VT_PAGE_SIZE - 1 |
| 179 | + print(f" {min(fetched, total if isinstance(total, int) else fetched)}/{total}") |
| 180 | + |
| 181 | + if "next" not in data: |
| 182 | + break |
| 183 | + start += VT_PAGE_SIZE |
| 184 | + |
| 185 | + print(f" {len(stocks)} unique tickers") |
| 186 | + return dict(sorted(stocks.items())) |
| 187 | + |
| 188 | + |
| 189 | +def _save(stocks: dict, path: Path) -> None: |
| 190 | + with open(path, "w") as f: |
| 191 | + json.dump(stocks, f, indent=4) |
| 192 | + print(f"Saved → {path}") |
| 193 | + |
| 194 | + |
| 195 | +if __name__ == "__main__": |
| 196 | + parser = argparse.ArgumentParser(description=__doc__) |
| 197 | + parser.add_argument("--sp500", action="store_true", help="Fetch S&P 500 only") |
| 198 | + parser.add_argument("--ftse", action="store_true", help="Fetch FTSE All-World only") |
| 199 | + args = parser.parse_args() |
| 200 | + run_all = not args.sp500 and not args.ftse |
| 201 | + |
| 202 | + if args.sp500 or run_all: |
| 203 | + _save(fetch_sp500(), SP500_FILE) |
| 204 | + |
| 205 | + if args.ftse or run_all: |
| 206 | + _save(fetch_ftse_all_world(), FTSE_AW_FILE) |
0 commit comments