insider_scraper.py

#!/usr/bin/env python3
"""
Korean Insider Trading Scraper

Collects insider purchase/sell data from DART API (elestock.json)
for tracked companies, enriches with stock prices, and stores in SQLite.

Usage:
    python insider_scraper.py                    # Scrape all semiconductor stocks
    python insider_scraper.py --broad            # Scrape KOSPI 200 + KOSDAQ 150 top stocks
    python insider_scraper.py --stock 005930     # Scrape specific stock
    python insider_scraper.py --enrich           # Update prices & returns
    python insider_scraper.py --summary          # Show summary of collected data
    python insider_scraper.py --purchases        # Show recent insider purchases
    python insider_scraper.py --opportunities    # Purchases where stock hasn't moved
"""

import warnings
warnings.filterwarnings('ignore')

import sys
import time
import argparse
import pandas as pd
from datetime import datetime, timedelta

from config import DART_API_KEY, DART_BASE_URL, SEMICONDUCTOR_STOCKS
from insider_storage import (
    init_insider_tables, save_insider_trades_bulk, update_prices,
    get_insider_purchases, get_trade_summary
)

import requests

SESSION = requests.Session()
SESSION.headers.update({
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
})
LAST_CALL = 0
RATE_LIMIT = 0.35  # seconds between calls


def _rate_limit():
    global LAST_CALL
    elapsed = time.time() - LAST_CALL
    if elapsed < RATE_LIMIT:
        time.sleep(RATE_LIMIT - elapsed)
    LAST_CALL = time.time()


def _parse_num(val):
    if val is None:
        return 0
    if isinstance(val, (int, float)):
        return val
    try:
        return float(str(val).replace(',', '').strip())
    except (ValueError, TypeError):
        return 0


# ── Corp code helpers ────────────────────────────────────────────────────────

def load_corp_codes():
    """Load corp_code <-> stock_code mapping."""
    try:
        df = pd.read_pickle('corp_codes.pkl')
        return df
    except FileNotFoundError:
        print("Downloading corp_codes from DART...")
        import zipfile, io
        import xml.etree.ElementTree as ET
        r = requests.get(f'{DART_BASE_URL}/corpCode.xml',
                         params={'crtfc_key': DART_API_KEY})
        zf = zipfile.ZipFile(io.BytesIO(r.content))
        xml_data = zf.read('CORPCODE.xml')
        tree = ET.XML(xml_data)
        records = [{sub.tag: sub.text for sub in child} for child in tree.findall('list')]
        df = pd.DataFrame(records)
        df.to_pickle('corp_codes.pkl')
        print(f"  Saved {len(df)} corp codes")
        return df


def get_corp_code(stock_code, corp_df):
    """Look up corp_code from stock_code."""
    match = corp_df[corp_df['stock_code'] == stock_code]
    if match.empty:
        return None
    return match.iloc[0]['corp_code']


# ── DART API calls ───────────────────────────────────────────────────────────

def fetch_elestock(corp_code):
    """Fetch insider ownership data from DART elestock.json endpoint."""
    _rate_limit()
    r = SESSION.get(f'{DART_BASE_URL}/elestock.json', params={
        'crtfc_key': DART_API_KEY,
        'corp_code': corp_code,
    })
    try:
        jo = r.json()
    except Exception:
        return []

    if jo.get('status') != '000':
        return []

    return jo.get('list', [])


def fetch_insider_filings(start_date='20200101', end_date=None, corp_code=None, max_pages=5):
    """Search DART for insider ownership filings (D002 type)."""
    if end_date is None:
        end_date = datetime.today().strftime('%Y%m%d')

    all_rows = []
    for page in range(1, max_pages + 1):
        _rate_limit()
        params = {
            'crtfc_key': DART_API_KEY,
            'bgn_de': start_date,
            'end_de': end_date,
            'pblntf_detail_ty': 'D002',
            'page_no': page,
            'page_count': 100,
            'sort': 'date',
            'sort_mth': 'desc',
        }
        if corp_code:
            params['corp_code'] = corp_code

        r = SESSION.get(f'{DART_BASE_URL}/list.json', params=params)
        try:
            jo = r.json()
        except Exception:
            break

        if jo.get('status') != '000':
            break

        rows = jo.get('list', [])
        all_rows.extend(rows)

        total_pages = jo.get('total_page', 1)
        if page >= total_pages:
            break

    return all_rows


# ── Parse & classify ─────────────────────────────────────────────────────────

def parse_elestock_record(record, stock_code, company_name, corp_code):
    """Parse a single elestock record into our standardized format."""
    shares_changed = _parse_num(record.get('sp_stock_lmp_irds_cnt', 0))
    shares_after = _parse_num(record.get('sp_stock_lmp_cnt', 0))
    pct_after = _parse_num(record.get('sp_stock_lmp_rate', 0))
    pct_change = _parse_num(record.get('sp_stock_lmp_irds_rate', 0))

    # Determine transaction type
    if shares_changed > 0:
        tx_type = 'BUY'
    elif shares_changed < 0:
        tx_type = 'SELL'
    else:
        tx_type = 'NO_CHANGE'

    return {
        'rcept_no': record.get('rcept_no', ''),
        'rcept_dt': record.get('rcept_dt', ''),
        'corp_code': corp_code,
        'stock_code': stock_code,
        'company_name': company_name,
        'insider_name': record.get('repror', '').strip(),
        'insider_position': record.get('isu_exctv_ofcps', '').strip(),
        'is_registered_exec': record.get('isu_exctv_rgist_at', '').strip(),
        'is_major_shareholder': record.get('isu_main_shrholdr', '').strip(),
        'transaction_type': tx_type,
        'shares_changed': abs(shares_changed),
        'shares_after': shares_after,
        'ownership_pct_after': pct_after,
        'ownership_pct_change': abs(pct_change),
        'price_at_filing': 0,
        'value_krw': 0,
        'price_current': 0,
        'return_since_filing': 0,
    }


# ── Stock price enrichment ───────────────────────────────────────────────────

def get_stock_price(stock_code, date_str):
    """Get stock close price on a given date using PyKrx."""
    try:
        from pykrx import stock
        dt = pd.to_datetime(date_str)
        # Try the filing date and a few days around it
        for offset in range(0, 5):
            try_date = (dt + timedelta(days=offset)).strftime('%Y%m%d')
            df = stock.get_market_ohlcv_by_date(try_date, try_date, stock_code)
            if not df.empty and df.iloc[0]['종가'] > 0:
                return float(df.iloc[0]['종가'])
        return 0
    except Exception:
        return 0


def get_current_price(stock_code):
    """Get latest stock price using PyKrx."""
    try:
        from pykrx import stock
        today = datetime.today()
        for offset in range(0, 10):
            try_date = (today - timedelta(days=offset)).strftime('%Y%m%d')
            df = stock.get_market_ohlcv_by_date(try_date, try_date, stock_code)
            if not df.empty and df.iloc[0]['종가'] > 0:
                return float(df.iloc[0]['종가'])
        return 0
    except Exception:
        return 0


# ── Main scraping functions ──────────────────────────────────────────────────

def scrape_company_insiders(stock_code, company_name, corp_code):
    """Scrape all insider data for a single company."""
    print(f"  {company_name} ({stock_code})...", end=' ', flush=True)

    records = fetch_elestock(corp_code)
    if not records:
        print("no data")
        return []

    trades = []
    for record in records:
        trade = parse_elestock_record(record, stock_code, company_name, corp_code)
        if trade['insider_name']:
            trades.append(trade)

    buys = sum(1 for t in trades if t['transaction_type'] == 'BUY')
    sells = sum(1 for t in trades if t['transaction_type'] == 'SELL')
    print(f"{len(trades)} records ({buys} buys, {sells} sells)")

    return trades


def scrape_all_semiconductor(enrich=False):
    """Scrape insider data for all semiconductor stocks."""
    print("\n" + "=" * 70)
    print("SCRAPING INSIDER DATA - SEMICONDUCTOR STOCKS")
    print("=" * 70)

    corp_df = load_corp_codes()
    all_trades = []

    for stock_code, company_name in SEMICONDUCTOR_STOCKS.items():
        corp_code = get_corp_code(stock_code, corp_df)
        if not corp_code:
            print(f"  {company_name} ({stock_code}): corp_code not found, skipping")
            continue

        trades = scrape_company_insiders(stock_code, company_name, corp_code)
        all_trades.extend(trades)
        time.sleep(0.1)

    if all_trades:
        saved = save_insider_trades_bulk(all_trades)
        print(f"\nTotal records: {len(all_trades)}, saved: {saved}")
    else:
        print("\nNo insider data found")

    return all_trades


def scrape_broad_market():
    """Scrape insider data for top KOSPI/KOSDAQ companies."""
    print("\n" + "=" * 70)
    print("SCRAPING INSIDER DATA - BROAD MARKET")
    print("=" * 70)

    corp_df = load_corp_codes()
    listed = corp_df[corp_df['stock_code'].notna() & (corp_df['stock_code'] != '') & (corp_df['stock_code'] != ' ')]

    # Get KOSPI/KOSDAQ market caps to pick top stocks
    try:
        from pykrx import stock
        today = datetime.today()
        for offset in range(0, 10):
            try_date = (today - timedelta(days=offset)).strftime('%Y%m%d')
            kospi = stock.get_market_cap_by_ticker(try_date, market="KOSPI")
            if not kospi.empty:
                break

        for offset in range(0, 10):
            try_date = (today - timedelta(days=offset)).strftime('%Y%m%d')
            kosdaq = stock.get_market_cap_by_ticker(try_date, market="KOSDAQ")
            if not kosdaq.empty:
                break

        # Top 200 KOSPI + top 100 KOSDAQ by market cap
        top_kospi = kospi.nlargest(200, '시가총액').index.tolist() if not kospi.empty else []
        top_kosdaq = kosdaq.nlargest(100, '시가총액').index.tolist() if not kosdaq.empty else []
        target_stocks = top_kospi + top_kosdaq
        print(f"Targeting {len(target_stocks)} stocks (top {len(top_kospi)} KOSPI + top {len(top_kosdaq)} KOSDAQ)")

    except Exception as e:
        print(f"Error getting market caps: {e}")
        print("Falling back to semiconductor stocks only")
        target_stocks = list(SEMICONDUCTOR_STOCKS.keys())

    all_trades = []
    for i, stock_code in enumerate(target_stocks):
        # Look up company info
        match = listed[listed['stock_code'] == stock_code]
        if match.empty:
            continue

        corp_code = match.iloc[0]['corp_code']
        company_name = match.iloc[0]['corp_name']

        if (i + 1) % 50 == 0:
            print(f"\n  --- Progress: {i + 1}/{len(target_stocks)} ---")

        trades = scrape_company_insiders(stock_code, company_name, corp_code)
        all_trades.extend(trades)
        time.sleep(0.1)

    if all_trades:
        saved = save_insider_trades_bulk(all_trades)
        print(f"\nTotal records: {len(all_trades)}, saved: {saved}")

    return all_trades


def enrich_with_prices():
    """Update all trades with stock prices and returns."""
    print("\n" + "=" * 70)
    print("ENRICHING INSIDER TRADES WITH STOCK PRICES")
    print("=" * 70)

    import sqlite3
    conn = sqlite3.connect('korean_fund_holdings.db')
    df = pd.read_sql("SELECT DISTINCT stock_code, company_name FROM insider_trades", conn)
    conn.close()

    if df.empty:
        print("No trades to enrich. Run scraper first.")
        return

    print(f"Enriching prices for {len(df)} stocks...")

    for _, row in df.iterrows():
        stock_code = row['stock_code']
        company_name = row['company_name']
        print(f"  {company_name} ({stock_code})...", end=' ', flush=True)

        # Get current price
        current_price = get_current_price(stock_code)
        if current_price <= 0:
            print("no price data")
            continue

        # Update current prices and returns
        update_prices(stock_code, current_price)

        # Also try to fill in filing-date prices for trades missing them
        conn = sqlite3.connect('korean_fund_holdings.db')
        missing = pd.read_sql(
            "SELECT id, rcept_dt FROM insider_trades WHERE stock_code = ? AND (price_at_filing = 0 OR price_at_filing IS NULL)",
            conn, params=(stock_code,)
        )
        conn.close()

        filled = 0
        for _, trade_row in missing.iterrows():
            price = get_stock_price(stock_code, trade_row['rcept_dt'])
            if price > 0:
                conn = sqlite3.connect('korean_fund_holdings.db')
                shares = pd.read_sql(
                    f"SELECT shares_changed FROM insider_trades WHERE id = {trade_row['id']}", conn
                ).iloc[0]['shares_changed']
                value = price * shares
                ret = ((current_price - price) / price) * 100 if price > 0 else 0
                conn.execute(
                    "UPDATE insider_trades SET price_at_filing = ?, value_krw = ?, price_current = ?, return_since_filing = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?",
                    (price, value, current_price, ret, trade_row['id'])
                )
                conn.commit()
                conn.close()
                filled += 1
            time.sleep(0.3)  # Be nice to KRX

        print(f"current={current_price:,.0f} KRW, filled {filled}/{len(missing)} filing prices")

    print("\nPrice enrichment complete.")


def show_summary():
    """Show summary of collected insider data."""
    print("\n" + "=" * 70)
    print("INSIDER TRADING DATA SUMMARY")
    print("=" * 70)

    summary = get_trade_summary()
    if summary.empty:
        print("No data. Run: python insider_scraper.py")
        return

    print("\nBy transaction type:")
    print(summary.to_string(index=False))

    import sqlite3
    conn = sqlite3.connect('korean_fund_holdings.db')

    # Top companies by insider activity
    top_co = pd.read_sql('''
        SELECT company_name, stock_code,
               COUNT(*) as total_filings,
               SUM(CASE WHEN transaction_type='BUY' THEN 1 ELSE 0 END) as buys,
               SUM(CASE WHEN transaction_type='SELL' THEN 1 ELSE 0 END) as sells
        FROM insider_trades
        GROUP BY stock_code
        ORDER BY buys DESC
        LIMIT 20
    ''', conn)
    print("\nTop companies by insider BUYS:")
    print(top_co.to_string(index=False))

    # Top insider buyers
    top_buyers = pd.read_sql('''
        SELECT insider_name, company_name, insider_position,
               COUNT(*) as buy_count,
               SUM(shares_changed) as total_shares,
               MAX(rcept_dt) as last_buy
        FROM insider_trades
        WHERE transaction_type = 'BUY'
        GROUP BY insider_name, company_name
        ORDER BY buy_count DESC
        LIMIT 20
    ''', conn)
    print("\nTop insider buyers (by frequency):")
    print(top_buyers.to_string(index=False))

    conn.close()


def show_recent_purchases(days=90, min_value=0):
    """Show recent insider purchases."""
    print(f"\n{'=' * 70}")
    print(f"INSIDER PURCHASES - LAST {days} DAYS")
    print(f"{'=' * 70}")

    df = get_insider_purchases(days=days, min_value=min_value)
    if df.empty:
        print("No purchases found. Run enrichment first: python insider_scraper.py --enrich")
        return

    cols = ['rcept_dt', 'company_name', 'insider_name', 'insider_position',
            'shares_changed', 'value_krw', 'price_at_filing', 'price_current', 'return_since_filing']
    available = [c for c in cols if c in df.columns]
    print(df[available].to_string(index=False))
    print(f"\nTotal: {len(df)} purchases")


def show_opportunities():
    """Show insider purchases where stock hasn't moved much (potential opportunities)."""
    print(f"\n{'=' * 70}")
    print(f"OPPORTUNITIES - INSIDER BUYS WHERE STOCK HASN'T MOVED")
    print(f"{'=' * 70}")

    import sqlite3
    conn = sqlite3.connect('korean_fund_holdings.db')
    df = pd.read_sql('''
        SELECT rcept_dt, company_name, stock_code, insider_name, insider_position,
               shares_changed, value_krw,
               price_at_filing, price_current, return_since_filing
        FROM insider_trades
        WHERE transaction_type = 'BUY'
          AND price_at_filing > 0
          AND price_current > 0
          AND return_since_filing BETWEEN -10 AND 10
        ORDER BY rcept_dt DESC
    ''', conn)
    conn.close()

    if df.empty:
        print("No opportunities found. Run enrichment first: python insider_scraper.py --enrich")
        return

    print(df.to_string(index=False))
    print(f"\nTotal: {len(df)} purchases within +/- 10% of filing price")


# ── Main ─────────────────────────────────────────────────────────────────────

def main():
    parser = argparse.ArgumentParser(description='Korean Insider Trading Scraper')
    parser.add_argument('--stock', type=str, help='Scrape specific stock ticker')
    parser.add_argument('--broad', action='store_true', help='Scrape top 300 KOSPI+KOSDAQ stocks')
    parser.add_argument('--enrich', action='store_true', help='Enrich trades with stock prices')
    parser.add_argument('--summary', action='store_true', help='Show data summary')
    parser.add_argument('--purchases', action='store_true', help='Show recent insider purchases')
    parser.add_argument('--days', type=int, default=90, help='Days to look back for purchases')
    parser.add_argument('--opportunities', action='store_true', help='Show opportunities (stock flat since buy)')

    args = parser.parse_args()
    init_insider_tables()

    if args.summary:
        show_summary()
        return

    if args.purchases:
        show_recent_purchases(days=args.days)
        return

    if args.opportunities:
        show_opportunities()
        return

    if args.enrich:
        enrich_with_prices()
        return

    if args.stock:
        corp_df = load_corp_codes()
        corp_code = get_corp_code(args.stock, corp_df)
        if not corp_code:
            print(f"Corp code not found for {args.stock}")
            return
        name = SEMICONDUCTOR_STOCKS.get(args.stock, args.stock)
        # Try to get name from corp_df
        match = corp_df[corp_df['stock_code'] == args.stock]
        if not match.empty:
            name = match.iloc[0]['corp_name']
        trades = scrape_company_insiders(args.stock, name, corp_code)
        if trades:
            saved = save_insider_trades_bulk(trades)
            print(f"Saved: {saved}")
        return

    if args.broad:
        scrape_broad_market()
    else:
        scrape_all_semiconductor()

    # Always show summary after scraping
    show_summary()


if __name__ == '__main__':
    main()