|
| 1 | +#!/usr/bin/env python3 |
| 2 | +""" |
| 3 | +Usage: |
| 4 | + ./ingest.py # incremental: from max(id) in CH -> HN maxitem |
| 5 | + ./ingest.py 38718864 # start from specific ID -> HN maxitem |
| 6 | + ./ingest.py --dry-run # estimate items to download without downloading |
| 7 | +""" |
| 8 | + |
| 9 | +import asyncio |
| 10 | +import json |
| 11 | +import logging |
| 12 | +import os |
| 13 | +import sys |
| 14 | +from typing import List, Optional |
| 15 | + |
| 16 | +import aiohttp |
| 17 | +from clickhouse_driver import Client |
| 18 | + |
| 19 | +# Configuration |
| 20 | +HN_BASE_URL = "https://hacker-news.firebaseio.com/v0" |
| 21 | +MAX_CONNECTIONS = int(os.getenv("WORKERS", "500")) |
| 22 | +BATCH_SIZE = int(os.getenv("BATCH_LINES", "500")) |
| 23 | +BLOCK_SIZE = int(os.getenv("BLOCK_SIZE", "10000")) |
| 24 | + |
| 25 | +# ClickHouse connection (native protocol, not HTTP) |
| 26 | +CH_HOST = os.getenv("CLICKHOUSE_HOST", "localhost") |
| 27 | +CH_PORT = int(os.getenv("CLICKHOUSE_PORT", "9000")) # Native protocol port, not 8123 |
| 28 | +CH_USER = os.getenv("CLICKHOUSE_USER", "default") |
| 29 | +CH_PASSWORD = os.getenv("CLICKHOUSE_PASSWORD", "") |
| 30 | +CH_DATABASE = os.getenv("CLICKHOUSE_DATABASE", "default") |
| 31 | +CH_SECURE = os.getenv("CLICKHOUSE_SECURE", False) |
| 32 | +TABLE_NAME = os.getenv("TABLE_NAME", "hackernews") |
| 33 | + |
| 34 | +# Logging |
| 35 | +logging.basicConfig( |
| 36 | + level=logging.INFO, |
| 37 | + format='%(asctime)s - %(levelname)s - %(message)s' |
| 38 | +) |
| 39 | +logger = logging.getLogger(__name__) |
| 40 | + |
| 41 | + |
| 42 | +async def fetch_item(session: aiohttp.ClientSession, item_id: int) -> Optional[dict]: |
| 43 | + """Fetch a single HN item by ID.""" |
| 44 | + url = f"{HN_BASE_URL}/item/{item_id}.json" |
| 45 | + try: |
| 46 | + async with session.get(url, timeout=10) as response: |
| 47 | + if response.status == 200: |
| 48 | + return await response.json() |
| 49 | + return None |
| 50 | + except (aiohttp.ClientError, asyncio.TimeoutError) as e: |
| 51 | + logger.debug(f"Failed to fetch item {item_id}: {e}") |
| 52 | + return None |
| 53 | + |
| 54 | + |
| 55 | +async def fetch_maxitem(session: aiohttp.ClientSession) -> int: |
| 56 | + """Fetch the current max item ID from HN.""" |
| 57 | + url = f"{HN_BASE_URL}/maxitem.json" |
| 58 | + async with session.get(url) as response: |
| 59 | + return await response.json() |
| 60 | + |
| 61 | + |
| 62 | +async def download_batch(session: aiohttp.ClientSession, start_id: int, end_id: int) -> List[dict]: |
| 63 | + """Download a batch of items in parallel.""" |
| 64 | + tasks = [fetch_item(session, item_id) for item_id in range(start_id, end_id + 1)] |
| 65 | + results = await asyncio.gather(*tasks) |
| 66 | + # Filter out None values and non-valid items |
| 67 | + return [item for item in results if item and isinstance(item, dict)] |
| 68 | + |
| 69 | + |
| 70 | +def transform_item(item: dict) -> tuple: |
| 71 | + """Transform HN item to ClickHouse row format.""" |
| 72 | + return ( |
| 73 | + item.get("id", 0), |
| 74 | + 1 if item.get("deleted", False) else 0, |
| 75 | + item.get("type", "story"), |
| 76 | + item.get("by", ""), |
| 77 | + item.get("time", 0), |
| 78 | + item.get("text", ""), |
| 79 | + 1 if item.get("dead", False) else 0, |
| 80 | + item.get("parent", 0), |
| 81 | + item.get("poll", 0), |
| 82 | + item.get("kids", []), |
| 83 | + item.get("url", ""), |
| 84 | + item.get("score", 0), |
| 85 | + item.get("title", ""), |
| 86 | + item.get("parts", []), |
| 87 | + item.get("descendants", 0) |
| 88 | + ) |
| 89 | + |
| 90 | + |
| 91 | +def insert_to_clickhouse(client: Client, items: List[dict]) -> int: |
| 92 | + """Insert items into ClickHouse.""" |
| 93 | + if not items: |
| 94 | + return 0 |
| 95 | + |
| 96 | + # Filter valid item types |
| 97 | + valid_types = {"story", "comment", "poll", "pollopt", "job"} |
| 98 | + filtered_items = [item for item in items if item.get("type") in valid_types] |
| 99 | + |
| 100 | + if not filtered_items: |
| 101 | + return 0 |
| 102 | + |
| 103 | + rows = [transform_item(item) for item in filtered_items] |
| 104 | + |
| 105 | + query = f""" |
| 106 | + INSERT INTO {TABLE_NAME} |
| 107 | + (id, deleted, type, by, time, text, dead, parent, poll, kids, url, score, title, parts, descendants) |
| 108 | + VALUES |
| 109 | + """ |
| 110 | + |
| 111 | + client.execute(query, rows) |
| 112 | + return len(rows) |
| 113 | + |
| 114 | + |
| 115 | +async def main(): |
| 116 | + # Check for dry-run mode |
| 117 | + dry_run = "--dry-run" in sys.argv |
| 118 | + |
| 119 | + # Determine starting ID |
| 120 | + client = Client( |
| 121 | + host=CH_HOST, |
| 122 | + port=CH_PORT, |
| 123 | + user=CH_USER, |
| 124 | + password=CH_PASSWORD, |
| 125 | + database=CH_DATABASE, |
| 126 | + secure=CH_SECURE |
| 127 | + ) |
| 128 | + |
| 129 | + # Parse arguments (skip --dry-run flag) |
| 130 | + args = [arg for arg in sys.argv[1:] if arg != "--dry-run"] |
| 131 | + |
| 132 | + if len(args) > 0: |
| 133 | + start_id = int(args[0]) |
| 134 | + logger.info(f"Starting from ID (parameter): {start_id}") |
| 135 | + else: |
| 136 | + result = client.execute(f"SELECT max(id) FROM {TABLE_NAME}") |
| 137 | + last_id = result[0][0] if result and result[0][0] else 0 |
| 138 | + start_id = last_id + 1 |
| 139 | + logger.info(f"Last downloaded ID: {last_id}") |
| 140 | + logger.info(f"Starting from ID: {start_id}") |
| 141 | + |
| 142 | + # Fetch maxitem |
| 143 | + connector = aiohttp.TCPConnector(limit=MAX_CONNECTIONS) |
| 144 | + async with aiohttp.ClientSession(connector=connector) as session: |
| 145 | + maxitem = await fetch_maxitem(session) |
| 146 | + logger.info(f"HN maxitem: {maxitem}") |
| 147 | + |
| 148 | + if start_id > maxitem: |
| 149 | + logger.info(f"Nothing to do: start ID {start_id} > maxitem {maxitem}") |
| 150 | + return |
| 151 | + |
| 152 | + # Calculate estimate |
| 153 | + items_to_download = maxitem - start_id + 1 |
| 154 | + logger.info(f"Items to download: {items_to_download:,} (IDs {start_id} -> {maxitem})") |
| 155 | + |
| 156 | + if dry_run: |
| 157 | + logger.info("=" * 60) |
| 158 | + logger.info("DRY RUN MODE - No actual download will be performed") |
| 159 | + logger.info("=" * 60) |
| 160 | + logger.info(f"Estimated items to process: {items_to_download:,}") |
| 161 | + logger.info(f"Estimated blocks: {(items_to_download + BLOCK_SIZE - 1) // BLOCK_SIZE}") |
| 162 | + logger.info(f"Block size: {BLOCK_SIZE:,} items") |
| 163 | + logger.info(f"Concurrent workers: {MAX_CONNECTIONS}") |
| 164 | + logger.info(f"Estimated time: ~{items_to_download / 1000:.1f}-{items_to_download / 500:.1f} seconds") |
| 165 | + logger.info("=" * 60) |
| 166 | + return |
| 167 | + |
| 168 | + logger.info(f"Starting download...") |
| 169 | + |
| 170 | + total_inserted = 0 |
| 171 | + current_id = start_id |
| 172 | + |
| 173 | + while current_id <= maxitem: |
| 174 | + batch_end = min(current_id + BLOCK_SIZE - 1, maxitem) |
| 175 | + logger.info(f"Processing block: {current_id}..{batch_end}") |
| 176 | + |
| 177 | + # Download batch |
| 178 | + items = await download_batch(session, current_id, batch_end) |
| 179 | + logger.info(f"Downloaded {len(items)} items") |
| 180 | + |
| 181 | + # Insert to ClickHouse |
| 182 | + inserted = insert_to_clickhouse(client, items) |
| 183 | + total_inserted += inserted |
| 184 | + logger.info(f"Inserted {inserted} items (total: {total_inserted})") |
| 185 | + |
| 186 | + current_id = batch_end + 1 |
| 187 | + |
| 188 | + logger.info(f"Done! Total inserted: {total_inserted}") |
| 189 | + |
| 190 | + |
| 191 | +if __name__ == "__main__": |
| 192 | + asyncio.run(main()) |
0 commit comments