crawl.py

import os
import time
import json
import requests
import random
from urllib.parse import urlparse, urljoin
import logging

# Set up logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

BASE = "https://www.loc.gov/audio/"
QUERY = ""
FACETS = [
    "partof:american folklife center",
    "original-format:personal narrative",
]
ITEM_ID_PREFIX = "afc"
PER_PAGE = 50
MAX_PAGES = 10 
SLEEP_S = 10.0 # Safer sleep between items to stay under 20 req/min
SUB_SLEEP_S = 3.0 # Safer sleep between metadata formats
OUTDIR = "stories"

os.makedirs(OUTDIR, exist_ok=True)

HEADERS = {
    "User-Agent": "StoryLogCrawler/1.0 (Educational research project; jason@example.com)"
}

def get_json(url, params=None, retries=5, backoff=10):
    for i in range(retries):
        try:
            r = requests.get(url, params=params, headers=HEADERS, timeout=60)
            if r.status_code == 429:
                # Check for Retry-After header
                retry_after = r.headers.get("Retry-After")
                if retry_after:
                    try:
                        wait = int(retry_after) + random.uniform(0, 5)
                    except ValueError:
                        # Sometimes it's a date
                        wait = 3600 + random.uniform(0, 60)
                else:
                    # LoC documentation says block is typically 1 hour
                    wait = 3600 + random.uniform(0, 60)
                
                logger.warning(f"Rate limited (429) on {url}. Library of Congress typically blocks for 1 hour. Waiting {wait/60:.1f} minutes...")
                time.sleep(wait)
                continue
            
            r.raise_for_status()
            return r.json()
        except Exception as e:
            if i == retries - 1:
                logger.error(f"Error fetching {url} after {retries} retries: {e}")
                return None
            
            wait = (backoff * (2 ** i)) + random.uniform(0, 5)
            logger.warning(f"Request failed ({e}). Retrying ({i+1}/{retries}) in {wait:.2f}s...")
            time.sleep(wait)
    return None

def get_content(url, retries=5, backoff=10):
    """Fetch raw content (like XML or binary)."""
    for i in range(retries):
        try:
            r = requests.get(url, headers=HEADERS, timeout=60)
            if r.status_code == 429:
                retry_after = r.headers.get("Retry-After")
                if retry_after:
                    try:
                        wait = int(retry_after) + random.uniform(0, 5)
                    except ValueError:
                        wait = 3600 + random.uniform(0, 60)
                else:
                    wait = 3600 + random.uniform(0, 60)
                
                logger.warning(f"Rate limited (429) on {url}. Waiting {wait/60:.1f} minutes...")
                time.sleep(wait)
                continue
                
            r.raise_for_status()
            return r.content
        except Exception as e:
            if i == retries - 1:
                logger.error(f"Error fetching {url} after {retries} retries: {e}")
                return None
            
            wait = (backoff * (2 ** i)) + random.uniform(0, 5)
            logger.warning(f"Request failed ({e}). Retrying ({i+1}/{retries}) in {wait:.2f}s...")
            time.sleep(wait)
    return None

def extract_audio_downloads(item_json):
    """Extract audio download URLs and their metadata."""
    downloads = []
    audio_extensions = {".mp3", ".wav", ".m4a", ".mp4", ".aac", ".flac"}
    
    for res in item_json.get("resources") or []:
        if res.get("download_restricted") is True:
            continue
        files = res.get("files") or []
        for group in files:
            for f in group or []:
                mimetype = (f.get("mimetype") or "").lower()
                dl_url = f.get("download") or f.get("url")
                
                if not dl_url:
                    continue

                is_audio = (
                    mimetype.startswith("audio/") or 
                    mimetype == "application/x-audio" or
                    any(dl_url.lower().endswith(ext) for ext in audio_extensions)
                )

                if is_audio:
                    downloads.append({
                        "url": dl_url,
                        "mimetype": mimetype,
                        "label": f.get("label", "audio")
                    })
    return downloads

def sanitize_filename(filename):
    return "".join([c for c in filename if c.isalnum() or c in (' ', '.', '_')]).strip()

def crawl():
    seen_urls = set()
    manifest_path = os.path.join(OUTDIR, "manifest.jsonl")
    
    if os.path.exists(manifest_path):
        with open(manifest_path, "r") as f:
            for line in f:
                try:
                    entry = json.loads(line)
                    seen_urls.add(entry.get("item_url"))
                except:
                    continue

    facet_info = " | ".join(FACETS) if FACETS else "none"
    logger.info(f"Starting crawl in {BASE} (query='{QUERY}', facets='{facet_info}')")
    
    total_downloaded = 0

    for sp in range(1, MAX_PAGES + 1):
        logger.info(f"Fetching search results page {sp}...")
        params = {"fo": "json", "c": PER_PAGE, "sp": sp}
        if QUERY:
            params["q"] = QUERY
        if FACETS:
            params["fa"] = "|".join(FACETS)
        data = get_json(BASE, params=params)
        if not data:
            break
            
        results = data.get("results") or []
        if not results:
            logger.info("No more results.")
            break

        for item in results:
            item_url = item.get("id") or item.get("url")
            if not item_url:
                continue
            
            item_url = item_url.replace("http://", "https://")
            # If we've already fully processed this item in a previous run, skip it entirely
            item_id = item.get("id", "").strip("/").split("/")[-1]
            if not item_id:
                item_id = sanitize_filename(item.get("title", "unknown"))[:50]
            if ITEM_ID_PREFIX and not item_id.lower().startswith(ITEM_ID_PREFIX):
                logger.info(f"Skipping item outside AFC prefix: {item_id}")
                continue
            
            item_dir = os.path.join(OUTDIR, item_id)
            if item_url in seen_urls or os.path.exists(os.path.join(item_dir, "metadata.json")):
                logger.info(f"Skipping item (already exists): {item_id}")
                if item_url not in seen_urls:
                    seen_urls.add(item_url)
                continue
            
            logger.info(f"Processing item: {item_url}")
            os.makedirs(item_dir, exist_ok=True)
            
            # 1. Fetch full item metadata if not exists
            metadata_path = os.path.join(item_dir, "metadata.json")
            full_api_path = os.path.join(item_dir, "full_api_response.json")
            
            if not os.path.exists(full_api_path):
                time.sleep(SUB_SLEEP_S + random.uniform(0, 1))
                item_data = get_json(item_url, params={"fo": "json"})
                if not item_data:
                    continue
                with open(full_api_path, "w") as f:
                    json.dump(item_data, f, indent=2)
                
                focused_metadata = {
                    "cite_this": item_data.get("cite_this"),
                    "item": item_data.get("item"),
                    "timestamp": item_data.get("timestamp")
                }
                with open(metadata_path, "w") as f:
                    json.dump(focused_metadata, f, indent=2)
            else:
                with open(full_api_path, "r") as f:
                    item_data = json.load(f)

            # 2. Fetch additional formats if not exist
            for fmt in ["mods", "marcxml", "dc"]:
                fmt_fn = f"metadata_{fmt}.xml"
                fmt_path = os.path.join(item_dir, fmt_fn)
                if not os.path.exists(fmt_path):
                    time.sleep(SUB_SLEEP_S + random.uniform(0, 1))
                    fmt_url = f"{item_url}?fo={fmt}"
                    fmt_content = get_content(fmt_url)
                    if fmt_content:
                        with open(fmt_path, "wb") as f:
                            f.write(fmt_content)
                        logger.info(f"Saved {fmt} metadata for {item_id}")

            # 3. Check for audio files
            audio_files = extract_audio_downloads(item_data)
            if not audio_files:
                logger.warning(f"No audio downloads found for {item_url}")
                continue

            # Deduplicate by URL (normalized)
            unique_audio = []
            seen_dl_urls = set()
            for a in audio_files:
                parsed = urlparse(a["url"])
                norm_url = f"https://{parsed.netloc}{parsed.path}".lower().rstrip("/")
                if norm_url not in seen_dl_urls:
                    unique_audio.append(a)
                    seen_dl_urls.add(norm_url)
            
            item_audio_paths = []
            for i, audio in enumerate(unique_audio):
                dl_url = audio["url"]
                ext = "mp3" if "mpeg" in audio["mimetype"] else "wav" if "wav" in audio["mimetype"] else "audio"
                fn = f"audio_{i}.{ext}"
                dest_path = os.path.join(item_dir, fn)
                
                # Sidecar metadata
                sidecar_path = dest_path + ".json"
                if not os.path.exists(sidecar_path):
                    item_info = item_data.get("item", {})
                    with open(sidecar_path, "w") as sf:
                        json.dump({
                            "item_url": item_url,
                            "download_url": dl_url,
                            "mimetype": audio["mimetype"],
                            "label": audio["label"],
                            "title": item_info.get("title") or item.get("title"),
                            "item_id": item_id,
                            "contributors": item_info.get("contributor_names"),
                            "date": item_info.get("date"),
                            "description": item_info.get("description"),
                            "subjects": item_info.get("subjects"),
                            "location": item_info.get("location"),
                            "notes": item_info.get("notes"),
                            "timestamp": time.time()
                        }, sf, indent=2)

                if os.path.exists(dest_path):
                    item_audio_paths.append(dest_path)
                    continue

                logger.info(f"Downloading {dl_url}...")
                time.sleep(SUB_SLEEP_S + random.uniform(0, 1))
                try:
                    with requests.get(dl_url, headers=HEADERS, stream=True, timeout=120) as r:
                        if r.status_code == 429:
                            logger.warning("Rate limited on download. Skipping this file for now.")
                            continue
                        r.raise_for_status()
                        with open(dest_path, "wb") as w:
                            for chunk in r.iter_content(chunk_size=1024 * 1024):
                                if chunk:
                                    w.write(chunk)
                    item_audio_paths.append(dest_path)
                    total_downloaded += 1
                except Exception as e:
                    logger.error(f"Failed to download {dl_url}: {e}")

            # Update manifest if we've completed the item
            if item_url not in seen_urls:
                with open(manifest_path, "a") as f:
                    manifest_entry = {
                        "item_url": item_url,
                        "item_id": item_id,
                        "title": item.get("title"),
                        "audio_files": item_audio_paths,
                        "timestamp": time.time()
                    }
                    f.write(json.dumps(manifest_entry) + "\n")
                seen_urls.add(item_url)
            
            time.sleep(SLEEP_S + random.uniform(0, 2))

        time.sleep(SLEEP_S * 2) # Extra sleep between pages

    logger.info(f"Crawl completed. Downloaded {total_downloaded} new files.")

if __name__ == "__main__":
    crawl()