|
| 1 | +"""Post-dedup PDF resolver — fill missing pdf_url from open-access aggregators. |
| 2 | +
|
| 3 | +Why this exists |
| 4 | +--------------- |
| 5 | +Most IEEE / ACM / Springer / Elsevier papers come back from their |
| 6 | +respective source plugins with ``pdf_url=None`` because the publisher |
| 7 | +sites are paywalled even when the paper itself is open access. The OA |
| 8 | +copy almost always exists somewhere else — the author's institutional |
| 9 | +repository, an arXiv preprint, ResearchGate, etc. — and Unpaywall |
| 10 | +indexes ~50M of them keyed by DOI. |
| 11 | +
|
| 12 | +This module runs after dedup and tries two strategies in order for |
| 13 | +every paper that still lacks a pdf_url: |
| 14 | +
|
| 15 | +1. **Unpaywall** (https://unpaywall.org/products/api). Free, no API |
| 16 | + key required, but needs an email in the query string per their |
| 17 | + politeness contract. Pulled from ``AUTOPAPERTOPPT_CONTACT_EMAIL`` |
| 18 | + (same env var Crossref / OpenAlex use); skipped silently when |
| 19 | + unset, with a one-time WARNING log so the user knows what they're |
| 20 | + missing. |
| 21 | +
|
| 22 | +2. **arXiv title search**. For papers without a DOI (or where |
| 23 | + Unpaywall returned no hit), search arXiv with the paper's title |
| 24 | + and accept the first result whose normalised title matches |
| 25 | + exactly. Covers the many CS papers that have an arXiv preprint |
| 26 | + but reach the pipeline via OpenAlex / Crossref / DBLP without |
| 27 | + the arXiv ID populated. |
| 28 | +
|
| 29 | +Both lookups are best-effort: any failure logs at DEBUG and the |
| 30 | +paper passes through unchanged. The resolver never raises. |
| 31 | +""" |
| 32 | + |
| 33 | +from __future__ import annotations |
| 34 | + |
| 35 | +import asyncio |
| 36 | +import dataclasses |
| 37 | +import os |
| 38 | +from typing import Any |
| 39 | + |
| 40 | +import httpx |
| 41 | + |
| 42 | +from autopapertoppt.core.exceptions import FetchError |
| 43 | +from autopapertoppt.core.models import Paper, PaperCollection, Query |
| 44 | +from autopapertoppt.fetchers.http import get_client |
| 45 | +from autopapertoppt.utils.logging import get_logger |
| 46 | + |
| 47 | +_LOG = get_logger(__name__) |
| 48 | + |
| 49 | +_UNPAYWALL_ENDPOINT = "https://api.unpaywall.org/v2" |
| 50 | +_UNPAYWALL_SOURCE = "unpaywall" |
| 51 | +_LOOKUP_TIMEOUT_SECONDS = 10.0 |
| 52 | +_CONCURRENCY = 5 |
| 53 | + |
| 54 | +# One-shot warning so we don't spam logs for every paper in a large run. |
| 55 | +_email_warning_emitted = False |
| 56 | + |
| 57 | + |
| 58 | +async def resolve_oa_pdfs(collection: PaperCollection) -> PaperCollection: |
| 59 | + """Try to fill ``pdf_url`` for every paper currently missing one. |
| 60 | +
|
| 61 | + Returns a new ``PaperCollection`` with the same query and same |
| 62 | + paper count. Papers that already have ``pdf_url`` pass through |
| 63 | + unchanged. |
| 64 | + """ |
| 65 | + missing = sum(1 for p in collection.papers if not p.pdf_url) |
| 66 | + if missing == 0: |
| 67 | + return collection |
| 68 | + |
| 69 | + _LOG.info( |
| 70 | + "OA resolver: looking up %d / %d papers without pdf_url", |
| 71 | + missing, |
| 72 | + len(collection.papers), |
| 73 | + ) |
| 74 | + |
| 75 | + semaphore = asyncio.Semaphore(_CONCURRENCY) |
| 76 | + resolved = await asyncio.gather( |
| 77 | + *(_resolve_one(paper, semaphore) for paper in collection.papers) |
| 78 | + ) |
| 79 | + found = sum( |
| 80 | + 1 |
| 81 | + for old, new in zip(collection.papers, resolved, strict=True) |
| 82 | + if not old.pdf_url and new.pdf_url |
| 83 | + ) |
| 84 | + if found: |
| 85 | + _LOG.info( |
| 86 | + "OA resolver: filled %d / %d missing pdf_url (Unpaywall + arXiv)", |
| 87 | + found, |
| 88 | + missing, |
| 89 | + ) |
| 90 | + return PaperCollection(query=collection.query, papers=tuple(resolved)) |
| 91 | + |
| 92 | + |
| 93 | +async def _resolve_one(paper: Paper, semaphore: asyncio.Semaphore) -> Paper: |
| 94 | + if paper.pdf_url: |
| 95 | + return paper |
| 96 | + async with semaphore: |
| 97 | + # 1. Unpaywall by DOI — fastest path, highest precision. |
| 98 | + if paper.doi: |
| 99 | + pdf = await _query_unpaywall(paper.doi) |
| 100 | + if pdf: |
| 101 | + _LOG.debug("Unpaywall hit for %s: %s", paper.bibtex_key(), pdf) |
| 102 | + return dataclasses.replace(paper, pdf_url=pdf) |
| 103 | + # 2. arXiv title search — covers DOI-less papers + DOIs missed |
| 104 | + # by Unpaywall. |
| 105 | + pdf = await _query_arxiv_title(paper) |
| 106 | + if pdf: |
| 107 | + _LOG.debug("arXiv title hit for %s: %s", paper.bibtex_key(), pdf) |
| 108 | + return dataclasses.replace(paper, pdf_url=pdf) |
| 109 | + return paper |
| 110 | + |
| 111 | + |
| 112 | +async def _query_unpaywall(doi: str) -> str | None: |
| 113 | + """Look up a DOI in Unpaywall; return the best OA PDF URL or None.""" |
| 114 | + email = os.environ.get("AUTOPAPERTOPPT_CONTACT_EMAIL", "").strip() |
| 115 | + if not email: |
| 116 | + _warn_once_about_email() |
| 117 | + return None |
| 118 | + client = await get_client(_UNPAYWALL_SOURCE) |
| 119 | + try: |
| 120 | + response = await asyncio.wait_for( |
| 121 | + client.get( |
| 122 | + f"{_UNPAYWALL_ENDPOINT}/{doi}", |
| 123 | + params={"email": email}, |
| 124 | + ), |
| 125 | + timeout=_LOOKUP_TIMEOUT_SECONDS, |
| 126 | + ) |
| 127 | + except (TimeoutError, httpx.HTTPError, FetchError) as err: |
| 128 | + _LOG.debug("Unpaywall lookup failed for %s: %s", doi, err) |
| 129 | + return None |
| 130 | + if response.status_code == 404: |
| 131 | + return None # not indexed |
| 132 | + if response.status_code != 200: |
| 133 | + _LOG.debug( |
| 134 | + "Unpaywall returned %s for %s: %s", |
| 135 | + response.status_code, doi, response.text[:128], |
| 136 | + ) |
| 137 | + return None |
| 138 | + try: |
| 139 | + data: dict[str, Any] = response.json() |
| 140 | + except ValueError: |
| 141 | + return None |
| 142 | + best_oa = data.get("best_oa_location") or {} |
| 143 | + candidate = (best_oa.get("url_for_pdf") or "").strip() |
| 144 | + if candidate.startswith("https://"): |
| 145 | + return candidate |
| 146 | + return None |
| 147 | + |
| 148 | + |
| 149 | +async def _query_arxiv_title(paper: Paper) -> str | None: |
| 150 | + """Search arXiv by title; return the matching paper's PDF URL or None. |
| 151 | +
|
| 152 | + Match is exact on the normalised title (alphanumeric + lowercase) |
| 153 | + so a "transformer" paper doesn't accidentally claim someone else's |
| 154 | + "transformer architecture for X" preprint. |
| 155 | + """ |
| 156 | + if not paper.title: |
| 157 | + return None |
| 158 | + # Skip the round-trip if the paper is already from arXiv — its |
| 159 | + # plugin would have populated pdf_url at parse time if a PDF |
| 160 | + # existed. |
| 161 | + if paper.source == "arxiv": |
| 162 | + return None |
| 163 | + try: |
| 164 | + from autopapertoppt.fetchers.base import load_fetcher |
| 165 | + except ImportError: |
| 166 | + return None |
| 167 | + try: |
| 168 | + fetcher = load_fetcher("arxiv") |
| 169 | + except Exception: # noqa: BLE001 — load failures must not break the resolver |
| 170 | + return None |
| 171 | + |
| 172 | + # arXiv's API supports field-restricted queries; ti:"<title>" looks |
| 173 | + # only at the title field. Pull the top 3 in case the first is a |
| 174 | + # later version of a different paper with similar words. |
| 175 | + query = Query( |
| 176 | + keywords=f'ti:"{paper.title}"', |
| 177 | + sources=("arxiv",), |
| 178 | + max_results=3, |
| 179 | + ) |
| 180 | + try: |
| 181 | + results = await fetcher.search(query) |
| 182 | + except Exception as err: # noqa: BLE001 — best-effort |
| 183 | + _LOG.debug("arXiv title search failed for %r: %s", paper.title, err) |
| 184 | + return None |
| 185 | + |
| 186 | + target = _normalise_title(paper.title) |
| 187 | + for candidate in results: |
| 188 | + if ( |
| 189 | + _normalise_title(candidate.title) == target |
| 190 | + and candidate.pdf_url |
| 191 | + and candidate.pdf_url.startswith("https://") |
| 192 | + ): |
| 193 | + return candidate.pdf_url |
| 194 | + return None |
| 195 | + |
| 196 | + |
| 197 | +def _normalise_title(text: str) -> str: |
| 198 | + """Lowercase + drop non-alphanumeric for fuzzy title comparison.""" |
| 199 | + return "".join(c.lower() for c in text if c.isalnum()) |
| 200 | + |
| 201 | + |
| 202 | +def _warn_once_about_email() -> None: |
| 203 | + """Log a single WARNING line when CONTACT_EMAIL is unset. |
| 204 | +
|
| 205 | + Module-global flag rather than logging-stdlib filter because we |
| 206 | + want the warning per-process, not per-logger-handler, and the |
| 207 | + fewer moving parts the better. |
| 208 | + """ |
| 209 | + global _email_warning_emitted # noqa: PLW0603 — intentional one-shot flag |
| 210 | + if _email_warning_emitted: |
| 211 | + return |
| 212 | + _email_warning_emitted = True |
| 213 | + _LOG.warning( |
| 214 | + "OA resolver: AUTOPAPERTOPPT_CONTACT_EMAIL is not set; " |
| 215 | + "Unpaywall lookups (the biggest PDF coverage win for IEEE / " |
| 216 | + "ACM / Springer / Elsevier papers) will be skipped. Set the " |
| 217 | + "env var to your email to enable them." |
| 218 | + ) |
0 commit comments