Big changes

app/services/domains/unpaywall/__init__.py

@@ -1,5 +1,10 @@
 from __future__ import annotations
 
-from app.services.domains.unpaywall.application import resolve_publication_pdf_urls
+
+async def resolve_publication_pdf_urls(*args, **kwargs):
+    from app.services.domains.unpaywall.application import resolve_publication_pdf_urls as _impl
+
+    return await _impl(*args, **kwargs)
+
 
 __all__ = ["resolve_publication_pdf_urls"]

app/services/domains/unpaywall/application.py

@@ -1,21 +1,44 @@
 from __future__ import annotations
 
+from dataclasses import dataclass
 import logging
 import re
+from typing import TYPE_CHECKING
 from urllib.parse import unquote
 
 from app.services.domains.crossref.application import discover_doi_for_publication
-from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
 from app.services.domains.doi.normalize import normalize_doi
+from app.services.domains.unpaywall.pdf_discovery import (
+    looks_like_pdf_url,
+    resolve_pdf_from_landing_page,
+)
+from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
 from app.settings import settings
 
+if TYPE_CHECKING:
+    from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+
 DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
 DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
 DOI_URL_RE = re.compile(r"(?:https?://)?(?:dx\.)?doi\.org/(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
 UNPAYWALL_URL_TEMPLATE = "https://api.unpaywall.org/v2/{doi}"
+FAILURE_MISSING_DOI = "missing_doi"
+FAILURE_NO_RECORD = "no_unpaywall_record"
+FAILURE_NO_PDF = "no_pdf_found"
+FAILURE_RESOLUTION_EXCEPTION = "resolution_exception"
 logger = logging.getLogger(__name__)
 
 
+@dataclass(frozen=True)
+class OaResolutionOutcome:
+    publication_id: int
+    doi: str | None
+    pdf_url: str | None
+    failure_reason: str | None
+    source: str | None
+    used_crossref: bool
+
+
 def _extract_doi_candidate(text: str | None) -> str | None:
     if not text:
         return None
@@ -57,21 +80,89 @@ def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str |
     )
 
 
-def _payload_pdf_url(payload: dict) -> str | None:
+def _payload_locations(payload: dict) -> list[dict]:
+    locations: list[dict] = []
     best = payload.get("best_oa_location")
     if isinstance(best, dict):
-        pdf_url = best.get("url_for_pdf")
-        if isinstance(pdf_url, str) and pdf_url.strip():
-            return pdf_url.strip()
-    locations = payload.get("oa_locations")
-    if not isinstance(locations, list):
+        locations.append(best)
+    oa_locations = payload.get("oa_locations")
+    if isinstance(oa_locations, list):
+        locations.extend(location for location in oa_locations if isinstance(location, dict))
+    return locations
+
+
+def _location_value(location: dict, key: str) -> str | None:
+    value = location.get(key)
+    if not isinstance(value, str):
         return None
-    for location in locations:
-        if not isinstance(location, dict):
+    normalized = value.strip()
+    return normalized or None
+
+
+def _payload_pdf_candidates(payload: dict) -> list[str]:
+    candidates: list[str] = []
+    seen: set[str] = set()
+    for location in _payload_locations(payload):
+        candidate = _location_value(location, "url_for_pdf")
+        if candidate is None or candidate in seen:
             continue
-        pdf_url = location.get("url_for_pdf")
-        if isinstance(pdf_url, str) and pdf_url.strip():
-            return pdf_url.strip()
+        seen.add(candidate)
+        candidates.append(candidate)
+    return candidates
+
+
+def _payload_landing_candidates(payload: dict) -> list[str]:
+    candidates: list[str] = []
+    seen: set[str] = set()
+    for location in _payload_locations(payload):
+        candidate = _location_value(location, "url")
+        if candidate is None or candidate in seen:
+            continue
+        seen.add(candidate)
+        candidates.append(candidate)
+    return candidates
+
+
+def _crawl_targets(
+    *,
+    payload: dict,
+    pdf_candidates: list[str],
+) -> list[str]:
+    targets = _payload_landing_candidates(payload)
+    seen = set(targets)
+    for candidate in pdf_candidates:
+        if candidate in seen:
+            continue
+        targets.append(candidate)
+        seen.add(candidate)
+    doi = normalize_doi(str(payload.get("doi") or ""))
+    doi_landing_url = f"https://doi.org/{doi}" if doi else None
+    if doi_landing_url and doi_landing_url not in seen:
+        targets.append(doi_landing_url)
+    return targets
+
+
+def _has_direct_payload_pdf(payload: dict) -> bool:
+    return any(looks_like_pdf_url(candidate) for candidate in _payload_pdf_candidates(payload))
+
+
+async def _resolved_pdf_url_from_payload(
+    payload: dict,
+    *,
+    client,
+) -> str | None:
+    pdf_candidates = _payload_pdf_candidates(payload)
+    for candidate in pdf_candidates:
+        if looks_like_pdf_url(candidate):
+            return candidate
+    for page_url in _crawl_targets(payload=payload, pdf_candidates=pdf_candidates)[:3]:
+        discovered = await resolve_pdf_from_landing_page(client, page_url=page_url)
+        if discovered:
+            logger.info(
+                "unpaywall.pdf_discovered_from_landing",
+                extra={"event": "unpaywall.pdf_discovered_from_landing", "landing_url": page_url},
+            )
+            return discovered
     return None
 
 
@@ -81,6 +172,7 @@ async def _fetch_unpaywall_payload_by_doi(
     doi: str,
     email: str,
 ) -> dict | None:
+    await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
     response = await client.get(
         UNPAYWALL_URL_TEMPLATE.format(doi=doi),
         params={"email": email},
@@ -130,7 +222,7 @@ async def _resolve_item_payload(
     payload: dict | None = None
     if doi:
         payload = await _fetch_unpaywall_payload_by_doi(client=client, doi=doi, email=email)
-        if payload is not None and _payload_pdf_url(payload):
+        if payload is not None and _has_direct_payload_pdf(payload):
             return payload, False
     if not allow_crossref or not settings.crossref_enabled:
         return payload, False
@@ -151,9 +243,13 @@ async def _resolve_item_payload(
     return payload, True
 
 
-def _doi_and_pdf_from_payload(payload: dict) -> tuple[str | None, str | None]:
+async def _doi_and_pdf_from_payload(
+    payload: dict,
+    *,
+    client,
+) -> tuple[str | None, str | None]:
     doi = normalize_doi(str(payload.get("doi") or ""))
-    return doi, _payload_pdf_url(payload)
+    return doi, await _resolved_pdf_url_from_payload(payload, client=client)
 
 
 def _resolution_targets(items: list[PublicationListItem]) -> list[PublicationListItem]:
@@ -164,11 +260,165 @@ def _crossref_budget_value() -> int:
     return max(int(settings.crossref_max_lookups_per_request), 0)
 
 
+def _outcome_with_failure(
+    *,
+    item: PublicationListItem,
+    failure_reason: str,
+    used_crossref: bool,
+) -> OaResolutionOutcome:
+    return OaResolutionOutcome(
+        publication_id=item.publication_id,
+        doi=_publication_doi(item),
+        pdf_url=None,
+        failure_reason=failure_reason,
+        source=None,
+        used_crossref=used_crossref,
+    )
+
+
+def _missing_payload_failure_reason(item: PublicationListItem, *, used_crossref: bool) -> str:
+    if _publication_doi(item):
+        return FAILURE_NO_RECORD
+    if used_crossref:
+        return FAILURE_NO_RECORD
+    return FAILURE_MISSING_DOI
+
+
+def _source_name(*, used_crossref: bool) -> str:
+    return "crossref+unpaywall" if used_crossref else "unpaywall"
+
+
+def _outcome_from_payload(
+    *,
+    item: PublicationListItem,
+    doi: str | None,
+    pdf_url: str | None,
+    used_crossref: bool,
+) -> OaResolutionOutcome:
+    return OaResolutionOutcome(
+        publication_id=item.publication_id,
+        doi=doi,
+        pdf_url=pdf_url,
+        failure_reason=None if pdf_url else FAILURE_NO_PDF,
+        source=_source_name(used_crossref=used_crossref),
+        used_crossref=used_crossref,
+    )
+
+
+def _resolved_pdf_count(outcomes: dict[int, OaResolutionOutcome]) -> int:
+    return sum(1 for outcome in outcomes.values() if outcome.pdf_url)
+
+
+def _outcome_metadata(outcomes: dict[int, OaResolutionOutcome]) -> dict[int, tuple[str | None, str | None]]:
+    return {
+        publication_id: (outcome.doi, outcome.pdf_url)
+        for publication_id, outcome in outcomes.items()
+    }
+
+
+async def _resolve_outcome_for_item(
+    *,
+    client,
+    item: PublicationListItem,
+    email: str,
+    allow_crossref: bool,
+) -> OaResolutionOutcome:
+    payload, used_crossref = await _resolve_item_payload(
+        client=client,
+        item=item,
+        email=email,
+        allow_crossref=allow_crossref,
+    )
+    if not isinstance(payload, dict):
+        return _outcome_with_failure(
+            item=item,
+            failure_reason=_missing_payload_failure_reason(item, used_crossref=used_crossref),
+            used_crossref=used_crossref,
+        )
+    doi, pdf_url = await _doi_and_pdf_from_payload(payload, client=client)
+    return _outcome_from_payload(
+        item=item,
+        doi=doi,
+        pdf_url=pdf_url,
+        used_crossref=used_crossref,
+    )
+
+
+def _doi_input_count(items: list[PublicationListItem]) -> int:
+    return len([item for item in items if _publication_doi(item)])
+
+
+def _search_attempt_count(*, targets: list[PublicationListItem]) -> int:
+    return len([item for item in targets if not _publication_doi(item)])
+
+
+async def _safe_outcome_for_item(
+    *,
+    client,
+    item: PublicationListItem,
+    email: str,
+    allow_crossref: bool,
+) -> OaResolutionOutcome:
+    try:
+        return await _resolve_outcome_for_item(
+            client=client,
+            item=item,
+            email=email,
+            allow_crossref=allow_crossref,
+        )
+    except Exception as exc:  # pragma: no cover - defensive network boundary
+        logger.warning(
+            "unpaywall.resolve_item_failed",
+            extra={
+                "event": "unpaywall.resolve_item_failed",
+                "publication_id": item.publication_id,
+                "error": str(exc),
+            },
+        )
+        return _outcome_with_failure(
+            item=item,
+            failure_reason=FAILURE_RESOLUTION_EXCEPTION,
+            used_crossref=allow_crossref and settings.crossref_enabled,
+        )
+
+
+async def _resolve_outcomes_with_client(
+    *,
+    client,
+    targets: list[PublicationListItem],
+    email: str,
+) -> dict[int, OaResolutionOutcome]:
+    outcomes: dict[int, OaResolutionOutcome] = {}
+    crossref_budget = _crossref_budget_value()
+    crossref_lookups = 0
+    for item in targets:
+        allow_crossref = crossref_budget > 0 and crossref_lookups < crossref_budget
+        outcome = await _safe_outcome_for_item(
+            client=client,
+            item=item,
+            email=email,
+            allow_crossref=allow_crossref,
+        )
+        if outcome.used_crossref:
+            crossref_lookups += 1
+        outcomes[item.publication_id] = outcome
+    return outcomes
+
+
 async def resolve_publication_oa_metadata(
     items: list[PublicationListItem],
     *,
     request_email: str | None = None,
 ) -> dict[int, tuple[str | None, str | None]]:
+    outcomes = await resolve_publication_oa_outcomes(items, request_email=request_email)
+    return _outcome_metadata(outcomes)
+
+
+async def resolve_publication_oa_outcomes(
+    items: list[PublicationListItem],
+    *,
+    request_email: str | None = None,
+) -> dict[int, OaResolutionOutcome]:
     if not settings.unpaywall_enabled:
         return {}
     email = _email_for_request(request_email)
@@ -178,35 +428,21 @@ async def resolve_publication_oa_metadata(
     import httpx
 
     timeout_seconds = max(float(settings.unpaywall_timeout_seconds), 0.5)
-    resolved: dict[int, tuple[str | None, str | None]] = {}
-    crossref_budget = _crossref_budget_value()
-    crossref_lookups = 0
     targets = _resolution_targets(items)[: max(int(settings.unpaywall_max_items_per_request), 0)]
     async with httpx.AsyncClient(timeout=timeout_seconds) as client:
-        for item in targets:
-            allow_crossref = crossref_budget > 0 and crossref_lookups < crossref_budget
-            payload, used_crossref = await _resolve_item_payload(
-                client=client,
-                item=item,
-                email=email,
-                allow_crossref=allow_crossref,
-            )
-            if used_crossref:
-                crossref_lookups += 1
-            if not isinstance(payload, dict):
-                continue
-            resolved[item.publication_id] = _doi_and_pdf_from_payload(payload)
-    resolved_count = sum(1 for _doi, pdf in resolved.values() if pdf)
-    doi_input_count = len([item for item in items if _publication_doi(item)])
-    target_doi_count = len([item for item in targets if _publication_doi(item)])
+        outcomes = await _resolve_outcomes_with_client(
+            client=client,
+            targets=targets,
+            email=email,
+        )
     _log_resolution_summary(
         publication_count=len(items),
-        doi_input_count=doi_input_count,
-        search_attempt_count=max(0, len(targets) - target_doi_count),
-        resolved_pdf_count=resolved_count,
+        doi_input_count=_doi_input_count(items),
+        search_attempt_count=_search_attempt_count(targets=targets),
+        resolved_pdf_count=_resolved_pdf_count(outcomes),
         email=email,
     )
-    return resolved
+    return outcomes
 
 
 async def resolve_publication_pdf_urls(

app/services/domains/unpaywall/pdf_discovery.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+from html.parser import HTMLParser
+import re
+from urllib.parse import urljoin, urlparse
+
+from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
+from app.settings import settings
+
+PDF_MIME = "application/pdf"
+URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.I)
+
+
+class _LandingPdfParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.base_href: str | None = None
+        self.candidates: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        attrs_map = {key.lower(): (value or "").strip() for key, value in attrs}
+        if tag == "base":
+            self.base_href = attrs_map.get("href") or self.base_href
+            return
+        if tag == "meta":
+            self._append_meta_candidate(attrs_map)
+            return
+        if tag == "link":
+            self._append_link_candidate(attrs_map)
+            return
+        if tag == "a":
+            self._append_anchor_candidate(attrs_map)
+
+    def _append_meta_candidate(self, attrs_map: dict[str, str]) -> None:
+        meta_name = (attrs_map.get("name") or attrs_map.get("property") or "").lower()
+        if meta_name != "citation_pdf_url":
+            return
+        content = attrs_map.get("content")
+        if content:
+            self.candidates.append(content)
+
+    def _append_link_candidate(self, attrs_map: dict[str, str]) -> None:
+        href = attrs_map.get("href")
+        link_type = (attrs_map.get("type") or "").lower()
+        if href and "pdf" in link_type:
+            self.candidates.append(href)
+
+    def _append_anchor_candidate(self, attrs_map: dict[str, str]) -> None:
+        href = attrs_map.get("href")
+        if href:
+            self.candidates.append(href)
+
+
+def looks_like_pdf_url(url: str | None) -> bool:
+    if not isinstance(url, str):
+        return False
+    value = url.strip()
+    if not value:
+        return False
+    parsed = urlparse(value)
+    path = (parsed.path or "").lower()
+    query = (parsed.query or "").lower()
+    return path.endswith(".pdf") or ".pdf" in query
+
+
+def _normalized_candidate_urls(*, page_url: str, html: str) -> list[str]:
+    parser = _LandingPdfParser()
+    parser.feed(html)
+    parser.close()
+    base_url = urljoin(page_url, parser.base_href) if parser.base_href else page_url
+    raw_candidates = [*parser.candidates, *_text_url_candidates(html)]
+    deduped: list[str] = []
+    seen: set[str] = set()
+    for raw in raw_candidates:
+        absolute = urljoin(base_url, raw.strip())
+        parsed = urlparse(absolute)
+        if parsed.scheme not in {"http", "https"}:
+            continue
+        if absolute in seen:
+            continue
+        seen.add(absolute)
+        deduped.append(absolute)
+    return sorted(deduped, key=_candidate_sort_key)
+
+
+def _text_url_candidates(html: str) -> list[str]:
+    candidates: list[str] = []
+    for match in URL_RE.findall(html):
+        cleaned = match.rstrip(".,);]}>")
+        if "pdf" not in cleaned.lower():
+            continue
+        candidates.append(cleaned)
+    return candidates
+
+
+def _candidate_sort_key(candidate: str) -> tuple[int, str]:
+    lowered = candidate.lower()
+    if looks_like_pdf_url(candidate):
+        return (0, lowered)
+    if any(token in lowered for token in ("doi", "full", "article", "download")):
+        return (1, lowered)
+    return (2, lowered)
+
+
+def _is_html_response(response) -> bool:
+    content_type = str(response.headers.get("content-type") or "").lower()
+    return "text/html" in content_type or "application/xhtml+xml" in content_type
+
+
+async def _fetch_page_html(client, *, page_url: str) -> str | None:
+    await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
+    response = await client.get(page_url, follow_redirects=True)
+    if response.status_code != 200 or not _is_html_response(response):
+        return None
+    text = response.text or ""
+    return text[: max(int(settings.unpaywall_pdf_discovery_max_html_bytes), 0)]
+
+
+async def _candidate_is_pdf(client, *, candidate_url: str) -> bool:
+    if looks_like_pdf_url(candidate_url):
+        return True
+    await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
+    response = await client.get(candidate_url, follow_redirects=True)
+    content_type = str(response.headers.get("content-type") or "").lower()
+    return response.status_code == 200 and PDF_MIME in content_type
+
+
+def _candidate_limit() -> int:
+    return max(int(settings.unpaywall_pdf_discovery_max_candidates), 1)
+
+
+async def _resolve_pdf_from_candidate_page(client, *, candidate_url: str) -> str | None:
+    html = await _fetch_page_html(client, page_url=candidate_url)
+    if not html:
+        return None
+    nested_candidates = _normalized_candidate_urls(page_url=candidate_url, html=html)
+    for nested in nested_candidates[: _candidate_limit()]:
+        if await _candidate_is_pdf(client, candidate_url=nested):
+            return nested
+    return None
+
+
+async def resolve_pdf_from_landing_page(client, *, page_url: str) -> str | None:
+    if not settings.unpaywall_pdf_discovery_enabled:
+        return None
+    html = await _fetch_page_html(client, page_url=page_url)
+    if not html:
+        return None
+    candidates = _normalized_candidate_urls(page_url=page_url, html=html)
+    for candidate in candidates[: _candidate_limit()]:
+        if await _candidate_is_pdf(client, candidate_url=candidate):
+            return candidate
+        nested_pdf = await _resolve_pdf_from_candidate_page(client, candidate_url=candidate)
+        if nested_pdf:
+            return nested_pdf
+    return None

app/services/domains/unpaywall/rate_limit.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import asyncio
+import time
+
+_REQUEST_LOCK = asyncio.Lock()
+_LAST_REQUEST_AT = 0.0
+
+
+async def wait_for_unpaywall_slot(*, min_interval_seconds: float) -> None:
+    global _LAST_REQUEST_AT
+    interval = max(float(min_interval_seconds), 0.0)
+    async with _REQUEST_LOCK:
+        elapsed = time.monotonic() - _LAST_REQUEST_AT
+        remaining = interval - elapsed
+        if remaining > 0:
+            await asyncio.sleep(remaining)
+        _LAST_REQUEST_AT = time.monotonic()