from __future__ import annotations import logging import re from urllib.parse import unquote from app.services.domains.crossref.application import discover_doi_for_publication from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem from app.services.domains.doi.normalize import normalize_doi from app.settings import settings DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I) DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I) DOI_URL_RE = re.compile(r"(?:https?://)?(?:dx\.)?doi\.org/(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I) UNPAYWALL_URL_TEMPLATE = "https://api.unpaywall.org/v2/{doi}" logger = logging.getLogger(__name__) def _extract_doi_candidate(text: str | None) -> str | None: if not text: return None decoded = unquote(text) match = DOI_PATTERN.search(decoded) if not match: return None return match.group(0).rstrip(" .;,)") def _extract_explicit_doi(text: str | None) -> str | None: if not text: return None decoded = unquote(text) url_match = DOI_URL_RE.search(decoded) if url_match: return normalize_doi(url_match.group(1)) prefix_match = DOI_PREFIX_RE.search(decoded) if prefix_match: return normalize_doi(prefix_match.group(1)) return None def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str | None: stored = normalize_doi(item.doi) if stored: in_metadata = any( normalize_doi(_extract_explicit_doi(value)) == stored for value in (item.pub_url, item.venue_text) ) if in_metadata: return stored pub_url_doi = _extract_doi_candidate(item.pub_url) if pub_url_doi: return normalize_doi(pub_url_doi) return ( _extract_explicit_doi(item.pub_url) or _extract_explicit_doi(item.venue_text) ) def _payload_pdf_url(payload: dict) -> str | None: best = payload.get("best_oa_location") if isinstance(best, dict): pdf_url = best.get("url_for_pdf") if isinstance(pdf_url, str) and pdf_url.strip(): return pdf_url.strip() locations = payload.get("oa_locations") if not isinstance(locations, list): return None for location in locations: if not isinstance(location, dict): continue pdf_url = location.get("url_for_pdf") if isinstance(pdf_url, str) and pdf_url.strip(): return pdf_url.strip() return None async def _fetch_unpaywall_payload_by_doi( *, client, doi: str, email: str, ) -> dict | None: response = await client.get( UNPAYWALL_URL_TEMPLATE.format(doi=doi), params={"email": email}, ) if response.status_code != 200: return None payload = response.json() if not isinstance(payload, dict): return None return payload def _email_for_request(request_email: str | None) -> str | None: email = (request_email or "").strip() or settings.unpaywall_email.strip() return email or None def _log_resolution_summary( *, publication_count: int, doi_input_count: int, search_attempt_count: int, resolved_pdf_count: int, email: str, ) -> None: logger.info( "unpaywall.resolve_completed", extra={ "event": "unpaywall.resolve_completed", "publication_count": publication_count, "doi_input_count": doi_input_count, "search_attempt_count": search_attempt_count, "resolved_pdf_count": resolved_pdf_count, "email_domain": email.split("@", 1)[-1] if "@" in email else None, }, ) async def _resolve_item_payload( *, client, item: PublicationListItem, email: str, allow_crossref: bool, ) -> tuple[dict | None, bool]: doi = _publication_doi(item) payload: dict | None = None if doi: payload = await _fetch_unpaywall_payload_by_doi(client=client, doi=doi, email=email) if payload is not None and _payload_pdf_url(payload): return payload, False if not allow_crossref or not settings.crossref_enabled: return payload, False crossref_doi = await discover_doi_for_publication( item=item, max_rows=settings.crossref_max_rows, email=email, ) if crossref_doi is None or crossref_doi == doi: return payload, crossref_doi is not None crossref_payload = await _fetch_unpaywall_payload_by_doi( client=client, doi=crossref_doi, email=email, ) if crossref_payload is not None: return crossref_payload, True return payload, True def _doi_and_pdf_from_payload(payload: dict) -> tuple[str | None, str | None]: doi = normalize_doi(str(payload.get("doi") or "")) return doi, _payload_pdf_url(payload) def _resolution_targets(items: list[PublicationListItem]) -> list[PublicationListItem]: return [item for item in items if not item.pdf_url] def _crossref_budget_value() -> int: return max(int(settings.crossref_max_lookups_per_request), 0) async def resolve_publication_oa_metadata( items: list[PublicationListItem], *, request_email: str | None = None, ) -> dict[int, tuple[str | None, str | None]]: if not settings.unpaywall_enabled: return {} email = _email_for_request(request_email) if email is None: logger.debug("unpaywall.resolve_skipped_missing_email") return {} import httpx timeout_seconds = max(float(settings.unpaywall_timeout_seconds), 0.5) resolved: dict[int, tuple[str | None, str | None]] = {} crossref_budget = _crossref_budget_value() crossref_lookups = 0 targets = _resolution_targets(items)[: max(int(settings.unpaywall_max_items_per_request), 0)] async with httpx.AsyncClient(timeout=timeout_seconds) as client: for item in targets: allow_crossref = crossref_budget > 0 and crossref_lookups < crossref_budget payload, used_crossref = await _resolve_item_payload( client=client, item=item, email=email, allow_crossref=allow_crossref, ) if used_crossref: crossref_lookups += 1 if not isinstance(payload, dict): continue resolved[item.publication_id] = _doi_and_pdf_from_payload(payload) resolved_count = sum(1 for _doi, pdf in resolved.values() if pdf) doi_input_count = len([item for item in items if _publication_doi(item)]) target_doi_count = len([item for item in targets if _publication_doi(item)]) _log_resolution_summary( publication_count=len(items), doi_input_count=doi_input_count, search_attempt_count=max(0, len(targets) - target_doi_count), resolved_pdf_count=resolved_count, email=email, ) return resolved async def resolve_publication_pdf_urls( items: list[PublicationListItem], *, request_email: str | None = None, ) -> dict[int, str | None]: resolved = await resolve_publication_oa_metadata(items, request_email=request_email) return {publication_id: pdf for publication_id, (_doi, pdf) in resolved.items()}