temp commit
This commit is contained in:
parent
8760f27b51
commit
0e9e49df16
193 changed files with 23228 additions and 935 deletions
|
|
@ -2,100 +2,114 @@ from __future__ import annotations
|
|||
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from typing import Any
|
||||
|
||||
from app.services.domains.arxiv.application import ArxivRateLimitError
|
||||
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
|
||||
from app.services.domains.publications.types import PublicationListItem
|
||||
from app.services.domains.scholar.publication_pdf import (
|
||||
ScholarPublicationLinkCandidate,
|
||||
ScholarPublicationLinkCandidates,
|
||||
fetch_link_candidates_from_scholar_publication_page,
|
||||
)
|
||||
from app.services.domains.unpaywall import pdf_discovery as pdf_discovery_service
|
||||
from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE = "scholar_publication_page"
|
||||
PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED = "scholar_publication_page_unlabeled_fallback"
|
||||
PDF_PATH_TOKEN = "/pdf/"
|
||||
HTTP_TIMEOUT_FLOOR_SECONDS = 0.5
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PipelineOutcome:
|
||||
outcome: OaResolutionOutcome | None
|
||||
scholar_candidates: ScholarPublicationLinkCandidates | None
|
||||
scholar_candidates: Any | None # Kept for backward compatibility with calling signatures
|
||||
|
||||
|
||||
async def resolve_publication_pdf_outcome_for_row(
|
||||
*,
|
||||
row: PublicationListItem,
|
||||
request_email: str | None,
|
||||
openalex_api_key: str | None = None,
|
||||
) -> PipelineOutcome:
|
||||
candidates = await _safe_scholar_candidates(row.pub_url)
|
||||
labeled = _labeled_candidate(candidates)
|
||||
if labeled is not None:
|
||||
return PipelineOutcome(_scholar_outcome(row=row, candidate=labeled), candidates)
|
||||
# 1. OpenAlex OA — raises OpenAlexBudgetExhaustedError if budget is gone
|
||||
openalex_outcome = await _openalex_outcome(row, request_email=request_email, openalex_api_key=openalex_api_key)
|
||||
if openalex_outcome and openalex_outcome.pdf_url:
|
||||
return PipelineOutcome(openalex_outcome, None)
|
||||
|
||||
# 2. arXiv
|
||||
arxiv_outcome = await _arxiv_outcome(row, request_email=request_email)
|
||||
if arxiv_outcome and arxiv_outcome.pdf_url:
|
||||
return PipelineOutcome(arxiv_outcome, None)
|
||||
|
||||
# 3. Unpaywall (which falls back to Crossref)
|
||||
oa_outcome = await _oa_outcome(row=row, request_email=request_email)
|
||||
if _oa_has_pdf(oa_outcome):
|
||||
return PipelineOutcome(oa_outcome, candidates)
|
||||
unlabeled = _unlabeled_candidate(candidates)
|
||||
if unlabeled is None:
|
||||
return PipelineOutcome(oa_outcome, candidates)
|
||||
fallback_outcome = await _unlabeled_fallback_outcome(row=row, candidate=unlabeled)
|
||||
if fallback_outcome is not None:
|
||||
return PipelineOutcome(fallback_outcome, candidates)
|
||||
return PipelineOutcome(oa_outcome, candidates)
|
||||
return PipelineOutcome(oa_outcome, None)
|
||||
|
||||
|
||||
async def _safe_scholar_candidates(pub_url: str | None) -> ScholarPublicationLinkCandidates | None:
|
||||
try:
|
||||
return await fetch_link_candidates_from_scholar_publication_page(pub_url)
|
||||
except Exception as exc: # pragma: no cover - defensive boundary
|
||||
logger.warning(
|
||||
"publications.pdf_resolution.scholar_candidates_failed",
|
||||
extra={"event": "publications.pdf_resolution.scholar_candidates_failed", "error": str(exc)},
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _labeled_candidate(
|
||||
candidates: ScholarPublicationLinkCandidates | None,
|
||||
) -> ScholarPublicationLinkCandidate | None:
|
||||
if candidates is None:
|
||||
return None
|
||||
return candidates.labeled_candidate
|
||||
|
||||
|
||||
def _unlabeled_candidate(
|
||||
candidates: ScholarPublicationLinkCandidates | None,
|
||||
) -> ScholarPublicationLinkCandidate | None:
|
||||
if candidates is None:
|
||||
return None
|
||||
return candidates.fallback_candidate
|
||||
|
||||
|
||||
def _scholar_outcome(
|
||||
*,
|
||||
async def _openalex_outcome(
|
||||
row: PublicationListItem,
|
||||
candidate: ScholarPublicationLinkCandidate,
|
||||
) -> OaResolutionOutcome:
|
||||
source = (
|
||||
PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE
|
||||
if candidate.label_present
|
||||
else PDF_SOURCE_SCHOLAR_PUBLICATION_PAGE_UNLABELED
|
||||
)
|
||||
return OaResolutionOutcome(
|
||||
publication_id=row.publication_id,
|
||||
doi=row.doi,
|
||||
pdf_url=candidate.url,
|
||||
failure_reason=None,
|
||||
source=source,
|
||||
used_crossref=False,
|
||||
)
|
||||
request_email: str | None,
|
||||
openalex_api_key: str | None = None,
|
||||
) -> OaResolutionOutcome | None:
|
||||
from app.services.domains.openalex.client import OpenAlexClient
|
||||
from app.services.domains.openalex.matching import find_best_match
|
||||
|
||||
if not row.title:
|
||||
return None
|
||||
|
||||
import re
|
||||
safe_title = re.sub(r"[^\w\s]", " ", row.title)
|
||||
safe_title = " ".join(safe_title.split())
|
||||
if not safe_title:
|
||||
return None
|
||||
|
||||
api_key = openalex_api_key or settings.openalex_api_key
|
||||
client = OpenAlexClient(api_key=api_key, mailto=request_email or settings.crossref_api_mailto)
|
||||
try:
|
||||
openalex_works = await client.get_works_by_filter({"title.search": safe_title}, limit=5)
|
||||
match = find_best_match(
|
||||
target_title=row.title,
|
||||
target_year=row.year,
|
||||
target_authors=row.scholar_label,
|
||||
candidates=openalex_works,
|
||||
)
|
||||
if match and match.oa_url:
|
||||
return OaResolutionOutcome(
|
||||
publication_id=row.publication_id,
|
||||
doi=match.doi,
|
||||
pdf_url=match.oa_url,
|
||||
failure_reason=None,
|
||||
source="openalex",
|
||||
used_crossref=False,
|
||||
)
|
||||
except OpenAlexBudgetExhaustedError:
|
||||
# Re-raise so the caller's batch loop can stop hitting the API.
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"publications.pdf_resolution.openalex_failed",
|
||||
extra={"event": "publications.pdf_resolution.openalex_failed", "error": str(exc)},
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def _arxiv_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None:
|
||||
from app.services.domains.arxiv.application import discover_arxiv_id_for_publication
|
||||
|
||||
try:
|
||||
arxiv_id = await discover_arxiv_id_for_publication(item=row, request_email=request_email)
|
||||
if arxiv_id:
|
||||
pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
|
||||
return OaResolutionOutcome(
|
||||
publication_id=row.publication_id,
|
||||
doi=None,
|
||||
pdf_url=pdf_url,
|
||||
failure_reason=None,
|
||||
source="arxiv",
|
||||
used_crossref=False,
|
||||
)
|
||||
except ArxivRateLimitError:
|
||||
raise # propagate so the batch loop can stop
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"publications.pdf_resolution.arxiv_failed",
|
||||
extra={"event": "publications.pdf_resolution.arxiv_failed", "error": str(exc)},
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def _oa_outcome(
|
||||
|
|
@ -105,46 +119,3 @@ async def _oa_outcome(
|
|||
) -> OaResolutionOutcome | None:
|
||||
outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email)
|
||||
return outcomes.get(row.publication_id)
|
||||
|
||||
|
||||
def _oa_has_pdf(outcome: OaResolutionOutcome | None) -> bool:
|
||||
return bool(outcome and outcome.pdf_url)
|
||||
|
||||
|
||||
async def _unlabeled_fallback_outcome(
|
||||
*,
|
||||
row: PublicationListItem,
|
||||
candidate: ScholarPublicationLinkCandidate,
|
||||
) -> OaResolutionOutcome | None:
|
||||
pdf_url = await _validated_pdf_url(candidate.url)
|
||||
if pdf_url is None:
|
||||
return None
|
||||
return _scholar_outcome(row=row, candidate=ScholarPublicationLinkCandidate(
|
||||
url=pdf_url,
|
||||
confidence_score=candidate.confidence_score,
|
||||
label_present=False,
|
||||
reason=candidate.reason,
|
||||
))
|
||||
|
||||
|
||||
async def _validated_pdf_url(candidate_url: str) -> str | None:
|
||||
if _looks_direct_pdf(candidate_url):
|
||||
return candidate_url
|
||||
timeout_seconds = _discovery_timeout_seconds()
|
||||
async with httpx.AsyncClient(timeout=timeout_seconds) as client:
|
||||
if await pdf_discovery_service._candidate_is_pdf(client, candidate_url=candidate_url):
|
||||
return candidate_url
|
||||
return await pdf_discovery_service.resolve_pdf_from_landing_page(client, page_url=candidate_url)
|
||||
|
||||
|
||||
def _looks_direct_pdf(url: str | None) -> bool:
|
||||
if pdf_discovery_service.looks_like_pdf_url(url):
|
||||
return True
|
||||
if not isinstance(url, str):
|
||||
return False
|
||||
path = (urlparse(url).path or "").lower()
|
||||
return PDF_PATH_TOKEN in path
|
||||
|
||||
|
||||
def _discovery_timeout_seconds() -> float:
|
||||
return max(float(settings.unpaywall_timeout_seconds), HTTP_TIMEOUT_FLOOR_SECONDS)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue