justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

Big changes

Browse source
This commit is contained in:
Justin Visser 2026-02-21 17:33:05 +01:00
parent 4240ad38e2
commit e3f0d63fec
99 changed files with 8804 additions and 1731 deletions
Download patch file Download diff file Expand all files Collapse all files

5
app/services/domains/dbops/__init__.py Normal file
View file

@ -0,0 +1,5 @@
from app.services.domains.dbops.application import run_publication_link_repair
from app.services.domains.dbops.integrity import collect_integrity_report
from app.services.domains.dbops.query import list_repair_jobs
__all__ = ["collect_integrity_report", "list_repair_jobs", "run_publication_link_repair"]

364
app/services/domains/dbops/application.py Normal file
View file

@ -0,0 +1,364 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from sqlalchemy import delete, exists, func, select, update
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import DataRepairJob, IngestionQueueItem, Publication, ScholarProfile, ScholarPublication
REPAIR_STATUS_PLANNED = "planned"
REPAIR_STATUS_RUNNING = "running"
REPAIR_STATUS_COMPLETED = "completed"
REPAIR_STATUS_FAILED = "failed"
SCOPE_MODE_SINGLE_USER = "single_user"
SCOPE_MODE_ALL_USERS = "all_users"
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
def _normalize_scope_mode(scope_mode: str) -> str:
normalized = scope_mode.strip().lower()
if normalized in {SCOPE_MODE_SINGLE_USER, SCOPE_MODE_ALL_USERS}:
return normalized
raise ValueError("Unknown scope mode.")
def _scope_user_id(*, scope_mode: str, user_id: int | None) -> int | None:
if scope_mode == SCOPE_MODE_SINGLE_USER:
if user_id is None:
raise ValueError("user_id is required when scope_mode=single_user.")
return int(user_id)
if user_id is not None:
raise ValueError("user_id must be omitted when scope_mode=all_users.")
return None
def _scope_payload(
*,
scope_mode: str,
user_id: int | None,
target_scholar_profile_ids: list[int],
orphan_gc: bool,
) -> dict[str, Any]:
payload: dict[str, Any] = {
"scope_mode": scope_mode,
"scholar_profile_ids": [int(value) for value in target_scholar_profile_ids],
"gc_orphan_publications": bool(orphan_gc),
}
if user_id is not None:
payload["user_id"] = int(user_id)
return payload
async def _target_scholar_profile_ids(
db_session: AsyncSession,
*,
scope_mode: str,
user_id: int | None,
scholar_profile_ids: list[int] | None,
) -> list[int]:
stmt = select(ScholarProfile.id)
if scope_mode == SCOPE_MODE_SINGLE_USER:
stmt = stmt.where(ScholarProfile.user_id == user_id)
if scholar_profile_ids:
normalized_ids = [int(value) for value in scholar_profile_ids]
stmt = stmt.where(ScholarProfile.id.in_(normalized_ids))
result = await db_session.execute(stmt.order_by(ScholarProfile.id.asc()))
ids = [int(row[0]) for row in result.all()]
if not ids:
raise ValueError("No target scholar profiles found for the requested scope.")
return ids
async def _count_scope(
db_session: AsyncSession,
*,
user_id: int | None,
target_scholar_profile_ids: list[int],
) -> dict[str, int]:
links_result = await db_session.execute(
select(func.count())
.select_from(ScholarPublication)
.where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids))
)
queue_stmt = (
select(func.count())
.select_from(IngestionQueueItem)
.where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids))
)
if user_id is not None:
queue_stmt = queue_stmt.where(IngestionQueueItem.user_id == user_id)
queue_result = await db_session.execute(queue_stmt)
return {
"target_scholar_count": len(target_scholar_profile_ids),
"links_in_scope": int(links_result.scalar_one() or 0),
"queue_items_in_scope": int(queue_result.scalar_one() or 0),
}
async def _count_orphan_publications(db_session: AsyncSession) -> int:
stmt = (
select(func.count())
.select_from(Publication)
.where(
~exists(
select(1).where(ScholarPublication.publication_id == Publication.id)
)
)
)
result = await db_session.execute(stmt)
return int(result.scalar_one() or 0)
async def _create_job(
db_session: AsyncSession,
*,
requested_by: str | None,
scope: dict[str, Any],
dry_run: bool,
) -> DataRepairJob:
job = DataRepairJob(
job_name="repair_publication_links",
requested_by=(requested_by or "").strip() or None,
scope=scope,
dry_run=dry_run,
status=REPAIR_STATUS_PLANNED,
summary={},
)
db_session.add(job)
await db_session.flush()
return job
def _job_summary(
*,
counts: dict[str, int],
dry_run: bool,
links_deleted: int,
queue_items_deleted: int,
scholars_reset: int,
orphan_publications_before: int,
orphan_publications_deleted: int,
) -> dict[str, Any]:
return {
**counts,
"dry_run": bool(dry_run),
"links_deleted": int(links_deleted),
"queue_items_deleted": int(queue_items_deleted),
"scholars_reset": int(scholars_reset),
"orphan_publications_before": int(orphan_publications_before),
"orphan_publications_deleted": int(orphan_publications_deleted),
}
async def _delete_links_for_targets(db_session: AsyncSession, *, target_scholar_profile_ids: list[int]) -> int:
result = await db_session.execute(
delete(ScholarPublication).where(
ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids)
)
)
return int(result.rowcount or 0)
async def _delete_queue_for_targets(
db_session: AsyncSession,
*,
user_id: int | None,
target_scholar_profile_ids: list[int],
) -> int:
stmt = delete(IngestionQueueItem).where(
IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids)
)
if user_id is not None:
stmt = stmt.where(IngestionQueueItem.user_id == user_id)
result = await db_session.execute(stmt)
return int(result.rowcount or 0)
async def _reset_scholar_tracking_state(
db_session: AsyncSession,
*,
user_id: int | None,
target_scholar_profile_ids: list[int],
) -> int:
stmt = update(ScholarProfile).where(ScholarProfile.id.in_(target_scholar_profile_ids))
if user_id is not None:
stmt = stmt.where(ScholarProfile.user_id == user_id)
result = await db_session.execute(
stmt.values(
baseline_completed=False,
last_initial_page_fingerprint_sha256=None,
last_initial_page_checked_at=None,
last_run_dt=None,
last_run_status=None,
)
)
return int(result.rowcount or 0)
async def _delete_orphan_publications(db_session: AsyncSession) -> int:
result = await db_session.execute(
delete(Publication).where(
~exists(select(1).where(ScholarPublication.publication_id == Publication.id))
)
)
return int(result.rowcount or 0)
async def _mutation_counts(
db_session: AsyncSession,
*,
user_id: int | None,
target_ids: list[int],
dry_run: bool,
gc_orphan_publications: bool,
) -> tuple[int, int, int, int]:
if dry_run:
return 0, 0, 0, 0
links_deleted = await _delete_links_for_targets(
db_session,
target_scholar_profile_ids=target_ids,
)
queue_deleted = await _delete_queue_for_targets(
db_session,
user_id=user_id,
target_scholar_profile_ids=target_ids,
)
scholars_reset = await _reset_scholar_tracking_state(
db_session,
user_id=user_id,
target_scholar_profile_ids=target_ids,
)
orphan_deleted = 0
if gc_orphan_publications:
orphan_deleted = await _delete_orphan_publications(db_session)
return links_deleted, queue_deleted, scholars_reset, orphan_deleted
def _result_payload(*, job: DataRepairJob, scope: dict[str, Any], summary: dict[str, Any]) -> dict[str, Any]:
return {
"job_id": int(job.id),
"status": job.status,
"scope": scope,
"summary": summary,
}
async def _complete_job(
db_session: AsyncSession,
*,
job: DataRepairJob,
summary: dict[str, Any],
scope: dict[str, Any],
) -> dict[str, Any]:
job.summary = summary
job.status = REPAIR_STATUS_COMPLETED
job.finished_at = _utcnow()
await db_session.commit()
return _result_payload(job=job, scope=scope, summary=summary)
async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None:
await db_session.rollback()
job.status = REPAIR_STATUS_FAILED
job.error_text = str(error)
job.finished_at = _utcnow()
db_session.add(job)
await db_session.commit()
async def _prepare_repair_job(
db_session: AsyncSession,
*,
scope_mode: str,
user_id: int | None,
scholar_profile_ids: list[int] | None,
dry_run: bool,
gc_orphan_publications: bool,
requested_by: str | None,
) -> tuple[int | None, list[int], dict[str, Any], DataRepairJob]:
normalized_scope = _normalize_scope_mode(scope_mode)
scope_user_id = _scope_user_id(scope_mode=normalized_scope, user_id=user_id)
target_ids = await _target_scholar_profile_ids(
db_session,
scope_mode=normalized_scope,
user_id=scope_user_id,
scholar_profile_ids=scholar_profile_ids,
)
scope = _scope_payload(
scope_mode=normalized_scope,
user_id=scope_user_id,
target_scholar_profile_ids=target_ids,
orphan_gc=gc_orphan_publications,
)
job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run)
job.status = REPAIR_STATUS_RUNNING
job.started_at = _utcnow()
return scope_user_id, target_ids, scope, job
async def _build_repair_summary(
db_session: AsyncSession,
*,
scope_user_id: int | None,
target_ids: list[int],
dry_run: bool,
gc_orphan_publications: bool,
) -> dict[str, Any]:
counts = await _count_scope(db_session, user_id=scope_user_id, target_scholar_profile_ids=target_ids)
orphan_before = await _count_orphan_publications(db_session)
links_deleted, queue_deleted, scholars_reset, orphan_deleted = await _mutation_counts(
db_session,
user_id=scope_user_id,
target_ids=target_ids,
dry_run=dry_run,
gc_orphan_publications=gc_orphan_publications,
)
return _job_summary(
counts=counts,
dry_run=dry_run,
links_deleted=links_deleted,
queue_items_deleted=queue_deleted,
scholars_reset=scholars_reset,
orphan_publications_before=orphan_before,
orphan_publications_deleted=orphan_deleted,
)
async def run_publication_link_repair(
db_session: AsyncSession,
*,
scope_mode: str = SCOPE_MODE_SINGLE_USER,
user_id: int | None = None,
scholar_profile_ids: list[int] | None = None,
dry_run: bool = True,
gc_orphan_publications: bool = False,
requested_by: str | None = None,
) -> dict[str, Any]:
scope_user_id, target_ids, scope, job = await _prepare_repair_job(
db_session,
scope_mode=scope_mode,
user_id=user_id,
scholar_profile_ids=scholar_profile_ids,
dry_run=dry_run,
gc_orphan_publications=gc_orphan_publications,
requested_by=requested_by,
)
try:
summary = await _build_repair_summary(
db_session,
scope_user_id=scope_user_id,
target_ids=target_ids,
dry_run=dry_run,
gc_orphan_publications=gc_orphan_publications,
)
return await _complete_job(db_session, job=job, summary=summary, scope=scope)
except Exception as exc:
await _fail_job(db_session, job=job, error=exc)
raise

175
app/services/domains/dbops/integrity.py Normal file
View file

@ -0,0 +1,175 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from sqlalchemy import func, select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Publication
INTEGRITY_CHECK_DEFS = (
(
"legacy_cluster_id_format",
"warning",
"Publications with non-namespaced cluster IDs.",
),
(
"negative_citation_count",
"failure",
"Publications with negative citation counts.",
),
(
"orphan_publications",
"warning",
"Publications with no scholar links.",
),
(
"orphan_scholar_publication_links",
"failure",
"Link rows missing parent scholar/publication.",
),
(
"duplicate_fingerprint_keys",
"failure",
"Duplicate publication fingerprint keys.",
),
(
"duplicate_cluster_ids",
"failure",
"Duplicate non-null publication cluster IDs.",
),
(
"missing_pdf_url",
"metric",
"Publications without a resolved PDF URL.",
),
)
async def _legacy_cluster_id_count(db_session: AsyncSession) -> int:
stmt = (
select(func.count())
.select_from(Publication)
.where(Publication.cluster_id.is_not(None))
.where(~Publication.cluster_id.like("cfv:%"))
.where(~Publication.cluster_id.like("cluster:%"))
)
result = await db_session.execute(stmt)
return int(result.scalar_one() or 0)
async def _negative_citation_count(db_session: AsyncSession) -> int:
result = await db_session.execute(
select(func.count()).select_from(Publication).where(Publication.citation_count < 0)
)
return int(result.scalar_one() or 0)
async def _missing_pdf_url_count(db_session: AsyncSession) -> int:
result = await db_session.execute(
select(func.count()).select_from(Publication).where(Publication.pdf_url.is_(None))
)
return int(result.scalar_one() or 0)
async def _count_from_sql(db_session: AsyncSession, *, sql: str) -> int:
result = await db_session.execute(text(sql))
return int(result.scalar_one() or 0)
def _issues_for_severity(*, checks: list[dict[str, Any]], severity: str) -> list[str]:
return [row["name"] for row in checks if row["severity"] == severity and row["count"] > 0]
def _check_row(*, name: str, count: int, severity: str, message: str) -> dict[str, Any]:
return {
"name": name,
"count": int(count),
"severity": severity,
"message": message,
}
async def _collect_counts(db_session: AsyncSession) -> dict[str, int]:
orphan_publications = await _count_from_sql(
db_session,
sql=(
"SELECT count(*) FROM publications p "
"LEFT JOIN scholar_publications sp ON sp.publication_id = p.id "
"WHERE sp.publication_id IS NULL"
),
)
orphan_links = await _count_from_sql(
db_session,
sql=(
"SELECT count(*) FROM scholar_publications sp "
"LEFT JOIN publications p ON p.id = sp.publication_id "
"LEFT JOIN scholar_profiles s ON s.id = sp.scholar_profile_id "
"WHERE p.id IS NULL OR s.id IS NULL"
),
)
duplicate_fingerprints = await _count_from_sql(
db_session,
sql=(
"SELECT count(*) FROM ("
"SELECT fingerprint_sha256 FROM publications "
"GROUP BY fingerprint_sha256 HAVING count(*) > 1"
") dup"
),
)
duplicate_cluster_ids = await _count_from_sql(
db_session,
sql=(
"SELECT count(*) FROM ("
"SELECT cluster_id FROM publications "
"WHERE cluster_id IS NOT NULL "
"GROUP BY cluster_id HAVING count(*) > 1"
") dup"
),
)
return {
"legacy_cluster_id_format": await _legacy_cluster_id_count(db_session),
"negative_citation_count": await _negative_citation_count(db_session),
"orphan_publications": orphan_publications,
"orphan_scholar_publication_links": orphan_links,
"duplicate_fingerprint_keys": duplicate_fingerprints,
"duplicate_cluster_ids": duplicate_cluster_ids,
"missing_pdf_url": await _missing_pdf_url_count(db_session),
}
def _build_checks(*, counts: dict[str, int]) -> list[dict[str, Any]]:
return [
_check_row(
name=name,
count=counts[name],
severity=severity,
message=message,
)
for name, severity, message in INTEGRITY_CHECK_DEFS
]
def _status_from_issues(*, failures: list[str], warnings: list[str]) -> str:
status = "ok"
if failures:
status = "failed"
elif warnings:
status = "warning"
return status
async def collect_integrity_report(db_session: AsyncSession) -> dict[str, Any]:
counts = await _collect_counts(db_session)
checks = _build_checks(counts=counts)
failures = _issues_for_severity(checks=checks, severity="failure")
warnings = _issues_for_severity(checks=checks, severity="warning")
return {
"status": _status_from_issues(failures=failures, warnings=warnings),
"checked_at": datetime.now(timezone.utc).isoformat(),
"failures": failures,
"warnings": warnings,
"checks": checks,
}

22
app/services/domains/dbops/query.py Normal file
View file

@ -0,0 +1,22 @@
from __future__ import annotations
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import DataRepairJob
def _bounded_limit(limit: int) -> int:
return max(1, min(int(limit), 200))
async def list_repair_jobs(
db_session: AsyncSession,
*,
limit: int = 50,
) -> list[DataRepairJob]:
bounded = _bounded_limit(limit)
result = await db_session.execute(
select(DataRepairJob).order_by(DataRepairJob.created_at.desc()).limit(bounded)
)
return list(result.scalars())

38
app/services/domains/ingestion/application.py
View file

@ -91,6 +91,35 @@ class ScholarIngestionService:
def __init__(self, *, source: ScholarSource) -> None:
self._source = source
@staticmethod
def _effective_request_delay_seconds(value: int) -> int:
policy_minimum = user_settings_service.resolve_request_delay_minimum(
settings.ingestion_min_request_delay_seconds
)
return max(policy_minimum, _int_or_default(value, policy_minimum))
@staticmethod
def _log_request_delay_coercion(
*,
user_id: int,
requested_request_delay_seconds: int,
effective_request_delay_seconds: int,
) -> None:
logger.warning(
"ingestion.request_delay_coerced_to_policy_floor",
extra={
"event": "ingestion.request_delay_coerced_to_policy_floor",
"user_id": user_id,
"requested_request_delay_seconds": requested_request_delay_seconds,
"effective_request_delay_seconds": effective_request_delay_seconds,
"policy_minimum_request_delay_seconds": user_settings_service.resolve_request_delay_minimum(
settings.ingestion_min_request_delay_seconds
),
"metric_name": "ingestion_request_delay_coerced_total",
"metric_value": 1,
},
)
async def _load_user_settings_for_run(
self,
db_session: AsyncSession,
@ -1278,7 +1307,14 @@ class ScholarIngestionService:
return failure_summary, alert_summary
async def run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, request_delay_seconds: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, max_pages_per_scholar: int = 30, page_size: int = 100, scholar_profile_ids: set[int] | None = None, start_cstart_by_scholar_id: dict[int, int] | None = None, auto_queue_continuations: bool = True, queue_delay_seconds: int = 60, idempotency_key: str | None = None, alert_blocked_failure_threshold: int = 1, alert_network_failure_threshold: int = 2, alert_retry_scheduled_threshold: int = 3) -> RunExecutionSummary:
paging_kwargs = self._paging_kwargs(request_delay_seconds=request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size)
effective_request_delay_seconds = self._effective_request_delay_seconds(request_delay_seconds)
if effective_request_delay_seconds != _int_or_default(request_delay_seconds, effective_request_delay_seconds):
self._log_request_delay_coercion(
user_id=user_id,
requested_request_delay_seconds=_int_or_default(request_delay_seconds, 0),
effective_request_delay_seconds=effective_request_delay_seconds,
)
paging_kwargs = self._paging_kwargs(request_delay_seconds=effective_request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size)
threshold_kwargs = self._threshold_kwargs(alert_blocked_failure_threshold=alert_blocked_failure_threshold, alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold)
user_settings, run, scholars, start_cstart_map = await self._initialize_run_for_user(
db_session,

22
app/services/domains/ingestion/scheduler.py
View file

@ -23,12 +23,28 @@ from app.services.domains.ingestion.application import (
RunBlockedBySafetyPolicyError,
ScholarIngestionService,
)
from app.services.domains.settings import application as user_settings_service
from app.services.domains.scholar.source import LiveScholarSource
from app.settings import settings
logger = logging.getLogger(__name__)
def _request_delay_floor_seconds() -> int:
return user_settings_service.resolve_request_delay_minimum(
settings.ingestion_min_request_delay_seconds
)
def _effective_request_delay_seconds(value: int | None) -> int:
floor = _request_delay_floor_seconds()
try:
parsed = int(value) if value is not None else floor
except (TypeError, ValueError):
parsed = floor
return max(floor, parsed)
@dataclass(frozen=True)
class _AutoRunCandidate:
user_id: int
@ -178,7 +194,7 @@ class SchedulerService:
return _AutoRunCandidate(
user_id=int(user_id),
run_interval_minutes=int(run_interval_minutes),
request_delay_seconds=int(request_delay_seconds),
request_delay_seconds=_effective_request_delay_seconds(request_delay_seconds),
cooldown_until=cooldown_until,
cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
)
@ -577,6 +593,4 @@ class SchedulerService:
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
)
delay = result.scalar_one_or_none()
if delay is None:
return 10
return max(1, int(delay))
return _effective_request_delay_seconds(delay)

20
app/services/domains/publications/application.py
View file

@ -2,6 +2,7 @@ from __future__ import annotations
from app.services.domains.publications.counts import (
count_for_user,
count_favorite_for_user,
count_latest_for_user,
count_unread_for_user,
)
@ -12,7 +13,9 @@ from app.services.domains.publications.listing import (
list_unread_for_user,
)
from app.services.domains.publications.enrichment import (
hydrate_pdf_enrichment_state,
schedule_missing_pdf_enrichment_for_user,
schedule_retry_pdf_enrichment_for_row,
)
from app.services.domains.publications.modes import (
MODE_ALL,
@ -30,6 +33,14 @@ from app.services.domains.publications.queries import (
from app.services.domains.publications.read_state import (
mark_all_unread_as_read_for_user,
mark_selected_as_read_for_user,
set_publication_favorite_for_user,
)
from app.services.domains.publications.pdf_queue import (
count_pdf_queue_items,
enqueue_all_missing_pdf_jobs,
enqueue_retry_pdf_job_for_publication_id,
list_pdf_queue_page,
list_pdf_queue_items,
)
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
@ -49,10 +60,19 @@ __all__ = [
"list_unread_for_user",
"list_new_for_latest_run_for_user",
"retry_pdf_for_user",
"hydrate_pdf_enrichment_state",
"schedule_retry_pdf_enrichment_for_row",
"list_pdf_queue_items",
"list_pdf_queue_page",
"count_pdf_queue_items",
"enqueue_all_missing_pdf_jobs",
"enqueue_retry_pdf_job_for_publication_id",
"schedule_missing_pdf_enrichment_for_user",
"count_for_user",
"count_favorite_for_user",
"count_unread_for_user",
"count_latest_for_user",
"mark_all_unread_as_read_for_user",
"mark_selected_as_read_for_user",
"set_publication_favorite_for_user",
]

22
app/services/domains/publications/counts.py
View file

@ -19,6 +19,7 @@ async def count_for_user(
user_id: int,
mode: str = MODE_ALL,
scholar_profile_id: int | None = None,
favorite_only: bool = False,
) -> int:
resolved_mode = resolve_publication_view_mode(mode)
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
@ -30,6 +31,8 @@ async def count_for_user(
)
if scholar_profile_id is not None:
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
if favorite_only:
stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
if resolved_mode == MODE_UNREAD:
stmt = stmt.where(ScholarPublication.is_read.is_(False))
if resolved_mode == MODE_LATEST:
@ -45,12 +48,14 @@ async def count_unread_for_user(
*,
user_id: int,
scholar_profile_id: int | None = None,
favorite_only: bool = False,
) -> int:
return await count_for_user(
db_session,
user_id=user_id,
mode=MODE_UNREAD,
scholar_profile_id=scholar_profile_id,
favorite_only=favorite_only,
)
@ -59,10 +64,27 @@ async def count_latest_for_user(
*,
user_id: int,
scholar_profile_id: int | None = None,
favorite_only: bool = False,
) -> int:
return await count_for_user(
db_session,
user_id=user_id,
mode=MODE_LATEST,
scholar_profile_id=scholar_profile_id,
favorite_only=favorite_only,
)
async def count_favorite_for_user(
db_session: AsyncSession,
*,
user_id: int,
scholar_profile_id: int | None = None,
) -> int:
return await count_for_user(
db_session,
user_id=user_id,
mode=MODE_ALL,
scholar_profile_id=scholar_profile_id,
favorite_only=True,
)

158
app/services/domains/publications/enrichment.py
View file

@ -1,135 +1,73 @@
from __future__ import annotations
import asyncio
import logging
import time
from app.db.session import get_session_factory
from app.services.domains.publications.listing import (
missing_pdf_items,
resolve_and_persist_oa_metadata,
from sqlalchemy.ext.asyncio import AsyncSession
from app.services.domains.publications.pdf_queue import (
enqueue_missing_pdf_jobs,
enqueue_retry_pdf_job,
overlay_pdf_job_state,
)
from app.services.domains.publications.types import PublicationListItem
from app.settings import settings
logger = logging.getLogger(__name__)
_enrichment_lock = asyncio.Lock()
_inflight_publications: set[tuple[int, int]] = set()
_recent_attempt_seconds: dict[tuple[int, int], float] = {}
_scheduled_tasks: set[asyncio.Task[None]] = set()
def _cooldown_seconds() -> float:
return max(float(settings.unpaywall_retry_cooldown_seconds), 1.0)
def _prune_recent_attempts(now_seconds: float, *, cooldown_seconds: float) -> None:
expiry = cooldown_seconds * 3
stale_keys = [
key for key, attempted_seconds in _recent_attempt_seconds.items()
if now_seconds - attempted_seconds >= expiry
]
for key in stale_keys:
_recent_attempt_seconds.pop(key, None)
async def _claim_items(
*,
user_id: int,
items: list[PublicationListItem],
max_items: int,
) -> list[PublicationListItem]:
candidates = missing_pdf_items(items, limit=max_items)
if not candidates:
return []
now_seconds = time.monotonic()
cooldown_seconds = _cooldown_seconds()
claimed: list[PublicationListItem] = []
async with _enrichment_lock:
_prune_recent_attempts(now_seconds, cooldown_seconds=cooldown_seconds)
for item in candidates:
key = (user_id, item.publication_id)
attempted_seconds = _recent_attempt_seconds.get(key)
if key in _inflight_publications:
continue
if attempted_seconds is not None and now_seconds - attempted_seconds < cooldown_seconds:
continue
_inflight_publications.add(key)
_recent_attempt_seconds[key] = now_seconds
claimed.append(item)
return claimed
async def _release_claims(*, user_id: int, publication_ids: list[int]) -> None:
async with _enrichment_lock:
for publication_id in publication_ids:
_inflight_publications.discard((user_id, publication_id))
def _on_task_done(task: asyncio.Task[None]) -> None:
_scheduled_tasks.discard(task)
try:
task.result()
except Exception:
logger.exception(
"publications.enrichment.task_failed",
extra={"event": "publications.enrichment.task_failed"},
)
async def _run_enrichment(
*,
user_id: int,
request_email: str | None,
items: list[PublicationListItem],
) -> None:
publication_ids = [item.publication_id for item in items]
try:
session_factory = get_session_factory()
async with session_factory() as db_session:
await resolve_and_persist_oa_metadata(
db_session,
rows=items,
unpaywall_email=request_email,
)
finally:
await _release_claims(user_id=user_id, publication_ids=publication_ids)
logger.info(
"publications.enrichment.completed",
extra={
"event": "publications.enrichment.completed",
"user_id": user_id,
"publication_count": len(items),
},
)
async def schedule_missing_pdf_enrichment_for_user(
db_session: AsyncSession,
*,
user_id: int,
request_email: str | None,
items: list[PublicationListItem],
max_items: int,
) -> int:
claimed_items = await _claim_items(user_id=user_id, items=items, max_items=max_items)
if not claimed_items:
return 0
task = asyncio.create_task(
_run_enrichment(
user_id=user_id,
request_email=request_email,
items=claimed_items,
)
queued_ids = await enqueue_missing_pdf_jobs(
db_session,
user_id=user_id,
request_email=request_email,
rows=items,
max_items=max_items,
)
_scheduled_tasks.add(task)
task.add_done_callback(_on_task_done)
logger.info(
"publications.enrichment.scheduled",
extra={
"event": "publications.enrichment.scheduled",
"user_id": user_id,
"publication_count": len(claimed_items),
"publication_count": len(queued_ids),
},
)
return len(claimed_items)
return len(queued_ids)
async def schedule_retry_pdf_enrichment_for_row(
db_session: AsyncSession,
*,
user_id: int,
request_email: str | None,
item: PublicationListItem,
) -> bool:
queued = await enqueue_retry_pdf_job(
db_session,
user_id=user_id,
request_email=request_email,
row=item,
)
logger.info(
"publications.enrichment.retry_scheduled",
extra={
"event": "publications.enrichment.retry_scheduled",
"user_id": user_id,
"publication_id": item.publication_id,
"queued": queued,
},
)
return queued
async def hydrate_pdf_enrichment_state(
db_session: AsyncSession,
*,
items: list[PublicationListItem],
) -> list[PublicationListItem]:
return await overlay_pdf_job_state(db_session, rows=items)

105
app/services/domains/publications/listing.py
View file

@ -16,94 +16,6 @@ from app.services.domains.publications.queries import (
unread_item_from_row,
)
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
from app.services.domains.unpaywall.application import resolve_publication_oa_metadata
from sqlalchemy import update
from app.db.models import Publication
def _with_oa_overrides(
rows: list[PublicationListItem],
oa_data: dict[int, tuple[str | None, str | None]],
) -> list[PublicationListItem]:
return [
PublicationListItem(
publication_id=row.publication_id,
scholar_profile_id=row.scholar_profile_id,
scholar_label=row.scholar_label,
title=row.title,
year=row.year,
citation_count=row.citation_count,
venue_text=row.venue_text,
pub_url=row.pub_url,
doi=(oa_data.get(row.publication_id) or (None, None))[0] or row.doi,
pdf_url=(oa_data.get(row.publication_id) or (None, None))[1] or row.pdf_url,
is_read=row.is_read,
first_seen_at=row.first_seen_at,
is_new_in_latest_run=row.is_new_in_latest_run,
)
for row in rows
]
def _resolved_fields(
*,
row: PublicationListItem,
resolved: tuple[str | None, str | None] | None,
) -> tuple[str | None, str | None]:
if resolved is None:
return row.doi, row.pdf_url
resolved_doi, resolved_pdf = resolved
return resolved_doi or row.doi, resolved_pdf or row.pdf_url
async def _persist_resolved_metadata(
db_session: AsyncSession,
*,
rows: list[PublicationListItem],
oa_data: dict[int, tuple[str | None, str | None]],
) -> None:
by_id = {row.publication_id: row for row in rows}
updates: list[tuple[int, str | None, str | None]] = []
for publication_id, resolved in oa_data.items():
existing = by_id.get(publication_id)
if existing is None:
continue
next_doi, next_pdf = _resolved_fields(row=existing, resolved=resolved)
if next_doi == existing.doi and next_pdf == existing.pdf_url:
continue
updates.append((publication_id, next_doi, next_pdf))
for publication_id, doi, pdf_url in updates:
await db_session.execute(
update(Publication)
.where(Publication.id == publication_id)
.values(doi=doi, pdf_url=pdf_url)
)
if updates:
await db_session.commit()
def missing_pdf_items(
rows: list[PublicationListItem],
*,
limit: int,
) -> list[PublicationListItem]:
bounded_limit = max(0, int(limit))
if bounded_limit == 0:
return []
return [row for row in rows if not row.pdf_url][:bounded_limit]
async def resolve_and_persist_oa_metadata(
db_session: AsyncSession,
*,
rows: list[PublicationListItem],
unpaywall_email: str | None = None,
) -> dict[int, tuple[str | None, str | None]]:
if not rows:
return {}
oa_data = await resolve_publication_oa_metadata(rows, request_email=unpaywall_email)
await _persist_resolved_metadata(db_session, rows=rows, oa_data=oa_data)
return oa_data
async def list_for_user(
@ -112,7 +24,9 @@ async def list_for_user(
user_id: int,
mode: str = MODE_ALL,
scholar_profile_id: int | None = None,
favorite_only: bool = False,
limit: int = 300,
offset: int = 0,
) -> list[PublicationListItem]:
resolved_mode = resolve_publication_view_mode(mode)
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
@ -122,7 +36,9 @@ async def list_for_user(
mode=resolved_mode,
latest_run_id=latest_run_id,
scholar_profile_id=scholar_profile_id,
favorite_only=favorite_only,
limit=limit,
offset=offset,
)
)
rows = [
@ -138,22 +54,13 @@ async def retry_pdf_for_user(
user_id: int,
scholar_profile_id: int,
publication_id: int,
unpaywall_email: str | None = None,
) -> PublicationListItem | None:
item = await get_publication_item_for_user(
return await get_publication_item_for_user(
db_session,
user_id=user_id,
scholar_profile_id=scholar_profile_id,
publication_id=publication_id,
)
if item is None:
return None
oa_data = await resolve_and_persist_oa_metadata(
db_session,
rows=[item],
unpaywall_email=unpaywall_email,
)
return _with_oa_overrides([item], oa_data)[0]
async def list_unread_for_user(
@ -168,7 +75,9 @@ async def list_unread_for_user(
mode=MODE_UNREAD,
latest_run_id=None,
scholar_profile_id=None,
favorite_only=False,
limit=limit,
offset=0,
)
)
return [unread_item_from_row(row) for row in result.all()]

877
app/services/domains/publications/pdf_queue.py Normal file
View file

@ -0,0 +1,877 @@
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from datetime import datetime, timezone
import logging
from sqlalchemy import Select, func, literal, or_, select, union_all
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import (
Publication,
PublicationPdfJob,
PublicationPdfJobEvent,
ScholarProfile,
ScholarPublication,
User,
)
from app.db.session import get_session_factory
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.unpaywall.application import (
FAILURE_RESOLUTION_EXCEPTION,
OaResolutionOutcome,
resolve_publication_oa_outcomes,
)
from app.settings import settings
PDF_STATUS_UNTRACKED = "untracked"
PDF_STATUS_QUEUED = "queued"
PDF_STATUS_RUNNING = "running"
PDF_STATUS_RESOLVED = "resolved"
PDF_STATUS_FAILED = "failed"
PDF_EVENT_QUEUED = "queued"
PDF_EVENT_ATTEMPT_STARTED = "attempt_started"
PDF_EVENT_RESOLVED = "resolved"
PDF_EVENT_FAILED = "failed"
logger = logging.getLogger(__name__)
_scheduled_tasks: set[asyncio.Task[None]] = set()
@dataclass(frozen=True)
class PdfQueueListItem:
publication_id: int
title: str
doi: str | None
pdf_url: str | None
status: str
attempt_count: int
last_failure_reason: str | None
last_failure_detail: str | None
last_source: str | None
requested_by_user_id: int | None
requested_by_email: str | None
queued_at: datetime | None
last_attempt_at: datetime | None
resolved_at: datetime | None
updated_at: datetime
@dataclass(frozen=True)
class PdfRequeueResult:
publication_exists: bool
queued: bool
@dataclass(frozen=True)
class PdfBulkQueueResult:
requested_count: int
queued_count: int
@dataclass(frozen=True)
class PdfQueuePage:
items: list[PdfQueueListItem]
total_count: int
limit: int
offset: int
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
def _publication_ids(rows: list[PublicationListItem]) -> list[int]:
return sorted({row.publication_id for row in rows})
def _status_from_job(row: PublicationListItem, job: PublicationPdfJob | None) -> str:
if row.pdf_url:
return PDF_STATUS_RESOLVED
if job is None:
return PDF_STATUS_UNTRACKED
return job.status
def _item_from_row_and_job(
row: PublicationListItem,
job: PublicationPdfJob | None,
) -> PublicationListItem:
return PublicationListItem(
publication_id=row.publication_id,
scholar_profile_id=row.scholar_profile_id,
scholar_label=row.scholar_label,
title=row.title,
year=row.year,
citation_count=row.citation_count,
venue_text=row.venue_text,
pub_url=row.pub_url,
doi=row.doi,
pdf_url=row.pdf_url,
is_read=row.is_read,
is_favorite=row.is_favorite,
first_seen_at=row.first_seen_at,
is_new_in_latest_run=row.is_new_in_latest_run,
pdf_status=_status_from_job(row, job),
pdf_attempt_count=int(job.attempt_count) if job is not None else 0,
pdf_failure_reason=job.last_failure_reason if job is not None else None,
pdf_failure_detail=job.last_failure_detail if job is not None else None,
)
def _queueable_rows(
rows: list[PublicationListItem],
*,
max_items: int,
) -> list[PublicationListItem]:
bounded = max(0, int(max_items))
if bounded == 0:
return []
candidates = [row for row in rows if not row.pdf_url]
return candidates[:bounded]
def _bounded_limit(limit: int, *, max_value: int = 500) -> int:
return max(1, min(int(limit), max_value))
def _bounded_offset(offset: int) -> int:
return max(int(offset), 0)
def _auto_retry_interval_seconds() -> int:
return max(int(settings.pdf_auto_retry_interval_seconds), 1)
def _auto_retry_first_interval_seconds() -> int:
return max(int(settings.pdf_auto_retry_first_interval_seconds), 1)
def _auto_retry_max_attempts() -> int:
return max(int(settings.pdf_auto_retry_max_attempts), 1)
def _retry_interval_seconds_for_attempt_count(attempt_count: int) -> int:
if int(attempt_count) <= 1:
return _auto_retry_first_interval_seconds()
return _auto_retry_interval_seconds()
def _cooldown_active(
*,
last_attempt_at: datetime | None,
attempt_count: int,
) -> bool:
if last_attempt_at is None:
return False
elapsed = (_utcnow() - last_attempt_at).total_seconds()
return elapsed < _retry_interval_seconds_for_attempt_count(int(attempt_count))
def _can_enqueue_job(
job: PublicationPdfJob | None,
*,
force_retry: bool,
) -> bool:
if job is None:
return True
if job.status in {PDF_STATUS_QUEUED, PDF_STATUS_RUNNING}:
return False
if force_retry:
return job.status in {PDF_STATUS_FAILED, PDF_STATUS_RESOLVED, PDF_STATUS_UNTRACKED}
if job.status == PDF_STATUS_RESOLVED:
return False
if int(job.attempt_count) >= _auto_retry_max_attempts():
return False
if _cooldown_active(
last_attempt_at=job.last_attempt_at,
attempt_count=int(job.attempt_count),
):
return False
return True
def _event_row(
*,
publication_id: int,
user_id: int | None,
event_type: str,
status: str | None,
source: str | None = None,
failure_reason: str | None = None,
message: str | None = None,
) -> PublicationPdfJobEvent:
return PublicationPdfJobEvent(
publication_id=publication_id,
user_id=user_id,
event_type=event_type,
status=status,
source=source,
failure_reason=failure_reason,
message=message,
)
def _queued_job(
*,
publication_id: int,
user_id: int,
) -> PublicationPdfJob:
now = _utcnow()
return PublicationPdfJob(
publication_id=publication_id,
status=PDF_STATUS_QUEUED,
queued_at=now,
last_requested_by_user_id=user_id,
)
def _mark_job_queued(job: PublicationPdfJob, *, user_id: int) -> None:
now = _utcnow()
job.status = PDF_STATUS_QUEUED
job.queued_at = now
job.last_requested_by_user_id = user_id
job.last_failure_reason = None
job.last_failure_detail = None
job.last_source = None
def _state_map(jobs: list[PublicationPdfJob]) -> dict[int, PublicationPdfJob]:
return {int(job.publication_id): job for job in jobs}
async def _jobs_for_publication_ids(
db_session: AsyncSession,
*,
publication_ids: list[int],
) -> dict[int, PublicationPdfJob]:
if not publication_ids:
return {}
result = await db_session.execute(
select(PublicationPdfJob).where(PublicationPdfJob.publication_id.in_(publication_ids))
)
return _state_map(list(result.scalars()))
async def overlay_pdf_job_state(
db_session: AsyncSession,
*,
rows: list[PublicationListItem],
) -> list[PublicationListItem]:
if not rows:
return []
jobs = await _jobs_for_publication_ids(
db_session,
publication_ids=_publication_ids(rows),
)
return [_item_from_row_and_job(row, jobs.get(row.publication_id)) for row in rows]
async def _enqueue_rows(
db_session: AsyncSession,
*,
user_id: int,
rows: list[PublicationListItem],
force_retry: bool,
) -> list[PublicationListItem]:
if not rows:
return []
queued: list[PublicationListItem] = []
jobs = await _jobs_for_publication_ids(
db_session,
publication_ids=_publication_ids(rows),
)
for row in rows:
job = jobs.get(row.publication_id)
if not _can_enqueue_job(job, force_retry=force_retry):
continue
if job is None:
job = _queued_job(publication_id=row.publication_id, user_id=user_id)
jobs[row.publication_id] = job
db_session.add(job)
else:
_mark_job_queued(job, user_id=user_id)
db_session.add(
_event_row(
publication_id=row.publication_id,
user_id=user_id,
event_type=PDF_EVENT_QUEUED,
status=PDF_STATUS_QUEUED,
)
)
queued.append(row)
if queued:
await db_session.commit()
return queued
def _register_task(task: asyncio.Task[None]) -> None:
_scheduled_tasks.add(task)
def _drop_finished_task(task: asyncio.Task[None]) -> None:
_scheduled_tasks.discard(task)
try:
task.result()
except Exception:
logger.exception(
"publications.pdf_queue.task_failed",
extra={"event": "publications.pdf_queue.task_failed"},
)
async def _mark_attempt_started(
*,
publication_id: int,
user_id: int,
) -> None:
session_factory = get_session_factory()
async with session_factory() as db_session:
job = await db_session.get(PublicationPdfJob, publication_id)
if job is None:
job = _queued_job(publication_id=publication_id, user_id=user_id)
db_session.add(job)
job.status = PDF_STATUS_RUNNING
job.last_attempt_at = _utcnow()
job.attempt_count = int(job.attempt_count) + 1
db_session.add(
_event_row(
publication_id=publication_id,
user_id=user_id,
event_type=PDF_EVENT_ATTEMPT_STARTED,
status=PDF_STATUS_RUNNING,
)
)
await db_session.commit()
def _failed_outcome(
*,
row: PublicationListItem,
) -> OaResolutionOutcome:
return OaResolutionOutcome(
publication_id=row.publication_id,
doi=row.doi,
pdf_url=None,
failure_reason=FAILURE_RESOLUTION_EXCEPTION,
source=None,
used_crossref=False,
)
async def _fetch_outcome_for_row(
*,
row: PublicationListItem,
request_email: str | None,
) -> OaResolutionOutcome:
outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email)
outcome = outcomes.get(row.publication_id)
if outcome is not None:
return outcome
return _failed_outcome(row=row)
def _apply_publication_update(
publication: Publication,
*,
doi: str | None,
pdf_url: str | None,
) -> None:
if doi and publication.doi != doi:
publication.doi = doi
if pdf_url and publication.pdf_url != pdf_url:
publication.pdf_url = pdf_url
def _apply_job_outcome(job: PublicationPdfJob, *, outcome: OaResolutionOutcome) -> None:
job.last_source = outcome.source
if outcome.pdf_url:
job.status = PDF_STATUS_RESOLVED
job.resolved_at = _utcnow()
job.last_failure_reason = None
job.last_failure_detail = None
return
job.status = PDF_STATUS_FAILED
job.last_failure_reason = outcome.failure_reason
job.last_failure_detail = outcome.failure_reason
def _result_event(outcome: OaResolutionOutcome) -> tuple[str, str]:
if outcome.pdf_url:
return PDF_EVENT_RESOLVED, PDF_STATUS_RESOLVED
return PDF_EVENT_FAILED, PDF_STATUS_FAILED
async def _persist_outcome(
*,
publication_id: int,
user_id: int,
outcome: OaResolutionOutcome,
) -> None:
session_factory = get_session_factory()
async with session_factory() as db_session:
publication = await db_session.get(Publication, publication_id)
job = await db_session.get(PublicationPdfJob, publication_id)
if publication is None or job is None:
return
_apply_publication_update(publication, doi=outcome.doi, pdf_url=outcome.pdf_url)
_apply_job_outcome(job, outcome=outcome)
event_type, status = _result_event(outcome)
db_session.add(
_event_row(
publication_id=publication_id,
user_id=user_id,
event_type=event_type,
status=status,
source=outcome.source,
failure_reason=outcome.failure_reason,
message=outcome.failure_reason,
)
)
await db_session.commit()
async def _resolve_publication_row(
*,
user_id: int,
request_email: str | None,
row: PublicationListItem,
) -> None:
await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
try:
outcome = await _fetch_outcome_for_row(row=row, request_email=request_email)
except Exception as exc: # pragma: no cover - defensive network boundary
logger.warning(
"publications.pdf_queue.resolve_failed",
extra={
"event": "publications.pdf_queue.resolve_failed",
"publication_id": row.publication_id,
"error": str(exc),
},
)
outcome = _failed_outcome(row=row)
await _persist_outcome(
publication_id=row.publication_id,
user_id=user_id,
outcome=outcome,
)
async def _run_resolution_task(
*,
user_id: int,
request_email: str | None,
rows: list[PublicationListItem],
) -> None:
for row in rows:
await _resolve_publication_row(
user_id=user_id,
request_email=request_email,
row=row,
)
def _schedule_rows(
*,
user_id: int,
request_email: str | None,
rows: list[PublicationListItem],
) -> None:
if not rows:
return
task = asyncio.create_task(
_run_resolution_task(
user_id=user_id,
request_email=request_email,
rows=rows,
)
)
_register_task(task)
task.add_done_callback(_drop_finished_task)
async def enqueue_missing_pdf_jobs(
db_session: AsyncSession,
*,
user_id: int,
request_email: str | None,
rows: list[PublicationListItem],
max_items: int,
) -> list[int]:
queueable = _queueable_rows(rows, max_items=max_items)
queued_rows = await _enqueue_rows(
db_session,
user_id=user_id,
rows=queueable,
force_retry=False,
)
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
return [row.publication_id for row in queued_rows]
async def enqueue_retry_pdf_job(
db_session: AsyncSession,
*,
user_id: int,
request_email: str | None,
row: PublicationListItem,
) -> bool:
queued_rows = await _enqueue_rows(
db_session,
user_id=user_id,
rows=[row],
force_retry=True,
)
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
return bool(queued_rows)
def _retry_item_label(display_name: str | None, scholar_id: str | None) -> str:
return str(display_name or scholar_id or "unknown")
def _retry_item_from_publication(
publication: Publication,
*,
link_row: tuple | None,
) -> PublicationListItem:
if link_row is None:
scholar_profile_id = 0
scholar_label = "unknown"
is_read = True
first_seen_at = publication.created_at or _utcnow()
else:
scholar_profile_id = int(link_row[0])
scholar_label = _retry_item_label(link_row[1], link_row[2])
is_read = bool(link_row[3])
first_seen_at = link_row[4] or publication.created_at or _utcnow()
return PublicationListItem(
publication_id=int(publication.id),
scholar_profile_id=scholar_profile_id,
scholar_label=scholar_label,
title=publication.title_raw,
year=publication.year,
citation_count=int(publication.citation_count or 0),
venue_text=publication.venue_text,
pub_url=publication.pub_url,
doi=publication.doi,
pdf_url=publication.pdf_url,
is_read=is_read,
first_seen_at=first_seen_at,
is_new_in_latest_run=False,
)
async def _retry_item_for_publication_id(
db_session: AsyncSession,
*,
publication_id: int,
) -> PublicationListItem | None:
publication = await db_session.get(Publication, publication_id)
if publication is None:
return None
result = await db_session.execute(
select(
ScholarProfile.id,
ScholarProfile.display_name,
ScholarProfile.scholar_id,
ScholarPublication.is_read,
ScholarPublication.created_at,
)
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
.where(ScholarPublication.publication_id == publication_id)
.order_by(ScholarPublication.created_at.asc())
.limit(1)
)
return _retry_item_from_publication(publication, link_row=result.one_or_none())
async def enqueue_retry_pdf_job_for_publication_id(
db_session: AsyncSession,
*,
user_id: int,
request_email: str | None,
publication_id: int,
) -> PdfRequeueResult:
row = await _retry_item_for_publication_id(
db_session,
publication_id=publication_id,
)
if row is None:
return PdfRequeueResult(publication_exists=False, queued=False)
queued = await enqueue_retry_pdf_job(
db_session,
user_id=user_id,
request_email=request_email,
row=row,
)
return PdfRequeueResult(publication_exists=True, queued=queued)
def _queue_candidate_from_publication(publication: Publication) -> PublicationListItem:
return PublicationListItem(
publication_id=int(publication.id),
scholar_profile_id=0,
scholar_label="admin",
title=publication.title_raw,
year=publication.year,
citation_count=int(publication.citation_count or 0),
venue_text=publication.venue_text,
pub_url=publication.pub_url,
doi=publication.doi,
pdf_url=publication.pdf_url,
is_read=True,
first_seen_at=publication.created_at or _utcnow(),
is_new_in_latest_run=False,
)
async def _missing_pdf_candidates(
db_session: AsyncSession,
*,
limit: int,
) -> list[PublicationListItem]:
bounded_limit = max(1, min(int(limit), 5000))
result = await db_session.execute(
select(Publication)
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
.where(Publication.pdf_url.is_(None))
.where(
or_(
PublicationPdfJob.publication_id.is_(None),
PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
)
)
.order_by(Publication.updated_at.desc(), Publication.id.desc())
.limit(bounded_limit)
)
return [
_queue_candidate_from_publication(publication)
for publication in result.scalars()
]
async def enqueue_all_missing_pdf_jobs(
db_session: AsyncSession,
*,
user_id: int,
request_email: str | None,
limit: int = 1000,
) -> PdfBulkQueueResult:
candidates = await _missing_pdf_candidates(db_session, limit=limit)
queued_rows = await _enqueue_rows(
db_session,
user_id=user_id,
rows=candidates,
force_retry=True,
)
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
return PdfBulkQueueResult(
requested_count=len(candidates),
queued_count=len(queued_rows),
)
def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]:
stmt = (
select(
PublicationPdfJob.publication_id,
Publication.title_raw,
Publication.doi,
Publication.pdf_url,
PublicationPdfJob.status,
PublicationPdfJob.attempt_count,
PublicationPdfJob.last_failure_reason,
PublicationPdfJob.last_failure_detail,
PublicationPdfJob.last_source,
PublicationPdfJob.last_requested_by_user_id,
User.email,
PublicationPdfJob.queued_at,
PublicationPdfJob.last_attempt_at,
PublicationPdfJob.resolved_at,
PublicationPdfJob.updated_at,
)
.join(Publication, Publication.id == PublicationPdfJob.publication_id)
.outerjoin(User, User.id == PublicationPdfJob.last_requested_by_user_id)
)
if status:
stmt = stmt.where(PublicationPdfJob.status == status)
return stmt
def _tracked_queue_select(*, limit: int, offset: int, status: str | None) -> Select[tuple]:
return (
_tracked_queue_select_base(status=status)
.order_by(PublicationPdfJob.updated_at.desc())
.limit(_bounded_limit(limit))
.offset(_bounded_offset(offset))
)
def _untracked_queue_select_base() -> Select[tuple]:
return (
select(
Publication.id,
Publication.title_raw,
Publication.doi,
Publication.pdf_url,
literal(PDF_STATUS_UNTRACKED),
literal(0),
literal(None),
literal(None),
literal(None),
literal(None),
literal(None),
literal(None),
literal(None),
literal(None),
Publication.updated_at,
)
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
.where(Publication.pdf_url.is_(None))
.where(PublicationPdfJob.publication_id.is_(None))
)
def _untracked_queue_select(*, limit: int, offset: int) -> Select[tuple]:
return (
_untracked_queue_select_base()
.order_by(Publication.updated_at.desc(), Publication.id.desc())
.limit(_bounded_limit(limit))
.offset(_bounded_offset(offset))
)
def _all_queue_select(*, limit: int, offset: int) -> Select[tuple]:
union_stmt = union_all(
_tracked_queue_select_base(status=None),
_untracked_queue_select_base(),
).subquery()
return (
select(union_stmt)
.order_by(union_stmt.c.updated_at.desc())
.limit(_bounded_limit(limit))
.offset(_bounded_offset(offset))
)
def _tracked_queue_count_select(*, status: str | None) -> Select[tuple]:
stmt = select(func.count()).select_from(PublicationPdfJob)
if status:
stmt = stmt.where(PublicationPdfJob.status == status)
return stmt
def _untracked_queue_count_select() -> Select[tuple]:
return (
select(func.count())
.select_from(Publication)
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
.where(Publication.pdf_url.is_(None))
.where(PublicationPdfJob.publication_id.is_(None))
)
def _queue_item_from_row(row: tuple) -> PdfQueueListItem:
return PdfQueueListItem(
publication_id=int(row[0]),
title=str(row[1] or ""),
doi=row[2],
pdf_url=row[3],
status=str(row[4] or PDF_STATUS_UNTRACKED),
attempt_count=int(row[5] or 0),
last_failure_reason=row[6],
last_failure_detail=row[7],
last_source=row[8],
requested_by_user_id=int(row[9]) if row[9] is not None else None,
requested_by_email=row[10],
queued_at=row[11],
last_attempt_at=row[12],
resolved_at=row[13],
updated_at=row[14],
)
async def list_pdf_queue_items(
db_session: AsyncSession,
*,
limit: int = 100,
offset: int = 0,
status: str | None = None,
) -> list[PdfQueueListItem]:
bounded_limit = _bounded_limit(limit)
bounded_offset = _bounded_offset(offset)
normalized_status = (status or "").strip().lower() or None
if normalized_status == PDF_STATUS_UNTRACKED:
result = await db_session.execute(
_untracked_queue_select(
limit=bounded_limit,
offset=bounded_offset,
)
)
return [_queue_item_from_row(row) for row in result.all()]
if normalized_status is None:
result = await db_session.execute(
_all_queue_select(
limit=bounded_limit,
offset=bounded_offset,
)
)
return [_queue_item_from_row(row) for row in result.all()]
result = await db_session.execute(
_tracked_queue_select(
limit=bounded_limit,
offset=bounded_offset,
status=normalized_status,
)
)
return [_queue_item_from_row(row) for row in result.all()]
async def count_pdf_queue_items(
db_session: AsyncSession,
*,
status: str | None = None,
) -> int:
normalized_status = (status or "").strip().lower() or None
if normalized_status == PDF_STATUS_UNTRACKED:
result = await db_session.execute(_untracked_queue_count_select())
return int(result.scalar_one() or 0)
tracked_result = await db_session.execute(
_tracked_queue_count_select(status=normalized_status)
)
tracked_count = int(tracked_result.scalar_one() or 0)
if normalized_status is not None:
return tracked_count
untracked_result = await db_session.execute(_untracked_queue_count_select())
untracked_count = int(untracked_result.scalar_one() or 0)
return tracked_count + untracked_count
async def list_pdf_queue_page(
db_session: AsyncSession,
*,
limit: int = 100,
offset: int = 0,
status: str | None = None,
) -> PdfQueuePage:
bounded_limit = _bounded_limit(limit)
bounded_offset = _bounded_offset(offset)
items = await list_pdf_queue_items(
db_session,
limit=bounded_limit,
offset=bounded_offset,
status=status,
)
total_count = await count_pdf_queue_items(
db_session,
status=status,
)
return PdfQueuePage(
items=items,
total_count=total_count,
limit=bounded_limit,
offset=bounded_offset,
)

10
app/services/domains/publications/queries.py
View file

@ -36,7 +36,9 @@ def publications_query(
mode: str,
latest_run_id: int | None,
scholar_profile_id: int | None,
favorite_only: bool,
limit: int,
offset: int = 0,
) -> Select[tuple]:
scholar_label = ScholarProfile.display_name
stmt = (
@ -53,6 +55,7 @@ def publications_query(
Publication.doi,
Publication.pdf_url,
ScholarPublication.is_read,
ScholarPublication.is_favorite,
ScholarPublication.first_seen_run_id,
ScholarPublication.created_at,
)
@ -60,10 +63,13 @@ def publications_query(
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
.where(ScholarProfile.user_id == user_id)
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
.offset(max(int(offset), 0))
.limit(limit)
)
if scholar_profile_id is not None:
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
if favorite_only:
stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
if mode == MODE_UNREAD:
stmt = stmt.where(ScholarPublication.is_read.is_(False))
if mode == MODE_LATEST:
@ -93,6 +99,7 @@ def publication_query_for_user(
Publication.doi,
Publication.pdf_url,
ScholarPublication.is_read,
ScholarPublication.is_favorite,
ScholarPublication.first_seen_run_id,
ScholarPublication.created_at,
)
@ -146,6 +153,7 @@ def publication_list_item_from_row(
doi,
pdf_url,
is_read,
is_favorite,
first_seen_run_id,
created_at,
) = row
@ -161,6 +169,7 @@ def publication_list_item_from_row(
doi=doi,
pdf_url=pdf_url,
is_read=bool(is_read),
is_favorite=bool(is_favorite),
first_seen_at=created_at,
is_new_in_latest_run=(
latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
@ -182,6 +191,7 @@ def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
doi,
pdf_url,
_is_read,
_is_favorite,
_first_seen_run_id,
_created_at,
) = row

43
app/services/domains/publications/read_state.py
View file

@ -16,16 +16,20 @@ def _normalized_selection_pairs(selections: list[tuple[int, int]]) -> set[tuple[
return pairs
def _scoped_scholar_ids_query(*, user_id: int):
return (
select(ScholarProfile.id)
.where(ScholarProfile.user_id == user_id)
.scalar_subquery()
)
async def mark_all_unread_as_read_for_user(
db_session: AsyncSession,
*,
user_id: int,
) -> int:
scholar_ids = (
select(ScholarProfile.id)
.where(ScholarProfile.user_id == user_id)
.scalar_subquery()
)
scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
stmt = (
update(ScholarPublication)
.where(
@ -49,11 +53,7 @@ async def mark_selected_as_read_for_user(
if not normalized_pairs:
return 0
scholar_ids = (
select(ScholarProfile.id)
.where(ScholarProfile.user_id == user_id)
.scalar_subquery()
)
scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
stmt = (
update(ScholarPublication)
.where(
@ -69,3 +69,26 @@ async def mark_selected_as_read_for_user(
result = await db_session.execute(stmt)
await db_session.commit()
return int(result.rowcount or 0)
async def set_publication_favorite_for_user(
db_session: AsyncSession,
*,
user_id: int,
scholar_profile_id: int,
publication_id: int,
is_favorite: bool,
) -> int:
scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
stmt = (
update(ScholarPublication)
.where(
ScholarPublication.scholar_profile_id.in_(scholar_ids),
ScholarPublication.scholar_profile_id == int(scholar_profile_id),
ScholarPublication.publication_id == int(publication_id),
)
.values(is_favorite=bool(is_favorite))
)
result = await db_session.execute(stmt)
await db_session.commit()
return int(result.rowcount or 0)

5
app/services/domains/publications/types.py
View file

@ -19,6 +19,11 @@ class PublicationListItem:
is_read: bool
first_seen_at: datetime
is_new_in_latest_run: bool
is_favorite: bool = False
pdf_status: str = "untracked"
pdf_attempt_count: int = 0
pdf_failure_reason: str | None = None
pdf_failure_detail: str | None = None
@dataclass(frozen=True)

7
app/services/domains/unpaywall/__init__.py
View file

@ -1,5 +1,10 @@
from __future__ import annotations
from app.services.domains.unpaywall.application import resolve_publication_pdf_urls
async def resolve_publication_pdf_urls(*args, **kwargs):
from app.services.domains.unpaywall.application import resolve_publication_pdf_urls as _impl
return await _impl(*args, **kwargs)
__all__ = ["resolve_publication_pdf_urls"]

312
app/services/domains/unpaywall/application.py
View file

@ -1,21 +1,44 @@
from __future__ import annotations
from dataclasses import dataclass
import logging
import re
from typing import TYPE_CHECKING
from urllib.parse import unquote
from app.services.domains.crossref.application import discover_doi_for_publication
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
from app.services.domains.doi.normalize import normalize_doi
from app.services.domains.unpaywall.pdf_discovery import (
looks_like_pdf_url,
resolve_pdf_from_landing_page,
)
from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
from app.settings import settings
if TYPE_CHECKING:
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
DOI_URL_RE = re.compile(r"(?:https?://)?(?:dx\.)?doi\.org/(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
UNPAYWALL_URL_TEMPLATE = "https://api.unpaywall.org/v2/{doi}"
FAILURE_MISSING_DOI = "missing_doi"
FAILURE_NO_RECORD = "no_unpaywall_record"
FAILURE_NO_PDF = "no_pdf_found"
FAILURE_RESOLUTION_EXCEPTION = "resolution_exception"
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class OaResolutionOutcome:
publication_id: int
doi: str | None
pdf_url: str | None
failure_reason: str | None
source: str | None
used_crossref: bool
def _extract_doi_candidate(text: str | None) -> str | None:
if not text:
return None
@ -57,21 +80,89 @@ def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str |
)
def _payload_pdf_url(payload: dict) -> str | None:
def _payload_locations(payload: dict) -> list[dict]:
locations: list[dict] = []
best = payload.get("best_oa_location")
if isinstance(best, dict):
pdf_url = best.get("url_for_pdf")
if isinstance(pdf_url, str) and pdf_url.strip():
return pdf_url.strip()
locations = payload.get("oa_locations")
if not isinstance(locations, list):
locations.append(best)
oa_locations = payload.get("oa_locations")
if isinstance(oa_locations, list):
locations.extend(location for location in oa_locations if isinstance(location, dict))
return locations
def _location_value(location: dict, key: str) -> str | None:
value = location.get(key)
if not isinstance(value, str):
return None
for location in locations:
if not isinstance(location, dict):
normalized = value.strip()
return normalized or None
def _payload_pdf_candidates(payload: dict) -> list[str]:
candidates: list[str] = []
seen: set[str] = set()
for location in _payload_locations(payload):
candidate = _location_value(location, "url_for_pdf")
if candidate is None or candidate in seen:
continue
pdf_url = location.get("url_for_pdf")
if isinstance(pdf_url, str) and pdf_url.strip():
return pdf_url.strip()
seen.add(candidate)
candidates.append(candidate)
return candidates
def _payload_landing_candidates(payload: dict) -> list[str]:
candidates: list[str] = []
seen: set[str] = set()
for location in _payload_locations(payload):
candidate = _location_value(location, "url")
if candidate is None or candidate in seen:
continue
seen.add(candidate)
candidates.append(candidate)
return candidates
def _crawl_targets(
*,
payload: dict,
pdf_candidates: list[str],
) -> list[str]:
targets = _payload_landing_candidates(payload)
seen = set(targets)
for candidate in pdf_candidates:
if candidate in seen:
continue
targets.append(candidate)
seen.add(candidate)
doi = normalize_doi(str(payload.get("doi") or ""))
doi_landing_url = f"https://doi.org/{doi}" if doi else None
if doi_landing_url and doi_landing_url not in seen:
targets.append(doi_landing_url)
return targets
def _has_direct_payload_pdf(payload: dict) -> bool:
return any(looks_like_pdf_url(candidate) for candidate in _payload_pdf_candidates(payload))
async def _resolved_pdf_url_from_payload(
payload: dict,
*,
client,
) -> str | None:
pdf_candidates = _payload_pdf_candidates(payload)
for candidate in pdf_candidates:
if looks_like_pdf_url(candidate):
return candidate
for page_url in _crawl_targets(payload=payload, pdf_candidates=pdf_candidates)[:3]:
discovered = await resolve_pdf_from_landing_page(client, page_url=page_url)
if discovered:
logger.info(
"unpaywall.pdf_discovered_from_landing",
extra={"event": "unpaywall.pdf_discovered_from_landing", "landing_url": page_url},
)
return discovered
return None
@ -81,6 +172,7 @@ async def _fetch_unpaywall_payload_by_doi(
doi: str,
email: str,
) -> dict | None:
await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
response = await client.get(
UNPAYWALL_URL_TEMPLATE.format(doi=doi),
params={"email": email},
@ -130,7 +222,7 @@ async def _resolve_item_payload(
payload: dict | None = None
if doi:
payload = await _fetch_unpaywall_payload_by_doi(client=client, doi=doi, email=email)
if payload is not None and _payload_pdf_url(payload):
if payload is not None and _has_direct_payload_pdf(payload):
return payload, False
if not allow_crossref or not settings.crossref_enabled:
return payload, False
@ -151,9 +243,13 @@ async def _resolve_item_payload(
return payload, True
def _doi_and_pdf_from_payload(payload: dict) -> tuple[str | None, str | None]:
async def _doi_and_pdf_from_payload(
payload: dict,
*,
client,
) -> tuple[str | None, str | None]:
doi = normalize_doi(str(payload.get("doi") or ""))
return doi, _payload_pdf_url(payload)
return doi, await _resolved_pdf_url_from_payload(payload, client=client)
def _resolution_targets(items: list[PublicationListItem]) -> list[PublicationListItem]:
@ -164,11 +260,165 @@ def _crossref_budget_value() -> int:
return max(int(settings.crossref_max_lookups_per_request), 0)
def _outcome_with_failure(
*,
item: PublicationListItem,
failure_reason: str,
used_crossref: bool,
) -> OaResolutionOutcome:
return OaResolutionOutcome(
publication_id=item.publication_id,
doi=_publication_doi(item),
pdf_url=None,
failure_reason=failure_reason,
source=None,
used_crossref=used_crossref,
)
def _missing_payload_failure_reason(item: PublicationListItem, *, used_crossref: bool) -> str:
if _publication_doi(item):
return FAILURE_NO_RECORD
if used_crossref:
return FAILURE_NO_RECORD
return FAILURE_MISSING_DOI
def _source_name(*, used_crossref: bool) -> str:
return "crossref+unpaywall" if used_crossref else "unpaywall"
def _outcome_from_payload(
*,
item: PublicationListItem,
doi: str | None,
pdf_url: str | None,
used_crossref: bool,
) -> OaResolutionOutcome:
return OaResolutionOutcome(
publication_id=item.publication_id,
doi=doi,
pdf_url=pdf_url,
failure_reason=None if pdf_url else FAILURE_NO_PDF,
source=_source_name(used_crossref=used_crossref),
used_crossref=used_crossref,
)
def _resolved_pdf_count(outcomes: dict[int, OaResolutionOutcome]) -> int:
return sum(1 for outcome in outcomes.values() if outcome.pdf_url)
def _outcome_metadata(outcomes: dict[int, OaResolutionOutcome]) -> dict[int, tuple[str | None, str | None]]:
return {
publication_id: (outcome.doi, outcome.pdf_url)
for publication_id, outcome in outcomes.items()
}
async def _resolve_outcome_for_item(
*,
client,
item: PublicationListItem,
email: str,
allow_crossref: bool,
) -> OaResolutionOutcome:
payload, used_crossref = await _resolve_item_payload(
client=client,
item=item,
email=email,
allow_crossref=allow_crossref,
)
if not isinstance(payload, dict):
return _outcome_with_failure(
item=item,
failure_reason=_missing_payload_failure_reason(item, used_crossref=used_crossref),
used_crossref=used_crossref,
)
doi, pdf_url = await _doi_and_pdf_from_payload(payload, client=client)
return _outcome_from_payload(
item=item,
doi=doi,
pdf_url=pdf_url,
used_crossref=used_crossref,
)
def _doi_input_count(items: list[PublicationListItem]) -> int:
return len([item for item in items if _publication_doi(item)])
def _search_attempt_count(*, targets: list[PublicationListItem]) -> int:
return len([item for item in targets if not _publication_doi(item)])
async def _safe_outcome_for_item(
*,
client,
item: PublicationListItem,
email: str,
allow_crossref: bool,
) -> OaResolutionOutcome:
try:
return await _resolve_outcome_for_item(
client=client,
item=item,
email=email,
allow_crossref=allow_crossref,
)
except Exception as exc: # pragma: no cover - defensive network boundary
logger.warning(
"unpaywall.resolve_item_failed",
extra={
"event": "unpaywall.resolve_item_failed",
"publication_id": item.publication_id,
"error": str(exc),
},
)
return _outcome_with_failure(
item=item,
failure_reason=FAILURE_RESOLUTION_EXCEPTION,
used_crossref=allow_crossref and settings.crossref_enabled,
)
async def _resolve_outcomes_with_client(
*,
client,
targets: list[PublicationListItem],
email: str,
) -> dict[int, OaResolutionOutcome]:
outcomes: dict[int, OaResolutionOutcome] = {}
crossref_budget = _crossref_budget_value()
crossref_lookups = 0
for item in targets:
allow_crossref = crossref_budget > 0 and crossref_lookups < crossref_budget
outcome = await _safe_outcome_for_item(
client=client,
item=item,
email=email,
allow_crossref=allow_crossref,
)
if outcome.used_crossref:
crossref_lookups += 1
outcomes[item.publication_id] = outcome
return outcomes
async def resolve_publication_oa_metadata(
items: list[PublicationListItem],
*,
request_email: str | None = None,
) -> dict[int, tuple[str | None, str | None]]:
outcomes = await resolve_publication_oa_outcomes(items, request_email=request_email)
return _outcome_metadata(outcomes)
async def resolve_publication_oa_outcomes(
items: list[PublicationListItem],
*,
request_email: str | None = None,
) -> dict[int, OaResolutionOutcome]:
if not settings.unpaywall_enabled:
return {}
email = _email_for_request(request_email)
@ -178,35 +428,21 @@ async def resolve_publication_oa_metadata(
import httpx
timeout_seconds = max(float(settings.unpaywall_timeout_seconds), 0.5)
resolved: dict[int, tuple[str | None, str | None]] = {}
crossref_budget = _crossref_budget_value()
crossref_lookups = 0
targets = _resolution_targets(items)[: max(int(settings.unpaywall_max_items_per_request), 0)]
async with httpx.AsyncClient(timeout=timeout_seconds) as client:
for item in targets:
allow_crossref = crossref_budget > 0 and crossref_lookups < crossref_budget
payload, used_crossref = await _resolve_item_payload(
client=client,
item=item,
email=email,
allow_crossref=allow_crossref,
)
if used_crossref:
crossref_lookups += 1
if not isinstance(payload, dict):
continue
resolved[item.publication_id] = _doi_and_pdf_from_payload(payload)
resolved_count = sum(1 for _doi, pdf in resolved.values() if pdf)
doi_input_count = len([item for item in items if _publication_doi(item)])
target_doi_count = len([item for item in targets if _publication_doi(item)])
outcomes = await _resolve_outcomes_with_client(
client=client,
targets=targets,
email=email,
)
_log_resolution_summary(
publication_count=len(items),
doi_input_count=doi_input_count,
search_attempt_count=max(0, len(targets) - target_doi_count),
resolved_pdf_count=resolved_count,
doi_input_count=_doi_input_count(items),
search_attempt_count=_search_attempt_count(targets=targets),
resolved_pdf_count=_resolved_pdf_count(outcomes),
email=email,
)
return resolved
return outcomes
async def resolve_publication_pdf_urls(

156
app/services/domains/unpaywall/pdf_discovery.py Normal file
View file

@ -0,0 +1,156 @@
from __future__ import annotations
from html.parser import HTMLParser
import re
from urllib.parse import urljoin, urlparse
from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
from app.settings import settings
PDF_MIME = "application/pdf"
URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.I)
class _LandingPdfParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.base_href: str | None = None
self.candidates: list[str] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
attrs_map = {key.lower(): (value or "").strip() for key, value in attrs}
if tag == "base":
self.base_href = attrs_map.get("href") or self.base_href
return
if tag == "meta":
self._append_meta_candidate(attrs_map)
return
if tag == "link":
self._append_link_candidate(attrs_map)
return
if tag == "a":
self._append_anchor_candidate(attrs_map)
def _append_meta_candidate(self, attrs_map: dict[str, str]) -> None:
meta_name = (attrs_map.get("name") or attrs_map.get("property") or "").lower()
if meta_name != "citation_pdf_url":
return
content = attrs_map.get("content")
if content:
self.candidates.append(content)
def _append_link_candidate(self, attrs_map: dict[str, str]) -> None:
href = attrs_map.get("href")
link_type = (attrs_map.get("type") or "").lower()
if href and "pdf" in link_type:
self.candidates.append(href)
def _append_anchor_candidate(self, attrs_map: dict[str, str]) -> None:
href = attrs_map.get("href")
if href:
self.candidates.append(href)
def looks_like_pdf_url(url: str | None) -> bool:
if not isinstance(url, str):
return False
value = url.strip()
if not value:
return False
parsed = urlparse(value)
path = (parsed.path or "").lower()
query = (parsed.query or "").lower()
return path.endswith(".pdf") or ".pdf" in query
def _normalized_candidate_urls(*, page_url: str, html: str) -> list[str]:
parser = _LandingPdfParser()
parser.feed(html)
parser.close()
base_url = urljoin(page_url, parser.base_href) if parser.base_href else page_url
raw_candidates = [*parser.candidates, *_text_url_candidates(html)]
deduped: list[str] = []
seen: set[str] = set()
for raw in raw_candidates:
absolute = urljoin(base_url, raw.strip())
parsed = urlparse(absolute)
if parsed.scheme not in {"http", "https"}:
continue
if absolute in seen:
continue
seen.add(absolute)
deduped.append(absolute)
return sorted(deduped, key=_candidate_sort_key)
def _text_url_candidates(html: str) -> list[str]:
candidates: list[str] = []
for match in URL_RE.findall(html):
cleaned = match.rstrip(".,);]}>")
if "pdf" not in cleaned.lower():
continue
candidates.append(cleaned)
return candidates
def _candidate_sort_key(candidate: str) -> tuple[int, str]:
lowered = candidate.lower()
if looks_like_pdf_url(candidate):
return (0, lowered)
if any(token in lowered for token in ("doi", "full", "article", "download")):
return (1, lowered)
return (2, lowered)
def _is_html_response(response) -> bool:
content_type = str(response.headers.get("content-type") or "").lower()
return "text/html" in content_type or "application/xhtml+xml" in content_type
async def _fetch_page_html(client, *, page_url: str) -> str | None:
await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
response = await client.get(page_url, follow_redirects=True)
if response.status_code != 200 or not _is_html_response(response):
return None
text = response.text or ""
return text[: max(int(settings.unpaywall_pdf_discovery_max_html_bytes), 0)]
async def _candidate_is_pdf(client, *, candidate_url: str) -> bool:
if looks_like_pdf_url(candidate_url):
return True
await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
response = await client.get(candidate_url, follow_redirects=True)
content_type = str(response.headers.get("content-type") or "").lower()
return response.status_code == 200 and PDF_MIME in content_type
def _candidate_limit() -> int:
return max(int(settings.unpaywall_pdf_discovery_max_candidates), 1)
async def _resolve_pdf_from_candidate_page(client, *, candidate_url: str) -> str | None:
html = await _fetch_page_html(client, page_url=candidate_url)
if not html:
return None
nested_candidates = _normalized_candidate_urls(page_url=candidate_url, html=html)
for nested in nested_candidates[: _candidate_limit()]:
if await _candidate_is_pdf(client, candidate_url=nested):
return nested
return None
async def resolve_pdf_from_landing_page(client, *, page_url: str) -> str | None:
if not settings.unpaywall_pdf_discovery_enabled:
return None
html = await _fetch_page_html(client, page_url=page_url)
if not html:
return None
candidates = _normalized_candidate_urls(page_url=page_url, html=html)
for candidate in candidates[: _candidate_limit()]:
if await _candidate_is_pdf(client, candidate_url=candidate):
return candidate
nested_pdf = await _resolve_pdf_from_candidate_page(client, candidate_url=candidate)
if nested_pdf:
return nested_pdf
return None

18
app/services/domains/unpaywall/rate_limit.py Normal file
View file

@ -0,0 +1,18 @@
from __future__ import annotations
import asyncio
import time
_REQUEST_LOCK = asyncio.Lock()
_LAST_REQUEST_AT = 0.0
async def wait_for_unpaywall_slot(*, min_interval_seconds: float) -> None:
global _LAST_REQUEST_AT
interval = max(float(min_interval_seconds), 0.0)
async with _REQUEST_LOCK:
elapsed = time.monotonic() - _LAST_REQUEST_AT
remaining = interval - elapsed
if remaining > 0:
await asyncio.sleep(remaining)
_LAST_REQUEST_AT = time.monotonic()