harden domain architecture and add lazy Crossref+Unpaywall PDF enrichment with publications workspace refactor

app/services/domains/publications/application.py

@@ -8,8 +8,12 @@ from app.services.domains.publications.counts import (
 from app.services.domains.publications.listing import (
     list_for_user,
     list_new_for_latest_run_for_user,
+    retry_pdf_for_user,
     list_unread_for_user,
 )
+from app.services.domains.publications.enrichment import (
+    schedule_missing_pdf_enrichment_for_user,
+)
 from app.services.domains.publications.modes import (
     MODE_ALL,
     MODE_LATEST,
@@ -20,6 +24,7 @@ from app.services.domains.publications.modes import (
 )
 from app.services.domains.publications.queries import (
     get_latest_completed_run_id_for_user,
+    get_publication_item_for_user,
     publications_query,
 )
 from app.services.domains.publications.read_state import (
@@ -39,9 +44,12 @@ __all__ = [
     "resolve_mode",
     "get_latest_completed_run_id_for_user",
     "publications_query",
+    "get_publication_item_for_user",
     "list_for_user",
     "list_unread_for_user",
     "list_new_for_latest_run_for_user",
+    "retry_pdf_for_user",
+    "schedule_missing_pdf_enrichment_for_user",
     "count_for_user",
     "count_unread_for_user",
     "count_latest_for_user",

app/services/domains/publications/enrichment.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+
+from app.db.session import get_session_factory
+from app.services.domains.publications.listing import (
+    missing_pdf_items,
+    resolve_and_persist_oa_metadata,
+)
+from app.services.domains.publications.types import PublicationListItem
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+_enrichment_lock = asyncio.Lock()
+_inflight_publications: set[tuple[int, int]] = set()
+_recent_attempt_seconds: dict[tuple[int, int], float] = {}
+_scheduled_tasks: set[asyncio.Task[None]] = set()
+
+
+def _cooldown_seconds() -> float:
+    return max(float(settings.unpaywall_retry_cooldown_seconds), 1.0)
+
+
+def _prune_recent_attempts(now_seconds: float, *, cooldown_seconds: float) -> None:
+    expiry = cooldown_seconds * 3
+    stale_keys = [
+        key for key, attempted_seconds in _recent_attempt_seconds.items()
+        if now_seconds - attempted_seconds >= expiry
+    ]
+    for key in stale_keys:
+        _recent_attempt_seconds.pop(key, None)
+
+
+async def _claim_items(
+    *,
+    user_id: int,
+    items: list[PublicationListItem],
+    max_items: int,
+) -> list[PublicationListItem]:
+    candidates = missing_pdf_items(items, limit=max_items)
+    if not candidates:
+        return []
+    now_seconds = time.monotonic()
+    cooldown_seconds = _cooldown_seconds()
+    claimed: list[PublicationListItem] = []
+    async with _enrichment_lock:
+        _prune_recent_attempts(now_seconds, cooldown_seconds=cooldown_seconds)
+        for item in candidates:
+            key = (user_id, item.publication_id)
+            attempted_seconds = _recent_attempt_seconds.get(key)
+            if key in _inflight_publications:
+                continue
+            if attempted_seconds is not None and now_seconds - attempted_seconds < cooldown_seconds:
+                continue
+            _inflight_publications.add(key)
+            _recent_attempt_seconds[key] = now_seconds
+            claimed.append(item)
+    return claimed
+
+
+async def _release_claims(*, user_id: int, publication_ids: list[int]) -> None:
+    async with _enrichment_lock:
+        for publication_id in publication_ids:
+            _inflight_publications.discard((user_id, publication_id))
+
+
+def _on_task_done(task: asyncio.Task[None]) -> None:
+    _scheduled_tasks.discard(task)
+    try:
+        task.result()
+    except Exception:
+        logger.exception(
+            "publications.enrichment.task_failed",
+            extra={"event": "publications.enrichment.task_failed"},
+        )
+
+
+async def _run_enrichment(
+    *,
+    user_id: int,
+    request_email: str | None,
+    items: list[PublicationListItem],
+) -> None:
+    publication_ids = [item.publication_id for item in items]
+    try:
+        session_factory = get_session_factory()
+        async with session_factory() as db_session:
+            await resolve_and_persist_oa_metadata(
+                db_session,
+                rows=items,
+                unpaywall_email=request_email,
+            )
+    finally:
+        await _release_claims(user_id=user_id, publication_ids=publication_ids)
+        logger.info(
+            "publications.enrichment.completed",
+            extra={
+                "event": "publications.enrichment.completed",
+                "user_id": user_id,
+                "publication_count": len(items),
+            },
+        )
+
+
+async def schedule_missing_pdf_enrichment_for_user(
+    *,
+    user_id: int,
+    request_email: str | None,
+    items: list[PublicationListItem],
+    max_items: int,
+) -> int:
+    claimed_items = await _claim_items(user_id=user_id, items=items, max_items=max_items)
+    if not claimed_items:
+        return 0
+    task = asyncio.create_task(
+        _run_enrichment(
+            user_id=user_id,
+            request_email=request_email,
+            items=claimed_items,
+        )
+    )
+    _scheduled_tasks.add(task)
+    task.add_done_callback(_on_task_done)
+    logger.info(
+        "publications.enrichment.scheduled",
+        extra={
+            "event": "publications.enrichment.scheduled",
+            "user_id": user_id,
+            "publication_count": len(claimed_items),
+        },
+    )
+    return len(claimed_items)

app/services/domains/publications/listing.py

@@ -10,11 +10,100 @@ from app.services.domains.publications.modes import (
 )
 from app.services.domains.publications.queries import (
     get_latest_completed_run_id_for_user,
+    get_publication_item_for_user,
     publication_list_item_from_row,
     publications_query,
     unread_item_from_row,
 )
 from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+from app.services.domains.unpaywall.application import resolve_publication_oa_metadata
+from sqlalchemy import update
+from app.db.models import Publication
+
+
+def _with_oa_overrides(
+    rows: list[PublicationListItem],
+    oa_data: dict[int, tuple[str | None, str | None]],
+) -> list[PublicationListItem]:
+    return [
+        PublicationListItem(
+            publication_id=row.publication_id,
+            scholar_profile_id=row.scholar_profile_id,
+            scholar_label=row.scholar_label,
+            title=row.title,
+            year=row.year,
+            citation_count=row.citation_count,
+            venue_text=row.venue_text,
+            pub_url=row.pub_url,
+            doi=(oa_data.get(row.publication_id) or (None, None))[0] or row.doi,
+            pdf_url=(oa_data.get(row.publication_id) or (None, None))[1] or row.pdf_url,
+            is_read=row.is_read,
+            first_seen_at=row.first_seen_at,
+            is_new_in_latest_run=row.is_new_in_latest_run,
+        )
+        for row in rows
+    ]
+
+
+def _resolved_fields(
+    *,
+    row: PublicationListItem,
+    resolved: tuple[str | None, str | None] | None,
+) -> tuple[str | None, str | None]:
+    if resolved is None:
+        return row.doi, row.pdf_url
+    resolved_doi, resolved_pdf = resolved
+    return resolved_doi or row.doi, resolved_pdf or row.pdf_url
+
+
+async def _persist_resolved_metadata(
+    db_session: AsyncSession,
+    *,
+    rows: list[PublicationListItem],
+    oa_data: dict[int, tuple[str | None, str | None]],
+) -> None:
+    by_id = {row.publication_id: row for row in rows}
+    updates: list[tuple[int, str | None, str | None]] = []
+    for publication_id, resolved in oa_data.items():
+        existing = by_id.get(publication_id)
+        if existing is None:
+            continue
+        next_doi, next_pdf = _resolved_fields(row=existing, resolved=resolved)
+        if next_doi == existing.doi and next_pdf == existing.pdf_url:
+            continue
+        updates.append((publication_id, next_doi, next_pdf))
+    for publication_id, doi, pdf_url in updates:
+        await db_session.execute(
+            update(Publication)
+            .where(Publication.id == publication_id)
+            .values(doi=doi, pdf_url=pdf_url)
+        )
+    if updates:
+        await db_session.commit()
+
+
+def missing_pdf_items(
+    rows: list[PublicationListItem],
+    *,
+    limit: int,
+) -> list[PublicationListItem]:
+    bounded_limit = max(0, int(limit))
+    if bounded_limit == 0:
+        return []
+    return [row for row in rows if not row.pdf_url][:bounded_limit]
+
+
+async def resolve_and_persist_oa_metadata(
+    db_session: AsyncSession,
+    *,
+    rows: list[PublicationListItem],
+    unpaywall_email: str | None = None,
+) -> dict[int, tuple[str | None, str | None]]:
+    if not rows:
+        return {}
+    oa_data = await resolve_publication_oa_metadata(rows, request_email=unpaywall_email)
+    await _persist_resolved_metadata(db_session, rows=rows, oa_data=oa_data)
+    return oa_data
 
 
 async def list_for_user(
@@ -36,10 +125,35 @@ async def list_for_user(
             limit=limit,
         )
     )
-    return [
+    rows = [
         publication_list_item_from_row(row, latest_run_id=latest_run_id)
         for row in result.all()
     ]
+    return rows
+
+
+async def retry_pdf_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+    publication_id: int,
+    unpaywall_email: str | None = None,
+) -> PublicationListItem | None:
+    item = await get_publication_item_for_user(
+        db_session,
+        user_id=user_id,
+        scholar_profile_id=scholar_profile_id,
+        publication_id=publication_id,
+    )
+    if item is None:
+        return None
+    oa_data = await resolve_and_persist_oa_metadata(
+        db_session,
+        rows=[item],
+        unpaywall_email=unpaywall_email,
+    )
+    return _with_oa_overrides([item], oa_data)[0]
 
 
 async def list_unread_for_user(
@@ -73,17 +187,19 @@ async def list_new_for_latest_run_for_user(
         scholar_profile_id=None,
         limit=limit,
     )
-    return [
-        UnreadPublicationItem(
-            publication_id=row.publication_id,
-            scholar_profile_id=row.scholar_profile_id,
-            scholar_label=row.scholar_label,
-            title=row.title,
-            year=row.year,
-            citation_count=row.citation_count,
-            venue_text=row.venue_text,
-            pub_url=row.pub_url,
-            pdf_url=row.pdf_url,
-        )
-        for row in rows
-    ]
+    return [_to_unread_item(row) for row in rows]
+
+
+def _to_unread_item(row: PublicationListItem) -> UnreadPublicationItem:
+    return UnreadPublicationItem(
+        publication_id=row.publication_id,
+        scholar_profile_id=row.scholar_profile_id,
+        scholar_label=row.scholar_label,
+        title=row.title,
+        year=row.year,
+        citation_count=row.citation_count,
+        venue_text=row.venue_text,
+        pub_url=row.pub_url,
+        doi=row.doi,
+        pdf_url=row.pdf_url,
+    )

app/services/domains/publications/queries.py

@@ -50,6 +50,7 @@ def publications_query(
             Publication.citation_count,
             Publication.venue_text,
             Publication.pub_url,
+            Publication.doi,
             Publication.pdf_url,
             ScholarPublication.is_read,
             ScholarPublication.first_seen_run_id,
@@ -72,6 +73,61 @@ def publications_query(
     return stmt
 
 
+def publication_query_for_user(
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+    publication_id: int,
+) -> Select[tuple]:
+    return (
+        select(
+            Publication.id,
+            ScholarProfile.id,
+            ScholarProfile.display_name,
+            ScholarProfile.scholar_id,
+            Publication.title_raw,
+            Publication.year,
+            Publication.citation_count,
+            Publication.venue_text,
+            Publication.pub_url,
+            Publication.doi,
+            Publication.pdf_url,
+            ScholarPublication.is_read,
+            ScholarPublication.first_seen_run_id,
+            ScholarPublication.created_at,
+        )
+        .join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
+        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
+        .where(
+            ScholarProfile.user_id == user_id,
+            ScholarProfile.id == scholar_profile_id,
+            Publication.id == publication_id,
+        )
+        .limit(1)
+    )
+
+
+async def get_publication_item_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+    publication_id: int,
+) -> PublicationListItem | None:
+    latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
+    result = await db_session.execute(
+        publication_query_for_user(
+            user_id=user_id,
+            scholar_profile_id=scholar_profile_id,
+            publication_id=publication_id,
+        )
+    )
+    row = result.one_or_none()
+    if row is None:
+        return None
+    return publication_list_item_from_row(row, latest_run_id=latest_run_id)
+
+
 def publication_list_item_from_row(
     row: tuple,
     *,
@@ -87,6 +143,7 @@ def publication_list_item_from_row(
         citation_count,
         venue_text,
         pub_url,
+        doi,
         pdf_url,
         is_read,
         first_seen_run_id,
@@ -101,6 +158,7 @@ def publication_list_item_from_row(
         citation_count=_normalized_citation_count(citation_count),
         venue_text=venue_text,
         pub_url=pub_url,
+        doi=doi,
         pdf_url=pdf_url,
         is_read=bool(is_read),
         first_seen_at=created_at,
@@ -121,6 +179,7 @@ def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
         citation_count,
         venue_text,
         pub_url,
+        doi,
         pdf_url,
         _is_read,
         _first_seen_run_id,
@@ -135,5 +194,6 @@ def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
         citation_count=_normalized_citation_count(citation_count),
         venue_text=venue_text,
         pub_url=pub_url,
+        doi=doi,
         pdf_url=pdf_url,
     )

app/services/domains/publications/types.py

@@ -14,6 +14,7 @@ class PublicationListItem:
     citation_count: int
     venue_text: str | None
     pub_url: str | None
+    doi: str | None
     pdf_url: str | None
     is_read: bool
     first_seen_at: datetime
@@ -30,4 +31,5 @@ class UnreadPublicationItem:
     citation_count: int
     venue_text: str | None
     pub_url: str | None
+    doi: str | None
     pdf_url: str | None