ci: add CodeQL security scanning and Dependabot

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

app/services/publications/__init__.py

@@ -0,0 +1,84 @@
+from app.services.domains.publications.application import (
+    MODE_ALL as MODE_ALL,
+)
+from app.services.domains.publications.application import (
+    MODE_LATEST as MODE_LATEST,
+)
+from app.services.domains.publications.application import (
+    MODE_NEW as MODE_NEW,
+)
+from app.services.domains.publications.application import (
+    MODE_UNREAD as MODE_UNREAD,
+)
+from app.services.domains.publications.application import (
+    PublicationListItem as PublicationListItem,
+)
+from app.services.domains.publications.application import (
+    UnreadPublicationItem as UnreadPublicationItem,
+)
+from app.services.domains.publications.application import (
+    count_favorite_for_user as count_favorite_for_user,
+)
+from app.services.domains.publications.application import (
+    count_for_user as count_for_user,
+)
+from app.services.domains.publications.application import (
+    count_latest_for_user as count_latest_for_user,
+)
+from app.services.domains.publications.application import (
+    count_pdf_queue_items as count_pdf_queue_items,
+)
+from app.services.domains.publications.application import (
+    count_unread_for_user as count_unread_for_user,
+)
+from app.services.domains.publications.application import (
+    enqueue_all_missing_pdf_jobs as enqueue_all_missing_pdf_jobs,
+)
+from app.services.domains.publications.application import (
+    enqueue_retry_pdf_job_for_publication_id as enqueue_retry_pdf_job_for_publication_id,
+)
+from app.services.domains.publications.application import (
+    get_latest_run_id_for_user as get_latest_run_id_for_user,
+)
+from app.services.domains.publications.application import (
+    get_publication_item_for_user as get_publication_item_for_user,
+)
+from app.services.domains.publications.application import (
+    hydrate_pdf_enrichment_state as hydrate_pdf_enrichment_state,
+)
+from app.services.domains.publications.application import (
+    list_for_user as list_for_user,
+)
+from app.services.domains.publications.application import (
+    list_pdf_queue_items as list_pdf_queue_items,
+)
+from app.services.domains.publications.application import (
+    list_pdf_queue_page as list_pdf_queue_page,
+)
+from app.services.domains.publications.application import (
+    list_unread_for_user as list_unread_for_user,
+)
+from app.services.domains.publications.application import (
+    mark_all_unread_as_read_for_user as mark_all_unread_as_read_for_user,
+)
+from app.services.domains.publications.application import (
+    mark_selected_as_read_for_user as mark_selected_as_read_for_user,
+)
+from app.services.domains.publications.application import (
+    publications_query as publications_query,
+)
+from app.services.domains.publications.application import (
+    resolve_publication_view_mode as resolve_publication_view_mode,
+)
+from app.services.domains.publications.application import (
+    retry_pdf_for_user as retry_pdf_for_user,
+)
+from app.services.domains.publications.application import (
+    schedule_missing_pdf_enrichment_for_user as schedule_missing_pdf_enrichment_for_user,
+)
+from app.services.domains.publications.application import (
+    schedule_retry_pdf_enrichment_for_row as schedule_retry_pdf_enrichment_for_row,
+)
+from app.services.domains.publications.application import (
+    set_publication_favorite_for_user as set_publication_favorite_for_user,
+)

app/services/publications/application.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from app.services.domains.publications.counts import (
+    count_favorite_for_user,
+    count_for_user,
+    count_latest_for_user,
+    count_unread_for_user,
+)
+from app.services.domains.publications.enrichment import (
+    hydrate_pdf_enrichment_state,
+    schedule_missing_pdf_enrichment_for_user,
+    schedule_retry_pdf_enrichment_for_row,
+)
+from app.services.domains.publications.listing import (
+    list_for_user,
+    list_unread_for_user,
+    retry_pdf_for_user,
+)
+from app.services.domains.publications.modes import (
+    MODE_ALL,
+    MODE_LATEST,
+    MODE_NEW,
+    MODE_UNREAD,
+    resolve_publication_view_mode,
+)
+from app.services.domains.publications.pdf_queue import (
+    count_pdf_queue_items,
+    enqueue_all_missing_pdf_jobs,
+    enqueue_retry_pdf_job_for_publication_id,
+    list_pdf_queue_items,
+    list_pdf_queue_page,
+)
+from app.services.domains.publications.queries import (
+    get_latest_run_id_for_user,
+    get_publication_item_for_user,
+    publications_query,
+)
+from app.services.domains.publications.read_state import (
+    mark_all_unread_as_read_for_user,
+    mark_selected_as_read_for_user,
+    set_publication_favorite_for_user,
+)
+from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+
+__all__ = [
+    "MODE_ALL",
+    "MODE_LATEST",
+    "MODE_NEW",
+    "MODE_UNREAD",
+    "PublicationListItem",
+    "UnreadPublicationItem",
+    "count_favorite_for_user",
+    "count_for_user",
+    "count_latest_for_user",
+    "count_pdf_queue_items",
+    "count_unread_for_user",
+    "enqueue_all_missing_pdf_jobs",
+    "enqueue_retry_pdf_job_for_publication_id",
+    "get_latest_run_id_for_user",
+    "get_publication_item_for_user",
+    "hydrate_pdf_enrichment_state",
+    "list_for_user",
+    "list_pdf_queue_items",
+    "list_pdf_queue_page",
+    "list_unread_for_user",
+    "mark_all_unread_as_read_for_user",
+    "mark_selected_as_read_for_user",
+    "publications_query",
+    "resolve_publication_view_mode",
+    "retry_pdf_for_user",
+    "schedule_missing_pdf_enrichment_for_user",
+    "schedule_retry_pdf_enrichment_for_row",
+    "set_publication_favorite_for_user",
+]

app/services/publications/counts.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy import distinct, func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import Publication, ScholarProfile, ScholarPublication
+from app.services.domains.publications.modes import (
+    MODE_ALL,
+    MODE_LATEST,
+    MODE_UNREAD,
+    resolve_publication_view_mode,
+)
+from app.services.domains.publications.queries import get_latest_run_id_for_user
+
+
+async def count_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    mode: str = MODE_ALL,
+    scholar_profile_id: int | None = None,
+    favorite_only: bool = False,
+    search: str | None = None,
+    snapshot_before: datetime | None = None,
+) -> int:
+    resolved_mode = resolve_publication_view_mode(mode)
+    latest_run_id = await get_latest_run_id_for_user(db_session, user_id=user_id)
+    stmt = (
+        select(func.count(distinct(ScholarPublication.publication_id)))
+        .select_from(ScholarPublication)
+        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
+        .join(Publication, Publication.id == ScholarPublication.publication_id)
+        .where(ScholarProfile.user_id == user_id)
+    )
+    stmt = _apply_search_filter(stmt, search=search)
+    if scholar_profile_id is not None:
+        stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
+    if favorite_only:
+        stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
+    if snapshot_before is not None:
+        stmt = stmt.where(ScholarPublication.created_at <= snapshot_before)
+    if resolved_mode == MODE_UNREAD:
+        stmt = stmt.where(ScholarPublication.is_read.is_(False))
+    if resolved_mode == MODE_LATEST:
+        if latest_run_id is None:
+            return 0
+        stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
+    result = await db_session.execute(stmt)
+    return int(result.scalar_one() or 0)
+
+
+def _apply_search_filter(stmt, *, search: str | None):
+    if not search:
+        return stmt
+    safe_search = search.replace("%", r"\%").replace("_", r"\_")
+    pattern = f"%{safe_search}%"
+    return stmt.where(
+        Publication.title_raw.ilike(pattern)
+        | ScholarProfile.display_name.ilike(pattern)
+        | Publication.venue_text.ilike(pattern)
+    )
+
+
+async def count_unread_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int | None = None,
+    favorite_only: bool = False,
+    search: str | None = None,
+    snapshot_before: datetime | None = None,
+) -> int:
+    return await count_for_user(
+        db_session,
+        user_id=user_id,
+        mode=MODE_UNREAD,
+        scholar_profile_id=scholar_profile_id,
+        favorite_only=favorite_only,
+        search=search,
+        snapshot_before=snapshot_before,
+    )
+
+
+async def count_latest_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int | None = None,
+    favorite_only: bool = False,
+    search: str | None = None,
+    snapshot_before: datetime | None = None,
+) -> int:
+    return await count_for_user(
+        db_session,
+        user_id=user_id,
+        mode=MODE_LATEST,
+        scholar_profile_id=scholar_profile_id,
+        favorite_only=favorite_only,
+        search=search,
+        snapshot_before=snapshot_before,
+    )
+
+
+async def count_favorite_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int | None = None,
+    search: str | None = None,
+    snapshot_before: datetime | None = None,
+) -> int:
+    return await count_for_user(
+        db_session,
+        user_id=user_id,
+        mode=MODE_ALL,
+        scholar_profile_id=scholar_profile_id,
+        favorite_only=True,
+        search=search,
+        snapshot_before=snapshot_before,
+    )

app/services/publications/dedup.py

@@ -0,0 +1,502 @@
+from __future__ import annotations
+
+import hashlib
+import logging
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+from sqlalchemy import delete, select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import aliased
+
+from app.db.models import Publication, PublicationIdentifier, ScholarPublication
+from app.logging_utils import structured_log
+from app.services.domains.ingestion.fingerprints import (
+    canonical_title_text_for_dedup,
+    canonical_title_tokens_for_dedup,
+    normalize_title,
+)
+
+logger = logging.getLogger(__name__)
+
+NEAR_DUP_DEFAULT_SIMILARITY_THRESHOLD = 0.78
+NEAR_DUP_DEFAULT_CONTAINMENT_THRESHOLD = 0.92
+NEAR_DUP_DEFAULT_MIN_SHARED_TOKENS = 3
+NEAR_DUP_DEFAULT_MAX_YEAR_DELTA = 1
+NEAR_DUP_MIN_TOKEN_LENGTH = 3
+NEAR_DUP_CLUSTER_KEY_LENGTH = 16
+NEAR_DUP_STOPWORDS = {
+    "a",
+    "an",
+    "and",
+    "approach",
+    "for",
+    "in",
+    "method",
+    "of",
+    "on",
+    "the",
+    "to",
+    "using",
+    "via",
+    "with",
+}
+
+
+@dataclass(frozen=True)
+class NearDuplicateMember:
+    publication_id: int
+    title: str
+    year: int | None
+    citation_count: int
+
+
+@dataclass(frozen=True)
+class NearDuplicateCluster:
+    cluster_key: str
+    winner_publication_id: int
+    similarity_score: float
+    members: tuple[NearDuplicateMember, ...]
+
+
+@dataclass(frozen=True)
+class _NearDuplicateCandidate:
+    publication_id: int
+    title: str
+    year: int | None
+    citation_count: int
+    canonical_text: str
+    tokens: frozenset[str]
+
+
+async def find_identifier_duplicate_pairs(
+    db_session: AsyncSession,
+) -> list[tuple[int, int]]:
+    """Return (winner_id, dup_id) pairs where two publications share the same identifier."""
+    pi1 = aliased(PublicationIdentifier, name="pi1")
+    pi2 = aliased(PublicationIdentifier, name="pi2")
+    rows = await db_session.execute(
+        select(pi1.publication_id, pi2.publication_id)
+        .join(
+            pi2,
+            (pi1.kind == pi2.kind)
+            & (pi1.value_normalized == pi2.value_normalized)
+            & (pi1.publication_id < pi2.publication_id),
+        )
+        .distinct()
+    )
+    return [(winner_id, dup_id) for winner_id, dup_id in rows]
+
+
+async def merge_duplicate_publication(
+    db_session: AsyncSession,
+    *,
+    winner_id: int,
+    dup_id: int,
+) -> None:
+    """Merge dup_id into winner_id: migrate metadata/links/identifiers, then delete dup."""
+    if winner_id == dup_id:
+        raise ValueError("winner_id and dup_id must differ.")
+    winner = await _load_publication(db_session, publication_id=winner_id)
+    dup = await _load_publication(db_session, publication_id=dup_id)
+    if winner is None or dup is None:
+        raise ValueError("winner_id and dup_id must both exist.")
+    _merge_publication_metadata(winner=winner, dup=dup)
+    await _migrate_scholar_links(db_session, winner_id=winner_id, dup_id=dup_id)
+    await _migrate_identifiers(db_session, winner_id=winner_id, dup_id=dup_id)
+    await db_session.execute(delete(Publication).where(Publication.id == dup_id))
+    structured_log(logger, "info", "publications.identifier_merge", winner_id=winner_id, dup_id=dup_id)
+
+
+async def _load_publication(
+    db_session: AsyncSession,
+    *,
+    publication_id: int,
+) -> Publication | None:
+    result = await db_session.execute(select(Publication).where(Publication.id == publication_id))
+    return result.scalar_one_or_none()
+
+
+def _merge_publication_metadata(*, winner: Publication, dup: Publication) -> None:
+    if winner.year is None and dup.year is not None:
+        winner.year = dup.year
+    winner.citation_count = max(int(winner.citation_count or 0), int(dup.citation_count or 0))
+    if not winner.author_text and dup.author_text:
+        winner.author_text = dup.author_text
+    if not winner.venue_text and dup.venue_text:
+        winner.venue_text = dup.venue_text
+    if not winner.pub_url and dup.pub_url:
+        winner.pub_url = dup.pub_url
+    if not winner.pdf_url and dup.pdf_url:
+        winner.pdf_url = dup.pdf_url
+    if not winner.cluster_id and dup.cluster_id:
+        winner.cluster_id = dup.cluster_id
+    if not winner.canonical_title_hash and dup.canonical_title_hash:
+        winner.canonical_title_hash = dup.canonical_title_hash
+    winner.title_raw = _preferred_title_text(winner=winner.title_raw, dup=dup.title_raw)
+    winner.title_normalized = normalize_title(winner.title_raw)
+
+
+def _preferred_title_text(*, winner: str, dup: str) -> str:
+    winner_score = len(canonical_title_text_for_dedup(winner))
+    dup_score = len(canonical_title_text_for_dedup(dup))
+    if dup_score > winner_score:
+        return dup
+    return winner
+
+
+async def _migrate_scholar_links(
+    db_session: AsyncSession,
+    *,
+    winner_id: int,
+    dup_id: int,
+) -> None:
+    """Move ScholarPublication links from dup to winner, dropping conflicts."""
+    dup_links_result = await db_session.execute(
+        select(ScholarPublication).where(ScholarPublication.publication_id == dup_id)
+    )
+    dup_links = dup_links_result.scalars().all()
+
+    winner_profiles_result = await db_session.execute(
+        select(ScholarPublication.scholar_profile_id).where(ScholarPublication.publication_id == winner_id)
+    )
+    winner_profiles: set[int] = {row for (row,) in winner_profiles_result}
+
+    for link in dup_links:
+        if link.scholar_profile_id in winner_profiles:
+            await db_session.delete(link)
+        else:
+            link.publication_id = winner_id
+
+
+async def _migrate_identifiers(
+    db_session: AsyncSession,
+    *,
+    winner_id: int,
+    dup_id: int,
+) -> None:
+    result = await db_session.execute(
+        select(PublicationIdentifier).where(PublicationIdentifier.publication_id == dup_id)
+    )
+    dup_identifiers = result.scalars().all()
+    for identifier in dup_identifiers:
+        existing = await _find_identifier(
+            db_session,
+            publication_id=winner_id,
+            kind=identifier.kind,
+            value_normalized=identifier.value_normalized,
+        )
+        if existing is None:
+            identifier.publication_id = winner_id
+            continue
+        _merge_identifier(existing=existing, dup=identifier)
+        await db_session.delete(identifier)
+
+
+async def _find_identifier(
+    db_session: AsyncSession,
+    *,
+    publication_id: int,
+    kind: str,
+    value_normalized: str,
+) -> PublicationIdentifier | None:
+    result = await db_session.execute(
+        select(PublicationIdentifier).where(
+            PublicationIdentifier.publication_id == publication_id,
+            PublicationIdentifier.kind == kind,
+            PublicationIdentifier.value_normalized == value_normalized,
+        )
+    )
+    return result.scalar_one_or_none()
+
+
+def _merge_identifier(*, existing: PublicationIdentifier, dup: PublicationIdentifier) -> None:
+    existing.confidence_score = max(
+        float(existing.confidence_score),
+        float(dup.confidence_score),
+    )
+    if not existing.evidence_url and dup.evidence_url:
+        existing.evidence_url = dup.evidence_url
+    if not existing.value_raw and dup.value_raw:
+        existing.value_raw = dup.value_raw
+
+
+async def sweep_identifier_duplicates(db_session: AsyncSession) -> int:
+    """Find publications sharing an identifier and merge duplicates into the winner."""
+    pairs = await find_identifier_duplicate_pairs(db_session)
+    if not pairs:
+        return 0
+
+    processed_dups: set[int] = set()
+    for winner_id, dup_id in pairs:
+        if dup_id in processed_dups:
+            continue
+        processed_dups.add(dup_id)
+        await merge_duplicate_publication(db_session, winner_id=winner_id, dup_id=dup_id)
+
+    await db_session.flush()
+    return len(processed_dups)
+
+
+async def find_near_duplicate_clusters(
+    db_session: AsyncSession,
+    *,
+    similarity_threshold: float = NEAR_DUP_DEFAULT_SIMILARITY_THRESHOLD,
+    min_shared_tokens: int = NEAR_DUP_DEFAULT_MIN_SHARED_TOKENS,
+    max_year_delta: int = NEAR_DUP_DEFAULT_MAX_YEAR_DELTA,
+) -> list[NearDuplicateCluster]:
+    candidates = await _load_near_duplicate_candidates(db_session)
+    if len(candidates) < 2:
+        return []
+    groups = _cluster_candidate_groups(
+        candidates,
+        similarity_threshold=similarity_threshold,
+        min_shared_tokens=min_shared_tokens,
+        max_year_delta=max_year_delta,
+    )
+    clusters = [_near_duplicate_cluster(group) for group in groups]
+    return sorted(clusters, key=lambda item: (-len(item.members), item.winner_publication_id))
+
+
+async def merge_near_duplicate_cluster(
+    db_session: AsyncSession,
+    *,
+    cluster: NearDuplicateCluster,
+) -> int:
+    winner_id = int(cluster.winner_publication_id)
+    merged = 0
+    for member in cluster.members:
+        if int(member.publication_id) == winner_id:
+            continue
+        await merge_duplicate_publication(
+            db_session,
+            winner_id=winner_id,
+            dup_id=int(member.publication_id),
+        )
+        merged += 1
+    return merged
+
+
+def near_duplicate_cluster_payload(cluster: NearDuplicateCluster) -> dict[str, object]:
+    members = [
+        {
+            "publication_id": int(member.publication_id),
+            "title": member.title,
+            "year": member.year,
+            "citation_count": int(member.citation_count),
+        }
+        for member in cluster.members
+    ]
+    return {
+        "cluster_key": cluster.cluster_key,
+        "winner_publication_id": int(cluster.winner_publication_id),
+        "member_count": len(cluster.members),
+        "similarity_score": float(cluster.similarity_score),
+        "members": members,
+    }
+
+
+async def _load_near_duplicate_candidates(
+    db_session: AsyncSession,
+) -> list[_NearDuplicateCandidate]:
+    result = await db_session.execute(
+        select(
+            Publication.id,
+            Publication.title_raw,
+            Publication.year,
+            Publication.citation_count,
+        )
+    )
+    records = [
+        _candidate_from_row(
+            publication_id=int(publication_id),
+            title=str(title_raw or ""),
+            year=year,
+            citation_count=int(citation_count or 0),
+        )
+        for publication_id, title_raw, year, citation_count in result.all()
+    ]
+    return [record for record in records if record is not None]
+
+
+def _candidate_from_row(
+    *,
+    publication_id: int,
+    title: str,
+    year: int | None,
+    citation_count: int,
+) -> _NearDuplicateCandidate | None:
+    canonical = canonical_title_text_for_dedup(title)
+    raw_tokens = canonical_title_tokens_for_dedup(title)
+    tokens = _normalized_tokens(raw_tokens)
+    if not canonical or not tokens:
+        return None
+    return _NearDuplicateCandidate(
+        publication_id=publication_id,
+        title=title,
+        year=year,
+        citation_count=citation_count,
+        canonical_text=canonical,
+        tokens=frozenset(tokens),
+    )
+
+
+def _normalized_tokens(tokens: Iterable[str]) -> set[str]:
+    return {token for token in tokens if len(token) >= NEAR_DUP_MIN_TOKEN_LENGTH and token not in NEAR_DUP_STOPWORDS}
+
+
+def _cluster_candidate_groups(
+    candidates: list[_NearDuplicateCandidate],
+    *,
+    similarity_threshold: float,
+    min_shared_tokens: int,
+    max_year_delta: int,
+) -> list[list[_NearDuplicateCandidate]]:
+    by_id = {candidate.publication_id: candidate for candidate in candidates}
+    token_index = _candidate_token_index(candidates)
+    parent = {candidate.publication_id: candidate.publication_id for candidate in candidates}
+    for candidate in candidates:
+        peers = _candidate_peer_ids(candidate=candidate, token_index=token_index)
+        for peer_id in sorted(peers):
+            if peer_id <= candidate.publication_id:
+                continue
+            peer = by_id[peer_id]
+            if _is_near_duplicate_pair(
+                candidate,
+                peer,
+                similarity_threshold=similarity_threshold,
+                min_shared_tokens=min_shared_tokens,
+                max_year_delta=max_year_delta,
+            ):
+                _union(parent, candidate.publication_id, peer_id)
+    return _grouped_candidates(candidates, parent)
+
+
+def _candidate_token_index(
+    candidates: list[_NearDuplicateCandidate],
+) -> dict[str, set[int]]:
+    index: dict[str, set[int]] = {}
+    for candidate in candidates:
+        for token in candidate.tokens:
+            index.setdefault(token, set()).add(candidate.publication_id)
+    return index
+
+
+def _candidate_peer_ids(
+    *,
+    candidate: _NearDuplicateCandidate,
+    token_index: dict[str, set[int]],
+) -> set[int]:
+    peers: set[int] = set()
+    for token in candidate.tokens:
+        peers.update(token_index.get(token, set()))
+    peers.discard(candidate.publication_id)
+    return peers
+
+
+def _is_near_duplicate_pair(
+    left: _NearDuplicateCandidate,
+    right: _NearDuplicateCandidate,
+    *,
+    similarity_threshold: float,
+    min_shared_tokens: int,
+    max_year_delta: int,
+) -> bool:
+    if left.canonical_text == right.canonical_text:
+        return True
+    if not _years_compatible(left.year, right.year, max_year_delta=max_year_delta):
+        return False
+    shared_tokens = len(left.tokens & right.tokens)
+    if shared_tokens < min_shared_tokens:
+        return False
+    jaccard = _jaccard(left.tokens, right.tokens)
+    containment = shared_tokens / max(1, min(len(left.tokens), len(right.tokens)))
+    return jaccard >= similarity_threshold or containment >= NEAR_DUP_DEFAULT_CONTAINMENT_THRESHOLD
+
+
+def _years_compatible(left: int | None, right: int | None, *, max_year_delta: int) -> bool:
+    if left is None or right is None:
+        return True
+    return abs(int(left) - int(right)) <= int(max_year_delta)
+
+
+def _jaccard(left: frozenset[str], right: frozenset[str]) -> float:
+    if not left or not right:
+        return 0.0
+    return len(left & right) / len(left | right)
+
+
+def _find_root(parent: dict[int, int], value: int) -> int:
+    root = parent[value]
+    while root != parent[root]:
+        root = parent[root]
+    while value != root:
+        next_value = parent[value]
+        parent[value] = root
+        value = next_value
+    return root
+
+
+def _union(parent: dict[int, int], left: int, right: int) -> None:
+    left_root = _find_root(parent, left)
+    right_root = _find_root(parent, right)
+    if left_root == right_root:
+        return
+    if left_root < right_root:
+        parent[right_root] = left_root
+        return
+    parent[left_root] = right_root
+
+
+def _grouped_candidates(
+    candidates: list[_NearDuplicateCandidate],
+    parent: dict[int, int],
+) -> list[list[_NearDuplicateCandidate]]:
+    groups: dict[int, list[_NearDuplicateCandidate]] = {}
+    for candidate in candidates:
+        root = _find_root(parent, candidate.publication_id)
+        groups.setdefault(root, []).append(candidate)
+    clustered = [members for members in groups.values() if len(members) > 1]
+    for members in clustered:
+        members.sort(key=lambda item: item.publication_id)
+    return clustered
+
+
+def _near_duplicate_cluster(members: list[_NearDuplicateCandidate]) -> NearDuplicateCluster:
+    winner = _winner_candidate(members)
+    member_ids = [member.publication_id for member in members]
+    joined = ",".join(str(publication_id) for publication_id in member_ids)
+    cluster_key = hashlib.sha256(joined.encode("utf-8")).hexdigest()[:NEAR_DUP_CLUSTER_KEY_LENGTH]
+    similarity_score = _cluster_similarity_score(members)
+    return NearDuplicateCluster(
+        cluster_key=cluster_key,
+        winner_publication_id=winner.publication_id,
+        similarity_score=similarity_score,
+        members=tuple(
+            NearDuplicateMember(
+                publication_id=member.publication_id,
+                title=member.title,
+                year=member.year,
+                citation_count=member.citation_count,
+            )
+            for member in members
+        ),
+    )
+
+
+def _winner_candidate(members: list[_NearDuplicateCandidate]) -> _NearDuplicateCandidate:
+    return min(
+        members,
+        key=lambda member: (-int(member.citation_count), member.publication_id),
+    )
+
+
+def _cluster_similarity_score(members: list[_NearDuplicateCandidate]) -> float:
+    best = 0.0
+    for index, left in enumerate(members):
+        for right in members[index + 1 :]:
+            shared_tokens = len(left.tokens & right.tokens)
+            jaccard = _jaccard(left.tokens, right.tokens)
+            containment = shared_tokens / max(1, min(len(left.tokens), len(right.tokens)))
+            best = max(best, jaccard, containment)
+    return round(best, 4)

app/services/publications/enrichment.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import logging
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.logging_utils import structured_log
+from app.services.domains.publications.pdf_queue import (
+    enqueue_missing_pdf_jobs,
+    enqueue_retry_pdf_job,
+    overlay_pdf_job_state,
+)
+from app.services.domains.publications.types import PublicationListItem
+
+logger = logging.getLogger(__name__)
+
+
+async def schedule_missing_pdf_enrichment_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    request_email: str | None,
+    items: list[PublicationListItem],
+    max_items: int,
+) -> int:
+    queued_ids = await enqueue_missing_pdf_jobs(
+        db_session,
+        user_id=user_id,
+        request_email=request_email,
+        rows=items,
+        max_items=max_items,
+    )
+    structured_log(
+        logger, "info", "publications.enrichment.scheduled", user_id=user_id, publication_count=len(queued_ids)
+    )
+    return len(queued_ids)
+
+
+async def schedule_retry_pdf_enrichment_for_row(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    request_email: str | None,
+    item: PublicationListItem,
+) -> bool:
+    queued = await enqueue_retry_pdf_job(
+        db_session,
+        user_id=user_id,
+        request_email=request_email,
+        row=item,
+    )
+    structured_log(
+        logger,
+        "info",
+        "publications.enrichment.retry_scheduled",
+        user_id=user_id,
+        publication_id=item.publication_id,
+        queued=queued,
+    )
+    return queued
+
+
+async def hydrate_pdf_enrichment_state(
+    db_session: AsyncSession,
+    *,
+    items: list[PublicationListItem],
+) -> list[PublicationListItem]:
+    return await overlay_pdf_job_state(db_session, rows=items)

app/services/publications/listing.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.services.domains.publication_identifiers import application as identifier_service
+from app.services.domains.publications.modes import (
+    MODE_ALL,
+    MODE_UNREAD,
+    resolve_publication_view_mode,
+)
+from app.services.domains.publications.queries import (
+    get_latest_run_id_for_user,
+    get_publication_item_for_user,
+    publication_list_item_from_row,
+    publications_query,
+    unread_item_from_row,
+)
+from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+
+
+async def list_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    mode: str = MODE_ALL,
+    scholar_profile_id: int | None = None,
+    favorite_only: bool = False,
+    search: str | None = None,
+    sort_by: str = "first_seen",
+    sort_dir: str = "desc",
+    limit: int = 100,
+    offset: int = 0,
+    snapshot_before: datetime | None = None,
+) -> list[PublicationListItem]:
+    resolved_mode = resolve_publication_view_mode(mode)
+    latest_run_id = await get_latest_run_id_for_user(db_session, user_id=user_id)
+    result = await db_session.execute(
+        publications_query(
+            user_id=user_id,
+            mode=resolved_mode,
+            latest_run_id=latest_run_id,
+            scholar_profile_id=scholar_profile_id,
+            favorite_only=favorite_only,
+            search=search,
+            sort_by=sort_by,
+            sort_dir=sort_dir,
+            limit=limit,
+            offset=offset,
+            snapshot_before=snapshot_before,
+        )
+    )
+    rows = [publication_list_item_from_row(row, latest_run_id=latest_run_id) for row in result.all()]
+    return await identifier_service.overlay_publication_items_with_display_identifiers(
+        db_session,
+        items=rows,
+    )
+
+
+async def retry_pdf_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+    publication_id: int,
+) -> PublicationListItem | None:
+    item = await get_publication_item_for_user(
+        db_session,
+        user_id=user_id,
+        scholar_profile_id=scholar_profile_id,
+        publication_id=publication_id,
+    )
+    if item is None:
+        return None
+    hydrated = await identifier_service.overlay_publication_items_with_display_identifiers(
+        db_session,
+        items=[item],
+    )
+    return hydrated[0] if hydrated else item
+
+
+async def list_unread_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    limit: int = 100,
+) -> list[UnreadPublicationItem]:
+    result = await db_session.execute(
+        publications_query(
+            user_id=user_id,
+            mode=MODE_UNREAD,
+            latest_run_id=None,
+            scholar_profile_id=None,
+            favorite_only=False,
+            limit=limit,
+            offset=0,
+        )
+    )
+    return [unread_item_from_row(row) for row in result.all()]

app/services/publications/modes.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+MODE_ALL = "all"
+MODE_UNREAD = "unread"
+MODE_LATEST = "latest"
+MODE_NEW = "new"  # compatibility alias for MODE_LATEST
+
+
+def resolve_publication_view_mode(value: str | None) -> str:
+    if value == MODE_UNREAD:
+        return MODE_UNREAD
+    if value in {MODE_LATEST, MODE_NEW}:
+        return MODE_LATEST
+    return MODE_ALL

app/services/publications/pdf_queue.py

@@ -0,0 +1,969 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import Select, and_, func, literal, or_, select, union_all
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import (
+    Publication,
+    PublicationPdfJob,
+    PublicationPdfJobEvent,
+    ScholarProfile,
+    ScholarPublication,
+    User,
+)
+from app.db.session import get_session_factory
+from app.logging_utils import structured_log
+from app.services.domains.publication_identifiers import application as identifier_service
+from app.services.domains.publication_identifiers.types import DisplayIdentifier
+from app.services.domains.publications.pdf_resolution_pipeline import (
+    resolve_publication_pdf_outcome_for_row,
+)
+from app.services.domains.publications.types import PublicationListItem
+from app.services.domains.unpaywall.application import (
+    FAILURE_RESOLUTION_EXCEPTION,
+    OaResolutionOutcome,
+)
+from app.settings import settings
+
+PDF_STATUS_UNTRACKED = "untracked"
+PDF_STATUS_QUEUED = "queued"
+PDF_STATUS_RUNNING = "running"
+PDF_STATUS_RESOLVED = "resolved"
+PDF_STATUS_FAILED = "failed"
+
+PDF_EVENT_QUEUED = "queued"
+PDF_EVENT_ATTEMPT_STARTED = "attempt_started"
+PDF_EVENT_RESOLVED = "resolved"
+PDF_EVENT_FAILED = "failed"
+
+logger = logging.getLogger(__name__)
+_scheduled_tasks: set[asyncio.Task[None]] = set()
+
+
+@dataclass(frozen=True)
+class PdfQueueListItem:
+    publication_id: int
+    title: str
+    pdf_url: str | None
+    status: str
+    attempt_count: int
+    last_failure_reason: str | None
+    last_failure_detail: str | None
+    last_source: str | None
+    requested_by_user_id: int | None
+    requested_by_email: str | None
+    queued_at: datetime | None
+    last_attempt_at: datetime | None
+    resolved_at: datetime | None
+    updated_at: datetime
+    display_identifier: DisplayIdentifier | None = None
+
+
+@dataclass(frozen=True)
+class PdfRequeueResult:
+    publication_exists: bool
+    queued: bool
+
+
+@dataclass(frozen=True)
+class PdfBulkQueueResult:
+    requested_count: int
+    queued_count: int
+
+
+@dataclass(frozen=True)
+class PdfQueuePage:
+    items: list[PdfQueueListItem]
+    total_count: int
+    limit: int
+    offset: int
+
+
+def _utcnow() -> datetime:
+    return datetime.now(UTC)
+
+
+def _publication_ids(rows: list[PublicationListItem]) -> list[int]:
+    return sorted({row.publication_id for row in rows})
+
+
+def _status_from_job(row: PublicationListItem, job: PublicationPdfJob | None) -> str:
+    if row.pdf_url:
+        return PDF_STATUS_RESOLVED
+    if job is None:
+        return PDF_STATUS_UNTRACKED
+    return job.status
+
+
+def _item_from_row_and_job(
+    row: PublicationListItem,
+    job: PublicationPdfJob | None,
+) -> PublicationListItem:
+    return PublicationListItem(
+        publication_id=row.publication_id,
+        scholar_profile_id=row.scholar_profile_id,
+        scholar_label=row.scholar_label,
+        title=row.title,
+        year=row.year,
+        citation_count=row.citation_count,
+        venue_text=row.venue_text,
+        pub_url=row.pub_url,
+        pdf_url=row.pdf_url,
+        is_read=row.is_read,
+        is_favorite=row.is_favorite,
+        first_seen_at=row.first_seen_at,
+        is_new_in_latest_run=row.is_new_in_latest_run,
+        pdf_status=_status_from_job(row, job),
+        pdf_attempt_count=int(job.attempt_count) if job is not None else 0,
+        pdf_failure_reason=job.last_failure_reason if job is not None else None,
+        pdf_failure_detail=job.last_failure_detail if job is not None else None,
+        display_identifier=row.display_identifier,
+    )
+
+
+def _queueable_rows(
+    rows: list[PublicationListItem],
+    *,
+    max_items: int,
+) -> list[PublicationListItem]:
+    bounded = max(0, int(max_items))
+    if bounded == 0:
+        return []
+    candidates = [row for row in rows if not row.pdf_url]
+    return candidates[:bounded]
+
+
+def _bounded_limit(limit: int, *, max_value: int = 500) -> int:
+    return max(1, min(int(limit), max_value))
+
+
+def _bounded_offset(offset: int) -> int:
+    return max(int(offset), 0)
+
+
+def _auto_retry_interval_seconds() -> int:
+    return max(int(settings.pdf_auto_retry_interval_seconds), 1)
+
+
+def _auto_retry_first_interval_seconds() -> int:
+    return max(int(settings.pdf_auto_retry_first_interval_seconds), 1)
+
+
+def _auto_retry_max_attempts() -> int:
+    return max(int(settings.pdf_auto_retry_max_attempts), 1)
+
+
+def _retry_interval_seconds_for_attempt_count(attempt_count: int) -> int:
+    if int(attempt_count) <= 1:
+        return _auto_retry_first_interval_seconds()
+    return _auto_retry_interval_seconds()
+
+
+def _cooldown_active(
+    *,
+    last_attempt_at: datetime | None,
+    attempt_count: int,
+) -> bool:
+    if last_attempt_at is None:
+        return False
+    elapsed = (_utcnow() - last_attempt_at).total_seconds()
+    return elapsed < _retry_interval_seconds_for_attempt_count(int(attempt_count))
+
+
+def _can_enqueue_job(
+    job: PublicationPdfJob | None,
+    *,
+    force_retry: bool,
+) -> bool:
+    if job is None:
+        return True
+    if job.status in {PDF_STATUS_QUEUED, PDF_STATUS_RUNNING}:
+        return False
+    if force_retry:
+        return job.status in {PDF_STATUS_FAILED, PDF_STATUS_RESOLVED, PDF_STATUS_UNTRACKED}
+    if job.status == PDF_STATUS_RESOLVED:
+        return False
+    if int(job.attempt_count) >= _auto_retry_max_attempts():
+        return False
+    return not _cooldown_active(
+        last_attempt_at=job.last_attempt_at,
+        attempt_count=int(job.attempt_count),
+    )
+
+
+def _event_row(
+    *,
+    publication_id: int,
+    user_id: int | None,
+    event_type: str,
+    status: str | None,
+    source: str | None = None,
+    failure_reason: str | None = None,
+    message: str | None = None,
+) -> PublicationPdfJobEvent:
+    return PublicationPdfJobEvent(
+        publication_id=publication_id,
+        user_id=user_id,
+        event_type=event_type,
+        status=status,
+        source=source,
+        failure_reason=failure_reason,
+        message=message,
+    )
+
+
+def _queued_job(
+    *,
+    publication_id: int,
+    user_id: int,
+) -> PublicationPdfJob:
+    now = _utcnow()
+    return PublicationPdfJob(
+        publication_id=publication_id,
+        status=PDF_STATUS_QUEUED,
+        queued_at=now,
+        last_requested_by_user_id=user_id,
+    )
+
+
+def _mark_job_queued(job: PublicationPdfJob, *, user_id: int) -> None:
+    now = _utcnow()
+    job.status = PDF_STATUS_QUEUED
+    job.queued_at = now
+    job.last_requested_by_user_id = user_id
+    job.last_failure_reason = None
+    job.last_failure_detail = None
+    job.last_source = None
+
+
+def _state_map(jobs: list[PublicationPdfJob]) -> dict[int, PublicationPdfJob]:
+    return {int(job.publication_id): job for job in jobs}
+
+
+async def _jobs_for_publication_ids(
+    db_session: AsyncSession,
+    *,
+    publication_ids: list[int],
+) -> dict[int, PublicationPdfJob]:
+    if not publication_ids:
+        return {}
+    result = await db_session.execute(
+        select(PublicationPdfJob).where(PublicationPdfJob.publication_id.in_(publication_ids))
+    )
+    return _state_map(list(result.scalars()))
+
+
+async def overlay_pdf_job_state(
+    db_session: AsyncSession,
+    *,
+    rows: list[PublicationListItem],
+) -> list[PublicationListItem]:
+    if not rows:
+        return []
+    jobs = await _jobs_for_publication_ids(
+        db_session,
+        publication_ids=_publication_ids(rows),
+    )
+    return [_item_from_row_and_job(row, jobs.get(row.publication_id)) for row in rows]
+
+
+async def _enqueue_rows(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    rows: list[PublicationListItem],
+    force_retry: bool,
+) -> list[PublicationListItem]:
+    if not rows:
+        return []
+    queued: list[PublicationListItem] = []
+    jobs = await _jobs_for_publication_ids(
+        db_session,
+        publication_ids=_publication_ids(rows),
+    )
+    for row in rows:
+        job = jobs.get(row.publication_id)
+        if not _can_enqueue_job(job, force_retry=force_retry):
+            continue
+        if job is None:
+            job = _queued_job(publication_id=row.publication_id, user_id=user_id)
+            jobs[row.publication_id] = job
+            db_session.add(job)
+        else:
+            _mark_job_queued(job, user_id=user_id)
+        db_session.add(
+            _event_row(
+                publication_id=row.publication_id,
+                user_id=user_id,
+                event_type=PDF_EVENT_QUEUED,
+                status=PDF_STATUS_QUEUED,
+            )
+        )
+        queued.append(row)
+    if queued:
+        await db_session.commit()
+    return queued
+
+
+def _register_task(task: asyncio.Task[None]) -> None:
+    _scheduled_tasks.add(task)
+
+
+def _drop_finished_task(task: asyncio.Task[None]) -> None:
+    _scheduled_tasks.discard(task)
+    try:
+        task.result()
+    except Exception:
+        logger.exception("publications.pdf_queue.task_failed")
+
+
+async def _mark_attempt_started(
+    *,
+    publication_id: int,
+    user_id: int,
+) -> None:
+    session_factory = get_session_factory()
+    async with session_factory() as db_session:
+        job = await db_session.get(PublicationPdfJob, publication_id)
+        if job is None:
+            job = _queued_job(publication_id=publication_id, user_id=user_id)
+            db_session.add(job)
+        job.status = PDF_STATUS_RUNNING
+        job.last_attempt_at = _utcnow()
+        job.attempt_count = int(job.attempt_count) + 1
+        db_session.add(
+            _event_row(
+                publication_id=publication_id,
+                user_id=user_id,
+                event_type=PDF_EVENT_ATTEMPT_STARTED,
+                status=PDF_STATUS_RUNNING,
+            )
+        )
+        await db_session.commit()
+
+
+def _failed_outcome(
+    *,
+    row: PublicationListItem,
+) -> OaResolutionOutcome:
+    return OaResolutionOutcome(
+        publication_id=row.publication_id,
+        doi=None,
+        pdf_url=None,
+        failure_reason=FAILURE_RESOLUTION_EXCEPTION,
+        source=None,
+        used_crossref=False,
+    )
+
+
+async def _fetch_outcome_for_row(
+    *,
+    row: PublicationListItem,
+    request_email: str | None,
+    openalex_api_key: str | None = None,
+    allow_arxiv_lookup: bool = True,
+) -> tuple[OaResolutionOutcome, bool]:
+    pipeline_result = await resolve_publication_pdf_outcome_for_row(
+        row=row,
+        request_email=request_email,
+        openalex_api_key=openalex_api_key,
+        allow_arxiv_lookup=allow_arxiv_lookup,
+    )
+    outcome = pipeline_result.outcome
+    if outcome is not None:
+        return outcome, bool(pipeline_result.arxiv_rate_limited)
+    return _failed_outcome(row=row), bool(pipeline_result.arxiv_rate_limited)
+
+
+def _apply_publication_update(
+    publication: Publication,
+    *,
+    pdf_url: str | None,
+) -> None:
+    if pdf_url and publication.pdf_url != pdf_url:
+        publication.pdf_url = pdf_url
+
+
+def _apply_job_outcome(job: PublicationPdfJob, *, outcome: OaResolutionOutcome) -> None:
+    job.last_source = outcome.source
+    if outcome.pdf_url:
+        job.status = PDF_STATUS_RESOLVED
+        job.resolved_at = _utcnow()
+        job.last_failure_reason = None
+        job.last_failure_detail = None
+        return
+    job.status = PDF_STATUS_FAILED
+    job.last_failure_reason = outcome.failure_reason
+    job.last_failure_detail = outcome.failure_reason
+
+
+def _result_event(outcome: OaResolutionOutcome) -> tuple[str, str]:
+    if outcome.pdf_url:
+        return PDF_EVENT_RESOLVED, PDF_STATUS_RESOLVED
+    return PDF_EVENT_FAILED, PDF_STATUS_FAILED
+
+
+async def _persist_outcome(
+    *,
+    publication_id: int,
+    user_id: int,
+    outcome: OaResolutionOutcome,
+) -> None:
+    session_factory = get_session_factory()
+    async with session_factory() as db_session:
+        publication = await db_session.get(Publication, publication_id)
+        job = await db_session.get(PublicationPdfJob, publication_id)
+        if publication is None or job is None:
+            return
+        _apply_publication_update(publication, pdf_url=outcome.pdf_url)
+        await identifier_service.sync_identifiers_for_publication_resolution(
+            db_session,
+            publication=publication,
+            source=outcome.source,
+        )
+        _apply_job_outcome(job, outcome=outcome)
+        event_type, status = _result_event(outcome)
+        db_session.add(
+            _event_row(
+                publication_id=publication_id,
+                user_id=user_id,
+                event_type=event_type,
+                status=status,
+                source=outcome.source,
+                failure_reason=outcome.failure_reason,
+                message=outcome.failure_reason,
+            )
+        )
+        await db_session.commit()
+
+
+async def _resolve_publication_row(
+    *,
+    user_id: int,
+    request_email: str | None,
+    row: PublicationListItem,
+    openalex_api_key: str | None = None,
+    allow_arxiv_lookup: bool = True,
+) -> bool:
+    from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
+
+    await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
+    try:
+        outcome, arxiv_rate_limited = await _fetch_outcome_for_row(
+            row=row,
+            request_email=request_email,
+            openalex_api_key=openalex_api_key,
+            allow_arxiv_lookup=allow_arxiv_lookup,
+        )
+    except OpenAlexBudgetExhaustedError:
+        # Persist a terminal outcome so jobs do not remain stuck in "running".
+        await _persist_outcome(
+            publication_id=row.publication_id,
+            user_id=user_id,
+            outcome=_failed_outcome(row=row),
+        )
+        # Propagate upward so the batch loop can stop immediately.
+        raise
+    except Exception as exc:  # pragma: no cover - defensive network boundary
+        structured_log(
+            logger,
+            "warning",
+            "publications.pdf_queue.resolve_failed",
+            publication_id=row.publication_id,
+            error=str(exc),
+        )
+        outcome = _failed_outcome(row=row)
+        arxiv_rate_limited = False
+    await _persist_outcome(
+        publication_id=row.publication_id,
+        user_id=user_id,
+        outcome=outcome,
+    )
+    return bool(arxiv_rate_limited)
+
+
+async def _run_resolution_task(
+    *,
+    user_id: int,
+    request_email: str | None,
+    rows: list[PublicationListItem],
+) -> None:
+    from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
+    from app.services.domains.settings import application as user_settings_service
+
+    # Resolve the best available API key: per-user setting → env var fallback.
+    openalex_api_key: str | None = None
+    try:
+        session_factory = get_session_factory()
+        async with session_factory() as key_session:
+            user_settings = await user_settings_service.get_or_create_settings(key_session, user_id=user_id)
+            openalex_api_key = getattr(user_settings, "openalex_api_key", None) or settings.openalex_api_key
+    except Exception:
+        openalex_api_key = settings.openalex_api_key
+
+    arxiv_lookup_allowed = True
+    for row in rows:
+        try:
+            arxiv_rate_limited = await _resolve_publication_row(
+                user_id=user_id,
+                request_email=request_email,
+                row=row,
+                openalex_api_key=openalex_api_key,
+                allow_arxiv_lookup=arxiv_lookup_allowed,
+            )
+            if arxiv_rate_limited and arxiv_lookup_allowed:
+                arxiv_lookup_allowed = False
+                structured_log(
+                    logger,
+                    "warning",
+                    "pdf_queue.arxiv_batch_disabled",
+                    detail="arXiv temporarily disabled for remaining batch after rate limit",
+                )
+        except OpenAlexBudgetExhaustedError:
+            structured_log(
+                logger,
+                "warning",
+                "pdf_queue.budget_exhausted",
+                detail="Stopping PDF resolution batch — OpenAlex daily budget exhausted",
+            )
+            break
+
+
+def _schedule_rows(
+    *,
+    user_id: int,
+    request_email: str | None,
+    rows: list[PublicationListItem],
+) -> None:
+    if not rows:
+        return
+    task = asyncio.create_task(
+        _run_resolution_task(
+            user_id=user_id,
+            request_email=request_email,
+            rows=rows,
+        )
+    )
+    _register_task(task)
+    task.add_done_callback(_drop_finished_task)
+
+
+async def enqueue_missing_pdf_jobs(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    request_email: str | None,
+    rows: list[PublicationListItem],
+    max_items: int,
+) -> list[int]:
+    queueable = _queueable_rows(rows, max_items=max_items)
+    queued_rows = await _enqueue_rows(
+        db_session,
+        user_id=user_id,
+        rows=queueable,
+        force_retry=False,
+    )
+    _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
+    return [row.publication_id for row in queued_rows]
+
+
+async def enqueue_retry_pdf_job(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    request_email: str | None,
+    row: PublicationListItem,
+) -> bool:
+    queued_rows = await _enqueue_rows(
+        db_session,
+        user_id=user_id,
+        rows=[row],
+        force_retry=True,
+    )
+    _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
+    return bool(queued_rows)
+
+
+def _retry_item_label(display_name: str | None, scholar_id: str | None) -> str:
+    return str(display_name or scholar_id or "unknown")
+
+
+def _retry_item_from_publication(
+    publication: Publication,
+    *,
+    link_row: tuple | None,
+) -> PublicationListItem:
+    if link_row is None:
+        scholar_profile_id = 0
+        scholar_label = "unknown"
+        is_read = True
+        first_seen_at = publication.created_at or _utcnow()
+    else:
+        scholar_profile_id = int(link_row[0])
+        scholar_label = _retry_item_label(link_row[1], link_row[2])
+        is_read = bool(link_row[3])
+        first_seen_at = link_row[4] or publication.created_at or _utcnow()
+    return PublicationListItem(
+        publication_id=int(publication.id),
+        scholar_profile_id=scholar_profile_id,
+        scholar_label=scholar_label,
+        title=publication.title_raw,
+        year=publication.year,
+        citation_count=int(publication.citation_count or 0),
+        venue_text=publication.venue_text,
+        pub_url=publication.pub_url,
+        pdf_url=publication.pdf_url,
+        is_read=is_read,
+        first_seen_at=first_seen_at,
+        is_new_in_latest_run=False,
+    )
+
+
+async def _retry_item_for_publication_id(
+    db_session: AsyncSession,
+    *,
+    publication_id: int,
+) -> PublicationListItem | None:
+    publication = await db_session.get(Publication, publication_id)
+    if publication is None:
+        return None
+    result = await db_session.execute(
+        select(
+            ScholarProfile.id,
+            ScholarProfile.display_name,
+            ScholarProfile.scholar_id,
+            ScholarPublication.is_read,
+            ScholarPublication.created_at,
+        )
+        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
+        .where(ScholarPublication.publication_id == publication_id)
+        .order_by(ScholarPublication.created_at.asc())
+        .limit(1)
+    )
+    return _retry_item_from_publication(publication, link_row=result.one_or_none())
+
+
+async def enqueue_retry_pdf_job_for_publication_id(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    request_email: str | None,
+    publication_id: int,
+) -> PdfRequeueResult:
+    row = await _retry_item_for_publication_id(
+        db_session,
+        publication_id=publication_id,
+    )
+    if row is None:
+        return PdfRequeueResult(publication_exists=False, queued=False)
+    queued = await enqueue_retry_pdf_job(
+        db_session,
+        user_id=user_id,
+        request_email=request_email,
+        row=row,
+    )
+    return PdfRequeueResult(publication_exists=True, queued=queued)
+
+
+def _queue_candidate_from_publication(publication: Publication) -> PublicationListItem:
+    return PublicationListItem(
+        publication_id=int(publication.id),
+        scholar_profile_id=0,
+        scholar_label="",
+        title=publication.title_raw,
+        year=publication.year,
+        citation_count=int(publication.citation_count or 0),
+        venue_text=publication.venue_text,
+        pub_url=publication.pub_url,
+        pdf_url=publication.pdf_url,
+        is_read=True,
+        first_seen_at=publication.created_at or _utcnow(),
+        is_new_in_latest_run=False,
+    )
+
+
+async def _missing_pdf_candidates(
+    db_session: AsyncSession,
+    *,
+    limit: int,
+) -> list[PublicationListItem]:
+    bounded_limit = max(1, min(int(limit), 5000))
+    now = datetime.now(UTC)
+    cooldown_threshold = now - timedelta(days=7)
+
+    result = await db_session.execute(
+        select(Publication)
+        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
+        .where(Publication.pdf_url.is_(None))
+        .where(
+            or_(
+                PublicationPdfJob.publication_id.is_(None),
+                and_(
+                    PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
+                    or_(
+                        PublicationPdfJob.last_attempt_at.is_(None),
+                        PublicationPdfJob.last_attempt_at < cooldown_threshold,
+                    ),
+                ),
+            )
+        )
+        .order_by(Publication.updated_at.desc(), Publication.id.desc())
+        .limit(bounded_limit)
+    )
+    return [_queue_candidate_from_publication(publication) for publication in result.scalars()]
+
+
+async def enqueue_all_missing_pdf_jobs(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    request_email: str | None,
+    limit: int = 1000,
+) -> PdfBulkQueueResult:
+    candidates = await _missing_pdf_candidates(db_session, limit=limit)
+    queued_rows = await _enqueue_rows(
+        db_session,
+        user_id=user_id,
+        rows=candidates,
+        force_retry=True,
+    )
+    _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
+    return PdfBulkQueueResult(
+        requested_count=len(candidates),
+        queued_count=len(queued_rows),
+    )
+
+
+def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]:
+    stmt = (
+        select(
+            PublicationPdfJob.publication_id,
+            Publication.title_raw,
+            Publication.pdf_url,
+            PublicationPdfJob.status,
+            PublicationPdfJob.attempt_count,
+            PublicationPdfJob.last_failure_reason,
+            PublicationPdfJob.last_failure_detail,
+            PublicationPdfJob.last_source,
+            PublicationPdfJob.last_requested_by_user_id,
+            User.email,
+            PublicationPdfJob.queued_at,
+            PublicationPdfJob.last_attempt_at,
+            PublicationPdfJob.resolved_at,
+            PublicationPdfJob.updated_at,
+        )
+        .join(Publication, Publication.id == PublicationPdfJob.publication_id)
+        .outerjoin(User, User.id == PublicationPdfJob.last_requested_by_user_id)
+    )
+    if status:
+        stmt = stmt.where(PublicationPdfJob.status == status)
+    return stmt
+
+
+def _tracked_queue_select(*, limit: int, offset: int, status: str | None) -> Select[tuple]:
+    return (
+        _tracked_queue_select_base(status=status)
+        .order_by(PublicationPdfJob.updated_at.desc())
+        .limit(_bounded_limit(limit))
+        .offset(_bounded_offset(offset))
+    )
+
+
+def _untracked_queue_select_base() -> Select[tuple]:
+    return (
+        select(
+            Publication.id,
+            Publication.title_raw,
+            Publication.pdf_url,
+            literal(PDF_STATUS_UNTRACKED),
+            literal(0),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            Publication.updated_at,
+        )
+        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
+        .where(Publication.pdf_url.is_(None))
+        .where(PublicationPdfJob.publication_id.is_(None))
+    )
+
+
+def _untracked_queue_select(*, limit: int, offset: int) -> Select[tuple]:
+    return (
+        _untracked_queue_select_base()
+        .order_by(Publication.updated_at.desc(), Publication.id.desc())
+        .limit(_bounded_limit(limit))
+        .offset(_bounded_offset(offset))
+    )
+
+
+def _all_queue_select(*, limit: int, offset: int) -> Select[tuple]:
+    union_stmt = union_all(
+        _tracked_queue_select_base(status=None),
+        _untracked_queue_select_base(),
+    ).subquery()
+    return (
+        select(union_stmt)
+        .order_by(union_stmt.c.updated_at.desc())
+        .limit(_bounded_limit(limit))
+        .offset(_bounded_offset(offset))
+    )
+
+
+def _tracked_queue_count_select(*, status: str | None) -> Select[tuple]:
+    stmt = select(func.count()).select_from(PublicationPdfJob)
+    if status:
+        stmt = stmt.where(PublicationPdfJob.status == status)
+    return stmt
+
+
+def _untracked_queue_count_select() -> Select[tuple]:
+    return (
+        select(func.count())
+        .select_from(Publication)
+        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
+        .where(Publication.pdf_url.is_(None))
+        .where(PublicationPdfJob.publication_id.is_(None))
+    )
+
+
+def _queue_item_from_row(row: tuple) -> PdfQueueListItem:
+    return PdfQueueListItem(
+        publication_id=int(row[0]),
+        title=str(row[1] or ""),
+        pdf_url=row[2],
+        status=str(row[3] or PDF_STATUS_UNTRACKED),
+        attempt_count=int(row[4] or 0),
+        last_failure_reason=row[5],
+        last_failure_detail=row[6],
+        last_source=row[7],
+        requested_by_user_id=int(row[8]) if row[8] is not None else None,
+        requested_by_email=row[9],
+        queued_at=row[10],
+        last_attempt_at=row[11],
+        resolved_at=row[12],
+        updated_at=row[13],
+    )
+
+
+async def _hydrated_queue_items(
+    db_session: AsyncSession,
+    *,
+    rows: list[tuple],
+) -> list[PdfQueueListItem]:
+    items = [_queue_item_from_row(row) for row in rows]
+    return await identifier_service.overlay_pdf_queue_items_with_display_identifiers(
+        db_session,
+        items=items,
+    )
+
+
+async def list_pdf_queue_items(
+    db_session: AsyncSession,
+    *,
+    limit: int = 100,
+    offset: int = 0,
+    status: str | None = None,
+) -> list[PdfQueueListItem]:
+    bounded_limit = _bounded_limit(limit)
+    bounded_offset = _bounded_offset(offset)
+    normalized_status = (status or "").strip().lower() or None
+    if normalized_status == PDF_STATUS_UNTRACKED:
+        result = await db_session.execute(
+            _untracked_queue_select(
+                limit=bounded_limit,
+                offset=bounded_offset,
+            )
+        )
+        return await _hydrated_queue_items(db_session, rows=list(result.all()))
+    if normalized_status is None:
+        result = await db_session.execute(
+            _all_queue_select(
+                limit=bounded_limit,
+                offset=bounded_offset,
+            )
+        )
+        return await _hydrated_queue_items(db_session, rows=list(result.all()))
+    result = await db_session.execute(
+        _tracked_queue_select(
+            limit=bounded_limit,
+            offset=bounded_offset,
+            status=normalized_status,
+        )
+    )
+    return await _hydrated_queue_items(db_session, rows=list(result.all()))
+
+
+async def count_pdf_queue_items(
+    db_session: AsyncSession,
+    *,
+    status: str | None = None,
+) -> int:
+    normalized_status = (status or "").strip().lower() or None
+    if normalized_status == PDF_STATUS_UNTRACKED:
+        result = await db_session.execute(_untracked_queue_count_select())
+        return int(result.scalar_one() or 0)
+    tracked_result = await db_session.execute(_tracked_queue_count_select(status=normalized_status))
+    tracked_count = int(tracked_result.scalar_one() or 0)
+    if normalized_status is not None:
+        return tracked_count
+    untracked_result = await db_session.execute(_untracked_queue_count_select())
+    untracked_count = int(untracked_result.scalar_one() or 0)
+    return tracked_count + untracked_count
+
+
+async def list_pdf_queue_page(
+    db_session: AsyncSession,
+    *,
+    limit: int = 100,
+    offset: int = 0,
+    status: str | None = None,
+) -> PdfQueuePage:
+    bounded_limit = _bounded_limit(limit)
+    bounded_offset = _bounded_offset(offset)
+    items = await list_pdf_queue_items(
+        db_session,
+        limit=bounded_limit,
+        offset=bounded_offset,
+        status=status,
+    )
+    total_count = await count_pdf_queue_items(
+        db_session,
+        status=status,
+    )
+    return PdfQueuePage(
+        items=items,
+        total_count=total_count,
+        limit=bounded_limit,
+        offset=bounded_offset,
+    )
+
+
+async def drain_ready_jobs(
+    db_session: AsyncSession,
+    *,
+    limit: int,
+    max_attempts: int,
+) -> int:
+    result = await db_session.execute(select(User.id).where(User.is_active.is_(True)).order_by(User.id.asc()).limit(1))
+    system_user_id = result.scalar_one_or_none()
+    if system_user_id is None:
+        return 0
+
+    bulk_result = await enqueue_all_missing_pdf_jobs(
+        db_session,
+        user_id=system_user_id,
+        request_email=settings.unpaywall_email,
+        limit=limit,
+    )
+    return bulk_result.queued_count

app/services/publications/pdf_resolution_pipeline.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import Any
+
+from app.logging_utils import structured_log
+from app.services.domains.arxiv.errors import ArxivRateLimitError
+from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
+from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
+from app.services.domains.publications.types import PublicationListItem
+from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class PipelineOutcome:
+    outcome: OaResolutionOutcome | None
+    scholar_candidates: Any | None  # Kept for backward compatibility with calling signatures
+    arxiv_rate_limited: bool = False
+
+
+async def resolve_publication_pdf_outcome_for_row(
+    *,
+    row: PublicationListItem,
+    request_email: str | None,
+    openalex_api_key: str | None = None,
+    allow_arxiv_lookup: bool = True,
+) -> PipelineOutcome:
+    # 1. OpenAlex OA — raises OpenAlexBudgetExhaustedError if budget is gone
+    openalex_outcome = await _openalex_outcome(row, request_email=request_email, openalex_api_key=openalex_api_key)
+    if openalex_outcome and openalex_outcome.pdf_url:
+        return PipelineOutcome(openalex_outcome, None)
+
+    # 2. arXiv
+    arxiv_rate_limited = False
+    try:
+        arxiv_outcome = await _arxiv_outcome(
+            row,
+            request_email=request_email,
+            allow_lookup=allow_arxiv_lookup,
+        )
+    except ArxivRateLimitError:
+        arxiv_rate_limited = True
+        arxiv_outcome = None
+        structured_log(logger, "warning", "pdf_resolution.arxiv_rate_limited", publication_id=int(row.publication_id))
+    if arxiv_outcome and arxiv_outcome.pdf_url:
+        return PipelineOutcome(arxiv_outcome, None, arxiv_rate_limited=arxiv_rate_limited)
+
+    # 3. Unpaywall (which falls back to Crossref)
+    oa_outcome = await _oa_outcome(row=row, request_email=request_email)
+    return PipelineOutcome(oa_outcome, None, arxiv_rate_limited=arxiv_rate_limited)
+
+
+async def _openalex_outcome(
+    row: PublicationListItem,
+    request_email: str | None,
+    openalex_api_key: str | None = None,
+) -> OaResolutionOutcome | None:
+    from app.services.domains.openalex.client import OpenAlexClient
+    from app.services.domains.openalex.matching import find_best_match
+
+    if not row.title:
+        return None
+
+    import re
+
+    safe_title = re.sub(r"[^\w\s]", " ", row.title)
+    safe_title = " ".join(safe_title.split())
+    if not safe_title:
+        return None
+
+    api_key = openalex_api_key or settings.openalex_api_key
+    client = OpenAlexClient(api_key=api_key, mailto=request_email or settings.crossref_api_mailto)
+    try:
+        openalex_works = await client.get_works_by_filter({"title.search": safe_title}, limit=5)
+        match = find_best_match(
+            target_title=row.title,
+            target_year=row.year,
+            target_authors=row.scholar_label,
+            candidates=openalex_works,
+        )
+        if match and match.oa_url:
+            return OaResolutionOutcome(
+                publication_id=row.publication_id,
+                doi=match.doi,
+                pdf_url=match.oa_url,
+                failure_reason=None,
+                source="openalex",
+                used_crossref=False,
+            )
+    except OpenAlexBudgetExhaustedError:
+        # Re-raise so the caller's batch loop can stop hitting the API.
+        raise
+    except Exception as exc:
+        structured_log(logger, "warning", "pdf_resolution.openalex_failed", error=str(exc))
+    return None
+
+
+async def _arxiv_outcome(
+    row: PublicationListItem,
+    *,
+    request_email: str | None,
+    allow_lookup: bool = True,
+) -> OaResolutionOutcome | None:
+    from app.services.domains.arxiv.application import discover_arxiv_id_for_publication
+
+    if not allow_lookup:
+        structured_log(
+            logger,
+            "info",
+            "pdf_resolution.arxiv_skipped",
+            publication_id=int(row.publication_id),
+            skip_reason="batch_arxiv_cooldown_active",
+        )
+        return None
+
+    skip_reason = arxiv_skip_reason_for_item(item=row)
+    if skip_reason is not None:
+        structured_log(
+            logger,
+            "info",
+            "pdf_resolution.arxiv_skipped",
+            publication_id=int(row.publication_id),
+            skip_reason=skip_reason,
+        )
+        return None
+
+    try:
+        arxiv_id = await discover_arxiv_id_for_publication(item=row, request_email=request_email)
+        if arxiv_id:
+            pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
+            return OaResolutionOutcome(
+                publication_id=row.publication_id,
+                doi=None,
+                pdf_url=pdf_url,
+                failure_reason=None,
+                source="arxiv",
+                used_crossref=False,
+            )
+    except ArxivRateLimitError:
+        raise  # propagate so orchestration can switch to non-arXiv fallback
+    except Exception as exc:
+        structured_log(logger, "warning", "pdf_resolution.arxiv_failed", error=str(exc))
+    return None
+
+
+async def _oa_outcome(
+    *,
+    row: PublicationListItem,
+    request_email: str | None,
+) -> OaResolutionOutcome | None:
+    outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email)
+    return outcomes.get(row.publication_id)

app/services/publications/queries.py

@@ -0,0 +1,262 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy import Select, case, func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import (
+    CrawlRun,
+    Publication,
+    PublicationPdfJob,
+    RunStatus,
+    ScholarProfile,
+    ScholarPublication,
+)
+from app.services.domains.publications.modes import MODE_LATEST, MODE_UNREAD
+from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+
+
+def _normalized_citation_count(value: object) -> int:
+    try:
+        return int(value or 0)  # type: ignore[call-overload]  # intentionally accepts any object
+    except (TypeError, ValueError):
+        return 0
+
+
+def _pdf_status_sort_rank():
+    return case(
+        (Publication.pdf_url.is_not(None), 4),
+        (PublicationPdfJob.status == "resolved", 4),
+        (PublicationPdfJob.status == "running", 3),
+        (PublicationPdfJob.status == "queued", 2),
+        (PublicationPdfJob.status == "failed", 0),
+        else_=1,
+    )
+
+
+def _sort_column(sort_by: str):
+    sort_columns = {
+        "first_seen": ScholarPublication.created_at,
+        "title": Publication.title_raw,
+        "year": Publication.year,
+        "citations": Publication.citation_count,
+        "scholar": ScholarProfile.display_name,
+        "pdf_status": _pdf_status_sort_rank(),
+    }
+    return sort_columns.get(sort_by, ScholarPublication.created_at)
+
+
+async def get_latest_run_id_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> int | None:
+    # We include RUNNING and RESOLVING statuses so that the "New" tab shows
+    # results in real-time as they are discovered.
+    result = await db_session.execute(
+        select(func.max(CrawlRun.id)).where(
+            CrawlRun.user_id == user_id,
+            CrawlRun.status.in_(
+                [
+                    RunStatus.RUNNING,
+                    RunStatus.RESOLVING,
+                    RunStatus.SUCCESS,
+                    RunStatus.PARTIAL_FAILURE,
+                ]
+            ),
+        )
+    )
+    latest_run_id = result.scalar_one_or_none()
+    return int(latest_run_id) if latest_run_id is not None else None
+
+
+def publications_query(
+    *,
+    user_id: int,
+    mode: str,
+    latest_run_id: int | None,
+    scholar_profile_id: int | None,
+    favorite_only: bool,
+    limit: int,
+    offset: int = 0,
+    search: str | None = None,
+    sort_by: str = "first_seen",
+    sort_dir: str = "desc",
+    snapshot_before: datetime | None = None,
+) -> Select[tuple]:
+    scholar_label = ScholarProfile.display_name
+    stmt = (
+        select(
+            Publication.id,
+            ScholarProfile.id,
+            scholar_label,
+            ScholarProfile.scholar_id,
+            Publication.title_raw,
+            Publication.year,
+            Publication.citation_count,
+            Publication.venue_text,
+            Publication.pub_url,
+            Publication.pdf_url,
+            ScholarPublication.is_read,
+            ScholarPublication.is_favorite,
+            ScholarPublication.first_seen_run_id,
+            ScholarPublication.created_at,
+        )
+        .join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
+        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
+        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
+        .where(ScholarProfile.user_id == user_id)
+    )
+    if search:
+        safe_search = search.replace("%", r"\%").replace("_", r"\_")
+        pat = f"%{safe_search}%"
+        stmt = stmt.where(
+            Publication.title_raw.ilike(pat)
+            | ScholarProfile.display_name.ilike(pat)
+            | Publication.venue_text.ilike(pat)
+        )
+    if scholar_profile_id is not None:
+        stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
+    if favorite_only:
+        stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
+    if mode == MODE_UNREAD:
+        stmt = stmt.where(ScholarPublication.is_read.is_(False))
+    if mode == MODE_LATEST:
+        if latest_run_id is None:
+            return stmt.where(False)
+        stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
+    if snapshot_before is not None:
+        stmt = stmt.where(ScholarPublication.created_at <= snapshot_before)
+
+    sort_col = _sort_column(sort_by)
+    order = sort_col.desc() if sort_dir == "desc" else sort_col.asc()
+    stmt = stmt.order_by(order, Publication.id.desc())
+
+    if limit is not None:
+        stmt = stmt.offset(max(int(offset), 0)).limit(limit)
+
+    return stmt
+
+
+def publication_query_for_user(
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+    publication_id: int,
+) -> Select[tuple]:
+    return (
+        select(
+            Publication.id,
+            ScholarProfile.id,
+            ScholarProfile.display_name,
+            ScholarProfile.scholar_id,
+            Publication.title_raw,
+            Publication.year,
+            Publication.citation_count,
+            Publication.venue_text,
+            Publication.pub_url,
+            Publication.pdf_url,
+            ScholarPublication.is_read,
+            ScholarPublication.is_favorite,
+            ScholarPublication.first_seen_run_id,
+            ScholarPublication.created_at,
+        )
+        .join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
+        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
+        .where(
+            ScholarProfile.user_id == user_id,
+            ScholarProfile.id == scholar_profile_id,
+            Publication.id == publication_id,
+        )
+        .limit(1)
+    )
+
+
+async def get_publication_item_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+    publication_id: int,
+) -> PublicationListItem | None:
+    latest_run_id = await get_latest_run_id_for_user(db_session, user_id=user_id)
+    result = await db_session.execute(
+        publication_query_for_user(
+            user_id=user_id,
+            scholar_profile_id=scholar_profile_id,
+            publication_id=publication_id,
+        )
+    )
+    row = result.one_or_none()
+    if row is None:
+        return None
+    return publication_list_item_from_row(row, latest_run_id=latest_run_id)
+
+
+def publication_list_item_from_row(
+    row: tuple,
+    *,
+    latest_run_id: int | None,
+) -> PublicationListItem:
+    (
+        publication_id,
+        scholar_profile_id,
+        display_name,
+        scholar_id,
+        title_raw,
+        year,
+        citation_count,
+        venue_text,
+        pub_url,
+        pdf_url,
+        is_read,
+        is_favorite,
+        first_seen_run_id,
+        created_at,
+    ) = row
+    return PublicationListItem(
+        publication_id=int(publication_id),
+        scholar_profile_id=int(scholar_profile_id),
+        scholar_label=(display_name or scholar_id),
+        title=title_raw,
+        year=year,
+        citation_count=_normalized_citation_count(citation_count),
+        venue_text=venue_text,
+        pub_url=pub_url,
+        pdf_url=pdf_url,
+        is_read=bool(is_read),
+        is_favorite=bool(is_favorite),
+        first_seen_at=created_at,
+        is_new_in_latest_run=(latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id),
+    )
+
+
+def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
+    (
+        publication_id,
+        scholar_profile_id,
+        display_name,
+        scholar_id,
+        title_raw,
+        year,
+        citation_count,
+        venue_text,
+        pub_url,
+        pdf_url,
+        _is_read,
+        _is_favorite,
+        _first_seen_run_id,
+        _created_at,
+    ) = row
+    return UnreadPublicationItem(
+        publication_id=int(publication_id),
+        scholar_profile_id=int(scholar_profile_id),
+        scholar_label=(display_name or scholar_id),
+        title=title_raw,
+        year=year,
+        citation_count=_normalized_citation_count(citation_count),
+        venue_text=venue_text,
+        pub_url=pub_url,
+        pdf_url=pdf_url,
+    )

app/services/publications/read_state.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+from sqlalchemy import select, tuple_, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import ScholarProfile, ScholarPublication
+
+
+def _normalized_selection_pairs(selections: list[tuple[int, int]]) -> set[tuple[int, int]]:
+    pairs: set[tuple[int, int]] = set()
+    for scholar_profile_id, publication_id in selections:
+        normalized = (int(scholar_profile_id), int(publication_id))
+        if normalized[0] <= 0 or normalized[1] <= 0:
+            continue
+        pairs.add(normalized)
+    return pairs
+
+
+def _scoped_scholar_ids_query(*, user_id: int):
+    return select(ScholarProfile.id).where(ScholarProfile.user_id == user_id).scalar_subquery()
+
+
+async def mark_all_unread_as_read_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> int:
+    scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
+    stmt = (
+        update(ScholarPublication)
+        .where(
+            ScholarPublication.scholar_profile_id.in_(scholar_ids),
+            ScholarPublication.is_read.is_(False),
+        )
+        .values(is_read=True)
+    )
+    result = await db_session.execute(stmt)
+    await db_session.commit()
+    return int(result.rowcount or 0)
+
+
+async def mark_selected_as_read_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    selections: list[tuple[int, int]],
+) -> int:
+    normalized_pairs = _normalized_selection_pairs(selections)
+    if not normalized_pairs:
+        return 0
+
+    scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
+    stmt = (
+        update(ScholarPublication)
+        .where(
+            ScholarPublication.scholar_profile_id.in_(scholar_ids),
+            tuple_(
+                ScholarPublication.scholar_profile_id,
+                ScholarPublication.publication_id,
+            ).in_(list(normalized_pairs)),
+            ScholarPublication.is_read.is_(False),
+        )
+        .values(is_read=True)
+    )
+    result = await db_session.execute(stmt)
+    await db_session.commit()
+    return int(result.rowcount or 0)
+
+
+async def set_publication_favorite_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+    publication_id: int,
+    is_favorite: bool,
+) -> int:
+    scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
+    stmt = (
+        update(ScholarPublication)
+        .where(
+            ScholarPublication.scholar_profile_id.in_(scholar_ids),
+            ScholarPublication.scholar_profile_id == int(scholar_profile_id),
+            ScholarPublication.publication_id == int(publication_id),
+        )
+        .values(is_favorite=bool(is_favorite))
+    )
+    result = await db_session.execute(stmt)
+    await db_session.commit()
+    return int(result.rowcount or 0)

app/services/publications/types.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+
+from app.services.domains.publication_identifiers.types import DisplayIdentifier
+
+
+@dataclass(frozen=True)
+class PublicationListItem:
+    publication_id: int
+    scholar_profile_id: int
+    scholar_label: str
+    title: str
+    year: int | None
+    citation_count: int
+    venue_text: str | None
+    pub_url: str | None
+    pdf_url: str | None
+    is_read: bool
+    first_seen_at: datetime
+    is_new_in_latest_run: bool
+    is_favorite: bool = False
+    pdf_status: str = "untracked"
+    pdf_attempt_count: int = 0
+    pdf_failure_reason: str | None = None
+    pdf_failure_detail: str | None = None
+    display_identifier: DisplayIdentifier | None = None
+
+
+@dataclass(frozen=True)
+class UnreadPublicationItem:
+    publication_id: int
+    scholar_profile_id: int
+    scholar_label: str
+    title: str
+    year: int | None
+    citation_count: int
+    venue_text: str | None
+    pub_url: str | None
+    pdf_url: str | None