Intermediate commit

app/services/domains/publications/counts.py

@@ -5,7 +5,7 @@ from datetime import datetime
 from sqlalchemy import distinct, func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.db.models import ScholarProfile, ScholarPublication
+from app.db.models import Publication, ScholarProfile, ScholarPublication
 from app.services.domains.publications.modes import (
     MODE_ALL,
     MODE_LATEST,
@@ -22,6 +22,7 @@ async def count_for_user(
     mode: str = MODE_ALL,
     scholar_profile_id: int | None = None,
     favorite_only: bool = False,
+    search: str | None = None,
     snapshot_before: datetime | None = None,
 ) -> int:
     resolved_mode = resolve_publication_view_mode(mode)
@@ -30,8 +31,10 @@ async def count_for_user(
         select(func.count(distinct(ScholarPublication.publication_id)))
         .select_from(ScholarPublication)
         .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
+        .join(Publication, Publication.id == ScholarPublication.publication_id)
         .where(ScholarProfile.user_id == user_id)
     )
+    stmt = _apply_search_filter(stmt, search=search)
     if scholar_profile_id is not None:
         stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
     if favorite_only:
@@ -48,12 +51,25 @@ async def count_for_user(
     return int(result.scalar_one() or 0)
 
 
+def _apply_search_filter(stmt, *, search: str | None):
+    if not search:
+        return stmt
+    safe_search = search.replace("%", r"\%").replace("_", r"\_")
+    pattern = f"%{safe_search}%"
+    return stmt.where(
+        Publication.title_raw.ilike(pattern)
+        | ScholarProfile.display_name.ilike(pattern)
+        | Publication.venue_text.ilike(pattern)
+    )
+
+
 async def count_unread_for_user(
     db_session: AsyncSession,
     *,
     user_id: int,
     scholar_profile_id: int | None = None,
     favorite_only: bool = False,
+    search: str | None = None,
     snapshot_before: datetime | None = None,
 ) -> int:
     return await count_for_user(
@@ -62,6 +78,7 @@ async def count_unread_for_user(
         mode=MODE_UNREAD,
         scholar_profile_id=scholar_profile_id,
         favorite_only=favorite_only,
+        search=search,
         snapshot_before=snapshot_before,
     )
 
@@ -72,6 +89,7 @@ async def count_latest_for_user(
     user_id: int,
     scholar_profile_id: int | None = None,
     favorite_only: bool = False,
+    search: str | None = None,
     snapshot_before: datetime | None = None,
 ) -> int:
     return await count_for_user(
@@ -80,6 +98,7 @@ async def count_latest_for_user(
         mode=MODE_LATEST,
         scholar_profile_id=scholar_profile_id,
         favorite_only=favorite_only,
+        search=search,
         snapshot_before=snapshot_before,
     )
 
@@ -89,6 +108,7 @@ async def count_favorite_for_user(
     *,
     user_id: int,
     scholar_profile_id: int | None = None,
+    search: str | None = None,
     snapshot_before: datetime | None = None,
 ) -> int:
     return await count_for_user(
@@ -97,5 +117,6 @@ async def count_favorite_for_user(
         mode=MODE_ALL,
         scholar_profile_id=scholar_profile_id,
         favorite_only=True,
+        search=search,
         snapshot_before=snapshot_before,
     )

app/services/domains/publications/dedup.py

@@ -1,24 +1,77 @@
 from __future__ import annotations
 
+from dataclasses import dataclass
+import hashlib
 import logging
+from typing import Iterable
 
 from sqlalchemy import delete, select
-from sqlalchemy.orm import aliased
 from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import aliased
 
 from app.db.models import Publication, PublicationIdentifier, ScholarPublication
+from app.services.domains.ingestion.fingerprints import (
+    canonical_title_text_for_dedup,
+    canonical_title_tokens_for_dedup,
+    normalize_title,
+)
 
 logger = logging.getLogger(__name__)
 
+NEAR_DUP_DEFAULT_SIMILARITY_THRESHOLD = 0.78
+NEAR_DUP_DEFAULT_CONTAINMENT_THRESHOLD = 0.92
+NEAR_DUP_DEFAULT_MIN_SHARED_TOKENS = 3
+NEAR_DUP_DEFAULT_MAX_YEAR_DELTA = 1
+NEAR_DUP_MIN_TOKEN_LENGTH = 3
+NEAR_DUP_CLUSTER_KEY_LENGTH = 16
+NEAR_DUP_STOPWORDS = {
+    "a",
+    "an",
+    "and",
+    "approach",
+    "for",
+    "in",
+    "method",
+    "of",
+    "on",
+    "the",
+    "to",
+    "using",
+    "via",
+    "with",
+}
+
+
+@dataclass(frozen=True)
+class NearDuplicateMember:
+    publication_id: int
+    title: str
+    year: int | None
+    citation_count: int
+
+
+@dataclass(frozen=True)
+class NearDuplicateCluster:
+    cluster_key: str
+    winner_publication_id: int
+    similarity_score: float
+    members: tuple[NearDuplicateMember, ...]
+
+
+@dataclass(frozen=True)
+class _NearDuplicateCandidate:
+    publication_id: int
+    title: str
+    year: int | None
+    citation_count: int
+    canonical_text: str
+    tokens: frozenset[str]
+
 
 async def find_identifier_duplicate_pairs(
     db_session: AsyncSession,
 ) -> list[tuple[int, int]]:
-    """Return (winner_id, dup_id) pairs where two publications share the same identifier.
-
-    Winner is always the lower publication_id (earlier-created). Uses the existing
-    ix_publication_identifiers_kind_value index for the self-join.
-    """
+    """Return (winner_id, dup_id) pairs where two publications share the same identifier."""
     pi1 = aliased(PublicationIdentifier, name="pi1")
     pi2 = aliased(PublicationIdentifier, name="pi2")
     rows = await db_session.execute(
@@ -40,11 +93,17 @@ async def merge_duplicate_publication(
     winner_id: int,
     dup_id: int,
 ) -> None:
-    """Merge dup_id into winner_id: migrate scholar links, then delete the dup."""
+    """Merge dup_id into winner_id: migrate metadata/links/identifiers, then delete dup."""
+    if winner_id == dup_id:
+        raise ValueError("winner_id and dup_id must differ.")
+    winner = await _load_publication(db_session, publication_id=winner_id)
+    dup = await _load_publication(db_session, publication_id=dup_id)
+    if winner is None or dup is None:
+        raise ValueError("winner_id and dup_id must both exist.")
+    _merge_publication_metadata(winner=winner, dup=dup)
     await _migrate_scholar_links(db_session, winner_id=winner_id, dup_id=dup_id)
-    await db_session.execute(
-        delete(Publication).where(Publication.id == dup_id)
-    )
+    await _migrate_identifiers(db_session, winner_id=winner_id, dup_id=dup_id)
+    await db_session.execute(delete(Publication).where(Publication.id == dup_id))
     logger.info(
         "publications.identifier_merge",
         extra={
@@ -55,6 +114,45 @@ async def merge_duplicate_publication(
     )
 
 
+async def _load_publication(
+    db_session: AsyncSession,
+    *,
+    publication_id: int,
+) -> Publication | None:
+    result = await db_session.execute(
+        select(Publication).where(Publication.id == publication_id)
+    )
+    return result.scalar_one_or_none()
+
+
+def _merge_publication_metadata(*, winner: Publication, dup: Publication) -> None:
+    if winner.year is None and dup.year is not None:
+        winner.year = dup.year
+    winner.citation_count = max(int(winner.citation_count or 0), int(dup.citation_count or 0))
+    if not winner.author_text and dup.author_text:
+        winner.author_text = dup.author_text
+    if not winner.venue_text and dup.venue_text:
+        winner.venue_text = dup.venue_text
+    if not winner.pub_url and dup.pub_url:
+        winner.pub_url = dup.pub_url
+    if not winner.pdf_url and dup.pdf_url:
+        winner.pdf_url = dup.pdf_url
+    if not winner.cluster_id and dup.cluster_id:
+        winner.cluster_id = dup.cluster_id
+    if not winner.canonical_title_hash and dup.canonical_title_hash:
+        winner.canonical_title_hash = dup.canonical_title_hash
+    winner.title_raw = _preferred_title_text(winner=winner.title_raw, dup=dup.title_raw)
+    winner.title_normalized = normalize_title(winner.title_raw)
+
+
+def _preferred_title_text(*, winner: str, dup: str) -> str:
+    winner_score = len(canonical_title_text_for_dedup(winner))
+    dup_score = len(canonical_title_text_for_dedup(dup))
+    if dup_score > winner_score:
+        return dup
+    return winner
+
+
 async def _migrate_scholar_links(
     db_session: AsyncSession,
     *,
@@ -81,17 +179,64 @@ async def _migrate_scholar_links(
             link.publication_id = winner_id
 
 
-async def sweep_identifier_duplicates(db_session: AsyncSession) -> int:
-    """Find publications sharing an identifier and merge duplicates into the winner.
+async def _migrate_identifiers(
+    db_session: AsyncSession,
+    *,
+    winner_id: int,
+    dup_id: int,
+) -> None:
+    result = await db_session.execute(
+        select(PublicationIdentifier).where(PublicationIdentifier.publication_id == dup_id)
+    )
+    dup_identifiers = result.scalars().all()
+    for identifier in dup_identifiers:
+        existing = await _find_identifier(
+            db_session,
+            publication_id=winner_id,
+            kind=identifier.kind,
+            value_normalized=identifier.value_normalized,
+        )
+        if existing is None:
+            identifier.publication_id = winner_id
+            continue
+        _merge_identifier(existing=existing, dup=identifier)
+        await db_session.delete(identifier)
 
-    Returns the number of duplicate publications removed.
-    """
+
+async def _find_identifier(
+    db_session: AsyncSession,
+    *,
+    publication_id: int,
+    kind: str,
+    value_normalized: str,
+) -> PublicationIdentifier | None:
+    result = await db_session.execute(
+        select(PublicationIdentifier).where(
+            PublicationIdentifier.publication_id == publication_id,
+            PublicationIdentifier.kind == kind,
+            PublicationIdentifier.value_normalized == value_normalized,
+        )
+    )
+    return result.scalar_one_or_none()
+
+
+def _merge_identifier(*, existing: PublicationIdentifier, dup: PublicationIdentifier) -> None:
+    existing.confidence_score = max(
+        float(existing.confidence_score),
+        float(dup.confidence_score),
+    )
+    if not existing.evidence_url and dup.evidence_url:
+        existing.evidence_url = dup.evidence_url
+    if not existing.value_raw and dup.value_raw:
+        existing.value_raw = dup.value_raw
+
+
+async def sweep_identifier_duplicates(db_session: AsyncSession) -> int:
+    """Find publications sharing an identifier and merge duplicates into the winner."""
     pairs = await find_identifier_duplicate_pairs(db_session)
     if not pairs:
         return 0
 
-    # Deduplicate the pairs — a dup may appear multiple times if it shares
-    # several identifiers with the winner; process each dup only once.
     processed_dups: set[int] = set()
     for winner_id, dup_id in pairs:
         if dup_id in processed_dups:
@@ -101,3 +246,271 @@ async def sweep_identifier_duplicates(db_session: AsyncSession) -> int:
 
     await db_session.flush()
     return len(processed_dups)
+
+
+async def find_near_duplicate_clusters(
+    db_session: AsyncSession,
+    *,
+    similarity_threshold: float = NEAR_DUP_DEFAULT_SIMILARITY_THRESHOLD,
+    min_shared_tokens: int = NEAR_DUP_DEFAULT_MIN_SHARED_TOKENS,
+    max_year_delta: int = NEAR_DUP_DEFAULT_MAX_YEAR_DELTA,
+) -> list[NearDuplicateCluster]:
+    candidates = await _load_near_duplicate_candidates(db_session)
+    if len(candidates) < 2:
+        return []
+    groups = _cluster_candidate_groups(
+        candidates,
+        similarity_threshold=similarity_threshold,
+        min_shared_tokens=min_shared_tokens,
+        max_year_delta=max_year_delta,
+    )
+    clusters = [_near_duplicate_cluster(group) for group in groups]
+    return sorted(clusters, key=lambda item: (-len(item.members), item.winner_publication_id))
+
+
+async def merge_near_duplicate_cluster(
+    db_session: AsyncSession,
+    *,
+    cluster: NearDuplicateCluster,
+) -> int:
+    winner_id = int(cluster.winner_publication_id)
+    merged = 0
+    for member in cluster.members:
+        if int(member.publication_id) == winner_id:
+            continue
+        await merge_duplicate_publication(
+            db_session,
+            winner_id=winner_id,
+            dup_id=int(member.publication_id),
+        )
+        merged += 1
+    return merged
+
+
+def near_duplicate_cluster_payload(cluster: NearDuplicateCluster) -> dict[str, object]:
+    members = [
+        {
+            "publication_id": int(member.publication_id),
+            "title": member.title,
+            "year": member.year,
+            "citation_count": int(member.citation_count),
+        }
+        for member in cluster.members
+    ]
+    return {
+        "cluster_key": cluster.cluster_key,
+        "winner_publication_id": int(cluster.winner_publication_id),
+        "member_count": len(cluster.members),
+        "similarity_score": float(cluster.similarity_score),
+        "members": members,
+    }
+
+
+async def _load_near_duplicate_candidates(
+    db_session: AsyncSession,
+) -> list[_NearDuplicateCandidate]:
+    result = await db_session.execute(
+        select(
+            Publication.id,
+            Publication.title_raw,
+            Publication.year,
+            Publication.citation_count,
+        )
+    )
+    records = [
+        _candidate_from_row(
+            publication_id=int(publication_id),
+            title=str(title_raw or ""),
+            year=year,
+            citation_count=int(citation_count or 0),
+        )
+        for publication_id, title_raw, year, citation_count in result.all()
+    ]
+    return [record for record in records if record is not None]
+
+
+def _candidate_from_row(
+    *,
+    publication_id: int,
+    title: str,
+    year: int | None,
+    citation_count: int,
+) -> _NearDuplicateCandidate | None:
+    canonical = canonical_title_text_for_dedup(title)
+    raw_tokens = canonical_title_tokens_for_dedup(title)
+    tokens = _normalized_tokens(raw_tokens)
+    if not canonical or not tokens:
+        return None
+    return _NearDuplicateCandidate(
+        publication_id=publication_id,
+        title=title,
+        year=year,
+        citation_count=citation_count,
+        canonical_text=canonical,
+        tokens=frozenset(tokens),
+    )
+
+
+def _normalized_tokens(tokens: Iterable[str]) -> set[str]:
+    return {
+        token
+        for token in tokens
+        if len(token) >= NEAR_DUP_MIN_TOKEN_LENGTH and token not in NEAR_DUP_STOPWORDS
+    }
+
+
+def _cluster_candidate_groups(
+    candidates: list[_NearDuplicateCandidate],
+    *,
+    similarity_threshold: float,
+    min_shared_tokens: int,
+    max_year_delta: int,
+) -> list[list[_NearDuplicateCandidate]]:
+    by_id = {candidate.publication_id: candidate for candidate in candidates}
+    token_index = _candidate_token_index(candidates)
+    parent = {candidate.publication_id: candidate.publication_id for candidate in candidates}
+    for candidate in candidates:
+        peers = _candidate_peer_ids(candidate=candidate, token_index=token_index)
+        for peer_id in sorted(peers):
+            if peer_id <= candidate.publication_id:
+                continue
+            peer = by_id[peer_id]
+            if _is_near_duplicate_pair(
+                candidate,
+                peer,
+                similarity_threshold=similarity_threshold,
+                min_shared_tokens=min_shared_tokens,
+                max_year_delta=max_year_delta,
+            ):
+                _union(parent, candidate.publication_id, peer_id)
+    return _grouped_candidates(candidates, parent)
+
+
+def _candidate_token_index(
+    candidates: list[_NearDuplicateCandidate],
+) -> dict[str, set[int]]:
+    index: dict[str, set[int]] = {}
+    for candidate in candidates:
+        for token in candidate.tokens:
+            index.setdefault(token, set()).add(candidate.publication_id)
+    return index
+
+
+def _candidate_peer_ids(
+    *,
+    candidate: _NearDuplicateCandidate,
+    token_index: dict[str, set[int]],
+) -> set[int]:
+    peers: set[int] = set()
+    for token in candidate.tokens:
+        peers.update(token_index.get(token, set()))
+    peers.discard(candidate.publication_id)
+    return peers
+
+
+def _is_near_duplicate_pair(
+    left: _NearDuplicateCandidate,
+    right: _NearDuplicateCandidate,
+    *,
+    similarity_threshold: float,
+    min_shared_tokens: int,
+    max_year_delta: int,
+) -> bool:
+    if left.canonical_text == right.canonical_text:
+        return True
+    if not _years_compatible(left.year, right.year, max_year_delta=max_year_delta):
+        return False
+    shared_tokens = len(left.tokens & right.tokens)
+    if shared_tokens < min_shared_tokens:
+        return False
+    jaccard = _jaccard(left.tokens, right.tokens)
+    containment = shared_tokens / max(1, min(len(left.tokens), len(right.tokens)))
+    return jaccard >= similarity_threshold or containment >= NEAR_DUP_DEFAULT_CONTAINMENT_THRESHOLD
+
+
+def _years_compatible(left: int | None, right: int | None, *, max_year_delta: int) -> bool:
+    if left is None or right is None:
+        return True
+    return abs(int(left) - int(right)) <= int(max_year_delta)
+
+
+def _jaccard(left: frozenset[str], right: frozenset[str]) -> float:
+    if not left or not right:
+        return 0.0
+    return len(left & right) / len(left | right)
+
+
+def _find_root(parent: dict[int, int], value: int) -> int:
+    root = parent[value]
+    while root != parent[root]:
+        root = parent[root]
+    while value != root:
+        next_value = parent[value]
+        parent[value] = root
+        value = next_value
+    return root
+
+
+def _union(parent: dict[int, int], left: int, right: int) -> None:
+    left_root = _find_root(parent, left)
+    right_root = _find_root(parent, right)
+    if left_root == right_root:
+        return
+    if left_root < right_root:
+        parent[right_root] = left_root
+        return
+    parent[left_root] = right_root
+
+
+def _grouped_candidates(
+    candidates: list[_NearDuplicateCandidate],
+    parent: dict[int, int],
+) -> list[list[_NearDuplicateCandidate]]:
+    groups: dict[int, list[_NearDuplicateCandidate]] = {}
+    for candidate in candidates:
+        root = _find_root(parent, candidate.publication_id)
+        groups.setdefault(root, []).append(candidate)
+    clustered = [members for members in groups.values() if len(members) > 1]
+    for members in clustered:
+        members.sort(key=lambda item: item.publication_id)
+    return clustered
+
+
+def _near_duplicate_cluster(members: list[_NearDuplicateCandidate]) -> NearDuplicateCluster:
+    winner = _winner_candidate(members)
+    member_ids = [member.publication_id for member in members]
+    joined = ",".join(str(publication_id) for publication_id in member_ids)
+    cluster_key = hashlib.sha256(joined.encode("utf-8")).hexdigest()[:NEAR_DUP_CLUSTER_KEY_LENGTH]
+    similarity_score = _cluster_similarity_score(members)
+    return NearDuplicateCluster(
+        cluster_key=cluster_key,
+        winner_publication_id=winner.publication_id,
+        similarity_score=similarity_score,
+        members=tuple(
+            NearDuplicateMember(
+                publication_id=member.publication_id,
+                title=member.title,
+                year=member.year,
+                citation_count=member.citation_count,
+            )
+            for member in members
+        ),
+    )
+
+
+def _winner_candidate(members: list[_NearDuplicateCandidate]) -> _NearDuplicateCandidate:
+    return min(
+        members,
+        key=lambda member: (-int(member.citation_count), member.publication_id),
+    )
+
+
+def _cluster_similarity_score(members: list[_NearDuplicateCandidate]) -> float:
+    best = 0.0
+    for index, left in enumerate(members):
+        for right in members[index + 1 :]:
+            shared_tokens = len(left.tokens & right.tokens)
+            jaccard = _jaccard(left.tokens, right.tokens)
+            containment = shared_tokens / max(1, min(len(left.tokens), len(right.tokens)))
+            best = max(best, jaccard, containment)
+    return round(best, 4)

app/services/domains/publications/pdf_queue.py

@@ -370,16 +370,18 @@ async def _fetch_outcome_for_row(
     row: PublicationListItem,
     request_email: str | None,
     openalex_api_key: str | None = None,
-) -> OaResolutionOutcome:
+    allow_arxiv_lookup: bool = True,
+) -> tuple[OaResolutionOutcome, bool]:
     pipeline_result = await resolve_publication_pdf_outcome_for_row(
         row=row,
         request_email=request_email,
         openalex_api_key=openalex_api_key,
+        allow_arxiv_lookup=allow_arxiv_lookup,
     )
     outcome = pipeline_result.outcome
     if outcome is not None:
-        return outcome
-    return _failed_outcome(row=row)
+        return outcome, bool(pipeline_result.arxiv_rate_limited)
+    return _failed_outcome(row=row), bool(pipeline_result.arxiv_rate_limited)
 
 
 def _apply_publication_update(
@@ -450,17 +452,19 @@ async def _resolve_publication_row(
     request_email: str | None,
     row: PublicationListItem,
     openalex_api_key: str | None = None,
-) -> None:
+    allow_arxiv_lookup: bool = True,
+) -> bool:
     from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
-    from app.services.domains.arxiv.application import ArxivRateLimitError
+
     await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
     try:
-        outcome = await _fetch_outcome_for_row(
+        outcome, arxiv_rate_limited = await _fetch_outcome_for_row(
             row=row,
             request_email=request_email,
             openalex_api_key=openalex_api_key,
+            allow_arxiv_lookup=allow_arxiv_lookup,
         )
-    except (OpenAlexBudgetExhaustedError, ArxivRateLimitError):
+    except OpenAlexBudgetExhaustedError:
         # Persist a terminal outcome so jobs do not remain stuck in "running".
         await _persist_outcome(
             publication_id=row.publication_id,
@@ -479,11 +483,13 @@ async def _resolve_publication_row(
             },
         )
         outcome = _failed_outcome(row=row)
+        arxiv_rate_limited = False
     await _persist_outcome(
         publication_id=row.publication_id,
         user_id=user_id,
         outcome=outcome,
     )
+    return bool(arxiv_rate_limited)
 
 
 async def _run_resolution_task(
@@ -493,7 +499,6 @@ async def _run_resolution_task(
     rows: list[PublicationListItem],
 ) -> None:
     from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
-    from app.services.domains.arxiv.application import ArxivRateLimitError
     from app.services.domains.settings import application as user_settings_service
 
     # Resolve the best available API key: per-user setting → env var fallback.
@@ -506,14 +511,25 @@ async def _run_resolution_task(
     except Exception:
         openalex_api_key = settings.openalex_api_key
 
+    arxiv_lookup_allowed = True
     for row in rows:
         try:
-            await _resolve_publication_row(
+            arxiv_rate_limited = await _resolve_publication_row(
                 user_id=user_id,
                 request_email=request_email,
                 row=row,
                 openalex_api_key=openalex_api_key,
+                allow_arxiv_lookup=arxiv_lookup_allowed,
             )
+            if arxiv_rate_limited and arxiv_lookup_allowed:
+                arxiv_lookup_allowed = False
+                logger.warning(
+                    "publications.pdf_queue.arxiv_batch_disabled",
+                    extra={
+                        "event": "publications.pdf_queue.arxiv_batch_disabled",
+                        "detail": "arXiv temporarily disabled for remaining batch after rate limit",
+                    },
+                )
         except OpenAlexBudgetExhaustedError:
             logger.warning(
                 "publications.pdf_queue.budget_exhausted",
@@ -521,15 +537,6 @@ async def _run_resolution_task(
                        "detail": "Stopping PDF resolution batch — OpenAlex daily budget exhausted"},
             )
             break
-        except ArxivRateLimitError:
-            logger.warning(
-                "publications.pdf_queue.arxiv_rate_limited",
-                extra={
-                    "event": "publications.pdf_queue.arxiv_rate_limited",
-                    "detail": "Stopping PDF resolution batch — arXiv rate limit hit (429)",
-                },
-            )
-            break
 
 
 def _schedule_rows(

app/services/domains/publications/pdf_resolution_pipeline.py

@@ -5,6 +5,7 @@ import logging
 from typing import Any
 
 from app.services.domains.arxiv.application import ArxivRateLimitError
+from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
 from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
 from app.services.domains.publications.types import PublicationListItem
 from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
@@ -17,6 +18,7 @@ logger = logging.getLogger(__name__)
 class PipelineOutcome:
     outcome: OaResolutionOutcome | None
     scholar_candidates: Any | None  # Kept for backward compatibility with calling signatures
+    arxiv_rate_limited: bool = False
 
 
 async def resolve_publication_pdf_outcome_for_row(
@@ -24,6 +26,7 @@ async def resolve_publication_pdf_outcome_for_row(
     row: PublicationListItem,
     request_email: str | None,
     openalex_api_key: str | None = None,
+    allow_arxiv_lookup: bool = True,
 ) -> PipelineOutcome:
     # 1. OpenAlex OA — raises OpenAlexBudgetExhaustedError if budget is gone
     openalex_outcome = await _openalex_outcome(row, request_email=request_email, openalex_api_key=openalex_api_key)
@@ -31,13 +34,29 @@ async def resolve_publication_pdf_outcome_for_row(
         return PipelineOutcome(openalex_outcome, None)
 
     # 2. arXiv
-    arxiv_outcome = await _arxiv_outcome(row, request_email=request_email)
+    arxiv_rate_limited = False
+    try:
+        arxiv_outcome = await _arxiv_outcome(
+            row,
+            request_email=request_email,
+            allow_lookup=allow_arxiv_lookup,
+        )
+    except ArxivRateLimitError:
+        arxiv_rate_limited = True
+        arxiv_outcome = None
+        logger.warning(
+            "publications.pdf_resolution.arxiv_rate_limited",
+            extra={
+                "event": "publications.pdf_resolution.arxiv_rate_limited",
+                "publication_id": int(row.publication_id),
+            },
+        )
     if arxiv_outcome and arxiv_outcome.pdf_url:
-        return PipelineOutcome(arxiv_outcome, None)
+        return PipelineOutcome(arxiv_outcome, None, arxiv_rate_limited=arxiv_rate_limited)
 
     # 3. Unpaywall (which falls back to Crossref)
     oa_outcome = await _oa_outcome(row=row, request_email=request_email)
-    return PipelineOutcome(oa_outcome, None)
+    return PipelineOutcome(oa_outcome, None, arxiv_rate_limited=arxiv_rate_limited)
 
 
 async def _openalex_outcome(
@@ -87,9 +106,23 @@ async def _openalex_outcome(
     return None
 
 
-async def _arxiv_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None:
+async def _arxiv_outcome(
+    row: PublicationListItem,
+    *,
+    request_email: str | None,
+    allow_lookup: bool = True,
+) -> OaResolutionOutcome | None:
     from app.services.domains.arxiv.application import discover_arxiv_id_for_publication
 
+    if not allow_lookup:
+        _log_arxiv_skip(row=row, skip_reason="batch_arxiv_cooldown_active")
+        return None
+
+    skip_reason = arxiv_skip_reason_for_item(item=row)
+    if skip_reason is not None:
+        _log_arxiv_skip(row=row, skip_reason=skip_reason)
+        return None
+
     try:
         arxiv_id = await discover_arxiv_id_for_publication(item=row, request_email=request_email)
         if arxiv_id:
@@ -103,7 +136,7 @@ async def _arxiv_outcome(row: PublicationListItem, request_email: str | None) ->
                 used_crossref=False,
             )
     except ArxivRateLimitError:
-        raise  # propagate so the batch loop can stop
+        raise  # propagate so orchestration can switch to non-arXiv fallback
     except Exception as exc:
         logger.warning(
             "publications.pdf_resolution.arxiv_failed",
@@ -112,6 +145,17 @@ async def _arxiv_outcome(row: PublicationListItem, request_email: str | None) ->
     return None
 
 
+def _log_arxiv_skip(*, row: PublicationListItem, skip_reason: str) -> None:
+    logger.info(
+        "publications.pdf_resolution.arxiv_skipped",
+        extra={
+            "event": "publications.pdf_resolution.arxiv_skipped",
+            "publication_id": int(row.publication_id),
+            "skip_reason": skip_reason,
+        },
+    )
+
+
 async def _oa_outcome(
     *,
     row: PublicationListItem,

app/services/domains/publications/queries.py

@@ -2,10 +2,17 @@ from __future__ import annotations
 
 from datetime import datetime
 
-from sqlalchemy import Select, func, select
+from sqlalchemy import Select, case, func, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.db.models import CrawlRun, Publication, RunStatus, ScholarProfile, ScholarPublication
+from app.db.models import (
+    CrawlRun,
+    Publication,
+    PublicationPdfJob,
+    RunStatus,
+    ScholarProfile,
+    ScholarPublication,
+)
 from app.services.domains.publications.modes import MODE_LATEST, MODE_UNREAD
 from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
 
@@ -17,6 +24,29 @@ def _normalized_citation_count(value: object) -> int:
         return 0
 
 
+def _pdf_status_sort_rank():
+    return case(
+        (Publication.pdf_url.is_not(None), 4),
+        (PublicationPdfJob.status == "resolved", 4),
+        (PublicationPdfJob.status == "running", 3),
+        (PublicationPdfJob.status == "queued", 2),
+        (PublicationPdfJob.status == "failed", 0),
+        else_=1,
+    )
+
+
+def _sort_column(sort_by: str):
+    sort_columns = {
+        "first_seen": ScholarPublication.created_at,
+        "title": Publication.title_raw,
+        "year": Publication.year,
+        "citations": Publication.citation_count,
+        "scholar": ScholarProfile.display_name,
+        "pdf_status": _pdf_status_sort_rank(),
+    }
+    return sort_columns.get(sort_by, ScholarPublication.created_at)
+
+
 async def get_latest_run_id_for_user(
     db_session: AsyncSession,
     *,
@@ -55,14 +85,6 @@ def publications_query(
     sort_dir: str = "desc",
     snapshot_before: datetime | None = None,
 ) -> Select[tuple]:
-    _SORT_COLUMNS = {
-        "first_seen": ScholarPublication.created_at,
-        "title": Publication.title_raw,
-        "year": Publication.year,
-        "citations": Publication.citation_count,
-        "scholar": ScholarProfile.display_name,
-    }
-
     scholar_label = ScholarProfile.display_name
     stmt = (
         select(
@@ -83,6 +105,7 @@ def publications_query(
         )
         .join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
         .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
+        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
         .where(ScholarProfile.user_id == user_id)
     )
     if search:
@@ -106,7 +129,7 @@ def publications_query(
     if snapshot_before is not None:
         stmt = stmt.where(ScholarPublication.created_at <= snapshot_before)
 
-    sort_col = _SORT_COLUMNS.get(sort_by, ScholarPublication.created_at)
+    sort_col = _sort_column(sort_by)
     order = sort_col.desc() if sort_dir == "desc" else sort_col.asc()
     stmt = stmt.order_by(order, Publication.id.desc())