ci: add CodeQL security scanning and Dependabot

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

app/services/arxiv/application.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from app.services.domains.arxiv.gateway import (
+    build_arxiv_query,
+    get_arxiv_gateway,
+)
+
+if TYPE_CHECKING:
+    from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+
+
+def _build_arxiv_query(title: str, author_surname: str | None) -> str | None:
+    return build_arxiv_query(title, author_surname)
+
+
+async def discover_arxiv_id_for_publication(
+    *,
+    item: PublicationListItem | UnreadPublicationItem,
+    request_email: str | None = None,
+    timeout_seconds: float | None = None,
+) -> str | None:
+    gateway = get_arxiv_gateway()
+    return await gateway.discover_arxiv_id_for_publication(
+        item=item,
+        request_email=request_email,
+        timeout_seconds=timeout_seconds,
+    )

app/services/arxiv/cache.py

@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+from collections.abc import Awaitable, Callable, Mapping
+from dataclasses import asdict
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from sqlalchemy import delete, func, select
+
+from app.db.models import ArxivQueryCacheEntry
+from app.db.session import get_session_factory
+from app.services.domains.arxiv.constants import ARXIV_CACHE_FINGERPRINT_VERSION
+from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
+
+_INFLIGHT_LOCK = asyncio.Lock()
+_INFLIGHT_FEEDS: dict[str, asyncio.Future[ArxivFeed]] = {}
+
+
+def build_query_fingerprint(*, params: Mapping[str, object]) -> str:
+    canonical = _canonical_cache_payload(params=params)
+    encoded = json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
+    payload = f"{ARXIV_CACHE_FINGERPRINT_VERSION}:{encoded}"
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+async def get_cached_feed(
+    *,
+    query_fingerprint: str,
+    now_utc: datetime | None = None,
+) -> ArxivFeed | None:
+    timestamp = _as_utc(now_utc)
+    session_factory = get_session_factory()
+    async with session_factory() as db_session, db_session.begin():
+        result = await db_session.execute(
+            select(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint == query_fingerprint)
+        )
+        entry = result.scalar_one_or_none()
+        return await _validate_cached_entry(db_session, entry=entry, now_utc=timestamp)
+
+
+async def set_cached_feed(
+    *,
+    query_fingerprint: str,
+    feed: ArxivFeed,
+    ttl_seconds: float,
+    max_entries: int,
+    now_utc: datetime | None = None,
+) -> None:
+    timestamp = _as_utc(now_utc)
+    session_factory = get_session_factory()
+    async with session_factory() as db_session, db_session.begin():
+        await _write_cached_entry(
+            db_session,
+            query_fingerprint=query_fingerprint,
+            feed=feed,
+            ttl_seconds=ttl_seconds,
+            max_entries=max_entries,
+            now_utc=timestamp,
+        )
+
+
+async def run_with_inflight_dedupe(
+    *,
+    query_fingerprint: str,
+    fetch_feed: Callable[[], Awaitable[ArxivFeed]],
+) -> ArxivFeed:
+    future, is_owner = await _reserve_inflight_future(query_fingerprint=query_fingerprint)
+    if not is_owner:
+        return await asyncio.shield(future)
+    try:
+        result = await fetch_feed()
+    except Exception as exc:
+        _complete_future(future, error=exc)
+        raise
+    finally:
+        await _release_inflight_future(query_fingerprint=query_fingerprint, future=future)
+    _complete_future(future, result=result)
+    return result
+
+
+def _canonical_cache_payload(*, params: Mapping[str, object]) -> dict[str, object]:
+    payload: dict[str, object] = {}
+    for key in sorted(params.keys()):
+        payload[str(key)] = _normalize_param_value(str(key), params[key])
+    return payload
+
+
+def _normalize_param_value(key: str, value: object) -> object:
+    if key == "search_query":
+        return _normalize_search_query(str(value or ""))
+    if key == "id_list":
+        return _normalize_id_list(str(value or ""))
+    if isinstance(value, str):
+        return " ".join(value.strip().split())
+    if isinstance(value, (int, float, bool)) or value is None:
+        return value
+    return str(value).strip()
+
+
+def _normalize_search_query(value: str) -> str:
+    return " ".join(value.strip().lower().split())
+
+
+def _normalize_id_list(value: str) -> str:
+    normalized = [item.strip().lower() for item in value.split(",") if item.strip()]
+    return ",".join(sorted(normalized))
+
+
+async def _validate_cached_entry(
+    db_session,
+    *,
+    entry: ArxivQueryCacheEntry | None,
+    now_utc: datetime,
+) -> ArxivFeed | None:
+    if entry is None:
+        return None
+    if _as_utc(entry.expires_at) <= now_utc:
+        await db_session.delete(entry)
+        return None
+    parsed = _deserialize_feed(entry.payload)
+    if parsed is None:
+        await db_session.delete(entry)
+        return None
+    return parsed
+
+
+async def _write_cached_entry(
+    db_session,
+    *,
+    query_fingerprint: str,
+    feed: ArxivFeed,
+    ttl_seconds: float,
+    max_entries: int,
+    now_utc: datetime,
+) -> None:
+    ttl = max(float(ttl_seconds), 0.0)
+    result = await db_session.execute(
+        select(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint == query_fingerprint)
+    )
+    existing = result.scalar_one_or_none()
+    if ttl <= 0.0:
+        if existing is not None:
+            await db_session.delete(existing)
+        return
+    expires_at = now_utc + timedelta(seconds=ttl)
+    payload = _serialize_feed(feed)
+    if existing is None:
+        db_session.add(
+            ArxivQueryCacheEntry(
+                query_fingerprint=query_fingerprint,
+                payload=payload,
+                expires_at=expires_at,
+                cached_at=now_utc,
+                updated_at=now_utc,
+            )
+        )
+    else:
+        existing.payload = payload
+        existing.expires_at = expires_at
+        existing.cached_at = now_utc
+        existing.updated_at = now_utc
+    await _prune_cache_entries(db_session, now_utc=now_utc, max_entries=max_entries)
+
+
+async def _prune_cache_entries(
+    db_session,
+    *,
+    now_utc: datetime,
+    max_entries: int,
+) -> None:
+    await db_session.execute(delete(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.expires_at <= now_utc))
+    bounded_max_entries = int(max_entries)
+    if bounded_max_entries <= 0:
+        return
+    count_result = await db_session.execute(select(func.count()).select_from(ArxivQueryCacheEntry))
+    entry_count = int(count_result.scalar_one() or 0)
+    overflow = max(0, entry_count - bounded_max_entries)
+    if overflow <= 0:
+        return
+    stale_result = await db_session.execute(
+        select(ArxivQueryCacheEntry.query_fingerprint).order_by(ArxivQueryCacheEntry.cached_at.asc()).limit(overflow)
+    )
+    stale_keys = [str(row[0]) for row in stale_result.all()]
+    if stale_keys:
+        await db_session.execute(
+            delete(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint.in_(stale_keys))
+        )
+
+
+def _serialize_feed(feed: ArxivFeed) -> dict[str, Any]:
+    return asdict(feed)
+
+
+def _deserialize_feed(payload: object) -> ArxivFeed | None:
+    if not isinstance(payload, dict):
+        return None
+    entries_payload = payload.get("entries")
+    opensearch_payload = payload.get("opensearch")
+    if not isinstance(entries_payload, list):
+        return None
+    entries: list[ArxivEntry] = []
+    for value in entries_payload:
+        entry = _deserialize_entry(value)
+        if entry is None:
+            return None
+        entries.append(entry)
+    opensearch = _deserialize_opensearch(opensearch_payload)
+    if opensearch is None:
+        return None
+    return ArxivFeed(entries=entries, opensearch=opensearch)
+
+
+def _deserialize_entry(value: object) -> ArxivEntry | None:
+    if not isinstance(value, dict):
+        return None
+    try:
+        return ArxivEntry(
+            entry_id_url=str(value["entry_id_url"]),
+            arxiv_id=_as_optional_string(value.get("arxiv_id")),
+            title=str(value["title"]),
+            summary=str(value["summary"]),
+            published=_as_optional_string(value.get("published")),
+            updated=_as_optional_string(value.get("updated")),
+            authors=_as_string_list(value.get("authors")),
+            links=_as_string_list(value.get("links")),
+            categories=_as_string_list(value.get("categories")),
+            primary_category=_as_optional_string(value.get("primary_category")),
+        )
+    except KeyError:
+        return None
+
+
+def _deserialize_opensearch(value: object) -> ArxivOpenSearchMeta | None:
+    if not isinstance(value, dict):
+        return None
+    try:
+        return ArxivOpenSearchMeta(
+            total_results=int(value.get("total_results", 0)),
+            start_index=int(value.get("start_index", 0)),
+            items_per_page=int(value.get("items_per_page", 0)),
+        )
+    except (TypeError, ValueError):
+        return None
+
+
+def _as_optional_string(value: object) -> str | None:
+    if value is None:
+        return None
+    normalized = str(value).strip()
+    return normalized or None
+
+
+def _as_string_list(value: object) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(item) for item in value]
+
+
+def _as_utc(value: datetime | None) -> datetime:
+    if value is None:
+        return datetime.now(UTC)
+    if value.tzinfo is None:
+        return value.replace(tzinfo=UTC)
+    return value
+
+
+async def _reserve_inflight_future(
+    *,
+    query_fingerprint: str,
+) -> tuple[asyncio.Future[ArxivFeed], bool]:
+    async with _INFLIGHT_LOCK:
+        existing = _INFLIGHT_FEEDS.get(query_fingerprint)
+        if existing is not None:
+            return existing, False
+        loop = asyncio.get_running_loop()
+        created = loop.create_future()
+        created.add_done_callback(_consume_unretrieved_future_exception)
+        _INFLIGHT_FEEDS[query_fingerprint] = created
+        return created, True
+
+
+async def _release_inflight_future(
+    *,
+    query_fingerprint: str,
+    future: asyncio.Future[ArxivFeed],
+) -> None:
+    async with _INFLIGHT_LOCK:
+        current = _INFLIGHT_FEEDS.get(query_fingerprint)
+        if current is future:
+            _INFLIGHT_FEEDS.pop(query_fingerprint, None)
+
+
+def _complete_future(
+    future: asyncio.Future[ArxivFeed],
+    *,
+    result: ArxivFeed | None = None,
+    error: Exception | None = None,
+) -> None:
+    if future.done():
+        return
+    if error is not None:
+        future.set_exception(error)
+        return
+    if result is None:
+        raise RuntimeError("in-flight future completion requires result or error")
+    future.set_result(result)
+
+
+def _consume_unretrieved_future_exception(future: asyncio.Future[ArxivFeed]) -> None:
+    if future.cancelled():
+        return
+    try:
+        _ = future.exception()
+    except Exception:
+        return

app/services/arxiv/client.py

@@ -0,0 +1,278 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import Awaitable, Callable
+
+import httpx
+
+from app.logging_utils import structured_log
+from app.services.domains.arxiv.cache import (
+    build_query_fingerprint,
+    get_cached_feed,
+    run_with_inflight_dedupe,
+    set_cached_feed,
+)
+from app.services.domains.arxiv.constants import (
+    ARXIV_SOURCE_PATH_LOOKUP_IDS,
+    ARXIV_SOURCE_PATH_SEARCH,
+    ARXIV_SOURCE_PATH_UNKNOWN,
+)
+from app.services.domains.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
+from app.services.domains.arxiv.parser import parse_arxiv_feed
+from app.services.domains.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
+from app.services.domains.arxiv.types import ArxivFeed
+from app.settings import settings
+
+_ARXIV_API_URL = "https://export.arxiv.org/api/query"
+_ARXIV_QUERY_START = 0
+_ARXIV_MAX_RESULTS_LIMIT = 30_000
+_ARXIV_SORT_BY_ALLOWED = {"relevance", "lastUpdatedDate", "submittedDate"}
+_ARXIV_SORT_ORDER_ALLOWED = {"ascending", "descending"}
+_FALLBACK_CONTACT_EMAIL = "unknown@example.com"
+
+ArxivRequestFn = Callable[..., Awaitable[httpx.Response]]
+logger = logging.getLogger(__name__)
+
+
+class ArxivClient:
+    def __init__(
+        self,
+        *,
+        request_fn: ArxivRequestFn | None = None,
+        cache_enabled: bool | None = None,
+    ) -> None:
+        self._request_fn = request_fn or _request_arxiv_feed
+        self._cache_enabled = _resolve_cache_enabled(
+            cache_enabled=cache_enabled,
+            request_fn=request_fn,
+        )
+        self._cache_ttl_seconds = _cache_ttl_seconds()
+        self._cache_max_entries = _cache_max_entries()
+
+    async def search(
+        self,
+        *,
+        query: str,
+        start: int = _ARXIV_QUERY_START,
+        max_results: int | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        request_email: str | None = None,
+        timeout_seconds: float | None = None,
+    ) -> ArxivFeed:
+        params = _search_params(
+            query=query,
+            start=start,
+            max_results=max_results,
+            sort_by=sort_by,
+            sort_order=sort_order,
+        )
+        return await self._fetch_feed(
+            params=params,
+            request_email=request_email,
+            timeout_seconds=timeout_seconds,
+            source_path=ARXIV_SOURCE_PATH_SEARCH,
+        )
+
+    async def lookup_ids(
+        self,
+        *,
+        id_list: list[str],
+        start: int = _ARXIV_QUERY_START,
+        max_results: int | None = None,
+        request_email: str | None = None,
+        timeout_seconds: float | None = None,
+    ) -> ArxivFeed:
+        params = _lookup_params(id_list=id_list, start=start, max_results=max_results)
+        return await self._fetch_feed(
+            params=params,
+            request_email=request_email,
+            timeout_seconds=timeout_seconds,
+            source_path=ARXIV_SOURCE_PATH_LOOKUP_IDS,
+        )
+
+    async def _fetch_feed(
+        self,
+        *,
+        params: dict[str, object],
+        request_email: str | None,
+        timeout_seconds: float | None,
+        source_path: str,
+    ) -> ArxivFeed:
+        query_fingerprint = build_query_fingerprint(params=params)
+        if self._cache_enabled:
+            cached = await get_cached_feed(query_fingerprint=query_fingerprint)
+            if cached is not None:
+                structured_log(
+                    logger, "info", "arxiv.cache_hit", query_fingerprint=query_fingerprint, source_path=source_path
+                )
+                return cached
+            structured_log(
+                logger, "info", "arxiv.cache_miss", query_fingerprint=query_fingerprint, source_path=source_path
+            )
+        return await run_with_inflight_dedupe(
+            query_fingerprint=query_fingerprint,
+            fetch_feed=lambda: self._fetch_live_feed(
+                params=params,
+                request_email=request_email,
+                timeout_seconds=timeout_seconds,
+                query_fingerprint=query_fingerprint,
+            ),
+        )
+
+    async def _fetch_live_feed(
+        self,
+        *,
+        params: dict[str, object],
+        request_email: str | None,
+        timeout_seconds: float | None,
+        query_fingerprint: str,
+    ) -> ArxivFeed:
+        response = await self._request_fn(
+            params=params,
+            request_email=request_email,
+            timeout_seconds=timeout_seconds,
+        )
+        response.raise_for_status()
+        feed = parse_arxiv_feed(response.text)
+        if self._cache_enabled:
+            await set_cached_feed(
+                query_fingerprint=query_fingerprint,
+                feed=feed,
+                ttl_seconds=self._cache_ttl_seconds,
+                max_entries=self._cache_max_entries,
+            )
+        return feed
+
+
+def _search_params(
+    *,
+    query: str,
+    start: int,
+    max_results: int | None,
+    sort_by: str | None,
+    sort_order: str | None,
+) -> dict[str, object]:
+    clean_query = query.strip()
+    if not clean_query:
+        raise ArxivClientValidationError("search query must not be empty")
+    params: dict[str, object] = {
+        "search_query": clean_query,
+        "start": _validate_start(start),
+        "max_results": _validate_max_results(max_results),
+    }
+    if sort_by is not None:
+        params["sortBy"] = _validate_sort_by(sort_by)
+    if sort_order is not None:
+        params["sortOrder"] = _validate_sort_order(sort_order)
+    return params
+
+
+def _lookup_params(*, id_list: list[str], start: int, max_results: int | None) -> dict[str, object]:
+    normalized_ids = [value.strip() for value in id_list if value and value.strip()]
+    if not normalized_ids:
+        raise ArxivClientValidationError("id_list must include at least one id")
+    return {
+        "id_list": ",".join(normalized_ids),
+        "start": _validate_start(start),
+        "max_results": _validate_max_results(max_results),
+    }
+
+
+def _validate_start(value: int) -> int:
+    start = int(value)
+    if start < 0:
+        raise ArxivClientValidationError("start must be >= 0")
+    return start
+
+
+def _validate_max_results(value: int | None) -> int:
+    if value is None:
+        default_value = int(settings.arxiv_default_max_results)
+        return max(default_value, 1)
+    parsed = int(value)
+    if parsed < 1:
+        raise ArxivClientValidationError("max_results must be >= 1")
+    if parsed > _ARXIV_MAX_RESULTS_LIMIT:
+        raise ArxivClientValidationError(f"max_results must be <= {_ARXIV_MAX_RESULTS_LIMIT}")
+    return parsed
+
+
+def _validate_sort_by(value: str) -> str:
+    if value not in _ARXIV_SORT_BY_ALLOWED:
+        raise ArxivClientValidationError(f"sort_by must be one of: {sorted(_ARXIV_SORT_BY_ALLOWED)!r}")
+    return value
+
+
+def _validate_sort_order(value: str) -> str:
+    if value not in _ARXIV_SORT_ORDER_ALLOWED:
+        raise ArxivClientValidationError(f"sort_order must be one of: {sorted(_ARXIV_SORT_ORDER_ALLOWED)!r}")
+    return value
+
+
+async def _request_arxiv_feed(
+    *,
+    params: dict[str, object],
+    request_email: str | None,
+    timeout_seconds: float | None,
+) -> httpx.Response:
+    source_path = _source_path_from_params(params)
+    cooldown_status = await get_arxiv_cooldown_status()
+    if cooldown_status.is_active:
+        structured_log(
+            logger,
+            "warning",
+            "arxiv.request_skipped_cooldown",
+            source_path=source_path,
+            cooldown_remaining_seconds=float(cooldown_status.remaining_seconds),
+        )
+        raise ArxivRateLimitError(f"arXiv global cooldown active ({cooldown_status.remaining_seconds:.0f}s remaining)")
+
+    async def _fetch() -> httpx.Response:
+        timeout_value = _timeout_seconds(timeout_seconds)
+        headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{_contact_email(request_email)})"}
+        async with httpx.AsyncClient(timeout=timeout_value, follow_redirects=True, headers=headers) as client:
+            return await client.get(_ARXIV_API_URL, params=params)
+
+    return await run_with_global_arxiv_limit(
+        fetch=_fetch,
+        source_path=source_path,
+    )
+
+
+def _timeout_seconds(timeout_seconds: float | None) -> float:
+    if timeout_seconds is not None:
+        return max(float(timeout_seconds), 0.5)
+    return max(float(settings.arxiv_timeout_seconds), 0.5)
+
+
+def _contact_email(request_email: str | None) -> str:
+    return request_email or settings.arxiv_mailto or settings.crossref_api_mailto or _FALLBACK_CONTACT_EMAIL
+
+
+def _resolve_cache_enabled(
+    *,
+    cache_enabled: bool | None,
+    request_fn: ArxivRequestFn | None,
+) -> bool:
+    if cache_enabled is not None:
+        return bool(cache_enabled)
+    if request_fn is not None:
+        return False
+    return _cache_ttl_seconds() > 0.0
+
+
+def _cache_ttl_seconds() -> float:
+    return max(float(settings.arxiv_cache_ttl_seconds), 0.0)
+
+
+def _cache_max_entries() -> int:
+    return max(int(settings.arxiv_cache_max_entries), 0)
+
+
+def _source_path_from_params(params: dict[str, object]) -> str:
+    if "search_query" in params:
+        return ARXIV_SOURCE_PATH_SEARCH
+    if "id_list" in params:
+        return ARXIV_SOURCE_PATH_LOOKUP_IDS
+    return ARXIV_SOURCE_PATH_UNKNOWN

app/services/arxiv/constants.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+ARXIV_RUNTIME_STATE_KEY = "global"
+ARXIV_RATE_LIMIT_LOCK_NAMESPACE = 91_100
+ARXIV_RATE_LIMIT_LOCK_KEY = 1
+ARXIV_SOURCE_PATH_SEARCH = "search"
+ARXIV_SOURCE_PATH_LOOKUP_IDS = "lookup_ids"
+ARXIV_SOURCE_PATH_UNKNOWN = "unknown"
+ARXIV_CACHE_FINGERPRINT_VERSION = "v1"
+ARXIV_TITLE_TOKEN_MIN_LENGTH = 3
+ARXIV_TITLE_MIN_TOKENS = 3
+ARXIV_TITLE_MIN_ALPHA_TOKENS = 2
+ARXIV_STRONG_IDENTIFIER_CONFIDENCE = 0.9

app/services/arxiv/errors.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+
+class ArxivRateLimitError(Exception):
+    """arXiv returned 429 or cooldown is active."""
+
+
+class ArxivClientValidationError(ValueError):
+    """arXiv client inputs are invalid."""
+
+
+class ArxivParseError(ValueError):
+    """arXiv API payload could not be parsed."""

app/services/arxiv/gateway.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import logging
+import re
+import unicodedata
+from typing import TYPE_CHECKING, Protocol
+
+from app.logging_utils import structured_log
+from app.services.domains.arxiv.client import ArxivClient
+from app.services.domains.arxiv.errors import ArxivRateLimitError
+from app.services.domains.arxiv.types import ArxivFeed
+from app.settings import settings
+
+if TYPE_CHECKING:
+    from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+
+logger = logging.getLogger(__name__)
+
+_default_gateway: ArxivGateway | None = None
+_MOJIBAKE_HINT_RE = re.compile(r"[ÃÂâ]")
+_NON_ALNUM_RE = re.compile(r"[^\w\s]+", re.UNICODE)
+_WHITESPACE_RE = re.compile(r"\s+")
+
+
+class ArxivGateway(Protocol):
+    async def discover_arxiv_id_for_publication(
+        self,
+        *,
+        item: PublicationListItem | UnreadPublicationItem,
+        request_email: str | None = None,
+        timeout_seconds: float | None = None,
+        max_results: int | None = None,
+    ) -> str | None: ...
+
+
+def build_arxiv_query(title: str, author_surname: str | None) -> str | None:
+    parts: list[str] = []
+    if title:
+        clean_title = _normalize_query_title(title)
+        if clean_title:
+            parts.append(f'ti:"{clean_title}"')
+    if author_surname:
+        clean_author = _normalize_query_title(author_surname)
+        if clean_author:
+            parts.append(f'au:"{clean_author}"')
+    if not parts:
+        return None
+    return " AND ".join(parts)
+
+
+def _normalize_query_title(value: str) -> str:
+    repaired = _repair_mojibake(value.strip())
+    normalized = unicodedata.normalize("NFKC", repaired)
+    stripped = _NON_ALNUM_RE.sub(" ", _MOJIBAKE_HINT_RE.sub(" ", normalized))
+    return _WHITESPACE_RE.sub(" ", stripped).strip()
+
+
+def _repair_mojibake(value: str) -> str:
+    if not value or not _MOJIBAKE_HINT_RE.search(value):
+        return value
+    try:
+        repaired = value.encode("latin1").decode("utf-8")
+    except UnicodeError:
+        return value
+    return repaired if _mojibake_score(repaired) < _mojibake_score(value) else value
+
+
+def _mojibake_score(value: str) -> int:
+    return len(_MOJIBAKE_HINT_RE.findall(value))
+
+
+def get_arxiv_gateway() -> ArxivGateway:
+    global _default_gateway
+    if _default_gateway is None:
+        _default_gateway = HttpArxivGateway()
+    return _default_gateway
+
+
+def set_arxiv_gateway(gateway: ArxivGateway | None) -> ArxivGateway | None:
+    global _default_gateway
+    previous = _default_gateway
+    _default_gateway = gateway
+    return previous
+
+
+class HttpArxivGateway:
+    def __init__(self, *, client: ArxivClient | None = None) -> None:
+        self._client = client or ArxivClient()
+
+    async def discover_arxiv_id_for_publication(
+        self,
+        *,
+        item: PublicationListItem | UnreadPublicationItem,
+        request_email: str | None = None,
+        timeout_seconds: float | None = None,
+        max_results: int | None = None,
+    ) -> str | None:
+        if not settings.arxiv_enabled:
+            return None
+        query = _query_for_item(item)
+        if query is None:
+            return None
+
+        try:
+            result = await self._client.search(
+                query=query,
+                start=0,
+                request_email=request_email,
+                timeout_seconds=timeout_seconds,
+                max_results=max_results,
+            )
+            return _first_discovered_id(result)
+        except ArxivRateLimitError:
+            raise
+        except Exception as exc:
+            structured_log(logger, "debug", "arxiv.query_failed", error=str(exc))
+            return None
+
+
+def _query_for_item(item: PublicationListItem | UnreadPublicationItem) -> str | None:
+    title = (item.title or "").strip()
+    if not title:
+        return None
+    author_surname = _author_surname(item.scholar_label)
+    return build_arxiv_query(title, author_surname)
+
+
+def _author_surname(scholar_label: str | None) -> str | None:
+    if not scholar_label:
+        return None
+    tokens = [token for token in scholar_label.strip().split() if token]
+    if not tokens:
+        return None
+    return tokens[-1].lower()
+
+
+def _first_discovered_id(result: ArxivFeed) -> str | None:
+    for entry in result.entries:
+        if entry.arxiv_id:
+            structured_log(logger, "debug", "arxiv.id_discovered", arxiv_id=entry.arxiv_id)
+            return entry.arxiv_id
+    return None

app/services/arxiv/guards.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+from app.services.domains.arxiv.constants import (
+    ARXIV_STRONG_IDENTIFIER_CONFIDENCE,
+    ARXIV_TITLE_MIN_ALPHA_TOKENS,
+    ARXIV_TITLE_MIN_TOKENS,
+    ARXIV_TITLE_TOKEN_MIN_LENGTH,
+)
+from app.services.domains.doi.normalize import normalize_doi
+from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
+
+if TYPE_CHECKING:
+    from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+
+_TITLE_TOKEN_RE = re.compile(r"[a-z0-9]+")
+
+
+def arxiv_skip_reason_for_item(
+    *,
+    item: PublicationListItem | UnreadPublicationItem,
+    has_strong_doi: bool = False,
+    has_existing_arxiv: bool = False,
+) -> str | None:
+    if has_existing_arxiv or _has_arxiv_identifier_evidence(item):
+        return "arxiv_identifier_present"
+    if has_strong_doi or _has_strong_doi_evidence(item):
+        return "strong_doi_present"
+    if not _title_passes_quality_guard(item.title):
+        return "title_quality_below_threshold"
+    return None
+
+
+def _has_arxiv_identifier_evidence(item: PublicationListItem | UnreadPublicationItem) -> bool:
+    if _display_identifier_matches(item, expected_kind="arxiv"):
+        return True
+    return _has_normalized_identifier(item, normalizer=normalize_arxiv_id)
+
+
+def _has_strong_doi_evidence(item: PublicationListItem | UnreadPublicationItem) -> bool:
+    if _display_identifier_matches(item, expected_kind="doi"):
+        return True
+    return _has_normalized_identifier(item, normalizer=normalize_doi)
+
+
+def _display_identifier_matches(
+    item: PublicationListItem | UnreadPublicationItem,
+    *,
+    expected_kind: str,
+) -> bool:
+    display = getattr(item, "display_identifier", None)
+    if display is None:
+        return False
+    if str(display.kind).lower() != expected_kind:
+        return False
+    return float(display.confidence_score) >= ARXIV_STRONG_IDENTIFIER_CONFIDENCE
+
+
+def _has_normalized_identifier(
+    item: PublicationListItem | UnreadPublicationItem,
+    *,
+    normalizer,
+) -> bool:
+    if normalizer(item.pub_url):
+        return True
+    return normalizer(item.pdf_url) is not None
+
+
+def _title_passes_quality_guard(title: str | None) -> bool:
+    tokens = _normalized_tokens(title or "")
+    if len(tokens) < ARXIV_TITLE_MIN_TOKENS:
+        return False
+    alpha_tokens = [token for token in tokens if _is_alpha_token(token)]
+    return len(alpha_tokens) >= ARXIV_TITLE_MIN_ALPHA_TOKENS
+
+
+def _normalized_tokens(value: str) -> list[str]:
+    return [token for token in _TITLE_TOKEN_RE.findall(value.lower()) if token]
+
+
+def _is_alpha_token(token: str) -> bool:
+    if len(token) < ARXIV_TITLE_TOKEN_MIN_LENGTH:
+        return False
+    return any(char.isalpha() for char in token)

app/services/arxiv/parser.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+
+from app.services.domains.arxiv.errors import ArxivParseError
+from app.services.domains.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
+from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
+
+_NAMESPACES = {
+    "atom": "http://www.w3.org/2005/Atom",
+    "opensearch": "http://a9.com/-/spec/opensearch/1.1/",
+    "arxiv": "http://arxiv.org/schemas/atom",
+}
+
+
+def parse_arxiv_feed(payload: str) -> ArxivFeed:
+    root = _parse_xml_root(payload)
+    opensearch = ArxivOpenSearchMeta(
+        total_results=_opensearch_int(root, "opensearch:totalResults"),
+        start_index=_opensearch_int(root, "opensearch:startIndex"),
+        items_per_page=_opensearch_int(root, "opensearch:itemsPerPage"),
+    )
+    entries = [_parse_entry(entry_elem) for entry_elem in root.findall("atom:entry", _NAMESPACES)]
+    return ArxivFeed(entries=entries, opensearch=opensearch)
+
+
+def _parse_xml_root(payload: str) -> ET.Element:
+    try:
+        return ET.fromstring(payload)
+    except ET.ParseError as exc:
+        raise ArxivParseError(f"Invalid arXiv XML payload: {exc}") from exc
+
+
+def _opensearch_int(root: ET.Element, path: str) -> int:
+    text = _optional_text(root, path)
+    if text is None:
+        return 0
+    try:
+        return int(text.strip())
+    except ValueError as exc:
+        raise ArxivParseError(f"Invalid integer value at {path}: {text!r}") from exc
+
+
+def _parse_entry(entry_elem: ET.Element) -> ArxivEntry:
+    entry_id_url = _required_text(entry_elem, "atom:id").strip()
+    arxiv_id = normalize_arxiv_id(entry_id_url)
+    title = _required_text(entry_elem, "atom:title").strip()
+    summary = (_optional_text(entry_elem, "atom:summary") or "").strip()
+    published = _optional_text(entry_elem, "atom:published")
+    updated = _optional_text(entry_elem, "atom:updated")
+    return ArxivEntry(
+        entry_id_url=entry_id_url,
+        arxiv_id=arxiv_id,
+        title=title,
+        summary=summary,
+        published=published,
+        updated=updated,
+        authors=_authors(entry_elem),
+        links=_links(entry_elem),
+        categories=_categories(entry_elem),
+        primary_category=_primary_category(entry_elem),
+    )
+
+
+def _required_text(elem: ET.Element, path: str) -> str:
+    text = _optional_text(elem, path)
+    if text is None or not text.strip():
+        raise ArxivParseError(f"Missing required field: {path}")
+    return text
+
+
+def _optional_text(elem: ET.Element, path: str) -> str | None:
+    node = elem.find(path, _NAMESPACES)
+    if node is None or node.text is None:
+        return None
+    return str(node.text)
+
+
+def _authors(entry_elem: ET.Element) -> list[str]:
+    authors: list[str] = []
+    for author in entry_elem.findall("atom:author", _NAMESPACES):
+        name = _optional_text(author, "atom:name")
+        if name:
+            authors.append(name.strip())
+    return authors
+
+
+def _links(entry_elem: ET.Element) -> list[str]:
+    values: list[str] = []
+    for link in entry_elem.findall("atom:link", _NAMESPACES):
+        href = str(link.attrib.get("href") or "").strip()
+        if href:
+            values.append(href)
+    return values
+
+
+def _categories(entry_elem: ET.Element) -> list[str]:
+    values: list[str] = []
+    for cat in entry_elem.findall("atom:category", _NAMESPACES):
+        term = str(cat.attrib.get("term") or "").strip()
+        if term:
+            values.append(term)
+    return values
+
+
+def _primary_category(entry_elem: ET.Element) -> str | None:
+    node = entry_elem.find("arxiv:primary_category", _NAMESPACES)
+    if node is None:
+        return None
+    value = str(node.attrib.get("term") or "").strip()
+    return value or None

app/services/arxiv/rate_limit.py

@@ -0,0 +1,203 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+
+import httpx
+from sqlalchemy import select, text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import ArxivRuntimeState
+from app.db.session import get_session_factory
+from app.logging_utils import structured_log
+from app.services.domains.arxiv.constants import (
+    ARXIV_RATE_LIMIT_LOCK_KEY,
+    ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
+    ARXIV_RUNTIME_STATE_KEY,
+    ARXIV_SOURCE_PATH_UNKNOWN,
+)
+from app.services.domains.arxiv.errors import ArxivRateLimitError
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class ArxivCooldownStatus:
+    is_active: bool
+    remaining_seconds: float
+    cooldown_until: datetime | None
+
+
+async def run_with_global_arxiv_limit(
+    *,
+    fetch: Callable[[], Awaitable[httpx.Response]],
+    source_path: str = ARXIV_SOURCE_PATH_UNKNOWN,
+) -> httpx.Response:
+    response, hit_rate_limit = await _run_serialized_fetch(
+        fetch=fetch,
+        source_path=source_path,
+    )
+    if hit_rate_limit:
+        raise ArxivRateLimitError("arXiv rate limit hit (429) — stopping batch")
+    return response
+
+
+async def get_arxiv_cooldown_status(*, now_utc: datetime | None = None) -> ArxivCooldownStatus:
+    timestamp = _normalize_datetime(now_utc) or datetime.now(UTC)
+    session_factory = get_session_factory()
+    async with session_factory() as db_session:
+        result = await db_session.execute(
+            select(ArxivRuntimeState.cooldown_until).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY)
+        )
+    cooldown_until = _normalize_datetime(result.scalar_one_or_none())
+    remaining_seconds = _cooldown_remaining_seconds(cooldown_until, now_utc=timestamp)
+    return ArxivCooldownStatus(
+        is_active=remaining_seconds > 0.0,
+        remaining_seconds=float(remaining_seconds),
+        cooldown_until=cooldown_until,
+    )
+
+
+async def _run_serialized_fetch(
+    *,
+    fetch: Callable[[], Awaitable[httpx.Response]],
+    source_path: str,
+) -> tuple[httpx.Response, bool]:
+    session_factory = get_session_factory()
+    async with session_factory() as db_session, db_session.begin():
+            await _acquire_arxiv_lock(db_session)
+            runtime_state = await _load_runtime_state_for_update(db_session)
+            wait_seconds = await _wait_for_allowed_slot_or_raise(
+                runtime_state,
+                source_path=source_path,
+            )
+            response = await fetch()
+            hit_rate_limit = _record_post_response_state(
+                runtime_state,
+                response_status=int(response.status_code),
+                source_path=source_path,
+            )
+            structured_log(
+                logger,
+                "info",
+                "arxiv.request_completed",
+                status_code=int(response.status_code),
+                wait_seconds=wait_seconds,
+                cooldown_remaining_seconds=_cooldown_remaining_seconds(
+                    runtime_state.cooldown_until, now_utc=datetime.now(UTC)
+                ),
+                source_path=source_path,
+            )
+            return response, hit_rate_limit
+
+
+async def _acquire_arxiv_lock(db_session: AsyncSession) -> None:
+    await db_session.execute(
+        text("SELECT pg_advisory_xact_lock(:namespace, :lock_key)"),
+        {
+            "namespace": ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
+            "lock_key": ARXIV_RATE_LIMIT_LOCK_KEY,
+        },
+    )
+
+
+async def _load_runtime_state_for_update(db_session: AsyncSession) -> ArxivRuntimeState:
+    result = await db_session.execute(
+        select(ArxivRuntimeState).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY).with_for_update()
+    )
+    state = result.scalar_one_or_none()
+    if state is not None:
+        return state
+    state = ArxivRuntimeState(state_key=ARXIV_RUNTIME_STATE_KEY)
+    db_session.add(state)
+    await db_session.flush()
+    return state
+
+
+async def _wait_for_allowed_slot_or_raise(
+    runtime_state: ArxivRuntimeState,
+    *,
+    source_path: str,
+) -> float:
+    now_utc = datetime.now(UTC)
+    cooldown_seconds = _cooldown_remaining_seconds(runtime_state.cooldown_until, now_utc=now_utc)
+    if cooldown_seconds > 0:
+        structured_log(
+            logger,
+            "info",
+            "arxiv.request_scheduled",
+            wait_seconds=0.0,
+            source_path=source_path,
+            cooldown_remaining_seconds=cooldown_seconds,
+        )
+        raise ArxivRateLimitError(f"arXiv global cooldown active ({cooldown_seconds:.0f}s remaining)")
+    wait_seconds = _next_allowed_wait_seconds(runtime_state.next_allowed_at, now_utc=now_utc)
+    structured_log(
+        logger,
+        "info",
+        "arxiv.request_scheduled",
+        wait_seconds=wait_seconds,
+        source_path=source_path,
+        cooldown_remaining_seconds=0.0,
+    )
+    if wait_seconds > 0:
+        await asyncio.sleep(wait_seconds)
+    return wait_seconds
+
+
+def _record_post_response_state(
+    runtime_state: ArxivRuntimeState,
+    *,
+    response_status: int,
+    source_path: str,
+) -> bool:
+    now_utc = datetime.now(UTC)
+    runtime_state.next_allowed_at = now_utc + timedelta(seconds=_min_interval_seconds())
+    if response_status == 429:
+        cooldown_seconds = _cooldown_seconds()
+        runtime_state.cooldown_until = now_utc + timedelta(seconds=cooldown_seconds)
+        structured_log(
+            logger,
+            "warning",
+            "arxiv.cooldown_activated",
+            cooldown_remaining_seconds=cooldown_seconds,
+            source_path=source_path,
+        )
+        return True
+    if _cooldown_remaining_seconds(runtime_state.cooldown_until, now_utc=now_utc) <= 0:
+        runtime_state.cooldown_until = None
+    return False
+
+
+def _cooldown_remaining_seconds(cooldown_until: datetime | None, *, now_utc: datetime) -> float:
+    bounded = _normalize_datetime(cooldown_until)
+    if bounded is None:
+        return 0.0
+    return max((bounded - now_utc).total_seconds(), 0.0)
+
+
+def _next_allowed_wait_seconds(next_allowed_at: datetime | None, *, now_utc: datetime) -> float:
+    bounded = _normalize_datetime(next_allowed_at)
+    if bounded is None:
+        return 0.0
+    return max((bounded - now_utc).total_seconds(), 0.0)
+
+
+def _normalize_datetime(value: datetime | None) -> datetime | None:
+    if value is None:
+        return None
+    if value.tzinfo is None:
+        return value.replace(tzinfo=UTC)
+    return value
+
+
+def _min_interval_seconds() -> float:
+    return max(float(settings.arxiv_min_interval_seconds), 0.0)
+
+
+def _cooldown_seconds() -> float:
+    return max(float(settings.arxiv_rate_limit_cooldown_seconds), 0.0)

app/services/arxiv/types.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Literal
+
+ArxivSortBy = Literal["relevance", "lastUpdatedDate", "submittedDate"]
+ArxivSortOrder = Literal["ascending", "descending"]
+
+
+@dataclass(frozen=True)
+class ArxivOpenSearchMeta:
+    total_results: int = 0
+    start_index: int = 0
+    items_per_page: int = 0
+
+
+@dataclass(frozen=True)
+class ArxivEntry:
+    entry_id_url: str
+    arxiv_id: str | None
+    title: str
+    summary: str
+    published: str | None
+    updated: str | None
+    authors: list[str] = field(default_factory=list)
+    links: list[str] = field(default_factory=list)
+    categories: list[str] = field(default_factory=list)
+    primary_category: str | None = None
+
+
+@dataclass(frozen=True)
+class ArxivFeed:
+    entries: list[ArxivEntry] = field(default_factory=list)
+    opensearch: ArxivOpenSearchMeta = field(default_factory=ArxivOpenSearchMeta)