temp commit

build/lib/app/services/__init__.py

@@ -0,0 +1,2 @@
+"""Service layer for scholarr application workflows."""
+

build/lib/app/services/domains/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

build/lib/app/services/domains/arxiv/application.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import TYPE_CHECKING
+import xml.etree.ElementTree as ET
+
+import httpx
+
+from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
+from app.settings import settings
+
+if TYPE_CHECKING:
+    from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+
+logger = logging.getLogger(__name__)
+
+
+def _build_arxiv_query(title: str, author_surname: str | None) -> str | None:
+    parts = []
+    if title:
+        # arXiv api allows strict title searching using ti:
+        clean_title = title.replace('"', '').replace("'", "")
+        parts.append(f'ti:"{clean_title}"')
+    if author_surname:
+        parts.append(f'au:"{author_surname}"')
+    
+    if not parts:
+        return None
+    return " AND ".join(parts)
+
+
+async def discover_arxiv_id_for_publication(
+    *,
+    item: PublicationListItem | UnreadPublicationItem,
+    request_email: str | None = None,
+    timeout_seconds: float = 3.0,
+) -> str | None:
+    title = (item.title or "").strip()
+    if not title:
+        return None
+        
+    author_surname = None
+    if item.scholar_label:
+        tokens = [t for t in item.scholar_label.strip().split() if t]
+        if tokens:
+            author_surname = tokens[-1].lower()
+
+    query = _build_arxiv_query(title, author_surname)
+    if not query:
+        return None
+
+    url = "https://export.arxiv.org/api/query"
+    params = {
+        "search_query": query,
+        "start": 0,
+        "max_results": 3,
+    }
+
+    headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{request_email or settings.crossref_api_mailto or 'unknown@example.com'})"}
+    
+    try:
+        async with httpx.AsyncClient(timeout=timeout_seconds, follow_redirects=True, headers=headers) as client:
+            response = await client.get(url, params=params)
+            response.raise_for_status()
+            
+        root = ET.fromstring(response.text)
+        namespace = {'atom': 'http://www.w3.org/2005/Atom'}
+        
+        for entry in root.findall('atom:entry', namespace):
+            id_elem = entry.find('atom:id', namespace)
+            if id_elem is not None and id_elem.text:
+                candidate = str(id_elem.text)
+                if '/abs/' in candidate:
+                    candidate = candidate.split('/abs/')[-1]
+                normalized = normalize_arxiv_id(candidate)
+                if normalized:
+                    logger.debug("arxiv.id_discovered", extra={"event": "arxiv.id_discovered", "arxiv_id": normalized})
+                    return normalized
+                    
+    except Exception as exc:
+        logger.debug(f"Failed to query arXiv API: {exc}")
+        
+    return None

build/lib/app/services/domains/crossref/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from app.services.domains.crossref.application import discover_doi_for_publication
+
+__all__ = ["discover_doi_for_publication"]

build/lib/app/services/domains/crossref/application.py

@@ -0,0 +1,382 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+import threading
+import time
+from typing import TYPE_CHECKING
+
+from crossref.restful import Etiquette, Works
+
+from app.services.domains.doi.normalize import normalize_doi
+from app.settings import settings
+
+if TYPE_CHECKING:
+    from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+
+TOKEN_RE = re.compile(r"[a-z0-9]+")
+NON_ALNUM_RE = re.compile(r"[^a-z0-9\\s]+")
+STOP_WORDS = {"the", "and", "for", "with", "from", "method", "study", "analysis"}
+_RATE_LOCK = threading.Lock()
+_LAST_REQUEST_AT = 0.0
+logger = logging.getLogger(__name__)
+STRICT_TITLE_MATCH_THRESHOLD = 0.75
+RELAXED_TITLE_MATCH_THRESHOLD = 0.85
+
+
+def _rate_limit_wait(min_interval_seconds: float) -> None:
+    global _LAST_REQUEST_AT
+    interval = max(float(min_interval_seconds), 0.0)
+    with _RATE_LOCK:
+        elapsed = time.monotonic() - _LAST_REQUEST_AT
+        remaining = interval - elapsed
+        if remaining > 0:
+            time.sleep(remaining)
+        _LAST_REQUEST_AT = time.monotonic()
+
+
+def _normalized_tokens(value: str) -> list[str]:
+    lowered = value.lower().replace("’", "'").replace("“", "\"").replace("”", "\"")
+    lowered = NON_ALNUM_RE.sub(" ", lowered)
+    return [token for token in TOKEN_RE.findall(lowered) if len(token) >= 3]
+
+
+def _normalized_query(value: str) -> str:
+    tokens = [token for token in _normalized_tokens(value) if token not in STOP_WORDS]
+    if len(tokens) < 3:
+        tokens = _normalized_tokens(value)
+    if len(tokens) < 3:
+        return ""
+    return " ".join(tokens[:12]).strip()
+
+
+def _query_author(value: str) -> str | None:
+    tokens = [token for token in value.strip().split() if token]
+    if len(tokens) < 2:
+        return None
+    return " ".join(tokens[:2])[:64]
+
+
+def _author_surname(value: str) -> str | None:
+    tokens = [token for token in value.strip().split() if token]
+    if not tokens:
+        return None
+    return NON_ALNUM_RE.sub("", tokens[-1].lower()) or None
+
+
+def _query_filters(year: int | None) -> list[tuple[str, str] | None]:
+    if year is None:
+        return [None]
+    return [
+        (f"{year - 1}-01-01", f"{year + 1}-12-31"),
+        (f"{year}-01-01", f"{year}-12-31"),
+        None,
+    ]
+
+
+def _candidate_title(item: dict) -> str:
+    titles = item.get("title")
+    if isinstance(titles, list) and titles:
+        return str(titles[0] or "")
+    return str(item.get("title") or "")
+
+
+def _title_match_score(source: str, candidate: str) -> float:
+    source_tokens = {token for token in _normalized_tokens(source) if len(token) >= 3}
+    candidate_tokens = {token for token in _normalized_tokens(candidate) if len(token) >= 3}
+    if not source_tokens or not candidate_tokens:
+        return 0.0
+    return len(source_tokens & candidate_tokens) / float(len(source_tokens))
+
+
+def _candidate_year(item: dict) -> int | None:
+    issued = item.get("issued")
+    if not isinstance(issued, dict):
+        return None
+    date_parts = issued.get("date-parts")
+    if not isinstance(date_parts, list) or not date_parts:
+        return None
+    first = date_parts[0]
+    if not isinstance(first, list) or not first:
+        return None
+    try:
+        return int(first[0])
+    except (TypeError, ValueError):
+        return None
+
+
+def _candidate_author_match(item: dict, surname: str | None) -> bool:
+    if not surname:
+        return True
+    authors = item.get("author")
+    if not isinstance(authors, list):
+        return False
+    for author in authors:
+        if not isinstance(author, dict):
+            continue
+        family = NON_ALNUM_RE.sub("", str(author.get("family") or "").lower())
+        if family and family == surname:
+            return True
+    return False
+
+
+def _candidate_rank(*, title: str, year: int | None, item: dict) -> tuple[float, str | None]:
+    doi = normalize_doi(str(item.get("DOI") or ""))
+    if doi is None:
+        return 0.0, None
+    score = _title_match_score(title, _candidate_title(item))
+    candidate_year = _candidate_year(item)
+    if year is not None and candidate_year is not None:
+        if abs(year - candidate_year) > 1:
+            return 0.0, None
+        score += 0.1
+    return score, doi
+
+
+def _year_delta(source_year: int | None, candidate_year: int | None) -> int | None:
+    if source_year is None or candidate_year is None:
+        return None
+    return abs(int(source_year) - int(candidate_year))
+
+
+def _candidate_rank_relaxed(
+    *,
+    title: str,
+    year: int | None,
+    item: dict,
+    author_surname: str | None,
+) -> tuple[float, str | None]:
+    doi = normalize_doi(str(item.get("DOI") or ""))
+    if doi is None:
+        return 0.0, None
+    score = _title_match_score(title, _candidate_title(item))
+    if score <= 0:
+        return 0.0, None
+    candidate_year = _candidate_year(item)
+    delta = _year_delta(year, candidate_year)
+    if delta is not None:
+        if delta <= 1:
+            score += 0.05
+        elif delta <= 3:
+            score += 0.0
+        elif delta <= 5:
+            score -= 0.03
+        else:
+            score -= 0.08
+    if _candidate_author_match(item, author_surname):
+        score += 0.03
+    return score, doi
+
+
+def _best_candidate_doi_strict(
+    *,
+    title: str,
+    year: int | None,
+    items: list[dict],
+    author_surname: str | None,
+) -> str | None:
+    best_score = 0.0
+    best_doi: str | None = None
+    best_year: int | None = None
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        if not _candidate_author_match(item, author_surname):
+            continue
+        score, doi = _candidate_rank(title=title, year=year, item=item)
+        candidate_year = _candidate_year(item)
+        if doi is None or score < STRICT_TITLE_MATCH_THRESHOLD:
+            continue
+        if score > best_score:
+            best_score = score
+            best_doi = doi
+            best_year = candidate_year
+            continue
+        if abs(score - best_score) > 0.02:
+            continue
+        if best_year is None or candidate_year is None:
+            continue
+        if candidate_year < best_year:
+            best_doi = doi
+            best_year = candidate_year
+    return best_doi
+
+
+def _best_candidate_doi_relaxed(
+    *,
+    title: str,
+    year: int | None,
+    items: list[dict],
+    author_surname: str | None,
+) -> str | None:
+    best_score = 0.0
+    best_doi: str | None = None
+    best_author_match = False
+    best_delta: int | None = None
+    best_year: int | None = None
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        score, doi = _candidate_rank_relaxed(
+            title=title,
+            year=year,
+            item=item,
+            author_surname=author_surname,
+        )
+        if doi is None or score < RELAXED_TITLE_MATCH_THRESHOLD:
+            continue
+        candidate_year = _candidate_year(item)
+        candidate_author_match = _candidate_author_match(item, author_surname)
+        candidate_delta = _year_delta(year, candidate_year)
+        if score > best_score:
+            best_score = score
+            best_doi = doi
+            best_author_match = candidate_author_match
+            best_delta = candidate_delta
+            best_year = candidate_year
+            continue
+        if abs(score - best_score) > 0.02:
+            continue
+        if candidate_author_match and not best_author_match:
+            best_doi = doi
+            best_author_match = True
+            best_delta = candidate_delta
+            best_year = candidate_year
+            continue
+        if best_delta is None and candidate_delta is not None:
+            best_doi = doi
+            best_author_match = candidate_author_match
+            best_delta = candidate_delta
+            best_year = candidate_year
+            continue
+        if best_delta is not None and candidate_delta is not None and candidate_delta < best_delta:
+            best_doi = doi
+            best_author_match = candidate_author_match
+            best_delta = candidate_delta
+            best_year = candidate_year
+            continue
+        if best_year is None or candidate_year is None:
+            continue
+        if candidate_year < best_year:
+            best_doi = doi
+            best_author_match = candidate_author_match
+            best_delta = candidate_delta
+            best_year = candidate_year
+    return best_doi
+
+
+def _best_candidate_doi(
+    *,
+    title: str,
+    year: int | None,
+    items: list[dict],
+    author_surname: str | None,
+) -> str | None:
+    strict_match = _best_candidate_doi_strict(
+        title=title,
+        year=year,
+        items=items,
+        author_surname=author_surname,
+    )
+    if strict_match:
+        return strict_match
+    return _best_candidate_doi_relaxed(
+        title=title,
+        year=year,
+        items=items,
+        author_surname=author_surname,
+    )
+
+
+def _works_client(email: str | None) -> Works:
+    if email:
+        etiquette = Etiquette(settings.app_name, "0.1.0", "https://scholarr.local", email)
+        return Works(etiquette=etiquette)
+    return Works()
+
+
+def _fetch_items_sync(
+    *,
+    query: str,
+    author: str | None,
+    date_range: tuple[str, str] | None,
+    max_rows: int,
+    email: str | None,
+    min_interval_seconds: float,
+) -> list[dict]:
+    _rate_limit_wait(min_interval_seconds)
+    works = _works_client(email)
+    params = {"bibliographic": query}
+    if author:
+        params["author"] = author
+    request = works.query(**params)
+    if date_range is not None:
+        from_date, until_date = date_range
+        request = request.filter(from_pub_date=from_date, until_pub_date=until_date)
+    request = request.select(["DOI", "title", "issued", "score", "author"])
+    items: list[dict] = []
+    for entry in request:
+        if isinstance(entry, dict):
+            items.append(entry)
+        if len(items) >= max(max_rows, 1):
+            break
+    return items
+
+
+async def _fetch_items(
+    *,
+    query: str,
+    author: str | None,
+    date_range: tuple[str, str] | None,
+    max_rows: int,
+    email: str | None,
+) -> list[dict]:
+    timeout = max(float(settings.crossref_timeout_seconds), 0.5)
+    try:
+        return await asyncio.wait_for(
+            asyncio.to_thread(
+                _fetch_items_sync,
+                query=query,
+                author=author,
+                date_range=date_range,
+                max_rows=max_rows,
+                email=email,
+                min_interval_seconds=settings.crossref_min_interval_seconds,
+            ),
+            timeout=timeout,
+        )
+    except Exception:
+        return []
+
+
+async def discover_doi_for_publication(
+    *,
+    item: PublicationListItem | UnreadPublicationItem,
+    max_rows: int = 10,
+    email: str | None = None,
+) -> str | None:
+    title = (item.title or "").strip()
+    query = _normalized_query(title)
+    if not query:
+        return None
+    author = _query_author(item.scholar_label)
+    author_surname = _author_surname(item.scholar_label)
+    for date_range in _query_filters(item.year):
+        items = await _fetch_items(
+            query=query,
+            author=author,
+            date_range=date_range,
+            max_rows=max_rows,
+            email=email,
+        )
+        doi = _best_candidate_doi(
+            title=title,
+            year=item.year,
+            items=items,
+            author_surname=author_surname,
+        )
+        if doi:
+            logger.debug("crossref.doi_discovered", extra={"event": "crossref.doi_discovered"})
+            return doi
+    return None

build/lib/app/services/domains/dbops/__init__.py

@@ -0,0 +1,5 @@
+from app.services.domains.dbops.application import run_publication_link_repair
+from app.services.domains.dbops.integrity import collect_integrity_report
+from app.services.domains.dbops.query import list_repair_jobs
+
+__all__ = ["collect_integrity_report", "list_repair_jobs", "run_publication_link_repair"]

build/lib/app/services/domains/dbops/application.py

@@ -0,0 +1,364 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+from sqlalchemy import delete, exists, func, select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import DataRepairJob, IngestionQueueItem, Publication, ScholarProfile, ScholarPublication
+
+
+REPAIR_STATUS_PLANNED = "planned"
+REPAIR_STATUS_RUNNING = "running"
+REPAIR_STATUS_COMPLETED = "completed"
+REPAIR_STATUS_FAILED = "failed"
+SCOPE_MODE_SINGLE_USER = "single_user"
+SCOPE_MODE_ALL_USERS = "all_users"
+
+
+def _utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _normalize_scope_mode(scope_mode: str) -> str:
+    normalized = scope_mode.strip().lower()
+    if normalized in {SCOPE_MODE_SINGLE_USER, SCOPE_MODE_ALL_USERS}:
+        return normalized
+    raise ValueError("Unknown scope mode.")
+
+
+def _scope_user_id(*, scope_mode: str, user_id: int | None) -> int | None:
+    if scope_mode == SCOPE_MODE_SINGLE_USER:
+        if user_id is None:
+            raise ValueError("user_id is required when scope_mode=single_user.")
+        return int(user_id)
+    if user_id is not None:
+        raise ValueError("user_id must be omitted when scope_mode=all_users.")
+    return None
+
+
+def _scope_payload(
+    *,
+    scope_mode: str,
+    user_id: int | None,
+    target_scholar_profile_ids: list[int],
+    orphan_gc: bool,
+) -> dict[str, Any]:
+    payload: dict[str, Any] = {
+        "scope_mode": scope_mode,
+        "scholar_profile_ids": [int(value) for value in target_scholar_profile_ids],
+        "gc_orphan_publications": bool(orphan_gc),
+    }
+    if user_id is not None:
+        payload["user_id"] = int(user_id)
+    return payload
+
+
+async def _target_scholar_profile_ids(
+    db_session: AsyncSession,
+    *,
+    scope_mode: str,
+    user_id: int | None,
+    scholar_profile_ids: list[int] | None,
+) -> list[int]:
+    stmt = select(ScholarProfile.id)
+    if scope_mode == SCOPE_MODE_SINGLE_USER:
+        stmt = stmt.where(ScholarProfile.user_id == user_id)
+    if scholar_profile_ids:
+        normalized_ids = [int(value) for value in scholar_profile_ids]
+        stmt = stmt.where(ScholarProfile.id.in_(normalized_ids))
+    result = await db_session.execute(stmt.order_by(ScholarProfile.id.asc()))
+    ids = [int(row[0]) for row in result.all()]
+    if not ids:
+        raise ValueError("No target scholar profiles found for the requested scope.")
+    return ids
+
+
+async def _count_scope(
+    db_session: AsyncSession,
+    *,
+    user_id: int | None,
+    target_scholar_profile_ids: list[int],
+) -> dict[str, int]:
+    links_result = await db_session.execute(
+        select(func.count())
+        .select_from(ScholarPublication)
+        .where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids))
+    )
+    queue_stmt = (
+        select(func.count())
+        .select_from(IngestionQueueItem)
+        .where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids))
+    )
+    if user_id is not None:
+        queue_stmt = queue_stmt.where(IngestionQueueItem.user_id == user_id)
+    queue_result = await db_session.execute(queue_stmt)
+    return {
+        "target_scholar_count": len(target_scholar_profile_ids),
+        "links_in_scope": int(links_result.scalar_one() or 0),
+        "queue_items_in_scope": int(queue_result.scalar_one() or 0),
+    }
+
+
+async def _count_orphan_publications(db_session: AsyncSession) -> int:
+    stmt = (
+        select(func.count())
+        .select_from(Publication)
+        .where(
+            ~exists(
+                select(1).where(ScholarPublication.publication_id == Publication.id)
+            )
+        )
+    )
+    result = await db_session.execute(stmt)
+    return int(result.scalar_one() or 0)
+
+
+async def _create_job(
+    db_session: AsyncSession,
+    *,
+    requested_by: str | None,
+    scope: dict[str, Any],
+    dry_run: bool,
+) -> DataRepairJob:
+    job = DataRepairJob(
+        job_name="repair_publication_links",
+        requested_by=(requested_by or "").strip() or None,
+        scope=scope,
+        dry_run=dry_run,
+        status=REPAIR_STATUS_PLANNED,
+        summary={},
+    )
+    db_session.add(job)
+    await db_session.flush()
+    return job
+
+
+def _job_summary(
+    *,
+    counts: dict[str, int],
+    dry_run: bool,
+    links_deleted: int,
+    queue_items_deleted: int,
+    scholars_reset: int,
+    orphan_publications_before: int,
+    orphan_publications_deleted: int,
+) -> dict[str, Any]:
+    return {
+        **counts,
+        "dry_run": bool(dry_run),
+        "links_deleted": int(links_deleted),
+        "queue_items_deleted": int(queue_items_deleted),
+        "scholars_reset": int(scholars_reset),
+        "orphan_publications_before": int(orphan_publications_before),
+        "orphan_publications_deleted": int(orphan_publications_deleted),
+    }
+
+
+async def _delete_links_for_targets(db_session: AsyncSession, *, target_scholar_profile_ids: list[int]) -> int:
+    result = await db_session.execute(
+        delete(ScholarPublication).where(
+            ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids)
+        )
+    )
+    return int(result.rowcount or 0)
+
+
+async def _delete_queue_for_targets(
+    db_session: AsyncSession,
+    *,
+    user_id: int | None,
+    target_scholar_profile_ids: list[int],
+) -> int:
+    stmt = delete(IngestionQueueItem).where(
+        IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids)
+    )
+    if user_id is not None:
+        stmt = stmt.where(IngestionQueueItem.user_id == user_id)
+    result = await db_session.execute(stmt)
+    return int(result.rowcount or 0)
+
+
+async def _reset_scholar_tracking_state(
+    db_session: AsyncSession,
+    *,
+    user_id: int | None,
+    target_scholar_profile_ids: list[int],
+) -> int:
+    stmt = update(ScholarProfile).where(ScholarProfile.id.in_(target_scholar_profile_ids))
+    if user_id is not None:
+        stmt = stmt.where(ScholarProfile.user_id == user_id)
+    result = await db_session.execute(
+        stmt.values(
+            baseline_completed=False,
+            last_initial_page_fingerprint_sha256=None,
+            last_initial_page_checked_at=None,
+            last_run_dt=None,
+            last_run_status=None,
+        )
+    )
+    return int(result.rowcount or 0)
+
+
+async def _delete_orphan_publications(db_session: AsyncSession) -> int:
+    result = await db_session.execute(
+        delete(Publication).where(
+            ~exists(select(1).where(ScholarPublication.publication_id == Publication.id))
+        )
+    )
+    return int(result.rowcount or 0)
+
+
+async def _mutation_counts(
+    db_session: AsyncSession,
+    *,
+    user_id: int | None,
+    target_ids: list[int],
+    dry_run: bool,
+    gc_orphan_publications: bool,
+) -> tuple[int, int, int, int]:
+    if dry_run:
+        return 0, 0, 0, 0
+    links_deleted = await _delete_links_for_targets(
+        db_session,
+        target_scholar_profile_ids=target_ids,
+    )
+    queue_deleted = await _delete_queue_for_targets(
+        db_session,
+        user_id=user_id,
+        target_scholar_profile_ids=target_ids,
+    )
+    scholars_reset = await _reset_scholar_tracking_state(
+        db_session,
+        user_id=user_id,
+        target_scholar_profile_ids=target_ids,
+    )
+    orphan_deleted = 0
+    if gc_orphan_publications:
+        orphan_deleted = await _delete_orphan_publications(db_session)
+    return links_deleted, queue_deleted, scholars_reset, orphan_deleted
+
+
+def _result_payload(*, job: DataRepairJob, scope: dict[str, Any], summary: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "job_id": int(job.id),
+        "status": job.status,
+        "scope": scope,
+        "summary": summary,
+    }
+
+
+async def _complete_job(
+    db_session: AsyncSession,
+    *,
+    job: DataRepairJob,
+    summary: dict[str, Any],
+    scope: dict[str, Any],
+) -> dict[str, Any]:
+    job.summary = summary
+    job.status = REPAIR_STATUS_COMPLETED
+    job.finished_at = _utcnow()
+    await db_session.commit()
+    return _result_payload(job=job, scope=scope, summary=summary)
+
+
+async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None:
+    await db_session.rollback()
+    job.status = REPAIR_STATUS_FAILED
+    job.error_text = str(error)
+    job.finished_at = _utcnow()
+    db_session.add(job)
+    await db_session.commit()
+
+
+async def _prepare_repair_job(
+    db_session: AsyncSession,
+    *,
+    scope_mode: str,
+    user_id: int | None,
+    scholar_profile_ids: list[int] | None,
+    dry_run: bool,
+    gc_orphan_publications: bool,
+    requested_by: str | None,
+) -> tuple[int | None, list[int], dict[str, Any], DataRepairJob]:
+    normalized_scope = _normalize_scope_mode(scope_mode)
+    scope_user_id = _scope_user_id(scope_mode=normalized_scope, user_id=user_id)
+    target_ids = await _target_scholar_profile_ids(
+        db_session,
+        scope_mode=normalized_scope,
+        user_id=scope_user_id,
+        scholar_profile_ids=scholar_profile_ids,
+    )
+    scope = _scope_payload(
+        scope_mode=normalized_scope,
+        user_id=scope_user_id,
+        target_scholar_profile_ids=target_ids,
+        orphan_gc=gc_orphan_publications,
+    )
+    job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run)
+    job.status = REPAIR_STATUS_RUNNING
+    job.started_at = _utcnow()
+    return scope_user_id, target_ids, scope, job
+
+
+async def _build_repair_summary(
+    db_session: AsyncSession,
+    *,
+    scope_user_id: int | None,
+    target_ids: list[int],
+    dry_run: bool,
+    gc_orphan_publications: bool,
+) -> dict[str, Any]:
+    counts = await _count_scope(db_session, user_id=scope_user_id, target_scholar_profile_ids=target_ids)
+    orphan_before = await _count_orphan_publications(db_session)
+    links_deleted, queue_deleted, scholars_reset, orphan_deleted = await _mutation_counts(
+        db_session,
+        user_id=scope_user_id,
+        target_ids=target_ids,
+        dry_run=dry_run,
+        gc_orphan_publications=gc_orphan_publications,
+    )
+    return _job_summary(
+        counts=counts,
+        dry_run=dry_run,
+        links_deleted=links_deleted,
+        queue_items_deleted=queue_deleted,
+        scholars_reset=scholars_reset,
+        orphan_publications_before=orphan_before,
+        orphan_publications_deleted=orphan_deleted,
+    )
+
+
+async def run_publication_link_repair(
+    db_session: AsyncSession,
+    *,
+    scope_mode: str = SCOPE_MODE_SINGLE_USER,
+    user_id: int | None = None,
+    scholar_profile_ids: list[int] | None = None,
+    dry_run: bool = True,
+    gc_orphan_publications: bool = False,
+    requested_by: str | None = None,
+) -> dict[str, Any]:
+    scope_user_id, target_ids, scope, job = await _prepare_repair_job(
+        db_session,
+        scope_mode=scope_mode,
+        user_id=user_id,
+        scholar_profile_ids=scholar_profile_ids,
+        dry_run=dry_run,
+        gc_orphan_publications=gc_orphan_publications,
+        requested_by=requested_by,
+    )
+
+    try:
+        summary = await _build_repair_summary(
+            db_session,
+            scope_user_id=scope_user_id,
+            target_ids=target_ids,
+            dry_run=dry_run,
+            gc_orphan_publications=gc_orphan_publications,
+        )
+        return await _complete_job(db_session, job=job, summary=summary, scope=scope)
+    except Exception as exc:
+        await _fail_job(db_session, job=job, error=exc)
+        raise

build/lib/app/services/domains/dbops/integrity.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+from sqlalchemy import func, select, text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import Publication
+
+INTEGRITY_CHECK_DEFS = (
+    (
+        "legacy_cluster_id_format",
+        "warning",
+        "Publications with non-namespaced cluster IDs.",
+    ),
+    (
+        "negative_citation_count",
+        "failure",
+        "Publications with negative citation counts.",
+    ),
+    (
+        "orphan_publications",
+        "warning",
+        "Publications with no scholar links.",
+    ),
+    (
+        "orphan_scholar_publication_links",
+        "failure",
+        "Link rows missing parent scholar/publication.",
+    ),
+    (
+        "duplicate_fingerprint_keys",
+        "failure",
+        "Duplicate publication fingerprint keys.",
+    ),
+    (
+        "duplicate_cluster_ids",
+        "failure",
+        "Duplicate non-null publication cluster IDs.",
+    ),
+    (
+        "missing_pdf_url",
+        "metric",
+        "Publications without a resolved PDF URL.",
+    ),
+)
+
+
+async def _legacy_cluster_id_count(db_session: AsyncSession) -> int:
+    stmt = (
+        select(func.count())
+        .select_from(Publication)
+        .where(Publication.cluster_id.is_not(None))
+        .where(~Publication.cluster_id.like("cfv:%"))
+        .where(~Publication.cluster_id.like("cluster:%"))
+    )
+    result = await db_session.execute(stmt)
+    return int(result.scalar_one() or 0)
+
+
+async def _negative_citation_count(db_session: AsyncSession) -> int:
+    result = await db_session.execute(
+        select(func.count()).select_from(Publication).where(Publication.citation_count < 0)
+    )
+    return int(result.scalar_one() or 0)
+
+
+async def _missing_pdf_url_count(db_session: AsyncSession) -> int:
+    result = await db_session.execute(
+        select(func.count()).select_from(Publication).where(Publication.pdf_url.is_(None))
+    )
+    return int(result.scalar_one() or 0)
+
+
+async def _count_from_sql(db_session: AsyncSession, *, sql: str) -> int:
+    result = await db_session.execute(text(sql))
+    return int(result.scalar_one() or 0)
+
+
+def _issues_for_severity(*, checks: list[dict[str, Any]], severity: str) -> list[str]:
+    return [row["name"] for row in checks if row["severity"] == severity and row["count"] > 0]
+
+
+def _check_row(*, name: str, count: int, severity: str, message: str) -> dict[str, Any]:
+    return {
+        "name": name,
+        "count": int(count),
+        "severity": severity,
+        "message": message,
+    }
+
+
+async def _collect_counts(db_session: AsyncSession) -> dict[str, int]:
+    orphan_publications = await _count_from_sql(
+        db_session,
+        sql=(
+            "SELECT count(*) FROM publications p "
+            "LEFT JOIN scholar_publications sp ON sp.publication_id = p.id "
+            "WHERE sp.publication_id IS NULL"
+        ),
+    )
+    orphan_links = await _count_from_sql(
+        db_session,
+        sql=(
+            "SELECT count(*) FROM scholar_publications sp "
+            "LEFT JOIN publications p ON p.id = sp.publication_id "
+            "LEFT JOIN scholar_profiles s ON s.id = sp.scholar_profile_id "
+            "WHERE p.id IS NULL OR s.id IS NULL"
+        ),
+    )
+    duplicate_fingerprints = await _count_from_sql(
+        db_session,
+        sql=(
+            "SELECT count(*) FROM ("
+            "SELECT fingerprint_sha256 FROM publications "
+            "GROUP BY fingerprint_sha256 HAVING count(*) > 1"
+            ") dup"
+        ),
+    )
+    duplicate_cluster_ids = await _count_from_sql(
+        db_session,
+        sql=(
+            "SELECT count(*) FROM ("
+            "SELECT cluster_id FROM publications "
+            "WHERE cluster_id IS NOT NULL "
+            "GROUP BY cluster_id HAVING count(*) > 1"
+            ") dup"
+        ),
+    )
+    return {
+        "legacy_cluster_id_format": await _legacy_cluster_id_count(db_session),
+        "negative_citation_count": await _negative_citation_count(db_session),
+        "orphan_publications": orphan_publications,
+        "orphan_scholar_publication_links": orphan_links,
+        "duplicate_fingerprint_keys": duplicate_fingerprints,
+        "duplicate_cluster_ids": duplicate_cluster_ids,
+        "missing_pdf_url": await _missing_pdf_url_count(db_session),
+    }
+
+
+def _build_checks(*, counts: dict[str, int]) -> list[dict[str, Any]]:
+    return [
+        _check_row(
+            name=name,
+            count=counts[name],
+            severity=severity,
+            message=message,
+        )
+        for name, severity, message in INTEGRITY_CHECK_DEFS
+    ]
+
+
+def _status_from_issues(*, failures: list[str], warnings: list[str]) -> str:
+    status = "ok"
+    if failures:
+        status = "failed"
+    elif warnings:
+        status = "warning"
+    return status
+
+
+async def collect_integrity_report(db_session: AsyncSession) -> dict[str, Any]:
+    counts = await _collect_counts(db_session)
+    checks = _build_checks(counts=counts)
+    failures = _issues_for_severity(checks=checks, severity="failure")
+    warnings = _issues_for_severity(checks=checks, severity="warning")
+
+    return {
+        "status": _status_from_issues(failures=failures, warnings=warnings),
+        "checked_at": datetime.now(timezone.utc).isoformat(),
+        "failures": failures,
+        "warnings": warnings,
+        "checks": checks,
+    }

build/lib/app/services/domains/dbops/query.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import DataRepairJob
+
+
+def _bounded_limit(limit: int) -> int:
+    return max(1, min(int(limit), 200))
+
+
+async def list_repair_jobs(
+    db_session: AsyncSession,
+    *,
+    limit: int = 50,
+) -> list[DataRepairJob]:
+    bounded = _bounded_limit(limit)
+    result = await db_session.execute(
+        select(DataRepairJob).order_by(DataRepairJob.created_at.desc()).limit(bounded)
+    )
+    return list(result.scalars())

build/lib/app/services/domains/doi/normalize.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import re
+from urllib.parse import unquote
+
+DOI_RE = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
+
+
+def normalize_doi(value: str | None) -> str | None:
+    if not value:
+        return None
+    match = DOI_RE.search(unquote(value))
+    if not match:
+        return None
+    return match.group(0).rstrip(" .;,)").lower()
+
+
+def first_doi_from_texts(*values: str | None) -> str | None:
+    for value in values:
+        doi = normalize_doi(value)
+        if doi:
+            return doi
+    return None

build/lib/app/services/domains/ingestion/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

build/lib/app/services/domains/ingestion/constants.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import re
+
+from app.services.domains.scholar.parser import ParseState
+
+TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+")
+WORD_RE = re.compile(r"[a-z0-9]+")
+HTML_TAG_RE = re.compile(r"<[^>]+>", re.S)
+SPACE_RE = re.compile(r"\s+")
+
+FAILED_STATES = {
+    ParseState.BLOCKED_OR_CAPTCHA.value,
+    ParseState.LAYOUT_CHANGED.value,
+    ParseState.NETWORK_ERROR.value,
+    "ingestion_error",
+}
+
+FAILURE_BUCKET_BLOCKED = "blocked_or_captcha"
+FAILURE_BUCKET_NETWORK = "network_error"
+FAILURE_BUCKET_LAYOUT = "layout_changed"
+FAILURE_BUCKET_INGESTION = "ingestion_error"
+FAILURE_BUCKET_OTHER = "other_failure"
+
+RUN_LOCK_NAMESPACE = 8217
+RESUMABLE_PARTIAL_REASONS = {
+    "max_pages_reached",
+    "pagination_cursor_stalled",
+}
+RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",)
+INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS = 30

build/lib/app/services/domains/ingestion/fingerprints.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from typing import Any
+from urllib.parse import urljoin
+
+from app.services.domains.ingestion.constants import (
+    HTML_TAG_RE,
+    INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS,
+    SPACE_RE,
+    TITLE_ALNUM_RE,
+    WORD_RE,
+)
+from app.services.domains.scholar.parser import ParseState, ParsedProfilePage, PublicationCandidate
+
+
+def normalize_title(value: str) -> str:
+    lowered = value.lower()
+    return TITLE_ALNUM_RE.sub("", lowered)
+
+
+def _first_author_last_name(authors_text: str | None) -> str:
+    if not authors_text:
+        return ""
+    first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
+    words = WORD_RE.findall(first_author)
+    if not words:
+        return ""
+    return words[-1]
+
+
+def _first_venue_word(venue_text: str | None) -> str:
+    if not venue_text:
+        return ""
+    words = WORD_RE.findall(venue_text.lower())
+    if not words:
+        return ""
+    return words[0]
+
+
+def build_publication_fingerprint(candidate: PublicationCandidate) -> str:
+    canonical = "|".join(
+        [
+            normalize_title(candidate.title),
+            str(candidate.year) if candidate.year is not None else "",
+            _first_author_last_name(candidate.authors_text),
+            _first_venue_word(candidate.venue_text),
+        ]
+    )
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def build_initial_page_fingerprint(parsed_page: ParsedProfilePage) -> str | None:
+    if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
+        return None
+
+    normalized_rows: list[dict[str, Any]] = []
+    for publication in parsed_page.publications[:INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS]:
+        normalized_rows.append(
+            {
+                "cluster_id": publication.cluster_id or "",
+                "title_normalized": normalize_title(publication.title),
+                "year": publication.year,
+                "citation_count": publication.citation_count,
+            }
+        )
+
+    payload = {
+        "state": parsed_page.state.value,
+        "articles_range": parsed_page.articles_range or "",
+        "has_show_more_button": parsed_page.has_show_more_button,
+        "profile_name": parsed_page.profile_name or "",
+        "publications": normalized_rows,
+    }
+    canonical = json.dumps(
+        payload,
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=True,
+    )
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def build_publication_url(path_or_url: str | None) -> str | None:
+    if not path_or_url:
+        return None
+    return urljoin("https://scholar.google.com", path_or_url)
+
+
+def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int:
+    if articles_range:
+        numbers = re.findall(r"\d+", articles_range)
+        if len(numbers) >= 2:
+            try:
+                return int(numbers[1])
+            except ValueError:
+                pass
+    return int(fallback)
+
+
+def _title_tokens(value: str) -> set[str]:
+    """Extract normalized word tokens for fuzzy title comparison."""
+    return set(WORD_RE.findall(value.lower()))
+
+
+def fuzzy_titles_match(
+    title_a: str,
+    title_b: str,
+    *,
+    threshold: float = 0.85,
+) -> bool:
+    """Return True if two titles are near-duplicates by token-level Jaccard similarity.
+
+    A threshold of 0.85 catches common academic duplicate patterns:
+    differences in punctuation, minor word variations, subtitle changes.
+    """
+    tokens_a = _title_tokens(title_a)
+    tokens_b = _title_tokens(title_b)
+    if not tokens_a or not tokens_b:
+        return False
+    intersection = tokens_a & tokens_b
+    union = tokens_a | tokens_b
+    return (len(intersection) / len(union)) >= threshold
+
+
+def _dedupe_publication_candidates(
+    publications: list[PublicationCandidate],
+) -> list[PublicationCandidate]:
+    deduped: list[PublicationCandidate] = []
+    seen: set[str] = set()
+    seen_titles: list[tuple[str, int]] = []  # (normalized_title, index into deduped)
+
+    for publication in publications:
+        if publication.cluster_id:
+            identity = f"cluster:{publication.cluster_id}"
+        else:
+            identity = "|".join(
+                [
+                    "fallback",
+                    normalize_title(publication.title),
+                    str(publication.year) if publication.year is not None else "",
+                    _first_author_last_name(publication.authors_text),
+                    _first_venue_word(publication.venue_text),
+                ]
+            )
+        if identity in seen:
+            continue
+
+        # Fuzzy title check — catch near-identical titles not caught by exact fingerprint
+        norm_title = normalize_title(publication.title)
+        is_fuzzy_dup = False
+        for existing_title, _idx in seen_titles:
+            if fuzzy_titles_match(norm_title, existing_title):
+                is_fuzzy_dup = True
+                break
+        if is_fuzzy_dup:
+            continue
+
+        seen.add(identity)
+        seen_titles.append((norm_title, len(deduped)))
+        deduped.append(publication)
+    return deduped
+
+
+def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None:
+    if not body:
+        return None
+    flattened = SPACE_RE.sub(" ", HTML_TAG_RE.sub(" ", body)).strip()
+    if not flattened:
+        return None
+    if len(flattened) <= max_chars:
+        return flattened
+    return f"{flattened[:max_chars - 1]}..."

build/lib/app/services/domains/ingestion/queue.py

@@ -0,0 +1,348 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import IngestionQueueItem, QueueItemStatus
+
+
+@dataclass(frozen=True)
+class ContinuationQueueJob:
+    id: int
+    user_id: int
+    scholar_profile_id: int
+    resume_cstart: int
+    reason: str
+    status: str
+    attempt_count: int
+    next_attempt_dt: datetime
+
+
+ACTIVE_QUEUE_STATUSES: tuple[str, ...] = (
+    QueueItemStatus.QUEUED.value,
+    QueueItemStatus.RETRYING.value,
+)
+
+
+def normalize_cstart(value: int | None) -> int:
+    if value is None:
+        return 0
+    return max(0, int(value))
+
+
+def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_seconds: int) -> int:
+    base = max(1, int(base_seconds))
+    attempts = max(1, int(attempt_count))
+    maximum = max(base, int(max_seconds))
+    seconds = base * (2 ** max(0, attempts - 1))
+    return min(seconds, maximum)
+
+
+async def _get_item_for_user_scholar(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+) -> IngestionQueueItem | None:
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(
+            IngestionQueueItem.user_id == user_id,
+            IngestionQueueItem.scholar_profile_id == scholar_profile_id,
+        )
+    )
+    return result.scalar_one_or_none()
+
+
+def _build_queue_item(
+    *,
+    now: datetime,
+    user_id: int,
+    scholar_profile_id: int,
+    normalized_cstart: int,
+    reason: str,
+    run_id: int | None,
+    next_attempt_dt: datetime,
+) -> IngestionQueueItem:
+    return IngestionQueueItem(
+        user_id=user_id,
+        scholar_profile_id=scholar_profile_id,
+        resume_cstart=normalized_cstart,
+        reason=reason,
+        status=QueueItemStatus.QUEUED.value,
+        attempt_count=0,
+        next_attempt_dt=next_attempt_dt,
+        last_run_id=run_id,
+        last_error=None,
+        dropped_reason=None,
+        dropped_at=None,
+        created_at=now,
+        updated_at=now,
+    )
+
+
+def _update_existing_queue_item(
+    *,
+    item: IngestionQueueItem,
+    now: datetime,
+    normalized_cstart: int,
+    reason: str,
+    run_id: int | None,
+    next_attempt_dt: datetime,
+) -> None:
+    item.resume_cstart = normalized_cstart
+    item.reason = reason
+    if item.status == QueueItemStatus.DROPPED.value:
+        item.attempt_count = 0
+    item.status = QueueItemStatus.QUEUED.value
+    item.next_attempt_dt = next_attempt_dt
+    item.last_run_id = run_id
+    item.last_error = None
+    item.dropped_reason = None
+    item.dropped_at = None
+    item.updated_at = now
+
+
+async def upsert_job(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+    resume_cstart: int,
+    reason: str,
+    run_id: int | None,
+    delay_seconds: int,
+) -> IngestionQueueItem:
+    now = datetime.now(timezone.utc)
+    next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds)))
+    item = await _get_item_for_user_scholar(
+        db_session,
+        user_id=user_id,
+        scholar_profile_id=scholar_profile_id,
+    )
+    normalized_cstart = normalize_cstart(resume_cstart)
+    if item is None:
+        item = _build_queue_item(
+            now=now,
+            user_id=user_id,
+            scholar_profile_id=scholar_profile_id,
+            normalized_cstart=normalized_cstart,
+            reason=reason,
+            run_id=run_id,
+            next_attempt_dt=next_attempt_dt,
+        )
+        db_session.add(item)
+        return item
+
+    _update_existing_queue_item(
+        item=item,
+        now=now,
+        normalized_cstart=normalized_cstart,
+        reason=reason,
+        run_id=run_id,
+        next_attempt_dt=next_attempt_dt,
+    )
+    return item
+
+
+async def clear_job_for_scholar(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+) -> bool:
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(
+            IngestionQueueItem.user_id == user_id,
+            IngestionQueueItem.scholar_profile_id == scholar_profile_id,
+        )
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return False
+    await db_session.delete(item)
+    return True
+
+
+async def delete_job_by_id(
+    db_session: AsyncSession,
+    *,
+    job_id: int,
+) -> bool:
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return False
+    await db_session.delete(item)
+    return True
+
+
+async def list_due_jobs(
+    db_session: AsyncSession,
+    *,
+    now: datetime,
+    limit: int,
+) -> list[ContinuationQueueJob]:
+    result = await db_session.execute(
+        select(IngestionQueueItem)
+        .where(
+            IngestionQueueItem.next_attempt_dt <= now,
+            IngestionQueueItem.status.in_(ACTIVE_QUEUE_STATUSES),
+        )
+        .order_by(
+            IngestionQueueItem.next_attempt_dt.asc(),
+            IngestionQueueItem.id.asc(),
+        )
+        .limit(limit)
+    )
+    rows = list(result.scalars().all())
+    jobs: list[ContinuationQueueJob] = []
+    for row in rows:
+        jobs.append(
+            ContinuationQueueJob(
+                id=int(row.id),
+                user_id=int(row.user_id),
+                scholar_profile_id=int(row.scholar_profile_id),
+                resume_cstart=normalize_cstart(row.resume_cstart),
+                reason=row.reason,
+                status=row.status,
+                attempt_count=int(row.attempt_count),
+                next_attempt_dt=row.next_attempt_dt,
+            )
+        )
+    return jobs
+
+
+async def increment_attempt_count(
+    db_session: AsyncSession,
+    *,
+    job_id: int,
+) -> IngestionQueueItem | None:
+    now = datetime.now(timezone.utc)
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    item.attempt_count = int(item.attempt_count or 0) + 1
+    item.updated_at = now
+    return item
+
+
+async def reset_attempt_count(
+    db_session: AsyncSession,
+    *,
+    job_id: int,
+) -> IngestionQueueItem | None:
+    now = datetime.now(timezone.utc)
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    item.attempt_count = 0
+    item.updated_at = now
+    return item
+
+
+async def reschedule_job(
+    db_session: AsyncSession,
+    *,
+    job_id: int,
+    delay_seconds: int,
+    reason: str,
+    error: str | None = None,
+) -> IngestionQueueItem | None:
+    now = datetime.now(timezone.utc)
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    item.next_attempt_dt = now + timedelta(seconds=max(1, int(delay_seconds)))
+    item.status = QueueItemStatus.QUEUED.value
+    item.reason = reason
+    item.last_error = error
+    item.dropped_reason = None
+    item.dropped_at = None
+    item.updated_at = now
+    return item
+
+
+async def mark_retrying(
+    db_session: AsyncSession,
+    *,
+    job_id: int,
+    reason: str | None = None,
+) -> IngestionQueueItem | None:
+    now = datetime.now(timezone.utc)
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    if item.status == QueueItemStatus.DROPPED.value:
+        return item
+    item.status = QueueItemStatus.RETRYING.value
+    if reason:
+        item.reason = reason
+    item.updated_at = now
+    return item
+
+
+async def mark_dropped(
+    db_session: AsyncSession,
+    *,
+    job_id: int,
+    reason: str,
+    error: str | None = None,
+) -> IngestionQueueItem | None:
+    now = datetime.now(timezone.utc)
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    item.status = QueueItemStatus.DROPPED.value
+    item.reason = "dropped"
+    item.dropped_reason = reason
+    item.dropped_at = now
+    if error is not None:
+        item.last_error = error
+    item.updated_at = now
+    return item
+
+
+async def mark_queued_now(
+    db_session: AsyncSession,
+    *,
+    job_id: int,
+    reason: str,
+    reset_attempt_count: bool = False,
+) -> IngestionQueueItem | None:
+    now = datetime.now(timezone.utc)
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    item.status = QueueItemStatus.QUEUED.value
+    item.reason = reason
+    item.next_attempt_dt = now
+    if reset_attempt_count:
+        item.attempt_count = 0
+    item.last_error = None
+    item.dropped_reason = None
+    item.dropped_at = None
+    item.updated_at = now
+    return item

build/lib/app/services/domains/ingestion/safety.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+from app.db.models import UserSetting
+
+COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD = "blocked_failure_threshold_exceeded"
+COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD = "network_failure_threshold_exceeded"
+
+_COUNTER_CONSECUTIVE_BLOCKED_RUNS = "consecutive_blocked_runs"
+_COUNTER_CONSECUTIVE_NETWORK_RUNS = "consecutive_network_runs"
+_COUNTER_COOLDOWN_ENTRY_COUNT = "cooldown_entry_count"
+_COUNTER_BLOCKED_START_COUNT = "blocked_start_count"
+_COUNTER_LAST_BLOCKED_FAILURE_COUNT = "last_blocked_failure_count"
+_COUNTER_LAST_NETWORK_FAILURE_COUNT = "last_network_failure_count"
+_COUNTER_LAST_EVALUATED_RUN_ID = "last_evaluated_run_id"
+
+
+def _utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _safe_int(value: Any, default: int = 0) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _safe_optional_int(value: Any) -> int | None:
+    if value is None:
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _normalize_datetime(value: datetime | None) -> datetime | None:
+    if value is None:
+        return None
+    if value.tzinfo is None:
+        return value.replace(tzinfo=timezone.utc)
+    return value.astimezone(timezone.utc)
+
+
+def _state_dict(settings: UserSetting) -> dict[str, Any]:
+    state = settings.scrape_safety_state
+    if isinstance(state, dict):
+        return state
+    return {}
+
+
+def _counters_from_state(settings: UserSetting) -> dict[str, Any]:
+    state = _state_dict(settings)
+    return {
+        _COUNTER_CONSECUTIVE_BLOCKED_RUNS: max(
+            0,
+            _safe_int(state.get(_COUNTER_CONSECUTIVE_BLOCKED_RUNS), 0),
+        ),
+        _COUNTER_CONSECUTIVE_NETWORK_RUNS: max(
+            0,
+            _safe_int(state.get(_COUNTER_CONSECUTIVE_NETWORK_RUNS), 0),
+        ),
+        _COUNTER_COOLDOWN_ENTRY_COUNT: max(
+            0,
+            _safe_int(state.get(_COUNTER_COOLDOWN_ENTRY_COUNT), 0),
+        ),
+        _COUNTER_BLOCKED_START_COUNT: max(
+            0,
+            _safe_int(state.get(_COUNTER_BLOCKED_START_COUNT), 0),
+        ),
+        _COUNTER_LAST_BLOCKED_FAILURE_COUNT: max(
+            0,
+            _safe_int(state.get(_COUNTER_LAST_BLOCKED_FAILURE_COUNT), 0),
+        ),
+        _COUNTER_LAST_NETWORK_FAILURE_COUNT: max(
+            0,
+            _safe_int(state.get(_COUNTER_LAST_NETWORK_FAILURE_COUNT), 0),
+        ),
+        _COUNTER_LAST_EVALUATED_RUN_ID: _safe_optional_int(
+            state.get(_COUNTER_LAST_EVALUATED_RUN_ID),
+        ),
+    }
+
+
+def _cooldown_reason_label(reason: str | None) -> str | None:
+    if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
+        return "Blocked responses exceeded safety threshold"
+    if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
+        return "Network failures exceeded safety threshold"
+    return None
+
+
+def _recommended_action(reason: str | None) -> str | None:
+    if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
+        return (
+            "Google Scholar appears to be blocking requests. Wait for cooldown to expire, "
+            "increase request delay, and avoid repeated manual retries."
+        )
+    if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
+        return (
+            "Network failures crossed the threshold. Verify connectivity and retry after cooldown."
+        )
+    return None
+
+
+def is_cooldown_active(
+    settings: UserSetting,
+    *,
+    now_utc: datetime | None = None,
+) -> bool:
+    now = now_utc or _utcnow()
+    cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
+    if cooldown_until is None:
+        return False
+    return cooldown_until > now
+
+
+def clear_expired_cooldown(
+    settings: UserSetting,
+    *,
+    now_utc: datetime | None = None,
+) -> bool:
+    now = now_utc or _utcnow()
+    cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
+    if cooldown_until is None:
+        return False
+    if cooldown_until > now:
+        return False
+    settings.scrape_cooldown_until = None
+    settings.scrape_cooldown_reason = None
+    return True
+
+
+def register_cooldown_blocked_start(
+    settings: UserSetting,
+    *,
+    now_utc: datetime | None = None,
+) -> dict[str, Any]:
+    now = now_utc or _utcnow()
+    counters = _counters_from_state(settings)
+    counters[_COUNTER_BLOCKED_START_COUNT] = int(counters[_COUNTER_BLOCKED_START_COUNT]) + 1
+    settings.scrape_safety_state = counters
+    return get_safety_state_payload(settings, now_utc=now)
+
+
+def _update_run_counters(
+    *,
+    counters: dict[str, Any],
+    run_id: int,
+    blocked_failure_count: int,
+    network_failure_count: int,
+) -> tuple[int, int]:
+    bounded_blocked_failures = max(0, int(blocked_failure_count))
+    bounded_network_failures = max(0, int(network_failure_count))
+    counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures
+    counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures
+    counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id)
+    counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = (
+        int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1
+        if bounded_blocked_failures > 0
+        else 0
+    )
+    counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = (
+        int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1
+        if bounded_network_failures > 0
+        else 0
+    )
+    return bounded_blocked_failures, bounded_network_failures
+
+
+def _resolve_cooldown_trigger(
+    *,
+    blocked_failures: int,
+    network_failures: int,
+    blocked_failure_threshold: int,
+    network_failure_threshold: int,
+    blocked_cooldown_seconds: int,
+    network_cooldown_seconds: int,
+) -> tuple[str | None, int]:
+    if blocked_failures >= max(1, int(blocked_failure_threshold)):
+        return COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD, max(60, int(blocked_cooldown_seconds))
+    if network_failures >= max(1, int(network_failure_threshold)):
+        return COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD, max(60, int(network_cooldown_seconds))
+    return None, 0
+
+
+def _apply_cooldown_decision(
+    *,
+    settings: UserSetting,
+    counters: dict[str, Any],
+    now: datetime,
+    reason: str | None,
+    cooldown_seconds: int,
+) -> None:
+    if reason is None:
+        clear_expired_cooldown(settings, now_utc=now)
+        return
+    settings.scrape_cooldown_reason = reason
+    settings.scrape_cooldown_until = now + timedelta(seconds=max(60, int(cooldown_seconds)))
+    counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1
+
+
+def apply_run_safety_outcome(
+    settings: UserSetting,
+    *,
+    run_id: int,
+    blocked_failure_count: int,
+    network_failure_count: int,
+    blocked_failure_threshold: int,
+    network_failure_threshold: int,
+    blocked_cooldown_seconds: int,
+    network_cooldown_seconds: int,
+    now_utc: datetime | None = None,
+) -> tuple[dict[str, Any], str | None]:
+    now = now_utc or _utcnow()
+    counters = _counters_from_state(settings)
+    blocked_failures, network_failures = _update_run_counters(
+        counters=counters,
+        run_id=run_id,
+        blocked_failure_count=blocked_failure_count,
+        network_failure_count=network_failure_count,
+    )
+    reason, cooldown_seconds = _resolve_cooldown_trigger(
+        blocked_failures=blocked_failures,
+        network_failures=network_failures,
+        blocked_failure_threshold=blocked_failure_threshold,
+        network_failure_threshold=network_failure_threshold,
+        blocked_cooldown_seconds=blocked_cooldown_seconds,
+        network_cooldown_seconds=network_cooldown_seconds,
+    )
+    _apply_cooldown_decision(
+        settings=settings,
+        counters=counters,
+        now=now,
+        reason=reason,
+        cooldown_seconds=cooldown_seconds,
+    )
+    settings.scrape_safety_state = counters
+    return get_safety_state_payload(settings, now_utc=now), reason
+
+
+def get_safety_state_payload(
+    settings: UserSetting,
+    *,
+    now_utc: datetime | None = None,
+) -> dict[str, Any]:
+    now = now_utc or _utcnow()
+    cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
+    cooldown_active = bool(cooldown_until is not None and cooldown_until > now)
+    cooldown_remaining_seconds = 0
+    if cooldown_active and cooldown_until is not None:
+        cooldown_remaining_seconds = max(0, int((cooldown_until - now).total_seconds()))
+
+    reason = settings.scrape_cooldown_reason if cooldown_active else None
+
+    return {
+        "cooldown_active": cooldown_active,
+        "cooldown_reason": reason,
+        "cooldown_reason_label": _cooldown_reason_label(reason),
+        "cooldown_until": cooldown_until,
+        "cooldown_remaining_seconds": cooldown_remaining_seconds,
+        "recommended_action": _recommended_action(reason),
+        "counters": _counters_from_state(settings),
+    }
+
+
+def get_safety_event_context(
+    settings: UserSetting,
+    *,
+    now_utc: datetime | None = None,
+) -> dict[str, Any]:
+    payload = get_safety_state_payload(settings, now_utc=now_utc)
+    return {
+        "cooldown_active": bool(payload.get("cooldown_active")),
+        "cooldown_reason": payload.get("cooldown_reason"),
+        "cooldown_until": payload.get("cooldown_until"),
+        "cooldown_remaining_seconds": int(payload.get("cooldown_remaining_seconds") or 0),
+        "safety_counters": payload.get("counters", {}),
+    }

build/lib/app/services/domains/ingestion/scheduler.py

@@ -0,0 +1,622 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+import logging
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import (
+    CrawlRun,
+    QueueItemStatus,
+    RunTriggerType,
+    ScholarProfile,
+    User,
+    UserSetting,
+)
+from app.db.session import get_session_factory
+from app.services.domains.ingestion import queue as queue_service
+from app.services.domains.ingestion.application import (
+    RunAlreadyInProgressError,
+    RunBlockedBySafetyPolicyError,
+    ScholarIngestionService,
+)
+from app.services.domains.settings import application as user_settings_service
+from app.services.domains.scholar.source import LiveScholarSource
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+def _request_delay_floor_seconds() -> int:
+    return user_settings_service.resolve_request_delay_minimum(
+        settings.ingestion_min_request_delay_seconds
+    )
+
+
+def _effective_request_delay_seconds(value: int | None) -> int:
+    floor = _request_delay_floor_seconds()
+    try:
+        parsed = int(value) if value is not None else floor
+    except (TypeError, ValueError):
+        parsed = floor
+    return max(floor, parsed)
+
+
+@dataclass(frozen=True)
+class _AutoRunCandidate:
+    user_id: int
+    run_interval_minutes: int
+    request_delay_seconds: int
+    cooldown_until: datetime | None
+    cooldown_reason: str | None
+
+
+class SchedulerService:
+    def __init__(
+        self,
+        *,
+        enabled: bool,
+        tick_seconds: int,
+        network_error_retries: int,
+        retry_backoff_seconds: float,
+        max_pages_per_scholar: int,
+        page_size: int,
+        continuation_queue_enabled: bool,
+        continuation_base_delay_seconds: int,
+        continuation_max_delay_seconds: int,
+        continuation_max_attempts: int,
+        queue_batch_size: int,
+    ) -> None:
+        self._enabled = enabled
+        self._tick_seconds = max(5, int(tick_seconds))
+        self._network_error_retries = max(0, int(network_error_retries))
+        self._retry_backoff_seconds = max(0.0, float(retry_backoff_seconds))
+        self._max_pages_per_scholar = max(1, int(max_pages_per_scholar))
+        self._page_size = max(1, int(page_size))
+        self._continuation_queue_enabled = bool(continuation_queue_enabled)
+        self._continuation_base_delay_seconds = max(1, int(continuation_base_delay_seconds))
+        self._continuation_max_delay_seconds = max(
+            self._continuation_base_delay_seconds,
+            int(continuation_max_delay_seconds),
+        )
+        self._continuation_max_attempts = max(1, int(continuation_max_attempts))
+        self._queue_batch_size = max(1, int(queue_batch_size))
+        self._task: asyncio.Task[None] | None = None
+        self._source = LiveScholarSource()
+
+    async def start(self) -> None:
+        if not self._enabled:
+            logger.info(
+                "scheduler.disabled",
+                extra={
+                    "event": "scheduler.disabled",
+                },
+            )
+            return
+        if self._task is not None:
+            return
+        self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler")
+        logger.info(
+            "scheduler.started",
+            extra={
+                "event": "scheduler.started",
+                "tick_seconds": self._tick_seconds,
+                "network_error_retries": self._network_error_retries,
+                "retry_backoff_seconds": self._retry_backoff_seconds,
+                "max_pages_per_scholar": self._max_pages_per_scholar,
+                "page_size": self._page_size,
+                "continuation_queue_enabled": self._continuation_queue_enabled,
+                "continuation_base_delay_seconds": self._continuation_base_delay_seconds,
+                "continuation_max_delay_seconds": self._continuation_max_delay_seconds,
+                "continuation_max_attempts": self._continuation_max_attempts,
+                "queue_batch_size": self._queue_batch_size,
+            },
+        )
+
+    async def stop(self) -> None:
+        if self._task is None:
+            return
+        self._task.cancel()
+        try:
+            await self._task
+        except asyncio.CancelledError:
+            pass
+        finally:
+            self._task = None
+        logger.info("scheduler.stopped", extra={"event": "scheduler.stopped"})
+
+    async def _run_loop(self) -> None:
+        while True:
+            try:
+                await self._tick_once()
+            except asyncio.CancelledError:
+                raise
+            except Exception:
+                logger.exception(
+                    "scheduler.tick_failed",
+                    extra={
+                        "event": "scheduler.tick_failed",
+                    },
+                )
+            await asyncio.sleep(float(self._tick_seconds))
+
+    async def _tick_once(self) -> None:
+        if self._continuation_queue_enabled:
+            await self._drain_continuation_queue()
+        
+        await self._drain_pdf_queue()
+
+        candidates = await self._load_candidates()
+        if not candidates:
+            return
+        now = datetime.now(timezone.utc)
+        for candidate in candidates:
+            if not await self._is_due(candidate, now=now):
+                continue
+            await self._run_candidate(candidate)
+
+    async def _load_candidate_rows(self) -> list[tuple]:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            result = await session.execute(
+                select(
+                    UserSetting.user_id,
+                    UserSetting.run_interval_minutes,
+                    UserSetting.request_delay_seconds,
+                    UserSetting.scrape_cooldown_until,
+                    UserSetting.scrape_cooldown_reason,
+                )
+                .join(User, User.id == UserSetting.user_id)
+                .where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True))
+                .order_by(UserSetting.user_id.asc())
+            )
+            return result.all()
+
+    @staticmethod
+    def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None:
+        user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row
+        if cooldown_until is not None and cooldown_until.tzinfo is None:
+            cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
+        if cooldown_until is not None and cooldown_until > now_utc:
+            logger.info(
+                "scheduler.run_skipped_safety_cooldown_precheck",
+                extra={
+                    "event": "scheduler.run_skipped_safety_cooldown_precheck",
+                    "user_id": int(user_id),
+                    "reason": cooldown_reason,
+                    "cooldown_until": cooldown_until,
+                    "cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()),
+                    "metric_name": "scheduler_run_skipped_safety_cooldown_total",
+                    "metric_value": 1,
+                },
+            )
+            return None
+        return _AutoRunCandidate(
+            user_id=int(user_id),
+            run_interval_minutes=int(run_interval_minutes),
+            request_delay_seconds=_effective_request_delay_seconds(request_delay_seconds),
+            cooldown_until=cooldown_until,
+            cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
+        )
+
+    async def _load_candidates(self) -> list[_AutoRunCandidate]:
+        if not settings.ingestion_automation_allowed:
+            return []
+        rows = await self._load_candidate_rows()
+        now_utc = datetime.now(timezone.utc)
+        candidates: list[_AutoRunCandidate] = []
+        for row in rows:
+            candidate = self._candidate_from_row(row, now_utc=now_utc)
+            if candidate is not None:
+                candidates.append(candidate)
+        return candidates
+
+    async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            result = await session.execute(
+                select(CrawlRun.start_dt)
+                .where(
+                    CrawlRun.user_id == candidate.user_id,
+                )
+                .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
+                .limit(1)
+            )
+            last_run = result.scalar_one_or_none()
+
+        if last_run is None:
+            return True
+
+        next_due_dt = last_run + timedelta(
+            minutes=candidate.run_interval_minutes
+        )
+        return now >= next_due_dt
+
+    async def _run_candidate_ingestion(
+        self,
+        *,
+        candidate: _AutoRunCandidate,
+    ):
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            ingestion = ScholarIngestionService(source=self._source)
+            try:
+                return await ingestion.run_for_user(
+                    session,
+                    user_id=candidate.user_id,
+                    trigger_type=RunTriggerType.SCHEDULED,
+                    request_delay_seconds=candidate.request_delay_seconds,
+                    network_error_retries=self._network_error_retries,
+                    retry_backoff_seconds=self._retry_backoff_seconds,
+                    max_pages_per_scholar=self._max_pages_per_scholar,
+                    page_size=self._page_size,
+                    auto_queue_continuations=self._continuation_queue_enabled,
+                    queue_delay_seconds=self._continuation_base_delay_seconds,
+                    alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
+                    alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
+                    alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
+                )
+            except RunAlreadyInProgressError:
+                await session.rollback()
+                logger.info("scheduler.run_skipped_locked", extra={"event": "scheduler.run_skipped_locked", "user_id": candidate.user_id})
+                return None
+            except RunBlockedBySafetyPolicyError as exc:
+                await session.rollback()
+                logger.info(
+                    "scheduler.run_skipped_safety_cooldown",
+                    extra={
+                        "event": "scheduler.run_skipped_safety_cooldown",
+                        "user_id": candidate.user_id,
+                        "reason": exc.safety_state.get("cooldown_reason"),
+                        "cooldown_until": exc.safety_state.get("cooldown_until"),
+                        "cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"),
+                        "metric_name": "scheduler_run_skipped_safety_cooldown_total",
+                        "metric_value": 1,
+                    },
+                )
+                return None
+            except Exception:
+                await session.rollback()
+                logger.exception("scheduler.run_failed", extra={"event": "scheduler.run_failed", "user_id": candidate.user_id})
+                return None
+
+    async def _run_candidate(self, candidate: _AutoRunCandidate) -> None:
+        run_summary = await self._run_candidate_ingestion(candidate=candidate)
+        if run_summary is None:
+            return
+        logger.info(
+            "scheduler.run_completed",
+            extra={
+                "event": "scheduler.run_completed",
+                "user_id": candidate.user_id,
+                "run_id": run_summary.crawl_run_id,
+                "status": run_summary.status.value,
+                "scholar_count": run_summary.scholar_count,
+                "new_publication_count": run_summary.new_publication_count,
+            },
+        )
+
+    async def _drain_continuation_queue(self) -> None:
+        now = datetime.now(timezone.utc)
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            jobs = await queue_service.list_due_jobs(
+                session,
+                now=now,
+                limit=self._queue_batch_size,
+            )
+        for job in jobs:
+            await self._run_queue_job(job)
+
+    async def _drop_queue_job_if_max_attempts(
+        self,
+        job: queue_service.ContinuationQueueJob,
+    ) -> bool:
+        if job.attempt_count < self._continuation_max_attempts:
+            return False
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            dropped = await queue_service.mark_dropped(
+                session,
+                job_id=job.id,
+                reason="max_attempts_reached",
+            )
+            await session.commit()
+        if dropped is not None:
+            logger.warning(
+                "scheduler.queue_item_dropped_max_attempts",
+                extra={
+                    "event": "scheduler.queue_item_dropped_max_attempts",
+                    "queue_item_id": job.id,
+                    "user_id": job.user_id,
+                    "scholar_profile_id": job.scholar_profile_id,
+                    "attempt_count": job.attempt_count,
+                    "max_attempts": self._continuation_max_attempts,
+                },
+            )
+        return True
+
+    async def _mark_queue_job_retrying(
+        self,
+        job: queue_service.ContinuationQueueJob,
+    ) -> bool:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            queue_item = await queue_service.mark_retrying(session, job_id=job.id)
+            await session.commit()
+        if queue_item is None:
+            return False
+        if queue_item.status == QueueItemStatus.DROPPED.value:
+            return False
+        return True
+
+    async def _queue_job_has_available_scholar(
+        self,
+        job: queue_service.ContinuationQueueJob,
+    ) -> bool:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            scholar_result = await session.execute(
+                select(ScholarProfile.id).where(
+                    ScholarProfile.user_id == job.user_id,
+                    ScholarProfile.id == job.scholar_profile_id,
+                    ScholarProfile.is_enabled.is_(True),
+                )
+            )
+            scholar_id = scholar_result.scalar_one_or_none()
+            if scholar_id is not None:
+                return True
+            dropped = await queue_service.mark_dropped(
+                session,
+                job_id=job.id,
+                reason="scholar_unavailable",
+            )
+            await session.commit()
+        if dropped is not None:
+            logger.info(
+                "scheduler.queue_item_dropped_scholar_unavailable",
+                extra={
+                    "event": "scheduler.queue_item_dropped_scholar_unavailable",
+                    "queue_item_id": job.id,
+                    "user_id": job.user_id,
+                    "scholar_profile_id": job.scholar_profile_id,
+                },
+            )
+        return False
+
+    async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
+        session_factory = get_session_factory()
+        async with session_factory() as recovery_session:
+            await queue_service.reschedule_job(
+                recovery_session,
+                job_id=job.id,
+                delay_seconds=max(self._tick_seconds, 15),
+                reason="user_run_lock_active",
+                error="run_already_in_progress",
+            )
+            await recovery_session.commit()
+        logger.info(
+            "scheduler.queue_item_deferred_lock",
+            extra={"event": "scheduler.queue_item_deferred_lock", "queue_item_id": job.id, "user_id": job.user_id},
+        )
+
+    async def _reschedule_queue_job_safety_cooldown(
+        self,
+        job: queue_service.ContinuationQueueJob,
+        exc: RunBlockedBySafetyPolicyError,
+    ) -> None:
+        cooldown_remaining_seconds = max(
+            self._tick_seconds,
+            int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
+        )
+        session_factory = get_session_factory()
+        async with session_factory() as recovery_session:
+            await queue_service.reschedule_job(
+                recovery_session,
+                job_id=job.id,
+                delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds),
+                reason="scrape_safety_cooldown",
+                error=str(exc.message),
+            )
+            await recovery_session.commit()
+        logger.info(
+            "scheduler.queue_item_deferred_safety_cooldown",
+            extra={
+                "event": "scheduler.queue_item_deferred_safety_cooldown",
+                "queue_item_id": job.id,
+                "user_id": job.user_id,
+                "reason": exc.safety_state.get("cooldown_reason"),
+                "cooldown_remaining_seconds": cooldown_remaining_seconds,
+                "metric_name": "scheduler_queue_item_deferred_safety_cooldown_total",
+                "metric_value": 1,
+            },
+        )
+
+    async def _reschedule_queue_job_after_exception(
+        self,
+        job: queue_service.ContinuationQueueJob,
+        *,
+        exc: Exception,
+    ) -> None:
+        session_factory = get_session_factory()
+        async with session_factory() as recovery_session:
+            queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
+            if queue_item is None:
+                await recovery_session.commit()
+                return
+            if int(queue_item.attempt_count) >= self._continuation_max_attempts:
+                await queue_service.mark_dropped(
+                    recovery_session,
+                    job_id=job.id,
+                    reason="scheduler_exception_max_attempts",
+                    error=str(exc),
+                )
+                await recovery_session.commit()
+                logger.warning(
+                    "scheduler.queue_item_dropped_after_exception",
+                    extra={"event": "scheduler.queue_item_dropped_after_exception", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count},
+                )
+                return
+            delay_seconds = queue_service.compute_backoff_seconds(
+                base_seconds=self._continuation_base_delay_seconds,
+                attempt_count=int(queue_item.attempt_count),
+                max_seconds=self._continuation_max_delay_seconds,
+            )
+            await queue_service.reschedule_job(
+                recovery_session,
+                job_id=job.id,
+                delay_seconds=delay_seconds,
+                reason="scheduler_exception",
+                error=str(exc),
+            )
+            await recovery_session.commit()
+        logger.exception("scheduler.queue_item_run_failed", extra={"event": "scheduler.queue_item_run_failed", "queue_item_id": job.id, "user_id": job.user_id})
+
+    async def _run_ingestion_for_queue_job(
+        self,
+        job: queue_service.ContinuationQueueJob,
+    ):
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id)
+            ingestion = ScholarIngestionService(source=self._source)
+            try:
+                return await ingestion.run_for_user(
+                    session,
+                    user_id=job.user_id,
+                    trigger_type=RunTriggerType.SCHEDULED,
+                    request_delay_seconds=request_delay_seconds,
+                    network_error_retries=self._network_error_retries,
+                    retry_backoff_seconds=self._retry_backoff_seconds,
+                    rate_limit_retries=settings.ingestion_rate_limit_retries,
+                    rate_limit_backoff_seconds=settings.ingestion_rate_limit_backoff_seconds,
+                    max_pages_per_scholar=self._max_pages_per_scholar,
+                    page_size=self._page_size,
+                    scholar_profile_ids={job.scholar_profile_id},
+                    start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart},
+                    auto_queue_continuations=self._continuation_queue_enabled,
+                    queue_delay_seconds=self._continuation_base_delay_seconds,
+                    alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
+                    alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
+                    alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
+                )
+            except RunAlreadyInProgressError:
+                await session.rollback()
+                await self._reschedule_queue_job_lock_active(job)
+            except RunBlockedBySafetyPolicyError as exc:
+                await session.rollback()
+                await self._reschedule_queue_job_safety_cooldown(job, exc)
+            except Exception as exc:
+                await session.rollback()
+                await self._reschedule_queue_job_after_exception(job, exc=exc)
+        return None
+
+    @staticmethod
+    def _log_queue_item_resolved(
+        *,
+        event_name: str,
+        job: queue_service.ContinuationQueueJob,
+        run_summary,
+        attempt_count: int | None = None,
+        delay_seconds: int | None = None,
+    ) -> None:
+        payload = {
+            "event": event_name,
+            "queue_item_id": job.id,
+            "user_id": job.user_id,
+            "run_id": run_summary.crawl_run_id,
+            "status": run_summary.status.value,
+        }
+        if attempt_count is not None:
+            payload["attempt_count"] = attempt_count
+        if delay_seconds is not None:
+            payload["delay_seconds"] = delay_seconds
+        logger.info(event_name, extra=payload)
+
+    async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            if int(run_summary.failed_count) <= 0:
+                queue_item = await queue_service.reset_attempt_count(session, job_id=job.id)
+                await session.commit()
+                if queue_item is None:
+                    self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
+                    return
+                self._log_queue_item_resolved(
+                    event_name="scheduler.queue_item_progressed",
+                    job=job,
+                    run_summary=run_summary,
+                    attempt_count=int(queue_item.attempt_count),
+                )
+                return
+            queue_item = await queue_service.increment_attempt_count(session, job_id=job.id)
+            if queue_item is None:
+                await session.commit()
+                self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
+                return
+            if int(queue_item.attempt_count) >= self._continuation_max_attempts:
+                await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
+                await session.commit()
+                logger.warning(
+                    "scheduler.queue_item_dropped_max_attempts_after_run",
+                    extra={"event": "scheduler.queue_item_dropped_max_attempts_after_run", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count, "run_id": run_summary.crawl_run_id, "status": run_summary.status.value},
+                )
+                return
+            delay_seconds = queue_service.compute_backoff_seconds(base_seconds=self._continuation_base_delay_seconds, attempt_count=int(queue_item.attempt_count), max_seconds=self._continuation_max_delay_seconds)
+            await queue_service.reschedule_job(session, job_id=job.id, delay_seconds=delay_seconds, reason=queue_item.reason, error=queue_item.last_error)
+            await session.commit()
+        self._log_queue_item_resolved(
+            event_name="scheduler.queue_item_rescheduled_failed",
+            job=job,
+            run_summary=run_summary,
+            attempt_count=int(queue_item.attempt_count),
+            delay_seconds=delay_seconds,
+        )
+
+    async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
+        if await self._drop_queue_job_if_max_attempts(job):
+            return
+        if not await self._mark_queue_job_retrying(job):
+            return
+        if not await self._queue_job_has_available_scholar(job):
+            return
+        run_summary = await self._run_ingestion_for_queue_job(job)
+        if run_summary is None:
+            return
+        await self._finalize_queue_job_after_run(job, run_summary)
+
+    async def _drain_pdf_queue(self) -> None:
+        from app.services.domains.publications.pdf_queue import drain_ready_jobs
+        
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            try:
+                processed = await drain_ready_jobs(
+                    session,
+                    limit=settings.scheduler_pdf_queue_batch_size,
+                    max_attempts=settings.pdf_auto_retry_max_attempts,
+                )
+                if processed > 0:
+                    logger.info("scheduler.pdf_queue_drain_completed", extra={
+                        "event": "scheduler.pdf_queue_drain_completed",
+                        "processed_count": processed,
+                    })
+            except Exception:
+                logger.exception("scheduler.pdf_queue_drain_failed", extra={
+                    "event": "scheduler.pdf_queue_drain_failed",
+                })
+
+    async def _load_request_delay_for_user(
+        self,
+        db_session: AsyncSession,
+        *,
+        user_id: int,
+    ) -> int:
+        result = await db_session.execute(
+            select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
+        )
+        delay = result.scalar_one_or_none()
+        return _effective_request_delay_seconds(delay)

build/lib/app/services/domains/ingestion/types.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from app.db.models import RunStatus
+from app.services.domains.scholar.parser import ParsedProfilePage, PublicationCandidate
+from app.services.domains.scholar.source import FetchResult
+
+
+@dataclass(frozen=True)
+class RunExecutionSummary:
+    crawl_run_id: int
+    status: RunStatus
+    scholar_count: int
+    succeeded_count: int
+    failed_count: int
+    partial_count: int
+    new_publication_count: int
+
+
+@dataclass(frozen=True)
+class PagedParseResult:
+    fetch_result: FetchResult
+    parsed_page: ParsedProfilePage
+    first_page_fetch_result: FetchResult
+    first_page_parsed_page: ParsedProfilePage
+    first_page_fingerprint_sha256: str | None
+    publications: list[PublicationCandidate]
+    attempt_log: list[dict[str, Any]]
+    page_logs: list[dict[str, Any]]
+    pages_fetched: int
+    pages_attempted: int
+    has_more_remaining: bool
+    pagination_truncated_reason: str | None
+    continuation_cstart: int | None
+    skipped_no_change: bool
+    discovered_publication_count: int
+
+
+@dataclass
+class RunProgress:
+    succeeded_count: int = 0
+    failed_count: int = 0
+    partial_count: int = 0
+    scholar_results: list[dict[str, Any]] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class ScholarProcessingOutcome:
+    result_entry: dict[str, Any]
+    succeeded_count_delta: int
+    failed_count_delta: int
+    partial_count_delta: int
+    discovered_publication_count: int
+
+
+@dataclass(frozen=True)
+class RunFailureSummary:
+    failed_state_counts: dict[str, int]
+    failed_reason_counts: dict[str, int]
+    scrape_failure_counts: dict[str, int]
+    retries_scheduled_count: int
+    scholars_with_retries_count: int
+    retry_exhausted_count: int
+
+
+@dataclass(frozen=True)
+class RunAlertSummary:
+    blocked_failure_count: int
+    network_failure_count: int
+    blocked_failure_threshold: int
+    network_failure_threshold: int
+    retry_scheduled_threshold: int
+    alert_flags: dict[str, bool]
+
+
+@dataclass
+class PagedLoopState:
+    fetch_result: FetchResult
+    parsed_page: ParsedProfilePage
+    attempt_log: list[dict[str, Any]]
+    page_logs: list[dict[str, Any]]
+    publications: list[PublicationCandidate]
+    pages_fetched: int
+    pages_attempted: int
+    current_cstart: int
+    next_cstart: int
+    has_more_remaining: bool = False
+    pagination_truncated_reason: str | None = None
+    continuation_cstart: int | None = None
+    discovered_publication_count: int = 0
+
+
+class RunAlreadyInProgressError(RuntimeError):
+    """Raised when a run lock for a user is already held by another process."""
+
+
+class RunBlockedBySafetyPolicyError(RuntimeError):
+    def __init__(
+        self,
+        *,
+        code: str,
+        message: str,
+        safety_state: dict[str, Any],
+    ) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+        self.safety_state = safety_state

build/lib/app/services/domains/openalex/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+import logging
+
+logger = logging.getLogger(__name__)

build/lib/app/services/domains/openalex/client.py

@@ -0,0 +1,135 @@
+import asyncio
+import logging
+from typing import Any, Mapping
+
+import httpx
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+from app.services.domains.openalex.types import OpenAlexWork
+
+logger = logging.getLogger(__name__)
+
+OPENALEX_BASE_URL = "https://api.openalex.org"
+
+
+class OpenAlexClientError(Exception):
+    pass
+
+
+class OpenAlexRateLimitError(OpenAlexClientError):
+    pass
+
+
+class OpenAlexClient:
+    def __init__(
+        self,
+        api_key: str | None = None,
+        mailto: str | None = None,
+        timeout: float = 10.0,
+    ) -> None:
+        self.api_key = api_key
+        self.mailto = mailto
+        self.timeout = timeout
+
+    @property
+    def _base_params(self) -> dict[str, str]:
+        params = {}
+        if self.mailto:
+            params["mailto"] = self.mailto
+        if self.api_key:
+            params["api_key"] = self.api_key
+        return params
+
+    @retry(
+        retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException, OpenAlexRateLimitError)),
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=2, max=10),
+        reraise=True,
+    )
+    async def get_work_by_doi(self, doi: str) -> OpenAlexWork | None:
+        """Fetch a single work by DOI directly."""
+        clean_doi = doi.replace("https://doi.org/", "")
+        if not clean_doi:
+            return None
+
+        url = f"{OPENALEX_BASE_URL}/works/{clean_doi}"
+        
+        headers = {}
+        if self.mailto:
+            headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
+        else:
+            headers["User-Agent"] = "scholar-scraper/1.0"
+
+        async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client:
+            response = await client.get(url, params=self._base_params)
+
+        if response.status_code == 404:
+            return None
+        if response.status_code == 429:
+            raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex work by DOI")
+        if response.status_code >= 400:
+            logger.warning("OpenAlex API error: %s %s", response.status_code, response.text)
+            raise OpenAlexClientError(f"API Error {response.status_code}")
+
+        data = response.json()
+        return OpenAlexWork.from_api_dict(data)
+
+    @retry(
+        retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException, OpenAlexRateLimitError)),
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=2, max=10),
+        reraise=True,
+    )
+    async def get_works_by_filter(
+        self,
+        filters: dict[str, str],
+        limit: int = 50,
+    ) -> list[OpenAlexWork]:
+        """
+        Fetch works using the ?filter= query parameter.
+        Supports fetching multiple records by joining filters with | (OR logic).
+        """
+        if not filters:
+            return []
+
+        # Example: {"doi": "10.foo|10.bar", "title.search": "query"}
+        filter_str = ",".join(f"{k}:{v}" for k, v in filters.items())
+        
+        params = self._base_params.copy()
+        params["filter"] = filter_str
+        params["per-page"] = str(limit)
+
+        url = f"{OPENALEX_BASE_URL}/works"
+        
+        headers = {}
+        if self.mailto:
+            headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
+        else:
+            headers["User-Agent"] = "scholar-scraper/1.0"
+
+        async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client:
+            response = await client.get(url, params=params)
+
+        if response.status_code == 429:
+            raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex works list")
+        if response.status_code >= 400:
+            logger.warning("OpenAlex API error (filters=%s): %s %s", filters, response.status_code, response.text)
+            raise OpenAlexClientError(f"API Error {response.status_code}")
+
+        data = response.json()
+        results = data.get("results") or []
+        
+        parsed_works = []
+        for raw_work in results:
+            try:
+                parsed_works.append(OpenAlexWork.from_api_dict(raw_work))
+            except Exception as e:
+                logger.warning("Failed to parse OpenAlex raw dict: %s", e)
+                continue
+
+        return parsed_works

build/lib/app/services/domains/openalex/matching.py

@@ -0,0 +1,108 @@
+import logging
+import re
+
+from rapidfuzz import fuzz
+
+from app.services.domains.openalex.types import OpenAlexWork
+
+logger = logging.getLogger(__name__)
+
+# A minimum similarity score out of 100 for a title to be considered a match candidate.
+TITLE_MATCH_THRESHOLD = 90.0
+# The margin within the top score where a secondary tiebreaker (author/year) is necessary.
+TIEBREAKER_MARGIN = 5.0
+
+
+def _clean_string(s: str | None) -> str:
+    if not s:
+        return ""
+    # Strip non-alphanumeric (keep spaces), lowercase, and collapse whitespace
+    cleaned = re.sub(r"[^a-z0-9\s]", " ", s.lower())
+    return " ".join(cleaned.split())
+
+
+def _author_overlap_score(target_authors: str | None, candidate_authors: list[str]) -> bool:
+    if not target_authors or not candidate_authors:
+        return False
+    
+    target_clean = _clean_string(target_authors)
+    if not target_clean:
+        return False
+        
+    for candidate in candidate_authors:
+        cand_clean = _clean_string(candidate)
+        if cand_clean and (cand_clean in target_clean or target_clean in cand_clean):
+            return True
+        # Alternatively check rapidfuzz token_set_ratio
+        if cand_clean and fuzz.token_set_ratio(target_clean, cand_clean) > 80:
+            return True
+            
+    return False
+
+
+def find_best_match(
+    target_title: str,
+    target_year: int | None,
+    target_authors: str | None,
+    candidates: list[OpenAlexWork],
+) -> OpenAlexWork | None:
+    """
+    Finds the best matching OpenAlexWork from a list of candidates, prioritizing title similarity (>90%)
+    with year and author overlap as tiebreakers for close candidates.
+    """
+    if not target_title or not candidates:
+        return None
+
+    clean_target = _clean_string(target_title)
+    if not clean_target:
+        return None
+
+    scored_candidates: list[tuple[float, OpenAlexWork]] = []
+
+    for cand in candidates:
+        if not cand.title:
+            continue
+            
+        clean_cand = _clean_string(cand.title)
+        
+        # Primary sort: string similarity ratio
+        score = fuzz.ratio(clean_target, clean_cand)
+        
+        if score >= TITLE_MATCH_THRESHOLD:
+            scored_candidates.append((score, cand))
+
+    if not scored_candidates:
+        return None
+
+    # Sort descending by score
+    scored_candidates.sort(key=lambda x: x[0], reverse=True)
+    
+    best_score = scored_candidates[0][0]
+    
+    # Extract all candidates within the tiebreaker margin
+    top_scored_candidates = [
+        (score, cand) for score, cand in scored_candidates 
+        if best_score - score <= TIEBREAKER_MARGIN
+    ]
+
+    if len(top_scored_candidates) == 1:
+        return top_scored_candidates[0][1]
+
+    # We have a tie or near-tie. Use year and author overlap to break the tie.
+    # Score candidates: +1 for year match (within 1 year), +1 for author overlap
+    tiebreaker_scores: list[tuple[int, float, OpenAlexWork]] = []
+    
+    for original_score, cand in top_scored_candidates:
+        tb_score = 0
+        if target_year is not None and cand.publication_year is not None:
+            if abs(target_year - cand.publication_year) <= 1:
+                tb_score += 1
+                
+        candidate_author_names = [a.display_name for a in cand.authors if a.display_name]
+        if _author_overlap_score(target_authors, candidate_author_names):
+            tb_score += 1
+            
+        tiebreaker_scores.append((tb_score, original_score, cand))
+        
+    tiebreaker_scores.sort(key=lambda x: (x[0], x[1]), reverse=True)
+    return tiebreaker_scores[0][2]

build/lib/app/services/domains/openalex/types.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Mapping
+
+
+@dataclass(frozen=True)
+class OpenAlexAuthor:
+    openalex_id: str | None
+    display_name: str | None
+
+
+@dataclass(frozen=True)
+class OpenAlexWork:
+    openalex_id: str
+    doi: str | None
+    pmid: str | None
+    pmcid: str | None
+    title: str | None
+    publication_year: int | None
+    cited_by_count: int
+    is_oa: bool
+    oa_url: str | None
+    authors: list[OpenAlexAuthor] = field(default_factory=list)
+    raw_data: Mapping[str, Any] = field(default_factory=dict, repr=False)
+
+    @classmethod
+    def from_api_dict(cls, data: Mapping[str, Any]) -> OpenAlexWork:
+        ids = data.get("ids") or {}
+        
+        # Extract DOI without the https://doi.org/ prefix
+        doi = ids.get("doi")
+        if doi and doi.startswith("https://doi.org/"):
+            doi = doi[16:]
+            
+        # Extract PMID without the url prefix
+        pmid = ids.get("pmid")
+        if pmid and pmid.startswith("https://pubmed.ncbi.nlm.nih.gov/"):
+            pmid = pmid[32:]
+            
+        # Extract PMCID without the url prefix
+        pmcid = ids.get("pmcid")
+        if pmcid and pmcid.startswith("https://www.ncbi.nlm.nih.gov/pmc/articles/"):
+            pmcid = pmcid[42:]
+
+        open_access = data.get("open_access") or {}
+        
+        authors = []
+        for authorship in data.get("authorships") or []:
+            author_data = authorship.get("author") or {}
+            authors.append(
+                OpenAlexAuthor(
+                    openalex_id=author_data.get("id"),
+                    display_name=author_data.get("display_name"),
+                )
+            )
+
+        return cls(
+            openalex_id=data.get("id", ""),
+            doi=doi,
+            pmid=pmid,
+            pmcid=pmcid,
+            title=data.get("title"),
+            publication_year=data.get("publication_year"),
+            cited_by_count=data.get("cited_by_count", 0),
+            is_oa=bool(open_access.get("is_oa")),
+            oa_url=open_access.get("oa_url"),
+            authors=authors,
+            raw_data=dict(data),
+        )

build/lib/app/services/domains/portability/__init__.py

@@ -0,0 +1 @@
+from app.services.domains.portability.application import *

build/lib/app/services/domains/portability/application.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import Publication
+from app.services.domains.portability.constants import (
+    EXPORT_SCHEMA_VERSION,
+    MAX_IMPORT_PUBLICATIONS,
+    MAX_IMPORT_SCHOLARS,
+)
+from app.services.domains.portability.exporting import export_user_data
+from app.services.domains.portability.normalize import _validate_import_sizes
+from app.services.domains.portability.publication_import import (
+    _build_imported_publication_input,
+    _initialize_import_counters,
+    _upsert_imported_publication,
+)
+from app.services.domains.portability.scholar_import import _upsert_imported_scholars
+from app.services.domains.portability.types import ImportExportError, ImportedPublicationInput
+
+
+async def import_user_data(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholars: list[dict[str, Any]],
+    publications: list[dict[str, Any]],
+) -> dict[str, int]:
+    _validate_import_sizes(scholars=scholars, publications=publications)
+    scholar_map, counters = await _upsert_imported_scholars(
+        db_session,
+        user_id=user_id,
+        scholars=scholars,
+    )
+    cluster_cache: dict[str, Publication | None] = {}
+    fingerprint_cache: dict[str, Publication | None] = {}
+    _initialize_import_counters(counters)
+    for item in publications:
+        parsed_item = _build_imported_publication_input(
+            item=item,
+            scholar_map=scholar_map,
+        )
+        if parsed_item is None:
+            counters["skipped_records"] += 1
+            continue
+        await _upsert_imported_publication(
+            db_session,
+            payload=parsed_item,
+            cluster_cache=cluster_cache,
+            fingerprint_cache=fingerprint_cache,
+            counters=counters,
+        )
+    await db_session.commit()
+    return counters
+
+
+__all__ = [
+    "EXPORT_SCHEMA_VERSION",
+    "MAX_IMPORT_SCHOLARS",
+    "MAX_IMPORT_PUBLICATIONS",
+    "ImportExportError",
+    "ImportedPublicationInput",
+    "export_user_data",
+    "import_user_data",
+]

build/lib/app/services/domains/portability/constants.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+import re
+
+EXPORT_SCHEMA_VERSION = 1
+MAX_IMPORT_SCHOLARS = 10_000
+MAX_IMPORT_PUBLICATIONS = 100_000
+WORD_RE = re.compile(r"[a-z0-9]+")
+SHA256_RE = re.compile(r"^[0-9a-f]{64}$")

build/lib/app/services/domains/portability/exporting.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import Publication, ScholarProfile, ScholarPublication
+from app.services.domains.portability.constants import EXPORT_SCHEMA_VERSION
+
+
+def _exported_at_iso() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+
+
+def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]:
+    return {
+        "scholar_id": profile.scholar_id,
+        "display_name": profile.display_name,
+        "is_enabled": bool(profile.is_enabled),
+        "profile_image_override_url": profile.profile_image_override_url,
+    }
+
+
+def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]:
+    (
+        scholar_id,
+        cluster_id,
+        fingerprint_sha256,
+        title_raw,
+        year,
+        citation_count,
+        author_text,
+        venue_text,
+        pub_url,
+        pdf_url,
+        is_read,
+    ) = row
+    return {
+        "scholar_id": scholar_id,
+        "cluster_id": cluster_id,
+        "fingerprint_sha256": fingerprint_sha256,
+        "title": title_raw,
+        "year": year,
+        "citation_count": int(citation_count or 0),
+        "author_text": author_text,
+        "venue_text": venue_text,
+        "pub_url": pub_url,
+        "pdf_url": pdf_url,
+        "is_read": bool(is_read),
+    }
+
+
+async def export_user_data(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> dict[str, Any]:
+    scholars_result = await db_session.execute(
+        select(ScholarProfile)
+        .where(ScholarProfile.user_id == user_id)
+        .order_by(ScholarProfile.id.asc())
+    )
+    publication_result = await db_session.execute(
+        select(
+            ScholarProfile.scholar_id,
+            Publication.cluster_id,
+            Publication.fingerprint_sha256,
+            Publication.title_raw,
+            Publication.year,
+            Publication.citation_count,
+            Publication.author_text,
+            Publication.venue_text,
+            Publication.pub_url,
+            Publication.pdf_url,
+            ScholarPublication.is_read,
+        )
+        .join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id)
+        .join(Publication, Publication.id == ScholarPublication.publication_id)
+        .where(ScholarProfile.user_id == user_id)
+        .order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
+    )
+    scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()]
+    publications = [_serialize_export_publication(row) for row in publication_result.all()]
+    return {
+        "schema_version": EXPORT_SCHEMA_VERSION,
+        "exported_at": _exported_at_iso(),
+        "scholars": scholars,
+        "publications": publications,
+    }

build/lib/app/services/domains/portability/normalize.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import hashlib
+from typing import Any
+
+from app.services.domains.ingestion.application import normalize_title
+from app.services.domains.portability.constants import (
+    MAX_IMPORT_PUBLICATIONS,
+    MAX_IMPORT_SCHOLARS,
+    SHA256_RE,
+    WORD_RE,
+)
+from app.services.domains.portability.types import ImportExportError
+
+
+def _normalize_optional_text(value: Any) -> str | None:
+    if value is None:
+        return None
+    normalized = str(value).strip()
+    return normalized or None
+
+
+def _normalize_optional_year(value: Any) -> int | None:
+    if value is None:
+        return None
+    try:
+        year = int(value)
+    except (TypeError, ValueError):
+        return None
+    if year < 1500 or year > 3000:
+        return None
+    return year
+
+
+def _normalize_citation_count(value: Any) -> int:
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError):
+        return 0
+    return max(0, parsed)
+
+
+def _first_author_last_name(authors_text: str | None) -> str:
+    if not authors_text:
+        return ""
+    first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
+    words = WORD_RE.findall(first_author)
+    if not words:
+        return ""
+    return words[-1]
+
+
+def _first_venue_word(venue_text: str | None) -> str:
+    if not venue_text:
+        return ""
+    words = WORD_RE.findall(venue_text.lower())
+    if not words:
+        return ""
+    return words[0]
+
+
+def _build_fingerprint(
+    *,
+    title: str,
+    year: int | None,
+    author_text: str | None,
+    venue_text: str | None,
+) -> str:
+    canonical = "|".join(
+        [
+            normalize_title(title),
+            str(year) if year is not None else "",
+            _first_author_last_name(author_text),
+            _first_venue_word(venue_text),
+        ]
+    )
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _resolve_fingerprint(
+    *,
+    title: str,
+    year: int | None,
+    author_text: str | None,
+    venue_text: str | None,
+    provided_fingerprint: Any,
+) -> str:
+    normalized = _normalize_optional_text(provided_fingerprint)
+    if normalized and SHA256_RE.fullmatch(normalized.lower()):
+        return normalized.lower()
+    return _build_fingerprint(
+        title=title,
+        year=year,
+        author_text=author_text,
+        venue_text=venue_text,
+    )
+
+
+def _validate_import_sizes(
+    *,
+    scholars: list[dict[str, Any]],
+    publications: list[dict[str, Any]],
+) -> None:
+    if len(scholars) > MAX_IMPORT_SCHOLARS:
+        raise ImportExportError(f"Import exceeds max scholars ({MAX_IMPORT_SCHOLARS}).")
+    if len(publications) > MAX_IMPORT_PUBLICATIONS:
+        raise ImportExportError(
+            f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS})."
+        )

build/lib/app/services/domains/portability/publication_import.py

@@ -0,0 +1,370 @@
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import Publication, ScholarProfile, ScholarPublication
+from app.services.domains.ingestion.application import build_publication_url, normalize_title
+from app.services.domains.doi.normalize import normalize_doi
+from app.services.domains.portability.normalize import (
+    _normalize_citation_count,
+    _normalize_optional_text,
+    _normalize_optional_year,
+    _resolve_fingerprint,
+)
+from app.services.domains.portability.types import ImportedPublicationInput
+
+
+async def _find_publication_by_cluster(
+    db_session: AsyncSession,
+    *,
+    cluster_id: str,
+) -> Publication | None:
+    result = await db_session.execute(
+        select(Publication).where(Publication.cluster_id == cluster_id)
+    )
+    return result.scalar_one_or_none()
+
+
+async def _find_publication_by_fingerprint(
+    db_session: AsyncSession,
+    *,
+    fingerprint_sha256: str,
+) -> Publication | None:
+    result = await db_session.execute(
+        select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256)
+    )
+    return result.scalar_one_or_none()
+
+
+async def _find_linked_publication_by_title(
+    db_session: AsyncSession,
+    *,
+    scholar_profile_id: int,
+    title: str,
+) -> Publication | None:
+    normalized_title = normalize_title(title)
+    result = await db_session.execute(
+        select(Publication)
+        .join(
+            ScholarPublication,
+            ScholarPublication.publication_id == Publication.id,
+        )
+        .where(
+            ScholarPublication.scholar_profile_id == scholar_profile_id,
+            Publication.title_normalized == normalized_title,
+        )
+        .order_by(Publication.id.asc())
+        .limit(1)
+    )
+    return result.scalar_one_or_none()
+
+
+def _apply_imported_publication_values(
+    *,
+    publication: Publication,
+    title: str,
+    year: int | None,
+    citation_count: int,
+    author_text: str | None,
+    venue_text: str | None,
+    pub_url: str | None,
+    pdf_url: str | None,
+    cluster_id: str | None,
+) -> bool:
+    updated = False
+    if cluster_id and publication.cluster_id != cluster_id:
+        publication.cluster_id = cluster_id
+        updated = True
+    if publication.title_raw != title:
+        publication.title_raw = title
+        publication.title_normalized = normalize_title(title)
+        updated = True
+    if publication.year != year:
+        publication.year = year
+        updated = True
+    if int(publication.citation_count or 0) != citation_count:
+        publication.citation_count = citation_count
+        updated = True
+    if publication.author_text != author_text:
+        publication.author_text = author_text
+        updated = True
+    if publication.venue_text != venue_text:
+        publication.venue_text = venue_text
+        updated = True
+    if pub_url and publication.pub_url != pub_url:
+        publication.pub_url = pub_url
+        updated = True
+    if pdf_url and publication.pdf_url != pdf_url:
+        publication.pdf_url = pdf_url
+        updated = True
+    return updated
+
+
+def _new_publication(
+    *,
+    cluster_id: str | None,
+    fingerprint_sha256: str,
+    title: str,
+    year: int | None,
+    citation_count: int,
+    author_text: str | None,
+    venue_text: str | None,
+    pub_url: str | None,
+    pdf_url: str | None,
+) -> Publication:
+    return Publication(
+        cluster_id=cluster_id,
+        fingerprint_sha256=fingerprint_sha256,
+        title_raw=title,
+        title_normalized=normalize_title(title),
+        year=year,
+        citation_count=citation_count,
+        author_text=author_text,
+        venue_text=venue_text,
+        pub_url=pub_url,
+        pdf_url=pdf_url,
+    )
+
+
+async def _resolve_publication_for_import(
+    db_session: AsyncSession,
+    *,
+    scholar_profile_id: int,
+    title: str,
+    cluster_id: str | None,
+    fingerprint_sha256: str,
+    cluster_cache: dict[str, Publication | None],
+    fingerprint_cache: dict[str, Publication | None],
+) -> Publication | None:
+    if cluster_id:
+        if cluster_id not in cluster_cache:
+            cluster_cache[cluster_id] = await _find_publication_by_cluster(
+                db_session,
+                cluster_id=cluster_id,
+            )
+        if cluster_cache[cluster_id] is not None:
+            return cluster_cache[cluster_id]
+    if fingerprint_sha256 not in fingerprint_cache:
+        fingerprint_cache[fingerprint_sha256] = await _find_publication_by_fingerprint(
+            db_session,
+            fingerprint_sha256=fingerprint_sha256,
+        )
+    if fingerprint_cache[fingerprint_sha256] is not None:
+        return fingerprint_cache[fingerprint_sha256]
+    return await _find_linked_publication_by_title(
+        db_session,
+        scholar_profile_id=scholar_profile_id,
+        title=title,
+    )
+
+
+async def _upsert_scholar_publication_link(
+    db_session: AsyncSession,
+    *,
+    scholar_profile_id: int,
+    publication_id: int,
+    is_read: bool,
+) -> tuple[bool, bool]:
+    result = await db_session.execute(
+        select(ScholarPublication).where(
+            ScholarPublication.scholar_profile_id == scholar_profile_id,
+            ScholarPublication.publication_id == publication_id,
+        )
+    )
+    link = result.scalar_one_or_none()
+    if link is None:
+        db_session.add(
+            ScholarPublication(
+                scholar_profile_id=scholar_profile_id,
+                publication_id=publication_id,
+                is_read=bool(is_read),
+            )
+        )
+        return True, False
+    if bool(link.is_read) == bool(is_read):
+        return False, False
+    link.is_read = bool(is_read)
+    return False, True
+
+
+def _initialize_import_counters(counters: dict[str, int]) -> None:
+    counters.update(
+        {
+            "publications_created": 0,
+            "publications_updated": 0,
+            "links_created": 0,
+            "links_updated": 0,
+        }
+    )
+
+
+def _build_imported_publication_input(
+    *,
+    item: dict[str, Any],
+    scholar_map: dict[str, ScholarProfile],
+) -> ImportedPublicationInput | None:
+    scholar_id = _normalize_optional_text(item.get("scholar_id"))
+    title = _normalize_optional_text(item.get("title"))
+    if not scholar_id or not title:
+        return None
+
+    profile = scholar_map.get(scholar_id)
+    if profile is None:
+        return None
+
+    year = _normalize_optional_year(item.get("year"))
+    author_text = _normalize_optional_text(item.get("author_text"))
+    venue_text = _normalize_optional_text(item.get("venue_text"))
+    return ImportedPublicationInput(
+        profile=profile,
+        title=title,
+        year=year,
+        citation_count=_normalize_citation_count(item.get("citation_count")),
+        author_text=author_text,
+        venue_text=venue_text,
+        cluster_id=_normalize_optional_text(item.get("cluster_id")),
+        pub_url=build_publication_url(_normalize_optional_text(item.get("pub_url"))),
+        doi=normalize_doi(_normalize_optional_text(item.get("doi"))),
+        pdf_url=build_publication_url(_normalize_optional_text(item.get("pdf_url"))),
+        fingerprint=_resolve_fingerprint(
+            title=title,
+            year=year,
+            author_text=author_text,
+            venue_text=venue_text,
+            provided_fingerprint=item.get("fingerprint_sha256"),
+        ),
+        is_read=bool(item.get("is_read", False)),
+    )
+
+
+def _update_link_counters(
+    *,
+    counters: dict[str, int],
+    link_created: bool,
+    link_updated: bool,
+) -> None:
+    if link_created:
+        counters["links_created"] += 1
+    if link_updated:
+        counters["links_updated"] += 1
+
+
+def _cache_resolved_publication(
+    *,
+    publication: Publication,
+    cluster_id: str | None,
+    fingerprint_sha256: str,
+    cluster_cache: dict[str, Publication | None],
+    fingerprint_cache: dict[str, Publication | None],
+) -> None:
+    if cluster_id:
+        cluster_cache[cluster_id] = publication
+    fingerprint_cache[fingerprint_sha256] = publication
+
+
+async def _create_import_publication(
+    db_session: AsyncSession,
+    *,
+    payload: ImportedPublicationInput,
+) -> Publication:
+    publication = _new_publication(
+        cluster_id=payload.cluster_id,
+        fingerprint_sha256=payload.fingerprint,
+        title=payload.title,
+        year=payload.year,
+        citation_count=payload.citation_count,
+        author_text=payload.author_text,
+        venue_text=payload.venue_text,
+        pub_url=payload.pub_url,
+        pdf_url=payload.pdf_url,
+    )
+    db_session.add(publication)
+    await db_session.flush()
+    return publication
+
+
+def _update_import_publication(
+    *,
+    publication: Publication,
+    payload: ImportedPublicationInput,
+) -> bool:
+    return _apply_imported_publication_values(
+        publication=publication,
+        title=payload.title,
+        year=payload.year,
+        citation_count=payload.citation_count,
+        author_text=payload.author_text,
+        venue_text=payload.venue_text,
+        pub_url=payload.pub_url,
+        pdf_url=payload.pdf_url,
+        cluster_id=payload.cluster_id,
+    )
+
+
+async def _upsert_publication_entity(
+    db_session: AsyncSession,
+    *,
+    payload: ImportedPublicationInput,
+    cluster_cache: dict[str, Publication | None],
+    fingerprint_cache: dict[str, Publication | None],
+) -> tuple[Publication, bool, bool]:
+    publication = await _resolve_publication_for_import(
+        db_session,
+        scholar_profile_id=int(payload.profile.id),
+        title=payload.title,
+        cluster_id=payload.cluster_id,
+        fingerprint_sha256=payload.fingerprint,
+        cluster_cache=cluster_cache,
+        fingerprint_cache=fingerprint_cache,
+    )
+    created = False
+    updated = False
+    if publication is None:
+        publication = await _create_import_publication(db_session, payload=payload)
+        created = True
+    else:
+        updated = _update_import_publication(publication=publication, payload=payload)
+
+    _cache_resolved_publication(
+        publication=publication,
+        cluster_id=payload.cluster_id,
+        fingerprint_sha256=payload.fingerprint,
+        cluster_cache=cluster_cache,
+        fingerprint_cache=fingerprint_cache,
+    )
+    return publication, created, updated
+
+
+async def _upsert_imported_publication(
+    db_session: AsyncSession,
+    *,
+    payload: ImportedPublicationInput,
+    cluster_cache: dict[str, Publication | None],
+    fingerprint_cache: dict[str, Publication | None],
+    counters: dict[str, int],
+) -> None:
+    publication, created, updated = await _upsert_publication_entity(
+        db_session,
+        payload=payload,
+        cluster_cache=cluster_cache,
+        fingerprint_cache=fingerprint_cache,
+    )
+    if created:
+        counters["publications_created"] += 1
+    if updated:
+        counters["publications_updated"] += 1
+
+    link_created, link_updated = await _upsert_scholar_publication_link(
+        db_session,
+        scholar_profile_id=int(payload.profile.id),
+        publication_id=int(publication.id),
+        is_read=payload.is_read,
+    )
+    _update_link_counters(
+        counters=counters,
+        link_created=link_created,
+        link_updated=link_updated,
+    )

build/lib/app/services/domains/portability/scholar_import.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import ScholarProfile
+from app.services.domains.portability.normalize import _normalize_optional_text
+from app.services.domains.scholars import application as scholar_service
+
+
+async def _load_user_scholar_map(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> dict[str, ScholarProfile]:
+    result = await db_session.execute(
+        select(ScholarProfile).where(ScholarProfile.user_id == user_id)
+    )
+    profiles = list(result.scalars().all())
+    return {profile.scholar_id: profile for profile in profiles}
+
+
+def _apply_imported_scholar_values(
+    *,
+    profile: ScholarProfile,
+    display_name: str | None,
+    profile_image_override_url: str | None,
+    is_enabled: bool,
+) -> bool:
+    updated = False
+    if display_name and profile.display_name != display_name:
+        profile.display_name = display_name
+        updated = True
+    if profile.profile_image_override_url != profile_image_override_url:
+        profile.profile_image_override_url = profile_image_override_url
+        updated = True
+    if bool(profile.is_enabled) != bool(is_enabled):
+        profile.is_enabled = bool(is_enabled)
+        updated = True
+    return updated
+
+
+def _new_scholar_profile(
+    *,
+    user_id: int,
+    scholar_id: str,
+    display_name: str | None,
+    profile_image_override_url: str | None,
+    is_enabled: bool,
+) -> ScholarProfile:
+    return ScholarProfile(
+        user_id=user_id,
+        scholar_id=scholar_id,
+        display_name=display_name,
+        profile_image_override_url=profile_image_override_url,
+        is_enabled=bool(is_enabled),
+    )
+
+
+async def _upsert_imported_scholars(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholars: list[dict[str, Any]],
+) -> tuple[dict[str, ScholarProfile], dict[str, int]]:
+    scholar_map = await _load_user_scholar_map(db_session, user_id=user_id)
+    counters = {"scholars_created": 0, "scholars_updated": 0, "skipped_records": 0}
+    for item in scholars:
+        try:
+            scholar_id = scholar_service.validate_scholar_id(str(item["scholar_id"]))
+            display_name = scholar_service.normalize_display_name(
+                str(item.get("display_name") or "")
+            )
+            override_url = scholar_service.normalize_profile_image_url(
+                _normalize_optional_text(item.get("profile_image_override_url"))
+            )
+        except (KeyError, scholar_service.ScholarServiceError):
+            counters["skipped_records"] += 1
+            continue
+
+        is_enabled = bool(item.get("is_enabled", True))
+        existing = scholar_map.get(scholar_id)
+        if existing is None:
+            profile = _new_scholar_profile(
+                user_id=user_id,
+                scholar_id=scholar_id,
+                display_name=display_name,
+                profile_image_override_url=override_url,
+                is_enabled=is_enabled,
+            )
+            db_session.add(profile)
+            scholar_map[scholar_id] = profile
+            counters["scholars_created"] += 1
+            continue
+
+        if _apply_imported_scholar_values(
+            profile=existing,
+            display_name=display_name,
+            profile_image_override_url=override_url,
+            is_enabled=is_enabled,
+        ):
+            counters["scholars_updated"] += 1
+
+    await db_session.flush()
+    return scholar_map, counters

build/lib/app/services/domains/portability/types.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from app.db.models import ScholarProfile
+
+
+class ImportExportError(ValueError):
+    """Raised when import/export payload constraints are violated."""
+
+
+@dataclass(frozen=True)
+class ImportedPublicationInput:
+    profile: ScholarProfile
+    title: str
+    year: int | None
+    citation_count: int
+    author_text: str | None
+    venue_text: str | None
+    cluster_id: str | None
+    pub_url: str | None
+    doi: str | None
+    pdf_url: str | None
+    fingerprint: str
+    is_read: bool

build/lib/app/services/domains/publication_identifiers/__init__.py

@@ -0,0 +1,17 @@
+from app.services.domains.publication_identifiers.application import (
+    DisplayIdentifier,
+    derive_display_identifier_from_values,
+    overlay_pdf_queue_items_with_display_identifiers,
+    overlay_publication_items_with_display_identifiers,
+    sync_identifiers_for_publication_fields,
+    sync_identifiers_for_publication_resolution,
+)
+
+__all__ = [
+    "DisplayIdentifier",
+    "derive_display_identifier_from_values",
+    "overlay_pdf_queue_items_with_display_identifiers",
+    "overlay_publication_items_with_display_identifiers",
+    "sync_identifiers_for_publication_fields",
+    "sync_identifiers_for_publication_resolution",
+]

build/lib/app/services/domains/publication_identifiers/application.py

@@ -0,0 +1,433 @@
+from __future__ import annotations
+
+from dataclasses import replace
+from typing import TYPE_CHECKING
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import Publication, PublicationIdentifier
+from app.services.domains.doi.normalize import normalize_doi
+from app.services.domains.publication_identifiers.normalize import (
+    normalize_arxiv_id,
+    normalize_pmcid,
+    normalize_pmid,
+)
+from app.services.domains.publication_identifiers.types import (
+    DisplayIdentifier,
+    IdentifierCandidate,
+    IdentifierKind,
+)
+
+if TYPE_CHECKING:
+    from app.services.domains.publications.pdf_queue import PdfQueueListItem
+    from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+
+CONFIDENCE_HIGH = 0.98
+CONFIDENCE_MEDIUM = 0.9
+CONFIDENCE_LOW = 0.6
+CONFIDENCE_FALLBACK = 0.4
+PRIORITY_DOI = 400
+PRIORITY_ARXIV = 300
+PRIORITY_PMCID = 200
+PRIORITY_PMID = 100
+
+
+def derive_display_identifier_from_values(
+    *,
+    doi: str | None,
+    pub_url: str | None = None,
+    pdf_url: str | None = None,
+) -> DisplayIdentifier | None:
+    candidates = _fallback_candidates_from_values(doi=doi, pub_url=pub_url, pdf_url=pdf_url)
+    return _best_display_identifier(candidates)
+
+
+def _fallback_candidates_from_values(
+    *,
+    doi: str | None,
+    pub_url: str | None,
+    pdf_url: str | None,
+) -> list[IdentifierCandidate]:
+    values = [value for value in [pub_url, pdf_url] if value]
+    candidates = []
+    if doi:
+        normalized_doi = normalize_doi(doi)
+        if normalized_doi:
+            candidates.append(_candidate(IdentifierKind.DOI, doi, normalized_doi, "legacy_doi", CONFIDENCE_HIGH, pub_url))
+    candidates.extend(_url_identifier_candidates(values=values, source="legacy_urls"))
+    return _dedup_candidates(candidates)
+
+
+def _url_identifier_candidates(*, values: list[str], source: str) -> list[IdentifierCandidate]:
+    candidates: list[IdentifierCandidate] = []
+    for value in values:
+        candidates.extend(_url_candidates_for_value(value=value, source=source))
+    return candidates
+
+
+def _url_candidates_for_value(*, value: str, source: str) -> list[IdentifierCandidate]:
+    candidates: list[IdentifierCandidate] = []
+    arxiv = normalize_arxiv_id(value)
+    if arxiv:
+        candidates.append(_candidate(IdentifierKind.ARXIV, value, arxiv, source, CONFIDENCE_MEDIUM, value))
+    pmcid = normalize_pmcid(value)
+    if pmcid:
+        candidates.append(_candidate(IdentifierKind.PMCID, value, pmcid, source, CONFIDENCE_LOW, value))
+    pmid = normalize_pmid(value)
+    if pmid:
+        candidates.append(_candidate(IdentifierKind.PMID, value, pmid, source, CONFIDENCE_FALLBACK, value))
+    return candidates
+
+
+def _candidate(
+    kind: IdentifierKind,
+    value_raw: str,
+    value_normalized: str,
+    source: str,
+    confidence_score: float,
+    evidence_url: str | None,
+) -> IdentifierCandidate:
+    return IdentifierCandidate(
+        kind=kind,
+        value_raw=value_raw,
+        value_normalized=value_normalized,
+        source=source,
+        confidence_score=float(confidence_score),
+        evidence_url=evidence_url,
+    )
+
+
+def _dedup_candidates(candidates: list[IdentifierCandidate]) -> list[IdentifierCandidate]:
+    deduped: dict[tuple[str, str], IdentifierCandidate] = {}
+    for candidate in candidates:
+        key = (candidate.kind.value, candidate.value_normalized)
+        current = deduped.get(key)
+        if current is None or candidate.confidence_score > current.confidence_score:
+            deduped[key] = candidate
+    return list(deduped.values())
+
+
+async def sync_identifiers_for_publication_fields(
+    db_session: AsyncSession,
+    *,
+    publication: Publication,
+) -> None:
+    candidates = _publication_field_candidates(publication)
+    await _upsert_publication_candidates(db_session, publication_id=int(publication.id), candidates=candidates)
+
+
+async def discover_and_sync_identifiers_for_publication(
+    db_session: AsyncSession,
+    *,
+    publication: Publication,
+    scholar_label: str,
+) -> None:
+    await sync_identifiers_for_publication_fields(db_session, publication=publication)
+    
+    existing_doi = await _existing_identifier_by_kind(
+        db_session,
+        publication_id=int(publication.id),
+        kind=IdentifierKind.DOI.value,
+    )
+    if existing_doi is not None:
+        return
+
+    from app.services.domains.crossref import application as crossref_service
+    from app.services.domains.publications.types import UnreadPublicationItem
+    
+    item = UnreadPublicationItem(
+        publication_id=int(publication.id),
+        scholar_profile_id=0,
+        scholar_label=scholar_label,
+        title=str(publication.title_raw or ""),
+        year=publication.year,
+        citation_count=publication.citation_count,
+        venue_text=publication.venue_text,
+        pub_url=publication.pub_url,
+        pdf_url=publication.pdf_url,
+    )
+    
+    discovered_doi = await crossref_service.discover_doi_for_publication(item=item)
+    if discovered_doi:
+        normalized_doi = normalize_doi(discovered_doi)
+        if normalized_doi:
+            candidate = _candidate(
+                IdentifierKind.DOI,
+                discovered_doi,
+                normalized_doi,
+                "crossref_api",
+                CONFIDENCE_MEDIUM,
+                None,
+            )
+            await _upsert_publication_candidate(db_session, publication_id=int(publication.id), candidate=candidate)
+
+    existing_arxiv = await _existing_identifier_by_kind(
+        db_session,
+        publication_id=int(publication.id),
+        kind=IdentifierKind.ARXIV.value,
+    )
+    if existing_arxiv is None:
+        from app.services.domains.arxiv import application as arxiv_service
+        discovered_arxiv = await arxiv_service.discover_arxiv_id_for_publication(item=item)
+        if discovered_arxiv:
+            normalized_arxiv = normalize_arxiv_id(discovered_arxiv)
+            if normalized_arxiv:
+                candidate = _candidate(
+                    IdentifierKind.ARXIV,
+                    discovered_arxiv,
+                    normalized_arxiv,
+                    "arxiv_api",
+                    CONFIDENCE_MEDIUM,
+                    None,
+                )
+                await _upsert_publication_candidate(db_session, publication_id=int(publication.id), candidate=candidate)
+
+
+def _publication_field_candidates(publication: Publication) -> list[IdentifierCandidate]:
+    return _fallback_candidates_from_values(
+        doi=None,
+        pub_url=publication.pub_url,
+        pdf_url=publication.pdf_url,
+    )
+
+
+async def sync_identifiers_for_publication_resolution(
+    db_session: AsyncSession,
+    *,
+    publication: Publication,
+    source: str | None,
+) -> None:
+    candidates = _publication_field_candidates(publication)
+    rewritten = [_candidate_with_source(candidate, source=source) for candidate in candidates]
+    await _upsert_publication_candidates(db_session, publication_id=int(publication.id), candidates=rewritten)
+
+
+def _candidate_with_source(candidate: IdentifierCandidate, *, source: str | None) -> IdentifierCandidate:
+    if not source:
+        return candidate
+    return IdentifierCandidate(
+        kind=candidate.kind,
+        value_raw=candidate.value_raw,
+        value_normalized=candidate.value_normalized,
+        source=source,
+        confidence_score=candidate.confidence_score,
+        evidence_url=candidate.evidence_url,
+    )
+
+
+async def _upsert_publication_candidates(
+    db_session: AsyncSession,
+    *,
+    publication_id: int,
+    candidates: list[IdentifierCandidate],
+) -> None:
+    for candidate in _dedup_candidates(candidates):
+        await _upsert_publication_candidate(db_session, publication_id=publication_id, candidate=candidate)
+
+
+async def _upsert_publication_candidate(
+    db_session: AsyncSession,
+    *,
+    publication_id: int,
+    candidate: IdentifierCandidate,
+) -> None:
+    existing = await _existing_identifier(
+        db_session,
+        publication_id=publication_id,
+        kind=candidate.kind.value,
+        value_normalized=candidate.value_normalized,
+    )
+    if existing is None:
+        db_session.add(_new_identifier_row(publication_id=publication_id, candidate=candidate))
+        return
+    _merge_identifier_row(existing, candidate=candidate)
+
+
+async def _existing_identifier(
+    db_session: AsyncSession,
+    *,
+    publication_id: int,
+    kind: str,
+    value_normalized: str,
+) -> PublicationIdentifier | None:
+    result = await db_session.execute(
+        select(PublicationIdentifier).where(
+            PublicationIdentifier.publication_id == publication_id,
+            PublicationIdentifier.kind == kind,
+            PublicationIdentifier.value_normalized == value_normalized,
+        )
+    )
+    return result.scalar_one_or_none()
+
+
+async def _existing_identifier_by_kind(
+    db_session: AsyncSession,
+    *,
+    publication_id: int,
+    kind: str,
+) -> PublicationIdentifier | None:
+    result = await db_session.execute(
+        select(PublicationIdentifier).where(
+            PublicationIdentifier.publication_id == publication_id,
+            PublicationIdentifier.kind == kind,
+        ).order_by(PublicationIdentifier.confidence_score.desc()).limit(1)
+    )
+    return result.scalar_one_or_none()
+
+
+def _new_identifier_row(
+    *,
+    publication_id: int,
+    candidate: IdentifierCandidate,
+) -> PublicationIdentifier:
+    return PublicationIdentifier(
+        publication_id=publication_id,
+        kind=candidate.kind.value,
+        value_raw=candidate.value_raw,
+        value_normalized=candidate.value_normalized,
+        source=candidate.source,
+        confidence_score=candidate.confidence_score,
+        evidence_url=candidate.evidence_url,
+    )
+
+
+def _merge_identifier_row(existing: PublicationIdentifier, *, candidate: IdentifierCandidate) -> None:
+    if candidate.confidence_score >= float(existing.confidence_score):
+        existing.value_raw = candidate.value_raw
+        existing.source = candidate.source
+        existing.confidence_score = candidate.confidence_score
+        if candidate.evidence_url:
+            existing.evidence_url = candidate.evidence_url
+
+
+async def overlay_publication_items_with_display_identifiers(
+    db_session: AsyncSession,
+    *,
+    items: list[PublicationListItem],
+) -> list[PublicationListItem]:
+    if not items:
+        return []
+    mapping = await _display_identifier_map(db_session, publication_ids=[item.publication_id for item in items])
+    return [_overlay_publication_item(item, mapping.get(item.publication_id)) for item in items]
+
+
+def _overlay_publication_item(
+    item: PublicationListItem,
+    display_identifier: DisplayIdentifier | None,
+) -> PublicationListItem:
+    fallback = display_identifier or derive_display_identifier_from_values(doi=None, pub_url=item.pub_url, pdf_url=item.pdf_url)
+    return replace(item, display_identifier=fallback)
+
+
+async def overlay_pdf_queue_items_with_display_identifiers(
+    db_session: AsyncSession,
+    *,
+    items: list[PdfQueueListItem],
+) -> list[PdfQueueListItem]:
+    if not items:
+        return []
+    mapping = await _display_identifier_map(db_session, publication_ids=[item.publication_id for item in items])
+    return [_overlay_queue_item(item, mapping.get(item.publication_id)) for item in items]
+
+
+def _overlay_queue_item(
+    item: PdfQueueListItem,
+    display_identifier: DisplayIdentifier | None,
+) -> PdfQueueListItem:
+    fallback = display_identifier or derive_display_identifier_from_values(doi=None, pdf_url=item.pdf_url)
+    return replace(item, display_identifier=fallback)
+
+
+async def _display_identifier_map(
+    db_session: AsyncSession,
+    *,
+    publication_ids: list[int],
+) -> dict[int, DisplayIdentifier]:
+    normalized_ids = sorted({int(value) for value in publication_ids if int(value) > 0})
+    if not normalized_ids:
+        return {}
+    result = await db_session.execute(
+        select(PublicationIdentifier).where(PublicationIdentifier.publication_id.in_(normalized_ids))
+    )
+    rows = list(result.scalars().all())
+    return _best_display_identifier_map(rows)
+
+
+def _best_display_identifier_map(rows: list[PublicationIdentifier]) -> dict[int, DisplayIdentifier]:
+    grouped: dict[int, list[IdentifierCandidate]] = {}
+    for row in rows:
+        grouped.setdefault(int(row.publication_id), []).append(_candidate_from_row(row))
+    return {
+        publication_id: display
+        for publication_id, display in (
+            (publication_id, _best_display_identifier(candidates))
+            for publication_id, candidates in grouped.items()
+        )
+        if display is not None
+    }
+
+
+def _candidate_from_row(row: PublicationIdentifier) -> IdentifierCandidate:
+    return IdentifierCandidate(
+        kind=IdentifierKind(str(row.kind)),
+        value_raw=str(row.value_raw),
+        value_normalized=str(row.value_normalized),
+        source=str(row.source),
+        confidence_score=float(row.confidence_score),
+        evidence_url=row.evidence_url,
+    )
+
+
+def _best_display_identifier(candidates: list[IdentifierCandidate]) -> DisplayIdentifier | None:
+    if not candidates:
+        return None
+    ordered = sorted(candidates, key=_display_sort_key, reverse=True)
+    return _display_identifier_from_candidate(ordered[0])
+
+
+def _display_sort_key(candidate: IdentifierCandidate) -> tuple[int, float]:
+    return (_kind_priority(candidate.kind), float(candidate.confidence_score))
+
+
+def _kind_priority(kind: IdentifierKind) -> int:
+    if kind == IdentifierKind.DOI:
+        return PRIORITY_DOI
+    if kind == IdentifierKind.ARXIV:
+        return PRIORITY_ARXIV
+    if kind == IdentifierKind.PMCID:
+        return PRIORITY_PMCID
+    return PRIORITY_PMID
+
+
+def _display_identifier_from_candidate(candidate: IdentifierCandidate) -> DisplayIdentifier:
+    value = candidate.value_normalized
+    return DisplayIdentifier(
+        kind=candidate.kind.value,
+        value=value,
+        label=_display_label(candidate.kind, value),
+        url=_identifier_url(candidate.kind, value),
+        confidence_score=float(candidate.confidence_score),
+    )
+
+
+def _display_label(kind: IdentifierKind, value: str) -> str:
+    if kind == IdentifierKind.DOI:
+        return f"DOI: {value}"
+    if kind == IdentifierKind.ARXIV:
+        return f"arXiv: {value}"
+    if kind == IdentifierKind.PMCID:
+        return f"PMCID: {value}"
+    return f"PMID: {value}"
+
+
+def _identifier_url(kind: IdentifierKind, value: str) -> str | None:
+    if kind == IdentifierKind.DOI:
+        return f"https://doi.org/{value}"
+    if kind == IdentifierKind.ARXIV:
+        return f"https://arxiv.org/abs/{value}"
+    if kind == IdentifierKind.PMCID:
+        return f"https://pmc.ncbi.nlm.nih.gov/articles/{value}/"
+    if kind == IdentifierKind.PMID:
+        return f"https://pubmed.ncbi.nlm.nih.gov/{value}/"
+    return None

build/lib/app/services/domains/publication_identifiers/normalize.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import re
+from urllib.parse import urlparse
+
+from app.services.domains.doi.normalize import normalize_doi
+from app.services.domains.publication_identifiers.types import IdentifierKind
+
+ARXIV_ABS_RE = re.compile(r"\barxiv:\s*([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?\b", re.I)
+ARXIV_PATH_RE = re.compile(r"^/(?:abs|pdf|html|ps|format)/([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?(?:\.pdf)?/?$", re.I)
+PMCID_RE = re.compile(r"\b(PMC\d+)\b", re.I)
+PUBMED_PATH_RE = re.compile(r"^/(\d+)/?$")
+
+
+def normalize_identifier(kind: IdentifierKind, value: str | None) -> str | None:
+    if kind == IdentifierKind.DOI:
+        return normalize_doi(value)
+    if kind == IdentifierKind.ARXIV:
+        return normalize_arxiv_id(value)
+    if kind == IdentifierKind.PMCID:
+        return normalize_pmcid(value)
+    if kind == IdentifierKind.PMID:
+        return normalize_pmid(value)
+    return None
+
+
+def normalize_arxiv_id(value: str | None) -> str | None:
+    text = (value or "").strip()
+    if not text:
+        return None
+    parsed = urlparse(text)
+    if parsed.scheme in {"http", "https"} and "arxiv.org" in parsed.netloc.lower():
+        return _arxiv_from_path(parsed.path)
+    match = ARXIV_ABS_RE.search(text)
+    if not match:
+        return None
+    version = (match.group(2) or "").lower()
+    return f"{match.group(1).lower()}{version}"
+
+
+def _arxiv_from_path(path: str) -> str | None:
+    match = ARXIV_PATH_RE.match(path or "")
+    if not match:
+        return None
+    version = (match.group(2) or "").lower()
+    return f"{match.group(1).lower()}{version}"
+
+
+def normalize_pmcid(value: str | None) -> str | None:
+    text = (value or "").strip()
+    if not text:
+        return None
+    parsed = urlparse(text)
+    if parsed.scheme in {"http", "https"} and "ncbi.nlm.nih.gov" in parsed.netloc.lower():
+        return _first_match(PMCID_RE, parsed.path)
+    return _first_match(PMCID_RE, text)
+
+
+def normalize_pmid(value: str | None) -> str | None:
+    text = (value or "").strip()
+    if not text:
+        return None
+    parsed = urlparse(text)
+    if parsed.scheme in {"http", "https"} and "pubmed.ncbi.nlm.nih.gov" in parsed.netloc.lower():
+        match = PUBMED_PATH_RE.match(parsed.path or "")
+        if not match:
+            return None
+        return match.group(1)
+    return None
+
+
+def _first_match(pattern: re.Pattern[str], value: str) -> str | None:
+    match = pattern.search(value)
+    if not match:
+        return None
+    return match.group(1).upper()

build/lib/app/services/domains/publication_identifiers/types.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+
+
+class IdentifierKind(StrEnum):
+    DOI = "doi"
+    ARXIV = "arxiv"
+    PMCID = "pmcid"
+    PMID = "pmid"
+
+
+@dataclass(frozen=True)
+class DisplayIdentifier:
+    kind: str
+    value: str
+    label: str
+    url: str | None
+    confidence_score: float
+
+
+@dataclass(frozen=True)
+class IdentifierCandidate:
+    kind: IdentifierKind
+    value_raw: str
+    value_normalized: str
+    source: str
+    confidence_score: float
+    evidence_url: str | None

build/lib/app/services/domains/publications/__init__.py

@@ -0,0 +1 @@
+from app.services.domains.publications.application import *

build/lib/app/services/domains/publications/application.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from app.services.domains.publications.counts import (
+    count_for_user,
+    count_favorite_for_user,
+    count_latest_for_user,
+    count_unread_for_user,
+)
+from app.services.domains.publications.listing import (
+    list_for_user,
+    retry_pdf_for_user,
+    list_unread_for_user,
+)
+from app.services.domains.publications.enrichment import (
+    hydrate_pdf_enrichment_state,
+    schedule_missing_pdf_enrichment_for_user,
+    schedule_retry_pdf_enrichment_for_row,
+)
+from app.services.domains.publications.modes import (
+    MODE_ALL,
+    MODE_LATEST,
+    MODE_NEW,
+    MODE_UNREAD,
+    resolve_publication_view_mode,
+)
+from app.services.domains.publications.queries import (
+    get_latest_completed_run_id_for_user,
+    get_publication_item_for_user,
+    publications_query,
+)
+from app.services.domains.publications.read_state import (
+    mark_all_unread_as_read_for_user,
+    mark_selected_as_read_for_user,
+    set_publication_favorite_for_user,
+)
+from app.services.domains.publications.pdf_queue import (
+    count_pdf_queue_items,
+    enqueue_all_missing_pdf_jobs,
+    enqueue_retry_pdf_job_for_publication_id,
+    list_pdf_queue_page,
+    list_pdf_queue_items,
+)
+from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+
+__all__ = [
+    "MODE_ALL",
+    "MODE_UNREAD",
+    "MODE_LATEST",
+    "MODE_NEW",
+    "PublicationListItem",
+    "UnreadPublicationItem",
+    "resolve_publication_view_mode",
+    "get_latest_completed_run_id_for_user",
+    "publications_query",
+    "get_publication_item_for_user",
+    "list_for_user",
+    "list_unread_for_user",
+    "retry_pdf_for_user",
+    "hydrate_pdf_enrichment_state",
+    "schedule_retry_pdf_enrichment_for_row",
+    "list_pdf_queue_items",
+    "list_pdf_queue_page",
+    "count_pdf_queue_items",
+    "enqueue_all_missing_pdf_jobs",
+    "enqueue_retry_pdf_job_for_publication_id",
+    "schedule_missing_pdf_enrichment_for_user",
+    "count_for_user",
+    "count_favorite_for_user",
+    "count_unread_for_user",
+    "count_latest_for_user",
+    "mark_all_unread_as_read_for_user",
+    "mark_selected_as_read_for_user",
+    "set_publication_favorite_for_user",
+]

build/lib/app/services/domains/publications/counts.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import ScholarProfile, ScholarPublication
+from app.services.domains.publications.modes import (
+    MODE_ALL,
+    MODE_LATEST,
+    MODE_UNREAD,
+    resolve_publication_view_mode,
+)
+from app.services.domains.publications.queries import get_latest_completed_run_id_for_user
+
+
+async def count_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    mode: str = MODE_ALL,
+    scholar_profile_id: int | None = None,
+    favorite_only: bool = False,
+) -> int:
+    resolved_mode = resolve_publication_view_mode(mode)
+    latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
+    stmt = (
+        select(func.count())
+        .select_from(ScholarPublication)
+        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
+        .where(ScholarProfile.user_id == user_id)
+    )
+    if scholar_profile_id is not None:
+        stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
+    if favorite_only:
+        stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
+    if resolved_mode == MODE_UNREAD:
+        stmt = stmt.where(ScholarPublication.is_read.is_(False))
+    if resolved_mode == MODE_LATEST:
+        if latest_run_id is None:
+            return 0
+        stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
+    result = await db_session.execute(stmt)
+    return int(result.scalar_one() or 0)
+
+
+async def count_unread_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int | None = None,
+    favorite_only: bool = False,
+) -> int:
+    return await count_for_user(
+        db_session,
+        user_id=user_id,
+        mode=MODE_UNREAD,
+        scholar_profile_id=scholar_profile_id,
+        favorite_only=favorite_only,
+    )
+
+
+async def count_latest_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int | None = None,
+    favorite_only: bool = False,
+) -> int:
+    return await count_for_user(
+        db_session,
+        user_id=user_id,
+        mode=MODE_LATEST,
+        scholar_profile_id=scholar_profile_id,
+        favorite_only=favorite_only,
+    )
+
+
+async def count_favorite_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int | None = None,
+) -> int:
+    return await count_for_user(
+        db_session,
+        user_id=user_id,
+        mode=MODE_ALL,
+        scholar_profile_id=scholar_profile_id,
+        favorite_only=True,
+    )

build/lib/app/services/domains/publications/enrichment.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import logging
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.services.domains.publications.pdf_queue import (
+    enqueue_missing_pdf_jobs,
+    enqueue_retry_pdf_job,
+    overlay_pdf_job_state,
+)
+from app.services.domains.publications.types import PublicationListItem
+
+logger = logging.getLogger(__name__)
+
+
+async def schedule_missing_pdf_enrichment_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    request_email: str | None,
+    items: list[PublicationListItem],
+    max_items: int,
+) -> int:
+    queued_ids = await enqueue_missing_pdf_jobs(
+        db_session,
+        user_id=user_id,
+        request_email=request_email,
+        rows=items,
+        max_items=max_items,
+    )
+    logger.info(
+        "publications.enrichment.scheduled",
+        extra={
+            "event": "publications.enrichment.scheduled",
+            "user_id": user_id,
+            "publication_count": len(queued_ids),
+        },
+    )
+    return len(queued_ids)
+
+
+async def schedule_retry_pdf_enrichment_for_row(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    request_email: str | None,
+    item: PublicationListItem,
+) -> bool:
+    queued = await enqueue_retry_pdf_job(
+        db_session,
+        user_id=user_id,
+        request_email=request_email,
+        row=item,
+    )
+    logger.info(
+        "publications.enrichment.retry_scheduled",
+        extra={
+            "event": "publications.enrichment.retry_scheduled",
+            "user_id": user_id,
+            "publication_id": item.publication_id,
+            "queued": queued,
+        },
+    )
+    return queued
+
+
+async def hydrate_pdf_enrichment_state(
+    db_session: AsyncSession,
+    *,
+    items: list[PublicationListItem],
+) -> list[PublicationListItem]:
+    return await overlay_pdf_job_state(db_session, rows=items)

build/lib/app/services/domains/publications/listing.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.services.domains.publication_identifiers import application as identifier_service
+from app.services.domains.publications.modes import (
+    MODE_ALL,
+    MODE_UNREAD,
+    resolve_publication_view_mode,
+)
+from app.services.domains.publications.queries import (
+    get_latest_completed_run_id_for_user,
+    get_publication_item_for_user,
+    publication_list_item_from_row,
+    publications_query,
+    unread_item_from_row,
+)
+from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+
+
+async def list_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    mode: str = MODE_ALL,
+    scholar_profile_id: int | None = None,
+    favorite_only: bool = False,
+    limit: int = 300,
+    offset: int = 0,
+) -> list[PublicationListItem]:
+    resolved_mode = resolve_publication_view_mode(mode)
+    latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
+    result = await db_session.execute(
+        publications_query(
+            user_id=user_id,
+            mode=resolved_mode,
+            latest_run_id=latest_run_id,
+            scholar_profile_id=scholar_profile_id,
+            favorite_only=favorite_only,
+            limit=limit,
+            offset=offset,
+        )
+    )
+    rows = [
+        publication_list_item_from_row(row, latest_run_id=latest_run_id)
+        for row in result.all()
+    ]
+    return await identifier_service.overlay_publication_items_with_display_identifiers(
+        db_session,
+        items=rows,
+    )
+
+
+async def retry_pdf_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+    publication_id: int,
+) -> PublicationListItem | None:
+    item = await get_publication_item_for_user(
+        db_session,
+        user_id=user_id,
+        scholar_profile_id=scholar_profile_id,
+        publication_id=publication_id,
+    )
+    if item is None:
+        return None
+    hydrated = await identifier_service.overlay_publication_items_with_display_identifiers(
+        db_session,
+        items=[item],
+    )
+    return hydrated[0] if hydrated else item
+
+
+async def list_unread_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    limit: int = 100,
+) -> list[UnreadPublicationItem]:
+    result = await db_session.execute(
+        publications_query(
+            user_id=user_id,
+            mode=MODE_UNREAD,
+            latest_run_id=None,
+            scholar_profile_id=None,
+            favorite_only=False,
+            limit=limit,
+            offset=0,
+        )
+    )
+    return [unread_item_from_row(row) for row in result.all()]

build/lib/app/services/domains/publications/modes.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+MODE_ALL = "all"
+MODE_UNREAD = "unread"
+MODE_LATEST = "latest"
+MODE_NEW = "new"  # compatibility alias for MODE_LATEST
+
+
+def resolve_publication_view_mode(value: str | None) -> str:
+    if value == MODE_UNREAD:
+        return MODE_UNREAD
+    if value in {MODE_LATEST, MODE_NEW}:
+        return MODE_LATEST
+    return MODE_ALL

build/lib/app/services/domains/publications/pdf_queue.py

@@ -0,0 +1,915 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from datetime import datetime, timezone
+import logging
+
+from sqlalchemy import Select, func, literal, or_, select, union_all
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import (
+    Publication,
+    PublicationPdfJob,
+    PublicationPdfJobEvent,
+    ScholarProfile,
+    ScholarPublication,
+    User,
+)
+from app.db.session import get_session_factory
+from app.services.domains.publication_identifiers import application as identifier_service
+from app.services.domains.publication_identifiers.types import DisplayIdentifier
+from app.services.domains.publications.pdf_resolution_pipeline import (
+    resolve_publication_pdf_outcome_for_row,
+)
+from app.services.domains.publications.types import PublicationListItem
+from app.services.domains.unpaywall.application import (
+    FAILURE_RESOLUTION_EXCEPTION,
+    OaResolutionOutcome,
+)
+from app.settings import settings
+
+PDF_STATUS_UNTRACKED = "untracked"
+PDF_STATUS_QUEUED = "queued"
+PDF_STATUS_RUNNING = "running"
+PDF_STATUS_RESOLVED = "resolved"
+PDF_STATUS_FAILED = "failed"
+
+PDF_EVENT_QUEUED = "queued"
+PDF_EVENT_ATTEMPT_STARTED = "attempt_started"
+PDF_EVENT_RESOLVED = "resolved"
+PDF_EVENT_FAILED = "failed"
+
+logger = logging.getLogger(__name__)
+_scheduled_tasks: set[asyncio.Task[None]] = set()
+
+
+@dataclass(frozen=True)
+class PdfQueueListItem:
+    publication_id: int
+    title: str
+    pdf_url: str | None
+    status: str
+    attempt_count: int
+    last_failure_reason: str | None
+    last_failure_detail: str | None
+    last_source: str | None
+    requested_by_user_id: int | None
+    requested_by_email: str | None
+    queued_at: datetime | None
+    last_attempt_at: datetime | None
+    resolved_at: datetime | None
+    updated_at: datetime
+    display_identifier: DisplayIdentifier | None = None
+
+
+@dataclass(frozen=True)
+class PdfRequeueResult:
+    publication_exists: bool
+    queued: bool
+
+
+@dataclass(frozen=True)
+class PdfBulkQueueResult:
+    requested_count: int
+    queued_count: int
+
+
+@dataclass(frozen=True)
+class PdfQueuePage:
+    items: list[PdfQueueListItem]
+    total_count: int
+    limit: int
+    offset: int
+
+
+def _utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _publication_ids(rows: list[PublicationListItem]) -> list[int]:
+    return sorted({row.publication_id for row in rows})
+
+
+def _status_from_job(row: PublicationListItem, job: PublicationPdfJob | None) -> str:
+    if row.pdf_url:
+        return PDF_STATUS_RESOLVED
+    if job is None:
+        return PDF_STATUS_UNTRACKED
+    return job.status
+
+
+def _item_from_row_and_job(
+    row: PublicationListItem,
+    job: PublicationPdfJob | None,
+) -> PublicationListItem:
+    return PublicationListItem(
+        publication_id=row.publication_id,
+        scholar_profile_id=row.scholar_profile_id,
+        scholar_label=row.scholar_label,
+        title=row.title,
+        year=row.year,
+        citation_count=row.citation_count,
+        venue_text=row.venue_text,
+        pub_url=row.pub_url,
+        pdf_url=row.pdf_url,
+        is_read=row.is_read,
+        is_favorite=row.is_favorite,
+        first_seen_at=row.first_seen_at,
+        is_new_in_latest_run=row.is_new_in_latest_run,
+        pdf_status=_status_from_job(row, job),
+        pdf_attempt_count=int(job.attempt_count) if job is not None else 0,
+        pdf_failure_reason=job.last_failure_reason if job is not None else None,
+        pdf_failure_detail=job.last_failure_detail if job is not None else None,
+        display_identifier=row.display_identifier,
+    )
+
+
+def _queueable_rows(
+    rows: list[PublicationListItem],
+    *,
+    max_items: int,
+) -> list[PublicationListItem]:
+    bounded = max(0, int(max_items))
+    if bounded == 0:
+        return []
+    candidates = [row for row in rows if not row.pdf_url]
+    return candidates[:bounded]
+
+
+def _bounded_limit(limit: int, *, max_value: int = 500) -> int:
+    return max(1, min(int(limit), max_value))
+
+
+def _bounded_offset(offset: int) -> int:
+    return max(int(offset), 0)
+
+
+def _auto_retry_interval_seconds() -> int:
+    return max(int(settings.pdf_auto_retry_interval_seconds), 1)
+
+
+def _auto_retry_first_interval_seconds() -> int:
+    return max(int(settings.pdf_auto_retry_first_interval_seconds), 1)
+
+
+def _auto_retry_max_attempts() -> int:
+    return max(int(settings.pdf_auto_retry_max_attempts), 1)
+
+
+def _retry_interval_seconds_for_attempt_count(attempt_count: int) -> int:
+    if int(attempt_count) <= 1:
+        return _auto_retry_first_interval_seconds()
+    return _auto_retry_interval_seconds()
+
+
+def _cooldown_active(
+    *,
+    last_attempt_at: datetime | None,
+    attempt_count: int,
+) -> bool:
+    if last_attempt_at is None:
+        return False
+    elapsed = (_utcnow() - last_attempt_at).total_seconds()
+    return elapsed < _retry_interval_seconds_for_attempt_count(int(attempt_count))
+
+
+def _can_enqueue_job(
+    job: PublicationPdfJob | None,
+    *,
+    force_retry: bool,
+) -> bool:
+    if job is None:
+        return True
+    if job.status in {PDF_STATUS_QUEUED, PDF_STATUS_RUNNING}:
+        return False
+    if force_retry:
+        return job.status in {PDF_STATUS_FAILED, PDF_STATUS_RESOLVED, PDF_STATUS_UNTRACKED}
+    if job.status == PDF_STATUS_RESOLVED:
+        return False
+    if int(job.attempt_count) >= _auto_retry_max_attempts():
+        return False
+    if _cooldown_active(
+        last_attempt_at=job.last_attempt_at,
+        attempt_count=int(job.attempt_count),
+    ):
+        return False
+    return True
+
+
+def _event_row(
+    *,
+    publication_id: int,
+    user_id: int | None,
+    event_type: str,
+    status: str | None,
+    source: str | None = None,
+    failure_reason: str | None = None,
+    message: str | None = None,
+) -> PublicationPdfJobEvent:
+    return PublicationPdfJobEvent(
+        publication_id=publication_id,
+        user_id=user_id,
+        event_type=event_type,
+        status=status,
+        source=source,
+        failure_reason=failure_reason,
+        message=message,
+    )
+
+
+def _queued_job(
+    *,
+    publication_id: int,
+    user_id: int,
+) -> PublicationPdfJob:
+    now = _utcnow()
+    return PublicationPdfJob(
+        publication_id=publication_id,
+        status=PDF_STATUS_QUEUED,
+        queued_at=now,
+        last_requested_by_user_id=user_id,
+    )
+
+
+def _mark_job_queued(job: PublicationPdfJob, *, user_id: int) -> None:
+    now = _utcnow()
+    job.status = PDF_STATUS_QUEUED
+    job.queued_at = now
+    job.last_requested_by_user_id = user_id
+    job.last_failure_reason = None
+    job.last_failure_detail = None
+    job.last_source = None
+
+
+def _state_map(jobs: list[PublicationPdfJob]) -> dict[int, PublicationPdfJob]:
+    return {int(job.publication_id): job for job in jobs}
+
+
+async def _jobs_for_publication_ids(
+    db_session: AsyncSession,
+    *,
+    publication_ids: list[int],
+) -> dict[int, PublicationPdfJob]:
+    if not publication_ids:
+        return {}
+    result = await db_session.execute(
+        select(PublicationPdfJob).where(PublicationPdfJob.publication_id.in_(publication_ids))
+    )
+    return _state_map(list(result.scalars()))
+
+
+async def overlay_pdf_job_state(
+    db_session: AsyncSession,
+    *,
+    rows: list[PublicationListItem],
+) -> list[PublicationListItem]:
+    if not rows:
+        return []
+    jobs = await _jobs_for_publication_ids(
+        db_session,
+        publication_ids=_publication_ids(rows),
+    )
+    return [_item_from_row_and_job(row, jobs.get(row.publication_id)) for row in rows]
+
+
+async def _enqueue_rows(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    rows: list[PublicationListItem],
+    force_retry: bool,
+) -> list[PublicationListItem]:
+    if not rows:
+        return []
+    queued: list[PublicationListItem] = []
+    jobs = await _jobs_for_publication_ids(
+        db_session,
+        publication_ids=_publication_ids(rows),
+    )
+    for row in rows:
+        job = jobs.get(row.publication_id)
+        if not _can_enqueue_job(job, force_retry=force_retry):
+            continue
+        if job is None:
+            job = _queued_job(publication_id=row.publication_id, user_id=user_id)
+            jobs[row.publication_id] = job
+            db_session.add(job)
+        else:
+            _mark_job_queued(job, user_id=user_id)
+        db_session.add(
+            _event_row(
+                publication_id=row.publication_id,
+                user_id=user_id,
+                event_type=PDF_EVENT_QUEUED,
+                status=PDF_STATUS_QUEUED,
+            )
+        )
+        queued.append(row)
+    if queued:
+        await db_session.commit()
+    return queued
+
+
+def _register_task(task: asyncio.Task[None]) -> None:
+    _scheduled_tasks.add(task)
+
+
+def _drop_finished_task(task: asyncio.Task[None]) -> None:
+    _scheduled_tasks.discard(task)
+    try:
+        task.result()
+    except Exception:
+        logger.exception(
+            "publications.pdf_queue.task_failed",
+            extra={"event": "publications.pdf_queue.task_failed"},
+        )
+
+
+async def _mark_attempt_started(
+    *,
+    publication_id: int,
+    user_id: int,
+) -> None:
+    session_factory = get_session_factory()
+    async with session_factory() as db_session:
+        job = await db_session.get(PublicationPdfJob, publication_id)
+        if job is None:
+            job = _queued_job(publication_id=publication_id, user_id=user_id)
+            db_session.add(job)
+        job.status = PDF_STATUS_RUNNING
+        job.last_attempt_at = _utcnow()
+        job.attempt_count = int(job.attempt_count) + 1
+        db_session.add(
+            _event_row(
+                publication_id=publication_id,
+                user_id=user_id,
+                event_type=PDF_EVENT_ATTEMPT_STARTED,
+                status=PDF_STATUS_RUNNING,
+            )
+        )
+        await db_session.commit()
+
+
+def _failed_outcome(
+    *,
+    row: PublicationListItem,
+) -> OaResolutionOutcome:
+    return OaResolutionOutcome(
+        publication_id=row.publication_id,
+        doi=None,
+        pdf_url=None,
+        failure_reason=FAILURE_RESOLUTION_EXCEPTION,
+        source=None,
+        used_crossref=False,
+    )
+
+
+async def _fetch_outcome_for_row(
+    *,
+    row: PublicationListItem,
+    request_email: str | None,
+) -> OaResolutionOutcome:
+    pipeline_result = await resolve_publication_pdf_outcome_for_row(
+        row=row,
+        request_email=request_email,
+    )
+    outcome = pipeline_result.outcome
+    if outcome is not None:
+        return outcome
+    return _failed_outcome(row=row)
+
+
+def _apply_publication_update(
+    publication: Publication,
+    *,
+    pdf_url: str | None,
+) -> None:
+    if pdf_url and publication.pdf_url != pdf_url:
+        publication.pdf_url = pdf_url
+
+
+def _apply_job_outcome(job: PublicationPdfJob, *, outcome: OaResolutionOutcome) -> None:
+    job.last_source = outcome.source
+    if outcome.pdf_url:
+        job.status = PDF_STATUS_RESOLVED
+        job.resolved_at = _utcnow()
+        job.last_failure_reason = None
+        job.last_failure_detail = None
+        return
+    job.status = PDF_STATUS_FAILED
+    job.last_failure_reason = outcome.failure_reason
+    job.last_failure_detail = outcome.failure_reason
+
+
+def _result_event(outcome: OaResolutionOutcome) -> tuple[str, str]:
+    if outcome.pdf_url:
+        return PDF_EVENT_RESOLVED, PDF_STATUS_RESOLVED
+    return PDF_EVENT_FAILED, PDF_STATUS_FAILED
+
+
+async def _persist_outcome(
+    *,
+    publication_id: int,
+    user_id: int,
+    outcome: OaResolutionOutcome,
+) -> None:
+    session_factory = get_session_factory()
+    async with session_factory() as db_session:
+        publication = await db_session.get(Publication, publication_id)
+        job = await db_session.get(PublicationPdfJob, publication_id)
+        if publication is None or job is None:
+            return
+        _apply_publication_update(publication, pdf_url=outcome.pdf_url)
+        await identifier_service.sync_identifiers_for_publication_resolution(
+            db_session,
+            publication=publication,
+            source=outcome.source,
+        )
+        _apply_job_outcome(job, outcome=outcome)
+        event_type, status = _result_event(outcome)
+        db_session.add(
+            _event_row(
+                publication_id=publication_id,
+                user_id=user_id,
+                event_type=event_type,
+                status=status,
+                source=outcome.source,
+                failure_reason=outcome.failure_reason,
+                message=outcome.failure_reason,
+            )
+        )
+        await db_session.commit()
+
+
+async def _resolve_publication_row(
+    *,
+    user_id: int,
+    request_email: str | None,
+    row: PublicationListItem,
+) -> None:
+    await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
+    try:
+        outcome = await _fetch_outcome_for_row(row=row, request_email=request_email)
+    except Exception as exc:  # pragma: no cover - defensive network boundary
+        logger.warning(
+            "publications.pdf_queue.resolve_failed",
+            extra={
+                "event": "publications.pdf_queue.resolve_failed",
+                "publication_id": row.publication_id,
+                "error": str(exc),
+            },
+        )
+        outcome = _failed_outcome(row=row)
+    await _persist_outcome(
+        publication_id=row.publication_id,
+        user_id=user_id,
+        outcome=outcome,
+    )
+
+
+async def _run_resolution_task(
+    *,
+    user_id: int,
+    request_email: str | None,
+    rows: list[PublicationListItem],
+) -> None:
+    for row in rows:
+        await _resolve_publication_row(
+            user_id=user_id,
+            request_email=request_email,
+            row=row,
+        )
+
+
+def _schedule_rows(
+    *,
+    user_id: int,
+    request_email: str | None,
+    rows: list[PublicationListItem],
+) -> None:
+    if not rows:
+        return
+    task = asyncio.create_task(
+        _run_resolution_task(
+            user_id=user_id,
+            request_email=request_email,
+            rows=rows,
+        )
+    )
+    _register_task(task)
+    task.add_done_callback(_drop_finished_task)
+
+
+async def enqueue_missing_pdf_jobs(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    request_email: str | None,
+    rows: list[PublicationListItem],
+    max_items: int,
+) -> list[int]:
+    queueable = _queueable_rows(rows, max_items=max_items)
+    queued_rows = await _enqueue_rows(
+        db_session,
+        user_id=user_id,
+        rows=queueable,
+        force_retry=False,
+    )
+    _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
+    return [row.publication_id for row in queued_rows]
+
+
+async def enqueue_retry_pdf_job(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    request_email: str | None,
+    row: PublicationListItem,
+) -> bool:
+    queued_rows = await _enqueue_rows(
+        db_session,
+        user_id=user_id,
+        rows=[row],
+        force_retry=True,
+    )
+    _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
+    return bool(queued_rows)
+
+
+def _retry_item_label(display_name: str | None, scholar_id: str | None) -> str:
+    return str(display_name or scholar_id or "unknown")
+
+
+def _retry_item_from_publication(
+    publication: Publication,
+    *,
+    link_row: tuple | None,
+) -> PublicationListItem:
+    if link_row is None:
+        scholar_profile_id = 0
+        scholar_label = "unknown"
+        is_read = True
+        first_seen_at = publication.created_at or _utcnow()
+    else:
+        scholar_profile_id = int(link_row[0])
+        scholar_label = _retry_item_label(link_row[1], link_row[2])
+        is_read = bool(link_row[3])
+        first_seen_at = link_row[4] or publication.created_at or _utcnow()
+    return PublicationListItem(
+        publication_id=int(publication.id),
+        scholar_profile_id=scholar_profile_id,
+        scholar_label=scholar_label,
+        title=publication.title_raw,
+        year=publication.year,
+        citation_count=int(publication.citation_count or 0),
+        venue_text=publication.venue_text,
+        pub_url=publication.pub_url,
+        pdf_url=publication.pdf_url,
+        is_read=is_read,
+        first_seen_at=first_seen_at,
+        is_new_in_latest_run=False,
+    )
+
+
+async def _retry_item_for_publication_id(
+    db_session: AsyncSession,
+    *,
+    publication_id: int,
+) -> PublicationListItem | None:
+    publication = await db_session.get(Publication, publication_id)
+    if publication is None:
+        return None
+    result = await db_session.execute(
+        select(
+            ScholarProfile.id,
+            ScholarProfile.display_name,
+            ScholarProfile.scholar_id,
+            ScholarPublication.is_read,
+            ScholarPublication.created_at,
+        )
+        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
+        .where(ScholarPublication.publication_id == publication_id)
+        .order_by(ScholarPublication.created_at.asc())
+        .limit(1)
+    )
+    return _retry_item_from_publication(publication, link_row=result.one_or_none())
+
+
+async def enqueue_retry_pdf_job_for_publication_id(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    request_email: str | None,
+    publication_id: int,
+) -> PdfRequeueResult:
+    row = await _retry_item_for_publication_id(
+        db_session,
+        publication_id=publication_id,
+    )
+    if row is None:
+        return PdfRequeueResult(publication_exists=False, queued=False)
+    queued = await enqueue_retry_pdf_job(
+        db_session,
+        user_id=user_id,
+        request_email=request_email,
+        row=row,
+    )
+    return PdfRequeueResult(publication_exists=True, queued=queued)
+
+
+def _queue_candidate_from_publication(publication: Publication) -> PublicationListItem:
+    return PublicationListItem(
+        publication_id=int(publication.id),
+        scholar_profile_id=0,
+        scholar_label="admin",
+        title=publication.title_raw,
+        year=publication.year,
+        citation_count=int(publication.citation_count or 0),
+        venue_text=publication.venue_text,
+        pub_url=publication.pub_url,
+        pdf_url=publication.pdf_url,
+        is_read=True,
+        first_seen_at=publication.created_at or _utcnow(),
+        is_new_in_latest_run=False,
+    )
+
+
+async def _missing_pdf_candidates(
+    db_session: AsyncSession,
+    *,
+    limit: int,
+) -> list[PublicationListItem]:
+    bounded_limit = max(1, min(int(limit), 5000))
+    result = await db_session.execute(
+        select(Publication)
+        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
+        .where(Publication.pdf_url.is_(None))
+        .where(
+            or_(
+                PublicationPdfJob.publication_id.is_(None),
+                PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
+            )
+        )
+        .order_by(Publication.updated_at.desc(), Publication.id.desc())
+        .limit(bounded_limit)
+    )
+    return [
+        _queue_candidate_from_publication(publication)
+        for publication in result.scalars()
+    ]
+
+
+async def enqueue_all_missing_pdf_jobs(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    request_email: str | None,
+    limit: int = 1000,
+) -> PdfBulkQueueResult:
+    candidates = await _missing_pdf_candidates(db_session, limit=limit)
+    queued_rows = await _enqueue_rows(
+        db_session,
+        user_id=user_id,
+        rows=candidates,
+        force_retry=True,
+    )
+    _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
+    return PdfBulkQueueResult(
+        requested_count=len(candidates),
+        queued_count=len(queued_rows),
+    )
+
+
+def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]:
+    stmt = (
+        select(
+            PublicationPdfJob.publication_id,
+            Publication.title_raw,
+            Publication.pdf_url,
+            PublicationPdfJob.status,
+            PublicationPdfJob.attempt_count,
+            PublicationPdfJob.last_failure_reason,
+            PublicationPdfJob.last_failure_detail,
+            PublicationPdfJob.last_source,
+            PublicationPdfJob.last_requested_by_user_id,
+            User.email,
+            PublicationPdfJob.queued_at,
+            PublicationPdfJob.last_attempt_at,
+            PublicationPdfJob.resolved_at,
+            PublicationPdfJob.updated_at,
+        )
+        .join(Publication, Publication.id == PublicationPdfJob.publication_id)
+        .outerjoin(User, User.id == PublicationPdfJob.last_requested_by_user_id)
+    )
+    if status:
+        stmt = stmt.where(PublicationPdfJob.status == status)
+    return stmt
+
+
+def _tracked_queue_select(*, limit: int, offset: int, status: str | None) -> Select[tuple]:
+    return (
+        _tracked_queue_select_base(status=status)
+        .order_by(PublicationPdfJob.updated_at.desc())
+        .limit(_bounded_limit(limit))
+        .offset(_bounded_offset(offset))
+    )
+
+
+def _untracked_queue_select_base() -> Select[tuple]:
+    return (
+        select(
+            Publication.id,
+            Publication.title_raw,
+            Publication.pdf_url,
+            literal(PDF_STATUS_UNTRACKED),
+            literal(0),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            Publication.updated_at,
+        )
+        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
+        .where(Publication.pdf_url.is_(None))
+        .where(PublicationPdfJob.publication_id.is_(None))
+    )
+
+
+def _untracked_queue_select(*, limit: int, offset: int) -> Select[tuple]:
+    return (
+        _untracked_queue_select_base()
+        .order_by(Publication.updated_at.desc(), Publication.id.desc())
+        .limit(_bounded_limit(limit))
+        .offset(_bounded_offset(offset))
+    )
+
+
+def _all_queue_select(*, limit: int, offset: int) -> Select[tuple]:
+    union_stmt = union_all(
+        _tracked_queue_select_base(status=None),
+        _untracked_queue_select_base(),
+    ).subquery()
+    return (
+        select(union_stmt)
+        .order_by(union_stmt.c.updated_at.desc())
+        .limit(_bounded_limit(limit))
+        .offset(_bounded_offset(offset))
+    )
+
+
+def _tracked_queue_count_select(*, status: str | None) -> Select[tuple]:
+    stmt = select(func.count()).select_from(PublicationPdfJob)
+    if status:
+        stmt = stmt.where(PublicationPdfJob.status == status)
+    return stmt
+
+
+def _untracked_queue_count_select() -> Select[tuple]:
+    return (
+        select(func.count())
+        .select_from(Publication)
+        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
+        .where(Publication.pdf_url.is_(None))
+        .where(PublicationPdfJob.publication_id.is_(None))
+    )
+
+
+def _queue_item_from_row(row: tuple) -> PdfQueueListItem:
+    return PdfQueueListItem(
+        publication_id=int(row[0]),
+        title=str(row[1] or ""),
+        pdf_url=row[2],
+        status=str(row[3] or PDF_STATUS_UNTRACKED),
+        attempt_count=int(row[4] or 0),
+        last_failure_reason=row[5],
+        last_failure_detail=row[6],
+        last_source=row[7],
+        requested_by_user_id=int(row[8]) if row[8] is not None else None,
+        requested_by_email=row[9],
+        queued_at=row[10],
+        last_attempt_at=row[11],
+        resolved_at=row[12],
+        updated_at=row[13],
+    )
+
+
+async def _hydrated_queue_items(
+    db_session: AsyncSession,
+    *,
+    rows: list[tuple],
+) -> list[PdfQueueListItem]:
+    items = [_queue_item_from_row(row) for row in rows]
+    return await identifier_service.overlay_pdf_queue_items_with_display_identifiers(
+        db_session,
+        items=items,
+    )
+
+
+async def list_pdf_queue_items(
+    db_session: AsyncSession,
+    *,
+    limit: int = 100,
+    offset: int = 0,
+    status: str | None = None,
+) -> list[PdfQueueListItem]:
+    bounded_limit = _bounded_limit(limit)
+    bounded_offset = _bounded_offset(offset)
+    normalized_status = (status or "").strip().lower() or None
+    if normalized_status == PDF_STATUS_UNTRACKED:
+        result = await db_session.execute(
+            _untracked_queue_select(
+                limit=bounded_limit,
+                offset=bounded_offset,
+            )
+        )
+        return await _hydrated_queue_items(db_session, rows=list(result.all()))
+    if normalized_status is None:
+        result = await db_session.execute(
+            _all_queue_select(
+                limit=bounded_limit,
+                offset=bounded_offset,
+            )
+        )
+        return await _hydrated_queue_items(db_session, rows=list(result.all()))
+    result = await db_session.execute(
+        _tracked_queue_select(
+            limit=bounded_limit,
+            offset=bounded_offset,
+            status=normalized_status,
+        )
+    )
+    return await _hydrated_queue_items(db_session, rows=list(result.all()))
+
+
+async def count_pdf_queue_items(
+    db_session: AsyncSession,
+    *,
+    status: str | None = None,
+) -> int:
+    normalized_status = (status or "").strip().lower() or None
+    if normalized_status == PDF_STATUS_UNTRACKED:
+        result = await db_session.execute(_untracked_queue_count_select())
+        return int(result.scalar_one() or 0)
+    tracked_result = await db_session.execute(
+        _tracked_queue_count_select(status=normalized_status)
+    )
+    tracked_count = int(tracked_result.scalar_one() or 0)
+    if normalized_status is not None:
+        return tracked_count
+    untracked_result = await db_session.execute(_untracked_queue_count_select())
+    untracked_count = int(untracked_result.scalar_one() or 0)
+    return tracked_count + untracked_count
+
+
+async def list_pdf_queue_page(
+    db_session: AsyncSession,
+    *,
+    limit: int = 100,
+    offset: int = 0,
+    status: str | None = None,
+) -> PdfQueuePage:
+    bounded_limit = _bounded_limit(limit)
+    bounded_offset = _bounded_offset(offset)
+    items = await list_pdf_queue_items(
+        db_session,
+        limit=bounded_limit,
+        offset=bounded_offset,
+        status=status,
+    )
+    total_count = await count_pdf_queue_items(
+        db_session,
+        status=status,
+    )
+    return PdfQueuePage(
+        items=items,
+        total_count=total_count,
+        limit=bounded_limit,
+        offset=bounded_offset,
+    )
+
+
+async def drain_ready_jobs(
+    db_session: AsyncSession,
+    *,
+    limit: int,
+    max_attempts: int,
+) -> int:
+    result = await db_session.execute(
+        select(User.id).where(User.is_active.is_(True)).order_by(User.id.asc()).limit(1)
+    )
+    system_user_id = result.scalar_one_or_none()
+    if system_user_id is None:
+        return 0
+
+    bulk_result = await enqueue_all_missing_pdf_jobs(
+        db_session,
+        user_id=system_user_id,
+        request_email=settings.unpaywall_email,
+        limit=limit,
+    )
+    return bulk_result.queued_count

build/lib/app/services/domains/publications/pdf_resolution_pipeline.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import logging
+from typing import Any
+
+from app.services.domains.publications.types import PublicationListItem
+from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class PipelineOutcome:
+    outcome: OaResolutionOutcome | None
+    scholar_candidates: Any | None  # Kept for backward compatibility with calling signatures
+
+
+async def resolve_publication_pdf_outcome_for_row(
+    *,
+    row: PublicationListItem,
+    request_email: str | None,
+) -> PipelineOutcome:
+    # 1. OpenAlex OA
+    openalex_outcome = await _openalex_outcome(row, request_email=request_email)
+    if openalex_outcome and openalex_outcome.pdf_url:
+        return PipelineOutcome(openalex_outcome, None)
+
+    # 2. arXiv
+    arxiv_outcome = await _arxiv_outcome(row, request_email=request_email)
+    if arxiv_outcome and arxiv_outcome.pdf_url:
+        return PipelineOutcome(arxiv_outcome, None)
+
+    # 3. Unpaywall (which falls back to Crossref)
+    oa_outcome = await _oa_outcome(row=row, request_email=request_email)
+    return PipelineOutcome(oa_outcome, None)
+
+
+async def _openalex_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None:
+    from app.services.domains.openalex.client import OpenAlexClient
+    from app.services.domains.openalex.matching import find_best_match
+
+    if not row.title:
+        return None
+
+    import re
+    safe_title = re.sub(r"[^\w\s]", " ", row.title)
+    safe_title = " ".join(safe_title.split())
+    if not safe_title:
+        return None
+
+    client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=request_email or settings.crossref_api_mailto)
+    try:
+        openalex_works = await client.get_works_by_filter({"title.search": safe_title}, limit=5)
+        match = find_best_match(
+            target_title=row.title,
+            target_year=row.year,
+            target_authors=row.scholar_label,
+            candidates=openalex_works,
+        )
+        if match and match.oa_url:
+            return OaResolutionOutcome(
+                publication_id=row.publication_id,
+                doi=match.doi,
+                pdf_url=match.oa_url,
+                failure_reason=None,
+                source="openalex",
+                used_crossref=False,
+            )
+    except Exception as exc:
+        logger.warning(
+            "publications.pdf_resolution.openalex_failed",
+            extra={"event": "publications.pdf_resolution.openalex_failed", "error": str(exc)},
+        )
+    return None
+
+
+async def _arxiv_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None:
+    from app.services.domains.arxiv.application import discover_arxiv_id_for_publication
+
+    try:
+        arxiv_id = await discover_arxiv_id_for_publication(item=row, request_email=request_email)
+        if arxiv_id:
+            pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
+            return OaResolutionOutcome(
+                publication_id=row.publication_id,
+                doi=None,
+                pdf_url=pdf_url,
+                failure_reason=None,
+                source="arxiv",
+                used_crossref=False,
+            )
+    except Exception as exc:
+        logger.warning(
+            "publications.pdf_resolution.arxiv_failed",
+            extra={"event": "publications.pdf_resolution.arxiv_failed", "error": str(exc)},
+        )
+    return None
+
+
+async def _oa_outcome(
+    *,
+    row: PublicationListItem,
+    request_email: str | None,
+) -> OaResolutionOutcome | None:
+    outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email)
+    return outcomes.get(row.publication_id)

build/lib/app/services/domains/publications/queries.py

@@ -0,0 +1,203 @@
+from __future__ import annotations
+
+from sqlalchemy import Select, func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import CrawlRun, Publication, RunStatus, ScholarProfile, ScholarPublication
+from app.services.domains.publications.modes import MODE_LATEST, MODE_UNREAD
+from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+
+
+def _normalized_citation_count(value: object) -> int:
+    try:
+        return int(value or 0)
+    except (TypeError, ValueError):
+        return 0
+
+
+async def get_latest_completed_run_id_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> int | None:
+    result = await db_session.execute(
+        select(func.max(CrawlRun.id)).where(
+            CrawlRun.user_id == user_id,
+            CrawlRun.status != RunStatus.RUNNING,
+        )
+    )
+    latest_run_id = result.scalar_one_or_none()
+    return int(latest_run_id) if latest_run_id is not None else None
+
+
+def publications_query(
+    *,
+    user_id: int,
+    mode: str,
+    latest_run_id: int | None,
+    scholar_profile_id: int | None,
+    favorite_only: bool,
+    limit: int,
+    offset: int = 0,
+) -> Select[tuple]:
+    scholar_label = ScholarProfile.display_name
+    stmt = (
+        select(
+            Publication.id,
+            ScholarProfile.id,
+            scholar_label,
+            ScholarProfile.scholar_id,
+            Publication.title_raw,
+            Publication.year,
+            Publication.citation_count,
+            Publication.venue_text,
+            Publication.pub_url,
+            Publication.pdf_url,
+            ScholarPublication.is_read,
+            ScholarPublication.is_favorite,
+            ScholarPublication.first_seen_run_id,
+            ScholarPublication.created_at,
+        )
+        .join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
+        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
+        .where(ScholarProfile.user_id == user_id)
+        .order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
+        .offset(max(int(offset), 0))
+        .limit(limit)
+    )
+    if scholar_profile_id is not None:
+        stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
+    if favorite_only:
+        stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
+    if mode == MODE_UNREAD:
+        stmt = stmt.where(ScholarPublication.is_read.is_(False))
+    if mode == MODE_LATEST:
+        if latest_run_id is None:
+            return stmt.where(False)
+        stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
+    return stmt
+
+
+def publication_query_for_user(
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+    publication_id: int,
+) -> Select[tuple]:
+    return (
+        select(
+            Publication.id,
+            ScholarProfile.id,
+            ScholarProfile.display_name,
+            ScholarProfile.scholar_id,
+            Publication.title_raw,
+            Publication.year,
+            Publication.citation_count,
+            Publication.venue_text,
+            Publication.pub_url,
+            Publication.pdf_url,
+            ScholarPublication.is_read,
+            ScholarPublication.is_favorite,
+            ScholarPublication.first_seen_run_id,
+            ScholarPublication.created_at,
+        )
+        .join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
+        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
+        .where(
+            ScholarProfile.user_id == user_id,
+            ScholarProfile.id == scholar_profile_id,
+            Publication.id == publication_id,
+        )
+        .limit(1)
+    )
+
+
+async def get_publication_item_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+    publication_id: int,
+) -> PublicationListItem | None:
+    latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
+    result = await db_session.execute(
+        publication_query_for_user(
+            user_id=user_id,
+            scholar_profile_id=scholar_profile_id,
+            publication_id=publication_id,
+        )
+    )
+    row = result.one_or_none()
+    if row is None:
+        return None
+    return publication_list_item_from_row(row, latest_run_id=latest_run_id)
+
+
+def publication_list_item_from_row(
+    row: tuple,
+    *,
+    latest_run_id: int | None,
+) -> PublicationListItem:
+    (
+        publication_id,
+        scholar_profile_id,
+        display_name,
+        scholar_id,
+        title_raw,
+        year,
+        citation_count,
+        venue_text,
+        pub_url,
+        pdf_url,
+        is_read,
+        is_favorite,
+        first_seen_run_id,
+        created_at,
+    ) = row
+    return PublicationListItem(
+        publication_id=int(publication_id),
+        scholar_profile_id=int(scholar_profile_id),
+        scholar_label=(display_name or scholar_id),
+        title=title_raw,
+        year=year,
+        citation_count=_normalized_citation_count(citation_count),
+        venue_text=venue_text,
+        pub_url=pub_url,
+        pdf_url=pdf_url,
+        is_read=bool(is_read),
+        is_favorite=bool(is_favorite),
+        first_seen_at=created_at,
+        is_new_in_latest_run=(
+            latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
+        ),
+    )
+
+
+def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
+    (
+        publication_id,
+        scholar_profile_id,
+        display_name,
+        scholar_id,
+        title_raw,
+        year,
+        citation_count,
+        venue_text,
+        pub_url,
+        pdf_url,
+        _is_read,
+        _is_favorite,
+        _first_seen_run_id,
+        _created_at,
+    ) = row
+    return UnreadPublicationItem(
+        publication_id=int(publication_id),
+        scholar_profile_id=int(scholar_profile_id),
+        scholar_label=(display_name or scholar_id),
+        title=title_raw,
+        year=year,
+        citation_count=_normalized_citation_count(citation_count),
+        venue_text=venue_text,
+        pub_url=pub_url,
+        pdf_url=pdf_url,
+    )

build/lib/app/services/domains/publications/read_state.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from sqlalchemy import select, tuple_, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import ScholarProfile, ScholarPublication
+
+
+def _normalized_selection_pairs(selections: list[tuple[int, int]]) -> set[tuple[int, int]]:
+    pairs: set[tuple[int, int]] = set()
+    for scholar_profile_id, publication_id in selections:
+        normalized = (int(scholar_profile_id), int(publication_id))
+        if normalized[0] <= 0 or normalized[1] <= 0:
+            continue
+        pairs.add(normalized)
+    return pairs
+
+
+def _scoped_scholar_ids_query(*, user_id: int):
+    return (
+        select(ScholarProfile.id)
+        .where(ScholarProfile.user_id == user_id)
+        .scalar_subquery()
+    )
+
+
+async def mark_all_unread_as_read_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> int:
+    scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
+    stmt = (
+        update(ScholarPublication)
+        .where(
+            ScholarPublication.scholar_profile_id.in_(scholar_ids),
+            ScholarPublication.is_read.is_(False),
+        )
+        .values(is_read=True)
+    )
+    result = await db_session.execute(stmt)
+    await db_session.commit()
+    return int(result.rowcount or 0)
+
+
+async def mark_selected_as_read_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    selections: list[tuple[int, int]],
+) -> int:
+    normalized_pairs = _normalized_selection_pairs(selections)
+    if not normalized_pairs:
+        return 0
+
+    scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
+    stmt = (
+        update(ScholarPublication)
+        .where(
+            ScholarPublication.scholar_profile_id.in_(scholar_ids),
+            tuple_(
+                ScholarPublication.scholar_profile_id,
+                ScholarPublication.publication_id,
+            ).in_(list(normalized_pairs)),
+            ScholarPublication.is_read.is_(False),
+        )
+        .values(is_read=True)
+    )
+    result = await db_session.execute(stmt)
+    await db_session.commit()
+    return int(result.rowcount or 0)
+
+
+async def set_publication_favorite_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+    publication_id: int,
+    is_favorite: bool,
+) -> int:
+    scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
+    stmt = (
+        update(ScholarPublication)
+        .where(
+            ScholarPublication.scholar_profile_id.in_(scholar_ids),
+            ScholarPublication.scholar_profile_id == int(scholar_profile_id),
+            ScholarPublication.publication_id == int(publication_id),
+        )
+        .values(is_favorite=bool(is_favorite))
+    )
+    result = await db_session.execute(stmt)
+    await db_session.commit()
+    return int(result.rowcount or 0)

build/lib/app/services/domains/publications/types.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+
+from app.services.domains.publication_identifiers.types import DisplayIdentifier
+
+
+@dataclass(frozen=True)
+class PublicationListItem:
+    publication_id: int
+    scholar_profile_id: int
+    scholar_label: str
+    title: str
+    year: int | None
+    citation_count: int
+    venue_text: str | None
+    pub_url: str | None
+    pdf_url: str | None
+    is_read: bool
+    first_seen_at: datetime
+    is_new_in_latest_run: bool
+    is_favorite: bool = False
+    pdf_status: str = "untracked"
+    pdf_attempt_count: int = 0
+    pdf_failure_reason: str | None = None
+    pdf_failure_detail: str | None = None
+    display_identifier: DisplayIdentifier | None = None
+
+
+@dataclass(frozen=True)
+class UnreadPublicationItem:
+    publication_id: int
+    scholar_profile_id: int
+    scholar_label: str
+    title: str
+    year: int | None
+    citation_count: int
+    venue_text: str | None
+    pub_url: str | None
+    pdf_url: str | None

build/lib/app/services/domains/runs/__init__.py

@@ -0,0 +1 @@
+from app.services.domains.runs.application import *

build/lib/app/services/domains/runs/application.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from app.services.domains.runs.queue_service import (
+    clear_queue_item_for_user,
+    drop_queue_item_for_user,
+    get_queue_item_for_user,
+    list_queue_items_for_user,
+    queue_status_counts_for_user,
+    retry_queue_item_for_user,
+)
+from app.services.domains.runs.runs_service import (
+    get_manual_run_by_idempotency_key,
+    get_run_for_user,
+    list_recent_runs_for_user,
+    list_runs_for_user,
+)
+from app.services.domains.runs.summary import extract_run_summary
+from app.services.domains.runs.types import (
+    QUEUE_STATUS_DROPPED,
+    QUEUE_STATUS_QUEUED,
+    QUEUE_STATUS_RETRYING,
+    QueueClearResult,
+    QueueListItem,
+    QueueTransitionError,
+)
+
+__all__ = [
+    "QUEUE_STATUS_QUEUED",
+    "QUEUE_STATUS_RETRYING",
+    "QUEUE_STATUS_DROPPED",
+    "QueueListItem",
+    "QueueClearResult",
+    "QueueTransitionError",
+    "extract_run_summary",
+    "list_recent_runs_for_user",
+    "list_runs_for_user",
+    "get_run_for_user",
+    "get_manual_run_by_idempotency_key",
+    "list_queue_items_for_user",
+    "get_queue_item_for_user",
+    "retry_queue_item_for_user",
+    "drop_queue_item_for_user",
+    "clear_queue_item_for_user",
+    "queue_status_counts_for_user",
+]

build/lib/app/services/domains/runs/events.py

@@ -0,0 +1,59 @@
+import asyncio
+import json
+import logging
+from typing import Any, AsyncGenerator, Dict, Set
+
+logger = logging.getLogger(__name__)
+
+class RunEventPublisher:
+    def __init__(self) -> None:
+        # Maps run_id to a set of subscriber queues
+        self._subscribers: Dict[int, Set[asyncio.Queue]] = {}
+
+    def subscribe(self, run_id: int) -> asyncio.Queue:
+        if run_id not in self._subscribers:
+            self._subscribers[run_id] = set()
+        queue: asyncio.Queue[Any] = asyncio.Queue()
+        self._subscribers[run_id].add(queue)
+        logger.debug(f"New subscriber for run {run_id}. Total: {len(self._subscribers[run_id])}")
+        return queue
+
+    def unsubscribe(self, run_id: int, queue: asyncio.Queue) -> None:
+        if run_id in self._subscribers:
+            self._subscribers[run_id].discard(queue)
+            if not self._subscribers[run_id]:
+                self._subscribers.pop(run_id, None)
+
+    async def publish(self, run_id: int, event_type: str, data: dict[str, Any]) -> None:
+        if run_id not in self._subscribers:
+            return
+        
+        message = {
+            "type": event_type,
+            "data": data
+        }
+        
+        # Fan-out to all active subscribers for this run
+        for queue in list(self._subscribers[run_id]):
+            try:
+                queue.put_nowait(message)
+            except asyncio.QueueFull:
+                logger.warning(f"Subscriber queue full for run {run_id}, dropping message")
+
+run_events = RunEventPublisher()
+
+async def event_generator(run_id: int) -> AsyncGenerator[str, None]:
+    queue = run_events.subscribe(run_id)
+    try:
+        while True:
+            # Wait for a new event
+            message = await queue.get()
+            # Server-Sent Events format: "event: <type>\ndata: <json>\n\n"
+            event_type = message["type"]
+            data_str = json.dumps(message["data"])
+            yield f"event: {event_type}\ndata: {data_str}\n\n"
+    except asyncio.CancelledError:
+        logger.debug(f"Client disconnected from SSE stream for run {run_id}")
+        raise
+    finally:
+        run_events.unsubscribe(run_id, queue)

build/lib/app/services/domains/runs/queue_queries.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from sqlalchemy import and_, select
+
+from app.db.models import IngestionQueueItem, ScholarProfile
+from app.services.domains.runs.types import QueueListItem
+
+
+def queue_item_columns() -> tuple:
+    return (
+        IngestionQueueItem.id,
+        IngestionQueueItem.scholar_profile_id,
+        ScholarProfile.display_name,
+        ScholarProfile.scholar_id,
+        IngestionQueueItem.status,
+        IngestionQueueItem.reason,
+        IngestionQueueItem.dropped_reason,
+        IngestionQueueItem.attempt_count,
+        IngestionQueueItem.resume_cstart,
+        IngestionQueueItem.next_attempt_dt,
+        IngestionQueueItem.updated_at,
+        IngestionQueueItem.last_error,
+        IngestionQueueItem.last_run_id,
+    )
+
+
+def queue_item_select(*, user_id: int):
+    return (
+        select(*queue_item_columns())
+        .join(
+            ScholarProfile,
+            and_(
+                ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
+                ScholarProfile.user_id == IngestionQueueItem.user_id,
+            ),
+        )
+        .where(IngestionQueueItem.user_id == user_id)
+    )
+
+
+def queue_list_item_from_row(row: tuple) -> QueueListItem:
+    (
+        item_id,
+        scholar_profile_id,
+        display_name,
+        scholar_id,
+        status,
+        reason,
+        dropped_reason,
+        attempt_count,
+        resume_cstart,
+        next_attempt_dt,
+        updated_at,
+        last_error,
+        last_run_id,
+    ) = row
+    return QueueListItem(
+        id=int(item_id),
+        scholar_profile_id=int(scholar_profile_id),
+        scholar_label=(display_name or scholar_id),
+        status=str(status),
+        reason=str(reason),
+        dropped_reason=dropped_reason,
+        attempt_count=int(attempt_count or 0),
+        resume_cstart=int(resume_cstart or 0),
+        next_attempt_dt=next_attempt_dt,
+        updated_at=updated_at,
+        last_error=last_error,
+        last_run_id=int(last_run_id) if last_run_id is not None else None,
+    )

build/lib/app/services/domains/runs/queue_service.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+from sqlalchemy import case, func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import IngestionQueueItem
+from app.services.domains.ingestion import queue as queue_mutations
+from app.services.domains.runs.queue_queries import queue_item_select, queue_list_item_from_row
+from app.services.domains.runs.types import (
+    QUEUE_STATUS_DROPPED,
+    QUEUE_STATUS_QUEUED,
+    QUEUE_STATUS_RETRYING,
+    QueueClearResult,
+    QueueListItem,
+    QueueTransitionError,
+)
+
+
+async def list_queue_items_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    limit: int = 200,
+) -> list[QueueListItem]:
+    result = await db_session.execute(
+        queue_item_select(user_id=user_id)
+        .order_by(
+            case((IngestionQueueItem.status == QUEUE_STATUS_DROPPED, 1), else_=0).asc(),
+            IngestionQueueItem.next_attempt_dt.asc(),
+            IngestionQueueItem.id.asc(),
+        )
+        .limit(limit)
+    )
+    return [queue_list_item_from_row(row) for row in result.all()]
+
+
+async def get_queue_item_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    queue_item_id: int,
+) -> QueueListItem | None:
+    result = await db_session.execute(
+        queue_item_select(user_id=user_id)
+        .where(IngestionQueueItem.id == queue_item_id)
+        .limit(1)
+    )
+    row = result.one_or_none()
+    if row is None:
+        return None
+    return queue_list_item_from_row(row)
+
+
+async def retry_queue_item_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    queue_item_id: int,
+) -> QueueListItem | None:
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(
+            IngestionQueueItem.id == queue_item_id,
+            IngestionQueueItem.user_id == user_id,
+        )
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    if item.status == QUEUE_STATUS_QUEUED:
+        raise QueueTransitionError(
+            code="queue_item_already_queued",
+            message="Queue item is already queued.",
+            current_status=item.status,
+        )
+    if item.status == QUEUE_STATUS_RETRYING:
+        raise QueueTransitionError(
+            code="queue_item_retrying",
+            message="Queue item is currently retrying.",
+            current_status=item.status,
+        )
+
+    await queue_mutations.mark_queued_now(
+        db_session,
+        job_id=item.id,
+        reason="manual_retry",
+        reset_attempt_count=(item.status == QUEUE_STATUS_DROPPED),
+    )
+    await db_session.commit()
+    return await get_queue_item_for_user(
+        db_session,
+        user_id=user_id,
+        queue_item_id=queue_item_id,
+    )
+
+
+async def drop_queue_item_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    queue_item_id: int,
+) -> QueueListItem | None:
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(
+            IngestionQueueItem.id == queue_item_id,
+            IngestionQueueItem.user_id == user_id,
+        )
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    if item.status == QUEUE_STATUS_DROPPED:
+        raise QueueTransitionError(
+            code="queue_item_already_dropped",
+            message="Queue item is already dropped.",
+            current_status=item.status,
+        )
+
+    await queue_mutations.mark_dropped(
+        db_session,
+        job_id=int(item.id),
+        reason="manual_drop",
+    )
+    await db_session.commit()
+    return await get_queue_item_for_user(
+        db_session,
+        user_id=user_id,
+        queue_item_id=queue_item_id,
+    )
+
+
+async def clear_queue_item_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    queue_item_id: int,
+) -> QueueClearResult | None:
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(
+            IngestionQueueItem.id == queue_item_id,
+            IngestionQueueItem.user_id == user_id,
+        )
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    if item.status != QUEUE_STATUS_DROPPED:
+        raise QueueTransitionError(
+            code="queue_item_not_dropped",
+            message="Queue item can only be cleared after it is dropped.",
+            current_status=item.status,
+        )
+
+    item_id = int(item.id)
+    previous_status = str(item.status)
+    deleted = await queue_mutations.delete_job_by_id(db_session, job_id=item_id)
+    await db_session.commit()
+    if not deleted:
+        return None
+    return QueueClearResult(queue_item_id=item_id, previous_status=previous_status)
+
+
+async def queue_status_counts_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> dict[str, int]:
+    result = await db_session.execute(
+        select(
+            IngestionQueueItem.status,
+            func.count(IngestionQueueItem.id),
+        )
+        .where(IngestionQueueItem.user_id == user_id)
+        .group_by(IngestionQueueItem.status)
+    )
+    counts: dict[str, int] = {
+        QUEUE_STATUS_QUEUED: 0,
+        QUEUE_STATUS_RETRYING: 0,
+        QUEUE_STATUS_DROPPED: 0,
+    }
+    for status, count in result.all():
+        counts[str(status)] = int(count or 0)
+    return counts

build/lib/app/services/domains/runs/runs_service.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from sqlalchemy import or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import CrawlRun, RunStatus, RunTriggerType
+
+
+async def list_recent_runs_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    limit: int = 20,
+) -> list[CrawlRun]:
+    result = await db_session.execute(
+        select(CrawlRun)
+        .where(CrawlRun.user_id == user_id)
+        .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
+        .limit(limit)
+    )
+    return list(result.scalars().all())
+
+
+async def list_runs_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    limit: int = 100,
+    failed_only: bool = False,
+) -> list[CrawlRun]:
+    stmt = (
+        select(CrawlRun)
+        .where(CrawlRun.user_id == user_id)
+        .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
+        .limit(limit)
+    )
+    if failed_only:
+        stmt = stmt.where(
+            CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE])
+        )
+    result = await db_session.execute(stmt)
+    return list(result.scalars().all())
+
+
+async def get_run_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    run_id: int,
+) -> CrawlRun | None:
+    result = await db_session.execute(
+        select(CrawlRun).where(
+            CrawlRun.user_id == user_id,
+            CrawlRun.id == run_id,
+        )
+    )
+    return result.scalar_one_or_none()
+
+
+async def get_manual_run_by_idempotency_key(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    idempotency_key: str,
+) -> CrawlRun | None:
+    result = await db_session.execute(
+        select(CrawlRun)
+        .where(
+            CrawlRun.user_id == user_id,
+            CrawlRun.trigger_type == RunTriggerType.MANUAL,
+            or_(
+                CrawlRun.idempotency_key == idempotency_key,
+                CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key,
+            ),
+        )
+        .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
+        .limit(1)
+    )
+    return result.scalar_one_or_none()

build/lib/app/services/domains/runs/summary.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+def _safe_int(value: object, default: int = 0) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _summary_dict(error_log: object) -> dict[str, Any]:
+    if not isinstance(error_log, dict):
+        return {}
+    summary = error_log.get("summary")
+    if not isinstance(summary, dict):
+        return {}
+    return summary
+
+
+def _summary_int_dict(summary: dict[str, Any], key: str) -> dict[str, int]:
+    value = summary.get(key)
+    if not isinstance(value, dict):
+        return {}
+    return {
+        str(item_key): _safe_int(item_value, 0)
+        for item_key, item_value in value.items()
+        if isinstance(item_key, str)
+    }
+
+
+def _summary_bool_dict(summary: dict[str, Any], key: str) -> dict[str, bool]:
+    value = summary.get(key)
+    if not isinstance(value, dict):
+        return {}
+    return {
+        str(item_key): bool(item_value)
+        for item_key, item_value in value.items()
+        if isinstance(item_key, str)
+    }
+
+
+def _retry_counts(summary: dict[str, Any]) -> dict[str, int]:
+    retry_counts = summary.get("retry_counts")
+    if not isinstance(retry_counts, dict):
+        retry_counts = {}
+    return {
+        "retries_scheduled_count": _safe_int(
+            retry_counts.get("retries_scheduled_count", 0),
+            0,
+        ),
+        "scholars_with_retries_count": _safe_int(
+            retry_counts.get("scholars_with_retries_count", 0),
+            0,
+        ),
+        "retry_exhausted_count": _safe_int(
+            retry_counts.get("retry_exhausted_count", 0),
+            0,
+        ),
+    }
+
+
+def extract_run_summary(error_log: object) -> dict[str, Any]:
+    summary = _summary_dict(error_log)
+    return {
+        "succeeded_count": _safe_int(summary.get("succeeded_count", 0)),
+        "failed_count": _safe_int(summary.get("failed_count", 0)),
+        "partial_count": _safe_int(summary.get("partial_count", 0)),
+        "failed_state_counts": _summary_int_dict(summary, "failed_state_counts"),
+        "failed_reason_counts": _summary_int_dict(summary, "failed_reason_counts"),
+        "scrape_failure_counts": _summary_int_dict(summary, "scrape_failure_counts"),
+        "retry_counts": _retry_counts(summary),
+        "alert_thresholds": _summary_int_dict(summary, "alert_thresholds"),
+        "alert_flags": _summary_bool_dict(summary, "alert_flags"),
+    }

build/lib/app/services/domains/runs/types.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+
+QUEUE_STATUS_QUEUED = "queued"
+QUEUE_STATUS_RETRYING = "retrying"
+QUEUE_STATUS_DROPPED = "dropped"
+
+
+@dataclass(frozen=True)
+class QueueListItem:
+    id: int
+    scholar_profile_id: int
+    scholar_label: str
+    status: str
+    reason: str
+    dropped_reason: str | None
+    attempt_count: int
+    resume_cstart: int
+    next_attempt_dt: datetime | None
+    updated_at: datetime
+    last_error: str | None
+    last_run_id: int | None
+
+
+@dataclass(frozen=True)
+class QueueClearResult:
+    queue_item_id: int
+    previous_status: str
+
+
+class QueueTransitionError(RuntimeError):
+    def __init__(self, *, code: str, message: str, current_status: str) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+        self.current_status = current_status

build/lib/app/services/domains/scholar/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

build/lib/app/services/domains/scholar/author_rows.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import re
+from html.parser import HTMLParser
+from typing import Any
+from urllib.parse import parse_qs, urlparse
+
+from app.services.domains.scholar.parser_constants import AUTHOR_SEARCH_MARKER_KEYS
+from app.services.domains.scholar.parser_types import ScholarSearchCandidate
+from app.services.domains.scholar.parser_utils import (
+    attr_class,
+    attr_href,
+    attr_src,
+    build_absolute_scholar_url,
+    normalize_space,
+)
+
+
+def parse_scholar_id_from_href(href: str | None) -> str | None:
+    if not href:
+        return None
+    parsed = urlparse(href)
+    query = parse_qs(parsed.query)
+    user_values = query.get("user")
+    if not user_values:
+        return None
+    candidate = user_values[0].strip()
+    return candidate or None
+
+
+def _extract_verified_email_domain(value: str | None) -> str | None:
+    if not value:
+        return None
+    match = re.search(r"verified email at\s+(.+)$", value.strip(), re.I)
+    if not match:
+        return None
+    domain = normalize_space(match.group(1))
+    return domain or None
+
+
+class ScholarAuthorSearchParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.candidates: list[ScholarSearchCandidate] = []
+        self._candidate: dict[str, Any] | None = None
+
+    def _begin_candidate(self) -> None:
+        self._candidate = {
+            "depth": 1,
+            "name_href": None,
+            "name_parts": [],
+            "aff_depth": 0,
+            "aff_parts": [],
+            "name_depth": 0,
+            "eml_depth": 0,
+            "eml_parts": [],
+            "cby_depth": 0,
+            "cby_parts": [],
+            "interest_depth": 0,
+            "interest_parts": [],
+            "interests": [],
+            "image_src": None,
+        }
+
+    def _increment_capture_depths(self) -> None:
+        if self._candidate is None:
+            return
+        for key in ("name_depth", "aff_depth", "eml_depth", "cby_depth", "interest_depth"):
+            if self._candidate[key] > 0:
+                self._candidate[key] += 1
+
+    def _finalize_candidate(self) -> None:
+        if self._candidate is None:
+            return
+
+        name = normalize_space("".join(self._candidate["name_parts"]))
+        scholar_id = parse_scholar_id_from_href(self._candidate["name_href"])
+        if not name or not scholar_id:
+            return
+
+        affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None
+        email_domain = _extract_verified_email_domain(
+            normalize_space("".join(self._candidate["eml_parts"])) or None
+        )
+        cited_by_text = normalize_space("".join(self._candidate["cby_parts"]))
+        cited_by_match = re.search(r"\d+", cited_by_text)
+        cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None
+
+        seen_interests: set[str] = set()
+        interests: list[str] = []
+        for interest in self._candidate["interests"]:
+            normalized = normalize_space(interest)
+            if not normalized or normalized in seen_interests:
+                continue
+            seen_interests.add(normalized)
+            interests.append(normalized)
+
+        profile_url = build_absolute_scholar_url(self._candidate["name_href"])
+        if not profile_url:
+            profile_url = (
+                "https://scholar.google.com/citations"
+                f"?hl=en&user={scholar_id}"
+            )
+
+        self.candidates.append(
+            ScholarSearchCandidate(
+                scholar_id=scholar_id,
+                display_name=name,
+                affiliation=affiliation,
+                email_domain=email_domain,
+                cited_by_count=cited_by_count,
+                interests=interests,
+                profile_url=profile_url,
+                profile_image_url=build_absolute_scholar_url(self._candidate["image_src"]),
+            )
+        )
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        classes = attr_class(attrs)
+
+        if self._candidate is None:
+            if tag == "div" and "gsc_1usr" in classes:
+                self._begin_candidate()
+            return
+
+        self._candidate["depth"] += 1
+        self._increment_capture_depths()
+
+        if tag == "a" and "gs_ai_name" in classes:
+            self._candidate["name_depth"] = 1
+            self._candidate["name_href"] = attr_href(attrs)
+            return
+
+        if tag == "div" and "gs_ai_aff" in classes:
+            self._candidate["aff_depth"] = 1
+            return
+
+        if tag == "div" and "gs_ai_eml" in classes:
+            self._candidate["eml_depth"] = 1
+            return
+
+        if tag == "div" and "gs_ai_cby" in classes:
+            self._candidate["cby_depth"] = 1
+            return
+
+        if tag == "a" and "gs_ai_one_int" in classes:
+            self._candidate["interest_depth"] = 1
+            self._candidate["interest_parts"] = []
+            return
+
+        if tag == "img" and self._candidate["image_src"] is None:
+            self._candidate["image_src"] = attr_src(attrs)
+
+    def handle_data(self, data: str) -> None:
+        if self._candidate is None:
+            return
+        if self._candidate["name_depth"] > 0:
+            self._candidate["name_parts"].append(data)
+        if self._candidate["aff_depth"] > 0:
+            self._candidate["aff_parts"].append(data)
+        if self._candidate["eml_depth"] > 0:
+            self._candidate["eml_parts"].append(data)
+        if self._candidate["cby_depth"] > 0:
+            self._candidate["cby_parts"].append(data)
+        if self._candidate["interest_depth"] > 0:
+            self._candidate["interest_parts"].append(data)
+
+    def _decrement_capture_depth(self, key: str) -> bool:
+        if self._candidate is None:
+            return False
+        if self._candidate[key] <= 0:
+            return False
+        self._candidate[key] -= 1
+        return self._candidate[key] == 0
+
+    def handle_endtag(self, _tag: str) -> None:
+        if self._candidate is None:
+            return
+
+        interest_closed = self._decrement_capture_depth("interest_depth")
+        self._decrement_capture_depth("name_depth")
+        self._decrement_capture_depth("aff_depth")
+        self._decrement_capture_depth("eml_depth")
+        self._decrement_capture_depth("cby_depth")
+
+        if interest_closed:
+            interest_text = normalize_space("".join(self._candidate["interest_parts"]))
+            if interest_text:
+                self._candidate["interests"].append(interest_text)
+            self._candidate["interest_parts"] = []
+
+        self._candidate["depth"] -= 1
+        if self._candidate["depth"] > 0:
+            return
+
+        self._finalize_candidate()
+        self._candidate = None
+
+
+def count_author_search_markers(html: str) -> dict[str, int]:
+    lowered = html.lower()
+    return {key: lowered.count(key.lower()) for key in AUTHOR_SEARCH_MARKER_KEYS}

build/lib/app/services/domains/scholar/parser.py

@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+from app.services.domains.scholar.author_rows import (
+    ScholarAuthorSearchParser,
+    count_author_search_markers,
+    parse_scholar_id_from_href,
+)
+from app.services.domains.scholar.parser_constants import SCRIPT_STYLE_RE
+from app.services.domains.scholar.parser_types import (
+    ParseState,
+    ParsedAuthorSearchPage,
+    ParsedProfilePage,
+    PublicationCandidate,
+    ScholarDomInvariantError,
+    ScholarMalformedDataError,
+    ScholarParserError,
+    ScholarSearchCandidate,
+)
+from app.services.domains.scholar.parser_utils import (
+    strip_tags,
+)
+from app.services.domains.scholar.profile_rows import (
+    ScholarRowParser,
+    count_markers,
+    extract_articles_range,
+    extract_profile_image_url,
+    extract_profile_name,
+    extract_rows,
+    has_operation_error_banner,
+    has_show_more_button,
+    parse_citation_count,
+    parse_cluster_id_from_href,
+    parse_publications,
+    parse_year,
+)
+from app.services.domains.scholar.source import FetchResult
+from app.services.domains.scholar.state_detection import (
+    classify_block_or_captcha_reason,
+    classify_network_error_reason,
+    detect_author_search_state,
+    detect_state,
+)
+
+
+def _raise_dom_error(code: str, message: str) -> None:
+    raise ScholarDomInvariantError(code=code, message=message)
+
+
+def _assert_profile_dom_invariants(
+    *,
+    fetch_result: FetchResult,
+    marker_counts: dict[str, int],
+    publications: list[PublicationCandidate],
+    warnings: list[str],
+    has_show_more_button_flag: bool,
+    articles_range: str | None,
+) -> None:
+    if fetch_result.status_code is None:
+        return
+    final_url = (fetch_result.final_url or "").lower()
+    if "accounts.google.com" in final_url or "sorry/index" in final_url:
+        return
+    if any(code.startswith("layout_") for code in warnings):
+        reason = next(code for code in warnings if code.startswith("layout_"))
+        _raise_dom_error(reason, f"Detected layout warning: {reason}")
+    if has_show_more_button_flag and not articles_range:
+        _raise_dom_error(
+            "layout_show_more_without_articles_range",
+            "Show-more control exists without an articles range marker.",
+        )
+    if marker_counts.get("gsc_a_tr", 0) > 0 and marker_counts.get("gsc_a_at", 0) <= 0:
+        _raise_dom_error(
+            "layout_missing_publication_title_anchor",
+            "Publication rows were present but title anchors were absent.",
+        )
+    if not publications:
+        has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
+        has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
+        if not has_profile_markers and not has_table_markers:
+            _raise_dom_error("layout_markers_missing", "Expected scholar profile markers were absent.")
+    for publication in publications:
+        if not publication.title.strip():
+            raise ScholarMalformedDataError(
+                code="malformed_publication_title",
+                message="Encountered a publication candidate with an empty title.",
+            )
+        if publication.citation_count is not None and int(publication.citation_count) < 0:
+            raise ScholarMalformedDataError(
+                code="malformed_publication_negative_citations",
+                message="Encountered a publication candidate with negative citations.",
+            )
+
+
+def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
+    publications, warnings = parse_publications(fetch_result.body)
+    marker_counts = count_markers(fetch_result.body)
+    visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
+
+    show_more = has_show_more_button(fetch_result.body)
+    operation_error_banner = has_operation_error_banner(fetch_result.body)
+    articles_range = extract_articles_range(fetch_result.body)
+
+    if show_more:
+        warnings.append("possible_partial_page_show_more_present")
+    if operation_error_banner:
+        warnings.append("operation_error_banner_present")
+
+    warnings = sorted(set(warnings))
+    _assert_profile_dom_invariants(
+        fetch_result=fetch_result,
+        marker_counts=marker_counts,
+        publications=publications,
+        warnings=warnings,
+        has_show_more_button_flag=show_more,
+        articles_range=articles_range,
+    )
+
+    state, state_reason = detect_state(
+        fetch_result,
+        publications,
+        marker_counts,
+        warnings=warnings,
+        has_show_more_button_flag=show_more,
+        articles_range=articles_range,
+        visible_text=visible_text,
+    )
+
+    return ParsedProfilePage(
+        state=state,
+        state_reason=state_reason,
+        profile_name=extract_profile_name(fetch_result.body),
+        profile_image_url=extract_profile_image_url(fetch_result.body),
+        publications=publications,
+        marker_counts=marker_counts,
+        warnings=warnings,
+        has_show_more_button=show_more,
+        has_operation_error_banner=operation_error_banner,
+        articles_range=articles_range,
+    )
+
+
+def parse_author_search_page(fetch_result: FetchResult) -> ParsedAuthorSearchPage:
+    parser = ScholarAuthorSearchParser()
+    parser.feed(fetch_result.body)
+
+    marker_counts = count_author_search_markers(fetch_result.body)
+    visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
+    warnings: list[str] = []
+    if not parser.candidates:
+        warnings.append("no_author_candidates_detected")
+
+    state, state_reason = detect_author_search_state(
+        fetch_result,
+        parser.candidates,
+        marker_counts,
+        visible_text=visible_text,
+    )
+
+    return ParsedAuthorSearchPage(
+        state=state,
+        state_reason=state_reason,
+        candidates=parser.candidates,
+        marker_counts=marker_counts,
+        warnings=warnings,
+    )

build/lib/app/services/domains/scholar/parser_constants.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import re
+
+BLOCKED_KEYWORDS = [
+    "unusual traffic",
+    "sorry/index",
+    "not a robot",
+    "our systems have detected",
+    "automated queries",
+    "recaptcha",
+    "captcha",
+]
+
+NO_RESULTS_KEYWORDS = [
+    "didn't match any articles",
+    "did not match any articles",
+    "no articles",
+    "no documents",
+]
+
+NO_AUTHOR_RESULTS_KEYWORDS = [
+    "didn't match any user profiles",
+    "did not match any user profiles",
+    "didn't match any scholars",
+    "did not match any scholars",
+    "no user profiles",
+]
+
+MARKER_KEYS = [
+    "gsc_a_tr",
+    "gsc_a_at",
+    "gsc_a_ac",
+    "gsc_a_h",
+    "gsc_a_y",
+    "gs_gray",
+    "gsc_prf_in",
+    "gsc_rsb_st",
+]
+
+AUTHOR_SEARCH_MARKER_KEYS = [
+    "gsc_1usr",
+    "gs_ai_name",
+    "gs_ai_aff",
+    "gs_ai_eml",
+    "gs_ai_cby",
+    "gs_ai_one_int",
+]
+
+NETWORK_DNS_ERROR_KEYWORDS = [
+    "temporary failure in name resolution",
+    "name or service not known",
+    "nodename nor servname provided",
+    "getaddrinfo failed",
+]
+
+NETWORK_TIMEOUT_KEYWORDS = [
+    "timed out",
+    "timeout",
+]
+
+NETWORK_TLS_ERROR_KEYWORDS = [
+    "ssl",
+    "tls",
+    "certificate verify failed",
+]
+
+TAG_RE = re.compile(r"<[^>]+>", re.S)
+SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.I | re.S)
+SHOW_MORE_BUTTON_RE = re.compile(
+    r"<button\b[^>]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>",
+    re.I | re.S,
+)
+
+PROFILE_ROW_PARSER_DIRECT_MARKERS = (
+    "gs_ggs",
+    "gs_ggsd",
+    "gs_ggsa",
+    "gs_or_ggsm",
+)
+
+PROFILE_ROW_DIRECT_LABEL_TOKENS = (
+    "pdf",
+    "[pdf]",
+    "full text",
+    "download",
+)

build/lib/app/services/domains/scholar/parser_types.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+
+
+class ParseState(StrEnum):
+    OK = "ok"
+    NO_RESULTS = "no_results"
+    BLOCKED_OR_CAPTCHA = "blocked_or_captcha"
+    LAYOUT_CHANGED = "layout_changed"
+    NETWORK_ERROR = "network_error"
+
+
+class ScholarParserError(RuntimeError):
+    code: str
+
+    def __init__(self, *, code: str, message: str) -> None:
+        super().__init__(message)
+        self.code = code
+
+
+class ScholarDomInvariantError(ScholarParserError):
+    pass
+
+
+class ScholarMalformedDataError(ScholarParserError):
+    pass
+
+
+@dataclass(frozen=True)
+class PublicationCandidate:
+    title: str
+    title_url: str | None
+    cluster_id: str | None
+    year: int | None
+    citation_count: int | None
+    authors_text: str | None
+    venue_text: str | None
+    pdf_url: str | None
+
+
+@dataclass(frozen=True)
+class ScholarSearchCandidate:
+    scholar_id: str
+    display_name: str
+    affiliation: str | None
+    email_domain: str | None
+    cited_by_count: int | None
+    interests: list[str]
+    profile_url: str
+    profile_image_url: str | None
+
+
+@dataclass(frozen=True)
+class ParsedProfilePage:
+    state: ParseState
+    state_reason: str
+    profile_name: str | None
+    profile_image_url: str | None
+    publications: list[PublicationCandidate]
+    marker_counts: dict[str, int]
+    warnings: list[str]
+    has_show_more_button: bool
+    has_operation_error_banner: bool
+    articles_range: str | None
+
+
+@dataclass(frozen=True)
+class ParsedAuthorSearchPage:
+    state: ParseState
+    state_reason: str
+    candidates: list[ScholarSearchCandidate]
+    marker_counts: dict[str, int]
+    warnings: list[str]

build/lib/app/services/domains/scholar/parser_utils.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from html import unescape
+
+from app.services.domains.scholar.parser_constants import TAG_RE
+
+
+def normalize_space(value: str) -> str:
+    return " ".join(unescape(value).split())
+
+
+def strip_tags(value: str) -> str:
+    return normalize_space(TAG_RE.sub(" ", value))
+
+
+def attr_class(attrs: list[tuple[str, str | None]]) -> str:
+    for name, raw_value in attrs:
+        if name.lower() == "class":
+            return raw_value or ""
+    return ""
+
+
+def attr_href(attrs: list[tuple[str, str | None]]) -> str | None:
+    for name, raw_value in attrs:
+        if name.lower() == "href":
+            return raw_value
+    return None
+
+
+def attr_src(attrs: list[tuple[str, str | None]]) -> str | None:
+    for name, raw_value in attrs:
+        if name.lower() == "src":
+            return raw_value
+    return None
+
+
+def build_absolute_scholar_url(path_or_url: str | None) -> str | None:
+    if not path_or_url:
+        return None
+    from urllib.parse import urljoin
+
+    return urljoin("https://scholar.google.com", path_or_url)

build/lib/app/services/domains/scholar/profile_rows.py

@@ -0,0 +1,270 @@
+from __future__ import annotations
+
+import re
+from html.parser import HTMLParser
+from urllib.parse import parse_qs, urlparse
+
+from app.services.domains.scholar.parser_constants import (
+    MARKER_KEYS,
+    SHOW_MORE_BUTTON_RE,
+)
+from app.services.domains.scholar.parser_types import PublicationCandidate
+from app.services.domains.scholar.parser_utils import (
+    attr_class,
+    attr_href,
+    attr_src,
+    build_absolute_scholar_url,
+    normalize_space,
+    strip_tags,
+)
+
+
+class ScholarRowParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.title_href: str | None = None
+        self.title_parts: list[str] = []
+        self.citation_parts: list[str] = []
+        self.year_parts: list[str] = []
+        self.gray_texts: list[str] = []
+
+        self._title_depth = 0
+        self._citation_depth = 0
+        self._year_depth = 0
+        self._gray_stack: list[dict[str, object]] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if self._title_depth > 0:
+            self._title_depth += 1
+        if self._citation_depth > 0:
+            self._citation_depth += 1
+        if self._year_depth > 0:
+            self._year_depth += 1
+        if self._gray_stack:
+            self._gray_stack[-1]["depth"] += 1
+
+        classes = attr_class(attrs)
+
+        if tag == "a" and "gsc_a_at" in classes:
+            self._title_depth = 1
+            self.title_href = attr_href(attrs)
+            return
+
+        if tag == "a" and "gsc_a_ac" in classes:
+            self._citation_depth = 1
+            return
+
+        if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes):
+            self._year_depth = 1
+            return
+
+        if tag == "div" and "gs_gray" in classes:
+            self._gray_stack.append({"depth": 1, "parts": []})
+            return
+
+    def handle_data(self, data: str) -> None:
+        if self._title_depth > 0:
+            self.title_parts.append(data)
+        if self._citation_depth > 0:
+            self.citation_parts.append(data)
+        if self._year_depth > 0:
+            self.year_parts.append(data)
+        if self._gray_stack:
+            self._gray_stack[-1]["parts"].append(data)
+
+    def handle_endtag(self, tag: str) -> None:
+        if self._title_depth > 0:
+            self._title_depth -= 1
+        if self._citation_depth > 0:
+            self._citation_depth -= 1
+        if self._year_depth > 0:
+            self._year_depth -= 1
+        if self._gray_stack:
+            self._gray_stack[-1]["depth"] -= 1
+            if self._gray_stack[-1]["depth"] == 0:
+                text_value = normalize_space("".join(self._gray_stack[-1]["parts"]))
+                if text_value:
+                    self.gray_texts.append(text_value)
+                self._gray_stack.pop()
+
+
+def extract_rows(html: str) -> list[str]:
+    pattern = re.compile(
+        r"<tr\b(?=[^>]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)</tr>",
+        re.I | re.S,
+    )
+    return [match.group(1) for match in pattern.finditer(html)]
+
+
+def parse_cluster_id_from_href(href: str | None) -> str | None:
+    if not href:
+        return None
+    parsed = urlparse(href)
+    query = parse_qs(parsed.query)
+
+    citation_for_view = query.get("citation_for_view")
+    if citation_for_view:
+        token = citation_for_view[0].strip()
+        if token:
+            return f"cfv:{token}"
+
+    cluster = query.get("cluster")
+    if cluster:
+        token = cluster[0].strip()
+        if token:
+            return f"cluster:{token}"
+    return None
+
+
+def parse_year(parts: list[str]) -> int | None:
+    text = normalize_space(" ".join(parts))
+    match = re.search(r"\b(19|20)\d{2}\b", text)
+    if not match:
+        return None
+    try:
+        return int(match.group(0))
+    except ValueError:
+        return None
+
+
+def parse_citation_count(parts: list[str]) -> int | None:
+    text = normalize_space(" ".join(parts))
+    if not text:
+        return 0
+    digits = re.sub(r"\D+", "", text)
+    if not digits:
+        return None
+    return int(digits)
+
+
+def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None, list[str]]:
+    parser = ScholarRowParser()
+    parser.feed(row_html)
+    warnings: list[str] = []
+    title = normalize_space("".join(parser.title_parts))
+    if not title:
+        warnings.append("row_missing_title")
+        return None, warnings
+    if not parser.title_href:
+        warnings.append("row_missing_title_href")
+
+    citation_text = normalize_space(" ".join(parser.citation_parts))
+    citation_count = parse_citation_count(parser.citation_parts)
+    if citation_text and citation_count is None:
+        warnings.append("layout_row_citation_unparseable")
+
+    year_text = normalize_space(" ".join(parser.year_parts))
+    year = parse_year(parser.year_parts)
+    if year_text and year is None:
+        warnings.append("layout_row_year_unparseable")
+
+    authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
+    venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
+    return (
+        PublicationCandidate(
+            title=title,
+            title_url=parser.title_href,
+            cluster_id=parse_cluster_id_from_href(parser.title_href),
+            year=year,
+            citation_count=citation_count,
+            authors_text=authors_text,
+            venue_text=venue_text,
+            pdf_url=None,
+        ),
+        warnings,
+    )
+
+
+def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]:
+    rows = extract_rows(html)
+    warnings: list[str] = []
+    publications: list[PublicationCandidate] = []
+
+    for row_html in rows:
+        publication, row_warnings = _parse_publication_row(row_html)
+        warnings.extend(row_warnings)
+        if publication is None:
+            continue
+        publications.append(publication)
+
+    if not rows:
+        warnings.append("no_rows_detected")
+    if rows and not publications:
+        warnings.append("layout_all_rows_unparseable")
+
+    return publications, sorted(set(warnings))
+
+
+def extract_profile_name(html: str) -> str | None:
+    pattern = re.compile(
+        r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)</[^>]+>",
+        re.I | re.S,
+    )
+    match = pattern.search(html)
+    if not match:
+        return None
+    value = strip_tags(match.group(1))
+    return value or None
+
+
+def extract_profile_image_url(html: str) -> str | None:
+    og_image_pattern = re.compile(
+        r"<meta[^>]+property=['\"]og:image['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>",
+        re.I | re.S,
+    )
+    og_match = og_image_pattern.search(html)
+    if og_match:
+        value = normalize_space(og_match.group(1))
+        absolute = build_absolute_scholar_url(value)
+        if absolute:
+            return absolute
+
+    image_pattern = re.compile(
+        r"<img[^>]*\bid=['\"]gsc_prf_pup-img['\"][^>]*\bsrc=['\"]([^'\"]+)['\"][^>]*>",
+        re.I | re.S,
+    )
+    image_match = image_pattern.search(html)
+    if not image_match:
+        return None
+
+    value = normalize_space(image_match.group(1))
+    return build_absolute_scholar_url(value)
+
+
+def extract_articles_range(html: str) -> str | None:
+    pattern = re.compile(
+        r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)</[^>]+>",
+        re.I | re.S,
+    )
+    match = pattern.search(html)
+    if not match:
+        return None
+    value = strip_tags(match.group(1))
+    return value or None
+
+
+def has_show_more_button(html: str) -> bool:
+    match = SHOW_MORE_BUTTON_RE.search(html)
+    if match is None:
+        return False
+
+    button_tag = match.group(0).lower()
+    if "disabled" in button_tag:
+        return False
+    if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag:
+        return False
+    if "gs_dis" in button_tag:
+        return False
+    return True
+
+
+def has_operation_error_banner(html: str) -> bool:
+    lowered = html.lower()
+    if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered:
+        return False
+    return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered
+
+
+def count_markers(html: str) -> dict[str, int]:
+    lowered = html.lower()
+    return {key: lowered.count(key.lower()) for key in MARKER_KEYS}

build/lib/app/services/domains/scholar/rate_limit.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import asyncio
+import time
+
+_REQUEST_LOCK = asyncio.Lock()
+_LAST_REQUEST_AT = 0.0
+
+
+async def wait_for_scholar_slot(*, min_interval_seconds: float) -> None:
+    global _LAST_REQUEST_AT
+    interval = max(float(min_interval_seconds), 0.0)
+    async with _REQUEST_LOCK:
+        elapsed = time.monotonic() - _LAST_REQUEST_AT
+        remaining = interval - elapsed
+        if remaining > 0:
+            await asyncio.sleep(remaining)
+        _LAST_REQUEST_AT = time.monotonic()

build/lib/app/services/domains/scholar/source.py

@@ -0,0 +1,235 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import random
+from dataclasses import dataclass
+from typing import Protocol
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations"
+DEFAULT_PAGE_SIZE = 100
+
+DEFAULT_USER_AGENTS = [
+    (
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+    ),
+    (
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 "
+        "(KHTML, like Gecko) Version/18.1 Safari/605.1.15"
+    ),
+    (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) "
+        "Gecko/20100101 Firefox/131.0"
+    ),
+]
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class FetchResult:
+    requested_url: str
+    status_code: int | None
+    final_url: str | None
+    body: str
+    error: str | None
+
+
+class ScholarSource(Protocol):
+    async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
+        ...
+
+    async def fetch_profile_page_html(
+        self,
+        scholar_id: str,
+        *,
+        cstart: int,
+        pagesize: int,
+    ) -> FetchResult:
+        ...
+
+    async def fetch_author_search_html(
+        self,
+        query: str,
+        *,
+        start: int,
+    ) -> FetchResult:
+        ...
+
+
+class LiveScholarSource:
+    def __init__(
+        self,
+        *,
+        timeout_seconds: float = 25.0,
+        user_agents: list[str] | None = None,
+    ) -> None:
+        self._timeout_seconds = timeout_seconds
+        self._user_agents = user_agents or DEFAULT_USER_AGENTS
+
+    async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
+        return await self.fetch_profile_page_html(
+            scholar_id,
+            cstart=0,
+            pagesize=DEFAULT_PAGE_SIZE,
+        )
+
+    async def fetch_profile_page_html(
+        self,
+        scholar_id: str,
+        *,
+        cstart: int,
+        pagesize: int = DEFAULT_PAGE_SIZE,
+    ) -> FetchResult:
+        requested_url = _build_profile_url(
+            scholar_id=scholar_id,
+            cstart=cstart,
+            pagesize=pagesize,
+        )
+        logger.debug(
+            "scholar_source.fetch_started",
+            extra={
+                "event": "scholar_source.fetch_started",
+                "scholar_id": scholar_id,
+                "requested_url": requested_url,
+                "cstart": cstart,
+                "pagesize": pagesize,
+            },
+        )
+        return await asyncio.to_thread(self._fetch_sync, requested_url)
+
+    async def fetch_author_search_html(
+        self,
+        query: str,
+        *,
+        start: int = 0,
+    ) -> FetchResult:
+        requested_url = _build_author_search_url(
+            query=query,
+            start=start,
+        )
+        logger.debug(
+            "scholar_source.search_fetch_started",
+            extra={
+                "event": "scholar_source.search_fetch_started",
+                "query": query,
+                "requested_url": requested_url,
+                "start": start,
+            },
+        )
+        return await asyncio.to_thread(self._fetch_sync, requested_url)
+
+    async def fetch_publication_html(self, publication_url: str) -> FetchResult:
+        logger.debug(
+            "scholar_source.publication_fetch_started",
+            extra={
+                "event": "scholar_source.publication_fetch_started",
+                "requested_url": publication_url,
+            },
+        )
+        return await asyncio.to_thread(self._fetch_sync, publication_url)
+
+    def _build_request(self, requested_url: str) -> Request:
+        return Request(
+            requested_url,
+            headers={
+                "User-Agent": random.choice(self._user_agents),
+                "Accept": "text/html,application/xhtml+xml",
+                "Accept-Language": "en-US,en;q=0.9",
+                "Connection": "close",
+            },
+        )
+
+    @staticmethod
+    def _http_error_body(exc: HTTPError) -> str:
+        try:
+            return exc.read().decode("utf-8", errors="replace")
+        except Exception:
+            return ""
+
+    @staticmethod
+    def _network_error_result(requested_url: str, exc: URLError) -> FetchResult:
+        logger.warning(
+            "scholar_source.fetch_network_error",
+            extra={"event": "scholar_source.fetch_network_error", "requested_url": requested_url},
+        )
+        return FetchResult(
+            requested_url=requested_url,
+            status_code=None,
+            final_url=None,
+            body="",
+            error=str(exc),
+        )
+
+    @staticmethod
+    def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult:
+        logger.warning(
+            "scholar_source.fetch_http_error",
+            extra={
+                "event": "scholar_source.fetch_http_error",
+                "requested_url": requested_url,
+                "status_code": exc.code,
+            },
+        )
+        return FetchResult(
+            requested_url=requested_url,
+            status_code=exc.code,
+            final_url=exc.geturl(),
+            body=LiveScholarSource._http_error_body(exc),
+            error=str(exc),
+        )
+
+    @staticmethod
+    def _success_result(requested_url: str, response) -> FetchResult:
+        body = response.read().decode("utf-8", errors="replace")
+        status_code = getattr(response, "status", 200)
+        logger.debug(
+            "scholar_source.fetch_succeeded",
+            extra={
+                "event": "scholar_source.fetch_succeeded",
+                "requested_url": requested_url,
+                "status_code": status_code,
+            },
+        )
+        return FetchResult(
+            requested_url=requested_url,
+            status_code=status_code,
+            final_url=response.geturl(),
+            body=body,
+            error=None,
+        )
+
+    def _fetch_sync(self, requested_url: str) -> FetchResult:
+        request = self._build_request(requested_url)
+
+        try:
+            with urlopen(request, timeout=self._timeout_seconds) as response:
+                return self._success_result(requested_url, response)
+        except HTTPError as exc:
+            return self._http_error_result(requested_url, exc)
+        except URLError as exc:
+            return self._network_error_result(requested_url, exc)
+
+
+def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:
+    query: dict[str, int | str] = {"hl": "en", "user": scholar_id}
+    if cstart > 0:
+        query["cstart"] = int(cstart)
+    if pagesize > 0:
+        query["pagesize"] = int(pagesize)
+    return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}"
+
+
+def _build_author_search_url(*, query: str, start: int) -> str:
+    params: dict[str, int | str] = {
+        "hl": "en",
+        "view_op": "search_authors",
+        "mauthors": query,
+    }
+    if start > 0:
+        params["astart"] = int(start)
+    return f"{SCHOLAR_PROFILE_URL}?{urlencode(params)}"

build/lib/app/services/domains/scholar/state_detection.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+from app.services.domains.scholar.parser_constants import (
+    BLOCKED_KEYWORDS,
+    NETWORK_DNS_ERROR_KEYWORDS,
+    NETWORK_TIMEOUT_KEYWORDS,
+    NETWORK_TLS_ERROR_KEYWORDS,
+    NO_AUTHOR_RESULTS_KEYWORDS,
+    NO_RESULTS_KEYWORDS,
+)
+from app.services.domains.scholar.parser_types import ParseState, PublicationCandidate, ScholarSearchCandidate
+from app.services.domains.scholar.source import FetchResult
+
+
+def classify_network_error_reason(fetch_error: str | None) -> str:
+    lowered = (fetch_error or "").lower()
+    if lowered:
+        if any(keyword in lowered for keyword in NETWORK_DNS_ERROR_KEYWORDS):
+            return "network_dns_resolution_failed"
+        if any(keyword in lowered for keyword in NETWORK_TIMEOUT_KEYWORDS):
+            return "network_timeout"
+        if any(keyword in lowered for keyword in NETWORK_TLS_ERROR_KEYWORDS):
+            return "network_tls_error"
+        if "connection reset" in lowered:
+            return "network_connection_reset"
+        if "connection refused" in lowered:
+            return "network_connection_refused"
+        if "network is unreachable" in lowered:
+            return "network_unreachable"
+    return "network_error_missing_status_code"
+
+
+def classify_block_or_captcha_reason(
+    *,
+    status_code: int,
+    final_url: str,
+    body_lowered: str,
+) -> str | None:
+    if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url):
+        return "blocked_accounts_redirect"
+    if status_code == 429:
+        return "blocked_http_429_rate_limited"
+    if status_code == 403:
+        if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url:
+            return "blocked_http_403_captcha_challenge"
+        return "blocked_http_403_forbidden"
+    if "sorry/index" in final_url or "sorry/index" in body_lowered:
+        return "blocked_google_sorry_challenge"
+    if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered:
+        return "blocked_unusual_traffic_detected"
+    if "automated queries" in body_lowered:
+        return "blocked_automated_queries_detected"
+    if "not a robot" in body_lowered:
+        return "blocked_not_a_robot_challenge"
+    if "recaptcha" in body_lowered:
+        return "blocked_recaptcha_challenge"
+    if "captcha" in body_lowered:
+        return "blocked_captcha_challenge"
+    if any(keyword in body_lowered for keyword in BLOCKED_KEYWORDS):
+        return "blocked_keyword_detected"
+    return None
+
+
+def _warnings_contain(warnings: list[str], code: str) -> bool:
+    return any(item == code for item in warnings)
+
+
+def _has_layout_row_failure(marker_counts: dict[str, int], warnings: list[str]) -> bool:
+    if _warnings_contain(warnings, "layout_all_rows_unparseable"):
+        return True
+    if marker_counts.get("gsc_a_tr", 0) <= 0:
+        return False
+    if _warnings_contain(warnings, "row_missing_title"):
+        return True
+    return marker_counts.get("gsc_a_at", 0) <= 0
+
+
+def _first_layout_warning(warnings: list[str]) -> str | None:
+    for warning in warnings:
+        if warning.startswith("layout_"):
+            return warning
+    return None
+
+
+def detect_state(
+    fetch_result: FetchResult,
+    publications: list[PublicationCandidate],
+    marker_counts: dict[str, int],
+    *,
+    warnings: list[str],
+    has_show_more_button_flag: bool,
+    articles_range: str | None,
+    visible_text: str,
+) -> tuple[ParseState, str]:
+    if fetch_result.status_code is None:
+        return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
+
+    lowered = fetch_result.body.lower()
+    final = (fetch_result.final_url or "").lower()
+    status_code = int(fetch_result.status_code)
+
+    block_reason = classify_block_or_captcha_reason(
+        status_code=status_code,
+        final_url=final,
+        body_lowered=lowered,
+    )
+    if block_reason is not None:
+        return ParseState.BLOCKED_OR_CAPTCHA, block_reason
+
+    if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS):
+        return ParseState.NO_RESULTS, "no_results_keyword_detected"
+
+    layout_warning = _first_layout_warning(warnings)
+    if layout_warning is not None:
+        return ParseState.LAYOUT_CHANGED, layout_warning
+
+    if _has_layout_row_failure(marker_counts, warnings):
+        return ParseState.LAYOUT_CHANGED, "layout_publication_rows_unparseable"
+
+    if has_show_more_button_flag and not articles_range:
+        return ParseState.LAYOUT_CHANGED, "layout_show_more_without_articles_range"
+
+    if not publications:
+        has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
+        has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
+        if not has_profile_markers and not has_table_markers:
+            return ParseState.LAYOUT_CHANGED, "layout_markers_missing"
+        return ParseState.OK, "no_rows_with_known_markers"
+
+    return ParseState.OK, "publications_extracted"
+
+
+def detect_author_search_state(
+    fetch_result: FetchResult,
+    candidates: list[ScholarSearchCandidate],
+    marker_counts: dict[str, int],
+    *,
+    visible_text: str,
+) -> tuple[ParseState, str]:
+    if fetch_result.status_code is None:
+        return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
+
+    lowered = fetch_result.body.lower()
+    final = (fetch_result.final_url or "").lower()
+    status_code = int(fetch_result.status_code)
+
+    block_reason = classify_block_or_captcha_reason(
+        status_code=status_code,
+        final_url=final,
+        body_lowered=lowered,
+    )
+    if block_reason is not None:
+        return ParseState.BLOCKED_OR_CAPTCHA, block_reason
+
+    if not candidates and any(keyword in visible_text for keyword in NO_AUTHOR_RESULTS_KEYWORDS):
+        return ParseState.NO_RESULTS, "no_results_keyword_detected"
+
+    if not candidates:
+        has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0
+        if not has_search_markers:
+            return ParseState.NO_RESULTS, "no_search_candidates_detected"
+        return ParseState.LAYOUT_CHANGED, "layout_author_candidates_unparseable"
+
+    return ParseState.OK, "author_candidates_extracted"

build/lib/app/services/domains/scholars/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

build/lib/app/services/domains/scholars/application.py

@@ -0,0 +1,974 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import replace
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
+import logging
+import os
+import random
+from uuid import uuid4
+
+from sqlalchemy import delete, func, select, text
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import AuthorSearchCacheEntry, AuthorSearchRuntimeState, ScholarProfile
+from app.services.domains.scholar.parser import (
+    ParseState,
+    ParsedAuthorSearchPage,
+    ScholarParserError,
+    parse_author_search_page,
+    parse_profile_page,
+)
+from app.services.domains.scholar.source import ScholarSource
+from app.services.domains.scholars.constants import (
+    ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES,
+    AUTHOR_SEARCH_LOCK_KEY,
+    AUTHOR_SEARCH_LOCK_NAMESPACE,
+    AUTHOR_SEARCH_RUNTIME_STATE_KEY,
+    DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS,
+    DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES,
+    DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD,
+    DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD,
+    DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS,
+    DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS,
+    MAX_AUTHOR_SEARCH_LIMIT,
+    DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS,
+    DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD,
+    SEARCH_CACHED_BLOCK_REASON,
+    SEARCH_COOLDOWN_REASON,
+    SEARCH_DISABLED_REASON,
+)
+from app.services.domains.scholars.exceptions import ScholarServiceError
+from app.services.domains.scholars.search_hints import (
+    _merge_warnings,
+    _policy_blocked_author_search_result,
+    _trim_author_search_result,
+    resolve_profile_image,
+    scrape_state_hint,
+)
+from app.services.domains.scholars.uploads import (
+    _ensure_upload_root,
+    _resolve_upload_path,
+    _safe_remove_upload,
+    resolve_upload_file_path,
+)
+from app.services.domains.scholars.validators import (
+    normalize_display_name,
+    normalize_profile_image_url,
+    validate_scholar_id,
+)
+logger = logging.getLogger(__name__)
+
+
+async def _acquire_author_search_lock(db_session: AsyncSession) -> None:
+    await db_session.execute(
+        text("SELECT pg_advisory_xact_lock(:namespace, :lock_key)"),
+        {
+            "namespace": AUTHOR_SEARCH_LOCK_NAMESPACE,
+            "lock_key": AUTHOR_SEARCH_LOCK_KEY,
+        },
+    )
+
+
+async def _load_runtime_state_for_update(
+    db_session: AsyncSession,
+) -> AuthorSearchRuntimeState:
+    result = await db_session.execute(
+        select(AuthorSearchRuntimeState)
+        .where(AuthorSearchRuntimeState.state_key == AUTHOR_SEARCH_RUNTIME_STATE_KEY)
+        .with_for_update()
+    )
+    state = result.scalar_one_or_none()
+    if state is not None:
+        return state
+
+    state = AuthorSearchRuntimeState(state_key=AUTHOR_SEARCH_RUNTIME_STATE_KEY)
+    db_session.add(state)
+    await db_session.flush()
+    return state
+
+
+def _serialize_parsed_author_search_page(parsed: ParsedAuthorSearchPage) -> dict:
+    return {
+        "state": parsed.state.value,
+        "state_reason": parsed.state_reason,
+        "marker_counts": {str(key): int(value) for key, value in parsed.marker_counts.items()},
+        "warnings": [str(value) for value in parsed.warnings],
+        "candidates": [
+            {
+                "scholar_id": candidate.scholar_id,
+                "display_name": candidate.display_name,
+                "affiliation": candidate.affiliation,
+                "email_domain": candidate.email_domain,
+                "cited_by_count": candidate.cited_by_count,
+                "interests": [str(interest) for interest in candidate.interests],
+                "profile_url": candidate.profile_url,
+                "profile_image_url": candidate.profile_image_url,
+            }
+            for candidate in parsed.candidates
+        ],
+    }
+
+
+def _payload_state(payload: dict[str, object]) -> ParseState | None:
+    state_raw = str(payload.get("state", "")).strip()
+    try:
+        return ParseState(state_raw)
+    except ValueError:
+        return None
+
+
+def _payload_marker_counts(payload: dict[str, object]) -> dict[str, int]:
+    marker_counts_payload = payload.get("marker_counts")
+    if not isinstance(marker_counts_payload, dict):
+        return {}
+    parsed: dict[str, int] = {}
+    for key, value in marker_counts_payload.items():
+        try:
+            parsed[str(key)] = int(value)
+        except (TypeError, ValueError):
+            continue
+    return parsed
+
+
+def _payload_warnings(payload: dict[str, object]) -> list[str]:
+    warnings_payload = payload.get("warnings")
+    if not isinstance(warnings_payload, list):
+        return []
+    return [str(value) for value in warnings_payload if isinstance(value, str)]
+
+
+def _parse_optional_string(value: object) -> str | None:
+    if value is None:
+        return None
+    normalized = str(value).strip()
+    return normalized or None
+
+
+def _parse_optional_int(value: object) -> int | None:
+    if isinstance(value, int):
+        return value
+    if isinstance(value, str) and value.strip():
+        try:
+            return int(value)
+        except ValueError:
+            return None
+    return None
+
+
+def _normalize_interests(value: object) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(item) for item in value if isinstance(item, str)]
+
+
+def _deserialize_candidate_payload(value: object) -> dict[str, object] | None:
+    if not isinstance(value, dict):
+        return None
+    scholar_id = str(value.get("scholar_id", "")).strip()
+    display_name = str(value.get("display_name", "")).strip()
+    profile_url = str(value.get("profile_url", "")).strip()
+    if not scholar_id or not display_name or not profile_url:
+        return None
+    return {
+        "scholar_id": scholar_id,
+        "display_name": display_name,
+        "affiliation": _parse_optional_string(value.get("affiliation")),
+        "email_domain": _parse_optional_string(value.get("email_domain")),
+        "cited_by_count": _parse_optional_int(value.get("cited_by_count")),
+        "interests": _normalize_interests(value.get("interests")),
+        "profile_url": profile_url,
+        "profile_image_url": _parse_optional_string(value.get("profile_image_url")),
+    }
+
+
+def _deserialize_candidates(payload: dict[str, object]) -> list[dict[str, object]]:
+    candidates_payload = payload.get("candidates")
+    if not isinstance(candidates_payload, list):
+        return []
+    normalized: list[dict[str, object]] = []
+    for value in candidates_payload:
+        candidate = _deserialize_candidate_payload(value)
+        if candidate is not None:
+            normalized.append(candidate)
+    return normalized
+
+
+def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearchPage | None:
+    if not isinstance(payload, dict):
+        return None
+
+    state = _payload_state(payload)
+    if state is None:
+        return None
+
+    marker_counts = _payload_marker_counts(payload)
+    warnings = _payload_warnings(payload)
+    from app.services.domains.scholar.parser import ScholarSearchCandidate
+
+    normalized_candidates = _deserialize_candidates(payload)
+
+    return ParsedAuthorSearchPage(
+        state=state,
+        state_reason=str(payload.get("state_reason", "")).strip() or "unknown",
+        candidates=[
+            ScholarSearchCandidate(
+                scholar_id=item["scholar_id"],
+                display_name=item["display_name"],
+                affiliation=item["affiliation"],
+                email_domain=item["email_domain"],
+                cited_by_count=item["cited_by_count"],
+                interests=item["interests"],
+                profile_url=item["profile_url"],
+                profile_image_url=item["profile_image_url"],
+            )
+            for item in normalized_candidates
+        ],
+        marker_counts=marker_counts,
+        warnings=warnings,
+    )
+
+
+async def _cache_get_author_search_result(
+    db_session: AsyncSession,
+    *,
+    query_key: str,
+    now_utc: datetime,
+) -> ParsedAuthorSearchPage | None:
+    result = await db_session.execute(
+        select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key)
+    )
+    entry = result.scalar_one_or_none()
+    if entry is None:
+        return None
+    expires_at = entry.expires_at
+    if expires_at.tzinfo is None:
+        expires_at = expires_at.replace(tzinfo=timezone.utc)
+    if expires_at <= now_utc:
+        await db_session.delete(entry)
+        return None
+    parsed = _deserialize_parsed_author_search_page(entry.payload)
+    if parsed is None:
+        await db_session.delete(entry)
+        return None
+    return parsed
+
+
+async def _cache_set_author_search_result(
+    db_session: AsyncSession,
+    *,
+    query_key: str,
+    parsed: ParsedAuthorSearchPage,
+    ttl_seconds: float,
+    max_entries: int,
+    now_utc: datetime,
+) -> None:
+    ttl = max(float(ttl_seconds), 0.0)
+    existing_result = await db_session.execute(
+        select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key)
+    )
+    existing = existing_result.scalar_one_or_none()
+
+    if ttl <= 0.0:
+        if existing is not None:
+            await db_session.delete(existing)
+        return
+
+    expires_at = now_utc + timedelta(seconds=ttl)
+    payload = _serialize_parsed_author_search_page(parsed)
+    if existing is None:
+        db_session.add(
+            AuthorSearchCacheEntry(
+                query_key=query_key,
+                payload=payload,
+                expires_at=expires_at,
+                cached_at=now_utc,
+                updated_at=now_utc,
+            )
+        )
+    else:
+        existing.payload = payload
+        existing.expires_at = expires_at
+        existing.cached_at = now_utc
+        existing.updated_at = now_utc
+
+    await _prune_author_search_cache(db_session, now_utc=now_utc, max_entries=max_entries)
+
+
+async def _prune_author_search_cache(
+    db_session: AsyncSession,
+    *,
+    now_utc: datetime,
+    max_entries: int,
+) -> None:
+    await db_session.execute(
+        delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.expires_at <= now_utc)
+    )
+    bounded_max_entries = max(1, int(max_entries))
+    count_result = await db_session.execute(
+        select(func.count()).select_from(AuthorSearchCacheEntry)
+    )
+    entry_count = int(count_result.scalar_one() or 0)
+    overflow = max(0, entry_count - bounded_max_entries)
+    if overflow <= 0:
+        return
+    stale_keys_result = await db_session.execute(
+        select(AuthorSearchCacheEntry.query_key)
+        .order_by(AuthorSearchCacheEntry.cached_at.asc())
+        .limit(overflow)
+    )
+    stale_keys = [str(row[0]) for row in stale_keys_result.all()]
+    if stale_keys:
+        await db_session.execute(
+            delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key.in_(stale_keys))
+        )
+
+
+def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool:
+    return parsed.state == ParseState.BLOCKED_OR_CAPTCHA
+
+
+def _author_search_cooldown_remaining_seconds(
+    runtime_state: AuthorSearchRuntimeState,
+    now_utc: datetime,
+) -> int:
+    cooldown_until = runtime_state.cooldown_until
+    if cooldown_until is None:
+        return 0
+    if cooldown_until.tzinfo is None:
+        cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
+    remaining_seconds = int((cooldown_until - now_utc).total_seconds())
+    return max(0, remaining_seconds)
+
+
+async def list_scholars_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> list[ScholarProfile]:
+    result = await db_session.execute(
+        select(ScholarProfile)
+        .where(ScholarProfile.user_id == user_id)
+        .order_by(ScholarProfile.created_at.desc(), ScholarProfile.id.desc())
+    )
+    return list(result.scalars().all())
+
+
+async def create_scholar_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_id: str,
+    display_name: str,
+    profile_image_url: str | None = None,
+) -> ScholarProfile:
+    profile = ScholarProfile(
+        user_id=user_id,
+        scholar_id=validate_scholar_id(scholar_id),
+        display_name=normalize_display_name(display_name),
+        profile_image_url=normalize_profile_image_url(profile_image_url),
+    )
+    db_session.add(profile)
+    try:
+        await db_session.commit()
+    except IntegrityError as exc:
+        await db_session.rollback()
+        raise ScholarServiceError("That scholar is already tracked for this account.") from exc
+    await db_session.refresh(profile)
+    return profile
+
+
+async def get_user_scholar_by_id(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+) -> ScholarProfile | None:
+    result = await db_session.execute(
+        select(ScholarProfile).where(
+            ScholarProfile.id == scholar_profile_id,
+            ScholarProfile.user_id == user_id,
+        )
+    )
+    return result.scalar_one_or_none()
+
+
+async def toggle_scholar_enabled(
+    db_session: AsyncSession,
+    *,
+    profile: ScholarProfile,
+) -> ScholarProfile:
+    profile.is_enabled = not profile.is_enabled
+    await db_session.commit()
+    await db_session.refresh(profile)
+    return profile
+
+
+async def delete_scholar(
+    db_session: AsyncSession,
+    *,
+    profile: ScholarProfile,
+    upload_dir: str | None = None,
+) -> None:
+    if upload_dir:
+        upload_root = _ensure_upload_root(upload_dir, create=True)
+        _safe_remove_upload(upload_root, profile.profile_image_upload_path)
+
+    await db_session.delete(profile)
+    await db_session.commit()
+
+
+def _normalize_author_search_inputs(query: str, limit: int) -> tuple[str, int, str]:
+    normalized_query = query.strip()
+    if len(normalized_query) < 2:
+        raise ScholarServiceError("Search query must be at least 2 characters.")
+    bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))
+    return normalized_query, bounded_limit, normalized_query.casefold()
+
+
+def _disabled_search_result(*, normalized_query: str, bounded_limit: int) -> ParsedAuthorSearchPage:
+    logger.warning(
+        "scholar_search.disabled_by_configuration",
+        extra={"event": "scholar_search.disabled_by_configuration", "query": normalized_query},
+    )
+    return _policy_blocked_author_search_result(
+        reason=SEARCH_DISABLED_REASON,
+        warning_codes=["author_search_disabled_by_configuration"],
+        limit=bounded_limit,
+    )
+
+
+def _normalize_runtime_cooldown_state(
+    runtime_state: AuthorSearchRuntimeState,
+    *,
+    now_utc: datetime,
+) -> bool:
+    if runtime_state.cooldown_until is None:
+        return False
+    cooldown_until = runtime_state.cooldown_until
+    updated = False
+    if cooldown_until.tzinfo is None:
+        cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
+        runtime_state.cooldown_until = cooldown_until
+        updated = True
+    if now_utc < cooldown_until:
+        return updated
+    logger.info(
+        "scholar_search.cooldown_expired",
+        extra={"event": "scholar_search.cooldown_expired", "cooldown_until_utc": cooldown_until.isoformat()},
+    )
+    runtime_state.cooldown_until = None
+    runtime_state.cooldown_rejection_count = 0
+    runtime_state.cooldown_alert_emitted = False
+    return True
+
+
+def _cooldown_warning_codes(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    cooldown_remaining_seconds: int,
+) -> list[str]:
+    warning_codes = [
+        "author_search_cooldown_active",
+        f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
+    ]
+    if bool(runtime_state.cooldown_alert_emitted):
+        warning_codes.append("author_search_cooldown_alert_threshold_exceeded")
+    return warning_codes
+
+
+def _emit_cooldown_threshold_alert(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    normalized_query: str,
+    cooldown_rejection_alert_threshold: int,
+) -> bool:
+    runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1
+    threshold = max(1, int(cooldown_rejection_alert_threshold))
+    if int(runtime_state.cooldown_rejection_count) < threshold:
+        return True
+    if bool(runtime_state.cooldown_alert_emitted):
+        return True
+    logger.error(
+        "scholar_search.cooldown_rejection_threshold_exceeded",
+        extra={
+            "event": "scholar_search.cooldown_rejection_threshold_exceeded",
+            "query": normalized_query,
+            "cooldown_rejection_count": int(runtime_state.cooldown_rejection_count),
+            "threshold": threshold,
+            "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
+        },
+    )
+    runtime_state.cooldown_alert_emitted = True
+    return True
+
+
+def _cooldown_block_result(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    normalized_query: str,
+    bounded_limit: int,
+    cooldown_rejection_alert_threshold: int,
+    cooldown_remaining_seconds: int,
+) -> ParsedAuthorSearchPage:
+    _emit_cooldown_threshold_alert(
+        runtime_state=runtime_state,
+        normalized_query=normalized_query,
+        cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
+    )
+    logger.warning(
+        "scholar_search.cooldown_active",
+        extra={
+            "event": "scholar_search.cooldown_active",
+            "query": normalized_query,
+            "cooldown_remaining_seconds": cooldown_remaining_seconds,
+            "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
+        },
+    )
+    return _policy_blocked_author_search_result(
+        reason=SEARCH_COOLDOWN_REASON,
+        warning_codes=_cooldown_warning_codes(
+            runtime_state=runtime_state,
+            cooldown_remaining_seconds=cooldown_remaining_seconds,
+        ),
+        limit=bounded_limit,
+    )
+
+
+async def _cache_hit_result(
+    db_session: AsyncSession,
+    *,
+    query_key: str,
+    now_utc: datetime,
+    normalized_query: str,
+    bounded_limit: int,
+) -> ParsedAuthorSearchPage | None:
+    cached = await _cache_get_author_search_result(
+        db_session,
+        query_key=query_key,
+        now_utc=now_utc,
+    )
+    if cached is None:
+        return None
+    logger.info(
+        "scholar_search.cache_hit",
+        extra={
+            "event": "scholar_search.cache_hit",
+            "query": normalized_query,
+            "state": cached.state.value,
+            "state_reason": cached.state_reason,
+        },
+    )
+    state_reason_override = SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None
+    return _trim_author_search_result(
+        cached,
+        limit=bounded_limit,
+        extra_warnings=["author_search_served_from_cache"],
+        state_reason_override=state_reason_override,
+    )
+
+
+def _throttle_sleep_seconds(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    now_utc: datetime,
+    min_interval_seconds: float,
+    interval_jitter_seconds: float,
+) -> tuple[float, bool]:
+    updated = False
+    if runtime_state.last_live_request_at is None:
+        enforced_wait_seconds = 0.0
+    else:
+        last_live_request_at = runtime_state.last_live_request_at
+        if last_live_request_at.tzinfo is None:
+            last_live_request_at = last_live_request_at.replace(tzinfo=timezone.utc)
+            runtime_state.last_live_request_at = last_live_request_at
+            updated = True
+        enforced_wait_seconds = (
+            last_live_request_at + timedelta(seconds=max(float(min_interval_seconds), 0.0)) - now_utc
+        ).total_seconds()
+    jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0))
+    return max(0.0, float(enforced_wait_seconds)) + jitter_seconds, updated
+
+
+async def _wait_for_author_search_throttle(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    normalized_query: str,
+    now_utc: datetime,
+    min_interval_seconds: float,
+    interval_jitter_seconds: float,
+) -> bool:
+    sleep_seconds, updated = _throttle_sleep_seconds(
+        runtime_state=runtime_state,
+        now_utc=now_utc,
+        min_interval_seconds=min_interval_seconds,
+        interval_jitter_seconds=interval_jitter_seconds,
+    )
+    if sleep_seconds <= 0.0:
+        return updated
+    logger.info(
+        "scholar_search.throttle_wait",
+        extra={"event": "scholar_search.throttle_wait", "query": normalized_query, "sleep_seconds": round(sleep_seconds, 3)},
+    )
+    await asyncio.sleep(sleep_seconds)
+    return True
+
+
+async def _fetch_author_search_with_retries(
+    *,
+    source: ScholarSource,
+    normalized_query: str,
+    network_error_retries: int,
+    retry_backoff_seconds: float,
+) -> tuple[ParsedAuthorSearchPage, int, list[str]]:
+    max_attempts = max(1, int(network_error_retries) + 1)
+    parsed: ParsedAuthorSearchPage | None = None
+    retry_warnings: list[str] = []
+    retry_scheduled_count = 0
+    for attempt_index in range(max_attempts):
+        fetch_result = await source.fetch_author_search_html(normalized_query, start=0)
+        try:
+            parsed = parse_author_search_page(fetch_result)
+        except ScholarParserError as exc:
+            parsed = ParsedAuthorSearchPage(
+                state=ParseState.LAYOUT_CHANGED,
+                state_reason=exc.code,
+                candidates=[],
+                marker_counts={},
+                warnings=[exc.code],
+            )
+        if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1:
+            break
+        retry_warnings.append("network_retry_scheduled_for_author_search")
+        retry_scheduled_count += 1
+        retry_sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index)
+        if retry_sleep_seconds > 0:
+            await asyncio.sleep(retry_sleep_seconds)
+    if parsed is None:
+        raise ScholarServiceError("Unable to complete scholar author search.")
+    return parsed, retry_scheduled_count, retry_warnings
+
+
+def _with_retry_warnings(
+    parsed: ParsedAuthorSearchPage,
+    *,
+    retry_warnings: list[str],
+    retry_scheduled_count: int,
+    retry_alert_threshold: int,
+    normalized_query: str,
+) -> ParsedAuthorSearchPage:
+    merged = replace(parsed, warnings=_merge_warnings(parsed.warnings, retry_warnings))
+    threshold = max(1, int(retry_alert_threshold))
+    if retry_scheduled_count < threshold:
+        return merged
+    logger.warning(
+        "scholar_search.retry_threshold_exceeded",
+        extra={
+            "event": "scholar_search.retry_threshold_exceeded",
+            "query": normalized_query,
+            "retry_scheduled_count": retry_scheduled_count,
+            "threshold": threshold,
+            "final_state": merged.state.value,
+            "final_state_reason": merged.state_reason,
+        },
+    )
+    return replace(
+        merged,
+        warnings=_merge_warnings(
+            merged.warnings,
+            [f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"],
+        ),
+    )
+
+
+def _apply_block_circuit_breaker(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    merged_parsed: ParsedAuthorSearchPage,
+    cooldown_block_threshold: int,
+    cooldown_seconds: int,
+    normalized_query: str,
+) -> ParsedAuthorSearchPage:
+    if not _is_author_search_block_state(merged_parsed):
+        runtime_state.consecutive_blocked_count = 0
+        return merged_parsed
+    runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1
+    logger.warning(
+        "scholar_search.block_detected",
+        extra={
+            "event": "scholar_search.block_detected",
+            "query": normalized_query,
+            "state_reason": merged_parsed.state_reason,
+            "consecutive_blocked_count": int(runtime_state.consecutive_blocked_count),
+        },
+    )
+    if int(runtime_state.consecutive_blocked_count) < max(1, int(cooldown_block_threshold)):
+        return merged_parsed
+    runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta(seconds=max(60, int(cooldown_seconds)))
+    runtime_state.consecutive_blocked_count = 0
+    runtime_state.cooldown_rejection_count = 0
+    runtime_state.cooldown_alert_emitted = False
+    logger.error(
+        "scholar_search.cooldown_activated",
+        extra={
+            "event": "scholar_search.cooldown_activated",
+            "query": normalized_query,
+            "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
+        },
+    )
+    return replace(
+        merged_parsed,
+        warnings=_merge_warnings(merged_parsed.warnings, ["author_search_circuit_breaker_armed"]),
+    )
+
+
+def _resolve_author_search_cache_ttl_seconds(
+    *,
+    merged_parsed: ParsedAuthorSearchPage,
+    blocked_cache_ttl_seconds: int,
+    cache_ttl_seconds: int,
+) -> int:
+    if _is_author_search_block_state(merged_parsed):
+        return min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds)))
+    return max(1, int(cache_ttl_seconds))
+
+
+async def _load_locked_runtime_state(
+    db_session: AsyncSession,
+) -> AuthorSearchRuntimeState:
+    await _acquire_author_search_lock(db_session)
+    return await _load_runtime_state_for_update(db_session)
+
+
+async def _cooldown_or_cache_result(
+    db_session: AsyncSession,
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    query_key: str,
+    normalized_query: str,
+    bounded_limit: int,
+    cooldown_rejection_alert_threshold: int,
+) -> tuple[ParsedAuthorSearchPage | None, bool]:
+    runtime_state_updated = _normalize_runtime_cooldown_state(
+        runtime_state,
+        now_utc=datetime.now(timezone.utc),
+    )
+    cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(
+        runtime_state,
+        datetime.now(timezone.utc),
+    )
+    if cooldown_remaining_seconds > 0:
+        return (
+            _cooldown_block_result(
+                runtime_state=runtime_state,
+                normalized_query=normalized_query,
+                bounded_limit=bounded_limit,
+                cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
+                cooldown_remaining_seconds=cooldown_remaining_seconds,
+            ),
+            True,
+        )
+    cached_result = await _cache_hit_result(
+        db_session,
+        query_key=query_key,
+        now_utc=datetime.now(timezone.utc),
+        normalized_query=normalized_query,
+        bounded_limit=bounded_limit,
+    )
+    return cached_result, runtime_state_updated
+
+
+async def _perform_live_author_search(db_session: AsyncSession, *, source: ScholarSource, runtime_state: AuthorSearchRuntimeState, normalized_query: str, query_key: str, network_error_retries: int, retry_backoff_seconds: float, min_interval_seconds: float, interval_jitter_seconds: float, retry_alert_threshold: int, cooldown_block_threshold: int, cooldown_seconds: int, blocked_cache_ttl_seconds: int, cache_ttl_seconds: int, cache_max_entries: int) -> tuple[ParsedAuthorSearchPage, bool]:
+    runtime_state_updated = await _wait_for_author_search_throttle(
+        runtime_state=runtime_state,
+        normalized_query=normalized_query,
+        now_utc=datetime.now(timezone.utc),
+        min_interval_seconds=min_interval_seconds,
+        interval_jitter_seconds=interval_jitter_seconds,
+    )
+    parsed, retry_count, retry_warnings = await _fetch_author_search_with_retries(
+        source=source,
+        normalized_query=normalized_query,
+        network_error_retries=network_error_retries,
+        retry_backoff_seconds=retry_backoff_seconds,
+    )
+    runtime_state.last_live_request_at = datetime.now(timezone.utc)
+    merged = _with_retry_warnings(
+        parsed,
+        retry_warnings=retry_warnings,
+        retry_scheduled_count=retry_count,
+        retry_alert_threshold=retry_alert_threshold,
+        normalized_query=normalized_query,
+    )
+    merged = _apply_block_circuit_breaker(
+        runtime_state=runtime_state,
+        merged_parsed=merged,
+        cooldown_block_threshold=cooldown_block_threshold,
+        cooldown_seconds=cooldown_seconds,
+        normalized_query=normalized_query,
+    )
+    ttl_seconds = _resolve_author_search_cache_ttl_seconds(
+        merged_parsed=merged,
+        blocked_cache_ttl_seconds=blocked_cache_ttl_seconds,
+        cache_ttl_seconds=cache_ttl_seconds,
+    )
+    await _cache_set_author_search_result(
+        db_session,
+        query_key=query_key,
+        parsed=merged,
+        ttl_seconds=float(ttl_seconds),
+        max_entries=cache_max_entries,
+        now_utc=datetime.now(timezone.utc),
+    )
+    return merged, True
+
+
+async def search_author_candidates(*, source: ScholarSource, db_session: AsyncSession, query: str, limit: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, search_enabled: bool = True, cache_ttl_seconds: int = 21_600, blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS, cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES, min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS, interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS, cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD, cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS, retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD) -> ParsedAuthorSearchPage:
+    normalized_query, bounded_limit, query_key = _normalize_author_search_inputs(query, limit)
+    if not search_enabled:
+        return _disabled_search_result(
+            normalized_query=normalized_query,
+            bounded_limit=bounded_limit,
+        )
+
+    runtime_state = await _load_locked_runtime_state(db_session)
+    early_result, runtime_state_updated = await _cooldown_or_cache_result(
+        db_session,
+        runtime_state=runtime_state,
+        query_key=query_key,
+        normalized_query=normalized_query,
+        bounded_limit=bounded_limit,
+        cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
+    )
+    if early_result is not None:
+        await db_session.commit()
+        return early_result
+
+    merged_parsed, live_runtime_updated = await _perform_live_author_search(
+        db_session,
+        source=source,
+        runtime_state=runtime_state,
+        normalized_query=normalized_query,
+        query_key=query_key,
+        network_error_retries=network_error_retries,
+        retry_backoff_seconds=retry_backoff_seconds,
+        min_interval_seconds=min_interval_seconds,
+        interval_jitter_seconds=interval_jitter_seconds,
+        retry_alert_threshold=retry_alert_threshold,
+        cooldown_block_threshold=cooldown_block_threshold,
+        cooldown_seconds=cooldown_seconds,
+        blocked_cache_ttl_seconds=blocked_cache_ttl_seconds,
+        cache_ttl_seconds=cache_ttl_seconds,
+        cache_max_entries=cache_max_entries,
+    )
+    runtime_state_updated = runtime_state_updated or live_runtime_updated
+    if runtime_state_updated:
+        await db_session.commit()
+    return _trim_author_search_result(merged_parsed, limit=bounded_limit)
+
+
+async def hydrate_profile_metadata(
+    db_session: AsyncSession,
+    *,
+    profile: ScholarProfile,
+    source: ScholarSource,
+) -> ScholarProfile:
+    fetch_result = await source.fetch_profile_html(profile.scholar_id)
+    try:
+        parsed_page = parse_profile_page(fetch_result)
+    except ScholarParserError:
+        return profile
+
+    if parsed_page.profile_name and not (profile.display_name or "").strip():
+        profile.display_name = parsed_page.profile_name
+    if parsed_page.profile_image_url and not profile.profile_image_url:
+        profile.profile_image_url = parsed_page.profile_image_url
+
+    await db_session.commit()
+    await db_session.refresh(profile)
+    return profile
+
+
+async def set_profile_image_override_url(
+    db_session: AsyncSession,
+    *,
+    profile: ScholarProfile,
+    image_url: str | None,
+    upload_dir: str,
+) -> ScholarProfile:
+    upload_root = _ensure_upload_root(upload_dir, create=True)
+    _safe_remove_upload(upload_root, profile.profile_image_upload_path)
+
+    profile.profile_image_upload_path = None
+    profile.profile_image_override_url = normalize_profile_image_url(image_url)
+
+    await db_session.commit()
+    await db_session.refresh(profile)
+    return profile
+
+
+async def clear_profile_image_customization(
+    db_session: AsyncSession,
+    *,
+    profile: ScholarProfile,
+    upload_dir: str,
+) -> ScholarProfile:
+    upload_root = _ensure_upload_root(upload_dir, create=True)
+    _safe_remove_upload(upload_root, profile.profile_image_upload_path)
+
+    profile.profile_image_upload_path = None
+    profile.profile_image_override_url = None
+
+    await db_session.commit()
+    await db_session.refresh(profile)
+    return profile
+
+
+async def set_profile_image_upload(
+    db_session: AsyncSession,
+    *,
+    profile: ScholarProfile,
+    content_type: str | None,
+    image_bytes: bytes,
+    upload_dir: str,
+    max_upload_bytes: int,
+) -> ScholarProfile:
+    normalized_content_type = (content_type or "").strip().lower()
+    extension = ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES.get(normalized_content_type)
+    if extension is None:
+        raise ScholarServiceError(
+            "Unsupported image type. Use JPEG, PNG, WEBP, or GIF."
+        )
+
+    if not image_bytes:
+        raise ScholarServiceError("Uploaded image file is empty.")
+
+    if len(image_bytes) > max_upload_bytes:
+        raise ScholarServiceError(
+            f"Uploaded image exceeds {max_upload_bytes} bytes."
+        )
+
+    upload_root = _ensure_upload_root(upload_dir, create=True)
+    user_dir = upload_root / str(profile.user_id)
+    user_dir.mkdir(parents=True, exist_ok=True)
+
+    filename = f"{profile.id}_{uuid4().hex}{extension}"
+    relative_path = os.path.join(str(profile.user_id), filename)
+    absolute_path = _resolve_upload_path(upload_root, relative_path)
+    absolute_path.write_bytes(image_bytes)
+
+    old_path = profile.profile_image_upload_path
+    profile.profile_image_upload_path = relative_path
+    profile.profile_image_override_url = None
+
+    await db_session.commit()
+    await db_session.refresh(profile)
+
+    if old_path and old_path != relative_path:
+        _safe_remove_upload(upload_root, old_path)
+
+    return profile

build/lib/app/services/domains/scholars/constants.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import re
+
+SCHOLAR_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]{12}$")
+MAX_IMAGE_URL_LENGTH = 2048
+MAX_AUTHOR_SEARCH_LIMIT = 25
+
+DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES = 512
+DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS = 300
+DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD = 1
+DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS = 1800
+DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS = 3.0
+DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS = 1.0
+DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD = 2
+DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD = 3
+
+AUTHOR_SEARCH_RUNTIME_STATE_KEY = "global"
+AUTHOR_SEARCH_LOCK_NAMESPACE = 3901
+AUTHOR_SEARCH_LOCK_KEY = 1
+
+ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES = {
+    "image/jpeg": ".jpg",
+    "image/png": ".png",
+    "image/webp": ".webp",
+    "image/gif": ".gif",
+}
+
+SEARCH_DISABLED_REASON = "search_disabled_by_configuration"
+SEARCH_COOLDOWN_REASON = "search_temporarily_disabled_due_to_repeated_blocks"
+SEARCH_CACHED_BLOCK_REASON = "search_temporarily_disabled_from_cached_blocked_response"
+
+STATE_REASON_HINTS: dict[str, str] = {
+    SEARCH_DISABLED_REASON: (
+        "Scholar name search is currently disabled by service policy. "
+        "Add scholars by profile URL or Scholar ID."
+    ),
+    SEARCH_COOLDOWN_REASON: (
+        "Scholar name search is temporarily paused after repeated block responses. "
+        "Use Scholar URL/ID adds until cooldown expires."
+    ),
+    SEARCH_CACHED_BLOCK_REASON: (
+        "A recent blocked response was cached to reduce traffic. "
+        "Retry later or add by Scholar URL/ID."
+    ),
+    "network_dns_resolution_failed": (
+        "DNS resolution failed while reaching scholar.google.com. "
+        "Verify container DNS/network and retry."
+    ),
+    "network_timeout": (
+        "Request timed out before Google Scholar responded. "
+        "Increase delay/backoff and retry."
+    ),
+    "network_tls_error": (
+        "TLS handshake/certificate validation failed. "
+        "Verify outbound TLS/network configuration."
+    ),
+    "blocked_http_429_rate_limited": (
+        "Google Scholar rate-limited the request. "
+        "Slow request cadence and retry later."
+    ),
+    "blocked_unusual_traffic_detected": (
+        "Google Scholar flagged traffic as unusual. "
+        "Increase delay/jitter and reduce concurrent scraping."
+    ),
+    "blocked_accounts_redirect": (
+        "Request was redirected to Google Account sign-in. "
+        "Treat as access block and retry later."
+    ),
+}

build/lib/app/services/domains/scholars/exceptions.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+
+class ScholarServiceError(ValueError):
+    """Raised for expected scholar-management validation failures."""

build/lib/app/services/domains/scholars/search_hints.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from app.db.models import ScholarProfile
+from app.services.domains.scholar.parser import ParseState, ParsedAuthorSearchPage
+from app.services.domains.scholars.constants import (
+    MAX_AUTHOR_SEARCH_LIMIT,
+    STATE_REASON_HINTS,
+)
+
+
+def resolve_profile_image(
+    profile: ScholarProfile,
+    *,
+    uploaded_image_url: str | None,
+) -> tuple[str | None, str]:
+    if profile.profile_image_upload_path and uploaded_image_url:
+        return uploaded_image_url, "upload"
+    if profile.profile_image_override_url:
+        return profile.profile_image_override_url, "override"
+    if profile.profile_image_url:
+        return profile.profile_image_url, "scraped"
+    return None, "none"
+
+
+def scrape_state_hint(*, state: ParseState, state_reason: str) -> str | None:
+    if state not in {ParseState.NETWORK_ERROR, ParseState.BLOCKED_OR_CAPTCHA}:
+        return None
+    return STATE_REASON_HINTS.get(state_reason)
+
+
+def _merge_warnings(base: list[str], extra: list[str]) -> list[str]:
+    if not extra:
+        return sorted(set(base))
+    return sorted(set(base + extra))
+
+
+def _trim_author_search_result(
+    parsed: ParsedAuthorSearchPage,
+    *,
+    limit: int,
+    extra_warnings: list[str] | None = None,
+    state_reason_override: str | None = None,
+) -> ParsedAuthorSearchPage:
+    bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))
+    return ParsedAuthorSearchPage(
+        state=parsed.state,
+        state_reason=state_reason_override or parsed.state_reason,
+        candidates=parsed.candidates[:bounded_limit],
+        marker_counts=parsed.marker_counts,
+        warnings=_merge_warnings(parsed.warnings, extra_warnings or []),
+    )
+
+
+def _policy_blocked_author_search_result(
+    *,
+    reason: str,
+    warning_codes: list[str],
+    limit: int,
+) -> ParsedAuthorSearchPage:
+    _ = limit
+    return ParsedAuthorSearchPage(
+        state=ParseState.BLOCKED_OR_CAPTCHA,
+        state_reason=reason,
+        candidates=[],
+        marker_counts={},
+        warnings=_merge_warnings([], warning_codes),
+    )

build/lib/app/services/domains/scholars/uploads.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from app.services.domains.scholars.exceptions import ScholarServiceError
+
+
+def _ensure_upload_root(upload_dir: str, *, create: bool) -> Path:
+    root = Path(upload_dir).expanduser().resolve()
+    if create:
+        root.mkdir(parents=True, exist_ok=True)
+    return root
+
+
+def _resolve_upload_path(upload_root: Path, relative_path: str) -> Path:
+    candidate = (upload_root / relative_path).resolve()
+    if upload_root != candidate and upload_root not in candidate.parents:
+        raise ScholarServiceError("Invalid scholar image path.")
+    return candidate
+
+
+def _safe_remove_upload(upload_root: Path, relative_path: str | None) -> None:
+    if not relative_path:
+        return
+    try:
+        file_path = _resolve_upload_path(upload_root, relative_path)
+    except ScholarServiceError:
+        return
+
+    try:
+        if file_path.exists() and file_path.is_file():
+            file_path.unlink()
+    except OSError:
+        return
+
+
+def resolve_upload_file_path(*, upload_dir: str, relative_path: str) -> Path:
+    root = _ensure_upload_root(upload_dir, create=False)
+    return _resolve_upload_path(root, relative_path)

build/lib/app/services/domains/scholars/validators.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+from app.services.domains.scholars.constants import MAX_IMAGE_URL_LENGTH, SCHOLAR_ID_PATTERN
+from app.services.domains.scholars.exceptions import ScholarServiceError
+
+
+def validate_scholar_id(value: str) -> str:
+    scholar_id = value.strip()
+    if not SCHOLAR_ID_PATTERN.fullmatch(scholar_id):
+        raise ScholarServiceError("Scholar ID must match [a-zA-Z0-9_-]{12}.")
+    return scholar_id
+
+
+def normalize_display_name(value: str) -> str | None:
+    normalized = value.strip()
+    return normalized if normalized else None
+
+
+def normalize_profile_image_url(value: str | None) -> str | None:
+    if value is None:
+        return None
+
+    candidate = value.strip()
+    if not candidate:
+        return None
+
+    if len(candidate) > MAX_IMAGE_URL_LENGTH:
+        raise ScholarServiceError(
+            f"Image URL must be {MAX_IMAGE_URL_LENGTH} characters or fewer."
+        )
+
+    parsed = urlparse(candidate)
+    if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc:
+        raise ScholarServiceError("Image URL must be an absolute http(s) URL.")
+
+    return candidate

build/lib/app/services/domains/settings/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

build/lib/app/services/domains/settings/application.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import UserSetting
+
+
+class UserSettingsServiceError(ValueError):
+    """Raised for expected settings-validation failures."""
+
+
+NAV_PAGE_DASHBOARD = "dashboard"
+NAV_PAGE_SCHOLARS = "scholars"
+NAV_PAGE_PUBLICATIONS = "publications"
+NAV_PAGE_SETTINGS = "settings"
+NAV_PAGE_STYLE_GUIDE = "style-guide"
+NAV_PAGE_RUNS = "runs"
+NAV_PAGE_USERS = "users"
+
+ALLOWED_NAV_PAGES = (
+    NAV_PAGE_DASHBOARD,
+    NAV_PAGE_SCHOLARS,
+    NAV_PAGE_PUBLICATIONS,
+    NAV_PAGE_SETTINGS,
+    NAV_PAGE_STYLE_GUIDE,
+    NAV_PAGE_RUNS,
+    NAV_PAGE_USERS,
+)
+REQUIRED_NAV_PAGES = (
+    NAV_PAGE_DASHBOARD,
+    NAV_PAGE_SCHOLARS,
+    NAV_PAGE_SETTINGS,
+)
+DEFAULT_NAV_VISIBLE_PAGES = list(ALLOWED_NAV_PAGES)
+HARD_MIN_RUN_INTERVAL_MINUTES = 15
+HARD_MIN_REQUEST_DELAY_SECONDS = 2
+
+
+def resolve_run_interval_minimum(configured_minimum: int | None) -> int:
+    try:
+        parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_RUN_INTERVAL_MINUTES
+    except (TypeError, ValueError):
+        parsed = HARD_MIN_RUN_INTERVAL_MINUTES
+    return max(HARD_MIN_RUN_INTERVAL_MINUTES, parsed)
+
+
+def resolve_request_delay_minimum(configured_minimum: int | None) -> int:
+    try:
+        parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_REQUEST_DELAY_SECONDS
+    except (TypeError, ValueError):
+        parsed = HARD_MIN_REQUEST_DELAY_SECONDS
+    return max(HARD_MIN_REQUEST_DELAY_SECONDS, parsed)
+
+
+def parse_run_interval_minutes(value: str, *, minimum: int = HARD_MIN_RUN_INTERVAL_MINUTES) -> int:
+    try:
+        parsed = int(value)
+    except ValueError as exc:
+        raise UserSettingsServiceError("Check interval must be a whole number.") from exc
+    effective_minimum = resolve_run_interval_minimum(minimum)
+    if parsed < effective_minimum:
+        raise UserSettingsServiceError(
+            f"Check interval must be at least {effective_minimum} minutes."
+        )
+    return parsed
+
+
+def parse_request_delay_seconds(value: str, *, minimum: int = HARD_MIN_REQUEST_DELAY_SECONDS) -> int:
+    try:
+        parsed = int(value)
+    except ValueError as exc:
+        raise UserSettingsServiceError("Request delay must be a whole number.") from exc
+    effective_minimum = resolve_request_delay_minimum(minimum)
+    if parsed < effective_minimum:
+        raise UserSettingsServiceError(
+            f"Request delay must be at least {effective_minimum} seconds."
+        )
+    return parsed
+
+
+def parse_nav_visible_pages(value: object) -> list[str]:
+    if not isinstance(value, list):
+        raise UserSettingsServiceError("Navigation visibility must be a list of page ids.")
+
+    deduped: list[str] = []
+    seen: set[str] = set()
+
+    for raw_page in value:
+        if not isinstance(raw_page, str):
+            raise UserSettingsServiceError("Navigation visibility entries must be strings.")
+
+        page_id = raw_page.strip()
+        if page_id not in ALLOWED_NAV_PAGES:
+            raise UserSettingsServiceError(f"Unsupported navigation page id: {page_id}")
+
+        if page_id in seen:
+            continue
+
+        seen.add(page_id)
+        deduped.append(page_id)
+
+    missing_required = [page for page in REQUIRED_NAV_PAGES if page not in seen]
+    if missing_required:
+        raise UserSettingsServiceError(
+            "Dashboard, Scholars, and Settings must remain visible."
+        )
+
+    return deduped
+
+
+from app.settings import settings as app_settings
+
+async def get_or_create_settings(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> UserSetting:
+    result = await db_session.execute(
+        select(UserSetting).where(UserSetting.user_id == user_id)
+    )
+    settings = result.scalar_one_or_none()
+    if settings is not None:
+        return settings
+
+    settings = UserSetting(
+        user_id=user_id,
+        openalex_api_key=app_settings.openalex_api_key,
+        crossref_api_token=app_settings.crossref_api_token,
+        crossref_api_mailto=app_settings.crossref_api_mailto,
+    )
+    db_session.add(settings)
+    await db_session.commit()
+    await db_session.refresh(settings)
+    return settings
+
+
+async def update_settings(
+    db_session: AsyncSession,
+    *,
+    settings: UserSetting,
+    auto_run_enabled: bool,
+    run_interval_minutes: int,
+    request_delay_seconds: int,
+    nav_visible_pages: list[str],
+    openalex_api_key: str | None,
+    crossref_api_token: str | None,
+    crossref_api_mailto: str | None,
+) -> UserSetting:
+    settings.auto_run_enabled = auto_run_enabled
+    settings.run_interval_minutes = run_interval_minutes
+    settings.request_delay_seconds = request_delay_seconds
+    settings.nav_visible_pages = nav_visible_pages
+    settings.openalex_api_key = openalex_api_key
+    settings.crossref_api_token = crossref_api_token
+    settings.crossref_api_mailto = crossref_api_mailto
+    await db_session.commit()
+    await db_session.refresh(settings)
+    return settings

build/lib/app/services/domains/unpaywall/__init__.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+
+async def resolve_publication_pdf_urls(*args, **kwargs):
+    from app.services.domains.unpaywall.application import resolve_publication_pdf_urls as _impl
+
+    return await _impl(*args, **kwargs)
+
+
+__all__ = ["resolve_publication_pdf_urls"]

build/lib/app/services/domains/unpaywall/application.py

@@ -0,0 +1,458 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import logging
+import re
+from typing import TYPE_CHECKING
+from urllib.parse import unquote
+
+from app.services.domains.crossref.application import discover_doi_for_publication
+from app.services.domains.doi.normalize import normalize_doi
+from app.services.domains.unpaywall.pdf_discovery import (
+    looks_like_pdf_url,
+    resolve_pdf_from_landing_page,
+)
+from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
+from app.settings import settings
+
+if TYPE_CHECKING:
+    from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+
+DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
+DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
+DOI_URL_RE = re.compile(r"(?:https?://)?(?:dx\.)?doi\.org/(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
+UNPAYWALL_URL_TEMPLATE = "https://api.unpaywall.org/v2/{doi}"
+FAILURE_MISSING_DOI = "missing_doi"
+FAILURE_NO_RECORD = "no_unpaywall_record"
+FAILURE_NO_PDF = "no_pdf_found"
+FAILURE_RESOLUTION_EXCEPTION = "resolution_exception"
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class OaResolutionOutcome:
+    publication_id: int
+    doi: str | None
+    pdf_url: str | None
+    failure_reason: str | None
+    source: str | None
+    used_crossref: bool
+
+
+def _extract_doi_candidate(text: str | None) -> str | None:
+    if not text:
+        return None
+    decoded = unquote(text)
+    match = DOI_PATTERN.search(decoded)
+    if not match:
+        return None
+    return match.group(0).rstrip(" .;,)")
+
+
+def _extract_explicit_doi(text: str | None) -> str | None:
+    if not text:
+        return None
+    decoded = unquote(text)
+    url_match = DOI_URL_RE.search(decoded)
+    if url_match:
+        return normalize_doi(url_match.group(1))
+    prefix_match = DOI_PREFIX_RE.search(decoded)
+    if prefix_match:
+        return normalize_doi(prefix_match.group(1))
+    return None
+
+
+def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str | None:
+    stored = None
+    if getattr(item, "display_identifier", None) and item.display_identifier.kind == "doi":
+        stored = normalize_doi(item.display_identifier.value)
+    
+    explicit_doi = (
+        _extract_explicit_doi(item.pub_url)
+        or _extract_explicit_doi(item.venue_text)
+    )
+    if explicit_doi:
+        return explicit_doi
+    pub_url_doi = _extract_doi_candidate(item.pub_url)
+    if pub_url_doi:
+        return normalize_doi(pub_url_doi)
+    return stored
+
+
+def _payload_locations(payload: dict) -> list[dict]:
+    locations: list[dict] = []
+    best = payload.get("best_oa_location")
+    if isinstance(best, dict):
+        locations.append(best)
+    oa_locations = payload.get("oa_locations")
+    if isinstance(oa_locations, list):
+        locations.extend(location for location in oa_locations if isinstance(location, dict))
+    return locations
+
+
+def _location_value(location: dict, key: str) -> str | None:
+    value = location.get(key)
+    if not isinstance(value, str):
+        return None
+    normalized = value.strip()
+    return normalized or None
+
+
+def _payload_pdf_candidates(payload: dict) -> list[str]:
+    candidates: list[str] = []
+    seen: set[str] = set()
+    for location in _payload_locations(payload):
+        candidate = _location_value(location, "url_for_pdf")
+        if candidate is None or candidate in seen:
+            continue
+        seen.add(candidate)
+        candidates.append(candidate)
+    return candidates
+
+
+def _payload_landing_candidates(payload: dict) -> list[str]:
+    candidates: list[str] = []
+    seen: set[str] = set()
+    for location in _payload_locations(payload):
+        candidate = _location_value(location, "url")
+        if candidate is None or candidate in seen:
+            continue
+        seen.add(candidate)
+        candidates.append(candidate)
+    return candidates
+
+
+def _crawl_targets(
+    *,
+    payload: dict,
+    pdf_candidates: list[str],
+) -> list[str]:
+    targets = _payload_landing_candidates(payload)
+    seen = set(targets)
+    for candidate in pdf_candidates:
+        if candidate in seen:
+            continue
+        targets.append(candidate)
+        seen.add(candidate)
+    doi = normalize_doi(str(payload.get("doi") or ""))
+    doi_landing_url = f"https://doi.org/{doi}" if doi else None
+    if doi_landing_url and doi_landing_url not in seen:
+        targets.append(doi_landing_url)
+    return targets
+
+
+def _has_direct_payload_pdf(payload: dict) -> bool:
+    return any(looks_like_pdf_url(candidate) for candidate in _payload_pdf_candidates(payload))
+
+
+async def _resolved_pdf_url_from_payload(
+    payload: dict,
+    *,
+    client,
+) -> str | None:
+    pdf_candidates = _payload_pdf_candidates(payload)
+    for candidate in pdf_candidates:
+        if looks_like_pdf_url(candidate):
+            return candidate
+    for page_url in _crawl_targets(payload=payload, pdf_candidates=pdf_candidates)[:3]:
+        discovered = await resolve_pdf_from_landing_page(client, page_url=page_url)
+        if discovered:
+            logger.info(
+                "unpaywall.pdf_discovered_from_landing",
+                extra={"event": "unpaywall.pdf_discovered_from_landing", "landing_url": page_url},
+            )
+            return discovered
+    return None
+
+
+async def _fetch_unpaywall_payload_by_doi(
+    *,
+    client,
+    doi: str,
+    email: str,
+) -> dict | None:
+    await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
+    headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{email})"}
+    response = await client.get(
+        UNPAYWALL_URL_TEMPLATE.format(doi=doi),
+        params={"email": email},
+        headers=headers,
+    )
+    if response.status_code != 200:
+        return None
+    payload = response.json()
+    if not isinstance(payload, dict):
+        return None
+    return payload
+
+
+def _email_for_request(request_email: str | None) -> str | None:
+    email = (request_email or "").strip() or settings.unpaywall_email.strip()
+    return email or None
+
+
+def _log_resolution_summary(
+    *,
+    publication_count: int,
+    doi_input_count: int,
+    search_attempt_count: int,
+    resolved_pdf_count: int,
+    email: str,
+) -> None:
+    logger.info(
+        "unpaywall.resolve_completed",
+        extra={
+            "event": "unpaywall.resolve_completed",
+            "publication_count": publication_count,
+            "doi_input_count": doi_input_count,
+            "search_attempt_count": search_attempt_count,
+            "resolved_pdf_count": resolved_pdf_count,
+            "email_domain": email.split("@", 1)[-1] if "@" in email else None,
+        },
+    )
+
+
+async def _resolve_item_payload(
+    *,
+    client,
+    item: PublicationListItem,
+    email: str,
+    allow_crossref: bool,
+) -> tuple[dict | None, bool, str | None]:
+    doi = _publication_doi(item)
+    payload: dict | None = None
+    if doi:
+        payload = await _fetch_unpaywall_payload_by_doi(client=client, doi=doi, email=email)
+        if payload is not None and _has_direct_payload_pdf(payload):
+            return payload, False, doi
+    if not allow_crossref or not settings.crossref_enabled:
+        return payload, False, doi
+    crossref_doi = await discover_doi_for_publication(
+        item=item,
+        max_rows=settings.crossref_max_rows,
+        email=email,
+    )
+    if crossref_doi is None or crossref_doi == doi:
+        return payload, crossref_doi is not None, doi or crossref_doi
+    crossref_payload = await _fetch_unpaywall_payload_by_doi(
+        client=client,
+        doi=crossref_doi,
+        email=email,
+    )
+    if crossref_payload is not None:
+        return crossref_payload, True, crossref_doi
+    return payload, True, crossref_doi
+
+
+async def _doi_and_pdf_from_payload(
+    payload: dict,
+    *,
+    client,
+) -> tuple[str | None, str | None]:
+    doi = normalize_doi(str(payload.get("doi") or ""))
+    return doi, await _resolved_pdf_url_from_payload(payload, client=client)
+
+
+def _resolution_targets(items: list[PublicationListItem]) -> list[PublicationListItem]:
+    return [item for item in items if not item.pdf_url]
+
+
+def _crossref_budget_value() -> int:
+    return max(int(settings.crossref_max_lookups_per_request), 0)
+
+
+def _outcome_with_failure(
+    *,
+    item: PublicationListItem,
+    failure_reason: str,
+    used_crossref: bool,
+    doi_override: str | None = None,
+) -> OaResolutionOutcome:
+    return OaResolutionOutcome(
+        publication_id=item.publication_id,
+        doi=normalize_doi(doi_override) if doi_override is not None else _publication_doi(item),
+        pdf_url=None,
+        failure_reason=failure_reason,
+        source=None,
+        used_crossref=used_crossref,
+    )
+
+
+def _missing_payload_failure_reason(item: PublicationListItem, *, used_crossref: bool) -> str:
+    if _publication_doi(item):
+        return FAILURE_NO_RECORD
+    if used_crossref:
+        return FAILURE_NO_RECORD
+    return FAILURE_MISSING_DOI
+
+
+def _source_name(*, used_crossref: bool) -> str:
+    return "crossref+unpaywall" if used_crossref else "unpaywall"
+
+
+def _outcome_from_payload(
+    *,
+    item: PublicationListItem,
+    doi: str | None,
+    pdf_url: str | None,
+    used_crossref: bool,
+) -> OaResolutionOutcome:
+    return OaResolutionOutcome(
+        publication_id=item.publication_id,
+        doi=doi,
+        pdf_url=pdf_url,
+        failure_reason=None if pdf_url else FAILURE_NO_PDF,
+        source=_source_name(used_crossref=used_crossref),
+        used_crossref=used_crossref,
+    )
+
+
+def _resolved_pdf_count(outcomes: dict[int, OaResolutionOutcome]) -> int:
+    return sum(1 for outcome in outcomes.values() if outcome.pdf_url)
+
+
+def _outcome_metadata(outcomes: dict[int, OaResolutionOutcome]) -> dict[int, tuple[str | None, str | None]]:
+    return {
+        publication_id: (outcome.doi, outcome.pdf_url)
+        for publication_id, outcome in outcomes.items()
+    }
+
+
+async def _resolve_outcome_for_item(
+    *,
+    client,
+    item: PublicationListItem,
+    email: str,
+    allow_crossref: bool,
+) -> OaResolutionOutcome:
+    payload, used_crossref, resolved_doi = await _resolve_item_payload(
+        client=client,
+        item=item,
+        email=email,
+        allow_crossref=allow_crossref,
+    )
+    if not isinstance(payload, dict):
+        return _outcome_with_failure(
+            item=item,
+            failure_reason=_missing_payload_failure_reason(item, used_crossref=used_crossref),
+            used_crossref=used_crossref,
+            doi_override=resolved_doi,
+        )
+    doi, pdf_url = await _doi_and_pdf_from_payload(payload, client=client)
+    return _outcome_from_payload(
+        item=item,
+        doi=doi,
+        pdf_url=pdf_url,
+        used_crossref=used_crossref,
+    )
+
+
+def _doi_input_count(items: list[PublicationListItem]) -> int:
+    return len([item for item in items if _publication_doi(item)])
+
+
+def _search_attempt_count(*, targets: list[PublicationListItem]) -> int:
+    return len([item for item in targets if not _publication_doi(item)])
+
+
+async def _safe_outcome_for_item(
+    *,
+    client,
+    item: PublicationListItem,
+    email: str,
+    allow_crossref: bool,
+) -> OaResolutionOutcome:
+    try:
+        return await _resolve_outcome_for_item(
+            client=client,
+            item=item,
+            email=email,
+            allow_crossref=allow_crossref,
+        )
+    except Exception as exc:  # pragma: no cover - defensive network boundary
+        logger.warning(
+            "unpaywall.resolve_item_failed",
+            extra={
+                "event": "unpaywall.resolve_item_failed",
+                "publication_id": item.publication_id,
+                "error": str(exc),
+            },
+        )
+        return _outcome_with_failure(
+            item=item,
+            failure_reason=FAILURE_RESOLUTION_EXCEPTION,
+            used_crossref=allow_crossref and settings.crossref_enabled,
+        )
+
+
+async def _resolve_outcomes_with_client(
+    *,
+    client,
+    targets: list[PublicationListItem],
+    email: str,
+) -> dict[int, OaResolutionOutcome]:
+    outcomes: dict[int, OaResolutionOutcome] = {}
+    crossref_budget = _crossref_budget_value()
+    crossref_lookups = 0
+    for item in targets:
+        allow_crossref = crossref_budget > 0 and crossref_lookups < crossref_budget
+        outcome = await _safe_outcome_for_item(
+            client=client,
+            item=item,
+            email=email,
+            allow_crossref=allow_crossref,
+        )
+        if outcome.used_crossref:
+            crossref_lookups += 1
+        outcomes[item.publication_id] = outcome
+    return outcomes
+
+
+async def resolve_publication_oa_metadata(
+    items: list[PublicationListItem],
+    *,
+    request_email: str | None = None,
+) -> dict[int, tuple[str | None, str | None]]:
+    outcomes = await resolve_publication_oa_outcomes(items, request_email=request_email)
+    return _outcome_metadata(outcomes)
+
+
+async def resolve_publication_oa_outcomes(
+    items: list[PublicationListItem],
+    *,
+    request_email: str | None = None,
+) -> dict[int, OaResolutionOutcome]:
+    if not settings.unpaywall_enabled:
+        return {}
+    email = _email_for_request(request_email)
+    if email is None:
+        logger.debug("unpaywall.resolve_skipped_missing_email")
+        return {}
+    import httpx
+
+    timeout_seconds = max(float(settings.unpaywall_timeout_seconds), 0.5)
+    targets = _resolution_targets(items)[: max(int(settings.unpaywall_max_items_per_request), 0)]
+    headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{email})"}
+    async with httpx.AsyncClient(timeout=timeout_seconds, follow_redirects=True, headers=headers) as client:
+        outcomes = await _resolve_outcomes_with_client(
+            client=client,
+            targets=targets,
+            email=email,
+        )
+    _log_resolution_summary(
+        publication_count=len(items),
+        doi_input_count=_doi_input_count(items),
+        search_attempt_count=_search_attempt_count(targets=targets),
+        resolved_pdf_count=_resolved_pdf_count(outcomes),
+        email=email,
+    )
+    return outcomes
+
+
+async def resolve_publication_pdf_urls(
+    items: list[PublicationListItem],
+    *,
+    request_email: str | None = None,
+) -> dict[int, str | None]:
+    resolved = await resolve_publication_oa_metadata(items, request_email=request_email)
+    return {publication_id: pdf for publication_id, (_doi, pdf) in resolved.items()}

build/lib/app/services/domains/unpaywall/pdf_discovery.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+from html.parser import HTMLParser
+import re
+from urllib.parse import urljoin, urlparse
+
+from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
+from app.settings import settings
+
+PDF_MIME = "application/pdf"
+URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.I)
+
+
+class _LandingPdfParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.base_href: str | None = None
+        self.candidates: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        attrs_map = {key.lower(): (value or "").strip() for key, value in attrs}
+        if tag == "base":
+            self.base_href = attrs_map.get("href") or self.base_href
+            return
+        if tag == "meta":
+            self._append_meta_candidate(attrs_map)
+            return
+        if tag == "link":
+            self._append_link_candidate(attrs_map)
+            return
+        if tag == "a":
+            self._append_anchor_candidate(attrs_map)
+
+    def _append_meta_candidate(self, attrs_map: dict[str, str]) -> None:
+        meta_name = (attrs_map.get("name") or attrs_map.get("property") or "").lower()
+        if meta_name != "citation_pdf_url":
+            return
+        content = attrs_map.get("content")
+        if content:
+            self.candidates.append(content)
+
+    def _append_link_candidate(self, attrs_map: dict[str, str]) -> None:
+        href = attrs_map.get("href")
+        link_type = (attrs_map.get("type") or "").lower()
+        if href and "pdf" in link_type:
+            self.candidates.append(href)
+
+    def _append_anchor_candidate(self, attrs_map: dict[str, str]) -> None:
+        href = attrs_map.get("href")
+        if href:
+            self.candidates.append(href)
+
+
+def looks_like_pdf_url(url: str | None) -> bool:
+    if not isinstance(url, str):
+        return False
+    value = url.strip()
+    if not value:
+        return False
+    parsed = urlparse(value)
+    path = (parsed.path or "").lower()
+    query = (parsed.query or "").lower()
+    return path.endswith(".pdf") or ".pdf" in query
+
+
+def _normalized_candidate_urls(*, page_url: str, html: str) -> list[str]:
+    parser = _LandingPdfParser()
+    parser.feed(html)
+    parser.close()
+    base_url = urljoin(page_url, parser.base_href) if parser.base_href else page_url
+    raw_candidates = [*parser.candidates, *_text_url_candidates(html)]
+    deduped: list[str] = []
+    seen: set[str] = set()
+    for raw in raw_candidates:
+        absolute = urljoin(base_url, raw.strip())
+        parsed = urlparse(absolute)
+        if parsed.scheme not in {"http", "https"}:
+            continue
+        if absolute in seen:
+            continue
+        seen.add(absolute)
+        deduped.append(absolute)
+    return sorted(deduped, key=_candidate_sort_key)
+
+
+def _text_url_candidates(html: str) -> list[str]:
+    candidates: list[str] = []
+    for match in URL_RE.findall(html):
+        cleaned = match.rstrip(".,);]}>")
+        if "pdf" not in cleaned.lower():
+            continue
+        candidates.append(cleaned)
+    return candidates
+
+
+def _candidate_sort_key(candidate: str) -> tuple[int, str]:
+    lowered = candidate.lower()
+    if looks_like_pdf_url(candidate):
+        return (0, lowered)
+    if any(token in lowered for token in ("doi", "full", "article", "download")):
+        return (1, lowered)
+    return (2, lowered)
+
+
+def _is_html_response(response) -> bool:
+    content_type = str(response.headers.get("content-type") or "").lower()
+    return "text/html" in content_type or "application/xhtml+xml" in content_type
+
+
+async def _fetch_page_html(client, *, page_url: str) -> str | None:
+    await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
+    response = await client.get(page_url, follow_redirects=True)
+    if response.status_code != 200 or not _is_html_response(response):
+        return None
+    text = response.text or ""
+    return text[: max(int(settings.unpaywall_pdf_discovery_max_html_bytes), 0)]
+
+
+async def _candidate_is_pdf(client, *, candidate_url: str) -> bool:
+    if looks_like_pdf_url(candidate_url):
+        return True
+    await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
+    response = await client.get(candidate_url, follow_redirects=True)
+    content_type = str(response.headers.get("content-type") or "").lower()
+    return response.status_code == 200 and PDF_MIME in content_type
+
+
+def _candidate_limit() -> int:
+    return max(int(settings.unpaywall_pdf_discovery_max_candidates), 1)
+
+
+async def _resolve_pdf_from_candidate_page(client, *, candidate_url: str) -> str | None:
+    html = await _fetch_page_html(client, page_url=candidate_url)
+    if not html:
+        return None
+    nested_candidates = _normalized_candidate_urls(page_url=candidate_url, html=html)
+    for nested in nested_candidates[: _candidate_limit()]:
+        if await _candidate_is_pdf(client, candidate_url=nested):
+            return nested
+    return None
+
+
+async def resolve_pdf_from_landing_page(client, *, page_url: str) -> str | None:
+    if not settings.unpaywall_pdf_discovery_enabled:
+        return None
+    html = await _fetch_page_html(client, page_url=page_url)
+    if not html:
+        return None
+    candidates = _normalized_candidate_urls(page_url=page_url, html=html)
+    for candidate in candidates[: _candidate_limit()]:
+        if await _candidate_is_pdf(client, candidate_url=candidate):
+            return candidate
+        nested_pdf = await _resolve_pdf_from_candidate_page(client, candidate_url=candidate)
+        if nested_pdf:
+            return nested_pdf
+    return None

build/lib/app/services/domains/unpaywall/rate_limit.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import asyncio
+import time
+
+_REQUEST_LOCK = asyncio.Lock()
+_LAST_REQUEST_AT = 0.0
+
+
+async def wait_for_unpaywall_slot(*, min_interval_seconds: float) -> None:
+    global _LAST_REQUEST_AT
+    interval = max(float(min_interval_seconds), 0.0)
+    async with _REQUEST_LOCK:
+        elapsed = time.monotonic() - _LAST_REQUEST_AT
+        remaining = interval - elapsed
+        if remaining > 0:
+            await asyncio.sleep(remaining)
+        _LAST_REQUEST_AT = time.monotonic()

build/lib/app/services/domains/users/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

build/lib/app/services/domains/users/application.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import re
+
+from sqlalchemy import select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import User
+
+EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
+
+
+class UserServiceError(ValueError):
+    """Raised for expected user-management validation failures."""
+
+
+def normalize_email(value: str) -> str:
+    return value.strip().lower()
+
+
+def validate_email(value: str) -> str:
+    email = normalize_email(value)
+    if not EMAIL_PATTERN.fullmatch(email):
+        raise UserServiceError("Enter a valid email address.")
+    return email
+
+
+def validate_password(value: str) -> str:
+    password = value.strip()
+    if len(password) < 8:
+        raise UserServiceError("Password must be at least 8 characters.")
+    return password
+
+
+async def get_user_by_id(db_session: AsyncSession, user_id: int) -> User | None:
+    result = await db_session.execute(select(User).where(User.id == user_id))
+    return result.scalar_one_or_none()
+
+
+async def get_user_by_email(db_session: AsyncSession, email: str) -> User | None:
+    result = await db_session.execute(
+        select(User).where(User.email == normalize_email(email))
+    )
+    return result.scalar_one_or_none()
+
+
+async def list_users(db_session: AsyncSession) -> list[User]:
+    result = await db_session.execute(select(User).order_by(User.email.asc()))
+    return list(result.scalars().all())
+
+
+async def create_user(
+    db_session: AsyncSession,
+    *,
+    email: str,
+    password_hash: str,
+    is_admin: bool,
+) -> User:
+    user = User(
+        email=validate_email(email),
+        password_hash=password_hash,
+        is_admin=is_admin,
+        is_active=True,
+    )
+    db_session.add(user)
+    try:
+        await db_session.commit()
+    except IntegrityError as exc:
+        await db_session.rollback()
+        raise UserServiceError("A user with that email already exists.") from exc
+    await db_session.refresh(user)
+    return user
+
+
+async def set_user_active(
+    db_session: AsyncSession,
+    *,
+    user: User,
+    is_active: bool,
+) -> User:
+    user.is_active = is_active
+    await db_session.commit()
+    await db_session.refresh(user)
+    return user
+
+
+async def set_user_password_hash(
+    db_session: AsyncSession,
+    *,
+    user: User,
+    password_hash: str,
+) -> User:
+    user.password_hash = password_hash
+    await db_session.commit()
+    await db_session.refresh(user)
+    return user
+