Merge pull request #10 from JustinZeus/feat/failfast-parser-pdf-import-export-dashboard-sync

feat: refactor backend services and add direct PDF links, import/expo…

README.md

@@ -78,6 +78,15 @@ docker compose -f docker-compose.yml -f docker-compose.dev.yml down
 - Scholar tracking is user-scoped: each account can track the same Scholar ID independently.
 - Publications are shared/global records deduplicated by Scholar cluster ID and normalized fingerprint.
 - Per-account visibility and read state is stored on scholar-publication links, not on the global publication row.
+- Publication records include both canonical Scholar detail URLs (`pub_url`) and direct download targets (`pdf_url`) when available.
+
+## Backend Architecture (Refactored)
+
+- `app/services/ingestion.py`: run orchestration, page retry/pagination logic, publication upsert/dedup, safety counters, continuation queue sync.
+- `app/services/scholar_parser.py`: strict DOM parsing with explicit layout-state detection and fail-fast warning codes for malformed Scholar markup.
+- `app/services/scheduler.py`: automatic/scheduled ingestion runner plus continuation queue draining and retry/drop policy.
+- `app/services/publications.py`: user-scoped publication listing/read-state update queries used by dashboard/publication UIs.
+- `app/services/import_export.py`: JSON import/export for tracked scholars and publication link state.
 
 ## Name Search Status
 
@@ -272,7 +281,13 @@ Scheduled fixture probes run in GitHub Actions via `.github/workflows/scheduled-
 - `GET /api/v1/publications` supports `mode=all|unread|latest` (plus temporary alias `mode=new`).
 - `unread` = actionable read-state (`is_read=false`).
 - `latest` = discovery-state (`first seen in the latest completed check`).
+- Publications include `pub_url` and `pdf_url` in API responses. `pdf_url` is a direct-download target extracted from Scholar result-side links when present.
 - Response counters:
   - `unread_count`: unread publications in current scope.
   - `latest_count`: publications discovered in latest completed check.
   - `new_count`: compatibility alias for `latest_count` (temporary).
+
+### Scholar Data Portability
+
+- `GET /api/v1/scholars/export`: exports tracked scholars plus scholar-publication link data as JSON.
+- `POST /api/v1/scholars/import`: imports that JSON payload, upserting scholars, global publication records, and per-scholar read state.

agents.md

@@ -24,5 +24,12 @@ These limits prevent IP bans and are not to be optimized away.
 
 ## 4. Current Environment & Stack
 * **Backend:** Python 3.12+, FastAPI, SQLAlchemy (Async/asyncpg), Alembic.
-* **Frontend:** TypeScript, React, Vite.
+* **Frontend:** TypeScript, Vue 3, Vite.
 * **Infrastructure:** Multi-stage Docker.
+
+## 5. Refactored Service Boundaries (Current)
+* **`app/services/scholar_parser.py`:** Parser contract is fail-fast. Layout drift must emit explicit `layout_*` reasons/warnings, never silent partial success.
+* **`app/services/ingestion.py`:** Orchestrates ingestion runs; validate parser outputs before persistence; enforce publication candidate constraints before upsert.
+* **`app/services/publications.py`:** Publication list/read-state query layer; include both `pub_url` and `pdf_url` for UI consumption.
+* **`app/services/import_export.py`:** Handles JSON import/export for user-scoped scholars and scholar-publication link state while preserving global publication dedup rules.
+* **`app/services/scheduler.py`:** Owns automatic runs and continuation queue retries/drops; do not bypass safety gate or cooldown logic.

app/api/routers/publications.py

@@ -94,6 +94,7 @@ async def list_publications(
                     "citation_count": item.citation_count,
                     "venue_text": item.venue_text,
                     "pub_url": item.pub_url,
+                    "pdf_url": item.pdf_url,
                     "is_read": item.is_read,
                     "first_seen_at": item.first_seen_at,
                     "is_new_in_latest_run": item.is_new_in_latest_run,

app/api/routers/scholars.py

@@ -13,6 +13,9 @@ from app.api.errors import ApiException
 from app.api.responses import success_payload
 from app.api.runtime_deps import get_scholar_source
 from app.api.schemas import (
+    DataExportEnvelope,
+    DataImportEnvelope,
+    DataImportRequest,
     MessageEnvelope,
     ScholarCreateRequest,
     ScholarEnvelope,
@@ -22,6 +25,7 @@ from app.api.schemas import (
 )
 from app.db.models import User
 from app.db.session import get_db_session
+from app.services import import_export as import_export_service
 from app.services import scholars as scholar_service
 from app.services.scholar_source import ScholarSource
 from app.settings import settings
@@ -81,6 +85,68 @@ async def list_scholars(
     )
 
 
+@router.get(
+    "/export",
+    response_model=DataExportEnvelope,
+)
+async def export_scholars_and_publications(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    data = await import_export_service.export_user_data(
+        db_session,
+        user_id=current_user.id,
+    )
+    return success_payload(request, data=data)
+
+
+@router.post(
+    "/import",
+    response_model=DataImportEnvelope,
+)
+async def import_scholars_and_publications(
+    payload: DataImportRequest,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    if (
+        payload.schema_version is not None
+        and int(payload.schema_version) != import_export_service.EXPORT_SCHEMA_VERSION
+    ):
+        raise ApiException(
+            status_code=400,
+            code="invalid_import_schema_version",
+            message=(
+                "Import schema version is not supported. "
+                f"Expected {import_export_service.EXPORT_SCHEMA_VERSION}."
+            ),
+        )
+    try:
+        result = await import_export_service.import_user_data(
+            db_session,
+            user_id=current_user.id,
+            scholars=[item.model_dump() for item in payload.scholars],
+            publications=[item.model_dump() for item in payload.publications],
+        )
+    except import_export_service.ImportExportError as exc:
+        raise ApiException(
+            status_code=400,
+            code="invalid_import_payload",
+            message=str(exc),
+        ) from exc
+    logger.info(
+        "api.scholars.imported",
+        extra={
+            "event": "api.scholars.imported",
+            "user_id": current_user.id,
+            **result,
+        },
+    )
+    return success_payload(request, data=result)
+
+
 @router.post(
     "",
     response_model=ScholarEnvelope,

app/api/schemas.py

@@ -186,6 +186,74 @@ class ScholarImageUrlUpdateRequest(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
 
+class ScholarExportItemData(BaseModel):
+    scholar_id: str
+    display_name: str | None = None
+    is_enabled: bool = True
+    profile_image_override_url: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationExportItemData(BaseModel):
+    scholar_id: str
+    cluster_id: str | None = None
+    fingerprint_sha256: str | None = None
+    title: str
+    year: int | None = None
+    citation_count: int = 0
+    author_text: str | None = None
+    venue_text: str | None = None
+    pub_url: str | None = None
+    pdf_url: str | None = None
+    is_read: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataExportData(BaseModel):
+    schema_version: int
+    exported_at: str
+    scholars: list[ScholarExportItemData]
+    publications: list[PublicationExportItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataExportEnvelope(BaseModel):
+    data: DataExportData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportRequest(BaseModel):
+    schema_version: int | None = None
+    scholars: list[ScholarExportItemData] = Field(default_factory=list)
+    publications: list[PublicationExportItemData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportResultData(BaseModel):
+    scholars_created: int
+    scholars_updated: int
+    publications_created: int
+    publications_updated: int
+    links_created: int
+    links_updated: int
+    skipped_records: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportEnvelope(BaseModel):
+    data: DataImportResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
 class RunListItemData(BaseModel):
     id: int
     trigger_type: str
@@ -505,6 +573,7 @@ class PublicationItemData(BaseModel):
     citation_count: int
     venue_text: str | None
     pub_url: str | None
+    pdf_url: str | None
     is_read: bool
     first_seen_at: datetime
     is_new_in_latest_run: bool

app/main.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from contextlib import asynccontextmanager
+import logging
 from pathlib import Path
 
 from fastapi import FastAPI, HTTPException
@@ -23,6 +24,9 @@ from app.security.csrf import CSRFMiddleware
 from app.services.scheduler import SchedulerService
 from app.settings import settings
 
+logger = logging.getLogger(__name__)
+BUILD_MARKER = "2026-02-19.phase2.direct-pdf-import-export-dashboard-sync"
+
 configure_logging(
     level=settings.log_level,
     log_format=settings.log_format,
@@ -45,8 +49,22 @@ scheduler_service = SchedulerService(
 )
 
 
+def _log_startup_build_marker() -> None:
+    logger.info(
+        "app.startup_build_marker",
+        extra={
+            "event": "app.startup_build_marker",
+            "build_marker": BUILD_MARKER,
+            "frontend_enabled": settings.frontend_enabled,
+            "scheduler_enabled": settings.scheduler_enabled,
+            "log_format": settings.log_format,
+        },
+    )
+
+
 @asynccontextmanager
 async def lifespan(_: FastAPI):
+    _log_startup_build_marker()
     await scheduler_service.start()
     yield
     await scheduler_service.stop()

app/services/continuation_queue.py

@@ -41,28 +41,32 @@ def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_second
     return min(seconds, maximum)
 
 
-async def upsert_job(
+async def _get_item_for_user_scholar(
     db_session: AsyncSession,
     *,
     user_id: int,
     scholar_profile_id: int,
-    resume_cstart: int,
+) -> IngestionQueueItem | None:
-    reason: str,
-    run_id: int | None,
-    delay_seconds: int,
-) -> IngestionQueueItem:
-    now = datetime.now(timezone.utc)
-    next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds)))
     result = await db_session.execute(
         select(IngestionQueueItem).where(
             IngestionQueueItem.user_id == user_id,
             IngestionQueueItem.scholar_profile_id == scholar_profile_id,
         )
     )
-    item = result.scalar_one_or_none()
+    return result.scalar_one_or_none()
-    normalized_cstart = normalize_cstart(resume_cstart)
+
-    if item is None:
+
-        item = IngestionQueueItem(
+def _build_queue_item(
+    *,
+    now: datetime,
+    user_id: int,
+    scholar_profile_id: int,
+    normalized_cstart: int,
+    reason: str,
+    run_id: int | None,
+    next_attempt_dt: datetime,
+) -> IngestionQueueItem:
+    return IngestionQueueItem(
         user_id=user_id,
         scholar_profile_id=scholar_profile_id,
         resume_cstart=normalized_cstart,
@@ -77,9 +81,17 @@ async def upsert_job(
         created_at=now,
         updated_at=now,
     )
-        db_session.add(item)
-        return item
 
+
+def _update_existing_queue_item(
+    *,
+    item: IngestionQueueItem,
+    now: datetime,
+    normalized_cstart: int,
+    reason: str,
+    run_id: int | None,
+    next_attempt_dt: datetime,
+) -> None:
     item.resume_cstart = normalized_cstart
     item.reason = reason
     if item.status == QueueItemStatus.DROPPED.value:
@@ -91,6 +103,47 @@ async def upsert_job(
     item.dropped_reason = None
     item.dropped_at = None
     item.updated_at = now
+
+
+async def upsert_job(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+    resume_cstart: int,
+    reason: str,
+    run_id: int | None,
+    delay_seconds: int,
+) -> IngestionQueueItem:
+    now = datetime.now(timezone.utc)
+    next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds)))
+    item = await _get_item_for_user_scholar(
+        db_session,
+        user_id=user_id,
+        scholar_profile_id=scholar_profile_id,
+    )
+    normalized_cstart = normalize_cstart(resume_cstart)
+    if item is None:
+        item = _build_queue_item(
+            now=now,
+            user_id=user_id,
+            scholar_profile_id=scholar_profile_id,
+            normalized_cstart=normalized_cstart,
+            reason=reason,
+            run_id=run_id,
+            next_attempt_dt=next_attempt_dt,
+        )
+        db_session.add(item)
+        return item
+
+    _update_existing_queue_item(
+        item=item,
+        now=now,
+        normalized_cstart=normalized_cstart,
+        reason=reason,
+        run_id=run_id,
+        next_attempt_dt=next_attempt_dt,
+    )
     return item
 
 

app/services/import_export.py

@@ -0,0 +1,667 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timezone
+import hashlib
+import re
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import Publication, ScholarProfile, ScholarPublication
+from app.services import scholars as scholar_service
+from app.services.ingestion import build_publication_url, normalize_title
+
+EXPORT_SCHEMA_VERSION = 1
+MAX_IMPORT_SCHOLARS = 10_000
+MAX_IMPORT_PUBLICATIONS = 100_000
+WORD_RE = re.compile(r"[a-z0-9]+")
+SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
+
+
+class ImportExportError(ValueError):
+    """Raised when import/export payload constraints are violated."""
+
+
+@dataclass(frozen=True)
+class ImportedPublicationInput:
+    profile: ScholarProfile
+    title: str
+    year: int | None
+    citation_count: int
+    author_text: str | None
+    venue_text: str | None
+    cluster_id: str | None
+    pub_url: str | None
+    pdf_url: str | None
+    fingerprint: str
+    is_read: bool
+
+
+def _normalize_optional_text(value: Any) -> str | None:
+    if value is None:
+        return None
+    normalized = str(value).strip()
+    return normalized or None
+
+
+def _normalize_optional_year(value: Any) -> int | None:
+    if value is None:
+        return None
+    try:
+        year = int(value)
+    except (TypeError, ValueError):
+        return None
+    if year < 1500 or year > 3000:
+        return None
+    return year
+
+
+def _normalize_citation_count(value: Any) -> int:
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError):
+        return 0
+    return max(0, parsed)
+
+
+def _first_author_last_name(authors_text: str | None) -> str:
+    if not authors_text:
+        return ""
+    first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
+    words = WORD_RE.findall(first_author)
+    if not words:
+        return ""
+    return words[-1]
+
+
+def _first_venue_word(venue_text: str | None) -> str:
+    if not venue_text:
+        return ""
+    words = WORD_RE.findall(venue_text.lower())
+    if not words:
+        return ""
+    return words[0]
+
+
+def _build_fingerprint(
+    *,
+    title: str,
+    year: int | None,
+    author_text: str | None,
+    venue_text: str | None,
+) -> str:
+    canonical = "|".join(
+        [
+            normalize_title(title),
+            str(year) if year is not None else "",
+            _first_author_last_name(author_text),
+            _first_venue_word(venue_text),
+        ]
+    )
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _resolve_fingerprint(
+    *,
+    title: str,
+    year: int | None,
+    author_text: str | None,
+    venue_text: str | None,
+    provided_fingerprint: Any,
+) -> str:
+    normalized = _normalize_optional_text(provided_fingerprint)
+    if normalized and SHA256_RE.fullmatch(normalized.lower()):
+        return normalized.lower()
+    return _build_fingerprint(
+        title=title,
+        year=year,
+        author_text=author_text,
+        venue_text=venue_text,
+    )
+
+
+def _validate_import_sizes(
+    *,
+    scholars: list[dict[str, Any]],
+    publications: list[dict[str, Any]],
+) -> None:
+    if len(scholars) > MAX_IMPORT_SCHOLARS:
+        raise ImportExportError(f"Import exceeds max scholars ({MAX_IMPORT_SCHOLARS}).")
+    if len(publications) > MAX_IMPORT_PUBLICATIONS:
+        raise ImportExportError(
+            f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS})."
+        )
+
+
+async def _load_user_scholar_map(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> dict[str, ScholarProfile]:
+    result = await db_session.execute(
+        select(ScholarProfile).where(ScholarProfile.user_id == user_id)
+    )
+    profiles = list(result.scalars().all())
+    return {profile.scholar_id: profile for profile in profiles}
+
+
+def _apply_imported_scholar_values(
+    *,
+    profile: ScholarProfile,
+    display_name: str | None,
+    profile_image_override_url: str | None,
+    is_enabled: bool,
+) -> bool:
+    updated = False
+    if display_name and profile.display_name != display_name:
+        profile.display_name = display_name
+        updated = True
+    if profile.profile_image_override_url != profile_image_override_url:
+        profile.profile_image_override_url = profile_image_override_url
+        updated = True
+    if bool(profile.is_enabled) != bool(is_enabled):
+        profile.is_enabled = bool(is_enabled)
+        updated = True
+    return updated
+
+
+def _new_scholar_profile(
+    *,
+    user_id: int,
+    scholar_id: str,
+    display_name: str | None,
+    profile_image_override_url: str | None,
+    is_enabled: bool,
+) -> ScholarProfile:
+    return ScholarProfile(
+        user_id=user_id,
+        scholar_id=scholar_id,
+        display_name=display_name,
+        profile_image_override_url=profile_image_override_url,
+        is_enabled=bool(is_enabled),
+    )
+
+
+async def _upsert_imported_scholars(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholars: list[dict[str, Any]],
+) -> tuple[dict[str, ScholarProfile], dict[str, int]]:
+    scholar_map = await _load_user_scholar_map(db_session, user_id=user_id)
+    counters = {"scholars_created": 0, "scholars_updated": 0, "skipped_records": 0}
+    for item in scholars:
+        try:
+            scholar_id = scholar_service.validate_scholar_id(str(item["scholar_id"]))
+            display_name = scholar_service.normalize_display_name(str(item.get("display_name") or ""))
+            override_url = scholar_service.normalize_profile_image_url(
+                _normalize_optional_text(item.get("profile_image_override_url"))
+            )
+        except (KeyError, scholar_service.ScholarServiceError):
+            counters["skipped_records"] += 1
+            continue
+        is_enabled = bool(item.get("is_enabled", True))
+        existing = scholar_map.get(scholar_id)
+        if existing is None:
+            profile = _new_scholar_profile(
+                user_id=user_id,
+                scholar_id=scholar_id,
+                display_name=display_name,
+                profile_image_override_url=override_url,
+                is_enabled=is_enabled,
+            )
+            db_session.add(profile)
+            scholar_map[scholar_id] = profile
+            counters["scholars_created"] += 1
+            continue
+        if _apply_imported_scholar_values(
+            profile=existing,
+            display_name=display_name,
+            profile_image_override_url=override_url,
+            is_enabled=is_enabled,
+        ):
+            counters["scholars_updated"] += 1
+    await db_session.flush()
+    return scholar_map, counters
+
+
+async def _find_publication_by_cluster(
+    db_session: AsyncSession,
+    *,
+    cluster_id: str,
+) -> Publication | None:
+    result = await db_session.execute(
+        select(Publication).where(Publication.cluster_id == cluster_id)
+    )
+    return result.scalar_one_or_none()
+
+
+async def _find_publication_by_fingerprint(
+    db_session: AsyncSession,
+    *,
+    fingerprint_sha256: str,
+) -> Publication | None:
+    result = await db_session.execute(
+        select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256)
+    )
+    return result.scalar_one_or_none()
+
+
+def _apply_imported_publication_values(
+    *,
+    publication: Publication,
+    title: str,
+    year: int | None,
+    citation_count: int,
+    author_text: str | None,
+    venue_text: str | None,
+    pub_url: str | None,
+    pdf_url: str | None,
+    cluster_id: str | None,
+) -> bool:
+    updated = False
+    if cluster_id and publication.cluster_id != cluster_id:
+        publication.cluster_id = cluster_id
+        updated = True
+    if publication.title_raw != title:
+        publication.title_raw = title
+        publication.title_normalized = normalize_title(title)
+        updated = True
+    if publication.year != year:
+        publication.year = year
+        updated = True
+    if int(publication.citation_count or 0) != citation_count:
+        publication.citation_count = citation_count
+        updated = True
+    if publication.author_text != author_text:
+        publication.author_text = author_text
+        updated = True
+    if publication.venue_text != venue_text:
+        publication.venue_text = venue_text
+        updated = True
+    if pub_url and publication.pub_url != pub_url:
+        publication.pub_url = pub_url
+        updated = True
+    if pdf_url and publication.pdf_url != pdf_url:
+        publication.pdf_url = pdf_url
+        updated = True
+    return updated
+
+
+def _new_publication(
+    *,
+    cluster_id: str | None,
+    fingerprint_sha256: str,
+    title: str,
+    year: int | None,
+    citation_count: int,
+    author_text: str | None,
+    venue_text: str | None,
+    pub_url: str | None,
+    pdf_url: str | None,
+) -> Publication:
+    return Publication(
+        cluster_id=cluster_id,
+        fingerprint_sha256=fingerprint_sha256,
+        title_raw=title,
+        title_normalized=normalize_title(title),
+        year=year,
+        citation_count=citation_count,
+        author_text=author_text,
+        venue_text=venue_text,
+        pub_url=pub_url,
+        pdf_url=pdf_url,
+    )
+
+
+async def _resolve_publication_for_import(
+    db_session: AsyncSession,
+    *,
+    cluster_id: str | None,
+    fingerprint_sha256: str,
+    cluster_cache: dict[str, Publication | None],
+    fingerprint_cache: dict[str, Publication | None],
+) -> Publication | None:
+    if cluster_id:
+        if cluster_id not in cluster_cache:
+            cluster_cache[cluster_id] = await _find_publication_by_cluster(
+                db_session,
+                cluster_id=cluster_id,
+            )
+        if cluster_cache[cluster_id] is not None:
+            return cluster_cache[cluster_id]
+    if fingerprint_sha256 not in fingerprint_cache:
+        fingerprint_cache[fingerprint_sha256] = await _find_publication_by_fingerprint(
+            db_session,
+            fingerprint_sha256=fingerprint_sha256,
+        )
+    return fingerprint_cache[fingerprint_sha256]
+
+
+async def _upsert_scholar_publication_link(
+    db_session: AsyncSession,
+    *,
+    scholar_profile_id: int,
+    publication_id: int,
+    is_read: bool,
+) -> tuple[bool, bool]:
+    result = await db_session.execute(
+        select(ScholarPublication).where(
+            ScholarPublication.scholar_profile_id == scholar_profile_id,
+            ScholarPublication.publication_id == publication_id,
+        )
+    )
+    link = result.scalar_one_or_none()
+    if link is None:
+        db_session.add(
+            ScholarPublication(
+                scholar_profile_id=scholar_profile_id,
+                publication_id=publication_id,
+                is_read=bool(is_read),
+            )
+        )
+        return True, False
+    if bool(link.is_read) == bool(is_read):
+        return False, False
+    link.is_read = bool(is_read)
+    return False, True
+
+
+def _initialize_import_counters(counters: dict[str, int]) -> None:
+    counters.update(
+        {
+            "publications_created": 0,
+            "publications_updated": 0,
+            "links_created": 0,
+            "links_updated": 0,
+        }
+    )
+
+
+def _build_imported_publication_input(
+    *,
+    item: dict[str, Any],
+    scholar_map: dict[str, ScholarProfile],
+) -> ImportedPublicationInput | None:
+    scholar_id = _normalize_optional_text(item.get("scholar_id"))
+    title = _normalize_optional_text(item.get("title"))
+    if not scholar_id or not title:
+        return None
+    profile = scholar_map.get(scholar_id)
+    if profile is None:
+        return None
+    year = _normalize_optional_year(item.get("year"))
+    author_text = _normalize_optional_text(item.get("author_text"))
+    venue_text = _normalize_optional_text(item.get("venue_text"))
+    return ImportedPublicationInput(
+        profile=profile,
+        title=title,
+        year=year,
+        citation_count=_normalize_citation_count(item.get("citation_count")),
+        author_text=author_text,
+        venue_text=venue_text,
+        cluster_id=_normalize_optional_text(item.get("cluster_id")),
+        pub_url=build_publication_url(_normalize_optional_text(item.get("pub_url"))),
+        pdf_url=build_publication_url(_normalize_optional_text(item.get("pdf_url"))),
+        fingerprint=_resolve_fingerprint(
+            title=title,
+            year=year,
+            author_text=author_text,
+            venue_text=venue_text,
+            provided_fingerprint=item.get("fingerprint_sha256"),
+        ),
+        is_read=bool(item.get("is_read", False)),
+    )
+
+
+def _update_link_counters(
+    *,
+    counters: dict[str, int],
+    link_created: bool,
+    link_updated: bool,
+) -> None:
+    if link_created:
+        counters["links_created"] += 1
+    if link_updated:
+        counters["links_updated"] += 1
+
+
+def _cache_resolved_publication(
+    *,
+    publication: Publication,
+    cluster_id: str | None,
+    fingerprint_sha256: str,
+    cluster_cache: dict[str, Publication | None],
+    fingerprint_cache: dict[str, Publication | None],
+) -> None:
+    if cluster_id:
+        cluster_cache[cluster_id] = publication
+    fingerprint_cache[fingerprint_sha256] = publication
+
+
+async def _create_import_publication(
+    db_session: AsyncSession,
+    *,
+    payload: ImportedPublicationInput,
+) -> Publication:
+    publication = _new_publication(
+        cluster_id=payload.cluster_id,
+        fingerprint_sha256=payload.fingerprint,
+        title=payload.title,
+        year=payload.year,
+        citation_count=payload.citation_count,
+        author_text=payload.author_text,
+        venue_text=payload.venue_text,
+        pub_url=payload.pub_url,
+        pdf_url=payload.pdf_url,
+    )
+    db_session.add(publication)
+    await db_session.flush()
+    return publication
+
+
+def _update_import_publication(
+    *,
+    publication: Publication,
+    payload: ImportedPublicationInput,
+) -> bool:
+    return _apply_imported_publication_values(
+        publication=publication,
+        title=payload.title,
+        year=payload.year,
+        citation_count=payload.citation_count,
+        author_text=payload.author_text,
+        venue_text=payload.venue_text,
+        pub_url=payload.pub_url,
+        pdf_url=payload.pdf_url,
+        cluster_id=payload.cluster_id,
+    )
+
+
+async def _upsert_publication_entity(
+    db_session: AsyncSession,
+    *,
+    payload: ImportedPublicationInput,
+    cluster_cache: dict[str, Publication | None],
+    fingerprint_cache: dict[str, Publication | None],
+) -> tuple[Publication, bool, bool]:
+    publication = await _resolve_publication_for_import(
+        db_session,
+        cluster_id=payload.cluster_id,
+        fingerprint_sha256=payload.fingerprint,
+        cluster_cache=cluster_cache,
+        fingerprint_cache=fingerprint_cache,
+    )
+    created = False
+    updated = False
+    if publication is None:
+        publication = await _create_import_publication(
+            db_session,
+            payload=payload,
+        )
+        created = True
+    else:
+        updated = _update_import_publication(
+            publication=publication,
+            payload=payload,
+        )
+    _cache_resolved_publication(
+        publication=publication,
+        cluster_id=payload.cluster_id,
+        fingerprint_sha256=payload.fingerprint,
+        cluster_cache=cluster_cache,
+        fingerprint_cache=fingerprint_cache,
+    )
+    return publication, created, updated
+
+
+async def _upsert_imported_publication(
+    db_session: AsyncSession,
+    *,
+    payload: ImportedPublicationInput,
+    cluster_cache: dict[str, Publication | None],
+    fingerprint_cache: dict[str, Publication | None],
+    counters: dict[str, int],
+) -> None:
+    publication, created, updated = await _upsert_publication_entity(
+        db_session,
+        payload=payload,
+        cluster_cache=cluster_cache,
+        fingerprint_cache=fingerprint_cache,
+    )
+    if created:
+        counters["publications_created"] += 1
+    if updated:
+        counters["publications_updated"] += 1
+    link_created, link_updated = await _upsert_scholar_publication_link(
+        db_session,
+        scholar_profile_id=int(payload.profile.id),
+        publication_id=int(publication.id),
+        is_read=payload.is_read,
+    )
+    _update_link_counters(
+        counters=counters,
+        link_created=link_created,
+        link_updated=link_updated,
+    )
+
+
+def _exported_at_iso() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+
+
+def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]:
+    return {
+        "scholar_id": profile.scholar_id,
+        "display_name": profile.display_name,
+        "is_enabled": bool(profile.is_enabled),
+        "profile_image_override_url": profile.profile_image_override_url,
+    }
+
+
+def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]:
+    (
+        scholar_id,
+        cluster_id,
+        fingerprint_sha256,
+        title_raw,
+        year,
+        citation_count,
+        author_text,
+        venue_text,
+        pub_url,
+        pdf_url,
+        is_read,
+    ) = row
+    return {
+        "scholar_id": scholar_id,
+        "cluster_id": cluster_id,
+        "fingerprint_sha256": fingerprint_sha256,
+        "title": title_raw,
+        "year": year,
+        "citation_count": int(citation_count or 0),
+        "author_text": author_text,
+        "venue_text": venue_text,
+        "pub_url": pub_url,
+        "pdf_url": pdf_url,
+        "is_read": bool(is_read),
+    }
+
+
+async def export_user_data(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> dict[str, Any]:
+    scholars_result = await db_session.execute(
+        select(ScholarProfile)
+        .where(ScholarProfile.user_id == user_id)
+        .order_by(ScholarProfile.id.asc())
+    )
+    publication_result = await db_session.execute(
+        select(
+            ScholarProfile.scholar_id,
+            Publication.cluster_id,
+            Publication.fingerprint_sha256,
+            Publication.title_raw,
+            Publication.year,
+            Publication.citation_count,
+            Publication.author_text,
+            Publication.venue_text,
+            Publication.pub_url,
+            Publication.pdf_url,
+            ScholarPublication.is_read,
+        )
+        .join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id)
+        .join(Publication, Publication.id == ScholarPublication.publication_id)
+        .where(ScholarProfile.user_id == user_id)
+        .order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
+    )
+    scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()]
+    publications = [
+        _serialize_export_publication(row)
+        for row in publication_result.all()
+    ]
+    return {
+        "schema_version": EXPORT_SCHEMA_VERSION,
+        "exported_at": _exported_at_iso(),
+        "scholars": scholars,
+        "publications": publications,
+    }
+
+
+async def import_user_data(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholars: list[dict[str, Any]],
+    publications: list[dict[str, Any]],
+) -> dict[str, int]:
+    _validate_import_sizes(scholars=scholars, publications=publications)
+    scholar_map, counters = await _upsert_imported_scholars(
+        db_session,
+        user_id=user_id,
+        scholars=scholars,
+    )
+    cluster_cache: dict[str, Publication | None] = {}
+    fingerprint_cache: dict[str, Publication | None] = {}
+    _initialize_import_counters(counters)
+    for item in publications:
+        parsed_item = _build_imported_publication_input(
+            item=item,
+            scholar_map=scholar_map,
+        )
+        if parsed_item is None:
+            counters["skipped_records"] += 1
+            continue
+        await _upsert_imported_publication(
+            db_session,
+            payload=parsed_item,
+            cluster_cache=cluster_cache,
+            fingerprint_cache=fingerprint_cache,
+            counters=counters,
+        )
+    await db_session.commit()
+    return counters

app/services/publications.py

@@ -30,6 +30,7 @@ class PublicationListItem:
     citation_count: int
     venue_text: str | None
     pub_url: str | None
+    pdf_url: str | None
     is_read: bool
     first_seen_at: datetime
     is_new_in_latest_run: bool
@@ -45,6 +46,7 @@ class UnreadPublicationItem:
     citation_count: int
     venue_text: str | None
     pub_url: str | None
+    pdf_url: str | None
 
 
 def resolve_publication_view_mode(value: str | None) -> str:
@@ -95,6 +97,7 @@ def publications_query(
             Publication.citation_count,
             Publication.venue_text,
             Publication.pub_url,
+            Publication.pdf_url,
             ScholarPublication.is_read,
             ScholarPublication.first_seen_run_id,
             ScholarPublication.created_at,
@@ -118,6 +121,44 @@ def publications_query(
     return stmt
 
 
+def _publication_list_item_from_row(
+    row: tuple,
+    *,
+    latest_run_id: int | None,
+) -> PublicationListItem:
+    (
+        publication_id,
+        scholar_profile_id,
+        display_name,
+        scholar_id,
+        title_raw,
+        year,
+        citation_count,
+        venue_text,
+        pub_url,
+        pdf_url,
+        is_read,
+        first_seen_run_id,
+        created_at,
+    ) = row
+    return PublicationListItem(
+        publication_id=int(publication_id),
+        scholar_profile_id=int(scholar_profile_id),
+        scholar_label=(display_name or scholar_id),
+        title=title_raw,
+        year=year,
+        citation_count=int(citation_count or 0),
+        venue_text=venue_text,
+        pub_url=pub_url,
+        pdf_url=pdf_url,
+        is_read=bool(is_read),
+        first_seen_at=created_at,
+        is_new_in_latest_run=(
+            latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
+        ),
+    )
+
+
 async def list_for_user(
     db_session: AsyncSession,
     *,
@@ -140,42 +181,10 @@ async def list_for_user(
             limit=limit,
         )
     )
-
+    return [
-    rows = result.all()
+        _publication_list_item_from_row(row, latest_run_id=latest_run_id)
-    items: list[PublicationListItem] = []
+        for row in result.all()
-    for row in rows:
+    ]
-        (
-            publication_id,
-            scholar_profile_id,
-            display_name,
-            scholar_id,
-            title_raw,
-            year,
-            citation_count,
-            venue_text,
-            pub_url,
-            is_read,
-            first_seen_run_id,
-            created_at,
-        ) = row
-        items.append(
-            PublicationListItem(
-                publication_id=int(publication_id),
-                scholar_profile_id=int(scholar_profile_id),
-                scholar_label=(display_name or scholar_id),
-                title=title_raw,
-                year=year,
-                citation_count=int(citation_count or 0),
-                venue_text=venue_text,
-                pub_url=pub_url,
-                is_read=bool(is_read),
-                first_seen_at=created_at,
-                is_new_in_latest_run=(
-                    latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
-                ),
-            )
-        )
-    return items
 
 
 async def list_unread_for_user(
@@ -206,6 +215,7 @@ async def list_unread_for_user(
             citation_count,
             venue_text,
             pub_url,
+            pdf_url,
             _is_read,
             _first_seen_run_id,
             _created_at,
@@ -220,6 +230,7 @@ async def list_unread_for_user(
                 citation_count=int(citation_count or 0),
                 venue_text=venue_text,
                 pub_url=pub_url,
+                pdf_url=pdf_url,
             )
         )
     return items
@@ -248,6 +259,7 @@ async def list_new_for_latest_run_for_user(
             citation_count=row.citation_count,
             venue_text=row.venue_text,
             pub_url=row.pub_url,
+            pdf_url=row.pdf_url,
         )
         for row in rows
     ]

app/services/run_safety.py

@@ -146,6 +146,63 @@ def register_cooldown_blocked_start(
     return get_safety_state_payload(settings, now_utc=now)
 
 
+def _update_run_counters(
+    *,
+    counters: dict[str, Any],
+    run_id: int,
+    blocked_failure_count: int,
+    network_failure_count: int,
+) -> tuple[int, int]:
+    bounded_blocked_failures = max(0, int(blocked_failure_count))
+    bounded_network_failures = max(0, int(network_failure_count))
+    counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures
+    counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures
+    counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id)
+    counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = (
+        int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1
+        if bounded_blocked_failures > 0
+        else 0
+    )
+    counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = (
+        int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1
+        if bounded_network_failures > 0
+        else 0
+    )
+    return bounded_blocked_failures, bounded_network_failures
+
+
+def _resolve_cooldown_trigger(
+    *,
+    blocked_failures: int,
+    network_failures: int,
+    blocked_failure_threshold: int,
+    network_failure_threshold: int,
+    blocked_cooldown_seconds: int,
+    network_cooldown_seconds: int,
+) -> tuple[str | None, int]:
+    if blocked_failures >= max(1, int(blocked_failure_threshold)):
+        return COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD, max(60, int(blocked_cooldown_seconds))
+    if network_failures >= max(1, int(network_failure_threshold)):
+        return COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD, max(60, int(network_cooldown_seconds))
+    return None, 0
+
+
+def _apply_cooldown_decision(
+    *,
+    settings: UserSetting,
+    counters: dict[str, Any],
+    now: datetime,
+    reason: str | None,
+    cooldown_seconds: int,
+) -> None:
+    if reason is None:
+        clear_expired_cooldown(settings, now_utc=now)
+        return
+    settings.scrape_cooldown_reason = reason
+    settings.scrape_cooldown_until = now + timedelta(seconds=max(60, int(cooldown_seconds)))
+    counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1
+
+
 def apply_run_safety_outcome(
     settings: UserSetting,
     *,
@@ -160,41 +217,27 @@ def apply_run_safety_outcome(
 ) -> tuple[dict[str, Any], str | None]:
     now = now_utc or _utcnow()
     counters = _counters_from_state(settings)
-
+    blocked_failures, network_failures = _update_run_counters(
-    bounded_blocked_failures = max(0, int(blocked_failure_count))
+        counters=counters,
-    bounded_network_failures = max(0, int(network_failure_count))
+        run_id=run_id,
-
+        blocked_failure_count=blocked_failure_count,
-    counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures
+        network_failure_count=network_failure_count,
-    counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures
-    counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id)
-    counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = (
-        int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1
-        if bounded_blocked_failures > 0
-        else 0
     )
-    counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = (
+    reason, cooldown_seconds = _resolve_cooldown_trigger(
-        int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1
+        blocked_failures=blocked_failures,
-        if bounded_network_failures > 0
+        network_failures=network_failures,
-        else 0
+        blocked_failure_threshold=blocked_failure_threshold,
+        network_failure_threshold=network_failure_threshold,
+        blocked_cooldown_seconds=blocked_cooldown_seconds,
+        network_cooldown_seconds=network_cooldown_seconds,
+    )
+    _apply_cooldown_decision(
+        settings=settings,
+        counters=counters,
+        now=now,
+        reason=reason,
+        cooldown_seconds=cooldown_seconds,
     )
-
-    reason: str | None = None
-    cooldown_seconds = 0
-
-    if bounded_blocked_failures >= max(1, int(blocked_failure_threshold)):
-        reason = COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD
-        cooldown_seconds = max(60, int(blocked_cooldown_seconds))
-    elif bounded_network_failures >= max(1, int(network_failure_threshold)):
-        reason = COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD
-        cooldown_seconds = max(60, int(network_cooldown_seconds))
-
-    if reason is not None:
-        settings.scrape_cooldown_reason = reason
-        settings.scrape_cooldown_until = now + timedelta(seconds=cooldown_seconds)
-        counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1
-    else:
-        clear_expired_cooldown(settings, now_utc=now)
-
     settings.scrape_safety_state = counters
     return get_safety_state_payload(settings, now_utc=now), reason
 

app/services/runs.py

@@ -129,6 +129,70 @@ def extract_run_summary(error_log: object) -> dict[str, Any]:
     }
 
 
+def _queue_item_columns() -> tuple:
+    return (
+        IngestionQueueItem.id,
+        IngestionQueueItem.scholar_profile_id,
+        ScholarProfile.display_name,
+        ScholarProfile.scholar_id,
+        IngestionQueueItem.status,
+        IngestionQueueItem.reason,
+        IngestionQueueItem.dropped_reason,
+        IngestionQueueItem.attempt_count,
+        IngestionQueueItem.resume_cstart,
+        IngestionQueueItem.next_attempt_dt,
+        IngestionQueueItem.updated_at,
+        IngestionQueueItem.last_error,
+        IngestionQueueItem.last_run_id,
+    )
+
+
+def _queue_item_select(*, user_id: int):
+    return (
+        select(*_queue_item_columns())
+        .join(
+            ScholarProfile,
+            and_(
+                ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
+                ScholarProfile.user_id == IngestionQueueItem.user_id,
+            ),
+        )
+        .where(IngestionQueueItem.user_id == user_id)
+    )
+
+
+def _queue_list_item_from_row(row: tuple) -> QueueListItem:
+    (
+        item_id,
+        scholar_profile_id,
+        display_name,
+        scholar_id,
+        status,
+        reason,
+        dropped_reason,
+        attempt_count,
+        resume_cstart,
+        next_attempt_dt,
+        updated_at,
+        last_error,
+        last_run_id,
+    ) = row
+    return QueueListItem(
+        id=int(item_id),
+        scholar_profile_id=int(scholar_profile_id),
+        scholar_label=(display_name or scholar_id),
+        status=str(status),
+        reason=str(reason),
+        dropped_reason=dropped_reason,
+        attempt_count=int(attempt_count or 0),
+        resume_cstart=int(resume_cstart or 0),
+        next_attempt_dt=next_attempt_dt,
+        updated_at=updated_at,
+        last_error=last_error,
+        last_run_id=int(last_run_id) if last_run_id is not None else None,
+    )
+
+
 async def list_recent_runs_for_user(
     db_session: AsyncSession,
     *,
@@ -209,29 +273,7 @@ async def list_queue_items_for_user(
     limit: int = 200,
 ) -> list[QueueListItem]:
     result = await db_session.execute(
-        select(
+        _queue_item_select(user_id=user_id)
-            IngestionQueueItem.id,
-            IngestionQueueItem.scholar_profile_id,
-            ScholarProfile.display_name,
-            ScholarProfile.scholar_id,
-            IngestionQueueItem.status,
-            IngestionQueueItem.reason,
-            IngestionQueueItem.dropped_reason,
-            IngestionQueueItem.attempt_count,
-            IngestionQueueItem.resume_cstart,
-            IngestionQueueItem.next_attempt_dt,
-            IngestionQueueItem.updated_at,
-            IngestionQueueItem.last_error,
-            IngestionQueueItem.last_run_id,
-        )
-        .join(
-            ScholarProfile,
-            and_(
-                ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
-                ScholarProfile.user_id == IngestionQueueItem.user_id,
-            ),
-        )
-        .where(IngestionQueueItem.user_id == user_id)
         .order_by(
             case((IngestionQueueItem.status == "dropped", 1), else_=0).asc(),
             IngestionQueueItem.next_attempt_dt.asc(),
@@ -239,41 +281,7 @@ async def list_queue_items_for_user(
         )
         .limit(limit)
     )
-
+    return [_queue_list_item_from_row(row) for row in result.all()]
-    items: list[QueueListItem] = []
-    for row in result.all():
-        (
-            item_id,
-            scholar_profile_id,
-            display_name,
-            scholar_id,
-            status,
-            reason,
-            dropped_reason,
-            attempt_count,
-            resume_cstart,
-            next_attempt_dt,
-            updated_at,
-            last_error,
-            last_run_id,
-        ) = row
-        items.append(
-            QueueListItem(
-                id=int(item_id),
-                scholar_profile_id=int(scholar_profile_id),
-                scholar_label=(display_name or scholar_id),
-                status=str(status),
-                reason=str(reason),
-                dropped_reason=dropped_reason,
-                attempt_count=int(attempt_count or 0),
-                resume_cstart=int(resume_cstart or 0),
-                next_attempt_dt=next_attempt_dt,
-                updated_at=updated_at,
-                last_error=last_error,
-                last_run_id=int(last_run_id) if last_run_id is not None else None,
-            )
-        )
-    return items
 
 
 async def get_queue_item_for_user(
@@ -283,66 +291,14 @@ async def get_queue_item_for_user(
     queue_item_id: int,
 ) -> QueueListItem | None:
     result = await db_session.execute(
-        select(
+        _queue_item_select(user_id=user_id)
-            IngestionQueueItem.id,
+        .where(IngestionQueueItem.id == queue_item_id)
-            IngestionQueueItem.scholar_profile_id,
-            ScholarProfile.display_name,
-            ScholarProfile.scholar_id,
-            IngestionQueueItem.status,
-            IngestionQueueItem.reason,
-            IngestionQueueItem.dropped_reason,
-            IngestionQueueItem.attempt_count,
-            IngestionQueueItem.resume_cstart,
-            IngestionQueueItem.next_attempt_dt,
-            IngestionQueueItem.updated_at,
-            IngestionQueueItem.last_error,
-            IngestionQueueItem.last_run_id,
-        )
-        .join(
-            ScholarProfile,
-            and_(
-                ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
-                ScholarProfile.user_id == IngestionQueueItem.user_id,
-            ),
-        )
-        .where(
-            IngestionQueueItem.user_id == user_id,
-            IngestionQueueItem.id == queue_item_id,
-        )
         .limit(1)
     )
     row = result.one_or_none()
     if row is None:
         return None
-    (
+    return _queue_list_item_from_row(row)
-        item_id,
-        scholar_profile_id,
-        display_name,
-        scholar_id,
-        status,
-        reason,
-        dropped_reason,
-        attempt_count,
-        resume_cstart,
-        next_attempt_dt,
-        updated_at,
-        last_error,
-        last_run_id,
-    ) = row
-    return QueueListItem(
-        id=int(item_id),
-        scholar_profile_id=int(scholar_profile_id),
-        scholar_label=(display_name or scholar_id),
-        status=str(status),
-        reason=str(reason),
-        dropped_reason=dropped_reason,
-        attempt_count=int(attempt_count or 0),
-        resume_cstart=int(resume_cstart or 0),
-        next_attempt_dt=next_attempt_dt,
-        updated_at=updated_at,
-        last_error=last_error,
-        last_run_id=int(last_run_id) if last_run_id is not None else None,
-    )
 
 
 async def retry_queue_item_for_user(

app/services/scheduler.py

@@ -139,10 +139,7 @@ class SchedulerService:
                 continue
             await self._run_candidate(candidate)
 
-    async def _load_candidates(self) -> list[_AutoRunCandidate]:
+    async def _load_candidate_rows(self) -> list[tuple]:
-        if not settings.ingestion_automation_allowed:
-            return []
-
         session_factory = get_session_factory()
         async with session_factory() as session:
             result = await session.execute(
@@ -154,16 +151,14 @@ class SchedulerService:
                     UserSetting.scrape_cooldown_reason,
                 )
                 .join(User, User.id == UserSetting.user_id)
-                .where(
+                .where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True))
-                    User.is_active.is_(True),
-                    UserSetting.auto_run_enabled.is_(True),
-                )
                 .order_by(UserSetting.user_id.asc())
             )
-            rows = result.all()
+            return result.all()
-        now_utc = datetime.now(timezone.utc)
+
-        candidates: list[_AutoRunCandidate] = []
+    @staticmethod
-        for user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason in rows:
+    def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None:
+        user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row
         if cooldown_until is not None and cooldown_until.tzinfo is None:
             cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
         if cooldown_until is not None and cooldown_until > now_utc:
@@ -179,16 +174,25 @@ class SchedulerService:
                     "metric_value": 1,
                 },
             )
-                continue
+            return None
-            candidates.append(
+        return _AutoRunCandidate(
-                _AutoRunCandidate(
             user_id=int(user_id),
             run_interval_minutes=int(run_interval_minutes),
             request_delay_seconds=int(request_delay_seconds),
             cooldown_until=cooldown_until,
             cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
         )
-            )
+
+    async def _load_candidates(self) -> list[_AutoRunCandidate]:
+        if not settings.ingestion_automation_allowed:
+            return []
+        rows = await self._load_candidate_rows()
+        now_utc = datetime.now(timezone.utc)
+        candidates: list[_AutoRunCandidate] = []
+        for row in rows:
+            candidate = self._candidate_from_row(row, now_utc=now_utc)
+            if candidate is not None:
+                candidates.append(candidate)
         return candidates
 
     async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
@@ -212,12 +216,16 @@ class SchedulerService:
         )
         return now >= next_due_dt
 
-    async def _run_candidate(self, candidate: _AutoRunCandidate) -> None:
+    async def _run_candidate_ingestion(
+        self,
+        *,
+        candidate: _AutoRunCandidate,
+    ):
         session_factory = get_session_factory()
         async with session_factory() as session:
             ingestion = ScholarIngestionService(source=self._source)
             try:
-                run_summary = await ingestion.run_for_user(
+                return await ingestion.run_for_user(
                     session,
                     user_id=candidate.user_id,
                     trigger_type=RunTriggerType.SCHEDULED,
@@ -234,14 +242,8 @@ class SchedulerService:
                 )
             except RunAlreadyInProgressError:
                 await session.rollback()
-                logger.info(
+                logger.info("scheduler.run_skipped_locked", extra={"event": "scheduler.run_skipped_locked", "user_id": candidate.user_id})
-                    "scheduler.run_skipped_locked",
+                return None
-                    extra={
-                        "event": "scheduler.run_skipped_locked",
-                        "user_id": candidate.user_id,
-                    },
-                )
-                return
             except RunBlockedBySafetyPolicyError as exc:
                 await session.rollback()
                 logger.info(
@@ -256,18 +258,16 @@ class SchedulerService:
                         "metric_value": 1,
                     },
                 )
-                return
+                return None
             except Exception:
                 await session.rollback()
-                logger.exception(
+                logger.exception("scheduler.run_failed", extra={"event": "scheduler.run_failed", "user_id": candidate.user_id})
-                    "scheduler.run_failed",
+                return None
-                    extra={
-                        "event": "scheduler.run_failed",
-                        "user_id": candidate.user_id,
-                    },
-                )
-                return
 
+    async def _run_candidate(self, candidate: _AutoRunCandidate) -> None:
+        run_summary = await self._run_candidate_ingestion(candidate=candidate)
+        if run_summary is None:
+            return
         logger.info(
             "scheduler.run_completed",
             extra={
@@ -292,8 +292,12 @@ class SchedulerService:
         for job in jobs:
             await self._run_queue_job(job)
 
-    async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
+    async def _drop_queue_job_if_max_attempts(
-        if job.attempt_count >= self._continuation_max_attempts:
+        self,
+        job: queue_service.ContinuationQueueJob,
+    ) -> bool:
+        if job.attempt_count < self._continuation_max_attempts:
+            return False
         session_factory = get_session_factory()
         async with session_factory() as session:
             dropped = await queue_service.mark_dropped(
@@ -314,19 +318,27 @@ class SchedulerService:
                     "max_attempts": self._continuation_max_attempts,
                 },
             )
-            return
+        return True
 
+    async def _mark_queue_job_retrying(
+        self,
+        job: queue_service.ContinuationQueueJob,
+    ) -> bool:
         session_factory = get_session_factory()
         async with session_factory() as session:
             queue_item = await queue_service.mark_retrying(session, job_id=job.id)
+            await session.commit()
         if queue_item is None:
-                await session.commit()
+            return False
-                return
         if queue_item.status == QueueItemStatus.DROPPED.value:
-                await session.commit()
+            return False
-                return
+        return True
-            await session.commit()
 
+    async def _queue_job_has_available_scholar(
+        self,
+        job: queue_service.ContinuationQueueJob,
+    ) -> bool:
+        session_factory = get_session_factory()
         async with session_factory() as session:
             scholar_result = await session.execute(
                 select(ScholarProfile.id).where(
@@ -336,7 +348,8 @@ class SchedulerService:
                 )
             )
             scholar_id = scholar_result.scalar_one_or_none()
-            if scholar_id is None:
+            if scholar_id is not None:
+                return True
             dropped = await queue_service.mark_dropped(
                 session,
                 job_id=job.id,
@@ -353,36 +366,10 @@ class SchedulerService:
                     "scholar_profile_id": job.scholar_profile_id,
                 },
             )
-                return
+        return False
 
-        async with session_factory() as session:
+    async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
-            request_delay_seconds = await self._load_request_delay_for_user(
+        session_factory = get_session_factory()
-                session,
-                user_id=job.user_id,
-            )
-            ingestion = ScholarIngestionService(source=self._source)
-            try:
-                run_summary = await ingestion.run_for_user(
-                    session,
-                    user_id=job.user_id,
-                    trigger_type=RunTriggerType.SCHEDULED,
-                    request_delay_seconds=request_delay_seconds,
-                    network_error_retries=self._network_error_retries,
-                    retry_backoff_seconds=self._retry_backoff_seconds,
-                    max_pages_per_scholar=self._max_pages_per_scholar,
-                    page_size=self._page_size,
-                    scholar_profile_ids={job.scholar_profile_id},
-                    start_cstart_by_scholar_id={
-                        job.scholar_profile_id: job.resume_cstart,
-                    },
-                    auto_queue_continuations=self._continuation_queue_enabled,
-                    queue_delay_seconds=self._continuation_base_delay_seconds,
-                    alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
-                    alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
-                    alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
-                )
-            except RunAlreadyInProgressError:
-                await session.rollback()
         async with session_factory() as recovery_session:
             await queue_service.reschedule_job(
                 recovery_session,
@@ -394,19 +381,19 @@ class SchedulerService:
             await recovery_session.commit()
         logger.info(
             "scheduler.queue_item_deferred_lock",
-                    extra={
+            extra={"event": "scheduler.queue_item_deferred_lock", "queue_item_id": job.id, "user_id": job.user_id},
-                        "event": "scheduler.queue_item_deferred_lock",
-                        "queue_item_id": job.id,
-                        "user_id": job.user_id,
-                    },
         )
-                return
+
-            except RunBlockedBySafetyPolicyError as exc:
+    async def _reschedule_queue_job_safety_cooldown(
-                await session.rollback()
+        self,
+        job: queue_service.ContinuationQueueJob,
+        exc: RunBlockedBySafetyPolicyError,
+    ) -> None:
         cooldown_remaining_seconds = max(
             self._tick_seconds,
             int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
         )
+        session_factory = get_session_factory()
         async with session_factory() as recovery_session:
             await queue_service.reschedule_job(
                 recovery_session,
@@ -428,14 +415,16 @@ class SchedulerService:
                 "metric_value": 1,
             },
         )
-                return
+
-            except Exception as exc:
+    async def _reschedule_queue_job_after_exception(
-                await session.rollback()
+        self,
+        job: queue_service.ContinuationQueueJob,
+        *,
+        exc: Exception,
+    ) -> None:
+        session_factory = get_session_factory()
         async with session_factory() as recovery_session:
-                    queue_item = await queue_service.increment_attempt_count(
+            queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
-                        recovery_session,
-                        job_id=job.id,
-                    )
             if queue_item is None:
                 await recovery_session.commit()
                 return
@@ -449,12 +438,7 @@ class SchedulerService:
                 await recovery_session.commit()
                 logger.warning(
                     "scheduler.queue_item_dropped_after_exception",
-                            extra={
+                    extra={"event": "scheduler.queue_item_dropped_after_exception", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count},
-                                "event": "scheduler.queue_item_dropped_after_exception",
-                                "queue_item_id": job.id,
-                                "user_id": job.user_id,
-                                "attempt_count": queue_item.attempt_count,
-                            },
                 )
                 return
             delay_seconds = queue_service.compute_backoff_seconds(
@@ -470,114 +454,119 @@ class SchedulerService:
                 error=str(exc),
             )
             await recovery_session.commit()
-                logger.exception(
+        logger.exception("scheduler.queue_item_run_failed", extra={"event": "scheduler.queue_item_run_failed", "queue_item_id": job.id, "user_id": job.user_id})
-                    "scheduler.queue_item_run_failed",
-                    extra={
-                        "event": "scheduler.queue_item_run_failed",
-                        "queue_item_id": job.id,
-                        "user_id": job.user_id,
-                    },
-                )
-                return
 
+    async def _run_ingestion_for_queue_job(
+        self,
+        job: queue_service.ContinuationQueueJob,
+    ):
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id)
+            ingestion = ScholarIngestionService(source=self._source)
+            try:
+                return await ingestion.run_for_user(
+                    session,
+                    user_id=job.user_id,
+                    trigger_type=RunTriggerType.SCHEDULED,
+                    request_delay_seconds=request_delay_seconds,
+                    network_error_retries=self._network_error_retries,
+                    retry_backoff_seconds=self._retry_backoff_seconds,
+                    max_pages_per_scholar=self._max_pages_per_scholar,
+                    page_size=self._page_size,
+                    scholar_profile_ids={job.scholar_profile_id},
+                    start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart},
+                    auto_queue_continuations=self._continuation_queue_enabled,
+                    queue_delay_seconds=self._continuation_base_delay_seconds,
+                    alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
+                    alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
+                    alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
+                )
+            except RunAlreadyInProgressError:
+                await session.rollback()
+                await self._reschedule_queue_job_lock_active(job)
+            except RunBlockedBySafetyPolicyError as exc:
+                await session.rollback()
+                await self._reschedule_queue_job_safety_cooldown(job, exc)
+            except Exception as exc:
+                await session.rollback()
+                await self._reschedule_queue_job_after_exception(job, exc=exc)
+        return None
+
+    @staticmethod
+    def _log_queue_item_resolved(
+        *,
+        event_name: str,
+        job: queue_service.ContinuationQueueJob,
+        run_summary,
+        attempt_count: int | None = None,
+        delay_seconds: int | None = None,
+    ) -> None:
+        payload = {
+            "event": event_name,
+            "queue_item_id": job.id,
+            "user_id": job.user_id,
+            "run_id": run_summary.crawl_run_id,
+            "status": run_summary.status.value,
+        }
+        if attempt_count is not None:
+            payload["attempt_count"] = attempt_count
+        if delay_seconds is not None:
+            payload["delay_seconds"] = delay_seconds
+        logger.info(event_name, extra=payload)
+
+    async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
+        session_factory = get_session_factory()
         async with session_factory() as session:
-            # Failed-attempt budget should advance only when continuation execution fails.
             if int(run_summary.failed_count) <= 0:
-                queue_item = await queue_service.reset_attempt_count(
+                queue_item = await queue_service.reset_attempt_count(session, job_id=job.id)
-                    session,
+                await session.commit()
-                    job_id=job.id,
+                if queue_item is None:
+                    self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
+                    return
+                self._log_queue_item_resolved(
+                    event_name="scheduler.queue_item_progressed",
+                    job=job,
+                    run_summary=run_summary,
+                    attempt_count=int(queue_item.attempt_count),
                 )
+                return
+            queue_item = await queue_service.increment_attempt_count(session, job_id=job.id)
             if queue_item is None:
                 await session.commit()
-                    logger.info(
+                self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
-                        "scheduler.queue_item_resolved",
-                        extra={
-                            "event": "scheduler.queue_item_resolved",
-                            "queue_item_id": job.id,
-                            "user_id": job.user_id,
-                            "run_id": run_summary.crawl_run_id,
-                            "status": run_summary.status.value,
-                        },
-                    )
                 return
-                await session.commit()
-                logger.info(
-                    "scheduler.queue_item_progressed",
-                    extra={
-                        "event": "scheduler.queue_item_progressed",
-                        "queue_item_id": job.id,
-                        "user_id": job.user_id,
-                        "attempt_count": int(queue_item.attempt_count),
-                        "run_id": run_summary.crawl_run_id,
-                        "status": run_summary.status.value,
-                    },
-                )
-                return
-
-            queue_item = await queue_service.increment_attempt_count(
-                session,
-                job_id=job.id,
-            )
-            if queue_item is None:
-                await session.commit()
-                logger.info(
-                    "scheduler.queue_item_resolved",
-                    extra={
-                        "event": "scheduler.queue_item_resolved",
-                        "queue_item_id": job.id,
-                        "user_id": job.user_id,
-                        "run_id": run_summary.crawl_run_id,
-                        "status": run_summary.status.value,
-                    },
-                )
-                return
-
             if int(queue_item.attempt_count) >= self._continuation_max_attempts:
-                await queue_service.mark_dropped(
+                await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
-                    session,
-                    job_id=job.id,
-                    reason="max_attempts_after_run",
-                )
                 await session.commit()
                 logger.warning(
                     "scheduler.queue_item_dropped_max_attempts_after_run",
-                    extra={
+                    extra={"event": "scheduler.queue_item_dropped_max_attempts_after_run", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count, "run_id": run_summary.crawl_run_id, "status": run_summary.status.value},
-                        "event": "scheduler.queue_item_dropped_max_attempts_after_run",
-                        "queue_item_id": job.id,
-                        "user_id": job.user_id,
-                        "attempt_count": queue_item.attempt_count,
-                        "run_id": run_summary.crawl_run_id,
-                        "status": run_summary.status.value,
-                    },
                 )
                 return
-
+            delay_seconds = queue_service.compute_backoff_seconds(base_seconds=self._continuation_base_delay_seconds, attempt_count=int(queue_item.attempt_count), max_seconds=self._continuation_max_delay_seconds)
-            delay_seconds = queue_service.compute_backoff_seconds(
+            await queue_service.reschedule_job(session, job_id=job.id, delay_seconds=delay_seconds, reason=queue_item.reason, error=queue_item.last_error)
-                base_seconds=self._continuation_base_delay_seconds,
-                attempt_count=int(queue_item.attempt_count),
-                max_seconds=self._continuation_max_delay_seconds,
-            )
-            await queue_service.reschedule_job(
-                session,
-                job_id=job.id,
-                delay_seconds=delay_seconds,
-                reason=queue_item.reason,
-                error=queue_item.last_error,
-            )
             await session.commit()
-            logger.info(
+        self._log_queue_item_resolved(
-                "scheduler.queue_item_rescheduled_failed",
+            event_name="scheduler.queue_item_rescheduled_failed",
-                extra={
+            job=job,
-                    "event": "scheduler.queue_item_rescheduled_failed",
+            run_summary=run_summary,
-                    "queue_item_id": job.id,
+            attempt_count=int(queue_item.attempt_count),
-                    "user_id": job.user_id,
+            delay_seconds=delay_seconds,
-                    "attempt_count": queue_item.attempt_count,
-                    "delay_seconds": delay_seconds,
-                    "run_id": run_summary.crawl_run_id,
-                    "status": run_summary.status.value,
-                },
         )
 
+    async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
+        if await self._drop_queue_job_if_max_attempts(job):
+            return
+        if not await self._mark_queue_job_retrying(job):
+            return
+        if not await self._queue_job_has_available_scholar(job):
+            return
+        run_summary = await self._run_ingestion_for_queue_job(job)
+        if run_summary is None:
+            return
+        await self._finalize_queue_job_after_run(job, run_summary)
+
     async def _load_request_delay_for_user(
         self,
         db_session: AsyncSession,

app/services/scholar_parser.py

@@ -79,6 +79,18 @@ SHOW_MORE_BUTTON_RE = re.compile(
     r"<button\b[^>]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>",
     re.I | re.S,
 )
+PROFILE_ROW_PARSER_DIRECT_MARKERS = (
+    "gs_ggs",
+    "gs_ggsd",
+    "gs_ggsa",
+    "gs_or_ggsm",
+)
+PROFILE_ROW_DIRECT_LABEL_TOKENS = (
+    "pdf",
+    "[pdf]",
+    "full text",
+    "download",
+)
 
 
 class ParseState(StrEnum):
@@ -98,6 +110,7 @@ class PublicationCandidate:
     citation_count: int | None
     authors_text: str | None
     venue_text: str | None
+    pdf_url: str | None
 
 
 @dataclass(frozen=True)
@@ -174,6 +187,7 @@ class ScholarRowParser(HTMLParser):
     def __init__(self) -> None:
         super().__init__(convert_charrefs=True)
         self.title_href: str | None = None
+        self.direct_download_href: str | None = None
         self.title_parts: list[str] = []
         self.citation_parts: list[str] = []
         self.year_parts: list[str] = []
@@ -183,6 +197,13 @@ class ScholarRowParser(HTMLParser):
         self._citation_depth = 0
         self._year_depth = 0
         self._gray_stack: list[dict[str, Any]] = []
+        self._direct_marker_depth = 0
+        self._aux_link_stack: list[dict[str, Any]] = []
+
+    @staticmethod
+    def _contains_direct_marker(classes: str) -> bool:
+        lowered = classes.lower()
+        return any(marker in lowered for marker in PROFILE_ROW_PARSER_DIRECT_MARKERS)
 
     def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
         if self._title_depth > 0:
@@ -193,8 +214,14 @@ class ScholarRowParser(HTMLParser):
             self._year_depth += 1
         if self._gray_stack:
             self._gray_stack[-1]["depth"] += 1
+        if self._direct_marker_depth > 0:
+            self._direct_marker_depth += 1
+        if self._aux_link_stack:
+            self._aux_link_stack[-1]["depth"] += 1
 
         classes = attr_class(attrs)
+        if tag in {"div", "span"} and self._contains_direct_marker(classes):
+            self._direct_marker_depth = 1
 
         if tag == "a" and "gsc_a_at" in classes:
             self._title_depth = 1
@@ -213,6 +240,16 @@ class ScholarRowParser(HTMLParser):
             self._gray_stack.append({"depth": 1, "parts": []})
             return
 
+        if tag == "a":
+            self._aux_link_stack.append(
+                {
+                    "depth": 1,
+                    "href": attr_href(attrs),
+                    "classes": classes,
+                    "parts": [],
+                }
+            )
+
     def handle_data(self, data: str) -> None:
         if self._title_depth > 0:
             self.title_parts.append(data)
@@ -222,6 +259,21 @@ class ScholarRowParser(HTMLParser):
             self.year_parts.append(data)
         if self._gray_stack:
             self._gray_stack[-1]["parts"].append(data)
+        if self._aux_link_stack:
+            self._aux_link_stack[-1]["parts"].append(data)
+
+    def _capture_direct_download_href(self, link: dict[str, Any]) -> None:
+        if self.direct_download_href:
+            return
+        href = link.get("href")
+        if not isinstance(href, str) or not href.strip():
+            return
+        label = normalize_space("".join(link.get("parts", []))).lower()
+        classes = str(link.get("classes", "")).lower()
+        label_match = any(token in label for token in PROFILE_ROW_DIRECT_LABEL_TOKENS)
+        marker_match = self._contains_direct_marker(classes) or self._direct_marker_depth > 0
+        if label_match or marker_match:
+            self.direct_download_href = href.strip()
 
     def handle_endtag(self, _tag: str) -> None:
         if self._title_depth > 0:
@@ -237,6 +289,13 @@ class ScholarRowParser(HTMLParser):
                 if text:
                     self.gray_texts.append(text)
                 self._gray_stack.pop()
+        if self._aux_link_stack:
+            self._aux_link_stack[-1]["depth"] -= 1
+            if self._aux_link_stack[-1]["depth"] == 0:
+                self._capture_direct_download_href(self._aux_link_stack[-1])
+                self._aux_link_stack.pop()
+        if self._direct_marker_depth > 0:
+            self._direct_marker_depth -= 1
 
 
 def extract_rows(html: str) -> list[str]:
@@ -302,37 +361,60 @@ def parse_citation_count(parts: list[str]) -> int | None:
     return int(match.group(0))
 
 
+def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None, list[str]]:
+    parser = ScholarRowParser()
+    parser.feed(row_html)
+    warnings: list[str] = []
+    title = normalize_space("".join(parser.title_parts))
+    if not title:
+        warnings.append("row_missing_title")
+        return None, warnings
+    if not parser.title_href:
+        warnings.append("row_missing_title_href")
+
+    citation_text = normalize_space(" ".join(parser.citation_parts))
+    citation_count = parse_citation_count(parser.citation_parts)
+    if citation_text and citation_count is None:
+        warnings.append("layout_row_citation_unparseable")
+
+    year_text = normalize_space(" ".join(parser.year_parts))
+    year = parse_year(parser.year_parts)
+    if year_text and year is None:
+        warnings.append("layout_row_year_unparseable")
+
+    authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
+    venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
+    return (
+        PublicationCandidate(
+            title=title,
+            title_url=parser.title_href,
+            cluster_id=parse_cluster_id_from_href(parser.title_href),
+            year=year,
+            citation_count=citation_count,
+            authors_text=authors_text,
+            venue_text=venue_text,
+            pdf_url=build_absolute_scholar_url(parser.direct_download_href),
+        ),
+        warnings,
+    )
+
+
 def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]:
     rows = extract_rows(html)
     warnings: list[str] = []
     publications: list[PublicationCandidate] = []
 
     for row_html in rows:
-        parser = ScholarRowParser()
+        publication, row_warnings = _parse_publication_row(row_html)
-        parser.feed(row_html)
+        warnings.extend(row_warnings)
-
+        if publication is None:
-        title = normalize_space("".join(parser.title_parts))
-        if not title:
-            warnings.append("row_missing_title")
             continue
-
+        publications.append(publication)
-        authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
-        venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
-
-        publications.append(
-            PublicationCandidate(
-                title=title,
-                title_url=parser.title_href,
-                cluster_id=parse_cluster_id_from_href(parser.title_href),
-                year=parse_year(parser.year_parts),
-                citation_count=parse_citation_count(parser.citation_parts),
-                authors_text=authors_text,
-                venue_text=venue_text,
-            )
-        )
 
     if not rows:
         warnings.append("no_rows_detected")
+    if rows and not publications:
+        warnings.append("layout_all_rows_unparseable")
 
     return publications, sorted(set(warnings))
 
@@ -635,11 +717,35 @@ def classify_block_or_captcha_reason(
     return None
 
 
+def _warnings_contain(warnings: list[str], code: str) -> bool:
+    return any(item == code for item in warnings)
+
+
+def _has_layout_row_failure(marker_counts: dict[str, int], warnings: list[str]) -> bool:
+    if _warnings_contain(warnings, "layout_all_rows_unparseable"):
+        return True
+    if marker_counts.get("gsc_a_tr", 0) <= 0:
+        return False
+    if _warnings_contain(warnings, "row_missing_title"):
+        return True
+    return marker_counts.get("gsc_a_at", 0) <= 0
+
+
+def _first_layout_warning(warnings: list[str]) -> str | None:
+    for warning in warnings:
+        if warning.startswith("layout_"):
+            return warning
+    return None
+
+
 def detect_state(
     fetch_result: FetchResult,
     publications: list[PublicationCandidate],
     marker_counts: dict[str, int],
     *,
+    warnings: list[str],
+    has_show_more_button_flag: bool,
+    articles_range: str | None,
     visible_text: str,
 ) -> tuple[ParseState, str]:
     if fetch_result.status_code is None:
@@ -660,6 +766,16 @@ def detect_state(
     if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS):
         return ParseState.NO_RESULTS, "no_results_keyword_detected"
 
+    layout_warning = _first_layout_warning(warnings)
+    if layout_warning is not None:
+        return ParseState.LAYOUT_CHANGED, layout_warning
+
+    if _has_layout_row_failure(marker_counts, warnings):
+        return ParseState.LAYOUT_CHANGED, "layout_publication_rows_unparseable"
+
+    if has_show_more_button_flag and not articles_range:
+        return ParseState.LAYOUT_CHANGED, "layout_show_more_without_articles_range"
+
     if not publications:
         has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
         has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
@@ -699,7 +815,7 @@ def detect_author_search_state(
         has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0
         if not has_search_markers:
             return ParseState.NO_RESULTS, "no_search_candidates_detected"
-        return ParseState.OK, "search_markers_present_with_empty_results"
+        return ParseState.LAYOUT_CHANGED, "layout_author_candidates_unparseable"
 
     return ParseState.OK, "author_candidates_extracted"
 
@@ -711,6 +827,7 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
 
     show_more = has_show_more_button(fetch_result.body)
     operation_error_banner = has_operation_error_banner(fetch_result.body)
+    articles_range = extract_articles_range(fetch_result.body)
 
     if show_more:
         warnings.append("possible_partial_page_show_more_present")
@@ -723,6 +840,9 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
         fetch_result,
         publications,
         marker_counts,
+        warnings=warnings,
+        has_show_more_button_flag=show_more,
+        articles_range=articles_range,
         visible_text=visible_text,
     )
 
@@ -736,7 +856,7 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
         warnings=warnings,
         has_show_more_button=show_more,
         has_operation_error_banner=operation_error_banner,
-        articles_range=extract_articles_range(fetch_result.body),
+        articles_range=articles_range,
     )
 
 

app/services/scholar_source.py

@@ -123,8 +123,8 @@ class LiveScholarSource:
         )
         return await asyncio.to_thread(self._fetch_sync, requested_url)
 
-    def _fetch_sync(self, requested_url: str) -> FetchResult:
+    def _build_request(self, requested_url: str) -> Request:
-        request = Request(
+        return Request(
             requested_url,
             headers={
                 "User-Agent": random.choice(self._user_agents),
@@ -134,8 +134,47 @@ class LiveScholarSource:
             },
         )
 
+    @staticmethod
+    def _http_error_body(exc: HTTPError) -> str:
         try:
-            with urlopen(request, timeout=self._timeout_seconds) as response:
+            return exc.read().decode("utf-8", errors="replace")
+        except Exception:
+            return ""
+
+    @staticmethod
+    def _network_error_result(requested_url: str, exc: URLError) -> FetchResult:
+        logger.warning(
+            "scholar_source.fetch_network_error",
+            extra={"event": "scholar_source.fetch_network_error", "requested_url": requested_url},
+        )
+        return FetchResult(
+            requested_url=requested_url,
+            status_code=None,
+            final_url=None,
+            body="",
+            error=str(exc),
+        )
+
+    @staticmethod
+    def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult:
+        logger.warning(
+            "scholar_source.fetch_http_error",
+            extra={
+                "event": "scholar_source.fetch_http_error",
+                "requested_url": requested_url,
+                "status_code": exc.code,
+            },
+        )
+        return FetchResult(
+            requested_url=requested_url,
+            status_code=exc.code,
+            final_url=exc.geturl(),
+            body=LiveScholarSource._http_error_body(exc),
+            error=str(exc),
+        )
+
+    @staticmethod
+    def _success_result(requested_url: str, response) -> FetchResult:
         body = response.read().decode("utf-8", errors="replace")
         status_code = getattr(response, "status", 200)
         logger.debug(
@@ -153,42 +192,17 @@ class LiveScholarSource:
             body=body,
             error=None,
         )
-        except HTTPError as exc:
+
-            body = ""
+    def _fetch_sync(self, requested_url: str) -> FetchResult:
+        request = self._build_request(requested_url)
+
         try:
-                body = exc.read().decode("utf-8", errors="replace")
+            with urlopen(request, timeout=self._timeout_seconds) as response:
-            except Exception:
+                return self._success_result(requested_url, response)
-                body = ""
+        except HTTPError as exc:
-            logger.warning(
+            return self._http_error_result(requested_url, exc)
-                "scholar_source.fetch_http_error",
-                extra={
-                    "event": "scholar_source.fetch_http_error",
-                    "requested_url": requested_url,
-                    "status_code": exc.code,
-                },
-            )
-            return FetchResult(
-                requested_url=requested_url,
-                status_code=exc.code,
-                final_url=exc.geturl(),
-                body=body,
-                error=str(exc),
-            )
         except URLError as exc:
-            logger.warning(
+            return self._network_error_result(requested_url, exc)
-                "scholar_source.fetch_network_error",
-                extra={
-                    "event": "scholar_source.fetch_network_error",
-                    "requested_url": requested_url,
-                },
-            )
-            return FetchResult(
-                requested_url=requested_url,
-                status_code=None,
-                final_url=None,
-                body="",
-                error=str(exc),
-            )
 
 
 def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:

app/services/scholars.py

@@ -269,73 +269,103 @@ def _serialize_parsed_author_search_page(parsed: ParsedAuthorSearchPage) -> dict
     }
 
 
-def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearchPage | None:
+def _payload_state(payload: dict[str, object]) -> ParseState | None:
-    if not isinstance(payload, dict):
-        return None
-
     state_raw = str(payload.get("state", "")).strip()
     try:
-        state = ParseState(state_raw)
+        return ParseState(state_raw)
     except ValueError:
         return None
 
-    marker_counts_payload = payload.get("marker_counts")
-    marker_counts = (
-        {str(key): int(value) for key, value in marker_counts_payload.items()}
-        if isinstance(marker_counts_payload, dict)
-        else {}
-    )
-    warnings_payload = payload.get("warnings")
-    warnings = (
-        [str(value) for value in warnings_payload if isinstance(value, str)]
-        if isinstance(warnings_payload, list)
-        else []
-    )
-    from app.services.scholar_parser import ScholarSearchCandidate
 
-    candidates_payload = payload.get("candidates")
+def _payload_marker_counts(payload: dict[str, object]) -> dict[str, int]:
-    normalized_candidates = []
+    marker_counts_payload = payload.get("marker_counts")
-    for value in candidates_payload if isinstance(candidates_payload, list) else []:
+    if not isinstance(marker_counts_payload, dict):
-        if not isinstance(value, dict):
+        return {}
+    parsed: dict[str, int] = {}
+    for key, value in marker_counts_payload.items():
+        try:
+            parsed[str(key)] = int(value)
+        except (TypeError, ValueError):
             continue
+    return parsed
+
+
+def _payload_warnings(payload: dict[str, object]) -> list[str]:
+    warnings_payload = payload.get("warnings")
+    if not isinstance(warnings_payload, list):
+        return []
+    return [str(value) for value in warnings_payload if isinstance(value, str)]
+
+
+def _parse_optional_string(value: object) -> str | None:
+    if value is None:
+        return None
+    normalized = str(value).strip()
+    return normalized or None
+
+
+def _parse_optional_int(value: object) -> int | None:
+    if isinstance(value, int):
+        return value
+    if isinstance(value, str) and value.strip():
+        try:
+            return int(value)
+        except ValueError:
+            return None
+    return None
+
+
+def _normalize_interests(value: object) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(item) for item in value if isinstance(item, str)]
+
+
+def _deserialize_candidate_payload(value: object) -> dict[str, object] | None:
+    if not isinstance(value, dict):
+        return None
     scholar_id = str(value.get("scholar_id", "")).strip()
     display_name = str(value.get("display_name", "")).strip()
     profile_url = str(value.get("profile_url", "")).strip()
     if not scholar_id or not display_name or not profile_url:
-            continue
+        return None
-        interests_payload = value.get("interests")
+    return {
-        interests = (
-            [str(item) for item in interests_payload if isinstance(item, str)]
-            if isinstance(interests_payload, list)
-            else []
-        )
-        cited_by_count = value.get("cited_by_count")
-        parsed_cited_by_count: int | None = None
-        if isinstance(cited_by_count, int):
-            parsed_cited_by_count = cited_by_count
-        elif isinstance(cited_by_count, str) and cited_by_count.strip():
-            try:
-                parsed_cited_by_count = int(cited_by_count)
-            except ValueError:
-                parsed_cited_by_count = None
-        normalized_candidates.append(
-            {
         "scholar_id": scholar_id,
         "display_name": display_name,
-                "affiliation": str(value.get("affiliation")).strip()
+        "affiliation": _parse_optional_string(value.get("affiliation")),
-                if value.get("affiliation") is not None
+        "email_domain": _parse_optional_string(value.get("email_domain")),
-                else None,
+        "cited_by_count": _parse_optional_int(value.get("cited_by_count")),
-                "email_domain": str(value.get("email_domain")).strip()
+        "interests": _normalize_interests(value.get("interests")),
-                if value.get("email_domain") is not None
-                else None,
-                "cited_by_count": parsed_cited_by_count,
-                "interests": interests,
         "profile_url": profile_url,
-                "profile_image_url": str(value.get("profile_image_url")).strip()
+        "profile_image_url": _parse_optional_string(value.get("profile_image_url")),
-                if value.get("profile_image_url") is not None
-                else None,
     }
-        )
+
+
+def _deserialize_candidates(payload: dict[str, object]) -> list[dict[str, object]]:
+    candidates_payload = payload.get("candidates")
+    if not isinstance(candidates_payload, list):
+        return []
+    normalized: list[dict[str, object]] = []
+    for value in candidates_payload:
+        candidate = _deserialize_candidate_payload(value)
+        if candidate is not None:
+            normalized.append(candidate)
+    return normalized
+
+
+def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearchPage | None:
+    if not isinstance(payload, dict):
+        return None
+
+    state = _payload_state(payload)
+    if state is None:
+        return None
+
+    marker_counts = _payload_marker_counts(payload)
+    warnings = _payload_warnings(payload)
+    from app.services.scholar_parser import ScholarSearchCandidate
+
+    normalized_candidates = _deserialize_candidates(payload)
 
     return ParsedAuthorSearchPage(
         state=state,
@@ -552,38 +582,18 @@ async def delete_scholar(
     await db_session.commit()
 
 
-async def search_author_candidates(
+def _normalize_author_search_inputs(query: str, limit: int) -> tuple[str, int, str]:
-    *,
-    source: ScholarSource,
-    db_session: AsyncSession,
-    query: str,
-    limit: int,
-    network_error_retries: int = 1,
-    retry_backoff_seconds: float = 1.0,
-    search_enabled: bool = True,
-    cache_ttl_seconds: int = 21_600,
-    blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS,
-    cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES,
-    min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS,
-    interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS,
-    cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD,
-    cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS,
-    retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD,
-    cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD,
-) -> ParsedAuthorSearchPage:
     normalized_query = query.strip()
     if len(normalized_query) < 2:
         raise ScholarServiceError("Search query must be at least 2 characters.")
     bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))
-    query_key = normalized_query.casefold()
+    return normalized_query, bounded_limit, normalized_query.casefold()
 
-    if not search_enabled:
+
+def _disabled_search_result(*, normalized_query: str, bounded_limit: int) -> ParsedAuthorSearchPage:
     logger.warning(
         "scholar_search.disabled_by_configuration",
-            extra={
+        extra={"event": "scholar_search.disabled_by_configuration", "query": normalized_query},
-                "event": "scholar_search.disabled_by_configuration",
-                "query": normalized_query,
-            },
     )
     return _policy_blocked_author_search_result(
         reason=SEARCH_DISABLED_REASON,
@@ -591,90 +601,119 @@ async def search_author_candidates(
         limit=bounded_limit,
     )
 
-    await _acquire_author_search_lock(db_session)
-    runtime_state = await _load_runtime_state_for_update(db_session)
-    runtime_state_updated = False
-    now_utc = datetime.now(timezone.utc)
 
-    if runtime_state.cooldown_until is not None:
+def _normalize_runtime_cooldown_state(
+    runtime_state: AuthorSearchRuntimeState,
+    *,
+    now_utc: datetime,
+) -> bool:
+    if runtime_state.cooldown_until is None:
+        return False
     cooldown_until = runtime_state.cooldown_until
+    updated = False
     if cooldown_until.tzinfo is None:
         cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
         runtime_state.cooldown_until = cooldown_until
-            runtime_state_updated = True
+        updated = True
-        if now_utc >= cooldown_until:
+    if now_utc < cooldown_until:
+        return updated
     logger.info(
         "scholar_search.cooldown_expired",
-                extra={
+        extra={"event": "scholar_search.cooldown_expired", "cooldown_until_utc": cooldown_until.isoformat()},
-                    "event": "scholar_search.cooldown_expired",
-                    "cooldown_until_utc": cooldown_until.isoformat(),
-                },
     )
     runtime_state.cooldown_until = None
     runtime_state.cooldown_rejection_count = 0
     runtime_state.cooldown_alert_emitted = False
-            runtime_state_updated = True
+    return True
 
-    cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(runtime_state, now_utc)
-    if cooldown_remaining_seconds > 0:
-        runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1
-        bounded_cooldown_rejection_alert_threshold = max(
-            1,
-            int(cooldown_rejection_alert_threshold),
-        )
-        if (
-            int(runtime_state.cooldown_rejection_count) >= bounded_cooldown_rejection_alert_threshold
-            and not bool(runtime_state.cooldown_alert_emitted)
-        ):
-            logger.error(
-                "scholar_search.cooldown_rejection_threshold_exceeded",
-                extra={
-                    "event": "scholar_search.cooldown_rejection_threshold_exceeded",
-                    "query": normalized_query,
-                    "cooldown_rejection_count": int(runtime_state.cooldown_rejection_count),
-                    "threshold": bounded_cooldown_rejection_alert_threshold,
-                    "cooldown_until_utc": runtime_state.cooldown_until.isoformat()
-                    if runtime_state.cooldown_until
-                    else None,
-                },
-            )
-            runtime_state.cooldown_alert_emitted = True
-        runtime_state_updated = True
 
-        logger.warning(
+def _cooldown_warning_codes(
-            "scholar_search.cooldown_active",
+    *,
-            extra={
+    runtime_state: AuthorSearchRuntimeState,
-                "event": "scholar_search.cooldown_active",
+    cooldown_remaining_seconds: int,
-                "query": normalized_query,
+) -> list[str]:
-                "cooldown_remaining_seconds": cooldown_remaining_seconds,
-                "cooldown_until_utc": runtime_state.cooldown_until.isoformat()
-                if runtime_state.cooldown_until
-                else None,
-            },
-        )
     warning_codes = [
         "author_search_cooldown_active",
         f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
     ]
     if bool(runtime_state.cooldown_alert_emitted):
         warning_codes.append("author_search_cooldown_alert_threshold_exceeded")
-        if runtime_state_updated:
+    return warning_codes
-            await db_session.commit()
+
+
+def _emit_cooldown_threshold_alert(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    normalized_query: str,
+    cooldown_rejection_alert_threshold: int,
+) -> bool:
+    runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1
+    threshold = max(1, int(cooldown_rejection_alert_threshold))
+    if int(runtime_state.cooldown_rejection_count) < threshold:
+        return True
+    if bool(runtime_state.cooldown_alert_emitted):
+        return True
+    logger.error(
+        "scholar_search.cooldown_rejection_threshold_exceeded",
+        extra={
+            "event": "scholar_search.cooldown_rejection_threshold_exceeded",
+            "query": normalized_query,
+            "cooldown_rejection_count": int(runtime_state.cooldown_rejection_count),
+            "threshold": threshold,
+            "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
+        },
+    )
+    runtime_state.cooldown_alert_emitted = True
+    return True
+
+
+def _cooldown_block_result(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    normalized_query: str,
+    bounded_limit: int,
+    cooldown_rejection_alert_threshold: int,
+    cooldown_remaining_seconds: int,
+) -> ParsedAuthorSearchPage:
+    _emit_cooldown_threshold_alert(
+        runtime_state=runtime_state,
+        normalized_query=normalized_query,
+        cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
+    )
+    logger.warning(
+        "scholar_search.cooldown_active",
+        extra={
+            "event": "scholar_search.cooldown_active",
+            "query": normalized_query,
+            "cooldown_remaining_seconds": cooldown_remaining_seconds,
+            "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
+        },
+    )
     return _policy_blocked_author_search_result(
         reason=SEARCH_COOLDOWN_REASON,
-            warning_codes=warning_codes,
+        warning_codes=_cooldown_warning_codes(
+            runtime_state=runtime_state,
+            cooldown_remaining_seconds=cooldown_remaining_seconds,
+        ),
         limit=bounded_limit,
     )
 
+
+async def _cache_hit_result(
+    db_session: AsyncSession,
+    *,
+    query_key: str,
+    now_utc: datetime,
+    normalized_query: str,
+    bounded_limit: int,
+) -> ParsedAuthorSearchPage | None:
     cached = await _cache_get_author_search_result(
         db_session,
         query_key=query_key,
         now_utc=now_utc,
     )
-    if cached is not None:
+    if cached is None:
-        state_reason_override = (
+        return None
-            SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None
-        )
     logger.info(
         "scholar_search.cache_hit",
         extra={
@@ -684,7 +723,7 @@ async def search_author_candidates(
             "state_reason": cached.state_reason,
         },
     )
-        await db_session.commit()
+    state_reason_override = SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None
     return _trim_author_search_result(
         cached,
         limit=bounded_limit,
@@ -692,81 +731,123 @@ async def search_author_candidates(
         state_reason_override=state_reason_override,
     )
 
-    if runtime_state.last_live_request_at is not None:
+
+def _throttle_sleep_seconds(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    now_utc: datetime,
+    min_interval_seconds: float,
+    interval_jitter_seconds: float,
+) -> tuple[float, bool]:
+    updated = False
+    if runtime_state.last_live_request_at is None:
+        enforced_wait_seconds = 0.0
+    else:
         last_live_request_at = runtime_state.last_live_request_at
         if last_live_request_at.tzinfo is None:
             last_live_request_at = last_live_request_at.replace(tzinfo=timezone.utc)
             runtime_state.last_live_request_at = last_live_request_at
-            runtime_state_updated = True
+            updated = True
         enforced_wait_seconds = (
             last_live_request_at + timedelta(seconds=max(float(min_interval_seconds), 0.0)) - now_utc
         ).total_seconds()
-    else:
-        enforced_wait_seconds = 0.0
-
     jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0))
-    sleep_seconds = max(0.0, float(enforced_wait_seconds)) + jitter_seconds
+    return max(0.0, float(enforced_wait_seconds)) + jitter_seconds, updated
-    if sleep_seconds > 0.0:
+
+
+async def _wait_for_author_search_throttle(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    normalized_query: str,
+    now_utc: datetime,
+    min_interval_seconds: float,
+    interval_jitter_seconds: float,
+) -> bool:
+    sleep_seconds, updated = _throttle_sleep_seconds(
+        runtime_state=runtime_state,
+        now_utc=now_utc,
+        min_interval_seconds=min_interval_seconds,
+        interval_jitter_seconds=interval_jitter_seconds,
+    )
+    if sleep_seconds <= 0.0:
+        return updated
     logger.info(
         "scholar_search.throttle_wait",
-            extra={
+        extra={"event": "scholar_search.throttle_wait", "query": normalized_query, "sleep_seconds": round(sleep_seconds, 3)},
-                "event": "scholar_search.throttle_wait",
-                "query": normalized_query,
-                "sleep_seconds": round(sleep_seconds, 3),
-            },
     )
     await asyncio.sleep(sleep_seconds)
+    return True
 
+
+async def _fetch_author_search_with_retries(
+    *,
+    source: ScholarSource,
+    normalized_query: str,
+    network_error_retries: int,
+    retry_backoff_seconds: float,
+) -> tuple[ParsedAuthorSearchPage, int, list[str]]:
     max_attempts = max(1, int(network_error_retries) + 1)
     parsed: ParsedAuthorSearchPage | None = None
     retry_warnings: list[str] = []
     retry_scheduled_count = 0
-
     for attempt_index in range(max_attempts):
         fetch_result = await source.fetch_author_search_html(normalized_query, start=0)
         parsed = parse_author_search_page(fetch_result)
         if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1:
             break
-
         retry_warnings.append("network_retry_scheduled_for_author_search")
         retry_scheduled_count += 1
         retry_sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index)
         if retry_sleep_seconds > 0:
             await asyncio.sleep(retry_sleep_seconds)
-
-    runtime_state.last_live_request_at = datetime.now(timezone.utc)
-    runtime_state_updated = True
-
     if parsed is None:
         raise ScholarServiceError("Unable to complete scholar author search.")
+    return parsed, retry_scheduled_count, retry_warnings
 
-    merged_parsed = replace(
-        parsed,
-        warnings=_merge_warnings(parsed.warnings, retry_warnings),
-    )
 
-    bounded_retry_alert_threshold = max(1, int(retry_alert_threshold))
+def _with_retry_warnings(
-    if retry_scheduled_count >= bounded_retry_alert_threshold:
+    parsed: ParsedAuthorSearchPage,
+    *,
+    retry_warnings: list[str],
+    retry_scheduled_count: int,
+    retry_alert_threshold: int,
+    normalized_query: str,
+) -> ParsedAuthorSearchPage:
+    merged = replace(parsed, warnings=_merge_warnings(parsed.warnings, retry_warnings))
+    threshold = max(1, int(retry_alert_threshold))
+    if retry_scheduled_count < threshold:
+        return merged
     logger.warning(
         "scholar_search.retry_threshold_exceeded",
         extra={
             "event": "scholar_search.retry_threshold_exceeded",
             "query": normalized_query,
             "retry_scheduled_count": retry_scheduled_count,
-                "threshold": bounded_retry_alert_threshold,
+            "threshold": threshold,
-                "final_state": merged_parsed.state.value,
+            "final_state": merged.state.value,
-                "final_state_reason": merged_parsed.state_reason,
+            "final_state_reason": merged.state_reason,
         },
     )
-        merged_parsed = replace(
+    return replace(
-            merged_parsed,
+        merged,
         warnings=_merge_warnings(
-                merged_parsed.warnings,
+            merged.warnings,
             [f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"],
         ),
     )
 
-    if _is_author_search_block_state(merged_parsed):
+
+def _apply_block_circuit_breaker(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    merged_parsed: ParsedAuthorSearchPage,
+    cooldown_block_threshold: int,
+    cooldown_seconds: int,
+    normalized_query: str,
+) -> ParsedAuthorSearchPage:
+    if not _is_author_search_block_state(merged_parsed):
+        runtime_state.consecutive_blocked_count = 0
+        return merged_parsed
     runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1
     logger.warning(
         "scholar_search.block_detected",
@@ -777,53 +858,169 @@ async def search_author_candidates(
             "consecutive_blocked_count": int(runtime_state.consecutive_blocked_count),
         },
     )
-        if int(runtime_state.consecutive_blocked_count) >= max(1, int(cooldown_block_threshold)):
+    if int(runtime_state.consecutive_blocked_count) < max(1, int(cooldown_block_threshold)):
-            runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta(
+        return merged_parsed
-                seconds=max(60, int(cooldown_seconds))
+    runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta(seconds=max(60, int(cooldown_seconds)))
-            )
     runtime_state.consecutive_blocked_count = 0
     runtime_state.cooldown_rejection_count = 0
     runtime_state.cooldown_alert_emitted = False
-            merged_parsed = replace(
-                merged_parsed,
-                warnings=_merge_warnings(
-                    merged_parsed.warnings,
-                    ["author_search_circuit_breaker_armed"],
-                ),
-            )
     logger.error(
         "scholar_search.cooldown_activated",
         extra={
             "event": "scholar_search.cooldown_activated",
             "query": normalized_query,
-                    "cooldown_until_utc": runtime_state.cooldown_until.isoformat()
+            "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
-                    if runtime_state.cooldown_until
-                    else None,
         },
     )
-    else:
+    return replace(
-        runtime_state.consecutive_blocked_count = 0
+        merged_parsed,
+        warnings=_merge_warnings(merged_parsed.warnings, ["author_search_circuit_breaker_armed"]),
+    )
 
-    ttl_seconds = (
+
-        min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds)))
+def _resolve_author_search_cache_ttl_seconds(
-        if _is_author_search_block_state(merged_parsed)
+    *,
-        else max(1, int(cache_ttl_seconds))
+    merged_parsed: ParsedAuthorSearchPage,
+    blocked_cache_ttl_seconds: int,
+    cache_ttl_seconds: int,
+) -> int:
+    if _is_author_search_block_state(merged_parsed):
+        return min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds)))
+    return max(1, int(cache_ttl_seconds))
+
+
+async def _load_locked_runtime_state(
+    db_session: AsyncSession,
+) -> AuthorSearchRuntimeState:
+    await _acquire_author_search_lock(db_session)
+    return await _load_runtime_state_for_update(db_session)
+
+
+async def _cooldown_or_cache_result(
+    db_session: AsyncSession,
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    query_key: str,
+    normalized_query: str,
+    bounded_limit: int,
+    cooldown_rejection_alert_threshold: int,
+) -> tuple[ParsedAuthorSearchPage | None, bool]:
+    runtime_state_updated = _normalize_runtime_cooldown_state(
+        runtime_state,
+        now_utc=datetime.now(timezone.utc),
+    )
+    cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(
+        runtime_state,
+        datetime.now(timezone.utc),
+    )
+    if cooldown_remaining_seconds > 0:
+        return (
+            _cooldown_block_result(
+                runtime_state=runtime_state,
+                normalized_query=normalized_query,
+                bounded_limit=bounded_limit,
+                cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
+                cooldown_remaining_seconds=cooldown_remaining_seconds,
+            ),
+            True,
+        )
+    cached_result = await _cache_hit_result(
+        db_session,
+        query_key=query_key,
+        now_utc=datetime.now(timezone.utc),
+        normalized_query=normalized_query,
+        bounded_limit=bounded_limit,
+    )
+    return cached_result, runtime_state_updated
+
+
+async def _perform_live_author_search(db_session: AsyncSession, *, source: ScholarSource, runtime_state: AuthorSearchRuntimeState, normalized_query: str, query_key: str, network_error_retries: int, retry_backoff_seconds: float, min_interval_seconds: float, interval_jitter_seconds: float, retry_alert_threshold: int, cooldown_block_threshold: int, cooldown_seconds: int, blocked_cache_ttl_seconds: int, cache_ttl_seconds: int, cache_max_entries: int) -> tuple[ParsedAuthorSearchPage, bool]:
+    runtime_state_updated = await _wait_for_author_search_throttle(
+        runtime_state=runtime_state,
+        normalized_query=normalized_query,
+        now_utc=datetime.now(timezone.utc),
+        min_interval_seconds=min_interval_seconds,
+        interval_jitter_seconds=interval_jitter_seconds,
+    )
+    parsed, retry_count, retry_warnings = await _fetch_author_search_with_retries(
+        source=source,
+        normalized_query=normalized_query,
+        network_error_retries=network_error_retries,
+        retry_backoff_seconds=retry_backoff_seconds,
+    )
+    runtime_state.last_live_request_at = datetime.now(timezone.utc)
+    merged = _with_retry_warnings(
+        parsed,
+        retry_warnings=retry_warnings,
+        retry_scheduled_count=retry_count,
+        retry_alert_threshold=retry_alert_threshold,
+        normalized_query=normalized_query,
+    )
+    merged = _apply_block_circuit_breaker(
+        runtime_state=runtime_state,
+        merged_parsed=merged,
+        cooldown_block_threshold=cooldown_block_threshold,
+        cooldown_seconds=cooldown_seconds,
+        normalized_query=normalized_query,
+    )
+    ttl_seconds = _resolve_author_search_cache_ttl_seconds(
+        merged_parsed=merged,
+        blocked_cache_ttl_seconds=blocked_cache_ttl_seconds,
+        cache_ttl_seconds=cache_ttl_seconds,
     )
     await _cache_set_author_search_result(
         db_session,
         query_key=query_key,
-        parsed=merged_parsed,
+        parsed=merged,
         ttl_seconds=float(ttl_seconds),
         max_entries=cache_max_entries,
         now_utc=datetime.now(timezone.utc),
     )
+    return merged, True
+
+
+async def search_author_candidates(*, source: ScholarSource, db_session: AsyncSession, query: str, limit: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, search_enabled: bool = True, cache_ttl_seconds: int = 21_600, blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS, cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES, min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS, interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS, cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD, cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS, retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD) -> ParsedAuthorSearchPage:
+    normalized_query, bounded_limit, query_key = _normalize_author_search_inputs(query, limit)
+    if not search_enabled:
+        return _disabled_search_result(
+            normalized_query=normalized_query,
+            bounded_limit=bounded_limit,
+        )
+
+    runtime_state = await _load_locked_runtime_state(db_session)
+    early_result, runtime_state_updated = await _cooldown_or_cache_result(
+        db_session,
+        runtime_state=runtime_state,
+        query_key=query_key,
+        normalized_query=normalized_query,
+        bounded_limit=bounded_limit,
+        cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
+    )
+    if early_result is not None:
+        await db_session.commit()
+        return early_result
+
+    merged_parsed, live_runtime_updated = await _perform_live_author_search(
+        db_session,
+        source=source,
+        runtime_state=runtime_state,
+        normalized_query=normalized_query,
+        query_key=query_key,
+        network_error_retries=network_error_retries,
+        retry_backoff_seconds=retry_backoff_seconds,
+        min_interval_seconds=min_interval_seconds,
+        interval_jitter_seconds=interval_jitter_seconds,
+        retry_alert_threshold=retry_alert_threshold,
+        cooldown_block_threshold=cooldown_block_threshold,
+        cooldown_seconds=cooldown_seconds,
+        blocked_cache_ttl_seconds=blocked_cache_ttl_seconds,
+        cache_ttl_seconds=cache_ttl_seconds,
+        cache_max_entries=cache_max_entries,
+    )
+    runtime_state_updated = runtime_state_updated or live_runtime_updated
     if runtime_state_updated:
         await db_session.commit()
-
+    return _trim_author_search_result(merged_parsed, limit=bounded_limit)
-    return _trim_author_search_result(
-        merged_parsed,
-        limit=bounded_limit,
-    )
 
 
 async def hydrate_profile_metadata(

frontend/src/features/publications/index.ts

@@ -11,6 +11,7 @@ export interface PublicationItem {
   citation_count: number;
   venue_text: string | null;
   pub_url: string | null;
+  pdf_url: string | null;
   is_read: boolean;
   first_seen_at: string;
   is_new_in_latest_run: boolean;

frontend/src/features/scholars/index.ts

@@ -37,6 +37,50 @@ export interface ScholarSearchResult {
   warnings: string[];
 }
 
+export interface ScholarExportItem {
+  scholar_id: string;
+  display_name: string | null;
+  is_enabled: boolean;
+  profile_image_override_url: string | null;
+}
+
+export interface PublicationExportItem {
+  scholar_id: string;
+  cluster_id: string | null;
+  fingerprint_sha256: string | null;
+  title: string;
+  year: number | null;
+  citation_count: number;
+  author_text: string | null;
+  venue_text: string | null;
+  pub_url: string | null;
+  pdf_url: string | null;
+  is_read: boolean;
+}
+
+export interface DataExportPayload {
+  schema_version: number;
+  exported_at: string;
+  scholars: ScholarExportItem[];
+  publications: PublicationExportItem[];
+}
+
+export interface DataImportPayload {
+  schema_version?: number;
+  scholars: ScholarExportItem[];
+  publications: PublicationExportItem[];
+}
+
+export interface DataImportResult {
+  scholars_created: number;
+  scholars_updated: number;
+  publications_created: number;
+  publications_updated: number;
+  links_created: number;
+  links_updated: number;
+  skipped_records: number;
+}
+
 interface ScholarsListData {
   scholars: ScholarProfile[];
 }
@@ -115,3 +159,20 @@ export async function clearScholarImage(
   });
   return response.data;
 }
+
+export async function exportScholarData(): Promise<DataExportPayload> {
+  const response = await apiRequest<DataExportPayload>("/scholars/export", {
+    method: "GET",
+  });
+  return response.data;
+}
+
+export async function importScholarData(
+  payload: DataImportPayload,
+): Promise<DataImportResult> {
+  const response = await apiRequest<DataImportResult>("/scholars/import", {
+    method: "POST",
+    body: payload,
+  });
+  return response.data;
+}

frontend/src/pages/DashboardPage.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { computed, onMounted, ref, watch } from "vue";
+import { computed, onMounted, onUnmounted, ref, watch } from "vue";
 
 import { fetchDashboardSnapshot, type DashboardSnapshot } from "@/features/dashboard";
 import { ApiRequestError } from "@/lib/api/errors";
@@ -14,7 +14,7 @@ import AppCard from "@/components/ui/AppCard.vue";
 import AppEmptyState from "@/components/ui/AppEmptyState.vue";
 import AppHelpHint from "@/components/ui/AppHelpHint.vue";
 import { useAuthStore } from "@/stores/auth";
-import { useRunStatusStore } from "@/stores/run_status";
+import { RUN_STATUS_POLL_INTERVAL_MS, useRunStatusStore } from "@/stores/run_status";
 import { useUserSettingsStore } from "@/stores/user_settings";
 
 const loading = ref(true);
@@ -26,6 +26,7 @@ const refreshingAfterCompletion = ref(false);
 const auth = useAuthStore();
 const runStatus = useRunStatusStore();
 const userSettings = useUserSettingsStore();
+let latestRunSyncTimer: ReturnType<typeof setInterval> | null = null;
 
 const isStartBlocked = computed(
   () =>
@@ -118,6 +119,39 @@ function formatDate(value: string | null): string {
   return asDate.toLocaleString();
 }
 
+function shouldRefreshAfterRunChange(
+  nextRun: typeof runStatus.latestRun,
+  previousRun: typeof runStatus.latestRun,
+): boolean {
+  if (!nextRun || !previousRun) {
+    return false;
+  }
+  if (nextRun.status === "running") {
+    return false;
+  }
+  if (nextRun.id !== previousRun.id) {
+    return true;
+  }
+  return previousRun.status === "running";
+}
+
+function startLatestRunSyncLoop(): void {
+  if (latestRunSyncTimer !== null) {
+    return;
+  }
+  latestRunSyncTimer = setInterval(() => {
+    void runStatus.syncLatest();
+  }, RUN_STATUS_POLL_INTERVAL_MS);
+}
+
+function stopLatestRunSyncLoop(): void {
+  if (latestRunSyncTimer === null) {
+    return;
+  }
+  clearInterval(latestRunSyncTimer);
+  latestRunSyncTimer = null;
+}
+
 async function loadSnapshot(): Promise<void> {
   loading.value = true;
   errorMessage.value = null;
@@ -172,6 +206,12 @@ async function onTriggerRun(): Promise<void> {
 
 onMounted(() => {
   void loadSnapshot();
+  void runStatus.syncLatest();
+  startLatestRunSyncLoop();
+});
+
+onUnmounted(() => {
+  stopLatestRunSyncLoop();
 });
 
 watch(
@@ -180,13 +220,7 @@ watch(
     if (refreshingAfterCompletion.value) {
       return;
     }
-    if (!nextRun || !previousRun) {
+    if (!shouldRefreshAfterRunChange(nextRun, previousRun)) {
-      return;
-    }
-    if (nextRun.id !== previousRun.id) {
-      return;
-    }
-    if (previousRun.status !== "running" || nextRun.status === "running") {
       return;
     }
 

frontend/src/pages/PublicationsPage.vue

@@ -99,6 +99,10 @@ function scholarLabel(item: ScholarProfile): string {
   return item.display_name || item.scholar_id;
 }
 
+function publicationPrimaryUrl(item: PublicationItem): string | null {
+  return item.pub_url || item.pdf_url;
+}
+
 const selectedScholarName = computed(() => {
   const selectedId = Number(selectedScholarFilter.value);
   if (!Number.isInteger(selectedId) || selectedId <= 0) {
@@ -590,9 +594,10 @@ watch(
                 />
               </td>
               <td>
+                <div class="grid gap-1">
                   <a
-                  v-if="item.pub_url"
+                    v-if="publicationPrimaryUrl(item)"
-                  :href="item.pub_url"
+                    :href="publicationPrimaryUrl(item) || ''"
                     target="_blank"
                     rel="noreferrer"
                     class="link-inline"
@@ -600,6 +605,16 @@ watch(
                     {{ item.title }}
                   </a>
                   <span v-else>{{ item.title }}</span>
+                  <a
+                    v-if="item.pdf_url"
+                    :href="item.pdf_url"
+                    target="_blank"
+                    rel="noreferrer"
+                    class="link-inline text-xs"
+                  >
+                    Direct PDF
+                  </a>
+                </div>
               </td>
               <td>{{ item.scholar_label }}</td>
               <td>{{ item.year ?? "n/a" }}</td>
@@ -607,7 +622,7 @@ watch(
               <td>
                 <div class="flex flex-wrap items-center gap-2">
                   <AppBadge :tone="item.is_new_in_latest_run ? 'info' : 'neutral'">
-                    {{ item.is_new_in_latest_run ? "New this check" : "Seen before" }}
+                    {{ item.is_new_in_latest_run ? "New" : "Seen before" }}
                   </AppBadge>
                   <AppBadge :tone="item.is_read ? 'success' : 'warning'">
                     {{ item.is_read ? "Read" : "Unread" }}

frontend/src/pages/ScholarsPage.vue

@@ -17,11 +17,15 @@ import {
   clearScholarImage,
   createScholar,
   deleteScholar,
+  exportScholarData,
+  importScholarData,
   listScholars,
   searchScholarsByName,
   setScholarImageUrl,
   toggleScholar,
   uploadScholarImage,
+  type DataImportPayload,
+  type DataImportResult,
   type ScholarProfile,
   type ScholarSearchCandidate,
   type ScholarSearchResult,
@@ -37,6 +41,9 @@ const imageSavingScholarId = ref<number | null>(null);
 const imageUploadingScholarId = ref<number | null>(null);
 const addingCandidateScholarId = ref<string | null>(null);
 const activeScholarSettingsId = ref<number | null>(null);
+const importingData = ref(false);
+const exportingData = ref(false);
+const importFileInput = ref<HTMLInputElement | null>(null);
 
 const scholars = ref<ScholarProfile[]>([]);
 const imageUrlDraftByScholarId = ref<Record<number, string>>({});
@@ -197,6 +204,30 @@ function scholarLabel(profile: ScholarProfile): string {
   return profile.display_name || profile.scholar_id;
 }
 
+function suggestExportFilename(exportedAt: string): string {
+  const date = exportedAt.slice(0, 10) || "unknown-date";
+  return `scholarr-export-${date}.json`;
+}
+
+function downloadJsonFile(filename: string, payload: unknown): void {
+  const blob = new Blob([JSON.stringify(payload, null, 2)], { type: "application/json" });
+  const url = URL.createObjectURL(blob);
+  const anchor = document.createElement("a");
+  anchor.href = url;
+  anchor.download = filename;
+  anchor.click();
+  URL.revokeObjectURL(url);
+}
+
+function importSummary(result: DataImportResult): string {
+  return (
+    `Import complete. Scholars +${result.scholars_created}` +
+    ` / updated ${result.scholars_updated}; publications +${result.publications_created}` +
+    ` / updated ${result.publications_updated}; links +${result.links_created}` +
+    ` / updated ${result.links_updated}; skipped ${result.skipped_records}.`
+  );
+}
+
 function isImageBusy(scholarProfileId: number): boolean {
   return (
     imageSavingScholarId.value === scholarProfileId ||
@@ -246,6 +277,74 @@ async function loadScholars(): Promise<void> {
   }
 }
 
+async function onExportData(): Promise<void> {
+  exportingData.value = true;
+  errorMessage.value = null;
+  errorRequestId.value = null;
+  successMessage.value = null;
+
+  try {
+    const payload = await exportScholarData();
+    downloadJsonFile(suggestExportFilename(payload.exported_at), payload);
+    successMessage.value = "Export complete.";
+  } catch (error) {
+    if (error instanceof ApiRequestError) {
+      errorMessage.value = error.message;
+      errorRequestId.value = error.requestId;
+    } else {
+      errorMessage.value = "Unable to export scholars and publications.";
+    }
+  } finally {
+    exportingData.value = false;
+  }
+}
+
+function onOpenImportPicker(): void {
+  importFileInput.value?.click();
+}
+
+function parseImportedJson(raw: string): DataImportPayload {
+  const parsed = JSON.parse(raw) as DataImportPayload;
+  if (!parsed || !Array.isArray(parsed.scholars) || !Array.isArray(parsed.publications)) {
+    throw new Error("Invalid import file: expected scholars[] and publications[] arrays.");
+  }
+  return parsed;
+}
+
+async function onImportFileSelected(event: Event): Promise<void> {
+  const input = event.target as HTMLInputElement | null;
+  const file = input?.files?.[0] ?? null;
+  if (!file) {
+    return;
+  }
+
+  importingData.value = true;
+  errorMessage.value = null;
+  errorRequestId.value = null;
+  successMessage.value = null;
+
+  try {
+    const payload = parseImportedJson(await file.text());
+    const result = await importScholarData(payload);
+    successMessage.value = importSummary(result);
+    await loadScholars();
+  } catch (error) {
+    if (error instanceof ApiRequestError) {
+      errorMessage.value = error.message;
+      errorRequestId.value = error.requestId;
+    } else if (error instanceof Error) {
+      errorMessage.value = error.message;
+    } else {
+      errorMessage.value = "Unable to import scholars and publications.";
+    }
+  } finally {
+    importingData.value = false;
+    if (input) {
+      input.value = "";
+    }
+  }
+}
+
 async function onAddScholars(): Promise<void> {
   saving.value = true;
   errorMessage.value = null;
@@ -706,11 +805,24 @@ onMounted(() => {
                   >
                     {{ trackedCountLabel }}
                   </span>
+                  <AppButton variant="secondary" :disabled="loading || exportingData" @click="onExportData">
+                    {{ exportingData ? "Exporting..." : "Export" }}
+                  </AppButton>
+                  <AppButton variant="secondary" :disabled="loading || importingData" @click="onOpenImportPicker">
+                    {{ importingData ? "Importing..." : "Import" }}
+                  </AppButton>
                   <AppButton variant="secondary" :disabled="loading || saving" @click="loadScholars">
                     {{ loading ? "Refreshing..." : "Refresh" }}
                   </AppButton>
                 </div>
               </div>
+              <input
+                ref="importFileInput"
+                type="file"
+                class="sr-only"
+                accept=".json,application/json"
+                @change="onImportFileSelected"
+              />
 
               <div class="grid gap-2 sm:grid-cols-[minmax(0,1fr)_12rem]">
                 <label class="grid gap-1 text-xs font-medium uppercase tracking-wide text-secondary" for="tracked-scholar-filter">

frontend/src/theme/presets/graphite.json

@@ -1,313 +0,0 @@
-{
-  "id": "graphite",
-  "label": "Graphite",
-  "description": "Neutral slate base with crisp indigo accents.",
-  "modes": {
-    "light": {
-      "scale": {
-        "brand": {
-          "50": "#f8fafc",
-          "100": "#f1f5f9",
-          "200": "#e2e8f0",
-          "300": "#cbd5e1",
-          "400": "#94a3b8",
-          "500": "#64748b",
-          "600": "#475569",
-          "700": "#334155",
-          "800": "#1e293b",
-          "900": "#0f172a",
-          "950": "#020617"
-        },
-        "info": {
-          "50": "#eef2ff",
-          "100": "#e0e7ff",
-          "200": "#c7d2fe",
-          "300": "#a5b4fc",
-          "400": "#818cf8",
-          "500": "#6366f1",
-          "600": "#4f46e5",
-          "700": "#4338ca",
-          "800": "#3730a3",
-          "900": "#312e81",
-          "950": "#1e1b4b"
-        },
-        "success": {
-          "50": "#ecfdf5",
-          "100": "#d1fae5",
-          "200": "#a7f3d0",
-          "300": "#6ee7b7",
-          "400": "#34d399",
-          "500": "#10b981",
-          "600": "#059669",
-          "700": "#047857",
-          "800": "#065f46",
-          "900": "#064e3b",
-          "950": "#022c22"
-        },
-        "warning": {
-          "50": "#fffbeb",
-          "100": "#fef3c7",
-          "200": "#fde68a",
-          "300": "#fcd34d",
-          "400": "#fbbf24",
-          "500": "#f59e0b",
-          "600": "#d97706",
-          "700": "#b45309",
-          "800": "#92400e",
-          "900": "#78350f",
-          "950": "#451a03"
-        },
-        "danger": {
-          "50": "#fff1f2",
-          "100": "#ffe4e6",
-          "200": "#fecdd3",
-          "300": "#fda4af",
-          "400": "#fb7185",
-          "500": "#f43f5e",
-          "600": "#e11d48",
-          "700": "#be123c",
-          "800": "#9f1239",
-          "900": "#881337",
-          "950": "#4c0519"
-        }
-      },
-      "surface": {
-        "app": "#f5f7fb",
-        "nav": "#ffffff",
-        "nav_active": "#334155",
-        "card": "#ffffff",
-        "card_muted": "#f1f5f9",
-        "table": "#ffffff",
-        "table_header": "#f8fafc",
-        "input": "#ffffff",
-        "overlay": "#0f172a"
-      },
-      "text": {
-        "primary": "#111827",
-        "secondary": "#374151",
-        "muted": "#6b7280",
-        "inverse": "#f9fafb",
-        "link": "#334155"
-      },
-      "border": {
-        "default": "#dbe2ea",
-        "strong": "#cbd5e1",
-        "subtle": "#eef2f7",
-        "interactive": "#94a3b8"
-      },
-      "focus": {
-        "ring": "#6366f1",
-        "ring_offset": "#f5f7fb"
-      },
-      "action": {
-        "primary": {
-          "bg": "#334155",
-          "border": "#334155",
-          "text": "#f9fafb",
-          "hover_bg": "#1e293b",
-          "hover_border": "#1e293b",
-          "hover_text": "#ffffff"
-        },
-        "secondary": {
-          "bg": "#f3f4f6",
-          "border": "#d1d5db",
-          "text": "#1f2937",
-          "hover_bg": "#e5e7eb",
-          "hover_border": "#cbd5e1",
-          "hover_text": "#111827"
-        },
-        "ghost": {
-          "bg": "#f8fafc",
-          "border": "#cbd5e1",
-          "text": "#334155",
-          "hover_bg": "#f1f5f9",
-          "hover_border": "#94a3b8",
-          "hover_text": "#1f2937"
-        },
-        "danger": {
-          "bg": "#ffe4e6",
-          "border": "#fecdd3",
-          "text": "#be123c",
-          "hover_bg": "#fecdd3",
-          "hover_border": "#fda4af",
-          "hover_text": "#9f1239"
-        }
-      },
-      "state": {
-        "info": {
-          "bg": "#eef2ff",
-          "border": "#c7d2fe",
-          "text": "#3730a3"
-        },
-        "success": {
-          "bg": "#ecfdf5",
-          "border": "#a7f3d0",
-          "text": "#047857"
-        },
-        "warning": {
-          "bg": "#fffbeb",
-          "border": "#fde68a",
-          "text": "#92400e"
-        },
-        "danger": {
-          "bg": "#fff1f2",
-          "border": "#fecdd3",
-          "text": "#be123c"
-        }
-      }
-    },
-    "dark": {
-      "scale": {
-        "brand": {
-          "50": "#f8fafc",
-          "100": "#f1f5f9",
-          "200": "#e2e8f0",
-          "300": "#cbd5e1",
-          "400": "#94a3b8",
-          "500": "#64748b",
-          "600": "#475569",
-          "700": "#334155",
-          "800": "#1e293b",
-          "900": "#0f172a",
-          "950": "#020617"
-        },
-        "info": {
-          "50": "#eef2ff",
-          "100": "#e0e7ff",
-          "200": "#c7d2fe",
-          "300": "#a5b4fc",
-          "400": "#818cf8",
-          "500": "#6366f1",
-          "600": "#4f46e5",
-          "700": "#4338ca",
-          "800": "#3730a3",
-          "900": "#312e81",
-          "950": "#1e1b4b"
-        },
-        "success": {
-          "50": "#ecfdf5",
-          "100": "#d1fae5",
-          "200": "#a7f3d0",
-          "300": "#6ee7b7",
-          "400": "#34d399",
-          "500": "#10b981",
-          "600": "#059669",
-          "700": "#047857",
-          "800": "#065f46",
-          "900": "#064e3b",
-          "950": "#022c22"
-        },
-        "warning": {
-          "50": "#fffbeb",
-          "100": "#fef3c7",
-          "200": "#fde68a",
-          "300": "#fcd34d",
-          "400": "#fbbf24",
-          "500": "#f59e0b",
-          "600": "#d97706",
-          "700": "#b45309",
-          "800": "#92400e",
-          "900": "#78350f",
-          "950": "#451a03"
-        },
-        "danger": {
-          "50": "#fff1f2",
-          "100": "#ffe4e6",
-          "200": "#fecdd3",
-          "300": "#fda4af",
-          "400": "#fb7185",
-          "500": "#f43f5e",
-          "600": "#e11d48",
-          "700": "#be123c",
-          "800": "#9f1239",
-          "900": "#881337",
-          "950": "#4c0519"
-        }
-      },
-      "surface": {
-        "app": "#0b1120",
-        "nav": "#111827",
-        "nav_active": "#64748b",
-        "card": "#111827",
-        "card_muted": "#1f2937",
-        "table": "#111827",
-        "table_header": "#1f2937",
-        "input": "#0f172a",
-        "overlay": "#020617"
-      },
-      "text": {
-        "primary": "#e5e7eb",
-        "secondary": "#cbd5e1",
-        "muted": "#94a3b8",
-        "inverse": "#0b1120",
-        "link": "#a5b4fc"
-      },
-      "border": {
-        "default": "#334155",
-        "strong": "#475569",
-        "subtle": "#1f2937",
-        "interactive": "#64748b"
-      },
-      "focus": {
-        "ring": "#818cf8",
-        "ring_offset": "#0b1120"
-      },
-      "action": {
-        "primary": {
-          "bg": "#64748b",
-          "border": "#64748b",
-          "text": "#020617",
-          "hover_bg": "#94a3b8",
-          "hover_border": "#94a3b8",
-          "hover_text": "#020617"
-        },
-        "secondary": {
-          "bg": "#1f2937",
-          "border": "#475569",
-          "text": "#e5e7eb",
-          "hover_bg": "#334155",
-          "hover_border": "#64748b",
-          "hover_text": "#f8fafc"
-        },
-        "ghost": {
-          "bg": "#0f172a",
-          "border": "#334155",
-          "text": "#cbd5e1",
-          "hover_bg": "#1f2937",
-          "hover_border": "#475569",
-          "hover_text": "#f8fafc"
-        },
-        "danger": {
-          "bg": "#4c0519",
-          "border": "#881337",
-          "text": "#fecdd3",
-          "hover_bg": "#9f1239",
-          "hover_border": "#be123c",
-          "hover_text": "#fff1f2"
-        }
-      },
-      "state": {
-        "info": {
-          "bg": "#1e1b4b",
-          "border": "#4338ca",
-          "text": "#c7d2fe"
-        },
-        "success": {
-          "bg": "#022c22",
-          "border": "#047857",
-          "text": "#a7f3d0"
-        },
-        "warning": {
-          "bg": "#451a03",
-          "border": "#b45309",
-          "text": "#fde68a"
-        },
-        "danger": {
-          "bg": "#4c0519",
-          "border": "#be123c",
-          "text": "#fecdd3"
-        }
-      }
-    }
-  }
-}

frontend/src/theme/presets/scholarly.json

@@ -1,313 +0,0 @@
-{
-  "id": "scholarly",
-  "label": "Scholarly",
-  "description": "Calm, academic blues with restrained contrast.",
-  "modes": {
-    "light": {
-      "scale": {
-        "brand": {
-          "50": "#f3f6fb",
-          "100": "#e7edf7",
-          "200": "#cedcee",
-          "300": "#afc6df",
-          "400": "#8eaac8",
-          "500": "#6f8fac",
-          "600": "#5a7490",
-          "700": "#495f77",
-          "800": "#3f5063",
-          "900": "#374554",
-          "950": "#242d38"
-        },
-        "info": {
-          "50": "#f0f9ff",
-          "100": "#e0f2fe",
-          "200": "#bae6fd",
-          "300": "#7dd3fc",
-          "400": "#38bdf8",
-          "500": "#0ea5e9",
-          "600": "#0284c7",
-          "700": "#0369a1",
-          "800": "#075985",
-          "900": "#0c4a6e",
-          "950": "#082f49"
-        },
-        "success": {
-          "50": "#ecfdf5",
-          "100": "#d1fae5",
-          "200": "#a7f3d0",
-          "300": "#6ee7b7",
-          "400": "#34d399",
-          "500": "#10b981",
-          "600": "#059669",
-          "700": "#047857",
-          "800": "#065f46",
-          "900": "#064e3b",
-          "950": "#022c22"
-        },
-        "warning": {
-          "50": "#fffbeb",
-          "100": "#fef3c7",
-          "200": "#fde68a",
-          "300": "#fcd34d",
-          "400": "#fbbf24",
-          "500": "#f59e0b",
-          "600": "#d97706",
-          "700": "#b45309",
-          "800": "#92400e",
-          "900": "#78350f",
-          "950": "#451a03"
-        },
-        "danger": {
-          "50": "#fff1f2",
-          "100": "#ffe4e6",
-          "200": "#fecdd3",
-          "300": "#fda4af",
-          "400": "#fb7185",
-          "500": "#f43f5e",
-          "600": "#e11d48",
-          "700": "#be123c",
-          "800": "#9f1239",
-          "900": "#881337",
-          "950": "#4c0519"
-        }
-      },
-      "surface": {
-        "app": "#f3f6fb",
-        "nav": "#ffffff",
-        "nav_active": "#495f77",
-        "card": "#ffffff",
-        "card_muted": "#e7edf7",
-        "table": "#ffffff",
-        "table_header": "#f3f6fb",
-        "input": "#ffffff",
-        "overlay": "#0f172a"
-      },
-      "text": {
-        "primary": "#1f2937",
-        "secondary": "#475569",
-        "muted": "#64748b",
-        "inverse": "#f8fafc",
-        "link": "#495f77"
-      },
-      "border": {
-        "default": "#cedcee",
-        "strong": "#afc6df",
-        "subtle": "#e7edf7",
-        "interactive": "#8eaac8"
-      },
-      "focus": {
-        "ring": "#6f8fac",
-        "ring_offset": "#f3f6fb"
-      },
-      "action": {
-        "primary": {
-          "bg": "#5a7490",
-          "border": "#5a7490",
-          "text": "#f8fafc",
-          "hover_bg": "#495f77",
-          "hover_border": "#495f77",
-          "hover_text": "#ffffff"
-        },
-        "secondary": {
-          "bg": "#eef2f7",
-          "border": "#cedcee",
-          "text": "#334155",
-          "hover_bg": "#e2e8f0",
-          "hover_border": "#afc6df",
-          "hover_text": "#1f2937"
-        },
-        "ghost": {
-          "bg": "#f3f6fb",
-          "border": "#cedcee",
-          "text": "#495f77",
-          "hover_bg": "#e7edf7",
-          "hover_border": "#afc6df",
-          "hover_text": "#374554"
-        },
-        "danger": {
-          "bg": "#ffe4e6",
-          "border": "#fecdd3",
-          "text": "#be123c",
-          "hover_bg": "#fecdd3",
-          "hover_border": "#fda4af",
-          "hover_text": "#9f1239"
-        }
-      },
-      "state": {
-        "info": {
-          "bg": "#f0f9ff",
-          "border": "#bae6fd",
-          "text": "#075985"
-        },
-        "success": {
-          "bg": "#ecfdf5",
-          "border": "#a7f3d0",
-          "text": "#047857"
-        },
-        "warning": {
-          "bg": "#fffbeb",
-          "border": "#fde68a",
-          "text": "#92400e"
-        },
-        "danger": {
-          "bg": "#fff1f2",
-          "border": "#fecdd3",
-          "text": "#be123c"
-        }
-      }
-    },
-    "dark": {
-      "scale": {
-        "brand": {
-          "50": "#f3f6fb",
-          "100": "#e7edf7",
-          "200": "#cedcee",
-          "300": "#afc6df",
-          "400": "#8eaac8",
-          "500": "#6f8fac",
-          "600": "#5a7490",
-          "700": "#495f77",
-          "800": "#3f5063",
-          "900": "#374554",
-          "950": "#242d38"
-        },
-        "info": {
-          "50": "#f0f9ff",
-          "100": "#e0f2fe",
-          "200": "#bae6fd",
-          "300": "#7dd3fc",
-          "400": "#38bdf8",
-          "500": "#0ea5e9",
-          "600": "#0284c7",
-          "700": "#0369a1",
-          "800": "#075985",
-          "900": "#0c4a6e",
-          "950": "#082f49"
-        },
-        "success": {
-          "50": "#ecfdf5",
-          "100": "#d1fae5",
-          "200": "#a7f3d0",
-          "300": "#6ee7b7",
-          "400": "#34d399",
-          "500": "#10b981",
-          "600": "#059669",
-          "700": "#047857",
-          "800": "#065f46",
-          "900": "#064e3b",
-          "950": "#022c22"
-        },
-        "warning": {
-          "50": "#fffbeb",
-          "100": "#fef3c7",
-          "200": "#fde68a",
-          "300": "#fcd34d",
-          "400": "#fbbf24",
-          "500": "#f59e0b",
-          "600": "#d97706",
-          "700": "#b45309",
-          "800": "#92400e",
-          "900": "#78350f",
-          "950": "#451a03"
-        },
-        "danger": {
-          "50": "#fff1f2",
-          "100": "#ffe4e6",
-          "200": "#fecdd3",
-          "300": "#fda4af",
-          "400": "#fb7185",
-          "500": "#f43f5e",
-          "600": "#e11d48",
-          "700": "#be123c",
-          "800": "#9f1239",
-          "900": "#881337",
-          "950": "#4c0519"
-        }
-      },
-      "surface": {
-        "app": "#0b1220",
-        "nav": "#111827",
-        "nav_active": "#8eaac8",
-        "card": "#111827",
-        "card_muted": "#1f2937",
-        "table": "#111827",
-        "table_header": "#1f2937",
-        "input": "#0f172a",
-        "overlay": "#020617"
-      },
-      "text": {
-        "primary": "#e5e7eb",
-        "secondary": "#cbd5e1",
-        "muted": "#94a3b8",
-        "inverse": "#0b1220",
-        "link": "#afc6df"
-      },
-      "border": {
-        "default": "#334155",
-        "strong": "#475569",
-        "subtle": "#1f2937",
-        "interactive": "#64748b"
-      },
-      "focus": {
-        "ring": "#8eaac8",
-        "ring_offset": "#0b1220"
-      },
-      "action": {
-        "primary": {
-          "bg": "#8eaac8",
-          "border": "#8eaac8",
-          "text": "#0b1220",
-          "hover_bg": "#afc6df",
-          "hover_border": "#afc6df",
-          "hover_text": "#020617"
-        },
-        "secondary": {
-          "bg": "#1f2937",
-          "border": "#475569",
-          "text": "#e5e7eb",
-          "hover_bg": "#334155",
-          "hover_border": "#64748b",
-          "hover_text": "#f8fafc"
-        },
-        "ghost": {
-          "bg": "#0f172a",
-          "border": "#334155",
-          "text": "#cbd5e1",
-          "hover_bg": "#1f2937",
-          "hover_border": "#475569",
-          "hover_text": "#f8fafc"
-        },
-        "danger": {
-          "bg": "#4c0519",
-          "border": "#881337",
-          "text": "#fecdd3",
-          "hover_bg": "#9f1239",
-          "hover_border": "#be123c",
-          "hover_text": "#fff1f2"
-        }
-      },
-      "state": {
-        "info": {
-          "bg": "#082f49",
-          "border": "#0369a1",
-          "text": "#bae6fd"
-        },
-        "success": {
-          "bg": "#022c22",
-          "border": "#047857",
-          "text": "#a7f3d0"
-        },
-        "warning": {
-          "bg": "#451a03",
-          "border": "#b45309",
-          "text": "#fde68a"
-        },
-        "danger": {
-          "bg": "#4c0519",
-          "border": "#be123c",
-          "text": "#fecdd3"
-        }
-      }
-    }
-  }
-}

frontend/src/theme/presets/tide.json

@@ -1,313 +0,0 @@
-{
-  "id": "tide",
-  "label": "Tide",
-  "description": "Ocean-inspired palette with teal emphasis.",
-  "modes": {
-    "light": {
-      "scale": {
-        "brand": {
-          "50": "#f0fdfa",
-          "100": "#ccfbf1",
-          "200": "#99f6e4",
-          "300": "#5eead4",
-          "400": "#2dd4bf",
-          "500": "#14b8a6",
-          "600": "#0d9488",
-          "700": "#0f766e",
-          "800": "#115e59",
-          "900": "#134e4a",
-          "950": "#042f2e"
-        },
-        "info": {
-          "50": "#f0f9ff",
-          "100": "#e0f2fe",
-          "200": "#bae6fd",
-          "300": "#7dd3fc",
-          "400": "#38bdf8",
-          "500": "#0ea5e9",
-          "600": "#0284c7",
-          "700": "#0369a1",
-          "800": "#075985",
-          "900": "#0c4a6e",
-          "950": "#082f49"
-        },
-        "success": {
-          "50": "#ecfdf5",
-          "100": "#d1fae5",
-          "200": "#a7f3d0",
-          "300": "#6ee7b7",
-          "400": "#34d399",
-          "500": "#10b981",
-          "600": "#059669",
-          "700": "#047857",
-          "800": "#065f46",
-          "900": "#064e3b",
-          "950": "#022c22"
-        },
-        "warning": {
-          "50": "#fffbeb",
-          "100": "#fef3c7",
-          "200": "#fde68a",
-          "300": "#fcd34d",
-          "400": "#fbbf24",
-          "500": "#f59e0b",
-          "600": "#d97706",
-          "700": "#b45309",
-          "800": "#92400e",
-          "900": "#78350f",
-          "950": "#451a03"
-        },
-        "danger": {
-          "50": "#fff1f2",
-          "100": "#ffe4e6",
-          "200": "#fecdd3",
-          "300": "#fda4af",
-          "400": "#fb7185",
-          "500": "#f43f5e",
-          "600": "#e11d48",
-          "700": "#be123c",
-          "800": "#9f1239",
-          "900": "#881337",
-          "950": "#4c0519"
-        }
-      },
-      "surface": {
-        "app": "#f0fdfa",
-        "nav": "#ffffff",
-        "nav_active": "#0f766e",
-        "card": "#ffffff",
-        "card_muted": "#ccfbf1",
-        "table": "#ffffff",
-        "table_header": "#f0fdfa",
-        "input": "#ffffff",
-        "overlay": "#0f172a"
-      },
-      "text": {
-        "primary": "#134e4a",
-        "secondary": "#0f766e",
-        "muted": "#115e59",
-        "inverse": "#f8fafc",
-        "link": "#0d9488"
-      },
-      "border": {
-        "default": "#99f6e4",
-        "strong": "#5eead4",
-        "subtle": "#ccfbf1",
-        "interactive": "#2dd4bf"
-      },
-      "focus": {
-        "ring": "#14b8a6",
-        "ring_offset": "#f0fdfa"
-      },
-      "action": {
-        "primary": {
-          "bg": "#0d9488",
-          "border": "#0d9488",
-          "text": "#f8fafc",
-          "hover_bg": "#0f766e",
-          "hover_border": "#0f766e",
-          "hover_text": "#ffffff"
-        },
-        "secondary": {
-          "bg": "#ecfeff",
-          "border": "#99f6e4",
-          "text": "#115e59",
-          "hover_bg": "#ccfbf1",
-          "hover_border": "#5eead4",
-          "hover_text": "#134e4a"
-        },
-        "ghost": {
-          "bg": "#f0fdfa",
-          "border": "#99f6e4",
-          "text": "#0f766e",
-          "hover_bg": "#ccfbf1",
-          "hover_border": "#5eead4",
-          "hover_text": "#115e59"
-        },
-        "danger": {
-          "bg": "#ffe4e6",
-          "border": "#fecdd3",
-          "text": "#be123c",
-          "hover_bg": "#fecdd3",
-          "hover_border": "#fda4af",
-          "hover_text": "#9f1239"
-        }
-      },
-      "state": {
-        "info": {
-          "bg": "#f0f9ff",
-          "border": "#bae6fd",
-          "text": "#075985"
-        },
-        "success": {
-          "bg": "#ecfdf5",
-          "border": "#a7f3d0",
-          "text": "#047857"
-        },
-        "warning": {
-          "bg": "#fffbeb",
-          "border": "#fde68a",
-          "text": "#92400e"
-        },
-        "danger": {
-          "bg": "#fff1f2",
-          "border": "#fecdd3",
-          "text": "#be123c"
-        }
-      }
-    },
-    "dark": {
-      "scale": {
-        "brand": {
-          "50": "#f0fdfa",
-          "100": "#ccfbf1",
-          "200": "#99f6e4",
-          "300": "#5eead4",
-          "400": "#2dd4bf",
-          "500": "#14b8a6",
-          "600": "#0d9488",
-          "700": "#0f766e",
-          "800": "#115e59",
-          "900": "#134e4a",
-          "950": "#042f2e"
-        },
-        "info": {
-          "50": "#f0f9ff",
-          "100": "#e0f2fe",
-          "200": "#bae6fd",
-          "300": "#7dd3fc",
-          "400": "#38bdf8",
-          "500": "#0ea5e9",
-          "600": "#0284c7",
-          "700": "#0369a1",
-          "800": "#075985",
-          "900": "#0c4a6e",
-          "950": "#082f49"
-        },
-        "success": {
-          "50": "#ecfdf5",
-          "100": "#d1fae5",
-          "200": "#a7f3d0",
-          "300": "#6ee7b7",
-          "400": "#34d399",
-          "500": "#10b981",
-          "600": "#059669",
-          "700": "#047857",
-          "800": "#065f46",
-          "900": "#064e3b",
-          "950": "#022c22"
-        },
-        "warning": {
-          "50": "#fffbeb",
-          "100": "#fef3c7",
-          "200": "#fde68a",
-          "300": "#fcd34d",
-          "400": "#fbbf24",
-          "500": "#f59e0b",
-          "600": "#d97706",
-          "700": "#b45309",
-          "800": "#92400e",
-          "900": "#78350f",
-          "950": "#451a03"
-        },
-        "danger": {
-          "50": "#fff1f2",
-          "100": "#ffe4e6",
-          "200": "#fecdd3",
-          "300": "#fda4af",
-          "400": "#fb7185",
-          "500": "#f43f5e",
-          "600": "#e11d48",
-          "700": "#be123c",
-          "800": "#9f1239",
-          "900": "#881337",
-          "950": "#4c0519"
-        }
-      },
-      "surface": {
-        "app": "#041f1f",
-        "nav": "#042f2e",
-        "nav_active": "#5eead4",
-        "card": "#042f2e",
-        "card_muted": "#134e4a",
-        "table": "#042f2e",
-        "table_header": "#134e4a",
-        "input": "#063a38",
-        "overlay": "#020617"
-      },
-      "text": {
-        "primary": "#e6fffb",
-        "secondary": "#c9fff3",
-        "muted": "#99f6e4",
-        "inverse": "#041f1f",
-        "link": "#5eead4"
-      },
-      "border": {
-        "default": "#115e59",
-        "strong": "#0f766e",
-        "subtle": "#134e4a",
-        "interactive": "#2dd4bf"
-      },
-      "focus": {
-        "ring": "#2dd4bf",
-        "ring_offset": "#041f1f"
-      },
-      "action": {
-        "primary": {
-          "bg": "#2dd4bf",
-          "border": "#2dd4bf",
-          "text": "#042f2e",
-          "hover_bg": "#5eead4",
-          "hover_border": "#5eead4",
-          "hover_text": "#042f2e"
-        },
-        "secondary": {
-          "bg": "#0f766e",
-          "border": "#14b8a6",
-          "text": "#e6fffb",
-          "hover_bg": "#0d9488",
-          "hover_border": "#2dd4bf",
-          "hover_text": "#ffffff"
-        },
-        "ghost": {
-          "bg": "#042f2e",
-          "border": "#0f766e",
-          "text": "#c9fff3",
-          "hover_bg": "#115e59",
-          "hover_border": "#14b8a6",
-          "hover_text": "#ffffff"
-        },
-        "danger": {
-          "bg": "#4c0519",
-          "border": "#881337",
-          "text": "#fecdd3",
-          "hover_bg": "#9f1239",
-          "hover_border": "#be123c",
-          "hover_text": "#fff1f2"
-        }
-      },
-      "state": {
-        "info": {
-          "bg": "#082f49",
-          "border": "#0369a1",
-          "text": "#bae6fd"
-        },
-        "success": {
-          "bg": "#022c22",
-          "border": "#047857",
-          "text": "#a7f3d0"
-        },
-        "warning": {
-          "bg": "#451a03",
-          "border": "#b45309",
-          "text": "#fde68a"
-        },
-        "danger": {
-          "bg": "#4c0519",
-          "border": "#be123c",
-          "text": "#fecdd3"
-        }
-      }
-    }
-  }
-}

tests/integration/test_api_v1.py

@@ -499,6 +499,177 @@ async def test_api_scholars_search_and_profile_image_management(
         )
 
 
+@pytest.mark.integration
+@pytest.mark.db
+@pytest.mark.asyncio
+async def test_api_scholar_import_export_round_trip(
+    db_session: AsyncSession,
+) -> None:
+    user_id = await insert_user(
+        db_session,
+        email="api-import-export@example.com",
+        password="api-password",
+    )
+    scholar_result = await db_session.execute(
+        text(
+            """
+            INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
+            VALUES (:user_id, :scholar_id, :display_name, true)
+            RETURNING id
+            """
+        ),
+        {
+            "user_id": user_id,
+            "scholar_id": "abcDEF123456",
+            "display_name": "Existing Scholar",
+        },
+    )
+    scholar_profile_id = int(scholar_result.scalar_one())
+    publication_result = await db_session.execute(
+        text(
+            """
+            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
+            VALUES (:fingerprint, :title_raw, :title_normalized, 1)
+            RETURNING id
+            """
+        ),
+        {
+            "fingerprint": f"{(user_id + 500):064x}",
+            "title_raw": "Existing Publication",
+            "title_normalized": "existingpublication",
+        },
+    )
+    publication_id = int(publication_result.scalar_one())
+    await db_session.execute(
+        text(
+            """
+            INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
+            VALUES (:scholar_profile_id, :publication_id, false)
+            """
+        ),
+        {
+            "scholar_profile_id": scholar_profile_id,
+            "publication_id": publication_id,
+        },
+    )
+    await db_session.commit()
+
+    client = TestClient(app)
+    login_user(client, email="api-import-export@example.com", password="api-password")
+    headers = _api_csrf_headers(client)
+
+    export_response = client.get("/api/v1/scholars/export")
+    assert export_response.status_code == 200
+    export_payload = export_response.json()["data"]
+    assert export_payload["schema_version"] == 1
+    assert len(export_payload["scholars"]) == 1
+    assert len(export_payload["publications"]) == 1
+
+    import_response = client.post(
+        "/api/v1/scholars/import",
+        json={
+            "schema_version": 1,
+            "scholars": [
+                {
+                    "scholar_id": "abcDEF123456",
+                    "display_name": "Updated Scholar",
+                    "is_enabled": False,
+                    "profile_image_override_url": "https://cdn.example.com/avatar.png",
+                },
+                {
+                    "scholar_id": "zzzYYY111222",
+                    "display_name": "Imported Scholar",
+                    "is_enabled": True,
+                    "profile_image_override_url": None,
+                },
+            ],
+            "publications": [
+                {
+                    "scholar_id": "abcDEF123456",
+                    "title": "Existing Publication",
+                    "year": 2024,
+                    "citation_count": 8,
+                    "author_text": "A. Author",
+                    "venue_text": "Test Venue",
+                    "pub_url": "https://example.org/existing",
+                    "pdf_url": "https://example.org/existing.pdf",
+                    "is_read": True,
+                },
+                {
+                    "scholar_id": "zzzYYY111222",
+                    "title": "Imported Publication",
+                    "year": 2025,
+                    "citation_count": 2,
+                    "author_text": "B. Author",
+                    "venue_text": "Another Venue",
+                    "pub_url": "https://example.org/imported",
+                    "pdf_url": "https://example.org/imported.pdf",
+                    "is_read": False,
+                },
+            ],
+        },
+        headers=headers,
+    )
+    assert import_response.status_code == 200
+    import_data = import_response.json()["data"]
+    assert int(import_data["scholars_created"]) == 1
+    assert int(import_data["scholars_updated"]) >= 1
+    assert int(import_data["publications_created"]) == 1
+    assert int(import_data["links_created"]) == 1
+
+    updated_scholar_result = await db_session.execute(
+        text(
+            """
+            SELECT display_name, is_enabled, profile_image_override_url
+            FROM scholar_profiles
+            WHERE user_id = :user_id AND scholar_id = :scholar_id
+            """
+        ),
+        {
+            "user_id": user_id,
+            "scholar_id": "abcDEF123456",
+        },
+    )
+    updated_scholar = updated_scholar_result.one()
+    assert updated_scholar[0] == "Updated Scholar"
+    assert updated_scholar[1] is False
+    assert updated_scholar[2] == "https://cdn.example.com/avatar.png"
+
+    imported_pub_result = await db_session.execute(
+        text(
+            """
+            SELECT p.title_raw, p.pdf_url
+            FROM publications p
+            WHERE p.title_raw = :title
+            """
+        ),
+        {"title": "Imported Publication"},
+    )
+    imported_pub = imported_pub_result.one()
+    assert imported_pub[0] == "Imported Publication"
+    assert imported_pub[1] == "https://example.org/imported.pdf"
+
+    updated_link_result = await db_session.execute(
+        text(
+            """
+            SELECT sp.is_read
+            FROM scholar_publications sp
+            JOIN scholar_profiles s ON s.id = sp.scholar_profile_id
+            JOIN publications p ON p.id = sp.publication_id
+            WHERE s.user_id = :user_id
+              AND s.scholar_id = :scholar_id
+              AND p.title_raw = :title
+            """
+        ),
+        {
+            "user_id": user_id,
+            "scholar_id": "abcDEF123456",
+            "title": "Existing Publication",
+        },
+    )
+    assert bool(updated_link_result.scalar_one()) is True
+
+
 @pytest.mark.integration
 @pytest.mark.db
 @pytest.mark.asyncio
@@ -1168,8 +1339,14 @@ async def test_api_publications_list_and_mark_read(db_session: AsyncSession) ->
     publication_a = await db_session.execute(
         text(
             """
-            INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
+            INSERT INTO publications (
-            VALUES (:fingerprint, :title_raw, :title_normalized, 10)
+                fingerprint_sha256,
+                title_raw,
+                title_normalized,
+                citation_count,
+                pdf_url
+            )
+            VALUES (:fingerprint, :title_raw, :title_normalized, 10, :pdf_url)
             RETURNING id
             """
         ),
@@ -1177,6 +1354,7 @@ async def test_api_publications_list_and_mark_read(db_session: AsyncSession) ->
             "fingerprint": f"{user_id:064x}",
             "title_raw": "Paper A",
             "title_normalized": "paper a",
+            "pdf_url": "https://example.org/paper-a.pdf",
         },
     )
     publication_a_id = int(publication_a.scalar_one())
@@ -1228,6 +1406,9 @@ async def test_api_publications_list_and_mark_read(db_session: AsyncSession) ->
     assert data["new_count"] == data["latest_count"]
     assert isinstance(data["publications"], list)
     assert len(data["publications"]) == 2
+    pdf_urls = {item["title"]: item["pdf_url"] for item in data["publications"]}
+    assert pdf_urls["Paper A"] == "https://example.org/paper-a.pdf"
+    assert pdf_urls["Paper B"] is None
 
     latest_response = client.get("/api/v1/publications?mode=latest")
     assert latest_response.status_code == 200

tests/unit/test_scholar_parser.py

@@ -98,6 +98,74 @@ def test_parse_profile_page_handles_missing_optional_metadata() -> None:
     assert publication.venue_text is None
 
 
+def test_parse_profile_page_extracts_direct_pdf_link() -> None:
+    html = """
+    <html>
+      <div id="gsc_prf_in">Direct PDF Test</div>
+      <span id="gsc_a_nn">Articles 1-1</span>
+      <table>
+        <tbody id="gsc_a_b">
+          <tr class="gsc_a_tr">
+            <td class="gsc_a_t">
+              <a class="gsc_a_at" href="/citations?view_op=view_citation&citation_for_view=abc:def999">Paper</a>
+              <div class="gs_ggs gs_fl">
+                <a href="https://example.org/paper.pdf">[PDF]</a>
+              </div>
+            </td>
+            <td class="gsc_a_c"><a class="gsc_a_ac">3</a></td>
+            <td class="gsc_a_y"><span class="gsc_a_h">2025</span></td>
+          </tr>
+        </tbody>
+      </table>
+    </html>
+    """
+    fetch_result = FetchResult(
+        requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
+        status_code=200,
+        final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
+        body=html,
+        error=None,
+    )
+
+    parsed = parse_profile_page(fetch_result)
+
+    assert parsed.state == ParseState.OK
+    assert len(parsed.publications) == 1
+    assert parsed.publications[0].pdf_url == "https://example.org/paper.pdf"
+
+
+def test_parse_profile_page_fails_fast_when_citation_markup_is_unparseable() -> None:
+    html = """
+    <html>
+      <div id="gsc_prf_in">Citation Parse Drift</div>
+      <span id="gsc_a_nn">Articles 1-1</span>
+      <table>
+        <tbody id="gsc_a_b">
+          <tr class="gsc_a_tr">
+            <td class="gsc_a_t">
+              <a class="gsc_a_at" href="/citations?view_op=view_citation&citation_for_view=abc:def777">Paper</a>
+            </td>
+            <td class="gsc_a_c"><a class="gsc_a_ac">Cited by none</a></td>
+            <td class="gsc_a_y"><span class="gsc_a_h">2025</span></td>
+          </tr>
+        </tbody>
+      </table>
+    </html>
+    """
+    fetch_result = FetchResult(
+        requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
+        status_code=200,
+        final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
+        body=html,
+        error=None,
+    )
+
+    parsed = parse_profile_page(fetch_result)
+
+    assert parsed.state == ParseState.LAYOUT_CHANGED
+    assert parsed.state_reason == "layout_row_citation_unparseable"
+
+
 def test_parse_profile_page_detects_layout_change_when_markers_absent() -> None:
     fetch_result = FetchResult(
         requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",