decompose application.py into enrichment, scholar processing, run completion

.env.example

@@ -115,6 +115,10 @@ SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD=1
 SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS=1800
 SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD=2
 SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD=3
+SCHOLAR_HTTP_USER_AGENT=
+SCHOLAR_HTTP_ROTATE_USER_AGENT=0
+SCHOLAR_HTTP_ACCEPT_LANGUAGE=en-US,en;q=0.9
+SCHOLAR_HTTP_COOKIE=
 
 # ------------------------------
 # OA Enrichment + PDF Resolution

DECOMPOSITION_SLICES.md

@@ -26,7 +26,7 @@ This file contains self-contained decomposition prompts ("slices") to bring the
 |---|-------|--------|
 | 1 | Schema split | **DONE** |
 | 2 | Ingestion: pagination + publication upsert | **DONE** |
-| 3 | Ingestion: enrichment + scholar processing + run completion | pending |
+| 3 | Ingestion: enrichment + scholar processing + run completion | **DONE** |
 | 4 | Scholars service + PDF queue | pending |
 | 5 | Routers + scheduler | pending |
 
@@ -63,7 +63,17 @@ Also fixed two external import paths (`portability/normalize.py`, `portability/p
 
 ---
 
-## Slice 3: Decompose `app/services/ingestion/application.py` — enrichment, scholar processing, run completion
+## Slice 3: Decompose `app/services/ingestion/application.py` — enrichment, scholar processing, run completion — DONE
+
+Completed. Extracted four logical chunks from `application.py` (2,085 → 635 lines):
+
+| File | Lines | Contents |
+|---|---|---|
+| `app/services/ingestion/enrichment.py` | 323 | `EnrichmentRunner` class — OpenAlex enrichment, identifier discovery, dedup sweep |
+| `app/services/ingestion/scholar_processing.py` | 614 | `process_scholar`, `run_scholar_iteration`, continuation queue, outcome resolution |
+| `app/services/ingestion/run_completion.py` | 417 | `complete_run_for_user`, failure summary, alert summary, safety outcome, progress tracking |
+
+Also updated two tests (`test_ingestion_arxiv_rate_limit.py`, `test_ingestion_progress_reporting.py`) and one integration test (`test_deferred_enrichment.py`) to import from new modules.
 
 **Why:** Three remaining logical chunks. Extracting all three brings `application.py` down to ~500 lines (the orchestration spine).
 

app/api/router.py

@@ -2,11 +2,12 @@ from __future__ import annotations
 
 from fastapi import APIRouter
 
-from app.api.routers import admin, admin_dbops, auth, publications, runs, scholars, settings
+from app.api.routers import admin, admin_dbops, admin_settings, auth, publications, runs, scholars, settings
 
 router = APIRouter(prefix="/api/v1")
 router.include_router(auth.router)
 router.include_router(admin.router)
+router.include_router(admin_settings.router)
 router.include_router(admin_dbops.router)
 router.include_router(scholars.router)
 router.include_router(settings.router)

app/api/routers/admin_settings.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import logging
+
+from fastapi import APIRouter, Depends, Request
+
+from app.api.deps import get_api_admin_user
+from app.api.errors import ApiException
+from app.api.responses import success_payload
+from app.api.schemas import (
+    AdminScholarHttpSettingsEnvelope,
+    AdminScholarHttpSettingsUpdateRequest,
+)
+from app.db.models import User
+from app.logging_utils import structured_log
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/admin/settings", tags=["api-admin-settings"])
+_MAX_USER_AGENT_LENGTH = 512
+_MAX_ACCEPT_LANGUAGE_LENGTH = 128
+_MAX_COOKIE_LENGTH = 8192
+
+
+def _normalize_header_value(value: str, *, field_name: str, max_length: int) -> str:
+    normalized = value.strip()
+    if len(normalized) > max_length:
+        raise ApiException(
+            status_code=400,
+            code="invalid_admin_setting",
+            message=f"{field_name} must be {max_length} characters or fewer.",
+        )
+    return normalized
+
+
+def _serialize_scholar_http_settings() -> dict[str, object]:
+    return {
+        "user_agent": settings.scholar_http_user_agent,
+        "rotate_user_agent": bool(settings.scholar_http_rotate_user_agent),
+        "accept_language": settings.scholar_http_accept_language,
+        "cookie": settings.scholar_http_cookie,
+    }
+
+
+def _apply_scholar_http_settings(payload: AdminScholarHttpSettingsUpdateRequest) -> None:
+    user_agent = _normalize_header_value(
+        payload.user_agent,
+        field_name="user_agent",
+        max_length=_MAX_USER_AGENT_LENGTH,
+    )
+    accept_language = _normalize_header_value(
+        payload.accept_language,
+        field_name="accept_language",
+        max_length=_MAX_ACCEPT_LANGUAGE_LENGTH,
+    )
+    cookie = _normalize_header_value(
+        payload.cookie,
+        field_name="cookie",
+        max_length=_MAX_COOKIE_LENGTH,
+    )
+    object.__setattr__(settings, "scholar_http_user_agent", user_agent)
+    object.__setattr__(settings, "scholar_http_rotate_user_agent", bool(payload.rotate_user_agent))
+    object.__setattr__(settings, "scholar_http_accept_language", accept_language)
+    object.__setattr__(settings, "scholar_http_cookie", cookie)
+
+
+@router.get(
+    "/scholar-http",
+    response_model=AdminScholarHttpSettingsEnvelope,
+)
+async def get_scholar_http_settings(
+    request: Request,
+    admin_user: User = Depends(get_api_admin_user),
+):
+    structured_log(
+        logger,
+        "info",
+        "api.admin.settings.scholar_http_read",
+        admin_user_id=int(admin_user.id),
+    )
+    return success_payload(request, data=_serialize_scholar_http_settings())
+
+
+@router.put(
+    "/scholar-http",
+    response_model=AdminScholarHttpSettingsEnvelope,
+)
+async def update_scholar_http_settings(
+    payload: AdminScholarHttpSettingsUpdateRequest,
+    request: Request,
+    admin_user: User = Depends(get_api_admin_user),
+):
+    _apply_scholar_http_settings(payload)
+    structured_log(
+        logger,
+        "info",
+        "api.admin.settings.scholar_http_updated",
+        admin_user_id=int(admin_user.id),
+        rotate_user_agent=bool(settings.scholar_http_rotate_user_agent),
+        user_agent_set=bool(settings.scholar_http_user_agent.strip()),
+        cookie_set=bool(settings.scholar_http_cookie.strip()),
+    )
+    return success_payload(request, data=_serialize_scholar_http_settings())

app/api/schemas.py

@@ -1,945 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Any, Literal
-
-from pydantic import BaseModel, ConfigDict, Field, model_validator
-
-
-class ApiMeta(BaseModel):
-    request_id: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ApiErrorData(BaseModel):
-    code: str
-    message: str
-    details: Any | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ApiErrorEnvelope(BaseModel):
-    error: ApiErrorData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessageData(BaseModel):
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessageEnvelope(BaseModel):
-    data: MessageData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SessionUserData(BaseModel):
-    id: int
-    email: str
-    is_admin: bool
-    is_active: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AuthMeData(BaseModel):
-    authenticated: bool
-    csrf_token: str
-    user: SessionUserData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AuthMeEnvelope(BaseModel):
-    data: AuthMeData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class CsrfBootstrapData(BaseModel):
-    csrf_token: str
-    authenticated: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class CsrfBootstrapEnvelope(BaseModel):
-    data: CsrfBootstrapData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginRequest(BaseModel):
-    email: str
-    password: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginData(BaseModel):
-    authenticated: bool
-    csrf_token: str
-    user: SessionUserData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginEnvelope(BaseModel):
-    data: LoginData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ChangePasswordRequest(BaseModel):
-    current_password: str
-    new_password: str
-    confirm_password: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarItemData(BaseModel):
-    id: int
-    scholar_id: str
-    display_name: str | None
-    profile_image_url: str | None
-    profile_image_source: str
-    is_enabled: bool
-    baseline_completed: bool
-    last_run_dt: datetime | None
-    last_run_status: str | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarsListData(BaseModel):
-    scholars: list[ScholarItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarsListEnvelope(BaseModel):
-    data: ScholarsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarCreateRequest(BaseModel):
-    scholar_id: str
-    profile_image_url: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarEnvelope(BaseModel):
-    data: ScholarItemData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchCandidateData(BaseModel):
-    scholar_id: str
-    display_name: str
-    affiliation: str | None
-    email_domain: str | None
-    cited_by_count: int | None
-    interests: list[str] = Field(default_factory=list)
-    profile_url: str
-    profile_image_url: str | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchData(BaseModel):
-    query: str
-    state: str
-    state_reason: str
-    action_hint: str | None = None
-    candidates: list[ScholarSearchCandidateData]
-    warnings: list[str] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchEnvelope(BaseModel):
-    data: ScholarSearchData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarImageUrlUpdateRequest(BaseModel):
-    image_url: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarExportItemData(BaseModel):
-    scholar_id: str
-    display_name: str | None = None
-    is_enabled: bool = True
-    profile_image_override_url: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationExportItemData(BaseModel):
-    scholar_id: str
-    cluster_id: str | None = None
-    fingerprint_sha256: str | None = None
-    title: str
-    year: int | None = None
-    citation_count: int = 0
-    author_text: str | None = None
-    venue_text: str | None = None
-    pub_url: str | None = None
-    pdf_url: str | None = None
-    is_read: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataExportData(BaseModel):
-    schema_version: int
-    exported_at: str
-    scholars: list[ScholarExportItemData]
-    publications: list[PublicationExportItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataExportEnvelope(BaseModel):
-    data: DataExportData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataImportRequest(BaseModel):
-    schema_version: int | None = None
-    scholars: list[ScholarExportItemData] = Field(default_factory=list)
-    publications: list[PublicationExportItemData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataImportResultData(BaseModel):
-    scholars_created: int
-    scholars_updated: int
-    publications_created: int
-    publications_updated: int
-    links_created: int
-    links_updated: int
-    skipped_records: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataImportEnvelope(BaseModel):
-    data: DataImportResultData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunListItemData(BaseModel):
-    id: int
-    trigger_type: str
-    status: str
-    start_dt: datetime
-    end_dt: datetime | None
-    scholar_count: int
-    new_publication_count: int
-    failed_count: int
-    partial_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunsListData(BaseModel):
-    runs: list[RunListItemData]
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunsListEnvelope(BaseModel):
-    data: RunsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunSummaryData(BaseModel):
-    succeeded_count: int
-    failed_count: int
-    partial_count: int
-    failed_state_counts: dict[str, int] = Field(default_factory=dict)
-    failed_reason_counts: dict[str, int] = Field(default_factory=dict)
-    scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
-    retry_counts: dict[str, int] = Field(default_factory=dict)
-    alert_thresholds: dict[str, int] = Field(default_factory=dict)
-    alert_flags: dict[str, bool] = Field(default_factory=dict)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScrapeSafetyCountersData(BaseModel):
-    consecutive_blocked_runs: int = 0
-    consecutive_network_runs: int = 0
-    cooldown_entry_count: int = 0
-    blocked_start_count: int = 0
-    last_blocked_failure_count: int = 0
-    last_network_failure_count: int = 0
-    last_evaluated_run_id: int | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScrapeSafetyStateData(BaseModel):
-    cooldown_active: bool
-    cooldown_reason: str | None = None
-    cooldown_reason_label: str | None = None
-    cooldown_until: datetime | None = None
-    cooldown_remaining_seconds: int = 0
-    recommended_action: str | None = None
-    counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunAttemptLogData(BaseModel):
-    attempt: int
-    cstart: int
-    state: str | None = None
-    state_reason: str | None = None
-    status_code: int | None = None
-    fetch_error: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunPageLogData(BaseModel):
-    page: int
-    cstart: int
-    state: str
-    state_reason: str | None = None
-    status_code: int | None = None
-    publication_count: int = 0
-    attempt_count: int = 0
-    has_show_more_button: bool = False
-    articles_range: str | None = None
-    warning_codes: list[str] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDebugData(BaseModel):
-    status_code: int | None = None
-    final_url: str | None = None
-    fetch_error: str | None = None
-    body_sha256: str | None = None
-    body_length: int | None = None
-    has_show_more_button: bool | None = None
-    articles_range: str | None = None
-    state_reason: str | None = None
-    warning_codes: list[str] = Field(default_factory=list)
-    marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
-    page_logs: list[RunPageLogData] = Field(default_factory=list)
-    attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunScholarResultData(BaseModel):
-    scholar_profile_id: int
-    scholar_id: str
-    state: str
-    state_reason: str | None = None
-    outcome: str
-    attempt_count: int = 0
-    publication_count: int = 0
-    start_cstart: int = 0
-    continuation_cstart: int | None = None
-    continuation_enqueued: bool = False
-    continuation_cleared: bool = False
-    warnings: list[str] = Field(default_factory=list)
-    error: str | None = None
-    debug: RunDebugData | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDetailData(BaseModel):
-    run: RunListItemData
-    summary: RunSummaryData
-    scholar_results: list[RunScholarResultData]
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDetailEnvelope(BaseModel):
-    data: RunDetailData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ManualRunData(BaseModel):
-    run_id: int
-    status: str
-    scholar_count: int
-    succeeded_count: int
-    failed_count: int
-    partial_count: int
-    new_publication_count: int
-    reused_existing_run: bool
-    idempotency_key: str | None = None
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ManualRunEnvelope(BaseModel):
-    data: ManualRunData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueItemData(BaseModel):
-    id: int
-    scholar_profile_id: int
-    scholar_label: str
-    status: str
-    reason: str
-    dropped_reason: str | None
-    attempt_count: int
-    resume_cstart: int
-    next_attempt_dt: datetime | None
-    updated_at: datetime
-    last_error: str | None
-    last_run_id: int | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueListData(BaseModel):
-    queue_items: list[QueueItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueListEnvelope(BaseModel):
-    data: QueueListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueItemEnvelope(BaseModel):
-    data: QueueItemData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueClearData(BaseModel):
-    queue_item_id: int
-    previous_status: str
-    status: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueClearEnvelope(BaseModel):
-    data: QueueClearData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserData(BaseModel):
-    id: int
-    email: str
-    is_active: bool
-    is_admin: bool
-    created_at: datetime
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUsersListData(BaseModel):
-    users: list[AdminUserData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUsersListEnvelope(BaseModel):
-    data: AdminUsersListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserCreateRequest(BaseModel):
-    email: str
-    password: str
-    is_admin: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserActiveUpdateRequest(BaseModel):
-    is_active: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserEnvelope(BaseModel):
-    data: AdminUserData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminResetPasswordRequest(BaseModel):
-    new_password: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityCheckData(BaseModel):
-    name: str
-    count: int
-    severity: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityData(BaseModel):
-    status: str
-    checked_at: datetime
-    failures: list[str] = Field(default_factory=list)
-    warnings: list[str] = Field(default_factory=list)
-    checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityEnvelope(BaseModel):
-    data: AdminDbIntegrityData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobData(BaseModel):
-    id: int
-    job_name: str
-    requested_by: str | None
-    dry_run: bool
-    status: str
-    scope: dict[str, Any] = Field(default_factory=dict)
-    summary: dict[str, Any] = Field(default_factory=dict)
-    error_text: str | None
-    started_at: datetime | None
-    finished_at: datetime | None
-    created_at: datetime
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobsData(BaseModel):
-    jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobsEnvelope(BaseModel):
-    data: AdminDbRepairJobsData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DisplayIdentifierData(BaseModel):
-    kind: str
-    value: str
-    label: str
-    url: str | None
-    confidence_score: float = Field(ge=0.0, le=1.0)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueItemData(BaseModel):
-    publication_id: int
-    title: str
-    display_identifier: DisplayIdentifierData | None = None
-    pdf_url: str | None
-    status: str
-    attempt_count: int
-    last_failure_reason: str | None
-    last_failure_detail: str | None
-    last_source: str | None
-    requested_by_user_id: int | None
-    requested_by_email: str | None
-    queued_at: datetime | None
-    last_attempt_at: datetime | None
-    resolved_at: datetime | None
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueData(BaseModel):
-    items: list[AdminPdfQueueItemData] = Field(default_factory=list)
-    total_count: int = 0
-    page: int = 1
-    page_size: int = 100
-    has_next: bool = False
-    has_prev: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueEnvelope(BaseModel):
-    data: AdminPdfQueueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueRequeueData(BaseModel):
-    publication_id: int
-    queued: bool
-    status: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueRequeueEnvelope(BaseModel):
-    data: AdminPdfQueueRequeueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueBulkEnqueueData(BaseModel):
-    requested_count: int
-    queued_count: int
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
-    data: AdminPdfQueueBulkEnqueueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationLinksRequest(BaseModel):
-    scope_mode: Literal["single_user", "all_users"] = "single_user"
-    user_id: int | None = Field(default=None, ge=1)
-    scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
-    dry_run: bool = True
-    gc_orphan_publications: bool = False
-    requested_by: str | None = None
-    confirmation_text: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-    @model_validator(mode="after")
-    def validate_scope(self) -> AdminRepairPublicationLinksRequest:
-        if self.scope_mode == "single_user" and self.user_id is None:
-            raise ValueError("user_id is required when scope_mode=single_user.")
-        if self.scope_mode == "all_users" and self.user_id is not None:
-            raise ValueError("user_id must be omitted when scope_mode=all_users.")
-        if not self.dry_run and self.scope_mode == "all_users":
-            expected = "REPAIR ALL USERS"
-            provided = (self.confirmation_text or "").strip()
-            if provided != expected:
-                raise ValueError("confirmation_text must equal 'REPAIR ALL USERS' when applying a repair to all users.")
-        return self
-
-
-class AdminRepairPublicationLinksResultData(BaseModel):
-    job_id: int
-    status: str
-    scope: dict[str, Any] = Field(default_factory=dict)
-    summary: dict[str, Any] = Field(default_factory=dict)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationLinksEnvelope(BaseModel):
-    data: AdminRepairPublicationLinksResultData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminNearDuplicateClusterMemberData(BaseModel):
-    publication_id: int
-    title: str
-    year: int | None
-    citation_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminNearDuplicateClusterData(BaseModel):
-    cluster_key: str
-    winner_publication_id: int
-    member_count: int
-    similarity_score: float = Field(ge=0.0, le=1.0)
-    members: list[AdminNearDuplicateClusterMemberData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationNearDuplicatesRequest(BaseModel):
-    dry_run: bool = True
-    similarity_threshold: float = Field(default=0.78, ge=0.5, le=1.0)
-    min_shared_tokens: int = Field(default=3, ge=1, le=8)
-    max_year_delta: int = Field(default=1, ge=0, le=5)
-    max_clusters: int = Field(default=25, ge=1, le=200)
-    selected_cluster_keys: list[str] = Field(default_factory=list, max_length=200)
-    requested_by: str | None = None
-    confirmation_text: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-    @model_validator(mode="after")
-    def validate_apply_mode(self) -> AdminRepairPublicationNearDuplicatesRequest:
-        if self.dry_run:
-            return self
-        if not self.selected_cluster_keys:
-            raise ValueError("selected_cluster_keys is required when dry_run=false.")
-        expected = "MERGE SELECTED DUPLICATES"
-        provided = (self.confirmation_text or "").strip()
-        if provided != expected:
-            raise ValueError(
-                "confirmation_text must equal 'MERGE SELECTED DUPLICATES' when applying near-duplicate merges."
-            )
-        return self
-
-
-class AdminRepairPublicationNearDuplicatesResultData(BaseModel):
-    job_id: int
-    status: str
-    scope: dict[str, Any] = Field(default_factory=dict)
-    summary: dict[str, Any] = Field(default_factory=dict)
-    clusters: list[AdminNearDuplicateClusterData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationNearDuplicatesEnvelope(BaseModel):
-    data: AdminRepairPublicationNearDuplicatesResultData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsPolicyData(BaseModel):
-    min_run_interval_minutes: int
-    min_request_delay_seconds: int
-    automation_allowed: bool
-    manual_run_allowed: bool
-    blocked_failure_threshold: int
-    network_failure_threshold: int
-    cooldown_blocked_seconds: int
-    cooldown_network_seconds: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsData(BaseModel):
-    auto_run_enabled: bool
-    run_interval_minutes: int
-    request_delay_seconds: int
-    nav_visible_pages: list[str]
-    policy: SettingsPolicyData
-    safety_state: ScrapeSafetyStateData
-
-    openalex_api_key: str | None = None
-    crossref_api_token: str | None = None
-    crossref_api_mailto: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsEnvelope(BaseModel):
-    data: SettingsData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsUpdateRequest(BaseModel):
-    auto_run_enabled: bool
-    run_interval_minutes: int
-    request_delay_seconds: int
-    nav_visible_pages: list[str] | None = None
-
-    openalex_api_key: str | None = None
-    crossref_api_token: str | None = None
-    crossref_api_mailto: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationItemData(BaseModel):
-    publication_id: int
-    scholar_profile_id: int
-    scholar_label: str
-    title: str
-    year: int | None
-    citation_count: int
-    venue_text: str | None
-    pub_url: str | None
-    display_identifier: DisplayIdentifierData | None = None
-    pdf_url: str | None
-    pdf_status: str = "untracked"
-    pdf_attempt_count: int = 0
-    pdf_failure_reason: str | None = None
-    pdf_failure_detail: str | None = None
-    is_read: bool
-    is_favorite: bool = False
-    first_seen_at: datetime
-    is_new_in_latest_run: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationsListData(BaseModel):
-    mode: str
-    favorite_only: bool = False
-    selected_scholar_profile_id: int | None
-    unread_count: int
-    favorites_count: int
-    latest_count: int
-    new_count: int
-    total_count: int
-    page: int
-    page_size: int
-    snapshot: str
-    has_next: bool = False
-    has_prev: bool = False
-    publications: list[PublicationItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationsListEnvelope(BaseModel):
-    data: PublicationsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkAllReadData(BaseModel):
-    message: str
-    updated_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkAllReadEnvelope(BaseModel):
-    data: MarkAllReadData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationSelectionItem(BaseModel):
-    scholar_profile_id: int
-    publication_id: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadRequest(BaseModel):
-    selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadData(BaseModel):
-    message: str
-    requested_count: int
-    updated_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadEnvelope(BaseModel):
-    data: MarkSelectedReadData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfRequest(BaseModel):
-    scholar_profile_id: int = Field(ge=1)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfData(BaseModel):
-    message: str
-    queued: bool
-    resolved_pdf: bool
-    publication: PublicationItemData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfEnvelope(BaseModel):
-    data: RetryPublicationPdfData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteRequest(BaseModel):
-    scholar_profile_id: int = Field(ge=1)
-    is_favorite: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteData(BaseModel):
-    message: str
-    publication: PublicationItemData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteEnvelope(BaseModel):
-    data: TogglePublicationFavoriteData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/__init__.py

@@ -0,0 +1,7 @@
+from app.api.schemas.admin import *  # noqa: F401, F403
+from app.api.schemas.auth import *  # noqa: F401, F403
+from app.api.schemas.common import *  # noqa: F401, F403
+from app.api.schemas.publications import *  # noqa: F401, F403
+from app.api.schemas.runs import *  # noqa: F401, F403
+from app.api.schemas.scholars import *  # noqa: F401, F403
+from app.api.schemas.settings import *  # noqa: F401, F403

app/api/schemas/admin.py

@@ -0,0 +1,314 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from app.api.schemas.common import ApiMeta
+from app.api.schemas.publications import DisplayIdentifierData
+
+
+class AdminUserData(BaseModel):
+    id: int
+    email: str
+    is_active: bool
+    is_admin: bool
+    created_at: datetime
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUsersListData(BaseModel):
+    users: list[AdminUserData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUsersListEnvelope(BaseModel):
+    data: AdminUsersListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserCreateRequest(BaseModel):
+    email: str
+    password: str
+    is_admin: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserActiveUpdateRequest(BaseModel):
+    is_active: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserEnvelope(BaseModel):
+    data: AdminUserData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminResetPasswordRequest(BaseModel):
+    new_password: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminScholarHttpSettingsData(BaseModel):
+    user_agent: str
+    rotate_user_agent: bool
+    accept_language: str
+    cookie: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminScholarHttpSettingsEnvelope(BaseModel):
+    data: AdminScholarHttpSettingsData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminScholarHttpSettingsUpdateRequest(BaseModel):
+    user_agent: str
+    rotate_user_agent: bool
+    accept_language: str
+    cookie: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityCheckData(BaseModel):
+    name: str
+    count: int
+    severity: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityData(BaseModel):
+    status: str
+    checked_at: datetime
+    failures: list[str] = Field(default_factory=list)
+    warnings: list[str] = Field(default_factory=list)
+    checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityEnvelope(BaseModel):
+    data: AdminDbIntegrityData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobData(BaseModel):
+    id: int
+    job_name: str
+    requested_by: str | None
+    dry_run: bool
+    status: str
+    scope: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any] = Field(default_factory=dict)
+    error_text: str | None
+    started_at: datetime | None
+    finished_at: datetime | None
+    created_at: datetime
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobsData(BaseModel):
+    jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobsEnvelope(BaseModel):
+    data: AdminDbRepairJobsData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueItemData(BaseModel):
+    publication_id: int
+    title: str
+    display_identifier: DisplayIdentifierData | None = None
+    pdf_url: str | None
+    status: str
+    attempt_count: int
+    last_failure_reason: str | None
+    last_failure_detail: str | None
+    last_source: str | None
+    requested_by_user_id: int | None
+    requested_by_email: str | None
+    queued_at: datetime | None
+    last_attempt_at: datetime | None
+    resolved_at: datetime | None
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueData(BaseModel):
+    items: list[AdminPdfQueueItemData] = Field(default_factory=list)
+    total_count: int = 0
+    page: int = 1
+    page_size: int = 100
+    has_next: bool = False
+    has_prev: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueEnvelope(BaseModel):
+    data: AdminPdfQueueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueRequeueData(BaseModel):
+    publication_id: int
+    queued: bool
+    status: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueRequeueEnvelope(BaseModel):
+    data: AdminPdfQueueRequeueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueBulkEnqueueData(BaseModel):
+    requested_count: int
+    queued_count: int
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
+    data: AdminPdfQueueBulkEnqueueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationLinksRequest(BaseModel):
+    scope_mode: Literal["single_user", "all_users"] = "single_user"
+    user_id: int | None = Field(default=None, ge=1)
+    scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
+    dry_run: bool = True
+    gc_orphan_publications: bool = False
+    requested_by: str | None = None
+    confirmation_text: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")
+    def validate_scope(self) -> AdminRepairPublicationLinksRequest:
+        if self.scope_mode == "single_user" and self.user_id is None:
+            raise ValueError("user_id is required when scope_mode=single_user.")
+        if self.scope_mode == "all_users" and self.user_id is not None:
+            raise ValueError("user_id must be omitted when scope_mode=all_users.")
+        if not self.dry_run and self.scope_mode == "all_users":
+            expected = "REPAIR ALL USERS"
+            provided = (self.confirmation_text or "").strip()
+            if provided != expected:
+                raise ValueError("confirmation_text must equal 'REPAIR ALL USERS' when applying a repair to all users.")
+        return self
+
+
+class AdminRepairPublicationLinksResultData(BaseModel):
+    job_id: int
+    status: str
+    scope: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationLinksEnvelope(BaseModel):
+    data: AdminRepairPublicationLinksResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminNearDuplicateClusterMemberData(BaseModel):
+    publication_id: int
+    title: str
+    year: int | None
+    citation_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminNearDuplicateClusterData(BaseModel):
+    cluster_key: str
+    winner_publication_id: int
+    member_count: int
+    similarity_score: float = Field(ge=0.0, le=1.0)
+    members: list[AdminNearDuplicateClusterMemberData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationNearDuplicatesRequest(BaseModel):
+    dry_run: bool = True
+    similarity_threshold: float = Field(default=0.78, ge=0.5, le=1.0)
+    min_shared_tokens: int = Field(default=3, ge=1, le=8)
+    max_year_delta: int = Field(default=1, ge=0, le=5)
+    max_clusters: int = Field(default=25, ge=1, le=200)
+    selected_cluster_keys: list[str] = Field(default_factory=list, max_length=200)
+    requested_by: str | None = None
+    confirmation_text: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")
+    def validate_apply_mode(self) -> AdminRepairPublicationNearDuplicatesRequest:
+        if self.dry_run:
+            return self
+        if not self.selected_cluster_keys:
+            raise ValueError("selected_cluster_keys is required when dry_run=false.")
+        expected = "MERGE SELECTED DUPLICATES"
+        provided = (self.confirmation_text or "").strip()
+        if provided != expected:
+            raise ValueError(
+                "confirmation_text must equal 'MERGE SELECTED DUPLICATES' when applying near-duplicate merges."
+            )
+        return self
+
+
+class AdminRepairPublicationNearDuplicatesResultData(BaseModel):
+    job_id: int
+    status: str
+    scope: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any] = Field(default_factory=dict)
+    clusters: list[AdminNearDuplicateClusterData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationNearDuplicatesEnvelope(BaseModel):
+    data: AdminRepairPublicationNearDuplicatesResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/auth.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict
+
+from app.api.schemas.common import ApiMeta
+
+
+class SessionUserData(BaseModel):
+    id: int
+    email: str
+    is_admin: bool
+    is_active: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AuthMeData(BaseModel):
+    authenticated: bool
+    csrf_token: str
+    user: SessionUserData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AuthMeEnvelope(BaseModel):
+    data: AuthMeData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class CsrfBootstrapData(BaseModel):
+    csrf_token: str
+    authenticated: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class CsrfBootstrapEnvelope(BaseModel):
+    data: CsrfBootstrapData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginRequest(BaseModel):
+    email: str
+    password: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginData(BaseModel):
+    authenticated: bool
+    csrf_token: str
+    user: SessionUserData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginEnvelope(BaseModel):
+    data: LoginData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ChangePasswordRequest(BaseModel):
+    current_password: str
+    new_password: str
+    confirm_password: str
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/common.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict
+
+
+class ApiMeta(BaseModel):
+    request_id: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ApiErrorData(BaseModel):
+    code: str
+    message: str
+    details: Any | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ApiErrorEnvelope(BaseModel):
+    error: ApiErrorData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessageData(BaseModel):
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessageEnvelope(BaseModel):
+    data: MessageData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/publications.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.api.schemas.common import ApiMeta
+
+
+class DisplayIdentifierData(BaseModel):
+    kind: str
+    value: str
+    label: str
+    url: str | None
+    confidence_score: float = Field(ge=0.0, le=1.0)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationItemData(BaseModel):
+    publication_id: int
+    scholar_profile_id: int
+    scholar_label: str
+    title: str
+    year: int | None
+    citation_count: int
+    venue_text: str | None
+    pub_url: str | None
+    display_identifier: DisplayIdentifierData | None = None
+    pdf_url: str | None
+    pdf_status: str = "untracked"
+    pdf_attempt_count: int = 0
+    pdf_failure_reason: str | None = None
+    pdf_failure_detail: str | None = None
+    is_read: bool
+    is_favorite: bool = False
+    first_seen_at: datetime
+    is_new_in_latest_run: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationsListData(BaseModel):
+    mode: str
+    favorite_only: bool = False
+    selected_scholar_profile_id: int | None
+    unread_count: int
+    favorites_count: int
+    latest_count: int
+    new_count: int
+    total_count: int
+    page: int
+    page_size: int
+    snapshot: str
+    has_next: bool = False
+    has_prev: bool = False
+    publications: list[PublicationItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationsListEnvelope(BaseModel):
+    data: PublicationsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkAllReadData(BaseModel):
+    message: str
+    updated_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkAllReadEnvelope(BaseModel):
+    data: MarkAllReadData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationSelectionItem(BaseModel):
+    scholar_profile_id: int
+    publication_id: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadRequest(BaseModel):
+    selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadData(BaseModel):
+    message: str
+    requested_count: int
+    updated_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadEnvelope(BaseModel):
+    data: MarkSelectedReadData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfRequest(BaseModel):
+    scholar_profile_id: int = Field(ge=1)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfData(BaseModel):
+    message: str
+    queued: bool
+    resolved_pdf: bool
+    publication: PublicationItemData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfEnvelope(BaseModel):
+    data: RetryPublicationPdfData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteRequest(BaseModel):
+    scholar_profile_id: int = Field(ge=1)
+    is_favorite: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteData(BaseModel):
+    message: str
+    publication: PublicationItemData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteEnvelope(BaseModel):
+    data: TogglePublicationFavoriteData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/runs.py

@@ -0,0 +1,226 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.api.schemas.common import ApiMeta
+
+
+class RunListItemData(BaseModel):
+    id: int
+    trigger_type: str
+    status: str
+    start_dt: datetime
+    end_dt: datetime | None
+    scholar_count: int
+    new_publication_count: int
+    failed_count: int
+    partial_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScrapeSafetyCountersData(BaseModel):
+    consecutive_blocked_runs: int = 0
+    consecutive_network_runs: int = 0
+    cooldown_entry_count: int = 0
+    blocked_start_count: int = 0
+    last_blocked_failure_count: int = 0
+    last_network_failure_count: int = 0
+    last_evaluated_run_id: int | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScrapeSafetyStateData(BaseModel):
+    cooldown_active: bool
+    cooldown_reason: str | None = None
+    cooldown_reason_label: str | None = None
+    cooldown_until: datetime | None = None
+    cooldown_remaining_seconds: int = 0
+    recommended_action: str | None = None
+    counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunsListData(BaseModel):
+    runs: list[RunListItemData]
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunsListEnvelope(BaseModel):
+    data: RunsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunSummaryData(BaseModel):
+    succeeded_count: int
+    failed_count: int
+    partial_count: int
+    failed_state_counts: dict[str, int] = Field(default_factory=dict)
+    failed_reason_counts: dict[str, int] = Field(default_factory=dict)
+    scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
+    retry_counts: dict[str, int] = Field(default_factory=dict)
+    alert_thresholds: dict[str, int] = Field(default_factory=dict)
+    alert_flags: dict[str, bool] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunAttemptLogData(BaseModel):
+    attempt: int
+    cstart: int
+    state: str | None = None
+    state_reason: str | None = None
+    status_code: int | None = None
+    fetch_error: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunPageLogData(BaseModel):
+    page: int
+    cstart: int
+    state: str
+    state_reason: str | None = None
+    status_code: int | None = None
+    publication_count: int = 0
+    attempt_count: int = 0
+    has_show_more_button: bool = False
+    articles_range: str | None = None
+    warning_codes: list[str] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDebugData(BaseModel):
+    status_code: int | None = None
+    final_url: str | None = None
+    fetch_error: str | None = None
+    body_sha256: str | None = None
+    body_length: int | None = None
+    has_show_more_button: bool | None = None
+    articles_range: str | None = None
+    state_reason: str | None = None
+    warning_codes: list[str] = Field(default_factory=list)
+    marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
+    page_logs: list[RunPageLogData] = Field(default_factory=list)
+    attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunScholarResultData(BaseModel):
+    scholar_profile_id: int
+    scholar_id: str
+    state: str
+    state_reason: str | None = None
+    outcome: str
+    attempt_count: int = 0
+    publication_count: int = 0
+    start_cstart: int = 0
+    continuation_cstart: int | None = None
+    continuation_enqueued: bool = False
+    continuation_cleared: bool = False
+    warnings: list[str] = Field(default_factory=list)
+    error: str | None = None
+    debug: RunDebugData | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDetailData(BaseModel):
+    run: RunListItemData
+    summary: RunSummaryData
+    scholar_results: list[RunScholarResultData]
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDetailEnvelope(BaseModel):
+    data: RunDetailData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ManualRunData(BaseModel):
+    run_id: int
+    status: str
+    scholar_count: int
+    succeeded_count: int
+    failed_count: int
+    partial_count: int
+    new_publication_count: int
+    reused_existing_run: bool
+    idempotency_key: str | None = None
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ManualRunEnvelope(BaseModel):
+    data: ManualRunData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueItemData(BaseModel):
+    id: int
+    scholar_profile_id: int
+    scholar_label: str
+    status: str
+    reason: str
+    dropped_reason: str | None
+    attempt_count: int
+    resume_cstart: int
+    next_attempt_dt: datetime | None
+    updated_at: datetime
+    last_error: str | None
+    last_run_id: int | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueListData(BaseModel):
+    queue_items: list[QueueItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueListEnvelope(BaseModel):
+    data: QueueListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueItemEnvelope(BaseModel):
+    data: QueueItemData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueClearData(BaseModel):
+    queue_item_id: int
+    previous_status: str
+    status: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueClearEnvelope(BaseModel):
+    data: QueueClearData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/scholars.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.api.schemas.common import ApiMeta
+
+
+class ScholarItemData(BaseModel):
+    id: int
+    scholar_id: str
+    display_name: str | None
+    profile_image_url: str | None
+    profile_image_source: str
+    is_enabled: bool
+    baseline_completed: bool
+    last_run_dt: datetime | None
+    last_run_status: str | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarsListData(BaseModel):
+    scholars: list[ScholarItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarsListEnvelope(BaseModel):
+    data: ScholarsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarCreateRequest(BaseModel):
+    scholar_id: str
+    profile_image_url: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarEnvelope(BaseModel):
+    data: ScholarItemData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchCandidateData(BaseModel):
+    scholar_id: str
+    display_name: str
+    affiliation: str | None
+    email_domain: str | None
+    cited_by_count: int | None
+    interests: list[str] = Field(default_factory=list)
+    profile_url: str
+    profile_image_url: str | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchData(BaseModel):
+    query: str
+    state: str
+    state_reason: str
+    action_hint: str | None = None
+    candidates: list[ScholarSearchCandidateData]
+    warnings: list[str] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchEnvelope(BaseModel):
+    data: ScholarSearchData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarImageUrlUpdateRequest(BaseModel):
+    image_url: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarExportItemData(BaseModel):
+    scholar_id: str
+    display_name: str | None = None
+    is_enabled: bool = True
+    profile_image_override_url: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationExportItemData(BaseModel):
+    scholar_id: str
+    cluster_id: str | None = None
+    fingerprint_sha256: str | None = None
+    title: str
+    year: int | None = None
+    citation_count: int = 0
+    author_text: str | None = None
+    venue_text: str | None = None
+    pub_url: str | None = None
+    pdf_url: str | None = None
+    is_read: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataExportData(BaseModel):
+    schema_version: int
+    exported_at: str
+    scholars: list[ScholarExportItemData]
+    publications: list[PublicationExportItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataExportEnvelope(BaseModel):
+    data: DataExportData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportRequest(BaseModel):
+    schema_version: int | None = None
+    scholars: list[ScholarExportItemData] = Field(default_factory=list)
+    publications: list[PublicationExportItemData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportResultData(BaseModel):
+    scholars_created: int
+    scholars_updated: int
+    publications_created: int
+    publications_updated: int
+    links_created: int
+    links_updated: int
+    skipped_records: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportEnvelope(BaseModel):
+    data: DataImportResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/settings.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict
+
+from app.api.schemas.common import ApiMeta
+from app.api.schemas.runs import ScrapeSafetyStateData
+
+
+class SettingsPolicyData(BaseModel):
+    min_run_interval_minutes: int
+    min_request_delay_seconds: int
+    automation_allowed: bool
+    manual_run_allowed: bool
+    blocked_failure_threshold: int
+    network_failure_threshold: int
+    cooldown_blocked_seconds: int
+    cooldown_network_seconds: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsData(BaseModel):
+    auto_run_enabled: bool
+    run_interval_minutes: int
+    request_delay_seconds: int
+    nav_visible_pages: list[str]
+    policy: SettingsPolicyData
+    safety_state: ScrapeSafetyStateData
+
+    openalex_api_key: str | None = None
+    crossref_api_token: str | None = None
+    crossref_api_mailto: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsEnvelope(BaseModel):
+    data: SettingsData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsUpdateRequest(BaseModel):
+    auto_run_enabled: bool
+    run_interval_minutes: int
+    request_delay_seconds: int
+    nav_visible_pages: list[str] | None = None
+
+    openalex_api_key: str | None = None
+    crossref_api_token: str | None = None
+    crossref_api_mailto: str | None = None
+
+    model_config = ConfigDict(extra="forbid")

app/services/ingestion/enrichment.py

@@ -0,0 +1,323 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import (
+    CrawlRun,
+    Publication,
+    RunStatus,
+    ScholarProfile,
+    ScholarPublication,
+)
+from app.logging_utils import structured_log
+from app.services.arxiv.errors import ArxivRateLimitError
+from app.services.publication_identifiers import application as identifier_service
+from app.services.runs.events import run_events
+from app.services.scholar.parser import PublicationCandidate
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+class EnrichmentRunner:
+    """Post-run OpenAlex enrichment logic.
+
+    Receives service dependencies at construction so it can be tested
+    independently of ``ScholarIngestionService``.
+    """
+
+    async def run_is_canceled(
+        self,
+        db_session: AsyncSession,
+        *,
+        run_id: int,
+    ) -> bool:
+        check_result = await db_session.execute(select(CrawlRun.status).where(CrawlRun.id == run_id))
+        status = check_result.scalar_one_or_none()
+        if status is None:
+            raise RuntimeError(f"Missing crawl_run for run_id={run_id}.")
+        return status == RunStatus.CANCELED
+
+    async def enrich_pending_publications(
+        self,
+        db_session: AsyncSession,
+        *,
+        run_id: int,
+        openalex_api_key: str | None = None,
+    ) -> None:
+        """Enrich unenriched publications with OpenAlex data.
+
+        Stops immediately on budget exhaustion (429 with $0 remaining).
+        Sleeps 60s and continues on transient rate limits.
+        """
+        from app.services.openalex.client import (
+            OpenAlexBudgetExhaustedError,
+            OpenAlexClient,
+            OpenAlexRateLimitError,
+        )
+        from app.services.openalex.matching import find_best_match
+
+        run_result = await db_session.execute(select(CrawlRun.user_id).where(CrawlRun.id == run_id))
+        user_id = run_result.scalar_one()
+
+        now = datetime.now(UTC)
+        cooldown_threshold = now - timedelta(days=7)
+
+        stmt = (
+            select(Publication)
+            .join(ScholarPublication)
+            .join(ScholarProfile, ScholarPublication.scholar_profile_id == ScholarProfile.id)
+            .where(
+                ScholarProfile.user_id == user_id,
+                Publication.openalex_enriched.is_(False),
+                or_(
+                    Publication.openalex_last_attempt_at.is_(None),
+                    Publication.openalex_last_attempt_at < cooldown_threshold,
+                ),
+            )
+            .distinct()
+        )
+        result = await db_session.execute(stmt)
+        publications = list(result.scalars().all())
+
+        if not publications:
+            return
+
+        resolved_key = openalex_api_key or settings.openalex_api_key
+        client = OpenAlexClient(api_key=resolved_key, mailto=settings.crossref_api_mailto)
+        batch_size = 25
+        arxiv_lookup_allowed = True
+
+        for i in range(0, len(publications), batch_size):
+            if await self.run_is_canceled(db_session, run_id=run_id):
+                logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id})
+                return
+            batch = publications[i : i + batch_size]
+            titles = [
+                " ".join(re.sub(r"[^\w\s]", " ", p.title_raw).split())
+                for p in batch
+                if p.title_raw and p.title_raw.strip()
+            ]
+
+            if not titles:
+                continue
+
+            try:
+                openalex_works = await client.get_works_by_filter(
+                    {"title.search": "|".join(titles)}, limit=batch_size * 3
+                )
+            except OpenAlexBudgetExhaustedError:
+                structured_log(
+                    logger,
+                    "warning",
+                    "ingestion.openalex_budget_exhausted",
+                    run_id=run_id,
+                )
+                break
+            except OpenAlexRateLimitError:
+                structured_log(
+                    logger,
+                    "warning",
+                    "ingestion.openalex_rate_limited",
+                    run_id=run_id,
+                )
+                await asyncio.sleep(60)
+                continue
+            except Exception as e:
+                structured_log(
+                    logger,
+                    "warning",
+                    "ingestion.openalex_enrichment_failed",
+                    error=str(e),
+                    run_id=run_id,
+                )
+                continue
+
+            for p in batch:
+                if await self.run_is_canceled(db_session, run_id=run_id):
+                    logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id})
+                    return
+
+                p.openalex_last_attempt_at = now
+                arxiv_lookup_allowed = await self._discover_identifiers_for_enrichment(
+                    db_session,
+                    publication=p,
+                    run_id=run_id,
+                    allow_arxiv_lookup=arxiv_lookup_allowed,
+                )
+
+                match = find_best_match(
+                    target_title=p.title_raw,
+                    target_year=p.year,
+                    target_authors=p.author_text or "",
+                    candidates=openalex_works,
+                )
+                if match:
+                    p.year = match.publication_year or p.year
+                    p.citation_count = match.cited_by_count or p.citation_count
+                    p.pdf_url = match.oa_url or p.pdf_url
+                    p.openalex_enriched = True
+
+        await db_session.flush()
+
+        from app.services.publications.dedup import sweep_identifier_duplicates
+
+        merge_count = await sweep_identifier_duplicates(db_session)
+        if merge_count:
+            structured_log(
+                logger,
+                "info",
+                "ingestion.identifier_dedup_sweep",
+                merged_count=merge_count,
+                run_id=run_id,
+            )
+
+    async def _discover_identifiers_for_enrichment(
+        self,
+        db_session: AsyncSession,
+        *,
+        publication: Publication,
+        run_id: int,
+        allow_arxiv_lookup: bool,
+    ) -> bool:
+        if not allow_arxiv_lookup:
+            await identifier_service.sync_identifiers_for_publication_fields(
+                db_session,
+                publication=publication,
+            )
+            await self._publish_identifier_update_event(
+                db_session,
+                run_id=run_id,
+                publication_id=int(publication.id),
+            )
+            return False
+        try:
+            await identifier_service.discover_and_sync_identifiers_for_publication(
+                db_session,
+                publication=publication,
+                scholar_label=publication.author_text or "",
+            )
+            await self._publish_identifier_update_event(
+                db_session,
+                run_id=run_id,
+                publication_id=int(publication.id),
+            )
+            return True
+        except ArxivRateLimitError:
+            structured_log(
+                logger,
+                "warning",
+                "ingestion.arxiv_rate_limited",
+                run_id=run_id,
+                publication_id=int(publication.id),
+                detail="arXiv temporarily disabled for remaining enrichment pass",
+            )
+            await identifier_service.sync_identifiers_for_publication_fields(
+                db_session,
+                publication=publication,
+            )
+            await self._publish_identifier_update_event(
+                db_session,
+                run_id=run_id,
+                publication_id=int(publication.id),
+            )
+            return False
+
+    async def _publish_identifier_update_event(
+        self,
+        db_session: AsyncSession,
+        *,
+        run_id: int,
+        publication_id: int,
+    ) -> None:
+        display = await identifier_service.display_identifier_for_publication_id(
+            db_session,
+            publication_id=publication_id,
+        )
+        if display is None:
+            return
+        await run_events.publish(
+            run_id=run_id,
+            event_type="identifier_updated",
+            data={
+                "publication_id": int(publication_id),
+                "display_identifier": {
+                    "kind": display.kind,
+                    "value": display.value,
+                    "label": display.label,
+                    "url": display.url,
+                    "confidence_score": float(display.confidence_score),
+                },
+            },
+        )
+
+    async def enrich_publications_with_openalex(
+        self,
+        scholar: ScholarProfile,
+        publications: list[PublicationCandidate],
+    ) -> list[PublicationCandidate]:
+        if not publications:
+            return publications
+
+        from app.services.openalex.client import OpenAlexClient
+        from app.services.openalex.matching import find_best_match
+
+        client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto)
+
+        batch_size = 25
+        enriched: list[PublicationCandidate] = []
+
+        for i in range(0, len(publications), batch_size):
+            batch = publications[i : i + batch_size]
+
+            titles = []
+            for p in batch:
+                if not p.title:
+                    continue
+                safe_title = re.sub(r"[^\w\s]", " ", p.title)
+                safe_title = " ".join(safe_title.split())
+                if safe_title:
+                    titles.append(safe_title)
+
+            if not titles:
+                enriched.extend(batch)
+                continue
+
+            query = "|".join(t for t in titles)
+            try:
+                openalex_works = await client.get_works_by_filter({"title.search": query}, limit=batch_size * 3)
+            except Exception as e:
+                logger.warning(
+                    "ingestion.openalex_enrichment_failed",
+                    extra={"error": str(e), "batch_size": len(batch), "scholar_id": scholar.id},
+                )
+                openalex_works = []
+
+            for p in batch:
+                match = find_best_match(
+                    target_title=p.title,
+                    target_year=p.year,
+                    target_authors=p.authors_text or (scholar.display_name or scholar.scholar_id),
+                    candidates=openalex_works,
+                )
+                if match:
+                    new_p = PublicationCandidate(
+                        title=p.title,
+                        title_url=p.title_url,
+                        cluster_id=p.cluster_id,
+                        year=match.publication_year or p.year,
+                        citation_count=match.cited_by_count,
+                        authors_text=p.authors_text,
+                        venue_text=p.venue_text,
+                        pdf_url=match.oa_url or p.pdf_url,
+                    )
+                    enriched.append(new_p)
+                else:
+                    enriched.append(p)
+        return enriched

app/services/ingestion/run_completion.py

@@ -0,0 +1,417 @@
+from __future__ import annotations
+
+import hashlib
+import logging
+from datetime import UTC, datetime
+from typing import Any
+
+from app.db.models import (
+    CrawlRun,
+    RunStatus,
+    ScholarProfile,
+)
+from app.logging_utils import structured_log
+from app.services.ingestion import safety as run_safety_service
+from app.services.ingestion.constants import (
+    FAILED_STATES,
+    FAILURE_BUCKET_BLOCKED,
+    FAILURE_BUCKET_INGESTION,
+    FAILURE_BUCKET_LAYOUT,
+    FAILURE_BUCKET_NETWORK,
+    FAILURE_BUCKET_OTHER,
+)
+from app.services.ingestion.fingerprints import _build_body_excerpt
+from app.services.ingestion.types import (
+    RunAlertSummary,
+    RunExecutionSummary,
+    RunFailureSummary,
+    RunProgress,
+    ScholarProcessingOutcome,
+)
+from app.services.scholar.parser import ParsedProfilePage, ParseState
+from app.services.scholar.source import FetchResult
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+def _int_or_default(value: Any, default: int = 0) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def classify_failure_bucket(*, state: str, state_reason: str) -> str:
+    reason = state_reason.strip().lower()
+    normalized_state = state.strip().lower()
+
+    if normalized_state == ParseState.BLOCKED_OR_CAPTCHA.value or reason.startswith("blocked_"):
+        return FAILURE_BUCKET_BLOCKED
+    if normalized_state == ParseState.NETWORK_ERROR.value or reason.startswith("network_"):
+        return FAILURE_BUCKET_NETWORK
+    if normalized_state == ParseState.LAYOUT_CHANGED.value:
+        return FAILURE_BUCKET_LAYOUT
+    if normalized_state == "ingestion_error":
+        return FAILURE_BUCKET_INGESTION
+    return FAILURE_BUCKET_OTHER
+
+
+def summarize_failures(
+    *,
+    scholar_results: list[dict[str, Any]],
+) -> RunFailureSummary:
+    failed_state_counts: dict[str, int] = {}
+    failed_reason_counts: dict[str, int] = {}
+    scrape_failure_counts: dict[str, int] = {}
+    retries_scheduled_count = 0
+    scholars_with_retries_count = 0
+    retry_exhausted_count = 0
+    for entry in scholar_results:
+        retries_for_entry = max(0, _int_or_default(entry.get("attempt_count"), 0) - 1)
+        if retries_for_entry > 0:
+            retries_scheduled_count += retries_for_entry
+            scholars_with_retries_count += 1
+        if str(entry.get("outcome", "")) != "failed":
+            continue
+        state = str(entry.get("state", "")).strip()
+        if state not in FAILED_STATES:
+            continue
+        failed_state_counts[state] = failed_state_counts.get(state, 0) + 1
+        reason = str(entry.get("state_reason", "")).strip()
+        if reason:
+            failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1
+        bucket = classify_failure_bucket(state=state, state_reason=reason)
+        scrape_failure_counts[bucket] = scrape_failure_counts.get(bucket, 0) + 1
+        if state == ParseState.NETWORK_ERROR.value and retries_for_entry > 0:
+            retry_exhausted_count += 1
+    return RunFailureSummary(
+        failed_state_counts=failed_state_counts,
+        failed_reason_counts=failed_reason_counts,
+        scrape_failure_counts=scrape_failure_counts,
+        retries_scheduled_count=retries_scheduled_count,
+        scholars_with_retries_count=scholars_with_retries_count,
+        retry_exhausted_count=retry_exhausted_count,
+    )
+
+
+def build_alert_summary(
+    *,
+    failure_summary: RunFailureSummary,
+    alert_blocked_failure_threshold: int,
+    alert_network_failure_threshold: int,
+    alert_retry_scheduled_threshold: int,
+) -> RunAlertSummary:
+    blocked_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_BLOCKED, 0))
+    network_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_NETWORK, 0))
+    blocked_threshold = max(1, int(alert_blocked_failure_threshold))
+    network_threshold = max(1, int(alert_network_failure_threshold))
+    retry_threshold = max(1, int(alert_retry_scheduled_threshold))
+    alert_flags = {
+        "blocked_failure_threshold_exceeded": blocked_failure_count >= blocked_threshold,
+        "network_failure_threshold_exceeded": network_failure_count >= network_threshold,
+        "retry_scheduled_threshold_exceeded": failure_summary.retries_scheduled_count >= retry_threshold,
+    }
+    return RunAlertSummary(
+        blocked_failure_count=blocked_failure_count,
+        network_failure_count=network_failure_count,
+        blocked_failure_threshold=blocked_threshold,
+        network_failure_threshold=network_threshold,
+        retry_scheduled_threshold=retry_threshold,
+        alert_flags=alert_flags,
+    )
+
+
+def apply_safety_outcome(
+    *,
+    user_settings: Any,
+    run: CrawlRun,
+    user_id: int,
+    alert_summary: RunAlertSummary,
+) -> None:
+    pre_apply_state = run_safety_service.get_safety_event_context(
+        user_settings,
+        now_utc=datetime.now(UTC),
+    )
+    safety_state, cooldown_trigger_reason = run_safety_service.apply_run_safety_outcome(
+        user_settings,
+        run_id=int(run.id),
+        blocked_failure_count=alert_summary.blocked_failure_count,
+        network_failure_count=alert_summary.network_failure_count,
+        blocked_failure_threshold=alert_summary.blocked_failure_threshold,
+        network_failure_threshold=alert_summary.network_failure_threshold,
+        blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds,
+        network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds,
+        now_utc=datetime.now(UTC),
+    )
+    if cooldown_trigger_reason is not None:
+        structured_log(
+            logger,
+            "warning",
+            "ingestion.safety_cooldown_entered",
+            user_id=user_id,
+            crawl_run_id=int(run.id),
+            reason=cooldown_trigger_reason,
+            blocked_failure_count=alert_summary.blocked_failure_count,
+            network_failure_count=alert_summary.network_failure_count,
+            blocked_failure_threshold=alert_summary.blocked_failure_threshold,
+            network_failure_threshold=alert_summary.network_failure_threshold,
+            cooldown_until=safety_state.get("cooldown_until"),
+            cooldown_remaining_seconds=safety_state.get("cooldown_remaining_seconds"),
+            safety_counters=safety_state.get("counters", {}),
+        )
+    elif pre_apply_state.get("cooldown_active") and not safety_state.get("cooldown_active"):
+        structured_log(
+            logger,
+            "info",
+            "ingestion.cooldown_cleared",
+            user_id=user_id,
+            crawl_run_id=int(run.id),
+            reason=pre_apply_state.get("cooldown_reason"),
+            cooldown_until=pre_apply_state.get("cooldown_until"),
+        )
+
+
+def finalize_run_record(
+    *,
+    run: CrawlRun,
+    scholars: list[ScholarProfile],
+    progress: RunProgress,
+    failure_summary: RunFailureSummary,
+    alert_summary: RunAlertSummary,
+    idempotency_key: str | None,
+    run_status: RunStatus,
+) -> None:
+    run.end_dt = datetime.now(UTC)
+    if run.status != RunStatus.CANCELED:
+        run.status = run_status
+    run.error_log = {
+        "scholar_results": progress.scholar_results,
+        "summary": {
+            "succeeded_count": progress.succeeded_count,
+            "failed_count": progress.failed_count,
+            "partial_count": progress.partial_count,
+            "failed_state_counts": failure_summary.failed_state_counts,
+            "failed_reason_counts": failure_summary.failed_reason_counts,
+            "scrape_failure_counts": failure_summary.scrape_failure_counts,
+            "retry_counts": {
+                "retries_scheduled_count": failure_summary.retries_scheduled_count,
+                "scholars_with_retries_count": failure_summary.scholars_with_retries_count,
+                "retry_exhausted_count": failure_summary.retry_exhausted_count,
+            },
+            "alert_thresholds": {
+                "blocked_failure_threshold": alert_summary.blocked_failure_threshold,
+                "network_failure_threshold": alert_summary.network_failure_threshold,
+                "retry_scheduled_threshold": alert_summary.retry_scheduled_threshold,
+            },
+            "alert_flags": alert_summary.alert_flags,
+        },
+        "meta": {"idempotency_key": idempotency_key} if idempotency_key else {},
+    }
+
+
+def resolve_run_status(
+    *,
+    scholar_count: int,
+    succeeded_count: int,
+    failed_count: int,
+    partial_count: int,
+) -> RunStatus:
+    if scholar_count == 0:
+        return RunStatus.SUCCESS
+    if failed_count == scholar_count:
+        return RunStatus.FAILED
+    if failed_count > 0 or partial_count > 0:
+        return RunStatus.PARTIAL_FAILURE
+    if succeeded_count > 0:
+        return RunStatus.SUCCESS
+    return RunStatus.FAILED
+
+
+def build_failure_debug_context(
+    *,
+    fetch_result: FetchResult,
+    parsed_page: ParsedProfilePage,
+    attempt_log: list[dict[str, Any]],
+    page_logs: list[dict[str, Any]] | None = None,
+    exception: Exception | None = None,
+) -> dict[str, Any]:
+    context: dict[str, Any] = {
+        "requested_url": fetch_result.requested_url,
+        "final_url": fetch_result.final_url,
+        "status_code": fetch_result.status_code,
+        "fetch_error": fetch_result.error,
+        "state_reason": parsed_page.state_reason,
+        "profile_name": parsed_page.profile_name,
+        "profile_image_url": parsed_page.profile_image_url,
+        "articles_range": parsed_page.articles_range,
+        "has_show_more_button": parsed_page.has_show_more_button,
+        "has_operation_error_banner": parsed_page.has_operation_error_banner,
+        "warning_codes": parsed_page.warnings,
+        "marker_counts_nonzero": {key: value for key, value in parsed_page.marker_counts.items() if value > 0},
+        "body_length": len(fetch_result.body),
+        "body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest() if fetch_result.body else None,
+        "body_excerpt": _build_body_excerpt(fetch_result.body),
+        "attempt_log": attempt_log,
+    }
+    if page_logs:
+        context["page_logs"] = page_logs
+    if exception is not None:
+        context["exception_type"] = type(exception).__name__
+        context["exception_message"] = str(exception)
+    return context
+
+
+def complete_run_for_user(
+    *,
+    user_settings: Any,
+    run: CrawlRun,
+    scholars: list[ScholarProfile],
+    user_id: int,
+    progress: RunProgress,
+    idempotency_key: str | None,
+    alert_blocked_failure_threshold: int,
+    alert_network_failure_threshold: int,
+    alert_retry_scheduled_threshold: int,
+) -> tuple[RunFailureSummary, RunAlertSummary]:
+    failure_summary = summarize_failures(scholar_results=progress.scholar_results)
+    alert_summary = build_alert_summary(
+        failure_summary=failure_summary,
+        alert_blocked_failure_threshold=alert_blocked_failure_threshold,
+        alert_network_failure_threshold=alert_network_failure_threshold,
+        alert_retry_scheduled_threshold=alert_retry_scheduled_threshold,
+    )
+    if alert_summary.alert_flags["blocked_failure_threshold_exceeded"]:
+        structured_log(
+            logger,
+            "warning",
+            "ingestion.alert_blocked_failure_threshold_exceeded",
+            user_id=user_id,
+            crawl_run_id=int(run.id),
+            blocked_failure_count=alert_summary.blocked_failure_count,
+            threshold=alert_summary.blocked_failure_threshold,
+        )
+    if alert_summary.alert_flags["network_failure_threshold_exceeded"]:
+        structured_log(
+            logger,
+            "warning",
+            "ingestion.alert_network_failure_threshold_exceeded",
+            user_id=user_id,
+            crawl_run_id=int(run.id),
+            network_failure_count=alert_summary.network_failure_count,
+            threshold=alert_summary.network_failure_threshold,
+        )
+    if alert_summary.alert_flags["retry_scheduled_threshold_exceeded"]:
+        structured_log(
+            logger,
+            "warning",
+            "ingestion.alert_retry_scheduled_threshold_exceeded",
+            user_id=user_id,
+            crawl_run_id=int(run.id),
+            retries_scheduled_count=failure_summary.retries_scheduled_count,
+            threshold=alert_summary.retry_scheduled_threshold,
+        )
+    apply_safety_outcome(user_settings=user_settings, run=run, user_id=user_id, alert_summary=alert_summary)
+    run_status = resolve_run_status(
+        scholar_count=len(scholars),
+        succeeded_count=progress.succeeded_count,
+        failed_count=progress.failed_count,
+        partial_count=progress.partial_count,
+    )
+    finalize_run_record(
+        run=run,
+        scholars=scholars,
+        progress=progress,
+        failure_summary=failure_summary,
+        alert_summary=alert_summary,
+        idempotency_key=idempotency_key,
+        run_status=run_status,
+    )
+    return failure_summary, alert_summary
+
+
+def result_counters(result_entry: dict[str, Any]) -> tuple[int, int, int]:
+    outcome = str(result_entry.get("outcome", "")).strip().lower()
+    if outcome == "success":
+        return 1, 0, 0
+    if outcome == "partial":
+        return 1, 0, 1
+    if outcome == "failed":
+        return 0, 1, 0
+    raise RuntimeError(f"Unexpected scholar outcome label: {outcome!r}")
+
+
+def find_scholar_result_index(
+    *,
+    scholar_results: list[dict[str, Any]],
+    scholar_profile_id: int,
+) -> int | None:
+    for index, result_entry in enumerate(scholar_results):
+        current_scholar_id = _int_or_default(result_entry.get("scholar_profile_id"), 0)
+        if current_scholar_id == scholar_profile_id:
+            return index
+    return None
+
+
+def adjust_progress_counts(
+    *,
+    progress: RunProgress,
+    succeeded_delta: int,
+    failed_delta: int,
+    partial_delta: int,
+) -> None:
+    progress.succeeded_count += succeeded_delta
+    progress.failed_count += failed_delta
+    progress.partial_count += partial_delta
+    if progress.succeeded_count < 0 or progress.failed_count < 0 or progress.partial_count < 0:
+        raise RuntimeError("RunProgress counters entered invalid negative state.")
+
+
+def apply_outcome_to_progress(
+    *,
+    progress: RunProgress,
+    outcome: ScholarProcessingOutcome,
+) -> None:
+    scholar_profile_id = _int_or_default(outcome.result_entry.get("scholar_profile_id"), 0)
+    if scholar_profile_id <= 0:
+        raise RuntimeError("Scholar outcome missing valid scholar_profile_id.")
+    prior_index = find_scholar_result_index(
+        scholar_results=progress.scholar_results,
+        scholar_profile_id=scholar_profile_id,
+    )
+    next_succeeded, next_failed, next_partial = result_counters(outcome.result_entry)
+    if prior_index is None:
+        progress.scholar_results.append(outcome.result_entry)
+        adjust_progress_counts(
+            progress=progress,
+            succeeded_delta=next_succeeded,
+            failed_delta=next_failed,
+            partial_delta=next_partial,
+        )
+        return
+    previous_entry = progress.scholar_results[prior_index]
+    prev_succeeded, prev_failed, prev_partial = result_counters(previous_entry)
+    progress.scholar_results[prior_index] = outcome.result_entry
+    adjust_progress_counts(
+        progress=progress,
+        succeeded_delta=next_succeeded - prev_succeeded,
+        failed_delta=next_failed - prev_failed,
+        partial_delta=next_partial - prev_partial,
+    )
+
+
+def run_execution_summary(
+    *,
+    run: CrawlRun,
+    scholars: list[ScholarProfile],
+    progress: RunProgress,
+) -> RunExecutionSummary:
+    return RunExecutionSummary(
+        crawl_run_id=run.id,
+        status=run.status,
+        scholar_count=len(scholars),
+        succeeded_count=progress.succeeded_count,
+        failed_count=progress.failed_count,
+        partial_count=progress.partial_count,
+        new_publication_count=run.new_pub_count,
+    )

app/services/ingestion/scholar_processing.py

@@ -0,0 +1,614 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import UTC, datetime
+from typing import Any
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import (
+    CrawlRun,
+    RunStatus,
+    ScholarProfile,
+)
+from app.logging_utils import structured_log
+from app.services.ingestion import queue as queue_service
+from app.services.ingestion.constants import (
+    RESUMABLE_PARTIAL_REASON_PREFIXES,
+    RESUMABLE_PARTIAL_REASONS,
+)
+from app.services.ingestion.pagination import PaginationEngine
+from app.services.ingestion.publication_upsert import upsert_profile_publications
+from app.services.ingestion.run_completion import apply_outcome_to_progress, build_failure_debug_context
+from app.services.ingestion.types import (
+    PagedParseResult,
+    RunProgress,
+    ScholarProcessingOutcome,
+)
+from app.services.scholar.parser import ParseState
+
+logger = logging.getLogger(__name__)
+
+
+def assert_valid_paged_parse_result(
+    *,
+    scholar_id: str,
+    paged_parse_result: PagedParseResult,
+) -> None:
+    parsed_page = paged_parse_result.parsed_page
+    if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} and any(
+        code.startswith("layout_") for code in parsed_page.warnings
+    ):
+        raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.")
+    for publication in paged_parse_result.publications:
+        if not publication.title.strip():
+            raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.")
+        if publication.citation_count is not None and int(publication.citation_count) < 0:
+            raise RuntimeError(f"Negative citation count for scholar_id={scholar_id}.")
+
+
+def apply_first_page_profile_metadata(
+    *,
+    scholar: ScholarProfile,
+    paged_parse_result: PagedParseResult,
+    run_dt: datetime,
+) -> None:
+    first_page = paged_parse_result.first_page_parsed_page
+    if first_page.profile_name and not (scholar.display_name or "").strip():
+        scholar.display_name = first_page.profile_name
+    if first_page.profile_image_url:
+        scholar.profile_image_url = first_page.profile_image_url
+    scholar.last_initial_page_checked_at = run_dt
+
+
+def build_result_entry(
+    *,
+    scholar: ScholarProfile,
+    start_cstart: int,
+    paged_parse_result: PagedParseResult,
+) -> dict[str, Any]:
+    parsed_page = paged_parse_result.parsed_page
+    return {
+        "scholar_profile_id": scholar.id,
+        "scholar_id": scholar.scholar_id,
+        "state": parsed_page.state.value,
+        "state_reason": parsed_page.state_reason,
+        "outcome": "failed",
+        "attempt_count": len(paged_parse_result.attempt_log),
+        "publication_count": len(paged_parse_result.publications),
+        "start_cstart": start_cstart,
+        "articles_range": parsed_page.articles_range,
+        "warnings": parsed_page.warnings,
+        "has_show_more_button": parsed_page.has_show_more_button,
+        "pages_fetched": paged_parse_result.pages_fetched,
+        "pages_attempted": paged_parse_result.pages_attempted,
+        "has_more_remaining": paged_parse_result.has_more_remaining,
+        "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason,
+        "continuation_cstart": paged_parse_result.continuation_cstart,
+        "skipped_no_change": paged_parse_result.skipped_no_change,
+        "initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256,
+    }
+
+
+def skipped_no_change_outcome(
+    *,
+    scholar: ScholarProfile,
+    run_dt: datetime,
+    paged_parse_result: PagedParseResult,
+    result_entry: dict[str, Any],
+) -> ScholarProcessingOutcome:
+    first_page = paged_parse_result.first_page_parsed_page
+    scholar.last_run_status = RunStatus.SUCCESS
+    scholar.last_run_dt = run_dt
+    result_entry["state"] = first_page.state.value
+    result_entry["state_reason"] = "no_change_initial_page_signature"
+    result_entry["outcome"] = "success"
+    result_entry["publication_count"] = 0
+    result_entry["warnings"] = first_page.warnings
+    result_entry["debug"] = {
+        "state_reason": "no_change_initial_page_signature",
+        "first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256,
+        "attempt_log": paged_parse_result.attempt_log,
+        "page_logs": paged_parse_result.page_logs,
+    }
+    return ScholarProcessingOutcome(
+        result_entry=result_entry,
+        succeeded_count_delta=1,
+        failed_count_delta=0,
+        partial_count_delta=0,
+        discovered_publication_count=0,
+    )
+
+
+async def upsert_publications_outcome(
+    db_session: AsyncSession,
+    *,
+    run: CrawlRun,
+    scholar: ScholarProfile,
+    run_dt: datetime,
+    paged_parse_result: PagedParseResult,
+    result_entry: dict[str, Any],
+) -> ScholarProcessingOutcome:
+    parsed_page = paged_parse_result.parsed_page
+    publications = paged_parse_result.publications
+    had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}
+    has_partial_set = len(publications) > 0 and had_page_failure
+    if (not had_page_failure) or has_partial_set:
+        return await _upsert_success_or_exception(
+            db_session,
+            run=run,
+            scholar=scholar,
+            run_dt=run_dt,
+            paged_parse_result=paged_parse_result,
+            result_entry=result_entry,
+            has_partial_publication_set=has_partial_set,
+        )
+    return _parse_failure_outcome(
+        scholar=scholar,
+        run_dt=run_dt,
+        paged_parse_result=paged_parse_result,
+        result_entry=result_entry,
+    )
+
+
+async def _upsert_success_or_exception(
+    db_session: AsyncSession,
+    *,
+    run: CrawlRun,
+    scholar: ScholarProfile,
+    run_dt: datetime,
+    paged_parse_result: PagedParseResult,
+    result_entry: dict[str, Any],
+    has_partial_publication_set: bool,
+) -> ScholarProcessingOutcome:
+    try:
+        return _upsert_success(
+            run=run,
+            scholar=scholar,
+            run_dt=run_dt,
+            paged_parse_result=paged_parse_result,
+            result_entry=result_entry,
+            has_partial_publication_set=has_partial_publication_set,
+        )
+    except Exception as exc:
+        return _upsert_exception_outcome(
+            run=run,
+            scholar=scholar,
+            run_dt=run_dt,
+            paged_parse_result=paged_parse_result,
+            result_entry=result_entry,
+            exc=exc,
+        )
+
+
+def _upsert_success(
+    *,
+    run: CrawlRun,
+    scholar: ScholarProfile,
+    run_dt: datetime,
+    paged_parse_result: PagedParseResult,
+    result_entry: dict[str, Any],
+    has_partial_publication_set: bool,
+) -> ScholarProcessingOutcome:
+    discovered_count = paged_parse_result.discovered_publication_count
+    is_partial = (
+        paged_parse_result.has_more_remaining
+        or paged_parse_result.pagination_truncated_reason is not None
+        or has_partial_publication_set
+    )
+    scholar.last_run_status = RunStatus.PARTIAL_FAILURE if is_partial else RunStatus.SUCCESS
+    scholar.last_run_dt = run_dt
+    if not is_partial and paged_parse_result.first_page_fingerprint_sha256:
+        scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256
+    result_entry["outcome"] = "partial" if is_partial else "success"
+    if is_partial:
+        result_entry["debug"] = build_failure_debug_context(
+            fetch_result=paged_parse_result.fetch_result,
+            parsed_page=paged_parse_result.parsed_page,
+            attempt_log=paged_parse_result.attempt_log,
+            page_logs=paged_parse_result.page_logs,
+        )
+    return ScholarProcessingOutcome(
+        result_entry=result_entry,
+        succeeded_count_delta=1,
+        failed_count_delta=0,
+        partial_count_delta=1 if is_partial else 0,
+        discovered_publication_count=discovered_count,
+    )
+
+
+def _upsert_exception_outcome(
+    *,
+    run: CrawlRun,
+    scholar: ScholarProfile,
+    run_dt: datetime,
+    paged_parse_result: PagedParseResult,
+    result_entry: dict[str, Any],
+    exc: Exception,
+) -> ScholarProcessingOutcome:
+    scholar.last_run_status = RunStatus.FAILED
+    scholar.last_run_dt = run_dt
+    result_entry["state"] = "ingestion_error"
+    result_entry["state_reason"] = "publication_upsert_exception"
+    result_entry["outcome"] = "failed"
+    result_entry["error"] = str(exc)
+    result_entry["debug"] = build_failure_debug_context(
+        fetch_result=paged_parse_result.fetch_result,
+        parsed_page=paged_parse_result.parsed_page,
+        attempt_log=paged_parse_result.attempt_log,
+        page_logs=paged_parse_result.page_logs,
+        exception=exc,
+    )
+    logger.exception(
+        "ingestion.scholar_failed",
+        extra={
+            "crawl_run_id": run.id,
+            "scholar_profile_id": scholar.id,
+            "scholar_id": scholar.scholar_id,
+        },
+    )
+    return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0)
+
+
+def _parse_failure_outcome(
+    *,
+    scholar: ScholarProfile,
+    run_dt: datetime,
+    paged_parse_result: PagedParseResult,
+    result_entry: dict[str, Any],
+) -> ScholarProcessingOutcome:
+    scholar.last_run_status = RunStatus.FAILED
+    scholar.last_run_dt = run_dt
+    result_entry["debug"] = build_failure_debug_context(
+        fetch_result=paged_parse_result.fetch_result,
+        parsed_page=paged_parse_result.parsed_page,
+        attempt_log=paged_parse_result.attempt_log,
+        page_logs=paged_parse_result.page_logs,
+    )
+    structured_log(
+        logger,
+        "warning",
+        "ingestion.scholar_parse_failed",
+        scholar_profile_id=scholar.id,
+        scholar_id=scholar.scholar_id,
+        state=paged_parse_result.parsed_page.state.value,
+        state_reason=paged_parse_result.parsed_page.state_reason,
+        status_code=paged_parse_result.fetch_result.status_code,
+    )
+    return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0)
+
+
+async def sync_continuation_queue(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar: ScholarProfile,
+    run: CrawlRun,
+    start_cstart: int,
+    result_entry: dict[str, Any],
+    paged_parse_result: PagedParseResult,
+    auto_queue_continuations: bool,
+    queue_delay_seconds: int,
+) -> None:
+    queue_reason, queue_cstart = resolve_continuation_queue_target(
+        outcome=str(result_entry.get("outcome", "")),
+        state=str(result_entry.get("state", "")),
+        pagination_truncated_reason=paged_parse_result.pagination_truncated_reason,
+        continuation_cstart=paged_parse_result.continuation_cstart,
+        fallback_cstart=start_cstart,
+    )
+    if auto_queue_continuations and queue_reason is not None:
+        await queue_service.upsert_job(
+            db_session,
+            user_id=user_id,
+            scholar_profile_id=scholar.id,
+            resume_cstart=queue_cstart,
+            reason=queue_reason,
+            run_id=run.id,
+            delay_seconds=queue_delay_seconds,
+        )
+        result_entry["continuation_enqueued"] = True
+        result_entry["continuation_reason"] = queue_reason
+        result_entry["continuation_cstart"] = queue_cstart
+        return
+    if await queue_service.clear_job_for_scholar(db_session, user_id=user_id, scholar_profile_id=scholar.id):
+        result_entry["continuation_cleared"] = True
+
+
+def resolve_continuation_queue_target(
+    *,
+    outcome: str,
+    state: str,
+    pagination_truncated_reason: str | None,
+    continuation_cstart: int | None,
+    fallback_cstart: int,
+) -> tuple[str | None, int]:
+    if outcome == "partial":
+        reason = (pagination_truncated_reason or "").strip()
+        if reason in RESUMABLE_PARTIAL_REASONS or reason.startswith(RESUMABLE_PARTIAL_REASON_PREFIXES):
+            return reason, queue_service.normalize_cstart(
+                continuation_cstart if continuation_cstart is not None else fallback_cstart
+            )
+        return None, queue_service.normalize_cstart(fallback_cstart)
+
+    if outcome == "failed" and state == ParseState.NETWORK_ERROR.value:
+        return "network_error_retry", queue_service.normalize_cstart(
+            continuation_cstart if continuation_cstart is not None else fallback_cstart
+        )
+
+    return None, queue_service.normalize_cstart(fallback_cstart)
+
+
+async def process_scholar(
+    db_session: AsyncSession,
+    *,
+    pagination: PaginationEngine,
+    run: CrawlRun,
+    scholar: ScholarProfile,
+    user_id: int,
+    request_delay_seconds: int,
+    network_error_retries: int,
+    retry_backoff_seconds: float,
+    rate_limit_retries: int,
+    rate_limit_backoff_seconds: float,
+    max_pages_per_scholar: int,
+    page_size: int,
+    start_cstart: int,
+    auto_queue_continuations: bool,
+    queue_delay_seconds: int,
+) -> ScholarProcessingOutcome:
+    try:
+        run_dt, paged_parse_result, result_entry = await _fetch_and_prepare_scholar_result(
+            db_session,
+            pagination=pagination,
+            run=run,
+            scholar=scholar,
+            user_id=user_id,
+            start_cstart=start_cstart,
+            request_delay_seconds=request_delay_seconds,
+            network_error_retries=network_error_retries,
+            retry_backoff_seconds=retry_backoff_seconds,
+            rate_limit_retries=rate_limit_retries,
+            rate_limit_backoff_seconds=rate_limit_backoff_seconds,
+            max_pages_per_scholar=max_pages_per_scholar,
+            page_size=page_size,
+        )
+        outcome = await _resolve_scholar_outcome(
+            db_session,
+            run=run,
+            scholar=scholar,
+            run_dt=run_dt,
+            paged_parse_result=paged_parse_result,
+            result_entry=result_entry,
+        )
+        await sync_continuation_queue(
+            db_session,
+            user_id=user_id,
+            scholar=scholar,
+            run=run,
+            start_cstart=start_cstart,
+            result_entry=outcome.result_entry,
+            paged_parse_result=paged_parse_result,
+            auto_queue_continuations=auto_queue_continuations,
+            queue_delay_seconds=queue_delay_seconds,
+        )
+        return outcome
+    except Exception as exc:
+        return unexpected_scholar_exception_outcome(
+            run=run,
+            scholar=scholar,
+            start_cstart=start_cstart,
+            exc=exc,
+        )
+
+
+async def _fetch_and_prepare_scholar_result(
+    db_session: AsyncSession,
+    *,
+    pagination: PaginationEngine,
+    run: CrawlRun,
+    scholar: ScholarProfile,
+    user_id: int,
+    start_cstart: int,
+    request_delay_seconds: int,
+    network_error_retries: int,
+    retry_backoff_seconds: float,
+    rate_limit_retries: int,
+    rate_limit_backoff_seconds: float,
+    max_pages_per_scholar: int,
+    page_size: int,
+) -> tuple[datetime, PagedParseResult, dict[str, Any]]:
+    run_dt = datetime.now(UTC)
+    paged_parse_result = await pagination.fetch_and_parse_all_pages(
+        scholar=scholar,
+        run=run,
+        db_session=db_session,
+        start_cstart=start_cstart,
+        request_delay_seconds=request_delay_seconds,
+        network_error_retries=network_error_retries,
+        retry_backoff_seconds=retry_backoff_seconds,
+        rate_limit_retries=rate_limit_retries,
+        rate_limit_backoff_seconds=rate_limit_backoff_seconds,
+        max_pages=max_pages_per_scholar,
+        page_size=page_size,
+        previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256,
+        upsert_publications_fn=upsert_profile_publications,
+    )
+    assert_valid_paged_parse_result(scholar_id=scholar.scholar_id, paged_parse_result=paged_parse_result)
+    apply_first_page_profile_metadata(scholar=scholar, paged_parse_result=paged_parse_result, run_dt=run_dt)
+    parsed_page = paged_parse_result.parsed_page
+    structured_log(
+        logger,
+        "info",
+        "ingestion.scholar_parsed",
+        user_id=user_id,
+        crawl_run_id=run.id,
+        scholar_profile_id=scholar.id,
+        scholar_id=scholar.scholar_id,
+        state=parsed_page.state.value,
+        publication_count=len(paged_parse_result.publications),
+        has_show_more_button=parsed_page.has_show_more_button,
+        pages_fetched=paged_parse_result.pages_fetched,
+        pages_attempted=paged_parse_result.pages_attempted,
+        has_more_remaining=paged_parse_result.has_more_remaining,
+        pagination_truncated_reason=paged_parse_result.pagination_truncated_reason,
+        warning_count=len(parsed_page.warnings),
+        skipped_no_change=paged_parse_result.skipped_no_change,
+    )
+    result_entry = build_result_entry(
+        scholar=scholar,
+        start_cstart=start_cstart,
+        paged_parse_result=paged_parse_result,
+    )
+    return run_dt, paged_parse_result, result_entry
+
+
+async def _resolve_scholar_outcome(
+    db_session: AsyncSession,
+    *,
+    run: CrawlRun,
+    scholar: ScholarProfile,
+    run_dt: datetime,
+    paged_parse_result: PagedParseResult,
+    result_entry: dict[str, Any],
+) -> ScholarProcessingOutcome:
+    if paged_parse_result.skipped_no_change:
+        return skipped_no_change_outcome(
+            scholar=scholar,
+            run_dt=run_dt,
+            paged_parse_result=paged_parse_result,
+            result_entry=result_entry,
+        )
+    return await upsert_publications_outcome(
+        db_session,
+        run=run,
+        scholar=scholar,
+        run_dt=run_dt,
+        paged_parse_result=paged_parse_result,
+        result_entry=result_entry,
+    )
+
+
+def unexpected_scholar_exception_outcome(
+    *,
+    run: CrawlRun,
+    scholar: ScholarProfile,
+    start_cstart: int,
+    exc: Exception,
+) -> ScholarProcessingOutcome:
+    scholar.last_run_status = RunStatus.FAILED
+    scholar.last_run_dt = datetime.now(UTC)
+    logger.exception(
+        "ingestion.scholar_unexpected_failure",
+        extra={
+            "crawl_run_id": run.id,
+            "scholar_profile_id": scholar.id,
+            "scholar_id": scholar.scholar_id,
+        },
+    )
+    return ScholarProcessingOutcome(
+        result_entry={
+            "scholar_profile_id": scholar.id,
+            "scholar_id": scholar.scholar_id,
+            "state": "ingestion_error",
+            "state_reason": "scholar_processing_exception",
+            "outcome": "failed",
+            "attempt_count": 0,
+            "publication_count": 0,
+            "start_cstart": start_cstart,
+            "warnings": [],
+            "error": str(exc),
+            "debug": {"exception_type": type(exc).__name__, "exception_message": str(exc)},
+        },
+        succeeded_count_delta=0,
+        failed_count_delta=1,
+        partial_count_delta=0,
+        discovered_publication_count=0,
+    )
+
+
+async def run_scholar_iteration(
+    db_session: AsyncSession,
+    *,
+    pagination: PaginationEngine,
+    run: CrawlRun,
+    scholars: list[ScholarProfile],
+    user_id: int,
+    start_cstart_map: dict[int, int],
+    request_delay_seconds: int,
+    network_error_retries: int,
+    retry_backoff_seconds: float,
+    rate_limit_retries: int,
+    rate_limit_backoff_seconds: float,
+    max_pages_per_scholar: int,
+    page_size: int,
+    auto_queue_continuations: bool,
+    queue_delay_seconds: int,
+) -> RunProgress:
+    progress = RunProgress()
+    scholar_kwargs = {
+        "request_delay_seconds": request_delay_seconds,
+        "network_error_retries": network_error_retries,
+        "retry_backoff_seconds": retry_backoff_seconds,
+        "rate_limit_retries": rate_limit_retries,
+        "rate_limit_backoff_seconds": rate_limit_backoff_seconds,
+        "page_size": page_size,
+    }
+
+    # ── Pass 1: first page of every scholar (breadth-first) ──────────
+    first_pass_cstarts: dict[int, int] = {}
+    for index, scholar in enumerate(scholars):
+        await db_session.refresh(run)
+        if run.status == RunStatus.CANCELED:
+            structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id)
+            return progress
+        if index > 0 and request_delay_seconds > 0:
+            await asyncio.sleep(float(request_delay_seconds))
+        start_cstart = int(start_cstart_map.get(int(scholar.id), 0))
+        outcome = await process_scholar(
+            db_session,
+            pagination=pagination,
+            run=run,
+            scholar=scholar,
+            user_id=user_id,
+            start_cstart=start_cstart,
+            max_pages_per_scholar=1,
+            auto_queue_continuations=False,
+            queue_delay_seconds=queue_delay_seconds,
+            **scholar_kwargs,
+        )
+        apply_outcome_to_progress(progress=progress, outcome=outcome)
+        resume_cstart = outcome.result_entry.get("continuation_cstart")
+        if resume_cstart is not None and int(resume_cstart) > start_cstart:
+            first_pass_cstarts[int(scholar.id)] = int(resume_cstart)
+
+    # ── Pass 2: remaining pages for each scholar (depth) ─────────────
+    remaining_max = max(max_pages_per_scholar - 1, 0)
+    if remaining_max <= 0:
+        return progress
+
+    for index, scholar in enumerate(scholars):
+        resume_cstart = first_pass_cstarts.get(int(scholar.id))
+        if resume_cstart is None:
+            continue
+        await db_session.refresh(run)
+        if run.status == RunStatus.CANCELED:
+            structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id)
+            break
+        if index > 0 and request_delay_seconds > 0:
+            await asyncio.sleep(float(request_delay_seconds))
+        outcome = await process_scholar(
+            db_session,
+            pagination=pagination,
+            run=run,
+            scholar=scholar,
+            user_id=user_id,
+            start_cstart=resume_cstart,
+            max_pages_per_scholar=remaining_max,
+            auto_queue_continuations=auto_queue_continuations,
+            queue_delay_seconds=queue_delay_seconds,
+            **scholar_kwargs,
+        )
+        apply_outcome_to_progress(progress=progress, outcome=outcome)
+    return progress

app/services/scholar/source.py

@@ -62,6 +62,7 @@ class LiveScholarSource:
         *,
         timeout_seconds: float = 25.0,
         min_interval_seconds: float | None = None,
+        rotate_user_agents: bool | None = None,
         user_agents: list[str] | None = None,
     ) -> None:
         self._timeout_seconds = timeout_seconds
@@ -72,6 +73,38 @@ class LiveScholarSource:
         )
         self._min_interval_seconds = max(configured_interval, 0.0)
         self._user_agents = user_agents or DEFAULT_USER_AGENTS
+        self._rotate_user_agents = (
+            bool(settings.scholar_http_rotate_user_agent)
+            if rotate_user_agents is None
+            else bool(rotate_user_agents)
+        )
+        configured_user_agent = settings.scholar_http_user_agent.strip()
+        self._configured_user_agent = configured_user_agent or None
+        self._accept_language = settings.scholar_http_accept_language.strip() or "en-US,en;q=0.9"
+        self._cookie_header = settings.scholar_http_cookie.strip() or None
+        self._stable_user_agent = self._resolve_initial_user_agent()
+
+    def _resolve_initial_user_agent(self) -> str:
+        if self._configured_user_agent is not None:
+            return self._configured_user_agent
+        return random.choice(self._user_agents)
+
+    def _resolve_user_agent_for_request(self) -> str:
+        if self._configured_user_agent is not None:
+            return self._configured_user_agent
+        if self._rotate_user_agents:
+            return random.choice(self._user_agents)
+        return self._stable_user_agent
+
+    def _request_headers(self) -> dict[str, str]:
+        headers = {
+            "User-Agent": self._resolve_user_agent_for_request(),
+            "Accept": "text/html,application/xhtml+xml",
+            "Accept-Language": self._accept_language,
+        }
+        if self._cookie_header is not None:
+            headers["Cookie"] = self._cookie_header
+        return headers
 
     async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
         return await self.fetch_profile_page_html(
@@ -116,15 +149,21 @@ class LiveScholarSource:
         return await asyncio.to_thread(self._fetch_sync, requested_url)
 
     def _build_request(self, requested_url: str) -> Request:
-        return Request(
+        return Request(requested_url, headers=self._request_headers())
-            requested_url,
+
-            headers={
+    @staticmethod
-                "User-Agent": random.choice(self._user_agents),
+    def _http_error_reason(*, status_code: int, final_url: str, body: str) -> str:
-                "Accept": "text/html,application/xhtml+xml",
+        lowered_url = final_url.lower()
-                "Accept-Language": "en-US,en;q=0.9",
+        lowered_body = body.lower()
-                "Connection": "close",
+        if "sorry/index" in lowered_url or "sorry/index" in lowered_body:
-            },
+            return "blocked_google_sorry_challenge"
-        )
+        if "our systems have detected" in lowered_body or "unusual traffic" in lowered_body:
+            return "blocked_unusual_traffic_detected"
+        if "automated queries" in lowered_body:
+            return "blocked_automated_queries_detected"
+        if status_code == 429:
+            return "blocked_http_429_rate_limited"
+        return f"http_error_status_{status_code}"
 
     @staticmethod
     def _http_error_body(exc: HTTPError) -> str:
@@ -151,18 +190,27 @@ class LiveScholarSource:
 
     @staticmethod
     def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult:
+        final_url = exc.geturl()
+        body = LiveScholarSource._http_error_body(exc)
+        block_reason = LiveScholarSource._http_error_reason(
+            status_code=exc.code,
+            final_url=final_url,
+            body=body,
+        )
         structured_log(
             logger,
             "warning",
             "scholar_source.fetch_http_error",
             requested_url=requested_url,
             status_code=exc.code,
+            final_url=final_url,
+            block_reason=block_reason,
         )
         return FetchResult(
             requested_url=requested_url,
             status_code=exc.code,
-            final_url=exc.geturl(),
+            final_url=final_url,
-            body=LiveScholarSource._http_error_body(exc),
+            body=body,
             error=str(exc),
         )
 

app/services/scholar/state_detection.py

@@ -38,18 +38,18 @@ def classify_block_or_captcha_reason(
 ) -> str | None:
     if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url):
         return "blocked_accounts_redirect"
-    if status_code == 429:
-        return "blocked_http_429_rate_limited"
-    if status_code == 403:
-        if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url:
-            return "blocked_http_403_captcha_challenge"
-        return "blocked_http_403_forbidden"
     if "sorry/index" in final_url or "sorry/index" in body_lowered:
         return "blocked_google_sorry_challenge"
     if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered:
         return "blocked_unusual_traffic_detected"
     if "automated queries" in body_lowered:
         return "blocked_automated_queries_detected"
+    if status_code == 429:
+        return "blocked_http_429_rate_limited"
+    if status_code == 403:
+        if "recaptcha" in body_lowered or "captcha" in body_lowered:
+            return "blocked_http_403_captcha_challenge"
+        return "blocked_http_403_forbidden"
     if "not a robot" in body_lowered:
         return "blocked_not_a_robot_challenge"
     if "recaptcha" in body_lowered:

app/settings.py

@@ -235,6 +235,13 @@ class Settings:
         "SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD",
         3,
     )
+    scholar_http_user_agent: str = _env_str("SCHOLAR_HTTP_USER_AGENT", "")
+    scholar_http_rotate_user_agent: bool = _env_bool("SCHOLAR_HTTP_ROTATE_USER_AGENT", False)
+    scholar_http_accept_language: str = _env_str(
+        "SCHOLAR_HTTP_ACCEPT_LANGUAGE",
+        "en-US,en;q=0.9",
+    )
+    scholar_http_cookie: str = _env_str("SCHOLAR_HTTP_COOKIE", "")
     unpaywall_enabled: bool = _env_bool("UNPAYWALL_ENABLED", True)
     unpaywall_email: str = _env_str("UNPAYWALL_EMAIL", "")
     unpaywall_timeout_seconds: float = _env_float("UNPAYWALL_TIMEOUT_SECONDS", 4.0)

frontend/src/features/settings/index.ts

@@ -34,6 +34,13 @@ export interface UserSettingsUpdate {
   crossref_api_mailto: string | null;
 }
 
+export interface AdminScholarHttpSettings {
+  user_agent: string;
+  rotate_user_agent: boolean;
+  accept_language: string;
+  cookie: string;
+}
+
 export interface ChangePasswordPayload {
   current_password: string;
   new_password: string;
@@ -60,3 +67,20 @@ export async function changePassword(payload: ChangePasswordPayload): Promise<{
   });
   return response.data;
 }
+
+export async function fetchAdminScholarHttpSettings(): Promise<AdminScholarHttpSettings> {
+  const response = await apiRequest<AdminScholarHttpSettings>("/admin/settings/scholar-http", {
+    method: "GET",
+  });
+  return response.data;
+}
+
+export async function updateAdminScholarHttpSettings(
+  payload: AdminScholarHttpSettings,
+): Promise<AdminScholarHttpSettings> {
+  const response = await apiRequest<AdminScholarHttpSettings>("/admin/settings/scholar-http", {
+    method: "PUT",
+    body: payload,
+  });
+  return response.data;
+}

frontend/src/pages/SettingsPage.vue

@@ -14,9 +14,12 @@ import AppTabs, { type AppTabItem } from "@/components/ui/AppTabs.vue";
 import SettingsAdminPanel from "@/features/settings/SettingsAdminPanel.vue";
 import {
   changePassword,
+  fetchAdminScholarHttpSettings,
   fetchSettings,
+  type AdminScholarHttpSettings,
   type UserSettings,
   type UserSettingsUpdate,
+  updateAdminScholarHttpSettings,
   updateSettings,
 } from "@/features/settings";
 import { ApiRequestError } from "@/lib/api/errors";
@@ -40,6 +43,7 @@ const router = useRouter();
 
 const loading = ref(true);
 const saving = ref(false);
+const savingScholarHttp = ref(false);
 const updatingPassword = ref(false);
 
 const autoRunEnabled = ref(false);
@@ -49,6 +53,10 @@ const navVisiblePages = ref<string[]>([]);
 const openalexApiKey = ref("");
 const crossrefApiToken = ref("");
 const crossrefApiMailto = ref("");
+const scholarHttpUserAgent = ref("");
+const scholarHttpRotateUserAgent = ref(false);
+const scholarHttpAcceptLanguage = ref("en-US,en;q=0.9");
+const scholarHttpCookie = ref("");
 
 const currentPassword = ref("");
 const newPassword = ref("");
@@ -142,6 +150,13 @@ function hydrateSettings(settings: UserSettings): void {
   runStatus.setSafetyState(settings.safety_state);
 }
 
+function hydrateScholarHttpSettings(settings: AdminScholarHttpSettings): void {
+  scholarHttpUserAgent.value = settings.user_agent;
+  scholarHttpRotateUserAgent.value = Boolean(settings.rotate_user_agent);
+  scholarHttpAcceptLanguage.value = settings.accept_language;
+  scholarHttpCookie.value = settings.cookie;
+}
+
 function parseBoundedInteger(value: string, label: string, minimum: number): number {
   const parsed = Number(value);
   if (!Number.isInteger(parsed)) {
@@ -160,6 +175,9 @@ async function loadSettings(): Promise<void> {
   try {
     const settings = await fetchSettings();
     hydrateSettings(settings);
+    if (auth.isAdmin) {
+      hydrateScholarHttpSettings(await fetchAdminScholarHttpSettings());
+    }
   } catch (error) {
     if (error instanceof ApiRequestError) {
       errorMessage.value = error.message;
@@ -172,6 +190,32 @@ async function loadSettings(): Promise<void> {
   }
 }
 
+async function onSaveScholarHttpSettings(): Promise<void> {
+  savingScholarHttp.value = true;
+  errorMessage.value = null;
+  errorRequestId.value = null;
+  successMessage.value = null;
+  try {
+    const updated = await updateAdminScholarHttpSettings({
+      user_agent: scholarHttpUserAgent.value.trim(),
+      rotate_user_agent: scholarHttpRotateUserAgent.value,
+      accept_language: scholarHttpAcceptLanguage.value.trim(),
+      cookie: scholarHttpCookie.value.trim(),
+    });
+    hydrateScholarHttpSettings(updated);
+    successMessage.value = "Scholar HTTP profile updated.";
+  } catch (error) {
+    if (error instanceof ApiRequestError) {
+      errorMessage.value = error.message;
+      errorRequestId.value = error.requestId;
+    } else {
+      errorMessage.value = "Unable to save Scholar HTTP profile.";
+    }
+  } finally {
+    savingScholarHttp.value = false;
+  }
+}
+
 async function onSaveSettings(): Promise<void> {
   saving.value = true;
   errorMessage.value = null;
@@ -404,6 +448,49 @@ onMounted(async () => {
                 <AppInput v-model="crossrefApiMailto" placeholder="e.g. admin@yourdomain.com" type="email" autocomplete="off" />
               </label>
             </div>
+
+            <div class="grid gap-2 rounded-lg border border-stroke-default bg-surface-card-muted p-3">
+              <h3 class="text-sm font-semibold text-ink-secondary">Scholar Request Profile</h3>
+              <p class="text-xs text-secondary mb-2">
+                Tune Scholar HTTP fingerprint behavior used by live scraper requests. Changes apply to new runs.
+              </p>
+              <label class="grid gap-1 text-sm font-medium text-ink-secondary">
+                <span>User-Agent override</span>
+                <AppInput
+                  v-model="scholarHttpUserAgent"
+                  placeholder="Leave empty to use built-in browser-like user agent"
+                  autocomplete="off"
+                />
+              </label>
+              <label class="grid gap-1 text-sm font-medium text-ink-secondary mt-2">
+                <span>Accept-Language</span>
+                <AppInput
+                  v-model="scholarHttpAcceptLanguage"
+                  placeholder="e.g. en-US,en;q=0.9"
+                  autocomplete="off"
+                />
+              </label>
+              <label class="grid gap-1 text-sm font-medium text-ink-secondary mt-2">
+                <span>Cookie header</span>
+                <AppInput
+                  v-model="scholarHttpCookie"
+                  placeholder="Optional. Leave empty to disable cookie passthrough"
+                  autocomplete="off"
+                />
+              </label>
+              <div class="mt-2">
+                <AppCheckbox
+                  id="scholar-http-rotate-user-agent"
+                  v-model="scholarHttpRotateUserAgent"
+                  label="Rotate user-agent per request"
+                />
+              </div>
+              <div>
+                <AppButton :disabled="savingScholarHttp" @click="onSaveScholarHttpSettings">
+                  {{ savingScholarHttp ? "Saving..." : "Save Scholar request profile" }}
+                </AppButton>
+              </div>
+            </div>
           </div>
 
           <div>

tests/integration/test_api_v1.py

@@ -391,6 +391,61 @@ async def test_api_admin_user_management_endpoints(db_session: AsyncSession) ->
     assert created_exists.scalar_one() == 1
 
 
+@pytest.mark.integration
+@pytest.mark.db
+@pytest.mark.asyncio
+async def test_api_admin_scholar_http_settings_endpoints(db_session: AsyncSession) -> None:
+    await insert_user(
+        db_session,
+        email="api-admin-http@example.com",
+        password="admin-password",
+        is_admin=True,
+    )
+    await insert_user(
+        db_session,
+        email="api-member-http@example.com",
+        password="member-password",
+        is_admin=False,
+    )
+    previous_user_agent = settings.scholar_http_user_agent
+    previous_rotate = settings.scholar_http_rotate_user_agent
+    previous_accept_language = settings.scholar_http_accept_language
+    previous_cookie = settings.scholar_http_cookie
+    try:
+        client = TestClient(app)
+        login_user(client, email="api-admin-http@example.com", password="admin-password")
+        headers = _api_csrf_headers(client)
+
+        read_response = client.get("/api/v1/admin/settings/scholar-http")
+        assert read_response.status_code == 200
+
+        update_response = client.put(
+            "/api/v1/admin/settings/scholar-http",
+            json={
+                "user_agent": "Mozilla/5.0 Test Runner",
+                "rotate_user_agent": False,
+                "accept_language": "en-US,en;q=0.8",
+                "cookie": "SID=test-cookie",
+            },
+            headers=headers,
+        )
+        assert update_response.status_code == 200
+        payload = update_response.json()["data"]
+        assert payload["user_agent"] == "Mozilla/5.0 Test Runner"
+        assert payload["cookie"] == "SID=test-cookie"
+        assert settings.scholar_http_user_agent == "Mozilla/5.0 Test Runner"
+
+        client.post("/api/v1/auth/logout", headers=headers)
+        login_user(client, email="api-member-http@example.com", password="member-password")
+        forbidden_response = client.get("/api/v1/admin/settings/scholar-http")
+        assert forbidden_response.status_code == 403
+    finally:
+        object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent)
+        object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate)
+        object.__setattr__(settings, "scholar_http_accept_language", previous_accept_language)
+        object.__setattr__(settings, "scholar_http_cookie", previous_cookie)
+
+
 @pytest.mark.integration
 @pytest.mark.db
 @pytest.mark.asyncio

tests/integration/test_deferred_enrichment.py

@@ -8,7 +8,7 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db.models import CrawlRun, Publication, RunStatus, RunTriggerType, ScholarProfile, ScholarPublication
-from app.services.ingestion.application import ScholarIngestionService
+from app.services.ingestion.enrichment import EnrichmentRunner
 from app.services.openalex.types import OpenAlexWork
 from tests.integration.helpers import insert_user
 
@@ -79,8 +79,7 @@ async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession
         oa_url="http://example.com/grover.pdf",
     )
 
-    mock_source = MagicMock()
+    runner = EnrichmentRunner()
-    service = ScholarIngestionService(source=mock_source)
 
     # We patch the client at its source, and also mock arXiv to avoid real HTTP calls
     with (
@@ -93,7 +92,7 @@ async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession
         mock_instance.get_works_by_filter = AsyncMock(return_value=[mock_work])
 
         # 5. Execute the enrichment pass for the NEW run
-        await service._enrich_pending_publications(db_session, run_id=new_run.id)
+        await runner.enrich_pending_publications(db_session, run_id=new_run.id)
         await db_session.commit()
 
     # 6. Verification: The publication from the FAILED run should now be enriched

tests/unit/test_ingestion_arxiv_rate_limit.py

@@ -5,7 +5,7 @@ from types import SimpleNamespace
 import pytest
 
 from app.services.arxiv.errors import ArxivRateLimitError
-from app.services.ingestion.application import ScholarIngestionService
+from app.services.ingestion.enrichment import EnrichmentRunner
 from app.services.publication_identifiers import application as identifier_service
 
 
@@ -13,7 +13,7 @@ from app.services.publication_identifiers import application as identifier_servi
 async def test_discover_identifiers_for_enrichment_disables_arxiv_on_rate_limit(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    service = ScholarIngestionService(source=object())
+    runner = EnrichmentRunner()
     publication = SimpleNamespace(id=11, author_text="Ada Lovelace")
     calls = {"sync": 0}
 
@@ -35,9 +35,9 @@ async def test_discover_identifiers_for_enrichment_disables_arxiv_on_rate_limit(
     async def _publish_noop(*args, **kwargs) -> None:
         _ = (args, kwargs)
 
-    monkeypatch.setattr(service, "_publish_identifier_update_event", _publish_noop)
+    monkeypatch.setattr(runner, "_publish_identifier_update_event", _publish_noop)
 
-    result = await service._discover_identifiers_for_enrichment(
+    result = await runner._discover_identifiers_for_enrichment(
         object(),
         publication=publication,
         run_id=321,

tests/unit/test_ingestion_progress_reporting.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import pytest
 
-from app.services.ingestion.application import ScholarIngestionService
+from app.services.ingestion.run_completion import apply_outcome_to_progress
 from app.services.ingestion.types import RunProgress, ScholarProcessingOutcome
 
 
@@ -31,11 +31,11 @@ def _outcome(*, scholar_profile_id: int, outcome_label: str) -> ScholarProcessin
 def test_apply_outcome_to_progress_replaces_previous_scholar_outcome() -> None:
     progress = RunProgress()
 
-    ScholarIngestionService._apply_outcome_to_progress(
+    apply_outcome_to_progress(
         progress=progress,
         outcome=_outcome(scholar_profile_id=42, outcome_label="partial"),
     )
-    ScholarIngestionService._apply_outcome_to_progress(
+    apply_outcome_to_progress(
         progress=progress,
         outcome=_outcome(scholar_profile_id=42, outcome_label="success"),
     )
@@ -58,4 +58,4 @@ def test_apply_outcome_to_progress_rejects_invalid_scholar_id() -> None:
     )
 
     with pytest.raises(RuntimeError, match="missing valid scholar_profile_id"):
-        ScholarIngestionService._apply_outcome_to_progress(progress=progress, outcome=invalid)
+        apply_outcome_to_progress(progress=progress, outcome=invalid)

tests/unit/test_scholar_parser.py

@@ -433,3 +433,18 @@ def test_parse_author_search_page_classifies_http_429_as_blocked() -> None:
 
     assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA
     assert parsed.state_reason == "blocked_http_429_rate_limited"
+
+
+def test_parse_author_search_page_prefers_sorry_challenge_reason_over_generic_429() -> None:
+    fetch_result = FetchResult(
+        requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
+        status_code=429,
+        final_url="https://www.google.com/sorry/index?continue=scholar",
+        body="<html><body>Our systems have detected unusual traffic.</body></html>",
+        error=None,
+    )
+
+    parsed = parse_author_search_page(fetch_result)
+
+    assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA
+    assert parsed.state_reason == "blocked_google_sorry_challenge"

tests/unit/test_scholar_source.py

@@ -2,6 +2,12 @@ import pytest
 
 from app.services.scholar import rate_limit as scholar_rate_limit
 from app.services.scholar.source import FetchResult, LiveScholarSource, _build_profile_url
+from app.settings import settings
+
+
+def _request_header(request, name: str) -> str | None:
+    headers = {key.lower(): value for key, value in request.header_items()}
+    return headers.get(name.lower())
 
 
 def test_build_profile_url_includes_pagesize_for_initial_page() -> None:
@@ -48,3 +54,72 @@ async def test_live_scholar_source_applies_global_throttle(monkeypatch: pytest.M
 
     assert result == expected_result
     assert captured_interval == [7.0]
+
+
+def test_http_error_reason_prefers_sorry_challenge_marker() -> None:
+    reason = LiveScholarSource._http_error_reason(
+        status_code=429,
+        final_url="https://www.google.com/sorry/index?continue=example",
+        body="",
+    )
+    assert reason == "blocked_google_sorry_challenge"
+
+
+def test_http_error_reason_falls_back_to_http_429_rate_limit() -> None:
+    reason = LiveScholarSource._http_error_reason(
+        status_code=429,
+        final_url="https://scholar.google.com/citations?hl=en&user=abc",
+        body="Too Many Requests",
+    )
+    assert reason == "blocked_http_429_rate_limited"
+
+
+def test_build_request_uses_stable_user_agent_by_default() -> None:
+    previous_user_agent = settings.scholar_http_user_agent
+    previous_rotate = settings.scholar_http_rotate_user_agent
+    previous_cookie = settings.scholar_http_cookie
+    previous_accept_language = settings.scholar_http_accept_language
+    object.__setattr__(settings, "scholar_http_user_agent", "")
+    object.__setattr__(settings, "scholar_http_rotate_user_agent", False)
+    object.__setattr__(settings, "scholar_http_cookie", "")
+    object.__setattr__(settings, "scholar_http_accept_language", "en-US,en;q=0.9")
+    try:
+        source = LiveScholarSource(user_agents=["UA-A", "UA-B"])
+        request_one = source._build_request("https://example.test/one")
+        request_two = source._build_request("https://example.test/two")
+        assert _request_header(request_one, "User-Agent") == _request_header(request_two, "User-Agent")
+        assert _request_header(request_one, "Connection") is None
+    finally:
+        object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent)
+        object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate)
+        object.__setattr__(settings, "scholar_http_cookie", previous_cookie)
+        object.__setattr__(settings, "scholar_http_accept_language", previous_accept_language)
+
+
+def test_build_request_rotates_user_agent_when_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
+    previous_user_agent = settings.scholar_http_user_agent
+    previous_rotate = settings.scholar_http_rotate_user_agent
+    object.__setattr__(settings, "scholar_http_user_agent", "")
+    object.__setattr__(settings, "scholar_http_rotate_user_agent", True)
+    choices = iter(["UA-STABLE", "UA-1", "UA-2"])
+    monkeypatch.setattr("app.services.scholar.source.random.choice", lambda _values: next(choices))
+    try:
+        source = LiveScholarSource(user_agents=["UA-A", "UA-B"])
+        request_one = source._build_request("https://example.test/one")
+        request_two = source._build_request("https://example.test/two")
+        assert _request_header(request_one, "User-Agent") == "UA-1"
+        assert _request_header(request_two, "User-Agent") == "UA-2"
+    finally:
+        object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent)
+        object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate)
+
+
+def test_build_request_includes_cookie_when_configured() -> None:
+    previous_cookie = settings.scholar_http_cookie
+    object.__setattr__(settings, "scholar_http_cookie", "SID=abc123")
+    try:
+        source = LiveScholarSource(user_agents=["UA-A"])
+        request = source._build_request("https://example.test/one")
+        assert _request_header(request, "Cookie") == "SID=abc123"
+    finally:
+        object.__setattr__(settings, "scholar_http_cookie", previous_cookie)