fix(frontend): confirm modal, settings display, race guard, and test coverage

- S1: Replace window.confirm() with AppConfirmModal for delete actions
- S4: Add formatHours helper to avoid ugly decimals in settings hints
- W5: Add generation counter to SettingsAdminPanel to prevent stale
  async results when section changes rapidly
- S7: Remove unused ApiRequestError import from useScholarBulkActions
- S2: Add error-path tests for bulk delete/toggle network failures
- S3: Add CSRF boundary tests for bulk-delete and bulk-toggle endpoints

.claude/settings.json

@@ -0,0 +1,33 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(wc -l /home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src/features/settings/SettingsAdminPanel.vue /home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src/features/settings/components/Admin*.vue)",
+      "Read(//home/jv/src/personal/playground/scholar_scraper/scholar-scraper/**)",
+      "Bash(while read line rest)",
+      "Bash(do echo \"$line $rest\")",
+      "Bash(done)",
+      "Bash(grep -c 'def test_' tests/unit/*.py tests/unit/**/*.py tests/integration/*.py)",
+      "Bash(sort -t: -k2 -rn)",
+      "Bash(wc -l frontend/src/pages/*.vue)",
+      "Bash(uv run ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
+      "Bash(ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
+      "Bash(npm install --package-lock-only)",
+      "Bash(python -m ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
+      "Bash(pip show ruff)",
+      "Bash(pip install ruff)",
+      "Bash(uvx ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
+      "Bash(git add app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py docs/website/package-lock.json)",
+      "Bash(git push)",
+      "Bash(uvx ruff format --check app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
+      "Bash(test -f docs/website/package-lock.json)",
+      "Bash(gh run list --branch openalex-size-reduction-scholar-hydration --limit 3 --json databaseId,status,conclusion,name,event)",
+      "Bash(gh run list --limit 10 --json databaseId,status,conclusion,name,event,headBranch)",
+      "Bash(gh run view 22554585841 --log-failed)",
+      "Bash(gh run rerun 22554585841 --failed)",
+      "Bash(bash scripts/premerge.sh)"
+    ],
+    "additionalDirectories": [
+      "/home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src"
+    ]
+  }
+}

.env.example

@@ -1,26 +1,53 @@
+# ------------------------------
+# Compose + Database
+# ------------------------------
 POSTGRES_DB=scholar
 POSTGRES_USER=scholar
 POSTGRES_PASSWORD=change-me
-
 DATABASE_URL=postgresql+asyncpg://scholar:scholar@db:5432/scholar
 # Optional override. If empty, tests derive "<DATABASE_URL db name>_test".
 TEST_DATABASE_URL=
 SCHOLARR_IMAGE=justinzeus/scholarr:latest
 
+# ------------------------------
+# App Runtime + Networking
+# ------------------------------
 APP_NAME=scholarr
 APP_HOST=0.0.0.0
 APP_PORT=8000
 APP_HOST_PORT=8000
 APP_RELOAD=0
+MIGRATE_ON_START=1
 FRONTEND_ENABLED=1
 FRONTEND_DIST_DIR=/app/frontend/dist
+
+# ------------------------------
+# Database Pool
+# ------------------------------
+DATABASE_POOL_MODE=auto
+DATABASE_POOL_SIZE=5
+DATABASE_POOL_MAX_OVERFLOW=10
+DATABASE_POOL_TIMEOUT_SECONDS=30
+DATABASE_RESERVED_API_CONNECTIONS=3
+
+# ------------------------------
+# Frontend Dev Overrides
+# ------------------------------
 FRONTEND_HOST_PORT=5173
 CHOKIDAR_USEPOLLING=1
 VITE_DEV_API_PROXY_TARGET=http://app:8000
-MIGRATE_ON_START=1
 
+# ------------------------------
+# Auth + Session
+# ------------------------------
 SESSION_SECRET_KEY=replace-with-a-long-random-secret-at-least-32-characters
 SESSION_COOKIE_SECURE=1
+LOGIN_RATE_LIMIT_ATTEMPTS=5
+LOGIN_RATE_LIMIT_WINDOW_SECONDS=60
+
+# ------------------------------
+# HTTP Security Headers + CSP
+# ------------------------------
 SECURITY_HEADERS_ENABLED=1
 SECURITY_X_CONTENT_TYPE_OPTIONS=nosniff
 SECURITY_X_FRAME_OPTIONS=DENY
@@ -36,22 +63,32 @@ SECURITY_STRICT_TRANSPORT_SECURITY_ENABLED=0
 SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE=31536000
 SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS=1
 SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD=0
-LOGIN_RATE_LIMIT_ATTEMPTS=5
-LOGIN_RATE_LIMIT_WINDOW_SECONDS=60
+
+# ------------------------------
+# Logging
+# ------------------------------
 LOG_LEVEL=INFO
 LOG_FORMAT=console
 LOG_REQUESTS=1
 LOG_UVICORN_ACCESS=0
 LOG_REQUEST_SKIP_PATHS=/healthz
 LOG_REDACT_FIELDS=
+
+# ------------------------------
+# Scheduler + Ingestion Safety
+# ------------------------------
 SCHEDULER_ENABLED=1
 SCHEDULER_TICK_SECONDS=60
+SCHEDULER_QUEUE_BATCH_SIZE=10
+SCHEDULER_PDF_QUEUE_BATCH_SIZE=15
 INGESTION_AUTOMATION_ALLOWED=1
 INGESTION_MANUAL_RUN_ALLOWED=1
 INGESTION_MIN_RUN_INTERVAL_MINUTES=15
 INGESTION_MIN_REQUEST_DELAY_SECONDS=2
 INGESTION_NETWORK_ERROR_RETRIES=1
 INGESTION_RETRY_BACKOFF_SECONDS=1.0
+INGESTION_RATE_LIMIT_RETRIES=3
+INGESTION_RATE_LIMIT_BACKOFF_SECONDS=30.0
 INGESTION_MAX_PAGES_PER_SCHOLAR=30
 INGESTION_PAGE_SIZE=100
 INGESTION_ALERT_BLOCKED_FAILURE_THRESHOLD=1
@@ -63,7 +100,10 @@ INGESTION_CONTINUATION_QUEUE_ENABLED=1
 INGESTION_CONTINUATION_BASE_DELAY_SECONDS=120
 INGESTION_CONTINUATION_MAX_DELAY_SECONDS=3600
 INGESTION_CONTINUATION_MAX_ATTEMPTS=6
-SCHEDULER_QUEUE_BATCH_SIZE=10
+
+# ------------------------------
+# Scholar Images + Name Search Safety
+# ------------------------------
 SCHOLAR_IMAGE_UPLOAD_DIR=/var/lib/scholarr/uploads
 SCHOLAR_IMAGE_UPLOAD_MAX_BYTES=2000000
 SCHOLAR_NAME_SEARCH_ENABLED=1
@@ -76,11 +116,49 @@ SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD=1
 SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS=1800
 SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD=2
 SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD=3
+SCHOLAR_HTTP_USER_AGENT=
+SCHOLAR_HTTP_ROTATE_USER_AGENT=0
+SCHOLAR_HTTP_ACCEPT_LANGUAGE=en-US,en;q=0.9
+SCHOLAR_HTTP_COOKIE=
 
+# ------------------------------
+# OA Enrichment + PDF Resolution
+# ------------------------------
+UNPAYWALL_ENABLED=1
+UNPAYWALL_EMAIL=
+UNPAYWALL_TIMEOUT_SECONDS=4.0
+UNPAYWALL_MIN_INTERVAL_SECONDS=0.6
+UNPAYWALL_MAX_ITEMS_PER_REQUEST=20
+UNPAYWALL_RETRY_COOLDOWN_SECONDS=1800
+UNPAYWALL_PDF_DISCOVERY_ENABLED=1
+UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES=5
+UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES=500000
+ARXIV_ENABLED=1
+ARXIV_TIMEOUT_SECONDS=3.0
+ARXIV_MIN_INTERVAL_SECONDS=4.0
+ARXIV_RATE_LIMIT_COOLDOWN_SECONDS=60.0
+ARXIV_DEFAULT_MAX_RESULTS=3
+ARXIV_CACHE_TTL_SECONDS=900
+ARXIV_CACHE_MAX_ENTRIES=512
+ARXIV_MAILTO=
+PDF_AUTO_RETRY_INTERVAL_SECONDS=86400
+PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS=3600
+PDF_AUTO_RETRY_MAX_ATTEMPTS=3
+CROSSREF_ENABLED=1
+CROSSREF_MAX_ROWS=10
+CROSSREF_TIMEOUT_SECONDS=8.0
+CROSSREF_MIN_INTERVAL_SECONDS=0.6
+CROSSREF_MAX_LOOKUPS_PER_REQUEST=8
+OPENALEX_API_KEY=
+CROSSREF_API_TOKEN=
+CROSSREF_API_MAILTO=
+
+# ------------------------------
+# Startup Bootstrap + DB Wait
+# ------------------------------
 BOOTSTRAP_ADMIN_ON_START=0
 BOOTSTRAP_ADMIN_EMAIL=
 BOOTSTRAP_ADMIN_PASSWORD=
 BOOTSTRAP_ADMIN_FORCE_PASSWORD=0
-
 DB_WAIT_TIMEOUT_SECONDS=60
 DB_WAIT_INTERVAL_SECONDS=2

.github/FUNDING.yml

@@ -0,0 +1 @@
+buy_me_a_coffee: justinzeus

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: npm
+    directory: /frontend
+    schedule:
+      interval: weekly
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly

.github/workflows/ci.yml

@@ -17,10 +17,31 @@ jobs:
       - name: Enforce no generated artifacts
         run: ./scripts/check_no_generated_artifacts.sh
 
+      - name: Enforce env contract parity
+        run: python3 scripts/check_env_contract.py
+
+  lint:
+    runs-on: ubuntu-latest
+    needs: [repo-hygiene]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: astral-sh/setup-uv@v4
+      - run: uv sync --extra dev
+      - name: Ruff check
+        run: uv run ruff check .
+      - name: Ruff format check
+        run: uv run ruff format --check .
+      - name: Mypy
+        run: uv run mypy app/ --ignore-missing-imports
+
   test:
     runs-on: ubuntu-latest
     needs:
       - repo-hygiene
+      - lint
     services:
       postgres:
         image: postgres:15
@@ -88,7 +109,7 @@ jobs:
 
       - name: Install frontend dependencies
         working-directory: frontend
-        run: npm install
+        run: npm ci
 
       - name: Frontend theme token policy
         working-directory: frontend
@@ -106,12 +127,33 @@ jobs:
         working-directory: frontend
         run: npm run build
 
+  docs-quality:
+    runs-on: ubuntu-latest
+    needs:
+      - repo-hygiene
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install docs dependencies
+        run: npm ci --prefix docs/website
+
+      - name: Docs build
+        run: npm --prefix docs/website run build
+
   docker-publish:
     runs-on: ubuntu-latest
     needs:
       - repo-hygiene
       - test
       - frontend-quality
+      - docs-quality
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     permissions:
       contents: read

.github/workflows/codeql.yml

@@ -0,0 +1,26 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "30 5 * * 1"
+
+jobs:
+  analyze:
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      contents: read
+    strategy:
+      matrix:
+        language: [python, javascript]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+      - uses: github/codeql-action/autobuild@v3
+      - uses: github/codeql-action/analyze@v3

.github/workflows/docs-pages.yml

@@ -0,0 +1,57 @@
+name: Docs Pages
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - docs/**
+      - .github/workflows/docs-pages.yml
+
+permissions:
+  contents: read
+
+concurrency:
+  group: docs-pages
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install docs dependencies
+        run: npm ci --prefix docs/website
+
+      - name: Build docs site
+        run: npm --prefix docs/website run build
+
+      - name: Upload pages artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: docs/website/build
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/release.yml

@@ -0,0 +1,32 @@
+name: Release
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    if: github.repository == 'JustinZeus/scholarr'
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: astral-sh/setup-uv@v4
+      - run: uv sync --extra dev
+      - name: Lint
+        run: uv run ruff check
+      - name: Test
+        run: uv run pytest
+      - name: Semantic Release
+        id: release
+        run: uv run semantic-release publish
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -12,6 +12,11 @@ htmlcov/
 *.log
 .DS_Store
 planning/
+build/
 frontend/node_modules/
 frontend/dist/
 frontend/.vite/
+docs/website/node_modules/
+docs/website/build/
+docs/website/.docusaurus/
+.claude/

AGENTS.MD

@@ -1,314 +0,0 @@
-# scholarr Agent Handbook
-
-This file is the consolidated source of truth for project scope, constraints, scrape contract, and UI implementation flow.
-
-## 1. Product Objective
-
-- Build a self-hosted scholar tracking system ("scholarr") with reliable, low-cost scraping and clear, actionable UI.
-- Keep the MVP scope intentionally small.
-- Prioritize reliability and ease of use over advanced/fancy processing.
-
-## 2. Locked Constraints and Preferences
-
-- Multi-user system with strict tenant isolation.
-- Users are admin-created only.
-- Users can change their own password.
-- Admin features are shown only to admin users.
-- Same-origin cookie session model with CSRF protection.
-- Container-first development workflow (`docker compose`, `uv`).
-- Test while developing; avoid shipping untested flows.
-- Keep backend modular and DRY.
-- UI is being rebuilt from scratch against API contracts.
-
-## 3. Current Backend Architecture
-
-- Runtime: Python + FastAPI.
-- DB: PostgreSQL + SQLAlchemy + Alembic migrations.
-- Scheduler: background scheduler + continuation queue processing.
-- Auth:
-  - Argon2 password hashing.
-  - Session cookie (`HttpOnly`, `SameSite=Lax`, secure flag configurable).
-  - CSRF required for unsafe methods via `X-CSRF-Token`.
-- Logging:
-  - structured request logging with request IDs.
-  - redaction controls for sensitive fields.
-
-## 4. Scraping Contract (Probe-Based)
-
-Status: sufficient for MVP implementation with graceful degradation.
-
-Target endpoint:
-- `https://scholar.google.com/citations?hl=en&user=<scholar_id>`
-
-Primary selectors:
-- Row: `tr.gsc_a_tr`
-- Title/link: `a.gsc_a_at`
-- Citation count: `a.gsc_a_ac`
-- Year: `span.gsc_a_h` (fallback regex on year cell text)
-- Metadata lines: first/second `div.gs_gray` -> authors/venue
-- Cluster ID: from `citation_for_view=<user>:<cluster_id>` in title URL
-
-Required parser states:
-- `ok`
-- `no_results`
-- `blocked_or_captcha`
-- `layout_changed`
-- `network_error`
-
-Required page flags:
-- `has_show_more_button`
-- `articles_range`
-
-Quality assumptions from probe:
-- title/cluster_id/citation/authors coverage observed near 100%
-- year and venue may be missing and must remain nullable
-
-Guardrails:
-- Never crash the full run when one scholar fails.
-- Persist structured failure/debug information for diagnosis.
-- Handle inaccessible/redirected IDs as states, not exceptions.
-
-## 5. Current API Scope
-
-Base path: `/api/v1`
-
-Envelope model:
-- success: `{"data": ..., "meta": {"request_id": "..."}}`
-- error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}`
-
-Auth/session:
-- `GET /api/v1/auth/csrf`
-- `POST /api/v1/auth/login`
-- `GET /api/v1/auth/me`
-- `POST /api/v1/auth/change-password`
-- `POST /api/v1/auth/logout`
-
-Admin users:
-- `GET /api/v1/admin/users`
-- `POST /api/v1/admin/users`
-- `PATCH /api/v1/admin/users/{id}/active`
-- `POST /api/v1/admin/users/{id}/reset-password`
-
-Scholars:
-- `GET /api/v1/scholars`
-- `POST /api/v1/scholars`
-- `GET /api/v1/scholars/search`
-- `PATCH /api/v1/scholars/{id}/toggle`
-- `PUT /api/v1/scholars/{id}/image/url`
-- `POST /api/v1/scholars/{id}/image/upload`
-- `GET /api/v1/scholars/{id}/image/upload`
-- `DELETE /api/v1/scholars/{id}/image`
-- `DELETE /api/v1/scholars/{id}`
-
-Settings:
-- `GET /api/v1/settings`
-- `PUT /api/v1/settings`
-
-Runs/diagnostics/queue:
-- `GET /api/v1/runs`
-- `GET /api/v1/runs/{id}`
-- `POST /api/v1/runs/manual` (`Idempotency-Key` supported)
-- `GET /api/v1/runs/queue/items`
-- `POST /api/v1/runs/queue/{id}/retry`
-- `POST /api/v1/runs/queue/{id}/drop`
-- `DELETE /api/v1/runs/queue/{id}`
-
-Publications:
-- `GET /api/v1/publications`
-- `POST /api/v1/publications/mark-read`
-- `POST /api/v1/publications/mark-all-read`
-
-Ops:
-- `GET /healthz`
-
-## 5.1 Runtime Semantics (Locked)
-
-Manual run idempotency:
-- `Idempotency-Key` is optional on `POST /api/v1/runs/manual`.
-- Idempotency scope is `(user_id, idempotency_key)`.
-- Replaying the same key for the same user returns the same run payload after completion.
-- If the keyed run is still in progress, respond with `409 run_in_progress` and include `run_id`.
-- Enforcement is database-backed, not best-effort in-memory logic.
-- Frontend triggers generate keys internally; key input is not user-facing.
-
-Continuation queue attempt policy:
-- `attempt_count` is a failed-attempt budget, not a total execution counter.
-- Increment attempts only when a continuation execution fails to make progress.
-- Reset attempts to `0` after successful progress (success or resumable partial progress).
-- Drop queue items only after failed-attempt threshold is reached.
-
-Scholar no-change skip policy:
-- Persist first-page fingerprint snapshots per scholar.
-- If the first-page fingerprint is unchanged for a normal run (`cstart=0`), skip deep pagination/upsert work.
-- Record run scholar result `state_reason` as `no_change_initial_page_signature`.
-
-Scholar create naming policy:
-- Do not accept custom display names during scholar create.
-- Hydrate names from scraped profile metadata.
-
-Scholar name-search safety policy:
-- Treat name search as best-effort only.
-- Apply single-flight execution, pacing/jitter, cache, and cooldown circuit breaker after repeated blocked responses.
-- Prefer Scholar URL/ID adds for reliable ingestion onboarding.
-
-## 6. Required UI Behavior (MVP)
-
-Core UX entities:
-- scholars
-- publications (`all` and `new`)
-- runs + run diagnostics
-- queue visibility/actions
-- settings/account
-- admin users (role-gated)
-
-Critical semantics:
-- Keep `is_new_in_latest_run` separate from `is_read`.
-- First-time ingestion should not imply user read-state.
-- Surface partial runs and warnings clearly.
-- Show loading, empty, and error states explicitly.
-- Support both light and dark modes with consistent status semantics.
-
-## 7. UI Information Architecture
-
-Public:
-- Login
-
-Authenticated:
-- Dashboard
-- Scholars
-- Publications
-- Settings
-- Admin Users (admin only)
-
-## 8. End-to-End UI Flow Plan
-
-## 8.1 Session bootstrap
-
-1. On load call `GET /api/v1/auth/csrf`.
-2. Call `GET /api/v1/auth/me`.
-3. Route to login or dashboard based on auth state.
-
-## 8.2 Login/logout
-
-1. Login form submits `POST /api/v1/auth/login` with `X-CSRF-Token`.
-2. Store CSRF token from response for future unsafe requests.
-3. Logout calls `POST /api/v1/auth/logout`.
-
-## 8.3 Dashboard
-1. Fetch latest publications summary from `GET /api/v1/publications?mode=new&limit=20`.
-2. Fetch recent runs from `GET /api/v1/runs?limit=5`.
-3. Fetch queue status from `GET /api/v1/runs/queue/items?limit=200` (aggregate in UI).
-4. Show:
-   - new publications count (`new_count`)
-   - total tracked publications (`total_count`)
-   - latest run status/started time
-   - queue badges (`queued`, `retrying`, `dropped`)
-5. Admin users additionally see a diagnostics shortcut block to full runs/queue page.
-6. Every dashboard error panel must display request-id for support.
-
-
-## 8.4 Scholars
-
-1. List via `GET /api/v1/scholars`.
-2. Add/toggle/delete actions mapped to scholar endpoints.
-3. Search by name via `GET /api/v1/scholars/search` with candidate add actions.
-4. Profile image controls:
-   - scraped image fallback from Scholar metadata
-   - custom URL override
-   - image upload + reset to scraped/default
-5. Per-row quick links:
-   - all publications filtered by scholar
-   - new publications filtered by scholar
-
-## 8.5 Publications
-
-1. Default view mode is `new` (`GET /api/v1/publications?mode=new`).
-2. Support mode toggle:
-   - `new`: scoped to latest completed run
-   - `all`: full history
-3. Support scholar filter via `scholar_profile_id` query param.
-4. Render both badges independently:
-   - `is_new_in_latest_run`
-   - `is_read`
-5. "Mark all read" action calls `POST /api/v1/publications/mark-all-read` with CSRF.
-6. Show explicit empty states:
-   - no publications tracked
-   - no new publications in latest run
-   - filtered scholar has no items
-7. Surface run recency context when mode is `new` (latest completed run semantics).
-
-## 8.6 Runs and diagnostics (admin)
-
-1. List runs from `GET /api/v1/runs`.
-2. Run details from `GET /api/v1/runs/{id}`:
-   - scholar result state/reason
-   - warnings
-   - page logs + attempt logs
-   - debug metadata
-3. Queue operations from queue endpoints (`retry`, `drop`, `clear`).
-
-## 8.7 Settings + account
-
-1. User ingest settings from `/api/v1/settings`.
-2. Password change via `/api/v1/auth/change-password`.
-
-## 8.8 Admin user management
-
-1. Show section only when `current_user.is_admin=true`.
-2. Use admin user endpoints for create/activate/deactivate/reset password.
-
-## 9. Development Method
-
-- API-contract-first frontend.
-- Vertical slices (complete flow before moving on).
-- Shared API client module:
-  - always send credentials
-  - inject CSRF header for unsafe methods
-  - normalize envelope parsing and error handling
-- Keep frontend modules separate (`auth`, `scholars`, `publications`, `runs`, `settings`, `admin`).
-- Write tests while implementing each slice.
-
-## 10. Definition of Done (UI Readiness)
-
-- All main flows above implemented and mapped to existing API endpoints.
-- Role-gated admin experience enforced in UI.
-- Distinct new-vs-read publication semantics visible.
-- Run/queue diagnostics visible and actionable.
-- Reliable loading/empty/error states with request-id surfaced for support.
-- No dependence on removed server-rendered UI layer.
-
-## 11. Open Items (Known Future Work)
-
-- API contract freeze/changelog discipline for external UI clients.
-- Additional API-first smoke coverage for end-to-end UI critical flows.
-- Optional background image enrichment/refresh jobs for already-tracked scholars.
-
-## 12. UI Rebuild Program Artifacts
-
-- Phase 0 through Phase 3 decisions are consolidated into this handbook and `README.md`.
-- Local scratch notes, probes, and temporary planning artifacts are intentionally not tracked in git.
-
-## 13. Frontend Scaffold Status (Implemented)
-
-- Frontend scaffold path: `frontend/`
-- Implemented core:
-  - App shell + routing + auth/role route guards
-  - Shared API client with envelope parsing, credentials, and CSRF header injection
-  - Pinia stores for auth, theme, and global UI errors
-  - Tailwind-first design foundation with light/dark themes and pre-paint theme initialization
-  - Page shells for login, dashboard, scholars, publications, runs, run detail, settings, and admin users
-  - Initial feature modules under `frontend/src/features/*` mapped to current API endpoints
-- Deployment boundary:
-  - No in-repo reverse-proxy implementation
-  - Frontend edge/proxy routing is managed externally (for example, Caddy)
-
-## 14. Frontend Quality Gates (Phase 3 Implemented)
-
-- CI now includes frontend quality gates in `.github/workflows/ci.yml`:
-  - API contract drift check (`scripts/check_frontend_api_contract.py`)
-  - frontend typecheck (`npm run typecheck`)
-  - frontend unit tests (`npm run test:run`)
-  - frontend build (`npm run build`)
-- Frontend test harness:
-  - `vitest` configured in `frontend/vitest.config.ts`
-  - baseline tests in `frontend/src/lib/api/envelope.test.ts` and `frontend/src/lib/api/errors.test.ts`

CONTRIBUTING.md

@@ -1,24 +0,0 @@
-# Contributing
-
-## Scope
-This project favors small, reviewable pull requests that keep runtime behavior clear and operationally safe.
-
-## Essential-File Policy
-Commit only source-of-truth files required to build, run, test, or document the app.
-
-Do not commit generated or local-only artifacts, including:
-- `__pycache__/`, `*.pyc`, `.pytest_cache/`, `.mypy_cache/`, `.ruff_cache/`
-- frontend build/install outputs like `frontend/dist/`, `frontend/node_modules/`, `frontend/.vite/`
-- coverage outputs (`.coverage`, `htmlcov/`)
-- packaging/build leftovers (`*.egg-info/`, `build/`, `dist/`)
-- local probe/scratch material (`planning/`)
-
-CI enforces this with `scripts/check_no_generated_artifacts.sh`.
-
-## Merge Checklist
-- [ ] Changes are minimal, purposeful, and remove obsolete/dead code in touched areas.
-- [ ] Backend tests pass (`uv run pytest tests/unit` and integration scope as needed).
-- [ ] Frontend checks pass (`npm run typecheck`, `npm run test:run`, `npm run build`).
-- [ ] API/behavior docs are updated when env vars, endpoints, or payloads change.
-- [ ] `README.md`, `.env.example`, and deployment notes stay aligned.
-- [ ] `scripts/check_no_generated_artifacts.sh` passes locally.

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 JustinZeus
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,268 +1,149 @@
-# scholarr
-
 <div align="center">
 
-Self-hosted scholar tracking with a single app image (API + frontend).
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset=".github/logo-dark.png" />
+  <source media="(prefers-color-scheme: light)" srcset="frontend/public/scholar_logo.png" />
+  <img src="frontend/public/scholar_logo.png" alt="Scholarr" width="120" />
+</picture>
 
-[![CI](https://img.shields.io/github/actions/workflow/status/justinzeus/scholarr/ci.yml?style=for-the-badge)](https://github.com/justinzeus/scholarr/actions/workflows/ci.yml)
+# Scholarr
+
+**Self-hosted academic publication tracker.**
+
+Track Google Scholar profiles, discover new papers automatically,
+resolve open-access PDFs, and stay on top of the literature you care about.
+
+[![CI](https://img.shields.io/github/actions/workflow/status/justinzeus/scholarr/ci.yml?style=for-the-badge)](https://github.com/JustinZeus/scholarr/actions/workflows/ci.yml)
+[![CodeQL](https://img.shields.io/github/actions/workflow/status/justinzeus/scholarr/codeql.yml?style=for-the-badge&label=CodeQL)](https://github.com/JustinZeus/scholarr/actions/workflows/codeql.yml)
+[![Release](https://img.shields.io/github/v/release/justinzeus/scholarr?style=for-the-badge)](https://github.com/JustinZeus/scholarr/releases)
 [![Docker Pulls](https://img.shields.io/docker/pulls/justinzeus/scholarr?style=for-the-badge&logo=docker)](https://hub.docker.com/r/justinzeus/scholarr)
-[![Docker Image](https://img.shields.io/badge/docker-justinzeus%2Fscholarr-2496ED?style=for-the-badge&logo=docker&logoColor=white)](https://hub.docker.com/r/justinzeus/scholarr)
+[![Python](https://img.shields.io/badge/python-3.12+-3776AB?style=for-the-badge&logo=python&logoColor=white)](https://www.python.org/)
+[![License](https://img.shields.io/github/license/justinzeus/scholarr?style=for-the-badge)](LICENSE)
+[![Docs](https://img.shields.io/badge/docs-scholarr-2e8555?style=for-the-badge)](https://justinzeus.github.io/scholarr/)
 
 </div>
 
+---
+
+<!-- TODO: Replace these placeholders with actual screenshots.
+     Drop PNG/JPEG files into docs/assets/ and update the paths below.
+
+     Recommended screenshots:
+       1. Dashboard / publications list (light or dark mode)
+       2. Scholar profile view
+       3. Run detail with progress
+
+     Ideal dimensions: 1200-1400px wide, PNG or WebP.
+-->
+
+<p align="center">
+  <img src="docs/assets/screenshot-dashboard.png" alt="Publications dashboard" width="720" />
+</p>
+
+<!--
+<p align="center">
+  <img src="docs/assets/screenshot-scholars.png" alt="Scholar profiles" width="360" />
+  <img src="docs/assets/screenshot-run.png" alt="Ingestion run" width="360" />
+</p>
+-->
+
+## Why Scholarr?
+
+Most researchers track new papers by manually checking Google Scholar, setting up email alerts, or juggling RSS feeds. Scholarr replaces all of that with a single self-hosted service:
+
+- **Add scholars once** -- by profile URL, Scholar ID, or name search
+- **Publications appear automatically** -- a background scheduler scrapes profiles on a configurable interval
+- **Open-access PDFs are resolved for you** -- Unpaywall and arXiv are queried automatically when a DOI is found
+- **Everything is deduplicated** -- publications are global records; no duplicates across scholars
+- **Your data stays yours** -- fully self-hosted, export/import your entire library at any time
+
+## Features
+
+| | |
+|---|---|
+| **Automated Ingestion** | Background scheduler with configurable intervals, continuation queue, and multi-page pagination |
+| **Identifier Resolution** | Cross-references arXiv, Crossref, and OpenAlex to gather DOIs, arXiv IDs, PMIDs |
+| **PDF Discovery** | Resolves open-access PDFs via Unpaywall API and arXiv, with automatic retry queue |
+| **Scrape Safety** | Rate limiting, cooldowns, and backoff strategies that prevent IP bans -- these are safety floors, not optional |
+| **Multi-User** | Session-based auth, admin user management, user-scoped scholar tracking |
+| **Theming** | 7 color presets with light/dark mode, tokenized component system |
+| **Import / Export** | Portable scholar data with full publication and read-state preservation |
+| **Single Container** | FastAPI backend + Vue 3 frontend ship as one Docker image |
+
 ## Quick Start
 
-1. Copy env template:
-
 ```bash
+# 1. Clone and configure
+git clone https://github.com/JustinZeus/scholarr.git
+cd scholarr
 cp .env.example .env
-```
 
-2. Set required values in `.env`:
-- `POSTGRES_PASSWORD`
-- `SESSION_SECRET_KEY`
+# 2. Set required secrets in .env
+#    POSTGRES_PASSWORD=<secure-password>
+#    SESSION_SECRET_KEY=<random-32-char-string>
 
-3. Choose a deploy method below.
-
-## Deploy Method A: Prebuilt Image (Recommended)
-
-Use this for normal self-hosted deployment.
-
-```bash
-docker compose pull
+# 3. Start
 docker compose up -d
+
+# 4. Open http://localhost:8000
 ```
 
-Open:
-- App/API: `http://localhost:8000`
-- Health: `http://localhost:8000/healthz`
-
-Upgrade:
+To bootstrap an admin account on first run, add to `.env`:
 
 ```bash
-docker compose pull
-docker compose up -d
+BOOTSTRAP_ADMIN_ON_START=1
+BOOTSTRAP_ADMIN_EMAIL=admin@example.com
+BOOTSTRAP_ADMIN_PASSWORD=<secure-password>
 ```
 
-## Deploy Method B: Local Source Build + Dev Frontend
+## How It Works
 
-Use this for development on this repository.
+```mermaid
+graph LR
+    UI[Vue 3 Dashboard] <-->|REST + SSE| API[FastAPI]
+    API --> Scheduler[Scheduler]
+    Scheduler -->|Scrape HTML| Scholar[Google Scholar]
+    Scholar -->|Parse & Deduplicate| DB[(PostgreSQL)]
+    Scholar -.->|Identify| Ext[arXiv / Crossref / OpenAlex]
+    Ext --> DB
+    DB -->|DOIs| PDF[PDF Resolution]
+    PDF -->|Unpaywall / arXiv| DB
+    API <--> DB
+```
+
+## Tech Stack
+
+| Layer | Technology |
+|-------|------------|
+| Backend | Python 3.12, FastAPI, SQLAlchemy 2.0 (async), Alembic |
+| Frontend | TypeScript, Vue 3, Vite, Tailwind CSS |
+| Database | PostgreSQL 15 |
+| Infrastructure | Multi-stage Docker, Docker Compose |
+
+## Documentation
+
+Full documentation: **[justinzeus.github.io/scholarr](https://justinzeus.github.io/scholarr/)**
+
+| Section | Covers |
+|---------|--------|
+| [User Guide](docs/user/overview.md) | Installation, configuration, all environment variables |
+| [Developer Guide](docs/developer/overview.md) | Architecture, local dev, contributing, testing |
+| [Operations](docs/operations/overview.md) | Deployment, database runbook, scrape safety |
+| [API Reference](docs/reference/api.md) | Envelope spec, all endpoints, DTO contracts |
+
+## Contributing
+
+Scholarr uses [conventional commits](https://www.conventionalcommits.org/) and [semantic versioning](https://semver.org/). See the [contributing guide](docs/developer/contributing.md) for PR process and code standards.
 
 ```bash
-docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
+# Dev environment
+docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build
+
+# Run tests (always in containers)
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app \
+  python -m pytest
 ```
 
-Open:
-- API (FastAPI): `http://localhost:8000`
-- Frontend dev server (Vite): `http://localhost:5173`
+## License
 
-Stop:
-
-```bash
-docker compose -f docker-compose.yml -f docker-compose.dev.yml down
-```
-
-## Essential Files
-
-- `docker-compose.yml`: deployment compose (prebuilt image).
-- `docker-compose.dev.yml`: dev override (source mounts + Vite dev server).
-- `.env.example`: full env variable template.
-- `Dockerfile`: multi-stage build (frontend + backend runtime).
-- `README.md`: deployment and operations reference.
-- `CONTRIBUTING.md`: contribution policy and merge checklist.
-- `docs/ops/scrape_safety_runbook.md`: scrape cooldown and blocked-IP operations guide.
-- `scripts/check_no_generated_artifacts.sh`: tracked-artifact guard used by CI.
-
-## Data Model Notes
-
-- Scholar tracking is user-scoped: each account can track the same Scholar ID independently.
-- Publications are shared/global records deduplicated by Scholar cluster ID and normalized fingerprint.
-- Per-account visibility and read state is stored on scholar-publication links, not on the global publication row.
-
-## Name Search Status
-
-- Scholar name search is intentionally WIP in the UI.
-- Current Google Scholar behavior often redirects name-search traffic to login/challenge flows, so production onboarding should use direct Scholar ID/profile URL adds.
-
-## Environment Variables (Complete Reference)
-
-Notes:
-- Boolean envs accept: `1/0`, `true/false`, `yes/no`, `on/off`.
-- Values shown are deployment defaults from `.env.example` and `docker-compose.yml`.
-- Some internal app fallbacks may differ for local test/dev safety when env vars are omitted.
-- `deploy` means used in regular deployment.
-- `dev` means used in local dev workflow.
-
-### Core Compose and Database
-
-| Variable | Default | Options | Scope | Description |
-| --- | --- | --- | --- | --- |
-| `POSTGRES_DB` | `scholar` | any valid DB name | deploy, dev | PostgreSQL database name. |
-| `POSTGRES_USER` | `scholar` | any valid DB user | deploy, dev | PostgreSQL user. |
-| `POSTGRES_PASSWORD` | `change-me` | strong password | deploy, dev | PostgreSQL password. Required. |
-| `DATABASE_URL` | `postgresql+asyncpg://scholar:scholar@db:5432/scholar` | valid SQLAlchemy asyncpg URL | deploy, dev, test | App database connection URL. |
-| `TEST_DATABASE_URL` | empty | valid SQLAlchemy asyncpg URL | test | Optional explicit integration test DB URL. |
-| `SCHOLARR_IMAGE` | `justinzeus/scholarr:latest` | any image ref | deploy | App image tag used by deployment compose. |
-
-### App Runtime and Networking
-
-| Variable | Default | Options | Scope | Description |
-| --- | --- | --- | --- | --- |
-| `APP_NAME` | `scholarr` | any string | deploy, dev | FastAPI app name/title. |
-| `APP_HOST` | `0.0.0.0` | bind host | deploy, dev | Uvicorn bind host inside container. |
-| `APP_PORT` | `8000` | integer | deploy, dev | Uvicorn bind port inside container. |
-| `APP_HOST_PORT` | `8000` | integer | deploy, dev | Host port mapped to app container port 8000. |
-| `APP_RELOAD` | `0` | boolean | deploy, dev | Enable uvicorn reload mode. Recommended `0` in deploy, `1` in dev. |
-| `MIGRATE_ON_START` | `1` | boolean | deploy, dev | Run Alembic migrations during app startup. |
-| `FRONTEND_ENABLED` | `1` | boolean | deploy, dev | Serve bundled frontend assets from FastAPI. |
-| `FRONTEND_DIST_DIR` | `/app/frontend/dist` | absolute path | deploy, dev | Path to built frontend assets inside app container. |
-
-### Dev Frontend Overrides
-
-| Variable | Default | Options | Scope | Description |
-| --- | --- | --- | --- | --- |
-| `FRONTEND_HOST_PORT` | `5173` | integer | dev | Host port mapped to Vite dev server. |
-| `CHOKIDAR_USEPOLLING` | `1` | boolean | dev | File watching mode for containerized Vite. |
-| `VITE_DEV_API_PROXY_TARGET` | `http://app:8000` | URL | dev | Vite proxy target for `/api` and `/healthz`. |
-
-### Auth and Session Security
-
-| Variable | Default | Options | Scope | Description |
-| --- | --- | --- | --- | --- |
-| `SESSION_SECRET_KEY` | `replace-with-a-long-random-secret-at-least-32-characters` | long random string | deploy, dev | Session signing secret. Required in deploy. |
-| `SESSION_COOKIE_SECURE` | `1` | boolean | deploy, dev | Mark session cookie as HTTPS-only. Set `1` behind HTTPS. |
-| `LOGIN_RATE_LIMIT_ATTEMPTS` | `5` | integer >= 1 | deploy, dev | Login attempts allowed per window. |
-| `LOGIN_RATE_LIMIT_WINDOW_SECONDS` | `60` | integer >= 1 | deploy, dev | Login rate limit window in seconds. |
-
-### HTTP Security Headers and CSP
-
-| Variable | Default | Options | Scope | Description |
-| --- | --- | --- | --- | --- |
-| `SECURITY_HEADERS_ENABLED` | `1` | boolean | deploy, dev | Global switch for response security headers middleware. |
-| `SECURITY_X_CONTENT_TYPE_OPTIONS` | `nosniff` | header value | deploy, dev | `X-Content-Type-Options` response header value. |
-| `SECURITY_X_FRAME_OPTIONS` | `DENY` | header value | deploy, dev | `X-Frame-Options` response header value. |
-| `SECURITY_REFERRER_POLICY` | `strict-origin-when-cross-origin` | header value | deploy, dev | `Referrer-Policy` response header value. |
-| `SECURITY_PERMISSIONS_POLICY` | `accelerometer=(), autoplay=(), camera=(), display-capture=(), geolocation=(), gyroscope=(), microphone=(), payment=(), usb=()` | permissions policy string | deploy, dev | `Permissions-Policy` response header value. |
-| `SECURITY_CROSS_ORIGIN_OPENER_POLICY` | `same-origin` | header value | deploy, dev | `Cross-Origin-Opener-Policy` response header value. |
-| `SECURITY_CROSS_ORIGIN_RESOURCE_POLICY` | `same-origin` | header value | deploy, dev | `Cross-Origin-Resource-Policy` response header value. |
-| `SECURITY_CSP_ENABLED` | `1` | boolean | deploy, dev | Enable Content Security Policy headers. |
-| `SECURITY_CSP_POLICY` | strict SPA/API default | CSP policy string | deploy, dev | CSP applied to app/API routes (excluding docs paths). |
-| `SECURITY_CSP_DOCS_POLICY` | relaxed docs default | CSP policy string | deploy, dev | CSP override for `/docs` and `/redoc` to keep Swagger/ReDoc usable. |
-| `SECURITY_CSP_REPORT_ONLY` | `0` | boolean | deploy, dev | Emit CSP in report-only mode instead of enforcement mode. |
-| `SECURITY_STRICT_TRANSPORT_SECURITY_ENABLED` | `0` | boolean | deploy, dev | Enable `Strict-Transport-Security` header. |
-| `SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE` | `31536000` | integer >= 0 | deploy, dev | `max-age` for HSTS header. |
-| `SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS` | `1` | boolean | deploy, dev | Add `includeSubDomains` directive to HSTS header. |
-| `SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD` | `0` | boolean | deploy, dev | Add `preload` directive to HSTS header. |
-
-### Logging
-
-| Variable | Default | Options | Scope | Description |
-| --- | --- | --- | --- | --- |
-| `LOG_LEVEL` | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` | deploy, dev | Application log level. |
-| `LOG_FORMAT` | `console` | `console`, `json` | deploy, dev | Log output format. |
-| `LOG_REQUESTS` | `1` | boolean | deploy, dev | Enable request start/completion logs. |
-| `LOG_UVICORN_ACCESS` | `0` | boolean | deploy, dev | Enable uvicorn access logs. |
-| `LOG_REQUEST_SKIP_PATHS` | `/healthz` | comma-separated path prefixes | deploy, dev | Request log skip list. |
-| `LOG_REDACT_FIELDS` | empty | comma-separated keys | deploy, dev | Extra fields to redact in structured logs. |
-
-### Scheduler and Ingestion
-
-| Variable | Default | Options | Scope | Description |
-| --- | --- | --- | --- | --- |
-| `SCHEDULER_ENABLED` | `1` | boolean | deploy, dev | Enable background scheduler loop. |
-| `SCHEDULER_TICK_SECONDS` | `60` | integer >= 1 | deploy, dev | Scheduler interval in seconds. |
-| `INGESTION_AUTOMATION_ALLOWED` | `1` | boolean | deploy, dev | Global safety switch for scheduled/automatic checks. When disabled, auto-run settings are forced off. |
-| `INGESTION_MANUAL_RUN_ALLOWED` | `1` | boolean | deploy, dev | Global safety switch for manual checks from API/UI. |
-| `INGESTION_MIN_RUN_INTERVAL_MINUTES` | `15` | integer >= 15 | deploy, dev | Server-enforced minimum for user-configured automatic check interval. |
-| `INGESTION_MIN_REQUEST_DELAY_SECONDS` | `2` | integer >= 2 | deploy, dev | Server-enforced minimum delay between scholar requests. |
-| `SCHEDULER_QUEUE_BATCH_SIZE` | `10` | integer >= 1 | deploy, dev | Queue items processed per scheduler tick. |
-| `INGESTION_NETWORK_ERROR_RETRIES` | `1` | integer >= 0 | deploy, dev | Retries for transient ingestion network errors. |
-| `INGESTION_RETRY_BACKOFF_SECONDS` | `1.0` | float >= 0 | deploy, dev | Backoff delay for retry attempts. |
-| `INGESTION_MAX_PAGES_PER_SCHOLAR` | `30` | integer >= 1 | deploy, dev | Upper bound of pages fetched per scholar run. |
-| `INGESTION_PAGE_SIZE` | `100` | integer >= 1 | deploy, dev | Requested Scholar page size. |
-| `INGESTION_ALERT_BLOCKED_FAILURE_THRESHOLD` | `1` | integer >= 1 | deploy, dev | Trigger blocked/captcha scrape alert flag when this many blocked failures occur in a run. |
-| `INGESTION_ALERT_NETWORK_FAILURE_THRESHOLD` | `2` | integer >= 1 | deploy, dev | Trigger network scrape alert flag when this many network failures occur in a run. |
-| `INGESTION_ALERT_RETRY_SCHEDULED_THRESHOLD` | `3` | integer >= 1 | deploy, dev | Trigger retry alert flag when scheduled retry count reaches this threshold in a run. |
-| `INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS` | `1800` | integer >= 60 | deploy, dev | Cooldown duration applied when blocked-failure threshold is exceeded. |
-| `INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS` | `900` | integer >= 60 | deploy, dev | Cooldown duration applied when network-failure threshold is exceeded. |
-| `INGESTION_CONTINUATION_QUEUE_ENABLED` | `1` | boolean | deploy, dev | Enable continuation queue for long runs. |
-| `INGESTION_CONTINUATION_BASE_DELAY_SECONDS` | `120` | integer >= 0 | deploy, dev | Initial delay before retrying continuation queue items. |
-| `INGESTION_CONTINUATION_MAX_DELAY_SECONDS` | `3600` | integer >= 0 | deploy, dev | Maximum continuation retry delay. |
-| `INGESTION_CONTINUATION_MAX_ATTEMPTS` | `6` | integer >= 1 | deploy, dev | Max failed continuation attempts before dropping item. |
-
-### Scholar Images and Name Search Safety
-
-| Variable | Default | Options | Scope | Description |
-| --- | --- | --- | --- | --- |
-| `SCHOLAR_IMAGE_UPLOAD_DIR` | `/var/lib/scholarr/uploads` | writable absolute path | deploy, dev | Storage path for uploaded scholar images in compose deployments. App fallback without env is `/tmp/scholarr_uploads/scholar_images`. |
-| `SCHOLAR_IMAGE_UPLOAD_MAX_BYTES` | `2000000` | integer >= 1 | deploy, dev | Max uploaded image size in bytes. |
-| `SCHOLAR_NAME_SEARCH_ENABLED` | `1` | boolean | deploy, dev | Enable name-search helper endpoint. |
-| `SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS` | `21600` | integer >= 1 | deploy, dev | Cache TTL for successful name-search responses. |
-| `SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS` | `300` | integer >= 1 | deploy, dev | Cache TTL for blocked/captcha responses. |
-| `SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES` | `512` | integer >= 1 | deploy, dev | Max cached search entries. |
-| `SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS` | `8.0` | float >= 0 | deploy, dev | Minimum interval between live name-search requests. |
-| `SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS` | `2.0` | float >= 0 | deploy, dev | Added jitter to reduce request burst patterns. |
-| `SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD` | `1` | integer >= 1 | deploy, dev | Consecutive blocked responses before cooldown starts. |
-| `SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS` | `1800` | integer >= 1 | deploy, dev | Cooldown duration after repeated blocked responses. |
-| `SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD` | `2` | integer >= 1 | deploy, dev | Emit retry-threshold observability warning when name-search retry count reaches this value. |
-| `SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD` | `3` | integer >= 1 | deploy, dev | Emit cooldown-threshold observability alert after this many requests are rejected during active cooldown. |
-
-### Scrape Safety Operations
-
-- Structured safety events are emitted for:
-  - policy/manual run blocks,
-  - cooldown entered/cleared transitions,
-  - blocked/network/retry threshold trips,
-  - scheduler cooldown skips/deferments.
-- Use `LOG_FORMAT=json` and ship logs to your preferred collector for alerting.
-- Runbook: `docs/ops/scrape_safety_runbook.md`.
-
-### Startup Bootstrap and DB Wait
-
-| Variable | Default | Options | Scope | Description |
-| --- | --- | --- | --- | --- |
-| `BOOTSTRAP_ADMIN_ON_START` | `0` | boolean | deploy, dev | Auto-create admin at startup. |
-| `BOOTSTRAP_ADMIN_EMAIL` | empty | valid email | deploy, dev | Bootstrap admin email. |
-| `BOOTSTRAP_ADMIN_PASSWORD` | empty | strong password | deploy, dev | Bootstrap admin password. |
-| `BOOTSTRAP_ADMIN_FORCE_PASSWORD` | `0` | boolean | deploy, dev | Force-reset bootstrap admin password if user already exists. |
-| `DB_WAIT_TIMEOUT_SECONDS` | `60` | integer >= 1 | deploy, dev | Max seconds to wait for DB readiness at startup. |
-| `DB_WAIT_INTERVAL_SECONDS` | `2` | integer >= 1 | deploy, dev | Interval between DB readiness checks. |
-
-## Quality and Test Commands
-
-Backend:
-
-```bash
-docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest tests/unit
-docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest -m integration
-```
-
-Frontend:
-
-```bash
-cd frontend
-npm install
-npm run typecheck
-npm run test:run
-npm run build
-```
-
-Contract drift:
-
-```bash
-python3 scripts/check_frontend_api_contract.py
-```
-
-Repository hygiene:
-
-```bash
-./scripts/check_no_generated_artifacts.sh
-```
-
-Scheduled fixture probes run in GitHub Actions via `.github/workflows/scheduled-probes.yml`.
-
-## API Contract
-
-- Base path: `/api/v1`
-- Success envelope: `{"data": ..., "meta": {"request_id": "..."}}`
-- Error envelope: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}`
+See [LICENSE](LICENSE) for details.

agents.md

@@ -0,0 +1,123 @@
+# AI Agent Instructions: Scholarr
+
+Adhere strictly to these constraints when working on this codebase.
+
+## 1. Code Quality
+
+- **Function length:** 50 lines max. Extract helpers ruthlessly.
+- **File length:** 400 lines target, 600 lines hard ceiling. Files above this must be decomposed before adding more code.
+- **DRY:** Abstract repeated logic immediately. No duplicate boilerplate for queries, responses, or error handling.
+- **Negative space programming:** Fail fast with explicit assertions and guard clauses. No silent failures, especially in DOM parsing.
+- **Cyclomatic complexity:** Flatten with early returns. No deep nesting. No magic numbers.
+- **No dead code:** Do not leave commented-out code, unused imports, or backward-compatibility shims. Delete cleanly.
+- **Variable naming:** Any variable outside of a lambda functions or iterators should be descriptive. Code should be readable for maintainers.
+
+## 2. Architecture
+
+### Data Model
+
+- Scholar tracking is **user-scoped**. Never assume global links between users and Scholar IDs.
+- Publications are **global, deduplicated records**. Deduplicate via cluster ID and normalized fingerprinting.
+- Read/unread, favorites, and visibility state live on the **scholar-publication link**, not the publication.
+
+### Service Boundaries
+
+All business logic lives in `app/services/<domain>/`. No flat files in `app/services/` root. Each domain owns its application service, types, and helpers.
+
+### API Envelope
+
+All `/api/v1` responses use this exact envelope:
+
+```
+Success: {"data": ..., "meta": {"request_id": "..."}}
+Error:   {"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}
+```
+
+Use the Pydantic envelope schemas in `app/api/schemas.py`. Do not construct raw dicts.
+
+## 3. Scrape Safety (Immutable)
+
+These constraints prevent IP bans. They are not tunable to zero and must not be optimized away.
+
+- Enforce `INGESTION_MIN_REQUEST_DELAY_SECONDS` (default 2s) between all external requests.
+- Default to direct ID or profile URL ingestion. Name searches trigger CAPTCHAs.
+- Respect `INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS` (1800s) and `INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS` (900s) upon threshold breaches.
+
+## 4. Logging
+
+Use `structured_log()` from `app/logging_utils.py` for all domain logging. Do not use raw `logger.info()` / `logger.warning()` calls.
+
+```python
+from app.logging_utils import structured_log
+
+structured_log(logger, "info", "ingestion.run_started", user_id=user_id, scholar_count=count)
+```
+
+Every event name should be dot-namespaced to its domain (e.g., `arxiv.cache_hit`, `ingestion.safety_cooldown_entered`).
+
+## 5. Stack & Tooling
+
+- **Backend:** Python 3.12+, FastAPI, SQLAlchemy 2.0 (async/asyncpg), Alembic
+- **Frontend:** TypeScript, Vue 3, Vite, Tailwind CSS
+- **Infrastructure:** Multi-stage Docker, Docker Compose
+- **Package manager:** `uv` (used in Dockerfile and CI; `uv run` prefix for all commands)
+- **Linting:** `ruff check .` and `ruff format --check .` (config in `pyproject.toml`)
+- **Type checking:** `mypy app/`
+- **Versioning:** python-semantic-release with conventional commits
+
+## 6. Commits
+
+Follow [Conventional Commits](https://www.conventionalcommits.org/):
+
+```
+<type>(<scope>): <description>
+```
+
+Types: `feat`, `fix`, `docs`, `ci`, `refactor`, `test`, `chore`, `perf`.
+
+## 7. Testing & Quality Gates
+
+All **local development** commands run inside containers. Do not run bare `npm`, `pytest`, or `ruff` on the host. CI workflows (`.github/workflows/`) use bare runners with `uv` and `npm` directly — this is intentional since CI has no Docker-in-Docker dependency.
+
+### Backend
+
+```bash
+# Unit tests (default, excludes integration)
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app python -m pytest
+
+# Integration tests
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app python -m pytest -m integration
+
+# Linting
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app ruff check .
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app ruff format --check .
+
+# Type checking
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app mypy app/
+```
+
+Pytest markers: `integration`, `db`, `migrations`, `schema`, `smoke`.
+
+### Frontend
+
+```bash
+# Type checking
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run typecheck
+
+# Unit tests
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run test:run
+
+# Lint
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run lint
+
+# Production build
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run build
+```
+
+All four gates must pass before a PR is considered ready.
+
+## 8. Frontend
+
+- Use the tokenized theme system (`frontend/src/theme/presets/`). Do not hardcode colors.
+- Integrate Tailwind with preset theme tokens. Reference `frontend/scripts/check_theme_tokens.mjs` for enforcement.
+- Every UI element must have a clear purpose. Clarity through styling and language.

alembic/env.py

@@ -1,13 +1,13 @@
-from logging.config import fileConfig
 import os
+from logging.config import fileConfig
 
-from alembic import context
 from sqlalchemy import pool
 from sqlalchemy.engine import Connection
 from sqlalchemy.ext.asyncio import async_engine_from_config
 
-from app.db.base import metadata
+from alembic import context
 from app.db import models  # noqa: F401
+from app.db.base import metadata
 
 config = context.config
 

alembic/versions/20260216_0001_multi_user_schema.py

@@ -8,10 +8,11 @@ Create Date: 2026-02-16 17:10:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision: str = "20260216_0001"
 down_revision: str | Sequence[str] | None = None

alembic/versions/20260216_0002_add_user_admin_flag.py

@@ -7,9 +7,10 @@ Create Date: 2026-02-16 17:40:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision: str = "20260216_0002"
 down_revision: str | Sequence[str] | None = "20260216_0001"
@@ -31,4 +32,3 @@ def upgrade() -> None:
 
 def downgrade() -> None:
     op.drop_column("users", "is_admin")
-

alembic/versions/20260216_0003_add_ingestion_queue.py

@@ -7,9 +7,10 @@ Create Date: 2026-02-16 23:45:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision: str = "20260216_0003"
 down_revision: str | Sequence[str] | None = "20260216_0002"

alembic/versions/20260217_0004_queue_status_fields.py

@@ -7,9 +7,10 @@ Create Date: 2026-02-17 10:15:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision: str = "20260217_0004"
 down_revision: str | Sequence[str] | None = "20260216_0003"
@@ -22,9 +23,7 @@ def upgrade() -> None:
     inspector = sa.inspect(bind)
     columns = {column["name"] for column in inspector.get_columns("ingestion_queue_items")}
     indexes = {index["name"] for index in inspector.get_indexes("ingestion_queue_items")}
-    checks = {
-        constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items")
-    }
+    checks = {constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items")}
 
     if "status" not in columns:
         op.add_column(
@@ -80,9 +79,7 @@ def downgrade() -> None:
     inspector = sa.inspect(bind)
     columns = {column["name"] for column in inspector.get_columns("ingestion_queue_items")}
     indexes = {index["name"] for index in inspector.get_indexes("ingestion_queue_items")}
-    checks = {
-        constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items")
-    }
+    checks = {constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items")}
 
     if "ix_ingestion_queue_status_next_attempt" in indexes:
         op.drop_index("ix_ingestion_queue_status_next_attempt", table_name="ingestion_queue_items")

alembic/versions/20260217_0005_manual_run_idempotency.py

@@ -7,9 +7,10 @@ Create Date: 2026-02-17 16:20:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision: str = "20260217_0005"
 down_revision: str | Sequence[str] | None = "20260217_0004"
@@ -70,9 +71,7 @@ def upgrade() -> None:
             "crawl_runs",
             ["user_id", "idempotency_key"],
             unique=True,
-            postgresql_where=sa.text(
-                "idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"
-            ),
+            postgresql_where=sa.text("idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"),
         )
 
 

alembic/versions/20260217_0006_scholar_profile_images.py

@@ -7,9 +7,10 @@ Create Date: 2026-02-17 18:30:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision: str = "20260217_0006"
 down_revision: str | Sequence[str] | None = "20260217_0005"

alembic/versions/20260217_0007_scholar_initial_page_fingerprint.py

@@ -7,9 +7,10 @@ Create Date: 2026-02-17 18:55:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision: str = "20260217_0007"
 down_revision: str | Sequence[str] | None = "20260217_0006"

alembic/versions/20260219_0008_user_settings_nav_visible_pages.py

@@ -7,10 +7,10 @@ Create Date: 2026-02-19 14:58:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "20260219_0008"
@@ -21,7 +21,7 @@ depends_on: str | Sequence[str] | None = None
 
 TABLE_NAME = "user_settings"
 DEFAULT_NAV_VISIBLE_PAGES_SQL = (
-    "'[\"dashboard\",\"scholars\",\"publications\",\"settings\",\"style-guide\",\"runs\",\"users\"]'::jsonb"
+    '\'["dashboard","scholars","publications","settings","style-guide","runs","users"]\'::jsonb'
 )
 
 

alembic/versions/20260219_0009_user_settings_scrape_safety_state.py

@@ -7,10 +7,10 @@ Create Date: 2026-02-19 21:20:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "20260219_0009"

alembic/versions/20260219_0010_author_search_shared_state.py

@@ -0,0 +1,115 @@
+"""Add shared author-search runtime and cache tables.
+
+Revision ID: 20260219_0010
+Revises: 20260219_0009
+Create Date: 2026-02-19 22:20:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260219_0010"
+down_revision: str | Sequence[str] | None = "20260219_0009"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    table_names = set(inspector.get_table_names())
+
+    if "author_search_runtime_state" not in table_names:
+        op.create_table(
+            "author_search_runtime_state",
+            sa.Column("state_key", sa.String(length=64), nullable=False),
+            sa.Column("last_live_request_at", sa.DateTime(timezone=True), nullable=True),
+            sa.Column("cooldown_until", sa.DateTime(timezone=True), nullable=True),
+            sa.Column(
+                "consecutive_blocked_count",
+                sa.Integer(),
+                nullable=False,
+                server_default=sa.text("0"),
+            ),
+            sa.Column(
+                "cooldown_rejection_count",
+                sa.Integer(),
+                nullable=False,
+                server_default=sa.text("0"),
+            ),
+            sa.Column(
+                "cooldown_alert_emitted",
+                sa.Boolean(),
+                nullable=False,
+                server_default=sa.text("false"),
+            ),
+            sa.Column(
+                "created_at",
+                sa.DateTime(timezone=True),
+                nullable=False,
+                server_default=sa.func.now(),
+            ),
+            sa.Column(
+                "updated_at",
+                sa.DateTime(timezone=True),
+                nullable=False,
+                server_default=sa.func.now(),
+            ),
+            sa.PrimaryKeyConstraint("state_key", name=op.f("pk_author_search_runtime_state")),
+        )
+
+    if "author_search_cache_entries" not in table_names:
+        op.create_table(
+            "author_search_cache_entries",
+            sa.Column("query_key", sa.String(length=256), nullable=False),
+            sa.Column(
+                "payload",
+                postgresql.JSONB(astext_type=sa.Text()),
+                nullable=False,
+            ),
+            sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
+            sa.Column(
+                "cached_at",
+                sa.DateTime(timezone=True),
+                nullable=False,
+                server_default=sa.func.now(),
+            ),
+            sa.Column(
+                "updated_at",
+                sa.DateTime(timezone=True),
+                nullable=False,
+                server_default=sa.func.now(),
+            ),
+            sa.PrimaryKeyConstraint("query_key", name=op.f("pk_author_search_cache_entries")),
+        )
+        op.create_index(
+            "ix_author_search_cache_expires_at",
+            "author_search_cache_entries",
+            ["expires_at"],
+            unique=False,
+        )
+        op.create_index(
+            "ix_author_search_cache_cached_at",
+            "author_search_cache_entries",
+            ["cached_at"],
+            unique=False,
+        )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    table_names = set(inspector.get_table_names())
+
+    if "author_search_cache_entries" in table_names:
+        op.drop_index("ix_author_search_cache_cached_at", table_name="author_search_cache_entries")
+        op.drop_index("ix_author_search_cache_expires_at", table_name="author_search_cache_entries")
+        op.drop_table("author_search_cache_entries")
+
+    if "author_search_runtime_state" in table_names:
+        op.drop_table("author_search_runtime_state")

alembic/versions/20260220_0011_add_publication_doi.py

@@ -0,0 +1,50 @@
+"""Add DOI column to publications.
+
+Revision ID: 20260220_0011
+Revises: 20260219_0010
+Create Date: 2026-02-20 13:35:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision: str = "20260220_0011"
+down_revision: str | Sequence[str] | None = "20260219_0010"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _has_column(inspector: sa.Inspector, table_name: str, column_name: str) -> bool:
+    for column in inspector.get_columns(table_name):
+        if str(column.get("name")) == column_name:
+            return True
+    return False
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    table_names = set(inspector.get_table_names())
+    if "publications" not in table_names:
+        return
+    if not _has_column(inspector, "publications", "doi"):
+        op.add_column("publications", sa.Column("doi", sa.String(length=255), nullable=True))
+    index_names = {index["name"] for index in inspector.get_indexes("publications")}
+    if "ix_publications_doi" not in index_names:
+        op.create_index("ix_publications_doi", "publications", ["doi"], unique=False)
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    table_names = set(inspector.get_table_names())
+    if "publications" not in table_names:
+        return
+    index_names = {index["name"] for index in inspector.get_indexes("publications")}
+    if "ix_publications_doi" in index_names:
+        op.drop_index("ix_publications_doi", table_name="publications")
+    if _has_column(inspector, "publications", "doi"):
+        op.drop_column("publications", "doi")

alembic/versions/20260220_0012_add_data_repair_jobs.py

@@ -0,0 +1,106 @@
+"""Add data repair job audit table.
+
+Revision ID: 20260220_0012
+Revises: 20260220_0011
+Create Date: 2026-02-20 23:35:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+from alembic import op
+
+revision: str = "20260220_0012"
+down_revision: str | Sequence[str] | None = "20260220_0011"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _table_names() -> set[str]:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    return set(inspector.get_table_names())
+
+
+def _create_data_repair_jobs_table() -> None:
+    op.create_table(
+        "data_repair_jobs",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("job_name", sa.String(length=64), nullable=False),
+        sa.Column("requested_by", sa.String(length=255), nullable=True),
+        sa.Column(
+            "scope",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=False,
+            server_default=sa.text("'{}'::jsonb"),
+        ),
+        sa.Column("dry_run", sa.Boolean(), nullable=False, server_default=sa.text("true")),
+        sa.Column(
+            "status",
+            sa.String(length=16),
+            nullable=False,
+            server_default=sa.text("'planned'"),
+        ),
+        sa.Column(
+            "summary",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=False,
+            server_default=sa.text("'{}'::jsonb"),
+        ),
+        sa.Column("error_text", sa.Text(), nullable=True),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.CheckConstraint(
+            "status IN ('planned', 'running', 'completed', 'failed')",
+            name="data_repair_jobs_status_valid",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_data_repair_jobs")),
+    )
+
+
+def _create_data_repair_jobs_indexes() -> None:
+    op.create_index(
+        "ix_data_repair_jobs_created_at",
+        "data_repair_jobs",
+        ["created_at"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_data_repair_jobs_job_name_created_at",
+        "data_repair_jobs",
+        ["job_name", "created_at"],
+        unique=False,
+    )
+
+
+def _drop_data_repair_jobs_indexes() -> None:
+    op.drop_index("ix_data_repair_jobs_job_name_created_at", table_name="data_repair_jobs")
+    op.drop_index("ix_data_repair_jobs_created_at", table_name="data_repair_jobs")
+
+
+def upgrade() -> None:
+    if "data_repair_jobs" in _table_names():
+        return
+    _create_data_repair_jobs_table()
+    _create_data_repair_jobs_indexes()
+
+
+def downgrade() -> None:
+    if "data_repair_jobs" not in _table_names():
+        return
+    _drop_data_repair_jobs_indexes()
+    op.drop_table("data_repair_jobs")

alembic/versions/20260221_0013_add_publication_pdf_jobs.py

@@ -0,0 +1,179 @@
+"""Add persistent publication PDF job tracking tables.
+
+Revision ID: 20260221_0013
+Revises: 20260220_0012
+Create Date: 2026-02-21 12:25:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision: str = "20260221_0013"
+down_revision: str | Sequence[str] | None = "20260220_0012"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _table_names() -> set[str]:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    return set(inspector.get_table_names())
+
+
+def _create_publication_pdf_jobs_table() -> None:
+    op.create_table(
+        "publication_pdf_jobs",
+        sa.Column("publication_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "status",
+            sa.String(length=16),
+            nullable=False,
+            server_default=sa.text("'queued'"),
+        ),
+        sa.Column("attempt_count", sa.Integer(), nullable=False, server_default=sa.text("0")),
+        sa.Column("queued_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_failure_reason", sa.String(length=64), nullable=True),
+        sa.Column("last_failure_detail", sa.Text(), nullable=True),
+        sa.Column("last_source", sa.String(length=32), nullable=True),
+        sa.Column("last_requested_by_user_id", sa.Integer(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.CheckConstraint(
+            "status IN ('queued', 'running', 'resolved', 'failed')",
+            name="publication_pdf_jobs_status_valid",
+        ),
+        sa.ForeignKeyConstraint(
+            ["last_requested_by_user_id"],
+            ["users.id"],
+            ondelete="SET NULL",
+        ),
+        sa.ForeignKeyConstraint(
+            ["publication_id"],
+            ["publications.id"],
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("publication_id", name=op.f("pk_publication_pdf_jobs")),
+    )
+
+
+def _create_publication_pdf_jobs_indexes() -> None:
+    op.create_index(
+        "ix_publication_pdf_jobs_status",
+        "publication_pdf_jobs",
+        ["status"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_publication_pdf_jobs_updated_at",
+        "publication_pdf_jobs",
+        ["updated_at"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_publication_pdf_jobs_queued_at",
+        "publication_pdf_jobs",
+        ["queued_at"],
+        unique=False,
+    )
+
+
+def _create_publication_pdf_job_events_table() -> None:
+    op.create_table(
+        "publication_pdf_job_events",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("publication_id", sa.Integer(), nullable=False),
+        sa.Column("user_id", sa.Integer(), nullable=True),
+        sa.Column("event_type", sa.String(length=32), nullable=False),
+        sa.Column("status", sa.String(length=16), nullable=True),
+        sa.Column("source", sa.String(length=32), nullable=True),
+        sa.Column("failure_reason", sa.String(length=64), nullable=True),
+        sa.Column("message", sa.Text(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.ForeignKeyConstraint(
+            ["publication_id"],
+            ["publications.id"],
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["users.id"],
+            ondelete="SET NULL",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_publication_pdf_job_events")),
+    )
+
+
+def _create_publication_pdf_job_events_indexes() -> None:
+    op.create_index(
+        "ix_publication_pdf_job_events_publication_created",
+        "publication_pdf_job_events",
+        ["publication_id", "created_at"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_publication_pdf_job_events_created_at",
+        "publication_pdf_job_events",
+        ["created_at"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_publication_pdf_job_events_event_type",
+        "publication_pdf_job_events",
+        ["event_type"],
+        unique=False,
+    )
+
+
+def _drop_publication_pdf_jobs_indexes() -> None:
+    op.drop_index("ix_publication_pdf_jobs_queued_at", table_name="publication_pdf_jobs")
+    op.drop_index("ix_publication_pdf_jobs_updated_at", table_name="publication_pdf_jobs")
+    op.drop_index("ix_publication_pdf_jobs_status", table_name="publication_pdf_jobs")
+
+
+def _drop_publication_pdf_job_events_indexes() -> None:
+    op.drop_index("ix_publication_pdf_job_events_event_type", table_name="publication_pdf_job_events")
+    op.drop_index("ix_publication_pdf_job_events_created_at", table_name="publication_pdf_job_events")
+    op.drop_index(
+        "ix_publication_pdf_job_events_publication_created",
+        table_name="publication_pdf_job_events",
+    )
+
+
+def upgrade() -> None:
+    tables = _table_names()
+    if "publication_pdf_jobs" not in tables:
+        _create_publication_pdf_jobs_table()
+        _create_publication_pdf_jobs_indexes()
+    if "publication_pdf_job_events" not in tables:
+        _create_publication_pdf_job_events_table()
+        _create_publication_pdf_job_events_indexes()
+
+
+def downgrade() -> None:
+    tables = _table_names()
+    if "publication_pdf_job_events" in tables:
+        _drop_publication_pdf_job_events_indexes()
+        op.drop_table("publication_pdf_job_events")
+    if "publication_pdf_jobs" in tables:
+        _drop_publication_pdf_jobs_indexes()
+        op.drop_table("publication_pdf_jobs")

alembic/versions/20260221_0014_add_scholar_publication_favorite_state.py

@@ -0,0 +1,44 @@
+"""add scholar publication favorite state
+
+Revision ID: 20260221_0014
+Revises: 20260221_0013
+Create Date: 2026-02-21 14:35:00
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "20260221_0014"
+down_revision = "20260221_0013"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "scholar_publications",
+        sa.Column(
+            "is_favorite",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.text("false"),
+        ),
+    )
+    op.create_index(
+        "ix_scholar_publications_is_favorite",
+        "scholar_publications",
+        ["is_favorite"],
+        unique=False,
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_scholar_publications_is_favorite",
+        table_name="scholar_publications",
+    )
+    op.drop_column("scholar_publications", "is_favorite")

alembic/versions/20260221_0015_enforce_request_delay_floor.py

@@ -0,0 +1,71 @@
+"""enforce request delay floor at two seconds
+
+Revision ID: 20260221_0015
+Revises: 20260221_0014
+Create Date: 2026-02-21 18:10:00
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260221_0015"
+down_revision: str | Sequence[str] | None = "20260221_0014"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+TABLE_NAME = "user_settings"
+CHECK_NAME_MIN_1 = "request_delay_seconds_min_1"
+CHECK_NAME_MIN_2 = "request_delay_seconds_min_2"
+LEGACY_CHECK_NAME_MIN_1 = "ck_user_settings_request_delay_seconds_min_1"
+LEGACY_CHECK_NAME_MIN_2 = "ck_user_settings_request_delay_seconds_min_2"
+
+
+def _check_constraints() -> set[str]:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    return {str(item.get("name")) for item in inspector.get_check_constraints(TABLE_NAME) if item.get("name")}
+
+
+def _drop_check_if_exists(name: str) -> None:
+    if name not in _check_constraints():
+        return
+    op.drop_constraint(
+        name,
+        TABLE_NAME,
+        type_="check",
+    )
+
+
+def upgrade() -> None:
+    op.execute(sa.text("UPDATE user_settings SET request_delay_seconds = 2 WHERE request_delay_seconds < 2"))
+
+    _drop_check_if_exists(CHECK_NAME_MIN_1)
+    _drop_check_if_exists(LEGACY_CHECK_NAME_MIN_1)
+
+    existing = _check_constraints()
+    if CHECK_NAME_MIN_2 not in existing and LEGACY_CHECK_NAME_MIN_2 not in existing:
+        op.create_check_constraint(
+            CHECK_NAME_MIN_2,
+            TABLE_NAME,
+            "request_delay_seconds >= 2",
+        )
+
+
+def downgrade() -> None:
+    _drop_check_if_exists(CHECK_NAME_MIN_2)
+    _drop_check_if_exists(LEGACY_CHECK_NAME_MIN_2)
+
+    existing = _check_constraints()
+    if CHECK_NAME_MIN_1 not in existing and LEGACY_CHECK_NAME_MIN_1 not in existing:
+        op.create_check_constraint(
+            CHECK_NAME_MIN_1,
+            TABLE_NAME,
+            "request_delay_seconds >= 1",
+        )

alembic/versions/20260222_0016_add_publication_identifiers.py

@@ -0,0 +1,104 @@
+"""Add publication identifiers table.
+
+Revision ID: 20260222_0016
+Revises: 20260221_0015
+Create Date: 2026-02-22 12:45:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision: str = "20260222_0016"
+down_revision: str | Sequence[str] | None = "20260221_0015"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _table_names() -> set[str]:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    return set(inspector.get_table_names())
+
+
+def _create_publication_identifiers_table() -> None:
+    op.create_table(
+        "publication_identifiers",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("publication_id", sa.Integer(), nullable=False),
+        sa.Column("kind", sa.String(length=32), nullable=False),
+        sa.Column("value_raw", sa.Text(), nullable=False),
+        sa.Column("value_normalized", sa.String(length=255), nullable=False),
+        sa.Column("source", sa.String(length=64), nullable=False),
+        sa.Column(
+            "confidence_score",
+            sa.Float(),
+            nullable=False,
+            server_default=sa.text("0"),
+        ),
+        sa.Column("evidence_url", sa.Text(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.CheckConstraint(
+            "confidence_score >= 0 AND confidence_score <= 1",
+            name="publication_identifiers_confidence_score_range",
+        ),
+        sa.ForeignKeyConstraint(
+            ["publication_id"],
+            ["publications.id"],
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_publication_identifiers")),
+        sa.UniqueConstraint(
+            "publication_id",
+            "kind",
+            "value_normalized",
+            name="uq_publication_identifiers_publication_kind_value",
+        ),
+    )
+
+
+def _create_publication_identifiers_indexes() -> None:
+    op.create_index(
+        "ix_publication_identifiers_kind_value",
+        "publication_identifiers",
+        ["kind", "value_normalized"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_publication_identifiers_publication_id",
+        "publication_identifiers",
+        ["publication_id"],
+        unique=False,
+    )
+
+
+def _drop_publication_identifiers_indexes() -> None:
+    op.drop_index("ix_publication_identifiers_publication_id", table_name="publication_identifiers")
+    op.drop_index("ix_publication_identifiers_kind_value", table_name="publication_identifiers")
+
+
+def upgrade() -> None:
+    if "publication_identifiers" in _table_names():
+        return
+    _create_publication_identifiers_table()
+    _create_publication_identifiers_indexes()
+
+
+def downgrade() -> None:
+    if "publication_identifiers" not in _table_names():
+        return
+    _drop_publication_identifiers_indexes()
+    op.drop_table("publication_identifiers")

alembic/versions/20260224_0018_add_openalex_enriched_to_publications.py

@@ -0,0 +1,33 @@
+"""Add openalex_enriched to publications
+
+Revision ID: 20260224_0018
+Revises: fa0c5e3262d5
+Create Date: 2026-02-24 19:10:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260224_0018"
+down_revision: str | Sequence[str] | None = "fa0c5e3262d5"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "publications", sa.Column("openalex_enriched", sa.Boolean(), server_default=sa.text("false"), nullable=False)
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("publications", "openalex_enriched")
+    # ### end Alembic commands ###

alembic/versions/20260224_0019_add_resolving_to_runstatus.py

@@ -0,0 +1,31 @@
+"""Add resolving to RunStatus enum
+
+Revision ID: 20260224_0019
+Revises: 20260224_0018
+Create Date: 2026-02-24 22:20:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260224_0019"
+down_revision: str | Sequence[str] | None = "20260224_0018"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # Use COMMIT to break out of the transaction block that Alembic creates in env.py.
+    # Postgres restricts ALTER TYPE ... ADD VALUE inside transactions when the type is in use.
+    # This is the robust, non-patchwork way to ensure the schema evolves correctly.
+    op.execute("COMMIT")
+    op.execute("ALTER TYPE run_status ADD VALUE IF NOT EXISTS 'resolving' AFTER 'running'")
+
+
+def downgrade() -> None:
+    # Enum values cannot be easily removed in Postgres without recreating the type,
+    # which is risky and usually avoided in simple migrations.
+    pass

alembic/versions/20260224_0020_add_openalex_last_attempt_at.py

@@ -0,0 +1,27 @@
+"""Add openalex_last_attempt_at to publications
+
+Revision ID: 20260224_0020
+Revises: 20260224_0019
+Create Date: 2026-02-24 22:50:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260224_0020"
+down_revision: str | Sequence[str] | None = "20260224_0019"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.add_column("publications", sa.Column("openalex_last_attempt_at", sa.DateTime(timezone=True), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("publications", "openalex_last_attempt_at")

alembic/versions/20260225_0021_enforce_single_active_run_per_user.py

@@ -0,0 +1,68 @@
+"""Enforce one active crawl run per user.
+
+Revision ID: 20260225_0021
+Revises: 20260224_0020
+Create Date: 2026-02-25 09:10:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260225_0021"
+down_revision: str | Sequence[str] | None = "20260224_0020"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+INDEX_NAME = "uq_crawl_runs_user_active"
+ACTIVE_STATUSES = ("running", "resolving")
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    indexes = {index["name"] for index in inspector.get_indexes("crawl_runs")}
+
+    op.execute("ALTER TYPE run_status ADD VALUE IF NOT EXISTS 'canceled'")
+
+    op.execute(
+        """
+        WITH ranked AS (
+            SELECT
+                id,
+                ROW_NUMBER() OVER (
+                    PARTITION BY user_id
+                    ORDER BY start_dt DESC, id DESC
+                ) AS rn
+            FROM crawl_runs
+            WHERE status IN ('running', 'resolving')
+        )
+        UPDATE crawl_runs AS runs
+        SET
+            status = 'failed',
+            end_dt = COALESCE(runs.end_dt, NOW())
+        FROM ranked
+        WHERE runs.id = ranked.id
+          AND ranked.rn > 1
+        """
+    )
+
+    if INDEX_NAME not in indexes:
+        op.create_index(
+            INDEX_NAME,
+            "crawl_runs",
+            ["user_id"],
+            unique=True,
+            postgresql_where=sa.text("status IN ('running'::run_status, 'resolving'::run_status)"),
+        )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    indexes = {index["name"] for index in inspector.get_indexes("crawl_runs")}
+    if INDEX_NAME in indexes:
+        op.drop_index(INDEX_NAME, table_name="crawl_runs")

alembic/versions/20260225_0022_add_canonical_title_hash_to_publications.py

@@ -0,0 +1,35 @@
+"""Add canonical_title_hash to publications for cross-scholar dedup.
+
+Revision ID: 20260225_0022
+Revises: 20260225_0021
+Create Date: 2026-02-25 10:00:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260225_0022"
+down_revision: str | Sequence[str] | None = "20260225_0021"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "publications",
+        sa.Column("canonical_title_hash", sa.String(64), nullable=True),
+    )
+    op.create_index(
+        "ix_publications_canonical_title_hash",
+        "publications",
+        ["canonical_title_hash"],
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_publications_canonical_title_hash", table_name="publications")
+    op.drop_column("publications", "canonical_title_hash")

alembic/versions/20260226_0023_add_arxiv_runtime_state.py

@@ -0,0 +1,55 @@
+"""Add shared arXiv runtime state table.
+
+Revision ID: 20260226_0023
+Revises: 20260225_0022
+Create Date: 2026-02-26 12:40:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260226_0023"
+down_revision: str | Sequence[str] | None = "20260225_0022"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    table_names = set(inspector.get_table_names())
+
+    if "arxiv_runtime_state" in table_names:
+        return
+
+    op.create_table(
+        "arxiv_runtime_state",
+        sa.Column("state_key", sa.String(length=64), nullable=False),
+        sa.Column("next_allowed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("cooldown_until", sa.DateTime(timezone=True), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.PrimaryKeyConstraint("state_key", name=op.f("pk_arxiv_runtime_state")),
+    )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    table_names = set(inspector.get_table_names())
+    if "arxiv_runtime_state" in table_names:
+        op.drop_table("arxiv_runtime_state")

alembic/versions/20260226_0024_add_arxiv_query_cache_entries.py

@@ -0,0 +1,70 @@
+"""Add arXiv query cache table.
+
+Revision ID: 20260226_0024
+Revises: 20260226_0023
+Create Date: 2026-02-26 14:20:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260226_0024"
+down_revision: str | Sequence[str] | None = "20260226_0023"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    table_names = set(inspector.get_table_names())
+    if "arxiv_query_cache_entries" in table_names:
+        return
+
+    op.create_table(
+        "arxiv_query_cache_entries",
+        sa.Column("query_fingerprint", sa.String(length=64), nullable=False),
+        sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+        sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column(
+            "cached_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.PrimaryKeyConstraint(
+            "query_fingerprint",
+            name=op.f("pk_arxiv_query_cache_entries"),
+        ),
+    )
+    op.create_index(
+        "ix_arxiv_query_cache_expires_at",
+        "arxiv_query_cache_entries",
+        ["expires_at"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_arxiv_query_cache_cached_at",
+        "arxiv_query_cache_entries",
+        ["cached_at"],
+        unique=False,
+    )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    table_names = set(inspector.get_table_names())
+    if "arxiv_query_cache_entries" in table_names:
+        op.drop_table("arxiv_query_cache_entries")

alembic/versions/44f7e10ef777_remove_doi_from_publications_and_.py

@@ -0,0 +1,27 @@
+"""remove_doi_from_publications_and_migrate_identifiers
+
+Revision ID: 44f7e10ef777
+Revises: 20260222_0016
+Create Date: 2026-02-22 17:03:56.261936
+
+"""
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '44f7e10ef777'
+down_revision: str | Sequence[str] | None = '20260222_0016'
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    pass
+
+
+def downgrade() -> None:
+    pass
+

alembic/versions/fa0c5e3262d5_add_api_integration_keys_to_usersetting.py

@@ -0,0 +1,39 @@
+"""Add API integration keys to UserSetting
+
+Revision ID: fa0c5e3262d5
+Revises: 44f7e10ef777
+Create Date: 2026-02-23 14:25:45.021512
+
+"""
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'fa0c5e3262d5'
+down_revision: str | Sequence[str] | None = '44f7e10ef777'
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_publications_doi'), table_name='publications')
+    op.drop_column('publications', 'doi')
+    op.add_column('user_settings', sa.Column('openalex_api_key', sa.String(length=255), nullable=True))
+    op.add_column('user_settings', sa.Column('crossref_api_token', sa.String(length=255), nullable=True))
+    op.add_column('user_settings', sa.Column('crossref_api_mailto', sa.String(length=255), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('user_settings', 'crossref_api_mailto')
+    op.drop_column('user_settings', 'crossref_api_token')
+    op.drop_column('user_settings', 'openalex_api_key')
+    op.add_column('publications', sa.Column('doi', sa.VARCHAR(length=255), autoincrement=False, nullable=True))
+    op.create_index(op.f('ix_publications_doi'), 'publications', ['doi'], unique=False)
+    # ### end Alembic commands ###
+

app/api/__init__.py

@@ -1,2 +1 @@
 from __future__ import annotations
-

app/api/errors.py

@@ -82,4 +82,3 @@ def register_api_exception_handlers(app: FastAPI) -> None:
             message="Request validation failed.",
             details=exc.errors(),
         )
-

app/api/media.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import mimetypes
+
+from fastapi import APIRouter, Depends
+from fastapi.responses import FileResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.deps import get_api_current_user
+from app.api.errors import ApiException
+from app.db.models import User
+from app.db.session import get_db_session
+from app.services.scholars import application as scholar_service
+from app.services.scholars import uploads as scholar_uploads
+from app.settings import settings
+
+router = APIRouter(tags=["media"])
+
+
+@router.get("/scholar-images/{scholar_profile_id}/upload")
+async def get_uploaded_scholar_image(
+    scholar_profile_id: int,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    profile = await scholar_service.get_user_scholar_by_id(
+        db_session,
+        user_id=current_user.id,
+        scholar_profile_id=scholar_profile_id,
+    )
+    if profile is None:
+        raise ApiException(
+            status_code=404,
+            code="scholar_not_found",
+            message="Scholar not found.",
+        )
+    if not profile.profile_image_upload_path:
+        raise ApiException(
+            status_code=404,
+            code="scholar_image_not_found",
+            message="Scholar image not found.",
+        )
+
+    try:
+        image_path = scholar_uploads.resolve_upload_file_path(
+            upload_dir=settings.scholar_image_upload_dir,
+            relative_path=profile.profile_image_upload_path,
+        )
+    except scholar_service.ScholarServiceError as exc:
+        raise ApiException(
+            status_code=404,
+            code="scholar_image_not_found",
+            message="Scholar image not found.",
+        ) from exc
+
+    if not image_path.exists() or not image_path.is_file():
+        raise ApiException(
+            status_code=404,
+            code="scholar_image_not_found",
+            message="Scholar image not found.",
+        )
+
+    media_type = mimetypes.guess_type(str(image_path))[0] or "application/octet-stream"
+    return FileResponse(path=image_path, media_type=media_type)

app/api/router.py

@@ -2,11 +2,13 @@ from __future__ import annotations
 
 from fastapi import APIRouter
 
-from app.api.routers import admin, auth, publications, runs, scholars, settings
+from app.api.routers import admin, admin_dbops, admin_settings, auth, publications, runs, scholars, settings
 
 router = APIRouter(prefix="/api/v1")
 router.include_router(auth.router)
 router.include_router(admin.router)
+router.include_router(admin_settings.router)
+router.include_router(admin_dbops.router)
 router.include_router(scholars.router)
 router.include_router(settings.router)
 router.include_router(runs.router)

app/api/routers/__init__.py

@@ -1,2 +1 @@
 from __future__ import annotations
-

app/api/routers/admin.py

@@ -20,7 +20,8 @@ from app.auth.deps import get_auth_service
 from app.auth.service import AuthService
 from app.db.models import User
 from app.db.session import get_db_session
-from app.services import users as user_service
+from app.logging_utils import structured_log
+from app.services.users import application as user_service
 
 logger = logging.getLogger(__name__)
 
@@ -48,14 +49,7 @@ async def list_users(
     admin_user: User = Depends(get_api_admin_user),
 ):
     users = await user_service.list_users(db_session)
-    logger.info(
-        "api.admin.users_listed",
-        extra={
-            "event": "api.admin.users_listed",
-            "admin_user_id": int(admin_user.id),
-            "user_count": len(users),
-        },
-    )
+    structured_log(logger, "info", "api.admin.users_listed", admin_user_id=int(admin_user.id), user_count=len(users))
     return success_payload(
         request,
         data={
@@ -92,14 +86,13 @@ async def create_user(
             message=str(exc),
         ) from exc
 
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.admin.user_created",
-        extra={
-            "event": "api.admin.user_created",
-            "admin_user_id": int(admin_user.id),
-            "target_user_id": int(created_user.id),
-            "target_is_admin": bool(created_user.is_admin),
-        },
+        admin_user_id=int(admin_user.id),
+        target_user_id=int(created_user.id),
+        target_is_admin=bool(created_user.is_admin),
     )
     return success_payload(
         request,
@@ -125,11 +118,7 @@ async def set_user_active(
             code="user_not_found",
             message="User not found.",
         )
-    if (
-        int(target_user.id) == int(admin_user.id)
-        and bool(target_user.is_active)
-        and not bool(payload.is_active)
-    ):
+    if int(target_user.id) == int(admin_user.id) and bool(target_user.is_active) and not bool(payload.is_active):
         raise ApiException(
             status_code=400,
             code="cannot_deactivate_self",
@@ -140,14 +129,13 @@ async def set_user_active(
         user=target_user,
         is_active=bool(payload.is_active),
     )
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.admin.user_active_updated",
-        extra={
-            "event": "api.admin.user_active_updated",
-            "admin_user_id": int(admin_user.id),
-            "target_user_id": int(updated_user.id),
-            "is_active": bool(updated_user.is_active),
-        },
+        admin_user_id=int(admin_user.id),
+        target_user_id=int(updated_user.id),
+        is_active=bool(updated_user.is_active),
     )
     return success_payload(
         request,
@@ -188,13 +176,12 @@ async def reset_user_password(
         user=target_user,
         password_hash=auth_service.hash_password(validated_password),
     )
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.admin.user_password_reset",
-        extra={
-            "event": "api.admin.user_password_reset",
-            "admin_user_id": int(admin_user.id),
-            "target_user_id": int(target_user.id),
-        },
+        admin_user_id=int(admin_user.id),
+        target_user_id=int(target_user.id),
     )
     return success_payload(
         request,

app/api/routers/admin_dbops.py

@@ -0,0 +1,445 @@
+from __future__ import annotations
+
+import logging
+
+from fastapi import APIRouter, Depends, Path, Query, Request
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.deps import get_api_admin_user, get_api_current_user
+from app.api.errors import ApiException
+from app.api.responses import success_payload
+from app.api.schemas import (
+    AdminDbIntegrityEnvelope,
+    AdminDbRepairJobsEnvelope,
+    AdminPdfQueueBulkEnqueueEnvelope,
+    AdminPdfQueueEnvelope,
+    AdminPdfQueueRequeueEnvelope,
+    AdminRepairPublicationLinksEnvelope,
+    AdminRepairPublicationLinksRequest,
+    AdminRepairPublicationNearDuplicatesEnvelope,
+    AdminRepairPublicationNearDuplicatesRequest,
+)
+from app.db.models import DataRepairJob, User
+from app.db.session import get_db_session
+from app.logging_utils import structured_log
+from app.services.dbops import (
+    collect_integrity_report,
+    list_repair_jobs,
+    run_publication_link_repair,
+    run_publication_near_duplicate_repair,
+)
+from app.services.publications import application as publication_service
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/admin/db", tags=["api-admin-dbops"])
+
+
+def _serialize_repair_job(job: DataRepairJob) -> dict[str, object]:
+    return {
+        "id": int(job.id),
+        "job_name": job.job_name,
+        "requested_by": job.requested_by,
+        "dry_run": bool(job.dry_run),
+        "status": job.status,
+        "scope": dict(job.scope or {}),
+        "summary": dict(job.summary or {}),
+        "error_text": job.error_text,
+        "started_at": job.started_at,
+        "finished_at": job.finished_at,
+        "created_at": job.created_at,
+        "updated_at": job.updated_at,
+    }
+
+
+def _requested_by_value(*, payload, admin_user: User) -> str:
+    from_payload = (payload.requested_by or "").strip()
+    return from_payload or admin_user.email
+
+
+def _serialize_pdf_queue_item(item, *, is_admin: bool = False) -> dict[str, object]:
+    return {
+        "publication_id": item.publication_id,
+        "title": item.title,
+        "display_identifier": _serialize_display_identifier(item.display_identifier),
+        "pdf_url": item.pdf_url,
+        "status": item.status,
+        "attempt_count": item.attempt_count,
+        "last_failure_reason": item.last_failure_reason,
+        "last_failure_detail": item.last_failure_detail,
+        "last_source": item.last_source,
+        "requested_by_user_id": item.requested_by_user_id,
+        "requested_by_email": item.requested_by_email if is_admin else None,
+        "queued_at": item.queued_at,
+        "last_attempt_at": item.last_attempt_at,
+        "resolved_at": item.resolved_at,
+        "updated_at": item.updated_at,
+    }
+
+
+def _serialize_display_identifier(value) -> dict[str, object] | None:
+    if value is None:
+        return None
+    return {
+        "kind": value.kind,
+        "value": value.value,
+        "label": value.label,
+        "url": value.url,
+        "confidence_score": float(value.confidence_score),
+    }
+
+
+def _requeue_response_state(*, queued: bool) -> tuple[str, str]:
+    if queued:
+        return "queued", "PDF lookup queued."
+    return "blocked", "PDF lookup is already queued or currently running."
+
+
+def _bulk_enqueue_message(*, queued_count: int, requested_count: int) -> str:
+    if requested_count == 0:
+        return "No missing-PDF publications were found."
+    if queued_count == 0:
+        return "No publications were queued; all candidates were already in-flight."
+    return f"Queued {queued_count} publication(s) for PDF lookup."
+
+
+def _resolve_pdf_queue_paging(
+    *,
+    page: int,
+    page_size: int,
+    limit: int | None,
+    offset: int | None,
+) -> tuple[int, int, int]:
+    resolved_limit = int(limit) if limit is not None else int(page_size)
+    resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0)
+    resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1)
+    return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0)
+
+
+def _pdf_queue_page_data(*, total_count: int, page: int, page_size: int, offset: int) -> dict[str, object]:
+    return {
+        "total_count": int(total_count),
+        "page": int(page),
+        "page_size": int(page_size),
+        "has_prev": int(offset) > 0,
+        "has_next": int(offset) + int(page_size) < int(total_count),
+    }
+
+
+@router.get(
+    "/integrity",
+    response_model=AdminDbIntegrityEnvelope,
+)
+async def get_integrity_report(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    admin_user: User = Depends(get_api_admin_user),
+):
+    report = await collect_integrity_report(db_session)
+    structured_log(
+        logger,
+        "info",
+        "api.admin.db.integrity_checked",
+        admin_user_id=int(admin_user.id),
+        status=report.get("status"),
+        failure_count=len(report.get("failures", [])),
+        warning_count=len(report.get("warnings", [])),
+    )
+    return success_payload(request, data=report)
+
+
+@router.get(
+    "/repair-jobs",
+    response_model=AdminDbRepairJobsEnvelope,
+)
+async def get_repair_jobs(
+    request: Request,
+    limit: int = Query(default=50, ge=1, le=200),
+    db_session: AsyncSession = Depends(get_db_session),
+    admin_user: User = Depends(get_api_admin_user),
+):
+    jobs = await list_repair_jobs(db_session, limit=limit)
+    structured_log(
+        logger,
+        "info",
+        "api.admin.db.repair_jobs_listed",
+        admin_user_id=int(admin_user.id),
+        limit=int(limit),
+        job_count=len(jobs),
+    )
+    return success_payload(
+        request,
+        data={"jobs": [_serialize_repair_job(job) for job in jobs]},
+    )
+
+
+@router.get(
+    "/pdf-queue",
+    response_model=AdminPdfQueueEnvelope,
+)
+async def get_pdf_queue(
+    request: Request,
+    page: int = Query(default=1, ge=1),
+    page_size: int = Query(default=100, ge=1, le=500),
+    limit: int | None = Query(default=None, ge=1, le=500),
+    offset: int | None = Query(default=None, ge=0),
+    status: str | None = Query(default=None),
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    normalized_status = (status or "").strip().lower() or None
+    resolved_page, resolved_limit, resolved_offset = _resolve_pdf_queue_paging(
+        page=page,
+        page_size=page_size,
+        limit=limit,
+        offset=offset,
+    )
+    queue_page = await publication_service.list_pdf_queue_page(
+        db_session,
+        limit=resolved_limit,
+        offset=resolved_offset,
+        status=normalized_status,
+    )
+    structured_log(
+        logger,
+        "info",
+        "api.admin.db.pdf_queue_listed",
+        user_id=int(current_user.id),
+        page=int(resolved_page),
+        page_size=int(resolved_limit),
+        offset=int(resolved_offset),
+        status=normalized_status,
+        item_count=len(queue_page.items),
+        total_count=int(queue_page.total_count),
+    )
+    return success_payload(
+        request,
+        data={
+            "items": [_serialize_pdf_queue_item(item, is_admin=current_user.is_admin) for item in queue_page.items],
+            **_pdf_queue_page_data(
+                total_count=queue_page.total_count,
+                page=resolved_page,
+                page_size=resolved_limit,
+                offset=resolved_offset,
+            ),
+        },
+    )
+
+
+@router.post(
+    "/pdf-queue/{publication_id}/requeue",
+    response_model=AdminPdfQueueRequeueEnvelope,
+)
+async def requeue_pdf_lookup(
+    request: Request,
+    publication_id: int = Path(ge=1),
+    db_session: AsyncSession = Depends(get_db_session),
+    admin_user: User = Depends(get_api_admin_user),
+):
+    result = await publication_service.enqueue_retry_pdf_job_for_publication_id(
+        db_session,
+        user_id=int(admin_user.id),
+        request_email=admin_user.email,
+        publication_id=publication_id,
+    )
+    if not result.publication_exists:
+        raise ApiException(
+            status_code=404,
+            code="publication_not_found",
+            message="Publication not found.",
+        )
+    status, message = _requeue_response_state(queued=result.queued)
+    structured_log(
+        logger,
+        "info",
+        "api.admin.db.pdf_requeued",
+        admin_user_id=int(admin_user.id),
+        publication_id=int(publication_id),
+        queued=bool(result.queued),
+    )
+    return success_payload(
+        request,
+        data={
+            "publication_id": int(publication_id),
+            "queued": bool(result.queued),
+            "status": status,
+            "message": message,
+        },
+    )
+
+
+@router.post(
+    "/pdf-queue/requeue-all",
+    response_model=AdminPdfQueueBulkEnqueueEnvelope,
+)
+async def requeue_all_missing_pdfs(
+    request: Request,
+    limit: int = Query(default=1000, ge=1, le=5000),
+    db_session: AsyncSession = Depends(get_db_session),
+    admin_user: User = Depends(get_api_admin_user),
+):
+    result = await publication_service.enqueue_all_missing_pdf_jobs(
+        db_session,
+        user_id=int(admin_user.id),
+        request_email=admin_user.email,
+        limit=limit,
+    )
+    structured_log(
+        logger,
+        "info",
+        "api.admin.db.pdf_queue_requeue_all",
+        admin_user_id=int(admin_user.id),
+        limit=int(limit),
+        requested_count=int(result.requested_count),
+        queued_count=int(result.queued_count),
+    )
+    return success_payload(
+        request,
+        data={
+            "requested_count": int(result.requested_count),
+            "queued_count": int(result.queued_count),
+            "message": _bulk_enqueue_message(
+                queued_count=int(result.queued_count),
+                requested_count=int(result.requested_count),
+            ),
+        },
+    )
+
+
+@router.post(
+    "/repairs/publication-links",
+    response_model=AdminRepairPublicationLinksEnvelope,
+)
+async def trigger_publication_link_repair(
+    payload: AdminRepairPublicationLinksRequest,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    admin_user: User = Depends(get_api_admin_user),
+):
+    try:
+        result = await run_publication_link_repair(
+            db_session,
+            scope_mode=payload.scope_mode,
+            user_id=payload.user_id,
+            scholar_profile_ids=payload.scholar_profile_ids,
+            dry_run=bool(payload.dry_run),
+            gc_orphan_publications=bool(payload.gc_orphan_publications),
+            requested_by=_requested_by_value(payload=payload, admin_user=admin_user),
+        )
+    except ValueError as exc:
+        raise ApiException(
+            status_code=400,
+            code="invalid_repair_scope",
+            message=str(exc),
+        ) from exc
+    structured_log(
+        logger,
+        "info",
+        "api.admin.db.publication_link_repair_triggered",
+        admin_user_id=int(admin_user.id),
+        scope_mode=payload.scope_mode,
+        target_user_id=int(payload.user_id) if payload.user_id is not None else None,
+        dry_run=bool(payload.dry_run),
+        gc_orphan_publications=bool(payload.gc_orphan_publications),
+        job_id=int(result["job_id"]),
+        status=result["status"],
+    )
+    return success_payload(request, data=result)
+
+
+@router.post(
+    "/repairs/publication-near-duplicates",
+    response_model=AdminRepairPublicationNearDuplicatesEnvelope,
+)
+async def trigger_publication_near_duplicate_repair(
+    payload: AdminRepairPublicationNearDuplicatesRequest,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    admin_user: User = Depends(get_api_admin_user),
+):
+    try:
+        result = await run_publication_near_duplicate_repair(
+            db_session,
+            dry_run=bool(payload.dry_run),
+            similarity_threshold=float(payload.similarity_threshold),
+            min_shared_tokens=int(payload.min_shared_tokens),
+            max_year_delta=int(payload.max_year_delta),
+            max_clusters=int(payload.max_clusters),
+            selected_cluster_keys=list(payload.selected_cluster_keys),
+            requested_by=_requested_by_value(payload=payload, admin_user=admin_user),
+        )
+    except ValueError as exc:
+        raise ApiException(
+            status_code=400,
+            code="invalid_near_duplicate_repair_request",
+            message=str(exc),
+        ) from exc
+    structured_log(
+        logger,
+        "info",
+        "api.admin.db.dedup_repair_triggered",
+        admin_user_id=int(admin_user.id),
+        dry_run=bool(payload.dry_run),
+        selected_cluster_count=len(payload.selected_cluster_keys),
+        job_id=int(result["job_id"]),
+        status=result["status"],
+    )
+    return success_payload(request, data=result)
+
+
+DROP_PUBLICATIONS_CONFIRMATION = "DROP ALL PUBLICATIONS"
+
+
+@router.post("/drop-all-publications")
+async def drop_all_publications(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    admin_user: User = Depends(get_api_admin_user),
+):
+    body = await request.json()
+    confirmation_text = (body.get("confirmation_text") or "").strip()
+    if confirmation_text != DROP_PUBLICATIONS_CONFIRMATION:
+        raise ApiException(
+            status_code=400,
+            code="confirmation_required",
+            message=f"Type '{DROP_PUBLICATIONS_CONFIRMATION}' to confirm this destructive action.",
+        )
+
+    from sqlalchemy import delete, func, select, update
+
+    from app.db.models import (
+        Publication,
+        PublicationIdentifier,
+        PublicationPdfJob,
+        PublicationPdfJobEvent,
+        ScholarProfile,
+        ScholarPublication,
+    )
+
+    count_result = await db_session.execute(select(func.count()).select_from(Publication))
+    total_publications = count_result.scalar_one()
+
+    await db_session.execute(delete(ScholarPublication))
+    await db_session.execute(delete(PublicationIdentifier))
+    await db_session.execute(delete(PublicationPdfJobEvent))
+    await db_session.execute(delete(PublicationPdfJob))
+    await db_session.execute(delete(Publication))
+    await db_session.execute(update(ScholarProfile).values(baseline_completed=False))
+    await db_session.commit()
+
+    structured_log(
+        logger,
+        "warning",
+        "api.admin.db.all_publications_dropped",
+        admin_user_id=int(admin_user.id),
+        admin_email=admin_user.email,
+        deleted_count=int(total_publications),
+    )
+    return success_payload(
+        request,
+        data={
+            "deleted_count": int(total_publications),
+            "message": f"Dropped {total_publications} publication(s) and all related data. "
+            "Scholar baselines have been reset; the next run will re-discover all publications.",
+        },
+    )

app/api/routers/admin_settings.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import logging
+
+from fastapi import APIRouter, Depends, Request
+
+from app.api.deps import get_api_admin_user
+from app.api.errors import ApiException
+from app.api.responses import success_payload
+from app.api.schemas import (
+    AdminScholarHttpSettingsEnvelope,
+    AdminScholarHttpSettingsUpdateRequest,
+)
+from app.db.models import User
+from app.logging_utils import structured_log
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/admin/settings", tags=["api-admin-settings"])
+_MAX_USER_AGENT_LENGTH = 512
+_MAX_ACCEPT_LANGUAGE_LENGTH = 128
+_MAX_COOKIE_LENGTH = 8192
+
+
+def _normalize_header_value(value: str, *, field_name: str, max_length: int) -> str:
+    normalized = value.strip()
+    if len(normalized) > max_length:
+        raise ApiException(
+            status_code=400,
+            code="invalid_admin_setting",
+            message=f"{field_name} must be {max_length} characters or fewer.",
+        )
+    return normalized
+
+
+def _serialize_scholar_http_settings() -> dict[str, object]:
+    return {
+        "user_agent": settings.scholar_http_user_agent,
+        "rotate_user_agent": bool(settings.scholar_http_rotate_user_agent),
+        "accept_language": settings.scholar_http_accept_language,
+        "cookie": settings.scholar_http_cookie,
+    }
+
+
+def _apply_scholar_http_settings(payload: AdminScholarHttpSettingsUpdateRequest) -> None:
+    user_agent = _normalize_header_value(
+        payload.user_agent,
+        field_name="user_agent",
+        max_length=_MAX_USER_AGENT_LENGTH,
+    )
+    accept_language = _normalize_header_value(
+        payload.accept_language,
+        field_name="accept_language",
+        max_length=_MAX_ACCEPT_LANGUAGE_LENGTH,
+    )
+    cookie = _normalize_header_value(
+        payload.cookie or "",
+        field_name="cookie",
+        max_length=_MAX_COOKIE_LENGTH,
+    )
+    object.__setattr__(settings, "scholar_http_user_agent", user_agent)
+    object.__setattr__(settings, "scholar_http_rotate_user_agent", bool(payload.rotate_user_agent))
+    object.__setattr__(settings, "scholar_http_accept_language", accept_language)
+    object.__setattr__(settings, "scholar_http_cookie", cookie)
+
+
+@router.get(
+    "/scholar-http",
+    response_model=AdminScholarHttpSettingsEnvelope,
+)
+async def get_scholar_http_settings(
+    request: Request,
+    admin_user: User = Depends(get_api_admin_user),
+):
+    structured_log(
+        logger,
+        "info",
+        "api.admin.settings.scholar_http_read",
+        admin_user_id=int(admin_user.id),
+    )
+    return success_payload(request, data=_serialize_scholar_http_settings())
+
+
+@router.put(
+    "/scholar-http",
+    response_model=AdminScholarHttpSettingsEnvelope,
+)
+async def update_scholar_http_settings(
+    payload: AdminScholarHttpSettingsUpdateRequest,
+    request: Request,
+    admin_user: User = Depends(get_api_admin_user),
+):
+    _apply_scholar_http_settings(payload)
+    structured_log(
+        logger,
+        "info",
+        "api.admin.settings.scholar_http_updated",
+        admin_user_id=int(admin_user.id),
+        rotate_user_agent=bool(settings.scholar_http_rotate_user_agent),
+        user_agent_set=bool(settings.scholar_http_user_agent.strip()),
+        cookie_set=bool(settings.scholar_http_cookie.strip()),
+    )
+    return success_payload(request, data=_serialize_scholar_http_settings())

app/api/routers/auth.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
 import logging
+from typing import NoReturn
 
 from fastapi import APIRouter, Depends, Request
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.api.errors import ApiException
 from app.api.deps import get_api_current_user
+from app.api.errors import ApiException
 from app.api.responses import success_payload
 from app.api.schemas import (
     AuthMeEnvelope,
@@ -16,21 +17,60 @@ from app.api.schemas import (
     LoginRequest,
     MessageEnvelope,
 )
+from app.auth import runtime as auth_runtime
 from app.auth.deps import get_auth_service, get_login_rate_limiter
 from app.auth.rate_limit import SlidingWindowRateLimiter
-from app.auth import runtime as auth_runtime
 from app.auth.service import AuthService
 from app.auth.session import set_session_user
 from app.db.models import User
 from app.db.session import get_db_session
+from app.logging_utils import structured_log
 from app.security.csrf import ensure_csrf_token
-from app.services import users as user_service
+from app.services.users import application as user_service
 
 logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/auth", tags=["api-auth"])
 
 
+def _login_limiter_key_and_email(request: Request, payload: LoginRequest) -> tuple[str, str]:
+    return auth_runtime.login_rate_limit_key(request, payload.email), payload.email.strip().lower()
+
+
+def _raise_rate_limited(normalized_email: str, retry_after_seconds: int) -> None:
+    structured_log(
+        logger,
+        "warning",
+        "api.auth.login_rate_limited",
+        email=normalized_email,
+        retry_after_seconds=retry_after_seconds,
+    )
+    raise ApiException(
+        status_code=429,
+        code="rate_limited",
+        message="Too many login attempts. Please try again later.",
+        details={"retry_after_seconds": retry_after_seconds},
+    )
+
+
+def _serialize_user_payload(user: User) -> dict[str, object]:
+    return {
+        "id": int(user.id),
+        "email": user.email,
+        "is_admin": bool(user.is_admin),
+        "is_active": bool(user.is_active),
+    }
+
+
+def _raise_invalid_credentials(*, normalized_email: str) -> NoReturn:
+    structured_log(logger, "info", "api.auth.login_failed", email=normalized_email)
+    raise ApiException(
+        status_code=401,
+        code="invalid_credentials",
+        message="Invalid email or password.",
+    )
+
+
 @router.post(
     "/login",
     response_model=LoginEnvelope,
@@ -42,24 +82,10 @@ async def login(
     auth_service: AuthService = Depends(get_auth_service),
     rate_limiter: SlidingWindowRateLimiter = Depends(get_login_rate_limiter),
 ):
-    limiter_key = auth_runtime.login_rate_limit_key(request, payload.email)
+    limiter_key, normalized_email = _login_limiter_key_and_email(request, payload)
     decision = rate_limiter.check(limiter_key)
-    normalized_email = payload.email.strip().lower()
     if not decision.allowed:
-        logger.warning(
-            "api.auth.login_rate_limited",
-            extra={
-                "event": "api.auth.login_rate_limited",
-                "email": normalized_email,
-                "retry_after_seconds": decision.retry_after_seconds,
-            },
-        )
-        raise ApiException(
-            status_code=429,
-            code="rate_limited",
-            message="Too many login attempts. Please try again later.",
-            details={"retry_after_seconds": decision.retry_after_seconds},
-        )
+        _raise_rate_limited(normalized_email, int(decision.retry_after_seconds))
 
     user = await auth_service.authenticate_user(
         db_session,
@@ -68,18 +94,7 @@ async def login(
     )
     if user is None:
         rate_limiter.record_failure(limiter_key)
-        logger.info(
-            "api.auth.login_failed",
-            extra={
-                "event": "api.auth.login_failed",
-                "email": normalized_email,
-            },
-        )
-        raise ApiException(
-            status_code=401,
-            code="invalid_credentials",
-            message="Invalid email or password.",
-        )
+        _raise_invalid_credentials(normalized_email=normalized_email)
 
     rate_limiter.reset(limiter_key)
     set_session_user(
@@ -88,25 +103,13 @@ async def login(
         email=user.email,
         is_admin=user.is_admin,
     )
-    logger.info(
-        "api.auth.login_succeeded",
-        extra={
-            "event": "api.auth.login_succeeded",
-            "user_id": user.id,
-            "is_admin": user.is_admin,
-        },
-    )
+    structured_log(logger, "info", "api.auth.login_succeeded", user_id=user.id, is_admin=user.is_admin)
     return success_payload(
         request,
         data={
             "authenticated": True,
             "csrf_token": ensure_csrf_token(request),
-            "user": {
-                "id": int(user.id),
-                "email": user.email,
-                "is_admin": bool(user.is_admin),
-                "is_active": bool(user.is_active),
-            },
+            "user": _serialize_user_payload(user),
         },
     )
 
@@ -124,12 +127,7 @@ async def get_current_session(
         data={
             "authenticated": True,
             "csrf_token": ensure_csrf_token(request),
-            "user": {
-                "id": int(current_user.id),
-                "email": current_user.email,
-                "is_admin": bool(current_user.is_admin),
-                "is_active": bool(current_user.is_active),
-            },
+            "user": _serialize_user_payload(current_user),
         },
     )
 
@@ -192,13 +190,7 @@ async def change_password(
         user=current_user,
         password_hash=auth_service.hash_password(validated_password),
     )
-    logger.info(
-        "api.auth.password_changed",
-        extra={
-            "event": "api.auth.password_changed",
-            "user_id": int(current_user.id),
-        },
-    )
+    structured_log(logger, "info", "api.auth.password_changed", user_id=int(current_user.id))
     return success_payload(
         request,
         data={"message": "Password updated successfully."},
@@ -215,12 +207,8 @@ async def logout(
 ):
     current_user = await auth_runtime.get_authenticated_user(request, db_session)
     auth_runtime.invalidate_session(request)
-    logger.info(
-        "api.auth.logout",
-        extra={
-            "event": "api.auth.logout",
-            "user_id": int(current_user.id) if current_user is not None else None,
-        },
+    structured_log(
+        logger, "info", "api.auth.logout", user_id=int(current_user.id) if current_user is not None else None
     )
     return success_payload(
         request,

app/api/routers/publications.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
 import logging
+from datetime import UTC, datetime
 from typing import Literal
 
-from fastapi import APIRouter, Depends, Query, Request
+from fastapi import APIRouter, Depends, Path, Query, Request
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.api.deps import get_api_current_user
@@ -14,88 +15,396 @@ from app.api.schemas import (
     MarkSelectedReadEnvelope,
     MarkSelectedReadRequest,
     PublicationsListEnvelope,
+    RetryPublicationPdfEnvelope,
+    RetryPublicationPdfRequest,
+    TogglePublicationFavoriteEnvelope,
+    TogglePublicationFavoriteRequest,
 )
 from app.db.models import User
 from app.db.session import get_db_session
-from app.services import publications as publication_service
-from app.services import scholars as scholar_service
+from app.logging_utils import structured_log
+from app.services.publication_identifiers import application as identifier_service
+from app.services.publications import application as publication_service
+from app.services.scholars import application as scholar_service
+from app.settings import settings
 
 logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/publications", tags=["api-publications"])
 
 
+async def _require_selected_profile(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    selected_scholar_id: int | None,
+) -> None:
+    if selected_scholar_id is None:
+        return
+    selected_profile = await scholar_service.get_user_scholar_by_id(
+        db_session,
+        user_id=user_id,
+        scholar_profile_id=selected_scholar_id,
+    )
+    if selected_profile is None:
+        raise ApiException(
+            status_code=404,
+            code="scholar_not_found",
+            message="Scholar filter not found.",
+        )
+
+
+def _serialize_publication_item(item) -> dict[str, object]:
+    return {
+        "publication_id": item.publication_id,
+        "scholar_profile_id": item.scholar_profile_id,
+        "scholar_label": item.scholar_label,
+        "title": item.title,
+        "year": item.year,
+        "citation_count": item.citation_count,
+        "venue_text": item.venue_text,
+        "pub_url": item.pub_url,
+        "display_identifier": _serialize_display_identifier(item.display_identifier),
+        "pdf_url": item.pdf_url,
+        "pdf_status": item.pdf_status,
+        "pdf_attempt_count": item.pdf_attempt_count,
+        "pdf_failure_reason": item.pdf_failure_reason,
+        "pdf_failure_detail": item.pdf_failure_detail,
+        "is_read": item.is_read,
+        "is_favorite": item.is_favorite,
+        "first_seen_at": item.first_seen_at,
+        "is_new_in_latest_run": item.is_new_in_latest_run,
+    }
+
+
+def _serialize_display_identifier(value) -> dict[str, object] | None:
+    if value is None:
+        return None
+    return {
+        "kind": value.kind,
+        "value": value.value,
+        "label": value.label,
+        "url": value.url,
+        "confidence_score": float(value.confidence_score),
+    }
+
+
+async def _publication_counts(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    selected_scholar_id: int | None,
+    favorite_only: bool,
+    search: str | None,
+    snapshot_before: datetime | None,
+) -> tuple[int, int, int, int]:
+    unread_count = await publication_service.count_unread_for_user(
+        db_session,
+        user_id=user_id,
+        scholar_profile_id=selected_scholar_id,
+        favorite_only=favorite_only,
+        snapshot_before=snapshot_before,
+    )
+    favorites_count = await publication_service.count_favorite_for_user(
+        db_session,
+        user_id=user_id,
+        scholar_profile_id=selected_scholar_id,
+        snapshot_before=snapshot_before,
+    )
+    latest_count = await publication_service.count_latest_for_user(
+        db_session,
+        user_id=user_id,
+        scholar_profile_id=selected_scholar_id,
+        favorite_only=favorite_only,
+        snapshot_before=snapshot_before,
+    )
+    total_count = await publication_service.count_for_user(
+        db_session,
+        user_id=user_id,
+        mode=publication_service.MODE_ALL,
+        scholar_profile_id=selected_scholar_id,
+        favorite_only=favorite_only,
+        search=search,
+        snapshot_before=snapshot_before,
+    )
+    return unread_count, favorites_count, latest_count, total_count
+
+
+async def _list_publications_for_request(
+    db_session: AsyncSession,
+    *,
+    current_user: User,
+    mode: Literal["all", "unread", "latest", "new"] | None,
+    favorite_only: bool,
+    scholar_profile_id: int | None,
+    search: str | None,
+    sort_by: str,
+    sort_dir: str,
+    limit: int,
+    offset: int,
+    snapshot_before: datetime | None,
+) -> tuple[str, int | None, list]:
+    resolved_mode = publication_service.resolve_publication_view_mode(mode)
+    selected_scholar_id = scholar_profile_id
+    await _require_selected_profile(
+        db_session,
+        user_id=current_user.id,
+        selected_scholar_id=selected_scholar_id,
+    )
+    publications = await publication_service.list_for_user(
+        db_session,
+        user_id=current_user.id,
+        mode=resolved_mode,
+        scholar_profile_id=selected_scholar_id,
+        favorite_only=favorite_only,
+        search=search,
+        sort_by=sort_by,
+        sort_dir=sort_dir,
+        limit=limit,
+        offset=offset,
+        snapshot_before=snapshot_before,
+    )
+    await publication_service.schedule_missing_pdf_enrichment_for_user(
+        db_session,
+        user_id=current_user.id,
+        request_email=current_user.email,
+        items=publications,
+        max_items=settings.unpaywall_max_items_per_request,
+    )
+    hydrated = await publication_service.hydrate_pdf_enrichment_state(
+        db_session,
+        items=publications,
+    )
+    return resolved_mode, selected_scholar_id, hydrated
+
+
+def _resolve_publications_snapshot(
+    *,
+    snapshot: str | None,
+) -> tuple[datetime, str]:
+    if snapshot is None:
+        now_utc = datetime.now(UTC)
+        return now_utc, now_utc.isoformat()
+    try:
+        parsed = datetime.fromisoformat(snapshot)
+    except ValueError as exc:
+        raise ApiException(
+            status_code=400,
+            code="invalid_snapshot",
+            message="Invalid publications snapshot cursor.",
+        ) from exc
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=UTC)
+    normalized = parsed.astimezone(UTC)
+    return normalized, normalized.isoformat()
+
+
+def _resolve_publications_paging(
+    *,
+    page: int,
+    page_size: int,
+    limit: int | None,
+    offset: int | None,
+) -> tuple[int, int, int]:
+    resolved_limit = int(limit) if limit is not None else int(page_size)
+    resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0)
+    resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1)
+    return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0)
+
+
+def _publications_list_data(
+    *,
+    mode: str,
+    favorite_only: bool,
+    selected_scholar_id: int | None,
+    unread_count: int,
+    favorites_count: int,
+    latest_count: int,
+    total_count: int,
+    publications: list,
+    page: int,
+    page_size: int,
+    offset: int,
+    snapshot: str,
+) -> dict[str, object]:
+    return {
+        "mode": mode,
+        "favorite_only": favorite_only,
+        "selected_scholar_profile_id": selected_scholar_id,
+        "unread_count": unread_count,
+        "favorites_count": favorites_count,
+        "latest_count": latest_count,
+        "new_count": latest_count,
+        "total_count": total_count,
+        "page": int(page),
+        "page_size": int(page_size),
+        "snapshot": snapshot,
+        "has_prev": int(offset) > 0,
+        "has_next": int(offset) + int(page_size) < int(total_count),
+        "publications": [_serialize_publication_item(item) for item in publications],
+    }
+
+
+def _retry_pdf_message(*, queued: bool, resolved_pdf: bool, pdf_status: str) -> str:
+    if resolved_pdf:
+        return "Open-access PDF link already resolved."
+    if queued:
+        return "PDF lookup queued."
+    if pdf_status in {"queued", "running"}:
+        return "PDF lookup is already queued."
+    return "No open-access PDF link found."
+
+
+def _favorite_message(*, is_favorite: bool) -> str:
+    if is_favorite:
+        return "Publication marked as favorite."
+    return "Publication removed from favorites."
+
+
+async def _retry_publication_state(
+    db_session: AsyncSession,
+    *,
+    current_user: User,
+    scholar_profile_id: int,
+    publication_id: int,
+):
+    publication = await publication_service.retry_pdf_for_user(
+        db_session,
+        user_id=current_user.id,
+        scholar_profile_id=scholar_profile_id,
+        publication_id=publication_id,
+    )
+    if publication is None:
+        raise ApiException(
+            status_code=404,
+            code="publication_not_found",
+            message="Publication not found.",
+        )
+    queued = False
+    if not publication.pdf_url:
+        queued = await publication_service.schedule_retry_pdf_enrichment_for_row(
+            db_session,
+            user_id=current_user.id,
+            request_email=current_user.email,
+            item=publication,
+        )
+    hydrated = await publication_service.hydrate_pdf_enrichment_state(
+        db_session,
+        items=[publication],
+    )
+    current = hydrated[0] if hydrated else publication
+    return current, queued
+
+
+async def _favorite_publication_state(
+    db_session: AsyncSession,
+    *,
+    current_user: User,
+    scholar_profile_id: int,
+    publication_id: int,
+    is_favorite: bool,
+):
+    updated_count = await publication_service.set_publication_favorite_for_user(
+        db_session,
+        user_id=current_user.id,
+        scholar_profile_id=scholar_profile_id,
+        publication_id=publication_id,
+        is_favorite=is_favorite,
+    )
+    if updated_count <= 0:
+        raise ApiException(
+            status_code=404,
+            code="publication_not_found",
+            message="Publication not found.",
+        )
+    publication = await publication_service.get_publication_item_for_user(
+        db_session,
+        user_id=current_user.id,
+        scholar_profile_id=scholar_profile_id,
+        publication_id=publication_id,
+    )
+    if publication is None:
+        raise ApiException(
+            status_code=404,
+            code="publication_not_found",
+            message="Publication not found.",
+        )
+    hydrated = await publication_service.hydrate_pdf_enrichment_state(
+        db_session,
+        items=[publication],
+    )
+    current = hydrated[0] if hydrated else publication
+    identifiers = await identifier_service.overlay_publication_items_with_display_identifiers(
+        db_session,
+        items=[current],
+    )
+    return identifiers[0] if identifiers else current
+
+
 @router.get(
     "",
     response_model=PublicationsListEnvelope,
 )
 async def list_publications(
     request: Request,
-    mode: Literal["all", "new"] | None = Query(default=None),
+    mode: Literal["all", "unread", "latest", "new"] | None = Query(default=None),
+    favorite_only: bool = Query(default=False),
     scholar_profile_id: int | None = Query(default=None, ge=1),
-    limit: int = Query(default=300, ge=1, le=1000),
+    search: str | None = Query(default=None, min_length=1, max_length=200),
+    sort_by: Literal["first_seen", "title", "year", "citations", "scholar", "pdf_status"] = Query(default="first_seen"),
+    sort_dir: Literal["asc", "desc"] = Query(default="desc"),
+    page: int = Query(default=1, ge=1),
+    page_size: int = Query(default=100, ge=1, le=500),
+    limit: int | None = Query(default=None, ge=1, le=1000),
+    offset: int | None = Query(default=None, ge=0),
+    snapshot: str | None = Query(default=None, min_length=1, max_length=64),
     db_session: AsyncSession = Depends(get_db_session),
     current_user: User = Depends(get_api_current_user),
 ):
-    resolved_mode = publication_service.resolve_mode(mode)
-    selected_scholar_id = scholar_profile_id
-    if selected_scholar_id is not None:
-        selected_profile = await scholar_service.get_user_scholar_by_id(
-            db_session,
-            user_id=current_user.id,
-            scholar_profile_id=selected_scholar_id,
-        )
-        if selected_profile is None:
-            raise ApiException(
-                status_code=404,
-                code="scholar_not_found",
-                message="Scholar filter not found.",
-            )
-
-    publications = await publication_service.list_for_user(
-        db_session,
-        user_id=current_user.id,
-        mode=resolved_mode,
-        scholar_profile_id=selected_scholar_id,
+    resolved_page, resolved_limit, resolved_offset = _resolve_publications_paging(
+        page=page,
+        page_size=page_size,
         limit=limit,
+        offset=offset,
     )
-    new_count = await publication_service.count_for_user(
+    snapshot_before, snapshot_cursor = _resolve_publications_snapshot(snapshot=snapshot)
+    normalized_search = (search or "").strip() or None
+    resolved_mode, selected_scholar_id, publications = await _list_publications_for_request(
+        db_session,
+        current_user=current_user,
+        mode=mode,
+        favorite_only=favorite_only,
+        scholar_profile_id=scholar_profile_id,
+        search=normalized_search,
+        sort_by=sort_by,
+        sort_dir=sort_dir,
+        limit=resolved_limit,
+        offset=resolved_offset,
+        snapshot_before=snapshot_before,
+    )
+    unread_count, favorites_count, latest_count, total_count = await _publication_counts(
         db_session,
         user_id=current_user.id,
-        mode=publication_service.MODE_NEW,
-        scholar_profile_id=selected_scholar_id,
+        selected_scholar_id=selected_scholar_id,
+        favorite_only=favorite_only,
+        search=normalized_search,
+        snapshot_before=snapshot_before,
     )
-    total_count = await publication_service.count_for_user(
-        db_session,
-        user_id=current_user.id,
-        mode=publication_service.MODE_ALL,
-        scholar_profile_id=selected_scholar_id,
-    )
-    return success_payload(
-        request,
-        data={
-            "mode": resolved_mode,
-            "selected_scholar_profile_id": selected_scholar_id,
-            "new_count": new_count,
-            "total_count": total_count,
-            "publications": [
-                {
-                    "publication_id": item.publication_id,
-                    "scholar_profile_id": item.scholar_profile_id,
-                    "scholar_label": item.scholar_label,
-                    "title": item.title,
-                    "year": item.year,
-                    "citation_count": item.citation_count,
-                    "venue_text": item.venue_text,
-                    "pub_url": item.pub_url,
-                    "is_read": item.is_read,
-                    "first_seen_at": item.first_seen_at,
-                    "is_new_in_latest_run": item.is_new_in_latest_run,
-                }
-                for item in publications
-            ],
-        },
+    data = _publications_list_data(
+        mode=resolved_mode,
+        favorite_only=favorite_only,
+        selected_scholar_id=selected_scholar_id,
+        unread_count=unread_count,
+        favorites_count=favorites_count,
+        latest_count=latest_count,
+        total_count=total_count,
+        publications=publications,
+        page=resolved_page,
+        page_size=resolved_limit,
+        offset=resolved_offset,
+        snapshot=snapshot_cursor,
     )
+    return success_payload(request, data=data)
 
 
 @router.post(
@@ -111,13 +420,8 @@ async def mark_all_publications_read(
         db_session,
         user_id=current_user.id,
     )
-    logger.info(
-        "api.publications.mark_all_read",
-        extra={
-            "event": "api.publications.mark_all_read",
-            "user_id": current_user.id,
-            "updated_count": updated_count,
-        },
+    structured_log(
+        logger, "info", "api.publications.mark_all_read", user_id=current_user.id, updated_count=updated_count
     )
     return success_payload(
         request,
@@ -138,25 +442,19 @@ async def mark_selected_publications_read(
     db_session: AsyncSession = Depends(get_db_session),
     current_user: User = Depends(get_api_current_user),
 ):
-    selection_pairs = sorted(
-        {
-            (int(item.scholar_profile_id), int(item.publication_id))
-            for item in payload.selections
-        }
-    )
+    selection_pairs = sorted({(int(item.scholar_profile_id), int(item.publication_id)) for item in payload.selections})
     updated_count = await publication_service.mark_selected_as_read_for_user(
         db_session,
         user_id=current_user.id,
         selections=selection_pairs,
     )
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.publications.mark_selected_read",
-        extra={
-            "event": "api.publications.mark_selected_read",
-            "user_id": current_user.id,
-            "requested_count": len(selection_pairs),
-            "updated_count": updated_count,
-        },
+        user_id=current_user.id,
+        requested_count=len(selection_pairs),
+        updated_count=updated_count,
     )
     return success_payload(
         request,
@@ -166,3 +464,84 @@ async def mark_selected_publications_read(
             "updated_count": updated_count,
         },
     )
+
+
+@router.post(
+    "/{publication_id}/retry-pdf",
+    response_model=RetryPublicationPdfEnvelope,
+)
+async def retry_publication_pdf(
+    payload: RetryPublicationPdfRequest,
+    request: Request,
+    publication_id: int = Path(ge=1),
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    current, queued = await _retry_publication_state(
+        db_session,
+        current_user=current_user,
+        scholar_profile_id=payload.scholar_profile_id,
+        publication_id=publication_id,
+    )
+    resolved_pdf = bool(current.pdf_url)
+    message = _retry_pdf_message(
+        queued=queued,
+        resolved_pdf=resolved_pdf,
+        pdf_status=current.pdf_status,
+    )
+    structured_log(
+        logger,
+        "info",
+        "api.publications.retry_pdf",
+        user_id=current_user.id,
+        scholar_profile_id=payload.scholar_profile_id,
+        publication_id=publication_id,
+        queued=queued,
+        resolved_pdf=resolved_pdf,
+        pdf_status=current.pdf_status,
+    )
+    return success_payload(
+        request,
+        data={
+            "message": message,
+            "queued": queued,
+            "resolved_pdf": resolved_pdf,
+            "publication": _serialize_publication_item(current),
+        },
+    )
+
+
+@router.post(
+    "/{publication_id}/favorite",
+    response_model=TogglePublicationFavoriteEnvelope,
+)
+async def toggle_publication_favorite(
+    payload: TogglePublicationFavoriteRequest,
+    request: Request,
+    publication_id: int = Path(ge=1),
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    current = await _favorite_publication_state(
+        db_session,
+        current_user=current_user,
+        scholar_profile_id=payload.scholar_profile_id,
+        publication_id=publication_id,
+        is_favorite=payload.is_favorite,
+    )
+    structured_log(
+        logger,
+        "info",
+        "api.publications.favorite",
+        user_id=current_user.id,
+        scholar_profile_id=payload.scholar_profile_id,
+        publication_id=publication_id,
+        is_favorite=bool(payload.is_favorite),
+    )
+    return success_payload(
+        request,
+        data={
+            "message": _favorite_message(is_favorite=bool(payload.is_favorite)),
+            "publication": _serialize_publication_item(current),
+        },
+    )

app/api/routers/run_manual.py

@@ -0,0 +1,243 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, NoReturn
+
+from fastapi import Request
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import ApiException
+from app.api.responses import success_payload
+from app.api.routers.run_serializers import manual_run_payload_from_run
+from app.db.models import RunStatus, RunTriggerType
+from app.logging_utils import structured_log
+from app.services.ingestion import application as ingestion_service
+from app.services.ingestion import safety as run_safety_service
+from app.services.runs import application as run_service
+from app.services.settings import application as user_settings_service
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+async def load_safety_state(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> dict[str, Any]:
+    user_settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=user_id,
+    )
+    previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
+    if run_safety_service.clear_expired_cooldown(user_settings):
+        await db_session.commit()
+        await db_session.refresh(user_settings)
+        structured_log(
+            logger,
+            "info",
+            "api.runs.safety_cooldown_cleared",
+            user_id=user_id,
+            reason=previous_safety_state.get("cooldown_reason"),
+            cooldown_until=previous_safety_state.get("cooldown_until"),
+        )
+    return run_safety_service.get_safety_state_payload(user_settings)
+
+
+def raise_manual_runs_disabled(*, user_id: int, safety_state: dict[str, Any]) -> None:
+    structured_log(
+        logger,
+        "warning",
+        "api.runs.manual_blocked_policy",
+        user_id=user_id,
+        policy={"manual_run_allowed": False},
+        safety_state=safety_state,
+    )
+    raise ApiException(
+        status_code=403,
+        code="manual_runs_disabled",
+        message="Manual checks are disabled by server policy.",
+        details={
+            "policy": {"manual_run_allowed": False},
+            "safety_state": safety_state,
+        },
+    )
+
+
+async def reused_manual_run_payload(
+    db_session: AsyncSession,
+    *,
+    request: Request,
+    user_id: int,
+    idempotency_key: str | None,
+    safety_state: dict[str, Any],
+) -> dict[str, Any] | None:
+    if idempotency_key is None:
+        return None
+    previous_run = await run_service.get_manual_run_by_idempotency_key(
+        db_session,
+        user_id=user_id,
+        idempotency_key=idempotency_key,
+    )
+    if previous_run is None:
+        return None
+    if previous_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING):
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run with this idempotency key is still in progress.",
+            details={"run_id": int(previous_run.id), "idempotency_key": idempotency_key},
+        )
+    return success_payload(
+        request,
+        data=manual_run_payload_from_run(
+            previous_run,
+            idempotency_key=idempotency_key,
+            reused_existing_run=True,
+            safety_state=safety_state,
+        ),
+    )
+
+
+async def run_ingestion_for_manual(
+    db_session: AsyncSession,
+    *,
+    ingest_service: ingestion_service.ScholarIngestionService,
+    user_id: int,
+    idempotency_key: str | None,
+):
+    user_settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=user_id,
+    )
+    return await ingest_service.run_for_user(
+        db_session,
+        user_id=user_id,
+        trigger_type=RunTriggerType.MANUAL,
+        request_delay_seconds=user_settings.request_delay_seconds,
+        network_error_retries=settings.ingestion_network_error_retries,
+        retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+        max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
+        page_size=settings.ingestion_page_size,
+        auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
+        queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
+        idempotency_key=idempotency_key,
+        alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
+        alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
+        alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
+    )
+
+
+async def recover_integrity_error(
+    db_session: AsyncSession,
+    *,
+    request: Request,
+    user_id: int,
+    idempotency_key: str | None,
+    original_exc: IntegrityError,
+) -> dict[str, Any]:
+    if idempotency_key is None:
+        structured_log(
+            logger,
+            "exception",
+            "api.runs.manual_integrity_error",
+            user_id=user_id,
+        )
+        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
+    existing_run = await run_service.get_manual_run_by_idempotency_key(
+        db_session,
+        user_id=user_id,
+        idempotency_key=idempotency_key,
+    )
+    if existing_run is None:
+        structured_log(
+            logger,
+            "exception",
+            "api.runs.manual_integrity_error",
+            user_id=user_id,
+        )
+        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
+    if existing_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING):
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run with this idempotency key is still in progress.",
+            details={"run_id": int(existing_run.id), "idempotency_key": idempotency_key},
+        ) from original_exc
+    return success_payload(
+        request,
+        data=manual_run_payload_from_run(
+            existing_run,
+            idempotency_key=idempotency_key,
+            reused_existing_run=True,
+            safety_state=await load_safety_state(db_session, user_id=user_id),
+        ),
+    )
+
+
+async def execute_manual_run(
+    db_session: AsyncSession,
+    *,
+    request: Request,
+    ingest_service: ingestion_service.ScholarIngestionService,
+    user_id: int,
+    idempotency_key: str | None,
+):
+    try:
+        return await run_ingestion_for_manual(
+            db_session,
+            ingest_service=ingest_service,
+            user_id=user_id,
+            idempotency_key=idempotency_key,
+        )
+    except ingestion_service.RunAlreadyInProgressError as exc:
+        await db_session.rollback()
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run is already in progress for this account.",
+        ) from exc
+    except ingestion_service.RunBlockedBySafetyPolicyError as exc:
+        await db_session.rollback()
+        raise_manual_blocked_safety(exc=exc, user_id=user_id)
+    except IntegrityError as exc:
+        await db_session.rollback()
+        return await recover_integrity_error(
+            db_session,
+            request=request,
+            user_id=user_id,
+            idempotency_key=idempotency_key,
+            original_exc=exc,
+        )
+    except Exception as exc:
+        await db_session.rollback()
+        raise_manual_failed(exc=exc, user_id=user_id)
+
+
+def raise_manual_blocked_safety(*, exc, user_id: int) -> NoReturn:
+    structured_log(
+        logger,
+        "info",
+        "api.runs.manual_blocked_safety",
+        user_id=user_id,
+        reason=exc.safety_state.get("cooldown_reason"),
+        cooldown_until=exc.safety_state.get("cooldown_until"),
+        cooldown_remaining_seconds=exc.safety_state.get("cooldown_remaining_seconds"),
+    )
+    raise ApiException(
+        status_code=429,
+        code=exc.code,
+        message=exc.message,
+        details={"safety_state": exc.safety_state},
+    ) from exc
+
+
+def raise_manual_failed(*, exc: Exception, user_id: int) -> NoReturn:
+    structured_log(
+        logger,
+        "exception",
+        "api.runs.manual_failed",
+        user_id=user_id,
+    )
+    raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from exc

app/api/routers/run_serializers.py

@@ -0,0 +1,238 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from app.api.errors import ApiException
+from app.services.runs import application as run_service
+
+IDEMPOTENCY_HEADER = "Idempotency-Key"
+IDEMPOTENCY_MAX_LENGTH = 128
+IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
+
+
+def _int_value(value: Any, default: int = 0) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _str_value(value: Any) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text if text else None
+
+
+def _bool_value(value: Any, default: bool = False) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        lowered = value.strip().lower()
+        if lowered in {"1", "true", "yes", "on"}:
+            return True
+        if lowered in {"0", "false", "no", "off"}:
+            return False
+    return default
+
+
+def normalize_idempotency_key(raw_value: str | None) -> str | None:
+    if raw_value is None:
+        return None
+    candidate = raw_value.strip()
+    if not candidate:
+        return None
+    if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate):
+        raise ApiException(
+            status_code=400,
+            code="invalid_idempotency_key",
+            message=("Invalid Idempotency-Key. Use 8-128 characters from: A-Z a-z 0-9 . _ : -"),
+        )
+    return candidate
+
+
+def serialize_run(run) -> dict[str, Any]:
+    summary = run_service.extract_run_summary(run.error_log)
+    return {
+        "id": int(run.id),
+        "trigger_type": run.trigger_type.value,
+        "status": run.status.value,
+        "start_dt": run.start_dt,
+        "end_dt": run.end_dt,
+        "scholar_count": int(run.scholar_count or 0),
+        "new_publication_count": int(run.new_pub_count or 0),
+        "failed_count": int(summary["failed_count"]),
+        "partial_count": int(summary["partial_count"]),
+    }
+
+
+def serialize_queue_item(item) -> dict[str, Any]:
+    return {
+        "id": int(item.id),
+        "scholar_profile_id": int(item.scholar_profile_id),
+        "scholar_label": item.scholar_label,
+        "status": item.status,
+        "reason": item.reason,
+        "dropped_reason": item.dropped_reason,
+        "attempt_count": int(item.attempt_count),
+        "resume_cstart": int(item.resume_cstart),
+        "next_attempt_dt": item.next_attempt_dt,
+        "updated_at": item.updated_at,
+        "last_error": item.last_error,
+        "last_run_id": item.last_run_id,
+    }
+
+
+def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]:
+    if not isinstance(value, list):
+        return []
+    normalized: list[dict[str, Any]] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        normalized.append(
+            {
+                "attempt": _int_value(item.get("attempt"), 0),
+                "cstart": _int_value(item.get("cstart"), 0),
+                "state": _str_value(item.get("state")),
+                "state_reason": _str_value(item.get("state_reason")),
+                "status_code": (_int_value(item.get("status_code")) if item.get("status_code") is not None else None),
+                "fetch_error": _str_value(item.get("fetch_error")),
+            }
+        )
+    return normalized
+
+
+def _normalize_page_logs(value: Any) -> list[dict[str, Any]]:
+    if not isinstance(value, list):
+        return []
+    normalized: list[dict[str, Any]] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        warning_codes = item.get("warning_codes")
+        normalized.append(
+            {
+                "page": _int_value(item.get("page"), 0),
+                "cstart": _int_value(item.get("cstart"), 0),
+                "state": _str_value(item.get("state")) or "unknown",
+                "state_reason": _str_value(item.get("state_reason")),
+                "status_code": (_int_value(item.get("status_code")) if item.get("status_code") is not None else None),
+                "publication_count": _int_value(item.get("publication_count"), 0),
+                "attempt_count": _int_value(item.get("attempt_count"), 0),
+                "has_show_more_button": _bool_value(item.get("has_show_more_button"), False),
+                "articles_range": _str_value(item.get("articles_range")),
+                "warning_codes": [str(code) for code in (warning_codes if isinstance(warning_codes, list) else [])],
+            }
+        )
+    return normalized
+
+
+def _normalize_debug(value: Any) -> dict[str, Any] | None:
+    if not isinstance(value, dict):
+        return None
+    marker_counts = value.get("marker_counts_nonzero")
+    warning_codes = value.get("warning_codes")
+    return {
+        "status_code": (_int_value(value.get("status_code")) if value.get("status_code") is not None else None),
+        "final_url": _str_value(value.get("final_url")),
+        "fetch_error": _str_value(value.get("fetch_error")),
+        "body_sha256": _str_value(value.get("body_sha256")),
+        "body_length": (_int_value(value.get("body_length")) if value.get("body_length") is not None else None),
+        "has_show_more_button": (
+            _bool_value(value.get("has_show_more_button"), False)
+            if value.get("has_show_more_button") is not None
+            else None
+        ),
+        "articles_range": _str_value(value.get("articles_range")),
+        "state_reason": _str_value(value.get("state_reason")),
+        "warning_codes": [str(code) for code in (warning_codes if isinstance(warning_codes, list) else [])],
+        "marker_counts_nonzero": {
+            str(key): _int_value(count, 0)
+            for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else [])
+        },
+        "page_logs": _normalize_page_logs(value.get("page_logs")),
+        "attempt_log": _normalize_attempt_log(value.get("attempt_log")),
+    }
+
+
+def normalize_scholar_result(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {
+            "scholar_profile_id": 0,
+            "scholar_id": "unknown",
+            "state": "unknown",
+            "state_reason": None,
+            "outcome": "failed",
+            "attempt_count": 0,
+            "publication_count": 0,
+            "start_cstart": 0,
+            "continuation_cstart": None,
+            "continuation_enqueued": False,
+            "continuation_cleared": False,
+            "warnings": [],
+            "error": None,
+            "debug": None,
+        }
+    warnings = value.get("warnings")
+    return {
+        "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0),
+        "scholar_id": _str_value(value.get("scholar_id")) or "unknown",
+        "state": _str_value(value.get("state")) or "unknown",
+        "state_reason": _str_value(value.get("state_reason")),
+        "outcome": _str_value(value.get("outcome")) or "failed",
+        "attempt_count": _int_value(value.get("attempt_count"), 0),
+        "publication_count": _int_value(value.get("publication_count"), 0),
+        "start_cstart": _int_value(value.get("start_cstart"), 0),
+        "continuation_cstart": (
+            _int_value(value.get("continuation_cstart")) if value.get("continuation_cstart") is not None else None
+        ),
+        "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False),
+        "continuation_cleared": _bool_value(value.get("continuation_cleared"), False),
+        "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])],
+        "error": _str_value(value.get("error")),
+        "debug": _normalize_debug(value.get("debug")),
+    }
+
+
+def manual_run_payload_from_run(
+    run,
+    *,
+    idempotency_key: str | None,
+    reused_existing_run: bool,
+    safety_state: dict[str, Any],
+) -> dict[str, Any]:
+    summary = run_service.extract_run_summary(run.error_log)
+    return {
+        "run_id": int(run.id),
+        "status": run.status.value,
+        "scholar_count": int(run.scholar_count or 0),
+        "succeeded_count": int(summary["succeeded_count"]),
+        "failed_count": int(summary["failed_count"]),
+        "partial_count": int(summary["partial_count"]),
+        "new_publication_count": int(run.new_pub_count or 0),
+        "reused_existing_run": reused_existing_run,
+        "idempotency_key": idempotency_key,
+        "safety_state": safety_state,
+    }
+
+
+def manual_run_success_payload(
+    *,
+    run_summary,
+    idempotency_key: str | None,
+    safety_state: dict[str, Any],
+) -> dict[str, Any]:
+    return {
+        "run_id": run_summary.crawl_run_id,
+        "status": run_summary.status.value,
+        "scholar_count": run_summary.scholar_count,
+        "succeeded_count": run_summary.succeeded_count,
+        "failed_count": run_summary.failed_count,
+        "partial_count": run_summary.partial_count,
+        "new_publication_count": run_summary.new_publication_count,
+        "reused_existing_run": False,
+        "idempotency_key": idempotency_key,
+        "safety_state": safety_state,
+    }

app/api/routers/runs.py

@@ -1,16 +1,33 @@
 from __future__ import annotations
 
+import asyncio
 import logging
-import re
 from typing import Any
 
 from fastapi import APIRouter, Depends, Query, Request
+from fastapi.responses import StreamingResponse
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.api.deps import get_api_current_user
 from app.api.errors import ApiException
 from app.api.responses import success_payload
+from app.api.routers.run_manual import (
+    load_safety_state,
+    raise_manual_blocked_safety,
+    raise_manual_failed,
+    raise_manual_runs_disabled,
+    recover_integrity_error,
+    reused_manual_run_payload,
+)
+from app.api.routers.run_serializers import (
+    IDEMPOTENCY_HEADER,
+    normalize_idempotency_key,
+    normalize_scholar_result,
+    serialize_queue_item,
+    serialize_run,
+)
+from app.api.runtime_deps import get_ingestion_service
 from app.api.schemas import (
     ManualRunEnvelope,
     QueueClearEnvelope,
@@ -20,282 +37,28 @@ from app.api.schemas import (
     RunsListEnvelope,
 )
 from app.db.models import RunStatus, RunTriggerType, User
-from app.db.session import get_db_session
-from app.services import ingestion as ingestion_service
-from app.services import run_safety as run_safety_service
-from app.services import runs as run_service
-from app.services import user_settings as user_settings_service
+from app.db.session import get_db_session, get_session_factory
+from app.logging_utils import structured_log
+from app.services.ingestion import application as ingestion_service
+from app.services.runs import application as run_service
+from app.services.runs.events import event_generator
+from app.services.settings import application as user_settings_service
 from app.settings import settings
-from app.api.runtime_deps import get_ingestion_service
 
 logger = logging.getLogger(__name__)
+_background_tasks: set[asyncio.Task[Any]] = set()
+
+
+def _drop_finished_task(task: asyncio.Task[Any]) -> None:
+    _background_tasks.discard(task)
+    try:
+        task.result()
+    except Exception:
+        structured_log(logger, "exception", "runs.background_task_failed")
+
 
 router = APIRouter(prefix="/runs", tags=["api-runs"])
-
-IDEMPOTENCY_HEADER = "Idempotency-Key"
-IDEMPOTENCY_MAX_LENGTH = 128
-IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
-
-
-def _int_value(value: Any, default: int = 0) -> int:
-    try:
-        return int(value)
-    except (TypeError, ValueError):
-        return default
-
-
-def _str_value(value: Any) -> str | None:
-    if value is None:
-        return None
-    text = str(value).strip()
-    return text if text else None
-
-
-def _bool_value(value: Any, default: bool = False) -> bool:
-    if isinstance(value, bool):
-        return value
-    if isinstance(value, str):
-        lowered = value.strip().lower()
-        if lowered in {"1", "true", "yes", "on"}:
-            return True
-        if lowered in {"0", "false", "no", "off"}:
-            return False
-    return default
-
-
-def _normalize_idempotency_key(raw_value: str | None) -> str | None:
-    if raw_value is None:
-        return None
-    candidate = raw_value.strip()
-    if not candidate:
-        return None
-    if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate):
-        raise ApiException(
-            status_code=400,
-            code="invalid_idempotency_key",
-            message=(
-                "Invalid Idempotency-Key. Use 8-128 characters from: "
-                "A-Z a-z 0-9 . _ : -"
-            ),
-        )
-    return candidate
-
-
-def _serialize_run(run) -> dict[str, Any]:
-    summary = run_service.extract_run_summary(run.error_log)
-    return {
-        "id": int(run.id),
-        "trigger_type": run.trigger_type.value,
-        "status": run.status.value,
-        "start_dt": run.start_dt,
-        "end_dt": run.end_dt,
-        "scholar_count": int(run.scholar_count or 0),
-        "new_publication_count": int(run.new_pub_count or 0),
-        "failed_count": int(summary["failed_count"]),
-        "partial_count": int(summary["partial_count"]),
-    }
-
-
-def _serialize_queue_item(item) -> dict[str, Any]:
-    return {
-        "id": int(item.id),
-        "scholar_profile_id": int(item.scholar_profile_id),
-        "scholar_label": item.scholar_label,
-        "status": item.status,
-        "reason": item.reason,
-        "dropped_reason": item.dropped_reason,
-        "attempt_count": int(item.attempt_count),
-        "resume_cstart": int(item.resume_cstart),
-        "next_attempt_dt": item.next_attempt_dt,
-        "updated_at": item.updated_at,
-        "last_error": item.last_error,
-        "last_run_id": item.last_run_id,
-    }
-
-
-def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]:
-    if not isinstance(value, list):
-        return []
-    normalized: list[dict[str, Any]] = []
-    for item in value:
-        if not isinstance(item, dict):
-            continue
-        normalized.append(
-            {
-                "attempt": _int_value(item.get("attempt"), 0),
-                "cstart": _int_value(item.get("cstart"), 0),
-                "state": _str_value(item.get("state")),
-                "state_reason": _str_value(item.get("state_reason")),
-                "status_code": (
-                    _int_value(item.get("status_code"))
-                    if item.get("status_code") is not None
-                    else None
-                ),
-                "fetch_error": _str_value(item.get("fetch_error")),
-            }
-        )
-    return normalized
-
-
-def _normalize_page_logs(value: Any) -> list[dict[str, Any]]:
-    if not isinstance(value, list):
-        return []
-    normalized: list[dict[str, Any]] = []
-    for item in value:
-        if not isinstance(item, dict):
-            continue
-        warning_codes = item.get("warning_codes")
-        normalized.append(
-            {
-                "page": _int_value(item.get("page"), 0),
-                "cstart": _int_value(item.get("cstart"), 0),
-                "state": _str_value(item.get("state")) or "unknown",
-                "state_reason": _str_value(item.get("state_reason")),
-                "status_code": (
-                    _int_value(item.get("status_code"))
-                    if item.get("status_code") is not None
-                    else None
-                ),
-                "publication_count": _int_value(item.get("publication_count"), 0),
-                "attempt_count": _int_value(item.get("attempt_count"), 0),
-                "has_show_more_button": _bool_value(item.get("has_show_more_button"), False),
-                "articles_range": _str_value(item.get("articles_range")),
-                "warning_codes": [
-                    str(code)
-                    for code in (warning_codes if isinstance(warning_codes, list) else [])
-                ],
-            }
-        )
-    return normalized
-
-
-def _normalize_debug(value: Any) -> dict[str, Any] | None:
-    if not isinstance(value, dict):
-        return None
-    marker_counts = value.get("marker_counts_nonzero")
-    warning_codes = value.get("warning_codes")
-    return {
-        "status_code": (
-            _int_value(value.get("status_code"))
-            if value.get("status_code") is not None
-            else None
-        ),
-        "final_url": _str_value(value.get("final_url")),
-        "fetch_error": _str_value(value.get("fetch_error")),
-        "body_sha256": _str_value(value.get("body_sha256")),
-        "body_length": (
-            _int_value(value.get("body_length"))
-            if value.get("body_length") is not None
-            else None
-        ),
-        "has_show_more_button": (
-            _bool_value(value.get("has_show_more_button"), False)
-            if value.get("has_show_more_button") is not None
-            else None
-        ),
-        "articles_range": _str_value(value.get("articles_range")),
-        "state_reason": _str_value(value.get("state_reason")),
-        "warning_codes": [
-            str(code)
-            for code in (warning_codes if isinstance(warning_codes, list) else [])
-        ],
-        "marker_counts_nonzero": {
-            str(key): _int_value(count, 0)
-            for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else [])
-        },
-        "page_logs": _normalize_page_logs(value.get("page_logs")),
-        "attempt_log": _normalize_attempt_log(value.get("attempt_log")),
-    }
-
-
-def _normalize_scholar_result(value: Any) -> dict[str, Any]:
-    if not isinstance(value, dict):
-        return {
-            "scholar_profile_id": 0,
-            "scholar_id": "unknown",
-            "state": "unknown",
-            "state_reason": None,
-            "outcome": "failed",
-            "attempt_count": 0,
-            "publication_count": 0,
-            "start_cstart": 0,
-            "continuation_cstart": None,
-            "continuation_enqueued": False,
-            "continuation_cleared": False,
-            "warnings": [],
-            "error": None,
-            "debug": None,
-        }
-    warnings = value.get("warnings")
-    return {
-        "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0),
-        "scholar_id": _str_value(value.get("scholar_id")) or "unknown",
-        "state": _str_value(value.get("state")) or "unknown",
-        "state_reason": _str_value(value.get("state_reason")),
-        "outcome": _str_value(value.get("outcome")) or "failed",
-        "attempt_count": _int_value(value.get("attempt_count"), 0),
-        "publication_count": _int_value(value.get("publication_count"), 0),
-        "start_cstart": _int_value(value.get("start_cstart"), 0),
-        "continuation_cstart": (
-            _int_value(value.get("continuation_cstart"))
-            if value.get("continuation_cstart") is not None
-            else None
-        ),
-        "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False),
-        "continuation_cleared": _bool_value(value.get("continuation_cleared"), False),
-        "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])],
-        "error": _str_value(value.get("error")),
-        "debug": _normalize_debug(value.get("debug")),
-    }
-
-
-def _manual_run_payload_from_run(
-    run,
-    *,
-    idempotency_key: str | None,
-    reused_existing_run: bool,
-    safety_state: dict[str, Any],
-) -> dict[str, Any]:
-    summary = run_service.extract_run_summary(run.error_log)
-    return {
-        "run_id": int(run.id),
-        "status": run.status.value,
-        "scholar_count": int(run.scholar_count or 0),
-        "succeeded_count": int(summary["succeeded_count"]),
-        "failed_count": int(summary["failed_count"]),
-        "partial_count": int(summary["partial_count"]),
-        "new_publication_count": int(run.new_pub_count or 0),
-        "reused_existing_run": reused_existing_run,
-        "idempotency_key": idempotency_key,
-        "safety_state": safety_state,
-    }
-
-
-async def _load_safety_state(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-) -> dict[str, Any]:
-    user_settings = await user_settings_service.get_or_create_settings(
-        db_session,
-        user_id=user_id,
-    )
-    previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
-    if run_safety_service.clear_expired_cooldown(user_settings):
-        await db_session.commit()
-        await db_session.refresh(user_settings)
-        logger.info(
-            "api.runs.safety_cooldown_cleared",
-            extra={
-                "event": "api.runs.safety_cooldown_cleared",
-                "user_id": user_id,
-                "reason": previous_safety_state.get("cooldown_reason"),
-                "cooldown_until": previous_safety_state.get("cooldown_until"),
-                "metric_name": "api_runs_safety_cooldown_cleared_total",
-                "metric_value": 1,
-            },
-        )
-    return run_safety_service.get_safety_state_payload(user_settings)
+ACTIVE_RUN_STATUSES = {RunStatus.RUNNING, RunStatus.RESOLVING}
 
 
 @router.get(
@@ -315,14 +78,14 @@ async def list_runs(
         limit=limit,
         failed_only=failed_only,
     )
-    safety_state = await _load_safety_state(
+    safety_state = await load_safety_state(
         db_session,
         user_id=current_user.id,
     )
     return success_payload(
         request,
         data={
-            "runs": [_serialize_run(run) for run in runs],
+            "runs": [serialize_run(run) for run in runs],
             "safety_state": safety_state,
         },
     )
@@ -353,21 +116,137 @@ async def get_run(
     scholar_results = error_log.get("scholar_results")
     if not isinstance(scholar_results, list):
         scholar_results = []
-    safety_state = await _load_safety_state(
+    safety_state = await load_safety_state(
         db_session,
         user_id=current_user.id,
     )
     return success_payload(
         request,
         data={
-            "run": _serialize_run(run),
+            "run": serialize_run(run),
             "summary": run_service.extract_run_summary(error_log),
-            "scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
+            "scholar_results": [normalize_scholar_result(item) for item in scholar_results],
             "safety_state": safety_state,
         },
     )
 
 
+@router.post(
+    "/{run_id}/cancel",
+    response_model=RunDetailEnvelope,
+)
+async def cancel_run(
+    run_id: int,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    run = await run_service.get_run_for_user(
+        db_session,
+        user_id=current_user.id,
+        run_id=run_id,
+    )
+    if run is None:
+        raise ApiException(
+            status_code=404,
+            code="run_not_found",
+            message="Run not found.",
+        )
+
+    if run.status in ACTIVE_RUN_STATUSES:
+        run.status = RunStatus.CANCELED
+        await db_session.commit()
+        await db_session.refresh(run)
+    else:
+        raise ApiException(
+            status_code=409,
+            code="run_not_cancelable",
+            message="Run is already terminal and cannot be canceled.",
+            details={"run_id": int(run.id), "status": run.status.value},
+        )
+
+    error_log = run.error_log if isinstance(run.error_log, dict) else {}
+    scholar_results = error_log.get("scholar_results")
+    if not isinstance(scholar_results, list):
+        scholar_results = []
+
+    safety_state = await load_safety_state(
+        db_session,
+        user_id=current_user.id,
+    )
+
+    return success_payload(
+        request,
+        data={
+            "run": serialize_run(run),
+            "summary": run_service.extract_run_summary(error_log),
+            "scholar_results": [normalize_scholar_result(item) for item in scholar_results],
+            "safety_state": safety_state,
+        },
+    )
+
+
+async def _start_manual_run(
+    db_session: AsyncSession,
+    *,
+    current_user: User,
+    ingest_service: ingestion_service.ScholarIngestionService,
+    idempotency_key: str | None,
+) -> tuple[Any, Any, list, dict]:
+    user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=current_user.id)
+    run, scholars, start_cstart_map = await ingest_service.initialize_run(
+        db_session,
+        user_id=current_user.id,
+        trigger_type=RunTriggerType.MANUAL,
+        request_delay_seconds=user_settings.request_delay_seconds,
+        network_error_retries=settings.ingestion_network_error_retries,
+        retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+        max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
+        page_size=settings.ingestion_page_size,
+        idempotency_key=idempotency_key,
+        alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
+        alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
+        alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
+    )
+    return user_settings, run, scholars, start_cstart_map
+
+
+def _spawn_background_execution(
+    ingest_service: ingestion_service.ScholarIngestionService,
+    *,
+    run: Any,
+    current_user: User,
+    scholars: list,
+    start_cstart_map: dict,
+    user_settings: Any,
+    idempotency_key: str | None,
+) -> None:
+    from app.db.background_session import background_session
+
+    task = asyncio.create_task(
+        ingest_service.execute_run(
+            session_factory=background_session,
+            run_id=run.id,
+            user_id=current_user.id,
+            scholars=scholars,
+            start_cstart_map=start_cstart_map,
+            request_delay_seconds=user_settings.request_delay_seconds,
+            network_error_retries=settings.ingestion_network_error_retries,
+            retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+            max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
+            page_size=settings.ingestion_page_size,
+            auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
+            queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
+            alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
+            alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
+            alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
+            idempotency_key=idempotency_key,
+        )
+    )
+    _background_tasks.add(task)
+    task.add_done_callback(_drop_finished_task)
+
+
 @router.post(
     "/manual",
     response_model=ManualRunEnvelope,
@@ -378,83 +257,57 @@ async def run_manual(
     current_user: User = Depends(get_api_current_user),
     ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
 ):
-    safety_state = await _load_safety_state(
-        db_session,
-        user_id=current_user.id,
-    )
+    user_id = int(current_user.id)
+    safety_state = await load_safety_state(db_session, user_id=user_id)
     if not settings.ingestion_manual_run_allowed:
-        logger.warning(
-            "api.runs.manual_blocked_policy",
-            extra={
-                "event": "api.runs.manual_blocked_policy",
-                "user_id": current_user.id,
-                "policy": {"manual_run_allowed": False},
-                "safety_state": safety_state,
-                "metric_name": "api_runs_manual_blocked_policy_total",
-                "metric_value": 1,
-            },
-        )
-        raise ApiException(
-            status_code=403,
-            code="manual_runs_disabled",
-            message="Manual checks are disabled by server policy.",
-            details={
-                "policy": {
-                    "manual_run_allowed": False,
-                },
-                "safety_state": safety_state,
-            },
-        )
+        raise_manual_runs_disabled(user_id=user_id, safety_state=safety_state)
 
-    idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
-    if idempotency_key is not None:
-        previous_run = await run_service.get_manual_run_by_idempotency_key(
-            db_session,
-            user_id=current_user.id,
-            idempotency_key=idempotency_key,
-        )
-        if previous_run is not None:
-            if previous_run.status == RunStatus.RUNNING:
-                raise ApiException(
-                    status_code=409,
-                    code="run_in_progress",
-                    message="A run with this idempotency key is still in progress.",
-                    details={
-                        "run_id": int(previous_run.id),
-                        "idempotency_key": idempotency_key,
-                    },
-                )
-            return success_payload(
-                request,
-                data=_manual_run_payload_from_run(
-                    previous_run,
-                    idempotency_key=idempotency_key,
-                    reused_existing_run=True,
-                    safety_state=safety_state,
-                ),
-            )
-
-    user_settings = await user_settings_service.get_or_create_settings(
+    idempotency_key = normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
+    reused_payload = await reused_manual_run_payload(
         db_session,
-        user_id=current_user.id,
+        request=request,
+        user_id=user_id,
+        idempotency_key=idempotency_key,
+        safety_state=safety_state,
     )
+    if reused_payload is not None:
+        return reused_payload
+
     try:
-        run_summary = await ingest_service.run_for_user(
+        user_settings, run, scholars, start_cstart_map = await _start_manual_run(
             db_session,
-            user_id=current_user.id,
-            trigger_type=RunTriggerType.MANUAL,
-            request_delay_seconds=user_settings.request_delay_seconds,
-            network_error_retries=settings.ingestion_network_error_retries,
-            retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
-            max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
-            page_size=settings.ingestion_page_size,
-            auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
-            queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
+            current_user=current_user,
+            ingest_service=ingest_service,
             idempotency_key=idempotency_key,
-            alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
-            alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
-            alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
         )
+        await db_session.commit()
+        _spawn_background_execution(
+            ingest_service,
+            run=run,
+            current_user=current_user,
+            scholars=scholars,
+            start_cstart_map=start_cstart_map,
+            user_settings=user_settings,
+            idempotency_key=idempotency_key,
+        )
+        return success_payload(
+            request,
+            data={
+                "run_id": int(run.id),
+                "status": run.status.value,
+                "scholar_count": int(run.scholar_count or 0),
+                "succeeded_count": 0,
+                "failed_count": 0,
+                "partial_count": 0,
+                "new_publication_count": 0,
+                "reused_existing_run": False,
+                "idempotency_key": idempotency_key,
+                "safety_state": await load_safety_state(db_session, user_id=user_id),
+            },
+        )
+    except ingestion_service.RunBlockedBySafetyPolicyError as exc:
+        await db_session.rollback()
+        raise_manual_blocked_safety(exc=exc, user_id=user_id)
     except ingestion_service.RunAlreadyInProgressError as exc:
         await db_session.rollback()
         raise ApiException(
@@ -462,105 +315,18 @@ async def run_manual(
             code="run_in_progress",
             message="A run is already in progress for this account.",
         ) from exc
-    except ingestion_service.RunBlockedBySafetyPolicyError as exc:
-        await db_session.rollback()
-        logger.info(
-            "api.runs.manual_blocked_safety",
-            extra={
-                "event": "api.runs.manual_blocked_safety",
-                "user_id": current_user.id,
-                "reason": exc.safety_state.get("cooldown_reason"),
-                "cooldown_until": exc.safety_state.get("cooldown_until"),
-                "cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"),
-                "metric_name": "api_runs_manual_blocked_safety_total",
-                "metric_value": 1,
-            },
-        )
-        raise ApiException(
-            status_code=429,
-            code=exc.code,
-            message=exc.message,
-            details={
-                "safety_state": exc.safety_state,
-            },
-        ) from exc
     except IntegrityError as exc:
         await db_session.rollback()
-        if idempotency_key is not None:
-            existing_run = await run_service.get_manual_run_by_idempotency_key(
-                db_session,
-                user_id=current_user.id,
-                idempotency_key=idempotency_key,
-            )
-            if existing_run is not None:
-                if existing_run.status == RunStatus.RUNNING:
-                    raise ApiException(
-                        status_code=409,
-                        code="run_in_progress",
-                        message="A run with this idempotency key is still in progress.",
-                        details={
-                            "run_id": int(existing_run.id),
-                            "idempotency_key": idempotency_key,
-                        },
-                    ) from exc
-                return success_payload(
-                    request,
-                    data=_manual_run_payload_from_run(
-                        existing_run,
-                        idempotency_key=idempotency_key,
-                        reused_existing_run=True,
-                        safety_state=await _load_safety_state(
-                            db_session,
-                            user_id=current_user.id,
-                        ),
-                    ),
-                )
-        logger.exception(
-            "api.runs.manual_integrity_error",
-            extra={
-                "event": "api.runs.manual_integrity_error",
-                "user_id": current_user.id,
-            },
+        return await recover_integrity_error(
+            db_session,
+            request=request,
+            user_id=user_id,
+            idempotency_key=idempotency_key,
+            original_exc=exc,
         )
-        raise ApiException(
-            status_code=500,
-            code="manual_run_failed",
-            message="Manual run failed.",
-        ) from exc
     except Exception as exc:
         await db_session.rollback()
-        logger.exception(
-            "api.runs.manual_failed",
-            extra={
-                "event": "api.runs.manual_failed",
-                "user_id": current_user.id,
-            },
-        )
-        raise ApiException(
-            status_code=500,
-            code="manual_run_failed",
-            message="Manual run failed.",
-        ) from exc
-
-    current_safety_state = await _load_safety_state(
-        db_session,
-        user_id=current_user.id,
-    )
-    return success_payload(
-        request,
-        data={
-            "run_id": run_summary.crawl_run_id,
-            "status": run_summary.status.value,
-            "scholar_count": run_summary.scholar_count,
-            "succeeded_count": run_summary.succeeded_count,
-            "failed_count": run_summary.failed_count,
-            "partial_count": run_summary.partial_count,
-            "new_publication_count": run_summary.new_publication_count,
-            "reused_existing_run": False,
-            "idempotency_key": idempotency_key,
-            "safety_state": current_safety_state,
-        },
-    )
+        raise_manual_failed(exc=exc, user_id=user_id)
 
 
 @router.get(
@@ -581,7 +347,7 @@ async def list_queue_items(
     return success_payload(
         request,
         data={
-            "queue_items": [_serialize_queue_item(item) for item in items],
+            "queue_items": [serialize_queue_item(item) for item in items],
         },
     )
 
@@ -617,7 +383,7 @@ async def retry_queue_item(
         )
     return success_payload(
         request,
-        data=_serialize_queue_item(queue_item),
+        data=serialize_queue_item(queue_item),
     )
 
 
@@ -652,7 +418,7 @@ async def drop_queue_item(
         )
     return success_payload(
         request,
-        data=_serialize_queue_item(dropped),
+        data=serialize_queue_item(dropped),
     )
 
 
@@ -694,3 +460,24 @@ async def clear_queue_item(
             "message": "Queue item cleared.",
         },
     )
+
+
+@router.get("/{run_id}/stream")
+async def stream_run_events(
+    run_id: int,
+    current_user: User = Depends(get_api_current_user),
+):
+    session_factory = get_session_factory()
+    async with session_factory() as db_session:
+        run = await run_service.get_run_for_user(
+            db_session,
+            user_id=current_user.id,
+            run_id=run_id,
+        )
+    if run is None:
+        raise ApiException(
+            status_code=404,
+            code="run_not_found",
+            message="Run not found.",
+        )
+    return StreamingResponse(event_generator(run_id), media_type="text/event-stream")

app/api/routers/scholar_helpers.py

@@ -0,0 +1,234 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+from fastapi import UploadFile
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import ApiException
+from app.logging_utils import structured_log
+from app.services.ingestion import queue as ingestion_queue_service
+from app.services.scholar import rate_limit as scholar_rate_limit
+from app.services.scholar.source import ScholarSource
+from app.services.scholars import application as scholar_service
+from app.services.scholars import search_hints as scholar_search_hints
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+CREATE_METADATA_HYDRATION_TIMEOUT_SECONDS = 5.0
+INITIAL_SCHOLAR_SCRAPE_QUEUE_DELAY_SECONDS = 0
+INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON = "scholar_added_initial_scrape"
+
+
+def needs_metadata_hydration(profile) -> bool:
+    if not profile.profile_image_url:
+        return True
+    return not (profile.display_name or "").strip()
+
+
+def is_create_hydration_rate_limited() -> tuple[bool, float]:
+    remaining_seconds = scholar_rate_limit.remaining_scholar_slot_seconds(
+        min_interval_seconds=float(settings.ingestion_min_request_delay_seconds),
+    )
+    return remaining_seconds > 0, remaining_seconds
+
+
+def auto_enqueue_new_scholar_enabled() -> bool:
+    if not settings.scheduler_enabled:
+        return False
+    if not settings.ingestion_automation_allowed:
+        return False
+    return bool(settings.ingestion_continuation_queue_enabled)
+
+
+async def enqueue_initial_scrape_job_for_scholar(
+    db_session: AsyncSession,
+    *,
+    profile,
+    user_id: int,
+) -> bool:
+    if not auto_enqueue_new_scholar_enabled():
+        return False
+    try:
+        await ingestion_queue_service.upsert_job(
+            db_session,
+            user_id=user_id,
+            scholar_profile_id=int(profile.id),
+            resume_cstart=0,
+            reason=INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON,
+            run_id=None,
+            delay_seconds=INITIAL_SCHOLAR_SCRAPE_QUEUE_DELAY_SECONDS,
+        )
+        await db_session.commit()
+    except Exception:
+        await db_session.rollback()
+        structured_log(
+            logger,
+            "warning",
+            "api.scholars.initial_scrape_enqueue_failed",
+            user_id=user_id,
+            scholar_profile_id=profile.id,
+        )
+        return False
+
+    structured_log(
+        logger,
+        "info",
+        "api.scholars.initial_scrape_enqueued",
+        user_id=user_id,
+        scholar_profile_id=profile.id,
+        reason=INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON,
+    )
+    return True
+
+
+def uploaded_image_media_path(scholar_profile_id: int) -> str:
+    return f"/scholar-images/{scholar_profile_id}/upload"
+
+
+def serialize_scholar(profile) -> dict[str, object]:
+    uploaded_image_url = None
+    if profile.profile_image_upload_path:
+        uploaded_image_url = uploaded_image_media_path(int(profile.id))
+
+    profile_image_url, profile_image_source = scholar_search_hints.resolve_profile_image(
+        profile,
+        uploaded_image_url=uploaded_image_url,
+    )
+
+    return {
+        "id": int(profile.id),
+        "scholar_id": profile.scholar_id,
+        "display_name": profile.display_name,
+        "profile_image_url": profile_image_url,
+        "profile_image_source": profile_image_source,
+        "is_enabled": bool(profile.is_enabled),
+        "baseline_completed": bool(profile.baseline_completed),
+        "last_run_dt": profile.last_run_dt,
+        "last_run_status": (profile.last_run_status.value if profile.last_run_status is not None else None),
+    }
+
+
+async def hydrate_scholar_metadata_if_needed(
+    db_session: AsyncSession,
+    *,
+    profile,
+    source: ScholarSource,
+    user_id: int,
+):
+    if not needs_metadata_hydration(profile):
+        return profile
+
+    should_skip, remaining_seconds = is_create_hydration_rate_limited()
+    if should_skip:
+        structured_log(
+            logger,
+            "info",
+            "api.scholars.create_metadata_hydration_skipped",
+            reason="scholar_request_throttle_active",
+            user_id=user_id,
+            scholar_profile_id=profile.id,
+            retry_after_seconds=round(remaining_seconds, 3),
+        )
+        return profile
+
+    try:
+        return await asyncio.wait_for(
+            scholar_service.hydrate_profile_metadata(
+                db_session,
+                profile=profile,
+                source=source,
+            ),
+            timeout=CREATE_METADATA_HYDRATION_TIMEOUT_SECONDS,
+        )
+    except TimeoutError:
+        structured_log(
+            logger,
+            "info",
+            "api.scholars.create_metadata_hydration_skipped",
+            reason="create_timeout",
+            user_id=user_id,
+            scholar_profile_id=profile.id,
+        )
+    except Exception:
+        structured_log(
+            logger,
+            "warning",
+            "api.scholars.create_metadata_hydration_failed",
+            user_id=user_id,
+            scholar_profile_id=profile.id,
+        )
+    return profile
+
+
+def search_kwargs() -> dict[str, Any]:
+    return {
+        "network_error_retries": settings.ingestion_network_error_retries,
+        "retry_backoff_seconds": settings.ingestion_retry_backoff_seconds,
+        "search_enabled": settings.scholar_name_search_enabled,
+        "cache_ttl_seconds": settings.scholar_name_search_cache_ttl_seconds,
+        "blocked_cache_ttl_seconds": settings.scholar_name_search_blocked_cache_ttl_seconds,
+        "cache_max_entries": settings.scholar_name_search_cache_max_entries,
+        "min_interval_seconds": settings.scholar_name_search_min_interval_seconds,
+        "interval_jitter_seconds": settings.scholar_name_search_interval_jitter_seconds,
+        "cooldown_block_threshold": settings.scholar_name_search_cooldown_block_threshold,
+        "cooldown_seconds": settings.scholar_name_search_cooldown_seconds,
+        "retry_alert_threshold": settings.scholar_name_search_alert_retry_count_threshold,
+        "cooldown_rejection_alert_threshold": (settings.scholar_name_search_alert_cooldown_rejections_threshold),
+    }
+
+
+def search_response_data(query: str, parsed) -> dict[str, object]:
+    return {
+        "query": query.strip(),
+        "state": parsed.state.value,
+        "state_reason": parsed.state_reason,
+        "action_hint": scholar_search_hints.scrape_state_hint(
+            state=parsed.state,
+            state_reason=parsed.state_reason,
+        ),
+        "candidates": [
+            {
+                "scholar_id": item.scholar_id,
+                "display_name": item.display_name,
+                "affiliation": item.affiliation,
+                "email_domain": item.email_domain,
+                "cited_by_count": item.cited_by_count,
+                "interests": item.interests,
+                "profile_url": item.profile_url,
+                "profile_image_url": item.profile_image_url,
+            }
+            for item in parsed.candidates
+        ],
+        "warnings": parsed.warnings,
+    }
+
+
+async def read_uploaded_image(image: UploadFile) -> bytes:
+    try:
+        return await image.read()
+    finally:
+        await image.close()
+
+
+async def require_user_profile(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+):
+    profile = await scholar_service.get_user_scholar_by_id(
+        db_session,
+        user_id=user_id,
+        scholar_profile_id=scholar_profile_id,
+    )
+    if profile is None:
+        raise ApiException(
+            status_code=404,
+            code="scholar_not_found",
+            message="Scholar not found.",
+        )
+    return profile

app/api/routers/scholars.py

@@ -1,19 +1,31 @@
 from __future__ import annotations
 
-import asyncio
 import logging
-import mimetypes
 
 from fastapi import APIRouter, Depends, File, Query, Request, UploadFile
-from fastapi.responses import FileResponse
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.api.deps import get_api_current_user
 from app.api.errors import ApiException
 from app.api.responses import success_payload
+from app.api.routers.scholar_helpers import (
+    enqueue_initial_scrape_job_for_scholar,
+    hydrate_scholar_metadata_if_needed,
+    read_uploaded_image,
+    require_user_profile,
+    search_kwargs,
+    search_response_data,
+    serialize_scholar,
+)
 from app.api.runtime_deps import get_scholar_source
 from app.api.schemas import (
+    DataExportEnvelope,
+    DataImportEnvelope,
+    DataImportRequest,
     MessageEnvelope,
+    ScholarBulkCountEnvelope,
+    ScholarBulkIdsRequest,
+    ScholarBulkToggleRequest,
     ScholarCreateRequest,
     ScholarEnvelope,
     ScholarImageUrlUpdateRequest,
@@ -22,8 +34,10 @@ from app.api.schemas import (
 )
 from app.db.models import User
 from app.db.session import get_db_session
-from app.services import scholars as scholar_service
-from app.services.scholar_source import ScholarSource
+from app.logging_utils import structured_log
+from app.services.portability import application as import_export_service
+from app.services.scholar.source import ScholarSource
+from app.services.scholars import application as scholar_service
 from app.settings import settings
 
 logger = logging.getLogger(__name__)
@@ -31,33 +45,20 @@ logger = logging.getLogger(__name__)
 router = APIRouter(prefix="/scholars", tags=["api-scholars"])
 
 
-def _uploaded_image_api_path(scholar_profile_id: int) -> str:
-    return f"/api/v1/scholars/{scholar_profile_id}/image/upload"
-
-
-def _serialize_scholar(profile) -> dict[str, object]:
-    uploaded_image_url = None
-    if profile.profile_image_upload_path:
-        uploaded_image_url = _uploaded_image_api_path(int(profile.id))
-
-    profile_image_url, profile_image_source = scholar_service.resolve_profile_image(
-        profile,
-        uploaded_image_url=uploaded_image_url,
-    )
-
-    return {
-        "id": int(profile.id),
-        "scholar_id": profile.scholar_id,
-        "display_name": profile.display_name,
-        "profile_image_url": profile_image_url,
-        "profile_image_source": profile_image_source,
-        "is_enabled": bool(profile.is_enabled),
-        "baseline_completed": bool(profile.baseline_completed),
-        "last_run_dt": profile.last_run_dt,
-        "last_run_status": (
-            profile.last_run_status.value if profile.last_run_status is not None else None
-        ),
-    }
+def _parse_ids_param(ids: str | None) -> list[int] | None:
+    if not ids:
+        return None
+    parts = [p.strip() for p in ids.split(",") if p.strip()]
+    if not parts:
+        return None
+    try:
+        return [int(p) for p in parts]
+    except ValueError as exc:
+        raise ApiException(
+            status_code=400,
+            code="invalid_ids_param",
+            message="The 'ids' parameter must be a comma-separated list of integers.",
+        ) from exc
 
 
 @router.get(
@@ -76,11 +77,130 @@ async def list_scholars(
     return success_payload(
         request,
         data={
-            "scholars": [_serialize_scholar(profile) for profile in scholars],
+            "scholars": [serialize_scholar(profile) for profile in scholars],
         },
     )
 
 
+@router.get(
+    "/export",
+    response_model=DataExportEnvelope,
+)
+async def export_scholars_and_publications(
+    request: Request,
+    ids: str | None = Query(None, description="Comma-separated scholar profile IDs to export"),
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    scholar_profile_ids = _parse_ids_param(ids)
+    data = await import_export_service.export_user_data(
+        db_session,
+        user_id=current_user.id,
+        scholar_profile_ids=scholar_profile_ids,
+    )
+    return success_payload(request, data=data)
+
+
+@router.post(
+    "/bulk-delete",
+    response_model=ScholarBulkCountEnvelope,
+)
+async def bulk_delete_scholars(
+    payload: ScholarBulkIdsRequest,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    try:
+        deleted_count = await scholar_service.bulk_delete_scholars(
+            db_session,
+            user_id=current_user.id,
+            scholar_profile_ids=payload.scholar_profile_ids,
+            upload_dir=settings.scholar_image_upload_dir,
+        )
+    except scholar_service.ScholarServiceError as exc:
+        raise ApiException(
+            status_code=409,
+            code="scholar_bulk_delete_failed",
+            message=str(exc),
+        ) from exc
+    structured_log(
+        logger,
+        "info",
+        "scholars.bulk_delete",
+        user_id=current_user.id,
+        requested_ids=payload.scholar_profile_ids,
+        deleted_count=deleted_count,
+    )
+    return success_payload(request, data={"deleted_count": deleted_count, "updated_count": 0})
+
+
+@router.post(
+    "/bulk-toggle",
+    response_model=ScholarBulkCountEnvelope,
+)
+async def bulk_toggle_scholars(
+    payload: ScholarBulkToggleRequest,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    updated_count = await scholar_service.bulk_toggle_scholars(
+        db_session,
+        user_id=current_user.id,
+        scholar_profile_ids=payload.scholar_profile_ids,
+        is_enabled=payload.is_enabled,
+    )
+    structured_log(
+        logger,
+        "info",
+        "scholars.bulk_toggle",
+        user_id=current_user.id,
+        requested_ids=payload.scholar_profile_ids,
+        is_enabled=payload.is_enabled,
+        updated_count=updated_count,
+    )
+    return success_payload(request, data={"deleted_count": 0, "updated_count": updated_count})
+
+
+@router.post(
+    "/import",
+    response_model=DataImportEnvelope,
+)
+async def import_scholars_and_publications(
+    payload: DataImportRequest,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    if (
+        payload.schema_version is not None
+        and int(payload.schema_version) != import_export_service.EXPORT_SCHEMA_VERSION
+    ):
+        raise ApiException(
+            status_code=400,
+            code="invalid_import_schema_version",
+            message=(
+                f"Import schema version is not supported. Expected {import_export_service.EXPORT_SCHEMA_VERSION}."
+            ),
+        )
+    try:
+        result = await import_export_service.import_user_data(
+            db_session,
+            user_id=current_user.id,
+            scholars=[item.model_dump() for item in payload.scholars],
+            publications=[item.model_dump() for item in payload.publications],
+        )
+    except import_export_service.ImportExportError as exc:
+        raise ApiException(
+            status_code=400,
+            code="invalid_import_payload",
+            message=str(exc),
+        ) from exc
+    structured_log(logger, "info", "api.scholars.imported", user_id=current_user.id, **result)
+    return success_payload(request, data=result)
+
+
 @router.post(
     "",
     response_model=ScholarEnvelope,
@@ -107,37 +227,22 @@ async def create_scholar(
             code="invalid_scholar",
             message=str(exc),
         ) from exc
-    logger.info(
-        "api.scholars.created",
-        extra={
-            "event": "api.scholars.created",
-            "user_id": current_user.id,
-            "scholar_profile_id": created.id,
-        },
+    structured_log(logger, "info", "api.scholars.created", user_id=current_user.id, scholar_profile_id=created.id)
+    await enqueue_initial_scrape_job_for_scholar(
+        db_session,
+        profile=created,
+        user_id=current_user.id,
+    )
+    created = await hydrate_scholar_metadata_if_needed(
+        db_session,
+        profile=created,
+        source=source,
+        user_id=current_user.id,
     )
-    try:
-        if not created.profile_image_url or not (created.display_name or "").strip():
-            created = await asyncio.wait_for(
-                scholar_service.hydrate_profile_metadata(
-                    db_session,
-                    profile=created,
-                    source=source,
-                ),
-                timeout=5.0,
-            )
-    except Exception:
-        logger.warning(
-            "api.scholars.create_metadata_hydration_failed",
-            extra={
-                "event": "api.scholars.create_metadata_hydration_failed",
-                "user_id": current_user.id,
-                "scholar_profile_id": created.id,
-            },
-        )
 
     return success_payload(
         request,
-        data=_serialize_scholar(created),
+        data=serialize_scholar(created),
     )
 
 
@@ -149,28 +254,17 @@ async def search_scholars(
     request: Request,
     query: str = Query(..., min_length=2, max_length=120),
     limit: int = Query(10, ge=1, le=25),
+    db_session: AsyncSession = Depends(get_db_session),
     source: ScholarSource = Depends(get_scholar_source),
     current_user: User = Depends(get_api_current_user),
 ):
     try:
         parsed = await scholar_service.search_author_candidates(
             source=source,
+            db_session=db_session,
             query=query,
             limit=limit,
-            network_error_retries=settings.ingestion_network_error_retries,
-            retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
-            search_enabled=settings.scholar_name_search_enabled,
-            cache_ttl_seconds=settings.scholar_name_search_cache_ttl_seconds,
-            blocked_cache_ttl_seconds=settings.scholar_name_search_blocked_cache_ttl_seconds,
-            cache_max_entries=settings.scholar_name_search_cache_max_entries,
-            min_interval_seconds=settings.scholar_name_search_min_interval_seconds,
-            interval_jitter_seconds=settings.scholar_name_search_interval_jitter_seconds,
-            cooldown_block_threshold=settings.scholar_name_search_cooldown_block_threshold,
-            cooldown_seconds=settings.scholar_name_search_cooldown_seconds,
-            retry_alert_threshold=settings.scholar_name_search_alert_retry_count_threshold,
-            cooldown_rejection_alert_threshold=(
-                settings.scholar_name_search_alert_cooldown_rejections_threshold
-            ),
+            **search_kwargs(),
         )
     except scholar_service.ScholarServiceError as exc:
         raise ApiException(
@@ -179,41 +273,18 @@ async def search_scholars(
             message=str(exc),
         ) from exc
 
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.scholars.search.completed",
-        extra={
-            "event": "api.scholars.search.completed",
-            "user_id": current_user.id,
-            "query": query.strip(),
-            "candidate_count": len(parsed.candidates),
-            "state": parsed.state.value,
-        },
+        user_id=current_user.id,
+        query=query.strip(),
+        candidate_count=len(parsed.candidates),
+        state=parsed.state.value,
     )
     return success_payload(
         request,
-        data={
-            "query": query.strip(),
-            "state": parsed.state.value,
-            "state_reason": parsed.state_reason,
-            "action_hint": scholar_service.scrape_state_hint(
-                state=parsed.state,
-                state_reason=parsed.state_reason,
-            ),
-            "candidates": [
-                {
-                    "scholar_id": item.scholar_id,
-                    "display_name": item.display_name,
-                    "affiliation": item.affiliation,
-                    "email_domain": item.email_domain,
-                    "cited_by_count": item.cited_by_count,
-                    "interests": item.interests,
-                    "profile_url": item.profile_url,
-                    "profile_image_url": item.profile_image_url,
-                }
-                for item in parsed.candidates
-            ],
-            "warnings": parsed.warnings,
-        },
+        data=search_response_data(query, parsed),
     )
 
 
@@ -239,18 +310,17 @@ async def toggle_scholar(
             message="Scholar not found.",
         )
     updated = await scholar_service.toggle_scholar_enabled(db_session, profile=profile)
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.scholars.toggled",
-        extra={
-            "event": "api.scholars.toggled",
-            "user_id": current_user.id,
-            "scholar_profile_id": updated.id,
-            "is_enabled": updated.is_enabled,
-        },
+        user_id=current_user.id,
+        scholar_profile_id=updated.id,
+        is_enabled=updated.is_enabled,
     )
     return success_payload(
         request,
-        data=_serialize_scholar(updated),
+        data=serialize_scholar(updated),
     )
 
 
@@ -275,18 +345,20 @@ async def delete_scholar(
             code="scholar_not_found",
             message="Scholar not found.",
         )
-    await scholar_service.delete_scholar(
-        db_session,
-        profile=profile,
-        upload_dir=settings.scholar_image_upload_dir,
-    )
-    logger.info(
-        "api.scholars.deleted",
-        extra={
-            "event": "api.scholars.deleted",
-            "user_id": current_user.id,
-            "scholar_profile_id": scholar_profile_id,
-        },
+    try:
+        await scholar_service.delete_scholar(
+            db_session,
+            profile=profile,
+            upload_dir=settings.scholar_image_upload_dir,
+        )
+    except scholar_service.ScholarServiceError as exc:
+        raise ApiException(
+            status_code=409,
+            code="scholar_delete_failed",
+            message=str(exc),
+        ) from exc
+    structured_log(
+        logger, "info", "api.scholars.deleted", user_id=current_user.id, scholar_profile_id=scholar_profile_id
     )
     return success_payload(
         request,
@@ -331,17 +403,12 @@ async def update_scholar_image_url(
             message=str(exc),
         ) from exc
 
-    logger.info(
-        "api.scholars.image_url_updated",
-        extra={
-            "event": "api.scholars.image_url_updated",
-            "user_id": current_user.id,
-            "scholar_profile_id": updated.id,
-        },
+    structured_log(
+        logger, "info", "api.scholars.image_url_updated", user_id=current_user.id, scholar_profile_id=updated.id
     )
     return success_payload(
         request,
-        data=_serialize_scholar(updated),
+        data=serialize_scholar(updated),
     )
 
 
@@ -356,20 +423,14 @@ async def upload_scholar_image(
     db_session: AsyncSession = Depends(get_db_session),
     current_user: User = Depends(get_api_current_user),
 ):
-    profile = await scholar_service.get_user_scholar_by_id(
+    profile = await require_user_profile(
         db_session,
         user_id=current_user.id,
         scholar_profile_id=scholar_profile_id,
     )
-    if profile is None:
-        raise ApiException(
-            status_code=404,
-            code="scholar_not_found",
-            message="Scholar not found.",
-        )
 
+    image_bytes = await read_uploaded_image(image)
     try:
-        image_bytes = await image.read()
         updated = await scholar_service.set_profile_image_upload(
             db_session,
             profile=profile,
@@ -384,22 +445,20 @@ async def upload_scholar_image(
             code="invalid_scholar_image",
             message=str(exc),
         ) from exc
-    finally:
-        await image.close()
 
-    logger.info(
+    image_size = len(image_bytes)
+    structured_log(
+        logger,
+        "info",
         "api.scholars.image_uploaded",
-        extra={
-            "event": "api.scholars.image_uploaded",
-            "user_id": current_user.id,
-            "scholar_profile_id": updated.id,
-            "content_type": image.content_type,
-            "size_bytes": len(image_bytes),
-        },
+        user_id=current_user.id,
+        scholar_profile_id=updated.id,
+        content_type=image.content_type,
+        size_bytes=image_size,
     )
     return success_payload(
         request,
-        data=_serialize_scholar(updated),
+        data=serialize_scholar(updated),
     )
 
 
@@ -430,64 +489,8 @@ async def clear_scholar_image_customization(
         profile=profile,
         upload_dir=settings.scholar_image_upload_dir,
     )
-    logger.info(
-        "api.scholars.image_customization_cleared",
-        extra={
-            "event": "api.scholars.image_customization_cleared",
-            "user_id": current_user.id,
-            "scholar_profile_id": updated.id,
-        },
-    )
+    structured_log(logger, "info", "api.scholars.image_cleared", user_id=current_user.id, scholar_profile_id=updated.id)
     return success_payload(
         request,
-        data=_serialize_scholar(updated),
+        data=serialize_scholar(updated),
     )
-
-
-@router.get(
-    "/{scholar_profile_id}/image/upload",
-)
-async def get_uploaded_scholar_image(
-    scholar_profile_id: int,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    profile = await scholar_service.get_user_scholar_by_id(
-        db_session,
-        user_id=current_user.id,
-        scholar_profile_id=scholar_profile_id,
-    )
-    if profile is None:
-        raise ApiException(
-            status_code=404,
-            code="scholar_not_found",
-            message="Scholar not found.",
-        )
-    if not profile.profile_image_upload_path:
-        raise ApiException(
-            status_code=404,
-            code="scholar_image_not_found",
-            message="Scholar image not found.",
-        )
-
-    try:
-        image_path = scholar_service.resolve_upload_file_path(
-            upload_dir=settings.scholar_image_upload_dir,
-            relative_path=profile.profile_image_upload_path,
-        )
-    except scholar_service.ScholarServiceError as exc:
-        raise ApiException(
-            status_code=404,
-            code="scholar_image_not_found",
-            message="Scholar image not found.",
-        ) from exc
-
-    if not image_path.exists() or not image_path.is_file():
-        raise ApiException(
-            status_code=404,
-            code="scholar_image_not_found",
-            message="Scholar image not found.",
-        )
-
-    media_type = mimetypes.guess_type(str(image_path))[0] or "application/octet-stream"
-    return FileResponse(path=image_path, media_type=media_type)

app/api/routers/settings.py

@@ -11,8 +11,9 @@ from app.api.responses import success_payload
 from app.api.schemas import SettingsEnvelope, SettingsUpdateRequest
 from app.db.models import User
 from app.db.session import get_db_session
-from app.services import run_safety as run_safety_service
-from app.services import user_settings as user_settings_service
+from app.logging_utils import structured_log
+from app.services.ingestion import safety as run_safety_service
+from app.services.settings import application as user_settings_service
 from app.settings import settings as settings_module
 
 logger = logging.getLogger(__name__)
@@ -21,12 +22,7 @@ router = APIRouter(prefix="/settings", tags=["api-settings"])
 
 
 def _serialize_settings(user_settings) -> dict[str, object]:
-    min_run_interval_minutes = user_settings_service.resolve_run_interval_minimum(
-        settings_module.ingestion_min_run_interval_minutes
-    )
-    min_request_delay_seconds = user_settings_service.resolve_request_delay_minimum(
-        settings_module.ingestion_min_request_delay_seconds
-    )
+    min_run_interval_minutes, min_request_delay_seconds = _minimum_policy()
     return {
         "auto_run_enabled": bool(user_settings.auto_run_enabled) and settings_module.ingestion_automation_allowed,
         "run_interval_minutes": int(user_settings.run_interval_minutes),
@@ -43,9 +39,65 @@ def _serialize_settings(user_settings) -> dict[str, object]:
             "cooldown_network_seconds": max(60, int(settings_module.ingestion_safety_cooldown_network_seconds)),
         },
         "safety_state": run_safety_service.get_safety_state_payload(user_settings),
+        "openalex_api_key": user_settings.openalex_api_key,
+        "crossref_api_token": user_settings.crossref_api_token,
+        "crossref_api_mailto": user_settings.crossref_api_mailto,
     }
 
 
+def _minimum_policy() -> tuple[int, int]:
+    min_run_interval_minutes = user_settings_service.resolve_run_interval_minimum(
+        settings_module.ingestion_min_run_interval_minutes
+    )
+    min_request_delay_seconds = user_settings_service.resolve_request_delay_minimum(
+        settings_module.ingestion_min_request_delay_seconds
+    )
+    return min_run_interval_minutes, min_request_delay_seconds
+
+
+def _parse_settings_payload(payload: SettingsUpdateRequest, user_settings) -> tuple[int, int, list[str]]:
+    min_run_interval_minutes, min_request_delay_seconds = _minimum_policy()
+    parsed_interval = user_settings_service.parse_run_interval_minutes(
+        str(payload.run_interval_minutes),
+        minimum=min_run_interval_minutes,
+    )
+    parsed_delay = user_settings_service.parse_request_delay_seconds(
+        str(payload.request_delay_seconds),
+        minimum=min_request_delay_seconds,
+    )
+    parsed_nav_visible_pages = user_settings_service.parse_nav_visible_pages(
+        payload.nav_visible_pages
+        if payload.nav_visible_pages is not None
+        else list(user_settings.nav_visible_pages or user_settings_service.DEFAULT_NAV_VISIBLE_PAGES)
+    )
+    return parsed_interval, parsed_delay, parsed_nav_visible_pages
+
+
+async def _clear_expired_cooldown_with_log(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    user_settings,
+) -> None:
+    previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
+    if not run_safety_service.clear_expired_cooldown(user_settings):
+        return
+    await db_session.commit()
+    await db_session.refresh(user_settings)
+    structured_log(
+        logger,
+        "info",
+        "api.settings.safety_cooldown_cleared",
+        user_id=user_id,
+        reason=previous_safety_state.get("cooldown_reason"),
+        cooldown_until=previous_safety_state.get("cooldown_until"),
+    )
+
+
+def _effective_auto_run_enabled(payload: SettingsUpdateRequest) -> bool:
+    return bool(payload.auto_run_enabled) and settings_module.ingestion_automation_allowed
+
+
 @router.get(
     "",
     response_model=SettingsEnvelope,
@@ -59,21 +111,11 @@ async def get_settings(
         db_session,
         user_id=current_user.id,
     )
-    previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
-    if run_safety_service.clear_expired_cooldown(user_settings):
-        await db_session.commit()
-        await db_session.refresh(user_settings)
-        logger.info(
-            "api.settings.safety_cooldown_cleared",
-            extra={
-                "event": "api.settings.safety_cooldown_cleared",
-                "user_id": current_user.id,
-                "reason": previous_safety_state.get("cooldown_reason"),
-                "cooldown_until": previous_safety_state.get("cooldown_until"),
-                "metric_name": "api_settings_safety_cooldown_cleared_total",
-                "metric_value": 1,
-            },
-        )
+    await _clear_expired_cooldown_with_log(
+        db_session,
+        user_id=current_user.id,
+        user_settings=user_settings,
+    )
     return success_payload(
         request,
         data=_serialize_settings(user_settings),
@@ -95,26 +137,10 @@ async def update_settings(
         user_id=current_user.id,
     )
 
-    min_run_interval_minutes = user_settings_service.resolve_run_interval_minimum(
-        settings_module.ingestion_min_run_interval_minutes
-    )
-    min_request_delay_seconds = user_settings_service.resolve_request_delay_minimum(
-        settings_module.ingestion_min_request_delay_seconds
-    )
-
     try:
-        parsed_interval = user_settings_service.parse_run_interval_minutes(
-            str(payload.run_interval_minutes),
-            minimum=min_run_interval_minutes,
-        )
-        parsed_delay = user_settings_service.parse_request_delay_seconds(
-            str(payload.request_delay_seconds),
-            minimum=min_request_delay_seconds,
-        )
-        parsed_nav_visible_pages = user_settings_service.parse_nav_visible_pages(
-            payload.nav_visible_pages
-            if payload.nav_visible_pages is not None
-            else list(user_settings.nav_visible_pages or user_settings_service.DEFAULT_NAV_VISIBLE_PAGES)
+        parsed_interval, parsed_delay, parsed_nav_visible_pages = _parse_settings_payload(
+            payload,
+            user_settings,
         )
     except user_settings_service.UserSettingsServiceError as exc:
         raise ApiException(
@@ -123,43 +149,32 @@ async def update_settings(
             message=str(exc),
         ) from exc
 
-    effective_auto_run_enabled = (
-        bool(payload.auto_run_enabled) and settings_module.ingestion_automation_allowed
-    )
-
     updated = await user_settings_service.update_settings(
         db_session,
         settings=user_settings,
-        auto_run_enabled=effective_auto_run_enabled,
+        auto_run_enabled=_effective_auto_run_enabled(payload),
         run_interval_minutes=parsed_interval,
         request_delay_seconds=parsed_delay,
         nav_visible_pages=parsed_nav_visible_pages,
+        openalex_api_key=payload.openalex_api_key,
+        crossref_api_token=payload.crossref_api_token,
+        crossref_api_mailto=payload.crossref_api_mailto,
     )
-    previous_safety_state = run_safety_service.get_safety_event_context(updated)
-    if run_safety_service.clear_expired_cooldown(updated):
-        await db_session.commit()
-        await db_session.refresh(updated)
-        logger.info(
-            "api.settings.safety_cooldown_cleared",
-            extra={
-                "event": "api.settings.safety_cooldown_cleared",
-                "user_id": current_user.id,
-                "reason": previous_safety_state.get("cooldown_reason"),
-                "cooldown_until": previous_safety_state.get("cooldown_until"),
-                "metric_name": "api_settings_safety_cooldown_cleared_total",
-                "metric_value": 1,
-            },
-        )
-    logger.info(
+    await _clear_expired_cooldown_with_log(
+        db_session,
+        user_id=current_user.id,
+        user_settings=updated,
+    )
+    structured_log(
+        logger,
+        "info",
         "api.settings.updated",
-        extra={
-            "event": "api.settings.updated",
-            "user_id": current_user.id,
-            "auto_run_enabled": updated.auto_run_enabled,
-            "run_interval_minutes": updated.run_interval_minutes,
-            "request_delay_seconds": updated.request_delay_seconds,
-            "nav_visible_pages": updated.nav_visible_pages,
-        },
+        user_id=current_user.id,
+        auto_run_enabled=updated.auto_run_enabled,
+        run_interval_minutes=updated.run_interval_minutes,
+        request_delay_seconds=updated.request_delay_seconds,
+        nav_visible_pages=updated.nav_visible_pages,
+        openalex_api_key="SET" if updated.openalex_api_key else "UNSET",
     )
     return success_payload(
         request,

app/api/runtime_deps.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 from fastapi import Depends
 
-from app.services import ingestion as ingestion_service
-from app.services import scholar_source as scholar_source_service
+from app.services.ingestion import application as ingestion_service
+from app.services.scholar import source as scholar_source_service
 
 
 def get_scholar_source() -> scholar_source_service.ScholarSource:

app/api/schemas.py

@@ -1,571 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Any
-
-from pydantic import BaseModel, ConfigDict, Field
-
-
-class ApiMeta(BaseModel):
-    request_id: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ApiErrorData(BaseModel):
-    code: str
-    message: str
-    details: Any | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ApiErrorEnvelope(BaseModel):
-    error: ApiErrorData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessageData(BaseModel):
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessageEnvelope(BaseModel):
-    data: MessageData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SessionUserData(BaseModel):
-    id: int
-    email: str
-    is_admin: bool
-    is_active: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AuthMeData(BaseModel):
-    authenticated: bool
-    csrf_token: str
-    user: SessionUserData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AuthMeEnvelope(BaseModel):
-    data: AuthMeData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class CsrfBootstrapData(BaseModel):
-    csrf_token: str
-    authenticated: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class CsrfBootstrapEnvelope(BaseModel):
-    data: CsrfBootstrapData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginRequest(BaseModel):
-    email: str
-    password: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginData(BaseModel):
-    authenticated: bool
-    csrf_token: str
-    user: SessionUserData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginEnvelope(BaseModel):
-    data: LoginData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ChangePasswordRequest(BaseModel):
-    current_password: str
-    new_password: str
-    confirm_password: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarItemData(BaseModel):
-    id: int
-    scholar_id: str
-    display_name: str | None
-    profile_image_url: str | None
-    profile_image_source: str
-    is_enabled: bool
-    baseline_completed: bool
-    last_run_dt: datetime | None
-    last_run_status: str | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarsListData(BaseModel):
-    scholars: list[ScholarItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarsListEnvelope(BaseModel):
-    data: ScholarsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarCreateRequest(BaseModel):
-    scholar_id: str
-    profile_image_url: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarEnvelope(BaseModel):
-    data: ScholarItemData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchCandidateData(BaseModel):
-    scholar_id: str
-    display_name: str
-    affiliation: str | None
-    email_domain: str | None
-    cited_by_count: int | None
-    interests: list[str] = Field(default_factory=list)
-    profile_url: str
-    profile_image_url: str | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchData(BaseModel):
-    query: str
-    state: str
-    state_reason: str
-    action_hint: str | None = None
-    candidates: list[ScholarSearchCandidateData]
-    warnings: list[str] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchEnvelope(BaseModel):
-    data: ScholarSearchData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarImageUrlUpdateRequest(BaseModel):
-    image_url: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunListItemData(BaseModel):
-    id: int
-    trigger_type: str
-    status: str
-    start_dt: datetime
-    end_dt: datetime | None
-    scholar_count: int
-    new_publication_count: int
-    failed_count: int
-    partial_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunsListData(BaseModel):
-    runs: list[RunListItemData]
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunsListEnvelope(BaseModel):
-    data: RunsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunSummaryData(BaseModel):
-    succeeded_count: int
-    failed_count: int
-    partial_count: int
-    failed_state_counts: dict[str, int] = Field(default_factory=dict)
-    failed_reason_counts: dict[str, int] = Field(default_factory=dict)
-    scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
-    retry_counts: dict[str, int] = Field(default_factory=dict)
-    alert_thresholds: dict[str, int] = Field(default_factory=dict)
-    alert_flags: dict[str, bool] = Field(default_factory=dict)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScrapeSafetyCountersData(BaseModel):
-    consecutive_blocked_runs: int = 0
-    consecutive_network_runs: int = 0
-    cooldown_entry_count: int = 0
-    blocked_start_count: int = 0
-    last_blocked_failure_count: int = 0
-    last_network_failure_count: int = 0
-    last_evaluated_run_id: int | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScrapeSafetyStateData(BaseModel):
-    cooldown_active: bool
-    cooldown_reason: str | None = None
-    cooldown_reason_label: str | None = None
-    cooldown_until: datetime | None = None
-    cooldown_remaining_seconds: int = 0
-    recommended_action: str | None = None
-    counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunAttemptLogData(BaseModel):
-    attempt: int
-    cstart: int
-    state: str | None = None
-    state_reason: str | None = None
-    status_code: int | None = None
-    fetch_error: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunPageLogData(BaseModel):
-    page: int
-    cstart: int
-    state: str
-    state_reason: str | None = None
-    status_code: int | None = None
-    publication_count: int = 0
-    attempt_count: int = 0
-    has_show_more_button: bool = False
-    articles_range: str | None = None
-    warning_codes: list[str] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDebugData(BaseModel):
-    status_code: int | None = None
-    final_url: str | None = None
-    fetch_error: str | None = None
-    body_sha256: str | None = None
-    body_length: int | None = None
-    has_show_more_button: bool | None = None
-    articles_range: str | None = None
-    state_reason: str | None = None
-    warning_codes: list[str] = Field(default_factory=list)
-    marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
-    page_logs: list[RunPageLogData] = Field(default_factory=list)
-    attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunScholarResultData(BaseModel):
-    scholar_profile_id: int
-    scholar_id: str
-    state: str
-    state_reason: str | None = None
-    outcome: str
-    attempt_count: int = 0
-    publication_count: int = 0
-    start_cstart: int = 0
-    continuation_cstart: int | None = None
-    continuation_enqueued: bool = False
-    continuation_cleared: bool = False
-    warnings: list[str] = Field(default_factory=list)
-    error: str | None = None
-    debug: RunDebugData | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDetailData(BaseModel):
-    run: RunListItemData
-    summary: RunSummaryData
-    scholar_results: list[RunScholarResultData]
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDetailEnvelope(BaseModel):
-    data: RunDetailData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ManualRunData(BaseModel):
-    run_id: int
-    status: str
-    scholar_count: int
-    succeeded_count: int
-    failed_count: int
-    partial_count: int
-    new_publication_count: int
-    reused_existing_run: bool
-    idempotency_key: str | None = None
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ManualRunEnvelope(BaseModel):
-    data: ManualRunData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueItemData(BaseModel):
-    id: int
-    scholar_profile_id: int
-    scholar_label: str
-    status: str
-    reason: str
-    dropped_reason: str | None
-    attempt_count: int
-    resume_cstart: int
-    next_attempt_dt: datetime | None
-    updated_at: datetime
-    last_error: str | None
-    last_run_id: int | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueListData(BaseModel):
-    queue_items: list[QueueItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueListEnvelope(BaseModel):
-    data: QueueListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueItemEnvelope(BaseModel):
-    data: QueueItemData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueClearData(BaseModel):
-    queue_item_id: int
-    previous_status: str
-    status: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueClearEnvelope(BaseModel):
-    data: QueueClearData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserData(BaseModel):
-    id: int
-    email: str
-    is_active: bool
-    is_admin: bool
-    created_at: datetime
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUsersListData(BaseModel):
-    users: list[AdminUserData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUsersListEnvelope(BaseModel):
-    data: AdminUsersListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserCreateRequest(BaseModel):
-    email: str
-    password: str
-    is_admin: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserActiveUpdateRequest(BaseModel):
-    is_active: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserEnvelope(BaseModel):
-    data: AdminUserData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminResetPasswordRequest(BaseModel):
-    new_password: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsPolicyData(BaseModel):
-    min_run_interval_minutes: int
-    min_request_delay_seconds: int
-    automation_allowed: bool
-    manual_run_allowed: bool
-    blocked_failure_threshold: int
-    network_failure_threshold: int
-    cooldown_blocked_seconds: int
-    cooldown_network_seconds: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsData(BaseModel):
-    auto_run_enabled: bool
-    run_interval_minutes: int
-    request_delay_seconds: int
-    nav_visible_pages: list[str]
-    policy: SettingsPolicyData
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsEnvelope(BaseModel):
-    data: SettingsData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsUpdateRequest(BaseModel):
-    auto_run_enabled: bool
-    run_interval_minutes: int
-    request_delay_seconds: int
-    nav_visible_pages: list[str] | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationItemData(BaseModel):
-    publication_id: int
-    scholar_profile_id: int
-    scholar_label: str
-    title: str
-    year: int | None
-    citation_count: int
-    venue_text: str | None
-    pub_url: str | None
-    is_read: bool
-    first_seen_at: datetime
-    is_new_in_latest_run: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationsListData(BaseModel):
-    mode: str
-    selected_scholar_profile_id: int | None
-    new_count: int
-    total_count: int
-    publications: list[PublicationItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationsListEnvelope(BaseModel):
-    data: PublicationsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkAllReadData(BaseModel):
-    message: str
-    updated_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkAllReadEnvelope(BaseModel):
-    data: MarkAllReadData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationSelectionItem(BaseModel):
-    scholar_profile_id: int
-    publication_id: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadRequest(BaseModel):
-    selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadData(BaseModel):
-    message: str
-    requested_count: int
-    updated_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadEnvelope(BaseModel):
-    data: MarkSelectedReadData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/__init__.py

@@ -0,0 +1,7 @@
+from app.api.schemas.admin import *  # noqa: F403
+from app.api.schemas.auth import *  # noqa: F403
+from app.api.schemas.common import *  # noqa: F403
+from app.api.schemas.publications import *  # noqa: F403
+from app.api.schemas.runs import *  # noqa: F403
+from app.api.schemas.scholars import *  # noqa: F403
+from app.api.schemas.settings import *  # noqa: F403

app/api/schemas/admin.py

@@ -0,0 +1,314 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from app.api.schemas.common import ApiMeta
+from app.api.schemas.publications import DisplayIdentifierData
+
+
+class AdminUserData(BaseModel):
+    id: int
+    email: str
+    is_active: bool
+    is_admin: bool
+    created_at: datetime
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUsersListData(BaseModel):
+    users: list[AdminUserData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUsersListEnvelope(BaseModel):
+    data: AdminUsersListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserCreateRequest(BaseModel):
+    email: str = Field(max_length=254)
+    password: str = Field(min_length=8, max_length=128)
+    is_admin: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserActiveUpdateRequest(BaseModel):
+    is_active: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserEnvelope(BaseModel):
+    data: AdminUserData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminResetPasswordRequest(BaseModel):
+    new_password: str = Field(min_length=8, max_length=128)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminScholarHttpSettingsData(BaseModel):
+    user_agent: str
+    rotate_user_agent: bool
+    accept_language: str
+    cookie: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminScholarHttpSettingsEnvelope(BaseModel):
+    data: AdminScholarHttpSettingsData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminScholarHttpSettingsUpdateRequest(BaseModel):
+    user_agent: str
+    rotate_user_agent: bool
+    accept_language: str
+    cookie: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityCheckData(BaseModel):
+    name: str
+    count: int
+    severity: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityData(BaseModel):
+    status: str
+    checked_at: datetime
+    failures: list[str] = Field(default_factory=list)
+    warnings: list[str] = Field(default_factory=list)
+    checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityEnvelope(BaseModel):
+    data: AdminDbIntegrityData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobData(BaseModel):
+    id: int
+    job_name: str
+    requested_by: str | None
+    dry_run: bool
+    status: str
+    scope: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any] = Field(default_factory=dict)
+    error_text: str | None
+    started_at: datetime | None
+    finished_at: datetime | None
+    created_at: datetime
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobsData(BaseModel):
+    jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobsEnvelope(BaseModel):
+    data: AdminDbRepairJobsData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueItemData(BaseModel):
+    publication_id: int
+    title: str
+    display_identifier: DisplayIdentifierData | None = None
+    pdf_url: str | None
+    status: str
+    attempt_count: int
+    last_failure_reason: str | None
+    last_failure_detail: str | None
+    last_source: str | None
+    requested_by_user_id: int | None
+    requested_by_email: str | None
+    queued_at: datetime | None
+    last_attempt_at: datetime | None
+    resolved_at: datetime | None
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueData(BaseModel):
+    items: list[AdminPdfQueueItemData] = Field(default_factory=list)
+    total_count: int = 0
+    page: int = 1
+    page_size: int = 100
+    has_next: bool = False
+    has_prev: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueEnvelope(BaseModel):
+    data: AdminPdfQueueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueRequeueData(BaseModel):
+    publication_id: int
+    queued: bool
+    status: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueRequeueEnvelope(BaseModel):
+    data: AdminPdfQueueRequeueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueBulkEnqueueData(BaseModel):
+    requested_count: int
+    queued_count: int
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
+    data: AdminPdfQueueBulkEnqueueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationLinksRequest(BaseModel):
+    scope_mode: Literal["single_user", "all_users"] = "single_user"
+    user_id: int | None = Field(default=None, ge=1)
+    scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
+    dry_run: bool = True
+    gc_orphan_publications: bool = False
+    requested_by: str | None = None
+    confirmation_text: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")
+    def validate_scope(self) -> AdminRepairPublicationLinksRequest:
+        if self.scope_mode == "single_user" and self.user_id is None:
+            raise ValueError("user_id is required when scope_mode=single_user.")
+        if self.scope_mode == "all_users" and self.user_id is not None:
+            raise ValueError("user_id must be omitted when scope_mode=all_users.")
+        if not self.dry_run and self.scope_mode == "all_users":
+            expected = "REPAIR ALL USERS"
+            provided = (self.confirmation_text or "").strip()
+            if provided != expected:
+                raise ValueError("confirmation_text must equal 'REPAIR ALL USERS' when applying a repair to all users.")
+        return self
+
+
+class AdminRepairPublicationLinksResultData(BaseModel):
+    job_id: int
+    status: str
+    scope: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationLinksEnvelope(BaseModel):
+    data: AdminRepairPublicationLinksResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminNearDuplicateClusterMemberData(BaseModel):
+    publication_id: int
+    title: str
+    year: int | None
+    citation_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminNearDuplicateClusterData(BaseModel):
+    cluster_key: str
+    winner_publication_id: int
+    member_count: int
+    similarity_score: float = Field(ge=0.0, le=1.0)
+    members: list[AdminNearDuplicateClusterMemberData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationNearDuplicatesRequest(BaseModel):
+    dry_run: bool = True
+    similarity_threshold: float = Field(default=0.78, ge=0.5, le=1.0)
+    min_shared_tokens: int = Field(default=3, ge=1, le=8)
+    max_year_delta: int = Field(default=1, ge=0, le=5)
+    max_clusters: int = Field(default=25, ge=1, le=200)
+    selected_cluster_keys: list[str] = Field(default_factory=list, max_length=200)
+    requested_by: str | None = None
+    confirmation_text: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")
+    def validate_apply_mode(self) -> AdminRepairPublicationNearDuplicatesRequest:
+        if self.dry_run:
+            return self
+        if not self.selected_cluster_keys:
+            raise ValueError("selected_cluster_keys is required when dry_run=false.")
+        expected = "MERGE SELECTED DUPLICATES"
+        provided = (self.confirmation_text or "").strip()
+        if provided != expected:
+            raise ValueError(
+                "confirmation_text must equal 'MERGE SELECTED DUPLICATES' when applying near-duplicate merges."
+            )
+        return self
+
+
+class AdminRepairPublicationNearDuplicatesResultData(BaseModel):
+    job_id: int
+    status: str
+    scope: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any] = Field(default_factory=dict)
+    clusters: list[AdminNearDuplicateClusterData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationNearDuplicatesEnvelope(BaseModel):
+    data: AdminRepairPublicationNearDuplicatesResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/auth.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.api.schemas.common import ApiMeta
+
+
+class SessionUserData(BaseModel):
+    id: int
+    email: str
+    is_admin: bool
+    is_active: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AuthMeData(BaseModel):
+    authenticated: bool
+    csrf_token: str
+    user: SessionUserData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AuthMeEnvelope(BaseModel):
+    data: AuthMeData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class CsrfBootstrapData(BaseModel):
+    csrf_token: str
+    authenticated: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class CsrfBootstrapEnvelope(BaseModel):
+    data: CsrfBootstrapData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginRequest(BaseModel):
+    email: str = Field(max_length=254)
+    password: str = Field(min_length=8, max_length=128)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginData(BaseModel):
+    authenticated: bool
+    csrf_token: str
+    user: SessionUserData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginEnvelope(BaseModel):
+    data: LoginData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ChangePasswordRequest(BaseModel):
+    current_password: str = Field(min_length=8, max_length=128)
+    new_password: str = Field(min_length=8, max_length=128)
+    confirm_password: str = Field(min_length=8, max_length=128)
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/common.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict
+
+
+class ApiMeta(BaseModel):
+    request_id: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ApiErrorData(BaseModel):
+    code: str
+    message: str
+    details: Any | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ApiErrorEnvelope(BaseModel):
+    error: ApiErrorData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessageData(BaseModel):
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessageEnvelope(BaseModel):
+    data: MessageData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/publications.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.api.schemas.common import ApiMeta
+
+
+class DisplayIdentifierData(BaseModel):
+    kind: str
+    value: str
+    label: str
+    url: str | None
+    confidence_score: float = Field(ge=0.0, le=1.0)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationItemData(BaseModel):
+    publication_id: int
+    scholar_profile_id: int
+    scholar_label: str
+    title: str
+    year: int | None
+    citation_count: int
+    venue_text: str | None
+    pub_url: str | None
+    display_identifier: DisplayIdentifierData | None = None
+    pdf_url: str | None
+    pdf_status: str = "untracked"
+    pdf_attempt_count: int = 0
+    pdf_failure_reason: str | None = None
+    pdf_failure_detail: str | None = None
+    is_read: bool
+    is_favorite: bool = False
+    first_seen_at: datetime
+    is_new_in_latest_run: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationsListData(BaseModel):
+    mode: str
+    favorite_only: bool = False
+    selected_scholar_profile_id: int | None
+    unread_count: int
+    favorites_count: int
+    latest_count: int
+    new_count: int
+    total_count: int
+    page: int
+    page_size: int
+    snapshot: str
+    has_next: bool = False
+    has_prev: bool = False
+    publications: list[PublicationItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationsListEnvelope(BaseModel):
+    data: PublicationsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkAllReadData(BaseModel):
+    message: str
+    updated_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkAllReadEnvelope(BaseModel):
+    data: MarkAllReadData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationSelectionItem(BaseModel):
+    scholar_profile_id: int
+    publication_id: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadRequest(BaseModel):
+    selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadData(BaseModel):
+    message: str
+    requested_count: int
+    updated_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadEnvelope(BaseModel):
+    data: MarkSelectedReadData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfRequest(BaseModel):
+    scholar_profile_id: int = Field(ge=1)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfData(BaseModel):
+    message: str
+    queued: bool
+    resolved_pdf: bool
+    publication: PublicationItemData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfEnvelope(BaseModel):
+    data: RetryPublicationPdfData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteRequest(BaseModel):
+    scholar_profile_id: int = Field(ge=1)
+    is_favorite: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteData(BaseModel):
+    message: str
+    publication: PublicationItemData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteEnvelope(BaseModel):
+    data: TogglePublicationFavoriteData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/runs.py

@@ -0,0 +1,226 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.api.schemas.common import ApiMeta
+
+
+class RunListItemData(BaseModel):
+    id: int
+    trigger_type: str
+    status: str
+    start_dt: datetime
+    end_dt: datetime | None
+    scholar_count: int
+    new_publication_count: int
+    failed_count: int
+    partial_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScrapeSafetyCountersData(BaseModel):
+    consecutive_blocked_runs: int = 0
+    consecutive_network_runs: int = 0
+    cooldown_entry_count: int = 0
+    blocked_start_count: int = 0
+    last_blocked_failure_count: int = 0
+    last_network_failure_count: int = 0
+    last_evaluated_run_id: int | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScrapeSafetyStateData(BaseModel):
+    cooldown_active: bool
+    cooldown_reason: str | None = None
+    cooldown_reason_label: str | None = None
+    cooldown_until: datetime | None = None
+    cooldown_remaining_seconds: int = 0
+    recommended_action: str | None = None
+    counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunsListData(BaseModel):
+    runs: list[RunListItemData]
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunsListEnvelope(BaseModel):
+    data: RunsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunSummaryData(BaseModel):
+    succeeded_count: int
+    failed_count: int
+    partial_count: int
+    failed_state_counts: dict[str, int] = Field(default_factory=dict)
+    failed_reason_counts: dict[str, int] = Field(default_factory=dict)
+    scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
+    retry_counts: dict[str, int] = Field(default_factory=dict)
+    alert_thresholds: dict[str, int] = Field(default_factory=dict)
+    alert_flags: dict[str, bool] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunAttemptLogData(BaseModel):
+    attempt: int
+    cstart: int
+    state: str | None = None
+    state_reason: str | None = None
+    status_code: int | None = None
+    fetch_error: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunPageLogData(BaseModel):
+    page: int
+    cstart: int
+    state: str
+    state_reason: str | None = None
+    status_code: int | None = None
+    publication_count: int = 0
+    attempt_count: int = 0
+    has_show_more_button: bool = False
+    articles_range: str | None = None
+    warning_codes: list[str] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDebugData(BaseModel):
+    status_code: int | None = None
+    final_url: str | None = None
+    fetch_error: str | None = None
+    body_sha256: str | None = None
+    body_length: int | None = None
+    has_show_more_button: bool | None = None
+    articles_range: str | None = None
+    state_reason: str | None = None
+    warning_codes: list[str] = Field(default_factory=list)
+    marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
+    page_logs: list[RunPageLogData] = Field(default_factory=list)
+    attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunScholarResultData(BaseModel):
+    scholar_profile_id: int
+    scholar_id: str
+    state: str
+    state_reason: str | None = None
+    outcome: str
+    attempt_count: int = 0
+    publication_count: int = 0
+    start_cstart: int = 0
+    continuation_cstart: int | None = None
+    continuation_enqueued: bool = False
+    continuation_cleared: bool = False
+    warnings: list[str] = Field(default_factory=list)
+    error: str | None = None
+    debug: RunDebugData | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDetailData(BaseModel):
+    run: RunListItemData
+    summary: RunSummaryData
+    scholar_results: list[RunScholarResultData]
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDetailEnvelope(BaseModel):
+    data: RunDetailData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ManualRunData(BaseModel):
+    run_id: int
+    status: str
+    scholar_count: int
+    succeeded_count: int
+    failed_count: int
+    partial_count: int
+    new_publication_count: int
+    reused_existing_run: bool
+    idempotency_key: str | None = None
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ManualRunEnvelope(BaseModel):
+    data: ManualRunData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueItemData(BaseModel):
+    id: int
+    scholar_profile_id: int
+    scholar_label: str
+    status: str
+    reason: str
+    dropped_reason: str | None
+    attempt_count: int
+    resume_cstart: int
+    next_attempt_dt: datetime | None
+    updated_at: datetime
+    last_error: str | None
+    last_run_id: int | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueListData(BaseModel):
+    queue_items: list[QueueItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueListEnvelope(BaseModel):
+    data: QueueListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueItemEnvelope(BaseModel):
+    data: QueueItemData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueClearData(BaseModel):
+    queue_item_id: int
+    previous_status: str
+    status: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueClearEnvelope(BaseModel):
+    data: QueueClearData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/scholars.py

@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from app.api.schemas.common import ApiMeta
+
+
+class ScholarItemData(BaseModel):
+    id: int
+    scholar_id: str
+    display_name: str | None
+    profile_image_url: str | None
+    profile_image_source: str
+    is_enabled: bool
+    baseline_completed: bool
+    last_run_dt: datetime | None
+    last_run_status: str | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarsListData(BaseModel):
+    scholars: list[ScholarItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarsListEnvelope(BaseModel):
+    data: ScholarsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarCreateRequest(BaseModel):
+    scholar_id: str
+    profile_image_url: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarEnvelope(BaseModel):
+    data: ScholarItemData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchCandidateData(BaseModel):
+    scholar_id: str
+    display_name: str
+    affiliation: str | None
+    email_domain: str | None
+    cited_by_count: int | None
+    interests: list[str] = Field(default_factory=list)
+    profile_url: str
+    profile_image_url: str | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchData(BaseModel):
+    query: str
+    state: str
+    state_reason: str
+    action_hint: str | None = None
+    candidates: list[ScholarSearchCandidateData]
+    warnings: list[str] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchEnvelope(BaseModel):
+    data: ScholarSearchData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarImageUrlUpdateRequest(BaseModel):
+    image_url: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarExportItemData(BaseModel):
+    scholar_id: str
+    display_name: str | None = None
+    is_enabled: bool = True
+    profile_image_override_url: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationExportItemData(BaseModel):
+    scholar_id: str
+    cluster_id: str | None = None
+    fingerprint_sha256: str | None = None
+    title: str
+    year: int | None = None
+    citation_count: int = 0
+    author_text: str | None = None
+    venue_text: str | None = None
+    pub_url: str | None = None
+    pdf_url: str | None = None
+    is_read: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataExportData(BaseModel):
+    schema_version: int
+    exported_at: str
+    scholars: list[ScholarExportItemData]
+    publications: list[PublicationExportItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataExportEnvelope(BaseModel):
+    data: DataExportData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarBulkIdsRequest(BaseModel):
+    scholar_profile_ids: list[int] = Field(..., min_length=1, max_length=500)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarBulkToggleRequest(BaseModel):
+    scholar_profile_ids: list[int] = Field(..., min_length=1, max_length=500)
+    is_enabled: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarBulkCountData(BaseModel):
+    deleted_count: int = 0
+    updated_count: int = 0
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarBulkCountEnvelope(BaseModel):
+    data: ScholarBulkCountData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportRequest(BaseModel):
+    schema_version: int | None = None
+    exported_at: str | None = None
+    scholars: list[ScholarExportItemData] = Field(default_factory=list)
+    publications: list[PublicationExportItemData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="before")
+    @classmethod
+    def unwrap_export_envelope(cls, values: dict) -> dict:
+        """Accept the full export envelope format (with data/meta wrapper)."""
+        if isinstance(values, dict) and "data" in values and isinstance(values["data"], dict):
+            return values["data"]
+        return values
+
+
+class DataImportResultData(BaseModel):
+    scholars_created: int
+    scholars_updated: int
+    publications_created: int
+    publications_updated: int
+    links_created: int
+    links_updated: int
+    skipped_records: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportEnvelope(BaseModel):
+    data: DataImportResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/settings.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict
+
+from app.api.schemas.common import ApiMeta
+from app.api.schemas.runs import ScrapeSafetyStateData
+
+
+class SettingsPolicyData(BaseModel):
+    min_run_interval_minutes: int
+    min_request_delay_seconds: int
+    automation_allowed: bool
+    manual_run_allowed: bool
+    blocked_failure_threshold: int
+    network_failure_threshold: int
+    cooldown_blocked_seconds: int
+    cooldown_network_seconds: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsData(BaseModel):
+    auto_run_enabled: bool
+    run_interval_minutes: int
+    request_delay_seconds: int
+    nav_visible_pages: list[str]
+    policy: SettingsPolicyData
+    safety_state: ScrapeSafetyStateData
+
+    openalex_api_key: str | None = None
+    crossref_api_token: str | None = None
+    crossref_api_mailto: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsEnvelope(BaseModel):
+    data: SettingsData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsUpdateRequest(BaseModel):
+    auto_run_enabled: bool
+    run_interval_minutes: int
+    request_delay_seconds: int
+    nav_visible_pages: list[str] | None = None
+
+    openalex_api_key: str | None = None
+    crossref_api_token: str | None = None
+    crossref_api_mailto: str | None = None
+
+    model_config = ConfigDict(extra="forbid")

app/auth/__init__.py

@@ -1,2 +1 @@
 """Authentication package for scholarr."""
-

app/auth/deps.py

@@ -24,4 +24,3 @@ def get_login_rate_limiter() -> SlidingWindowRateLimiter:
         max_attempts=settings.login_rate_limit_attempts,
         window_seconds=settings.login_rate_limit_window_seconds,
     )
-

app/auth/rate_limit.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from collections import defaultdict, deque
+from collections import deque
 from collections.abc import Callable
 from dataclasses import dataclass
 from math import ceil
@@ -25,14 +25,19 @@ class SlidingWindowRateLimiter:
         self._max_attempts = max_attempts
         self._window_seconds = window_seconds
         self._now = now
-        self._attempts: dict[str, deque[float]] = defaultdict(deque)
+        self._attempts: dict[str, deque[float]] = {}
         self._lock = Lock()
 
     def check(self, key: str) -> RateLimitDecision:
         with self._lock:
             now_value = self._now()
-            attempts = self._attempts[key]
+            attempts = self._attempts.get(key)
+            if attempts is None:
+                return RateLimitDecision(allowed=True)
             self._trim_expired(attempts, now_value)
+            if not attempts:
+                self._attempts.pop(key, None)
+                return RateLimitDecision(allowed=True)
             if len(attempts) >= self._max_attempts:
                 retry_after = self._window_seconds - (now_value - attempts[0])
                 return RateLimitDecision(
@@ -44,7 +49,10 @@ class SlidingWindowRateLimiter:
     def record_failure(self, key: str) -> None:
         with self._lock:
             now_value = self._now()
-            attempts = self._attempts[key]
+            attempts = self._attempts.get(key)
+            if attempts is None:
+                attempts = deque()
+                self._attempts[key] = attempts
             self._trim_expired(attempts, now_value)
             attempts.append(now_value)
 
@@ -59,4 +67,3 @@ class SlidingWindowRateLimiter:
     def _trim_expired(self, attempts: deque[float], now_value: float) -> None:
         while attempts and now_value - attempts[0] >= self._window_seconds:
             attempts.popleft()
-

app/auth/runtime.py

@@ -7,8 +7,9 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.auth.session import clear_session_user, get_session_user, set_session_user
 from app.db.models import User
+from app.logging_utils import structured_log
 from app.security.csrf import CSRF_SESSION_KEY
-from app.services import users as user_service
+from app.services.users import application as user_service
 
 logger = logging.getLogger(__name__)
 
@@ -28,13 +29,7 @@ async def get_authenticated_user(
 
     user = await user_service.get_user_by_id(db_session, session_user.id)
     if user is None or not user.is_active:
-        logger.info(
-            "auth.session_invalidated",
-            extra={
-                "event": "auth.session_invalidated",
-                "session_user_id": session_user.id,
-            },
-        )
+        structured_log(logger, "info", "auth.session_invalidated", session_user_id=session_user.id)
         invalidate_session(request)
         return None
 

app/auth/security.py

@@ -16,4 +16,3 @@ class PasswordService:
             return bool(self._hasher.verify(password_hash, password))
         except (InvalidHashError, VerificationError):
             return False
-

app/auth/service.py

@@ -21,9 +21,7 @@ class AuthService:
         normalized_email = email.strip().lower()
         if not normalized_email or not password:
             return None
-        result = await db_session.execute(
-            select(User).where(User.email == normalized_email)
-        )
+        result = await db_session.execute(select(User).where(User.email == normalized_email))
         user = result.scalar_one_or_none()
         if user is None or not user.is_active:
             return None

app/auth/session.py

@@ -4,7 +4,6 @@ from dataclasses import dataclass
 
 from starlette.requests import Request
 
-
 SESSION_USER_ID_KEY = "auth_user_id"
 SESSION_USER_EMAIL_KEY = "auth_user_email"
 SESSION_USER_IS_ADMIN_KEY = "auth_user_is_admin"

app/db/background_session.py

@@ -0,0 +1,57 @@
+"""Semaphore-gated DB sessions for background tasks.
+
+Background tasks (ingestion, PDF resolution, scheduler) acquire a semaphore
+permit before checking out a database connection.  The semaphore cap is
+``pool_size + max_overflow - reserved_for_api``, which guarantees that at
+least *reserved_for_api* connections remain available for API request
+handlers at all times.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.session import get_session_factory
+from app.logging_utils import structured_log
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+_semaphore: asyncio.Semaphore | None = None
+
+
+def _build_semaphore() -> asyncio.Semaphore:
+    pool_capacity = max(1, settings.database_pool_size) + max(0, settings.database_pool_max_overflow)
+    reserved = max(0, settings.database_reserved_api_connections)
+    limit = max(1, pool_capacity - reserved)
+    structured_log(
+        logger,
+        "info",
+        "db.background_semaphore_initialized",
+        pool_capacity=pool_capacity,
+        reserved_for_api=reserved,
+        background_limit=limit,
+    )
+    return asyncio.Semaphore(limit)
+
+
+def get_background_semaphore() -> asyncio.Semaphore:
+    global _semaphore
+    if _semaphore is None:
+        _semaphore = _build_semaphore()
+    return _semaphore
+
+
+@asynccontextmanager
+async def background_session() -> AsyncIterator[AsyncSession]:
+    """Yield a DB session after acquiring the background semaphore."""
+    semaphore = get_background_semaphore()
+    async with semaphore:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            yield session

app/db/base.py

@@ -1,9 +1,8 @@
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 
 from sqlalchemy import MetaData, event
 from sqlalchemy.orm import DeclarativeBase
 
-
 NAMING_CONVENTION = {
     "ix": "ix_%(column_0_label)s",
     "uq": "uq_%(table_name)s_%(column_0_name)s",
@@ -21,7 +20,7 @@ class Base(DeclarativeBase):
 def _set_updated_at_before_update(_mapper, _connection, target) -> None:
     # Keep audit timestamps current for ORM-managed updates.
     if hasattr(target, "updated_at"):
-        target.updated_at = datetime.now(timezone.utc)
+        target.updated_at = datetime.now(UTC)
 
 
 metadata = Base.metadata

app/db/models.py

@@ -8,6 +8,7 @@ from sqlalchemy import (
     CheckConstraint,
     DateTime,
     Enum,
+    Float,
     ForeignKey,
     Index,
     Integer,
@@ -30,9 +31,11 @@ class RunTriggerType(StrEnum):
 
 class RunStatus(StrEnum):
     RUNNING = "running"
+    RESOLVING = "resolving"
     SUCCESS = "success"
     PARTIAL_FAILURE = "partial_failure"
     FAILED = "failed"
+    CANCELED = "canceled"
 
 
 class QueueItemStatus(StrEnum):
@@ -59,18 +62,10 @@ class User(Base):
     id: Mapped[int] = mapped_column(primary_key=True)
     email: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
     password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
-    is_active: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, server_default=text("true")
-    )
-    is_admin: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, server_default=text("false")
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
+    is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("true"))
+    is_admin: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
 
 
 class UserSetting(Base):
@@ -81,23 +76,15 @@ class UserSetting(Base):
             name="run_interval_minutes_min_15",
         ),
         CheckConstraint(
-            "request_delay_seconds >= 1",
-            name="request_delay_seconds_min_1",
+            "request_delay_seconds >= 2",
+            name="request_delay_seconds_min_2",
         ),
     )
 
-    user_id: Mapped[int] = mapped_column(
-        ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
-    )
-    auto_run_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, server_default=text("false")
-    )
-    run_interval_minutes: Mapped[int] = mapped_column(
-        Integer, nullable=False, server_default=text("1440")
-    )
-    request_delay_seconds: Mapped[int] = mapped_column(
-        Integer, nullable=False, server_default=text("10")
-    )
+    user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), primary_key=True)
+    auto_run_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
+    run_interval_minutes: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("1440"))
+    request_delay_seconds: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("10"))
     nav_visible_pages: Mapped[list[str]] = mapped_column(
         JSONB,
         nullable=False,
@@ -112,12 +99,13 @@ class UserSetting(Base):
     )
     scrape_cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     scrape_cooldown_reason: Mapped[str | None] = mapped_column(String(64))
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
+
+    openalex_api_key: Mapped[str | None] = mapped_column(String(255))
+    crossref_api_token: Mapped[str | None] = mapped_column(String(255))
+    crossref_api_mailto: Mapped[str | None] = mapped_column(String(255))
+
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
 
 
 class ScholarProfile(Base):
@@ -128,9 +116,7 @@ class ScholarProfile(Base):
     )
 
     id: Mapped[int] = mapped_column(primary_key=True)
-    user_id: Mapped[int] = mapped_column(
-        ForeignKey("users.id", ondelete="CASCADE"), nullable=False
-    )
+    user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
     scholar_id: Mapped[str] = mapped_column(String(64), nullable=False)
     display_name: Mapped[str | None] = mapped_column(String(255))
     profile_image_url: Mapped[str | None] = mapped_column(Text)
@@ -138,69 +124,47 @@ class ScholarProfile(Base):
     profile_image_upload_path: Mapped[str | None] = mapped_column(Text)
     last_initial_page_fingerprint_sha256: Mapped[str | None] = mapped_column(String(64))
     last_initial_page_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    is_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, server_default=text("true")
-    )
-    baseline_completed: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, server_default=text("false")
-    )
+    is_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("true"))
+    baseline_completed: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
     last_run_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     last_run_status: Mapped[RunStatus | None] = mapped_column(
         RUN_STATUS_DB_ENUM,
     )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
 
 
 class CrawlRun(Base):
     __tablename__ = "crawl_runs"
     __table_args__ = (
         Index("ix_crawl_runs_user_start", "user_id", "start_dt"),
+        Index(
+            "uq_crawl_runs_user_active",
+            "user_id",
+            unique=True,
+            postgresql_where=text("status IN ('running'::run_status, 'resolving'::run_status)"),
+        ),
         Index(
             "uq_crawl_runs_user_manual_idempotency_key",
             "user_id",
             "idempotency_key",
             unique=True,
-            postgresql_where=text(
-                "idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"
-            ),
+            postgresql_where=text("idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"),
         ),
     )
 
     id: Mapped[int] = mapped_column(primary_key=True)
-    user_id: Mapped[int] = mapped_column(
-        ForeignKey("users.id", ondelete="CASCADE"), nullable=False
-    )
-    trigger_type: Mapped[RunTriggerType] = mapped_column(
-        RUN_TRIGGER_TYPE_DB_ENUM, nullable=False
-    )
-    status: Mapped[RunStatus] = mapped_column(
-        RUN_STATUS_DB_ENUM, nullable=False
-    )
-    start_dt: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
+    user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
+    trigger_type: Mapped[RunTriggerType] = mapped_column(RUN_TRIGGER_TYPE_DB_ENUM, nullable=False)
+    status: Mapped[RunStatus] = mapped_column(RUN_STATUS_DB_ENUM, nullable=False)
+    start_dt: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
     end_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    scholar_count: Mapped[int] = mapped_column(
-        Integer, nullable=False, server_default=text("0")
-    )
-    new_pub_count: Mapped[int] = mapped_column(
-        Integer, nullable=False, server_default=text("0")
-    )
+    scholar_count: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
+    new_pub_count: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
     idempotency_key: Mapped[str | None] = mapped_column(String(128))
-    error_log: Mapped[dict] = mapped_column(
-        JSONB, nullable=False, server_default=text("'{}'::jsonb")
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
+    error_log: Mapped[dict] = mapped_column(JSONB, nullable=False, server_default=text("'{}'::jsonb"))
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
 
 
 class Publication(Base):
@@ -221,25 +185,129 @@ class Publication(Base):
     title_raw: Mapped[str] = mapped_column(Text, nullable=False)
     title_normalized: Mapped[str] = mapped_column(Text, nullable=False)
     year: Mapped[int | None] = mapped_column(Integer)
-    citation_count: Mapped[int] = mapped_column(
-        Integer, nullable=False, server_default=text("0")
-    )
+    citation_count: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
     author_text: Mapped[str | None] = mapped_column(Text)
     venue_text: Mapped[str | None] = mapped_column(Text)
     pub_url: Mapped[str | None] = mapped_column(Text)
     pdf_url: Mapped[str | None] = mapped_column(Text)
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+    canonical_title_hash: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
+    openalex_enriched: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
+    openalex_last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+
+
+class PublicationIdentifier(Base):
+    __tablename__ = "publication_identifiers"
+    __table_args__ = (
+        UniqueConstraint(
+            "publication_id",
+            "kind",
+            "value_normalized",
+            name="uq_publication_identifiers_publication_kind_value",
+        ),
+        CheckConstraint(
+            "confidence_score >= 0 AND confidence_score <= 1",
+            name="publication_identifiers_confidence_score_range",
+        ),
+        Index(
+            "ix_publication_identifiers_kind_value",
+            "kind",
+            "value_normalized",
+        ),
+        Index(
+            "ix_publication_identifiers_publication_id",
+            "publication_id",
+        ),
     )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    publication_id: Mapped[int] = mapped_column(
+        ForeignKey("publications.id", ondelete="CASCADE"),
+        nullable=False,
     )
+    kind: Mapped[str] = mapped_column(String(32), nullable=False)
+    value_raw: Mapped[str] = mapped_column(Text, nullable=False)
+    value_normalized: Mapped[str] = mapped_column(String(255), nullable=False)
+    source: Mapped[str] = mapped_column(String(64), nullable=False)
+    confidence_score: Mapped[float] = mapped_column(
+        Float,
+        nullable=False,
+        server_default=text("0"),
+    )
+    evidence_url: Mapped[str | None] = mapped_column(Text)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+
+
+class PublicationPdfJob(Base):
+    __tablename__ = "publication_pdf_jobs"
+    __table_args__ = (
+        CheckConstraint(
+            "status IN ('queued', 'running', 'resolved', 'failed')",
+            name="publication_pdf_jobs_status_valid",
+        ),
+        Index("ix_publication_pdf_jobs_status", "status"),
+        Index("ix_publication_pdf_jobs_updated_at", "updated_at"),
+        Index("ix_publication_pdf_jobs_queued_at", "queued_at"),
+    )
+
+    publication_id: Mapped[int] = mapped_column(
+        ForeignKey("publications.id", ondelete="CASCADE"),
+        primary_key=True,
+    )
+    status: Mapped[str] = mapped_column(
+        String(16),
+        nullable=False,
+        server_default=text("'queued'"),
+    )
+    attempt_count: Mapped[int] = mapped_column(
+        Integer,
+        nullable=False,
+        server_default=text("0"),
+    )
+    queued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    last_failure_reason: Mapped[str | None] = mapped_column(String(64))
+    last_failure_detail: Mapped[str | None] = mapped_column(Text)
+    last_source: Mapped[str | None] = mapped_column(String(32))
+    last_requested_by_user_id: Mapped[int | None] = mapped_column(
+        ForeignKey("users.id", ondelete="SET NULL"),
+    )
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+
+
+class PublicationPdfJobEvent(Base):
+    __tablename__ = "publication_pdf_job_events"
+    __table_args__ = (
+        Index("ix_publication_pdf_job_events_publication_created", "publication_id", "created_at"),
+        Index("ix_publication_pdf_job_events_created_at", "created_at"),
+        Index("ix_publication_pdf_job_events_event_type", "event_type"),
+    )
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    publication_id: Mapped[int] = mapped_column(
+        ForeignKey("publications.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    user_id: Mapped[int | None] = mapped_column(
+        ForeignKey("users.id", ondelete="SET NULL"),
+    )
+    event_type: Mapped[str] = mapped_column(String(32), nullable=False)
+    status: Mapped[str | None] = mapped_column(String(16))
+    source: Mapped[str | None] = mapped_column(String(32))
+    failure_reason: Mapped[str | None] = mapped_column(String(64))
+    message: Mapped[str | None] = mapped_column(Text)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
 
 
 class ScholarPublication(Base):
     __tablename__ = "scholar_publications"
     __table_args__ = (
         Index("ix_scholar_publications_is_read", "is_read"),
+        Index("ix_scholar_publications_is_favorite", "is_favorite"),
     )
 
     scholar_profile_id: Mapped[int] = mapped_column(
@@ -250,15 +318,10 @@ class ScholarPublication(Base):
         ForeignKey("publications.id", ondelete="CASCADE"),
         primary_key=True,
     )
-    is_read: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, server_default=text("false")
-    )
-    first_seen_run_id: Mapped[int | None] = mapped_column(
-        ForeignKey("crawl_runs.id", ondelete="SET NULL")
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
+    is_read: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
+    is_favorite: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
+    first_seen_run_id: Mapped[int | None] = mapped_column(ForeignKey("crawl_runs.id", ondelete="SET NULL"))
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
 
 
 class IngestionQueueItem(Base):
@@ -313,9 +376,121 @@ class IngestionQueueItem(Base):
     last_error: Mapped[str | None] = mapped_column(Text)
     dropped_reason: Mapped[str | None] = mapped_column(String(128))
     dropped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+
+
+class DataRepairJob(Base):
+    __tablename__ = "data_repair_jobs"
+    __table_args__ = (
+        CheckConstraint(
+            "status IN ('planned', 'running', 'completed', 'failed')",
+            name="data_repair_jobs_status_valid",
+        ),
+        Index("ix_data_repair_jobs_created_at", "created_at"),
+        Index("ix_data_repair_jobs_job_name_created_at", "job_name", "created_at"),
     )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    job_name: Mapped[str] = mapped_column(String(64), nullable=False)
+    requested_by: Mapped[str | None] = mapped_column(String(255))
+    scope: Mapped[dict] = mapped_column(
+        JSONB,
+        nullable=False,
+        server_default=text("'{}'::jsonb"),
     )
+    dry_run: Mapped[bool] = mapped_column(
+        Boolean,
+        nullable=False,
+        server_default=text("true"),
+    )
+    status: Mapped[str] = mapped_column(
+        String(16),
+        nullable=False,
+        server_default=text("'planned'"),
+    )
+    summary: Mapped[dict] = mapped_column(
+        JSONB,
+        nullable=False,
+        server_default=text("'{}'::jsonb"),
+    )
+    error_text: Mapped[str | None] = mapped_column(Text)
+    started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+
+
+class AuthorSearchRuntimeState(Base):
+    __tablename__ = "author_search_runtime_state"
+
+    state_key: Mapped[str] = mapped_column(String(64), primary_key=True)
+    last_live_request_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    consecutive_blocked_count: Mapped[int] = mapped_column(
+        Integer,
+        nullable=False,
+        server_default=text("0"),
+    )
+    cooldown_rejection_count: Mapped[int] = mapped_column(
+        Integer,
+        nullable=False,
+        server_default=text("0"),
+    )
+    cooldown_alert_emitted: Mapped[bool] = mapped_column(
+        Boolean,
+        nullable=False,
+        server_default=text("false"),
+    )
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+
+
+class ArxivRuntimeState(Base):
+    __tablename__ = "arxiv_runtime_state"
+
+    state_key: Mapped[str] = mapped_column(String(64), primary_key=True)
+    next_allowed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+
+
+class ArxivQueryCacheEntry(Base):
+    __tablename__ = "arxiv_query_cache_entries"
+    __table_args__ = (
+        Index("ix_arxiv_query_cache_expires_at", "expires_at"),
+        Index("ix_arxiv_query_cache_cached_at", "cached_at"),
+    )
+
+    query_fingerprint: Mapped[str] = mapped_column(String(64), primary_key=True)
+    payload: Mapped[dict] = mapped_column(
+        JSONB,
+        nullable=False,
+    )
+    expires_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        nullable=False,
+    )
+    cached_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+
+
+class AuthorSearchCacheEntry(Base):
+    __tablename__ = "author_search_cache_entries"
+    __table_args__ = (
+        Index("ix_author_search_cache_expires_at", "expires_at"),
+        Index("ix_author_search_cache_cached_at", "cached_at"),
+    )
+
+    query_key: Mapped[str] = mapped_column(String(256), primary_key=True)
+    payload: Mapped[dict] = mapped_column(
+        JSONB,
+        nullable=False,
+    )
+    expires_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        nullable=False,
+    )
+    cached_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())

app/db/session.py

@@ -1,5 +1,6 @@
-from collections.abc import AsyncIterator
 import logging
+import os
+from collections.abc import AsyncIterator
 
 from sqlalchemy import text
 from sqlalchemy.ext.asyncio import (
@@ -10,6 +11,7 @@ from sqlalchemy.ext.asyncio import (
 )
 from sqlalchemy.pool import NullPool
 
+from app.logging_utils import structured_log
 from app.settings import settings
 
 logger = logging.getLogger(__name__)
@@ -18,16 +20,48 @@ _engine: AsyncEngine | None = None
 _session_factory: async_sessionmaker[AsyncSession] | None = None
 
 
+def _normalized_pool_mode(raw_mode: str) -> str:
+    mode = (raw_mode or "").strip().lower()
+    if mode == "auto":
+        if os.getenv("PYTEST_CURRENT_TEST"):
+            return "null"
+        app_env = (os.getenv("APP_ENV") or "").strip().lower()
+        if app_env in {"test", "development", "dev", "local"}:
+            return "null"
+        return "queue"
+    if mode in {"null", "queue"}:
+        return mode
+    structured_log(
+        logger,
+        "warning",
+        "db.invalid_pool_mode_fallback",
+        database_pool_mode=raw_mode,
+        fallback_mode="queue",
+    )
+    return "queue"
+
+
 def get_engine() -> AsyncEngine:
     global _engine
     if _engine is None:
-        # NullPool avoids cross-event-loop connection reuse in tests and dev tools.
-        _engine = create_async_engine(
-            settings.database_url,
-            pool_pre_ping=True,
-            poolclass=NullPool,
+        pool_mode = _normalized_pool_mode(settings.database_pool_mode)
+        engine_kwargs: dict[str, object] = {
+            "pool_pre_ping": True,
+        }
+        if pool_mode == "null":
+            engine_kwargs["poolclass"] = NullPool
+        else:
+            engine_kwargs["pool_size"] = max(1, int(settings.database_pool_size))
+            engine_kwargs["max_overflow"] = max(0, int(settings.database_pool_max_overflow))
+            engine_kwargs["pool_timeout"] = max(1, int(settings.database_pool_timeout_seconds))
+
+        _engine = create_async_engine(settings.database_url, **engine_kwargs)
+        structured_log(
+            logger,
+            "info",
+            "db.engine_initialized",
+            pool_mode=pool_mode,
         )
-        logger.info("db.engine_initialized", extra={"event": "db.engine_initialized"})
     return _engine
 
 
@@ -51,7 +85,7 @@ async def check_database() -> bool:
             result = await conn.execute(text("SELECT 1"))
             return result.scalar_one() == 1
     except Exception:
-        logger.exception("db.healthcheck_failed", extra={"event": "db.healthcheck_failed"})
+        structured_log(logger, "exception", "db.healthcheck_failed")
         return False
 
 
@@ -59,6 +93,6 @@ async def close_engine() -> None:
     global _engine, _session_factory
     if _engine is not None:
         await _engine.dispose()
-        logger.info("db.engine_disposed", extra={"event": "db.engine_disposed"})
+        structured_log(logger, "info", "db.engine_disposed")
         _engine = None
         _session_factory = None

app/http/middleware.py

@@ -1,14 +1,15 @@
 from __future__ import annotations
 
-from secrets import token_urlsafe
 import logging
 import time
+from secrets import token_urlsafe
 
 from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.requests import Request
 from starlette.responses import Response
 
 from app.logging_context import set_request_id
+from app.logging_utils import structured_log
 
 REQUEST_ID_HEADER = "X-Request-ID"
 DEFAULT_PERMISSIONS_POLICY = (
@@ -61,42 +62,39 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
         start = time.perf_counter()
         should_log = self._log_requests and not self._is_skipped_path(request.url.path)
         if should_log:
-            logger.info(
+            structured_log(
+                logger,
+                "debug",
                 "request.started",
-                extra={
-                    "event": "request.started",
-                    "method": request.method,
-                    "path": request.url.path,
-                },
+                method=request.method,
+                path=request.url.path,
             )
 
         try:
             response = await call_next(request)
         except Exception:
             duration_ms = int((time.perf_counter() - start) * 1000)
-            logger.exception(
+            structured_log(
+                logger,
+                "exception",
                 "request.failed",
-                extra={
-                    "event": "request.failed",
-                    "method": request.method,
-                    "path": request.url.path,
-                    "duration_ms": duration_ms,
-                },
+                method=request.method,
+                path=request.url.path,
+                duration_ms=duration_ms,
             )
             raise
         else:
             duration_ms = int((time.perf_counter() - start) * 1000)
             response.headers[REQUEST_ID_HEADER] = request_id
             if should_log:
-                logger.info(
+                structured_log(
+                    logger,
+                    "debug",
                     "request.completed",
-                    extra={
-                        "event": "request.completed",
-                        "method": request.method,
-                        "path": request.url.path,
-                        "status_code": response.status_code,
-                        "duration_ms": duration_ms,
-                    },
+                    method=request.method,
+                    path=request.url.path,
+                    status_code=response.status_code,
+                    duration_ms=duration_ms,
                 )
             return response
         finally:
@@ -170,11 +168,7 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
 
         csp_policy = self._csp_policy_for_path(request.url.path)
         if self._csp_enabled and csp_policy:
-            csp_header = (
-                "Content-Security-Policy-Report-Only"
-                if self._csp_report_only
-                else "Content-Security-Policy"
-            )
+            csp_header = "Content-Security-Policy-Report-Only" if self._csp_report_only else "Content-Security-Policy"
             response.headers.setdefault(csp_header, csp_policy)
 
         hsts = self._strict_transport_security_value()

app/logging_config.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 
-from datetime import datetime, timezone
 import json
 import logging
 import sys
+from datetime import UTC, datetime
 from typing import Any
 
 from app.logging_context import get_request_id
@@ -102,12 +102,22 @@ class JsonLogFormatter(logging.Formatter):
         if key.lower() in self._redact_fields:
             return "[REDACTED]"
         if isinstance(value, dict):
-            return {nested_key: self._redact_value(nested_key, nested_value) for nested_key, nested_value in value.items()}
+            return {
+                nested_key: self._redact_value(nested_key, nested_value) for nested_key, nested_value in value.items()
+            }
         if isinstance(value, (list, tuple)):
             return [self._redact_value(key, item) for item in value]
         return value
 
 
+_CONSOLE_SHORT_KEYS = {
+    "user_id": "user",
+    "scholar_id": "scholar",
+    "crawl_run_id": "run",
+    "run_id": "run",
+}
+
+
 class ConsoleLogFormatter(logging.Formatter):
     def __init__(self, *, redact_fields: set[str]) -> None:
         super().__init__()
@@ -140,7 +150,8 @@ class ConsoleLogFormatter(logging.Formatter):
         for key in sorted(payload.keys()):
             if key in {"timestamp", "level", "logger", "event", "exception"}:
                 continue
-            parts.append(f"{key}={payload[key]}")
+            display_key = _CONSOLE_SHORT_KEYS.get(key, key)
+            parts.append(f"{display_key}={payload[key]}")
 
         if "exception" in payload:
             parts.append(f"exception={payload['exception']}")
@@ -168,7 +179,7 @@ def _normalize_level(level: str) -> int:
 
 
 def _format_timestamp(created_ts: float) -> str:
-    dt = datetime.fromtimestamp(created_ts, tz=timezone.utc)
+    dt = datetime.fromtimestamp(created_ts, tz=UTC)
     return dt.strftime("%Y-%m-%d %H:%M:%SZ")
 
 

app/logging_context.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 from contextvars import ContextVar
 
-
 _request_id_ctx: ContextVar[str | None] = ContextVar("request_id", default=None)
 
 
@@ -12,4 +11,3 @@ def get_request_id() -> str | None:
 
 def set_request_id(value: str | None) -> None:
     _request_id_ctx.set(value)
-

app/logging_utils.py

@@ -0,0 +1,29 @@
+"""Structured logging utility — eliminates boilerplate across domain services."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+
+def structured_log(
+    logger: logging.Logger,
+    level: str,
+    event: str,
+    /,
+    **fields: Any,
+) -> None:
+    """Emit a structured log entry.
+
+    The event name is passed as the log message. The JsonLogFormatter in
+    logging_config.py extracts it via record.getMessage() when no explicit
+    'event' key exists in extra — so we do NOT duplicate it.
+
+    Usage:
+        structured_log(logger, "info", "ingestion.run_started", user_id=1, scholar_count=5)
+    """
+    fields.pop("metric_name", None)
+    fields.pop("metric_value", None)
+
+    log_method = getattr(logger, level.lower())
+    log_method(event, extra=fields)

app/main.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 from contextlib import asynccontextmanager
 from pathlib import Path
 
@@ -9,20 +10,23 @@ from fastapi.staticfiles import StaticFiles
 from starlette.middleware.sessions import SessionMiddleware
 
 from app.api.errors import register_api_exception_handlers
+from app.api.media import router as media_router
 from app.api.router import router as api_router
-from app.api.runtime_deps import get_ingestion_service, get_scholar_source
-from app.db.session import check_database
-from app.db.session import close_engine
+from app.db.session import check_database, close_engine
 from app.http.middleware import (
     RequestLoggingMiddleware,
     SecurityHeadersMiddleware,
     parse_skip_paths,
 )
 from app.logging_config import configure_logging, parse_redact_fields
+from app.logging_utils import structured_log
 from app.security.csrf import CSRFMiddleware
-from app.services.scheduler import SchedulerService
+from app.services.ingestion.scheduler import SchedulerService
 from app.settings import settings
 
+logger = logging.getLogger(__name__)
+BUILD_MARKER = "2026-02-19.phase2.direct-pdf-import-export-dashboard-sync"
+
 configure_logging(
     level=settings.log_level,
     log_format=settings.log_format,
@@ -47,6 +51,56 @@ scheduler_service = SchedulerService(
 
 @asynccontextmanager
 async def lifespan(_: FastAPI):
+    structured_log(
+        logger,
+        "info",
+        "app.startup_build_marker",
+        build_marker=BUILD_MARKER,
+        frontend_enabled=settings.frontend_enabled,
+        scheduler_enabled=settings.scheduler_enabled,
+        log_format=settings.log_format,
+    )
+
+    from sqlalchemy import text
+
+    from app.db.session import get_session_factory
+
+    try:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            await session.execute(
+                text("UPDATE crawl_runs SET status = 'failed' WHERE status::text IN ('running', 'resolving')")
+            )
+            await session.commit()
+            structured_log(logger, "info", "app.startup_orphaned_runs_cleaned")
+    except Exception as exc:
+        structured_log(
+            logger,
+            "error",
+            "app.startup_orphaned_runs_cleanup_failed",
+            error=str(exc),
+        )
+
+    try:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            await session.execute(
+                text(
+                    "UPDATE publication_pdf_jobs SET status = 'queued'"
+                    " WHERE status = 'running'"
+                    " AND (last_attempt_at IS NULL OR last_attempt_at < NOW() - INTERVAL '10 minutes')"
+                )
+            )
+            await session.commit()
+            structured_log(logger, "info", "app.startup_stuck_pdf_jobs_recovered")
+    except Exception as exc:
+        structured_log(
+            logger,
+            "error",
+            "app.startup_stuck_pdf_jobs_recovery_failed",
+            error=str(exc),
+        )
+
     await scheduler_service.start()
     yield
     await scheduler_service.stop()
@@ -82,12 +136,11 @@ app.add_middleware(
     content_security_policy_report_only=settings.security_csp_report_only,
     strict_transport_security_enabled=settings.security_strict_transport_security_enabled,
     strict_transport_security_max_age=settings.security_strict_transport_security_max_age,
-    strict_transport_security_include_subdomains=(
-        settings.security_strict_transport_security_include_subdomains
-    ),
+    strict_transport_security_include_subdomains=(settings.security_strict_transport_security_include_subdomains),
     strict_transport_security_preload=settings.security_strict_transport_security_preload,
 )
 app.include_router(api_router)
+app.include_router(media_router)
 
 
 @app.get("/healthz")

app/security/__init__.py

@@ -1,2 +1 @@
 """Security middleware and helpers for scholarr."""
-

app/security/csrf.py

@@ -6,9 +6,11 @@ from urllib.parse import parse_qs
 
 from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.requests import Request
-from starlette.responses import JSONResponse, PlainTextResponse, Response
+from starlette.responses import PlainTextResponse, Response
 from starlette.types import Message
 
+from app.api.responses import error_response
+from app.logging_utils import structured_log
 
 CSRF_SESSION_KEY = "csrf_token"
 CSRF_FORM_FIELD = "csrf_token"
@@ -36,13 +38,12 @@ class CSRFMiddleware(BaseHTTPMiddleware):
 
         session_token = request.session.get(CSRF_SESSION_KEY)
         if not session_token:
-            logger.warning(
+            structured_log(
+                logger,
+                "warning",
                 "csrf.missing_session_token",
-                extra={
-                    "event": "csrf.missing_session_token",
-                    "method": request.method,
-                    "path": request.url.path,
-                },
+                method=request.method,
+                path=request.url.path,
             )
             return self._csrf_error_response(
                 request,
@@ -54,16 +55,15 @@ class CSRFMiddleware(BaseHTTPMiddleware):
         if request_token is None and self._is_form_payload(request):
             body = await request.body()
             request_token = self._token_from_form_body(request, body)
-            request._receive = self._build_receive(body)  # type: ignore[attr-defined]
+            request._receive = self._build_receive(body)
 
         if not request_token or not compare_digest(str(session_token), str(request_token)):
-            logger.warning(
+            structured_log(
+                logger,
+                "warning",
                 "csrf.invalid_token",
-                extra={
-                    "event": "csrf.invalid_token",
-                    "method": request.method,
-                    "path": request.url.path,
-                },
+                method=request.method,
+                path=request.url.path,
             )
             return self._csrf_error_response(
                 request,
@@ -89,7 +89,10 @@ class CSRFMiddleware(BaseHTTPMiddleware):
         content_type = request.headers.get("content-type", "")
         if not content_type.startswith("application/x-www-form-urlencoded"):
             return None
-        parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True)
+        try:
+            parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True)
+        except (UnicodeDecodeError, ValueError):
+            return None
         values = parsed.get(CSRF_FORM_FIELD)
         if not values:
             return None
@@ -115,19 +118,11 @@ class CSRFMiddleware(BaseHTTPMiddleware):
         message: str,
     ) -> Response:
         if request.url.path.startswith("/api/"):
-            request_state = getattr(request, "state", None)
-            request_id = getattr(request_state, "request_id", None) if request_state else None
-            return JSONResponse(
-                {
-                    "error": {
-                        "code": code,
-                        "message": message,
-                        "details": None,
-                    },
-                    "meta": {
-                        "request_id": request_id,
-                    },
-                },
+            return error_response(
+                request,
                 status_code=403,
+                code=code,
+                message=message,
+                details=None,
             )
         return PlainTextResponse(message, status_code=403)

app/services/__init__.py

@@ -1,2 +1 @@
 """Service layer for scholarr application workflows."""
-

app/services/arxiv/application.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from app.services.arxiv.gateway import (
+    build_arxiv_query,
+    get_arxiv_gateway,
+)
+
+if TYPE_CHECKING:
+    from app.services.publications.types import PublicationListItem, UnreadPublicationItem
+
+
+def _build_arxiv_query(title: str, author_surname: str | None) -> str | None:
+    return build_arxiv_query(title, author_surname)
+
+
+async def discover_arxiv_id_for_publication(
+    *,
+    item: PublicationListItem | UnreadPublicationItem,
+    request_email: str | None = None,
+    timeout_seconds: float | None = None,
+) -> str | None:
+    gateway = get_arxiv_gateway()
+    return await gateway.discover_arxiv_id_for_publication(
+        item=item,
+        request_email=request_email,
+        timeout_seconds=timeout_seconds,
+    )

app/services/arxiv/cache.py

@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+from collections.abc import Awaitable, Callable, Mapping
+from dataclasses import asdict
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from sqlalchemy import delete, func, select
+
+from app.db.models import ArxivQueryCacheEntry
+from app.db.session import get_session_factory
+from app.services.arxiv.constants import ARXIV_CACHE_FINGERPRINT_VERSION
+from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
+
+_INFLIGHT_LOCK = asyncio.Lock()
+_INFLIGHT_FEEDS: dict[str, asyncio.Future[ArxivFeed]] = {}
+
+
+def build_query_fingerprint(*, params: Mapping[str, object]) -> str:
+    canonical = _canonical_cache_payload(params=params)
+    encoded = json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
+    payload = f"{ARXIV_CACHE_FINGERPRINT_VERSION}:{encoded}"
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+async def get_cached_feed(
+    *,
+    query_fingerprint: str,
+    now_utc: datetime | None = None,
+) -> ArxivFeed | None:
+    timestamp = _as_utc(now_utc)
+    session_factory = get_session_factory()
+    async with session_factory() as db_session, db_session.begin():
+        result = await db_session.execute(
+            select(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint == query_fingerprint)
+        )
+        entry = result.scalar_one_or_none()
+        return await _validate_cached_entry(db_session, entry=entry, now_utc=timestamp)
+
+
+async def set_cached_feed(
+    *,
+    query_fingerprint: str,
+    feed: ArxivFeed,
+    ttl_seconds: float,
+    max_entries: int,
+    now_utc: datetime | None = None,
+) -> None:
+    timestamp = _as_utc(now_utc)
+    session_factory = get_session_factory()
+    async with session_factory() as db_session, db_session.begin():
+        await _write_cached_entry(
+            db_session,
+            query_fingerprint=query_fingerprint,
+            feed=feed,
+            ttl_seconds=ttl_seconds,
+            max_entries=max_entries,
+            now_utc=timestamp,
+        )
+
+
+async def run_with_inflight_dedupe(
+    *,
+    query_fingerprint: str,
+    fetch_feed: Callable[[], Awaitable[ArxivFeed]],
+) -> ArxivFeed:
+    future, is_owner = await _reserve_inflight_future(query_fingerprint=query_fingerprint)
+    if not is_owner:
+        return await asyncio.shield(future)
+    try:
+        result = await fetch_feed()
+    except Exception as exc:
+        _complete_future(future, error=exc)
+        raise
+    finally:
+        await _release_inflight_future(query_fingerprint=query_fingerprint, future=future)
+    _complete_future(future, result=result)
+    return result
+
+
+def _canonical_cache_payload(*, params: Mapping[str, object]) -> dict[str, object]:
+    payload: dict[str, object] = {}
+    for key in sorted(params.keys()):
+        payload[str(key)] = _normalize_param_value(str(key), params[key])
+    return payload
+
+
+def _normalize_param_value(key: str, value: object) -> object:
+    if key == "search_query":
+        return _normalize_search_query(str(value or ""))
+    if key == "id_list":
+        return _normalize_id_list(str(value or ""))
+    if isinstance(value, str):
+        return " ".join(value.strip().split())
+    if isinstance(value, (int, float, bool)) or value is None:
+        return value
+    return str(value).strip()
+
+
+def _normalize_search_query(value: str) -> str:
+    return " ".join(value.strip().lower().split())
+
+
+def _normalize_id_list(value: str) -> str:
+    normalized = [item.strip().lower() for item in value.split(",") if item.strip()]
+    return ",".join(sorted(normalized))
+
+
+async def _validate_cached_entry(
+    db_session,
+    *,
+    entry: ArxivQueryCacheEntry | None,
+    now_utc: datetime,
+) -> ArxivFeed | None:
+    if entry is None:
+        return None
+    if _as_utc(entry.expires_at) <= now_utc:
+        await db_session.delete(entry)
+        return None
+    parsed = _deserialize_feed(entry.payload)
+    if parsed is None:
+        await db_session.delete(entry)
+        return None
+    return parsed
+
+
+async def _write_cached_entry(
+    db_session,
+    *,
+    query_fingerprint: str,
+    feed: ArxivFeed,
+    ttl_seconds: float,
+    max_entries: int,
+    now_utc: datetime,
+) -> None:
+    ttl = max(float(ttl_seconds), 0.0)
+    result = await db_session.execute(
+        select(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint == query_fingerprint)
+    )
+    existing = result.scalar_one_or_none()
+    if ttl <= 0.0:
+        if existing is not None:
+            await db_session.delete(existing)
+        return
+    expires_at = now_utc + timedelta(seconds=ttl)
+    payload = _serialize_feed(feed)
+    if existing is None:
+        db_session.add(
+            ArxivQueryCacheEntry(
+                query_fingerprint=query_fingerprint,
+                payload=payload,
+                expires_at=expires_at,
+                cached_at=now_utc,
+                updated_at=now_utc,
+            )
+        )
+    else:
+        existing.payload = payload
+        existing.expires_at = expires_at
+        existing.cached_at = now_utc
+        existing.updated_at = now_utc
+    await _prune_cache_entries(db_session, now_utc=now_utc, max_entries=max_entries)
+
+
+async def _prune_cache_entries(
+    db_session,
+    *,
+    now_utc: datetime,
+    max_entries: int,
+) -> None:
+    await db_session.execute(delete(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.expires_at <= now_utc))
+    bounded_max_entries = int(max_entries)
+    if bounded_max_entries <= 0:
+        return
+    count_result = await db_session.execute(select(func.count()).select_from(ArxivQueryCacheEntry))
+    entry_count = int(count_result.scalar_one() or 0)
+    overflow = max(0, entry_count - bounded_max_entries)
+    if overflow <= 0:
+        return
+    stale_result = await db_session.execute(
+        select(ArxivQueryCacheEntry.query_fingerprint).order_by(ArxivQueryCacheEntry.cached_at.asc()).limit(overflow)
+    )
+    stale_keys = [str(row[0]) for row in stale_result.all()]
+    if stale_keys:
+        await db_session.execute(
+            delete(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint.in_(stale_keys))
+        )
+
+
+def _serialize_feed(feed: ArxivFeed) -> dict[str, Any]:
+    return asdict(feed)
+
+
+def _deserialize_feed(payload: object) -> ArxivFeed | None:
+    if not isinstance(payload, dict):
+        return None
+    entries_payload = payload.get("entries")
+    opensearch_payload = payload.get("opensearch")
+    if not isinstance(entries_payload, list):
+        return None
+    entries: list[ArxivEntry] = []
+    for value in entries_payload:
+        entry = _deserialize_entry(value)
+        if entry is None:
+            return None
+        entries.append(entry)
+    opensearch = _deserialize_opensearch(opensearch_payload)
+    if opensearch is None:
+        return None
+    return ArxivFeed(entries=entries, opensearch=opensearch)
+
+
+def _deserialize_entry(value: object) -> ArxivEntry | None:
+    if not isinstance(value, dict):
+        return None
+    try:
+        return ArxivEntry(
+            entry_id_url=str(value["entry_id_url"]),
+            arxiv_id=_as_optional_string(value.get("arxiv_id")),
+            title=str(value["title"]),
+            summary=str(value["summary"]),
+            published=_as_optional_string(value.get("published")),
+            updated=_as_optional_string(value.get("updated")),
+            authors=_as_string_list(value.get("authors")),
+            links=_as_string_list(value.get("links")),
+            categories=_as_string_list(value.get("categories")),
+            primary_category=_as_optional_string(value.get("primary_category")),
+        )
+    except KeyError:
+        return None
+
+
+def _deserialize_opensearch(value: object) -> ArxivOpenSearchMeta | None:
+    if not isinstance(value, dict):
+        return None
+    try:
+        return ArxivOpenSearchMeta(
+            total_results=int(value.get("total_results", 0)),
+            start_index=int(value.get("start_index", 0)),
+            items_per_page=int(value.get("items_per_page", 0)),
+        )
+    except (TypeError, ValueError):
+        return None
+
+
+def _as_optional_string(value: object) -> str | None:
+    if value is None:
+        return None
+    normalized = str(value).strip()
+    return normalized or None
+
+
+def _as_string_list(value: object) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(item) for item in value]
+
+
+def _as_utc(value: datetime | None) -> datetime:
+    if value is None:
+        return datetime.now(UTC)
+    if value.tzinfo is None:
+        return value.replace(tzinfo=UTC)
+    return value
+
+
+async def _reserve_inflight_future(
+    *,
+    query_fingerprint: str,
+) -> tuple[asyncio.Future[ArxivFeed], bool]:
+    async with _INFLIGHT_LOCK:
+        existing = _INFLIGHT_FEEDS.get(query_fingerprint)
+        if existing is not None:
+            return existing, False
+        loop = asyncio.get_running_loop()
+        created = loop.create_future()
+        created.add_done_callback(_consume_unretrieved_future_exception)
+        _INFLIGHT_FEEDS[query_fingerprint] = created
+        return created, True
+
+
+async def _release_inflight_future(
+    *,
+    query_fingerprint: str,
+    future: asyncio.Future[ArxivFeed],
+) -> None:
+    async with _INFLIGHT_LOCK:
+        current = _INFLIGHT_FEEDS.get(query_fingerprint)
+        if current is future:
+            _INFLIGHT_FEEDS.pop(query_fingerprint, None)
+
+
+def _complete_future(
+    future: asyncio.Future[ArxivFeed],
+    *,
+    result: ArxivFeed | None = None,
+    error: Exception | None = None,
+) -> None:
+    if future.done():
+        return
+    if error is not None:
+        future.set_exception(error)
+        return
+    if result is None:
+        raise RuntimeError("in-flight future completion requires result or error")
+    future.set_result(result)
+
+
+def _consume_unretrieved_future_exception(future: asyncio.Future[ArxivFeed]) -> None:
+    if future.cancelled():
+        return
+    try:
+        _ = future.exception()
+    except Exception:
+        return

app/services/arxiv/client.py

@@ -0,0 +1,278 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import Awaitable, Callable
+
+import httpx
+
+from app.logging_utils import structured_log
+from app.services.arxiv.cache import (
+    build_query_fingerprint,
+    get_cached_feed,
+    run_with_inflight_dedupe,
+    set_cached_feed,
+)
+from app.services.arxiv.constants import (
+    ARXIV_SOURCE_PATH_LOOKUP_IDS,
+    ARXIV_SOURCE_PATH_SEARCH,
+    ARXIV_SOURCE_PATH_UNKNOWN,
+)
+from app.services.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
+from app.services.arxiv.parser import parse_arxiv_feed
+from app.services.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
+from app.services.arxiv.types import ArxivFeed
+from app.settings import settings
+
+_ARXIV_API_URL = "https://export.arxiv.org/api/query"
+_ARXIV_QUERY_START = 0
+_ARXIV_MAX_RESULTS_LIMIT = 30_000
+_ARXIV_SORT_BY_ALLOWED = {"relevance", "lastUpdatedDate", "submittedDate"}
+_ARXIV_SORT_ORDER_ALLOWED = {"ascending", "descending"}
+_FALLBACK_CONTACT_EMAIL = "unknown@example.com"
+
+ArxivRequestFn = Callable[..., Awaitable[httpx.Response]]
+logger = logging.getLogger(__name__)
+
+
+class ArxivClient:
+    def __init__(
+        self,
+        *,
+        request_fn: ArxivRequestFn | None = None,
+        cache_enabled: bool | None = None,
+    ) -> None:
+        self._request_fn = request_fn or _request_arxiv_feed
+        self._cache_enabled = _resolve_cache_enabled(
+            cache_enabled=cache_enabled,
+            request_fn=request_fn,
+        )
+        self._cache_ttl_seconds = _cache_ttl_seconds()
+        self._cache_max_entries = _cache_max_entries()
+
+    async def search(
+        self,
+        *,
+        query: str,
+        start: int = _ARXIV_QUERY_START,
+        max_results: int | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        request_email: str | None = None,
+        timeout_seconds: float | None = None,
+    ) -> ArxivFeed:
+        params = _search_params(
+            query=query,
+            start=start,
+            max_results=max_results,
+            sort_by=sort_by,
+            sort_order=sort_order,
+        )
+        return await self._fetch_feed(
+            params=params,
+            request_email=request_email,
+            timeout_seconds=timeout_seconds,
+            source_path=ARXIV_SOURCE_PATH_SEARCH,
+        )
+
+    async def lookup_ids(
+        self,
+        *,
+        id_list: list[str],
+        start: int = _ARXIV_QUERY_START,
+        max_results: int | None = None,
+        request_email: str | None = None,
+        timeout_seconds: float | None = None,
+    ) -> ArxivFeed:
+        params = _lookup_params(id_list=id_list, start=start, max_results=max_results)
+        return await self._fetch_feed(
+            params=params,
+            request_email=request_email,
+            timeout_seconds=timeout_seconds,
+            source_path=ARXIV_SOURCE_PATH_LOOKUP_IDS,
+        )
+
+    async def _fetch_feed(
+        self,
+        *,
+        params: dict[str, object],
+        request_email: str | None,
+        timeout_seconds: float | None,
+        source_path: str,
+    ) -> ArxivFeed:
+        query_fingerprint = build_query_fingerprint(params=params)
+        if self._cache_enabled:
+            cached = await get_cached_feed(query_fingerprint=query_fingerprint)
+            if cached is not None:
+                structured_log(
+                    logger, "info", "arxiv.cache_hit", query_fingerprint=query_fingerprint, source_path=source_path
+                )
+                return cached
+            structured_log(
+                logger, "info", "arxiv.cache_miss", query_fingerprint=query_fingerprint, source_path=source_path
+            )
+        return await run_with_inflight_dedupe(
+            query_fingerprint=query_fingerprint,
+            fetch_feed=lambda: self._fetch_live_feed(
+                params=params,
+                request_email=request_email,
+                timeout_seconds=timeout_seconds,
+                query_fingerprint=query_fingerprint,
+            ),
+        )
+
+    async def _fetch_live_feed(
+        self,
+        *,
+        params: dict[str, object],
+        request_email: str | None,
+        timeout_seconds: float | None,
+        query_fingerprint: str,
+    ) -> ArxivFeed:
+        response = await self._request_fn(
+            params=params,
+            request_email=request_email,
+            timeout_seconds=timeout_seconds,
+        )
+        response.raise_for_status()
+        feed = parse_arxiv_feed(response.text)
+        if self._cache_enabled:
+            await set_cached_feed(
+                query_fingerprint=query_fingerprint,
+                feed=feed,
+                ttl_seconds=self._cache_ttl_seconds,
+                max_entries=self._cache_max_entries,
+            )
+        return feed
+
+
+def _search_params(
+    *,
+    query: str,
+    start: int,
+    max_results: int | None,
+    sort_by: str | None,
+    sort_order: str | None,
+) -> dict[str, object]:
+    clean_query = query.strip()
+    if not clean_query:
+        raise ArxivClientValidationError("search query must not be empty")
+    params: dict[str, object] = {
+        "search_query": clean_query,
+        "start": _validate_start(start),
+        "max_results": _validate_max_results(max_results),
+    }
+    if sort_by is not None:
+        params["sortBy"] = _validate_sort_by(sort_by)
+    if sort_order is not None:
+        params["sortOrder"] = _validate_sort_order(sort_order)
+    return params
+
+
+def _lookup_params(*, id_list: list[str], start: int, max_results: int | None) -> dict[str, object]:
+    normalized_ids = [value.strip() for value in id_list if value and value.strip()]
+    if not normalized_ids:
+        raise ArxivClientValidationError("id_list must include at least one id")
+    return {
+        "id_list": ",".join(normalized_ids),
+        "start": _validate_start(start),
+        "max_results": _validate_max_results(max_results),
+    }
+
+
+def _validate_start(value: int) -> int:
+    start = int(value)
+    if start < 0:
+        raise ArxivClientValidationError("start must be >= 0")
+    return start
+
+
+def _validate_max_results(value: int | None) -> int:
+    if value is None:
+        default_value = int(settings.arxiv_default_max_results)
+        return max(default_value, 1)
+    parsed = int(value)
+    if parsed < 1:
+        raise ArxivClientValidationError("max_results must be >= 1")
+    if parsed > _ARXIV_MAX_RESULTS_LIMIT:
+        raise ArxivClientValidationError(f"max_results must be <= {_ARXIV_MAX_RESULTS_LIMIT}")
+    return parsed
+
+
+def _validate_sort_by(value: str) -> str:
+    if value not in _ARXIV_SORT_BY_ALLOWED:
+        raise ArxivClientValidationError(f"sort_by must be one of: {sorted(_ARXIV_SORT_BY_ALLOWED)!r}")
+    return value
+
+
+def _validate_sort_order(value: str) -> str:
+    if value not in _ARXIV_SORT_ORDER_ALLOWED:
+        raise ArxivClientValidationError(f"sort_order must be one of: {sorted(_ARXIV_SORT_ORDER_ALLOWED)!r}")
+    return value
+
+
+async def _request_arxiv_feed(
+    *,
+    params: dict[str, object],
+    request_email: str | None,
+    timeout_seconds: float | None,
+) -> httpx.Response:
+    source_path = _source_path_from_params(params)
+    cooldown_status = await get_arxiv_cooldown_status()
+    if cooldown_status.is_active:
+        structured_log(
+            logger,
+            "warning",
+            "arxiv.request_skipped_cooldown",
+            source_path=source_path,
+            cooldown_remaining_seconds=float(cooldown_status.remaining_seconds),
+        )
+        raise ArxivRateLimitError(f"arXiv global cooldown active ({cooldown_status.remaining_seconds:.0f}s remaining)")
+
+    async def _fetch() -> httpx.Response:
+        timeout_value = _timeout_seconds(timeout_seconds)
+        headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{_contact_email(request_email)})"}
+        async with httpx.AsyncClient(timeout=timeout_value, follow_redirects=True, headers=headers) as client:
+            return await client.get(_ARXIV_API_URL, params=params)  # type: ignore[arg-type]
+
+    return await run_with_global_arxiv_limit(
+        fetch=_fetch,
+        source_path=source_path,
+    )
+
+
+def _timeout_seconds(timeout_seconds: float | None) -> float:
+    if timeout_seconds is not None:
+        return max(float(timeout_seconds), 0.5)
+    return max(float(settings.arxiv_timeout_seconds), 0.5)
+
+
+def _contact_email(request_email: str | None) -> str:
+    return request_email or settings.arxiv_mailto or settings.crossref_api_mailto or _FALLBACK_CONTACT_EMAIL
+
+
+def _resolve_cache_enabled(
+    *,
+    cache_enabled: bool | None,
+    request_fn: ArxivRequestFn | None,
+) -> bool:
+    if cache_enabled is not None:
+        return bool(cache_enabled)
+    if request_fn is not None:
+        return False
+    return _cache_ttl_seconds() > 0.0
+
+
+def _cache_ttl_seconds() -> float:
+    return max(float(settings.arxiv_cache_ttl_seconds), 0.0)
+
+
+def _cache_max_entries() -> int:
+    return max(int(settings.arxiv_cache_max_entries), 0)
+
+
+def _source_path_from_params(params: dict[str, object]) -> str:
+    if "search_query" in params:
+        return ARXIV_SOURCE_PATH_SEARCH
+    if "id_list" in params:
+        return ARXIV_SOURCE_PATH_LOOKUP_IDS
+    return ARXIV_SOURCE_PATH_UNKNOWN

app/services/arxiv/constants.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+ARXIV_RUNTIME_STATE_KEY = "global"
+ARXIV_RATE_LIMIT_LOCK_NAMESPACE = 91_100
+ARXIV_RATE_LIMIT_LOCK_KEY = 1
+ARXIV_SOURCE_PATH_SEARCH = "search"
+ARXIV_SOURCE_PATH_LOOKUP_IDS = "lookup_ids"
+ARXIV_SOURCE_PATH_UNKNOWN = "unknown"
+ARXIV_CACHE_FINGERPRINT_VERSION = "v1"
+ARXIV_TITLE_TOKEN_MIN_LENGTH = 3
+ARXIV_TITLE_MIN_TOKENS = 3
+ARXIV_TITLE_MIN_ALPHA_TOKENS = 2
+ARXIV_STRONG_IDENTIFIER_CONFIDENCE = 0.9

app/services/arxiv/errors.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+
+class ArxivRateLimitError(Exception):
+    """arXiv returned 429 or cooldown is active."""
+
+
+class ArxivClientValidationError(ValueError):
+    """arXiv client inputs are invalid."""
+
+
+class ArxivParseError(ValueError):
+    """arXiv API payload could not be parsed."""

app/services/arxiv/gateway.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import logging
+import re
+import unicodedata
+from typing import TYPE_CHECKING, Protocol
+
+from app.logging_utils import structured_log
+from app.services.arxiv.client import ArxivClient
+from app.services.arxiv.errors import ArxivRateLimitError
+from app.services.arxiv.types import ArxivFeed
+from app.settings import settings
+
+if TYPE_CHECKING:
+    from app.services.publications.types import PublicationListItem, UnreadPublicationItem
+
+logger = logging.getLogger(__name__)
+
+_default_gateway: ArxivGateway | None = None
+_MOJIBAKE_HINT_RE = re.compile(r"[ÃÂâ]")
+_NON_ALNUM_RE = re.compile(r"[^\w\s]+", re.UNICODE)
+_WHITESPACE_RE = re.compile(r"\s+")
+
+
+class ArxivGateway(Protocol):
+    async def discover_arxiv_id_for_publication(
+        self,
+        *,
+        item: PublicationListItem | UnreadPublicationItem,
+        request_email: str | None = None,
+        timeout_seconds: float | None = None,
+        max_results: int | None = None,
+    ) -> str | None: ...
+
+
+def build_arxiv_query(title: str, author_surname: str | None) -> str | None:
+    parts: list[str] = []
+    if title:
+        clean_title = _normalize_query_title(title)
+        if clean_title:
+            parts.append(f'ti:"{clean_title}"')
+    if author_surname:
+        clean_author = _normalize_query_title(author_surname)
+        if clean_author:
+            parts.append(f'au:"{clean_author}"')
+    if not parts:
+        return None
+    return " AND ".join(parts)
+
+
+def _normalize_query_title(value: str) -> str:
+    repaired = _repair_mojibake(value.strip())
+    normalized = unicodedata.normalize("NFKC", repaired)
+    stripped = _NON_ALNUM_RE.sub(" ", _MOJIBAKE_HINT_RE.sub(" ", normalized))
+    return _WHITESPACE_RE.sub(" ", stripped).strip()
+
+
+def _repair_mojibake(value: str) -> str:
+    if not value or not _MOJIBAKE_HINT_RE.search(value):
+        return value
+    try:
+        repaired = value.encode("latin1").decode("utf-8")
+    except UnicodeError:
+        return value
+    return repaired if _mojibake_score(repaired) < _mojibake_score(value) else value
+
+
+def _mojibake_score(value: str) -> int:
+    return len(_MOJIBAKE_HINT_RE.findall(value))
+
+
+def get_arxiv_gateway() -> ArxivGateway:
+    global _default_gateway
+    if _default_gateway is None:
+        _default_gateway = HttpArxivGateway()
+    return _default_gateway
+
+
+def set_arxiv_gateway(gateway: ArxivGateway | None) -> ArxivGateway | None:
+    global _default_gateway
+    previous = _default_gateway
+    _default_gateway = gateway
+    return previous
+
+
+class HttpArxivGateway:
+    def __init__(self, *, client: ArxivClient | None = None) -> None:
+        self._client = client or ArxivClient()
+
+    async def discover_arxiv_id_for_publication(
+        self,
+        *,
+        item: PublicationListItem | UnreadPublicationItem,
+        request_email: str | None = None,
+        timeout_seconds: float | None = None,
+        max_results: int | None = None,
+    ) -> str | None:
+        if not settings.arxiv_enabled:
+            return None
+        query = _query_for_item(item)
+        if query is None:
+            return None
+
+        try:
+            result = await self._client.search(
+                query=query,
+                start=0,
+                request_email=request_email,
+                timeout_seconds=timeout_seconds,
+                max_results=max_results,
+            )
+            return _first_discovered_id(result)
+        except ArxivRateLimitError:
+            raise
+        except Exception as exc:
+            structured_log(logger, "debug", "arxiv.query_failed", error=str(exc))
+            return None
+
+
+def _query_for_item(item: PublicationListItem | UnreadPublicationItem) -> str | None:
+    title = (item.title or "").strip()
+    if not title:
+        return None
+    author_surname = _author_surname(item.scholar_label)
+    return build_arxiv_query(title, author_surname)
+
+
+def _author_surname(scholar_label: str | None) -> str | None:
+    if not scholar_label:
+        return None
+    tokens = [token for token in scholar_label.strip().split() if token]
+    if not tokens:
+        return None
+    return tokens[-1].lower()
+
+
+def _first_discovered_id(result: ArxivFeed) -> str | None:
+    for entry in result.entries:
+        if entry.arxiv_id:
+            structured_log(logger, "debug", "arxiv.id_discovered", arxiv_id=entry.arxiv_id)
+            return entry.arxiv_id
+    return None

app/services/arxiv/guards.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+from app.services.arxiv.constants import (
+    ARXIV_STRONG_IDENTIFIER_CONFIDENCE,
+    ARXIV_TITLE_MIN_ALPHA_TOKENS,
+    ARXIV_TITLE_MIN_TOKENS,
+    ARXIV_TITLE_TOKEN_MIN_LENGTH,
+)
+from app.services.doi.normalize import normalize_doi
+from app.services.publication_identifiers.normalize import normalize_arxiv_id
+
+if TYPE_CHECKING:
+    from app.services.publications.types import PublicationListItem, UnreadPublicationItem
+
+_TITLE_TOKEN_RE = re.compile(r"[a-z0-9]+")
+
+
+def arxiv_skip_reason_for_item(
+    *,
+    item: PublicationListItem | UnreadPublicationItem,
+    has_strong_doi: bool = False,
+    has_existing_arxiv: bool = False,
+) -> str | None:
+    if has_existing_arxiv or _has_arxiv_identifier_evidence(item):
+        return "arxiv_identifier_present"
+    if has_strong_doi or _has_strong_doi_evidence(item):
+        return "strong_doi_present"
+    if not _title_passes_quality_guard(item.title):
+        return "title_quality_below_threshold"
+    return None
+
+
+def _has_arxiv_identifier_evidence(item: PublicationListItem | UnreadPublicationItem) -> bool:
+    if _display_identifier_matches(item, expected_kind="arxiv"):
+        return True
+    return _has_normalized_identifier(item, normalizer=normalize_arxiv_id)
+
+
+def _has_strong_doi_evidence(item: PublicationListItem | UnreadPublicationItem) -> bool:
+    if _display_identifier_matches(item, expected_kind="doi"):
+        return True
+    return _has_normalized_identifier(item, normalizer=normalize_doi)
+
+
+def _display_identifier_matches(
+    item: PublicationListItem | UnreadPublicationItem,
+    *,
+    expected_kind: str,
+) -> bool:
+    display = getattr(item, "display_identifier", None)
+    if display is None:
+        return False
+    if str(display.kind).lower() != expected_kind:
+        return False
+    return float(display.confidence_score) >= ARXIV_STRONG_IDENTIFIER_CONFIDENCE
+
+
+def _has_normalized_identifier(
+    item: PublicationListItem | UnreadPublicationItem,
+    *,
+    normalizer,
+) -> bool:
+    if normalizer(item.pub_url):
+        return True
+    return normalizer(item.pdf_url) is not None
+
+
+def _title_passes_quality_guard(title: str | None) -> bool:
+    tokens = _normalized_tokens(title or "")
+    if len(tokens) < ARXIV_TITLE_MIN_TOKENS:
+        return False
+    alpha_tokens = [token for token in tokens if _is_alpha_token(token)]
+    return len(alpha_tokens) >= ARXIV_TITLE_MIN_ALPHA_TOKENS
+
+
+def _normalized_tokens(value: str) -> list[str]:
+    return [token for token in _TITLE_TOKEN_RE.findall(value.lower()) if token]
+
+
+def _is_alpha_token(token: str) -> bool:
+    if len(token) < ARXIV_TITLE_TOKEN_MIN_LENGTH:
+        return False
+    return any(char.isalpha() for char in token)

app/services/arxiv/parser.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+
+from app.services.arxiv.errors import ArxivParseError
+from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
+from app.services.publication_identifiers.normalize import normalize_arxiv_id
+
+_NAMESPACES = {
+    "atom": "http://www.w3.org/2005/Atom",
+    "opensearch": "http://a9.com/-/spec/opensearch/1.1/",
+    "arxiv": "http://arxiv.org/schemas/atom",
+}
+
+
+def parse_arxiv_feed(payload: str) -> ArxivFeed:
+    root = _parse_xml_root(payload)
+    opensearch = ArxivOpenSearchMeta(
+        total_results=_opensearch_int(root, "opensearch:totalResults"),
+        start_index=_opensearch_int(root, "opensearch:startIndex"),
+        items_per_page=_opensearch_int(root, "opensearch:itemsPerPage"),
+    )
+    entries = [_parse_entry(entry_elem) for entry_elem in root.findall("atom:entry", _NAMESPACES)]
+    return ArxivFeed(entries=entries, opensearch=opensearch)
+
+
+def _parse_xml_root(payload: str) -> ET.Element:
+    try:
+        return ET.fromstring(payload)
+    except ET.ParseError as exc:
+        raise ArxivParseError(f"Invalid arXiv XML payload: {exc}") from exc
+
+
+def _opensearch_int(root: ET.Element, path: str) -> int:
+    text = _optional_text(root, path)
+    if text is None:
+        return 0
+    try:
+        return int(text.strip())
+    except ValueError as exc:
+        raise ArxivParseError(f"Invalid integer value at {path}: {text!r}") from exc
+
+
+def _parse_entry(entry_elem: ET.Element) -> ArxivEntry:
+    entry_id_url = _required_text(entry_elem, "atom:id").strip()
+    arxiv_id = normalize_arxiv_id(entry_id_url)
+    title = _required_text(entry_elem, "atom:title").strip()
+    summary = (_optional_text(entry_elem, "atom:summary") or "").strip()
+    published = _optional_text(entry_elem, "atom:published")
+    updated = _optional_text(entry_elem, "atom:updated")
+    return ArxivEntry(
+        entry_id_url=entry_id_url,
+        arxiv_id=arxiv_id,
+        title=title,
+        summary=summary,
+        published=published,
+        updated=updated,
+        authors=_authors(entry_elem),
+        links=_links(entry_elem),
+        categories=_categories(entry_elem),
+        primary_category=_primary_category(entry_elem),
+    )
+
+
+def _required_text(elem: ET.Element, path: str) -> str:
+    text = _optional_text(elem, path)
+    if text is None or not text.strip():
+        raise ArxivParseError(f"Missing required field: {path}")
+    return text
+
+
+def _optional_text(elem: ET.Element, path: str) -> str | None:
+    node = elem.find(path, _NAMESPACES)
+    if node is None or node.text is None:
+        return None
+    return str(node.text)
+
+
+def _authors(entry_elem: ET.Element) -> list[str]:
+    authors: list[str] = []
+    for author in entry_elem.findall("atom:author", _NAMESPACES):
+        name = _optional_text(author, "atom:name")
+        if name:
+            authors.append(name.strip())
+    return authors
+
+
+def _links(entry_elem: ET.Element) -> list[str]:
+    values: list[str] = []
+    for link in entry_elem.findall("atom:link", _NAMESPACES):
+        href = str(link.attrib.get("href") or "").strip()
+        if href:
+            values.append(href)
+    return values
+
+
+def _categories(entry_elem: ET.Element) -> list[str]:
+    values: list[str] = []
+    for cat in entry_elem.findall("atom:category", _NAMESPACES):
+        term = str(cat.attrib.get("term") or "").strip()
+        if term:
+            values.append(term)
+    return values
+
+
+def _primary_category(entry_elem: ET.Element) -> str | None:
+    node = entry_elem.find("arxiv:primary_category", _NAMESPACES)
+    if node is None:
+        return None
+    value = str(node.attrib.get("term") or "").strip()
+    return value or None

app/services/arxiv/rate_limit.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+
+import httpx
+from sqlalchemy import select, text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import ArxivRuntimeState
+from app.db.session import get_session_factory
+from app.logging_utils import structured_log
+from app.services.arxiv.constants import (
+    ARXIV_RATE_LIMIT_LOCK_KEY,
+    ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
+    ARXIV_RUNTIME_STATE_KEY,
+    ARXIV_SOURCE_PATH_UNKNOWN,
+)
+from app.services.arxiv.errors import ArxivRateLimitError
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class ArxivCooldownStatus:
+    is_active: bool
+    remaining_seconds: float
+    cooldown_until: datetime | None
+
+
+async def run_with_global_arxiv_limit(
+    *,
+    fetch: Callable[[], Awaitable[httpx.Response]],
+    source_path: str = ARXIV_SOURCE_PATH_UNKNOWN,
+) -> httpx.Response:
+    response, hit_rate_limit = await _run_serialized_fetch(
+        fetch=fetch,
+        source_path=source_path,
+    )
+    if hit_rate_limit:
+        raise ArxivRateLimitError("arXiv rate limit hit (429) — stopping batch")
+    return response
+
+
+async def get_arxiv_cooldown_status(*, now_utc: datetime | None = None) -> ArxivCooldownStatus:
+    timestamp = _normalize_datetime(now_utc) or datetime.now(UTC)
+    session_factory = get_session_factory()
+    async with session_factory() as db_session:
+        result = await db_session.execute(
+            select(ArxivRuntimeState.cooldown_until).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY)
+        )
+        cooldown_until = _normalize_datetime(result.scalar_one_or_none())
+    remaining_seconds = _cooldown_remaining_seconds(cooldown_until, now_utc=timestamp)
+    return ArxivCooldownStatus(
+        is_active=remaining_seconds > 0.0,
+        remaining_seconds=float(remaining_seconds),
+        cooldown_until=cooldown_until,
+    )
+
+
+async def _run_serialized_fetch(
+    *,
+    fetch: Callable[[], Awaitable[httpx.Response]],
+    source_path: str,
+) -> tuple[httpx.Response, bool]:
+    session_factory = get_session_factory()
+    async with session_factory() as lock_session:
+        await _acquire_arxiv_lock(lock_session)
+        try:
+            async with session_factory() as db_session, db_session.begin():
+                runtime_state = await _load_runtime_state_for_update(db_session)
+                wait_seconds = await _wait_for_allowed_slot_or_raise(
+                    runtime_state,
+                    source_path=source_path,
+                )
+                runtime_state.next_allowed_at = datetime.now(UTC) + timedelta(seconds=_min_interval_seconds())
+
+            if wait_seconds > 0:
+                await asyncio.sleep(wait_seconds)
+            response = await fetch()
+
+            async with session_factory() as db_session, db_session.begin():
+                runtime_state = await _load_runtime_state_for_update(db_session)
+                hit_rate_limit = _record_post_response_state(
+                    runtime_state,
+                    response_status=int(response.status_code),
+                    source_path=source_path,
+                )
+                cooldown_until_value = runtime_state.cooldown_until
+        finally:
+            await _release_arxiv_lock(lock_session)
+    structured_log(
+        logger,
+        "info",
+        "arxiv.request_completed",
+        status_code=int(response.status_code),
+        wait_seconds=wait_seconds,
+        cooldown_remaining_seconds=_cooldown_remaining_seconds(cooldown_until_value, now_utc=datetime.now(UTC)),
+        source_path=source_path,
+    )
+    return response, hit_rate_limit
+
+
+async def _acquire_arxiv_lock(db_session: AsyncSession) -> None:
+    await db_session.execute(
+        text("SELECT pg_advisory_lock(:namespace, :lock_key)"),
+        {
+            "namespace": ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
+            "lock_key": ARXIV_RATE_LIMIT_LOCK_KEY,
+        },
+    )
+
+
+async def _release_arxiv_lock(db_session: AsyncSession) -> None:
+    await db_session.execute(
+        text("SELECT pg_advisory_unlock(:namespace, :lock_key)"),
+        {
+            "namespace": ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
+            "lock_key": ARXIV_RATE_LIMIT_LOCK_KEY,
+        },
+    )
+
+
+async def _load_runtime_state_for_update(db_session: AsyncSession) -> ArxivRuntimeState:
+    result = await db_session.execute(
+        select(ArxivRuntimeState).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY).with_for_update()
+    )
+    state = result.scalar_one_or_none()
+    if state is not None:
+        return state
+    state = ArxivRuntimeState(state_key=ARXIV_RUNTIME_STATE_KEY)
+    db_session.add(state)
+    await db_session.flush()
+    return state
+
+
+async def _wait_for_allowed_slot_or_raise(
+    runtime_state: ArxivRuntimeState,
+    *,
+    source_path: str,
+) -> float:
+    now_utc = datetime.now(UTC)
+    cooldown_seconds = _cooldown_remaining_seconds(runtime_state.cooldown_until, now_utc=now_utc)
+    if cooldown_seconds > 0:
+        structured_log(
+            logger,
+            "info",
+            "arxiv.request_scheduled",
+            wait_seconds=0.0,
+            source_path=source_path,
+            cooldown_remaining_seconds=cooldown_seconds,
+        )
+        raise ArxivRateLimitError(f"arXiv global cooldown active ({cooldown_seconds:.0f}s remaining)")
+    wait_seconds = _next_allowed_wait_seconds(runtime_state.next_allowed_at, now_utc=now_utc)
+    structured_log(
+        logger,
+        "info",
+        "arxiv.request_scheduled",
+        wait_seconds=wait_seconds,
+        source_path=source_path,
+        cooldown_remaining_seconds=0.0,
+    )
+    return wait_seconds
+
+
+def _record_post_response_state(
+    runtime_state: ArxivRuntimeState,
+    *,
+    response_status: int,
+    source_path: str,
+) -> bool:
+    now_utc = datetime.now(UTC)
+    runtime_state.next_allowed_at = now_utc + timedelta(seconds=_min_interval_seconds())
+    if response_status == 429:
+        cooldown_seconds = _cooldown_seconds()
+        runtime_state.cooldown_until = now_utc + timedelta(seconds=cooldown_seconds)
+        structured_log(
+            logger,
+            "warning",
+            "arxiv.cooldown_activated",
+            cooldown_remaining_seconds=cooldown_seconds,
+            source_path=source_path,
+        )
+        return True
+    if _cooldown_remaining_seconds(runtime_state.cooldown_until, now_utc=now_utc) <= 0:
+        runtime_state.cooldown_until = None
+    return False
+
+
+def _cooldown_remaining_seconds(cooldown_until: datetime | None, *, now_utc: datetime) -> float:
+    bounded = _normalize_datetime(cooldown_until)
+    if bounded is None:
+        return 0.0
+    return max((bounded - now_utc).total_seconds(), 0.0)
+
+
+def _next_allowed_wait_seconds(next_allowed_at: datetime | None, *, now_utc: datetime) -> float:
+    bounded = _normalize_datetime(next_allowed_at)
+    if bounded is None:
+        return 0.0
+    return max((bounded - now_utc).total_seconds(), 0.0)
+
+
+def _normalize_datetime(value: datetime | None) -> datetime | None:
+    if value is None:
+        return None
+    if value.tzinfo is None:
+        return value.replace(tzinfo=UTC)
+    return value
+
+
+def _min_interval_seconds() -> float:
+    return max(float(settings.arxiv_min_interval_seconds), 0.0)
+
+
+def _cooldown_seconds() -> float:
+    return max(float(settings.arxiv_rate_limit_cooldown_seconds), 0.0)

app/services/arxiv/types.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Literal
+
+ArxivSortBy = Literal["relevance", "lastUpdatedDate", "submittedDate"]
+ArxivSortOrder = Literal["ascending", "descending"]
+
+
+@dataclass(frozen=True)
+class ArxivOpenSearchMeta:
+    total_results: int = 0
+    start_index: int = 0
+    items_per_page: int = 0
+
+
+@dataclass(frozen=True)
+class ArxivEntry:
+    entry_id_url: str
+    arxiv_id: str | None
+    title: str
+    summary: str
+    published: str | None
+    updated: str | None
+    authors: list[str] = field(default_factory=list)
+    links: list[str] = field(default_factory=list)
+    categories: list[str] = field(default_factory=list)
+    primary_category: str | None = None
+
+
+@dataclass(frozen=True)
+class ArxivFeed:
+    entries: list[ArxivEntry] = field(default_factory=list)
+    opensearch: ArxivOpenSearchMeta = field(default_factory=ArxivOpenSearchMeta)

app/services/crossref/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from app.services.crossref.application import discover_doi_for_publication
+
+__all__ = ["discover_doi_for_publication"]

app/services/crossref/application.py

@@ -0,0 +1,388 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+import threading
+import time
+from importlib.metadata import version as pkg_version
+from typing import TYPE_CHECKING
+
+from crossref.restful import Etiquette, Works
+
+from app.logging_utils import structured_log
+from app.services.doi.normalize import normalize_doi
+from app.settings import settings
+
+_APP_VERSION = pkg_version("scholarr")
+
+if TYPE_CHECKING:
+    from app.services.publications.types import PublicationListItem, UnreadPublicationItem
+
+TOKEN_RE = re.compile(r"[a-z0-9]+")
+NON_ALNUM_RE = re.compile(r"[^a-z0-9\s]+")
+STOP_WORDS = {"the", "and", "for", "with", "from", "method", "study", "analysis"}
+_RATE_LOCK = threading.Lock()
+_LAST_REQUEST_AT = 0.0
+logger = logging.getLogger(__name__)
+STRICT_TITLE_MATCH_THRESHOLD = 0.75
+RELAXED_TITLE_MATCH_THRESHOLD = 0.85
+
+
+def _rate_limit_wait(min_interval_seconds: float) -> None:
+    global _LAST_REQUEST_AT
+    interval = max(float(min_interval_seconds), 0.0)
+    with _RATE_LOCK:
+        elapsed = time.monotonic() - _LAST_REQUEST_AT
+        remaining = interval - elapsed
+    if remaining > 0:
+        time.sleep(remaining)
+    with _RATE_LOCK:
+        _LAST_REQUEST_AT = time.monotonic()
+
+
+def _normalized_tokens(value: str) -> list[str]:
+    lowered = value.lower().replace("’", "'").replace("“", '"').replace("”", '"')
+    lowered = NON_ALNUM_RE.sub(" ", lowered)
+    return [token for token in TOKEN_RE.findall(lowered) if len(token) >= 3]
+
+
+def _normalized_query(value: str) -> str:
+    tokens = [token for token in _normalized_tokens(value) if token not in STOP_WORDS]
+    if len(tokens) < 3:
+        tokens = _normalized_tokens(value)
+    if len(tokens) < 3:
+        return ""
+    return " ".join(tokens[:12]).strip()
+
+
+def _query_author(value: str) -> str | None:
+    tokens = [token for token in value.strip().split() if token]
+    if len(tokens) < 2:
+        return None
+    return " ".join(tokens[:2])[:64]
+
+
+def _author_surname(value: str) -> str | None:
+    tokens = [token for token in value.strip().split() if token]
+    if not tokens:
+        return None
+    return NON_ALNUM_RE.sub("", tokens[-1].lower()) or None
+
+
+def _query_filters(year: int | None) -> list[tuple[str, str] | None]:
+    if year is None:
+        return [None]
+    return [
+        (f"{year - 1}-01-01", f"{year + 1}-12-31"),
+        (f"{year}-01-01", f"{year}-12-31"),
+        None,
+    ]
+
+
+def _candidate_title(item: dict) -> str:
+    titles = item.get("title")
+    if isinstance(titles, list) and titles:
+        return str(titles[0] or "")
+    return str(item.get("title") or "")
+
+
+def _title_match_score(source: str, candidate: str) -> float:
+    source_tokens = {token for token in _normalized_tokens(source) if len(token) >= 3}
+    candidate_tokens = {token for token in _normalized_tokens(candidate) if len(token) >= 3}
+    if not source_tokens or not candidate_tokens:
+        return 0.0
+    return len(source_tokens & candidate_tokens) / float(len(source_tokens))
+
+
+def _candidate_year(item: dict) -> int | None:
+    issued = item.get("issued")
+    if not isinstance(issued, dict):
+        return None
+    date_parts = issued.get("date-parts")
+    if not isinstance(date_parts, list) or not date_parts:
+        return None
+    first = date_parts[0]
+    if not isinstance(first, list) or not first:
+        return None
+    try:
+        return int(first[0])
+    except (TypeError, ValueError):
+        return None
+
+
+def _candidate_author_match(item: dict, surname: str | None) -> bool:
+    if not surname:
+        return True
+    authors = item.get("author")
+    if not isinstance(authors, list):
+        return False
+    for author in authors:
+        if not isinstance(author, dict):
+            continue
+        family = NON_ALNUM_RE.sub("", str(author.get("family") or "").lower())
+        if family and family == surname:
+            return True
+    return False
+
+
+def _candidate_rank(*, title: str, year: int | None, item: dict) -> tuple[float, str | None]:
+    doi = normalize_doi(str(item.get("DOI") or ""))
+    if doi is None:
+        return 0.0, None
+    score = _title_match_score(title, _candidate_title(item))
+    candidate_year = _candidate_year(item)
+    if year is not None and candidate_year is not None:
+        if abs(year - candidate_year) > 1:
+            return 0.0, None
+        score += 0.1
+    return score, doi
+
+
+def _year_delta(source_year: int | None, candidate_year: int | None) -> int | None:
+    if source_year is None or candidate_year is None:
+        return None
+    return abs(int(source_year) - int(candidate_year))
+
+
+def _candidate_rank_relaxed(
+    *,
+    title: str,
+    year: int | None,
+    item: dict,
+    author_surname: str | None,
+) -> tuple[float, str | None]:
+    doi = normalize_doi(str(item.get("DOI") or ""))
+    if doi is None:
+        return 0.0, None
+    score = _title_match_score(title, _candidate_title(item))
+    if score <= 0:
+        return 0.0, None
+    candidate_year = _candidate_year(item)
+    delta = _year_delta(year, candidate_year)
+    if delta is not None:
+        if delta <= 1:
+            score += 0.05
+        elif delta <= 3:
+            score += 0.0
+        elif delta <= 5:
+            score -= 0.03
+        else:
+            score -= 0.08
+    if _candidate_author_match(item, author_surname):
+        score += 0.03
+    return score, doi
+
+
+def _best_candidate_doi_strict(
+    *,
+    title: str,
+    year: int | None,
+    items: list[dict],
+    author_surname: str | None,
+) -> str | None:
+    best_score = 0.0
+    best_doi: str | None = None
+    best_year: int | None = None
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        if not _candidate_author_match(item, author_surname):
+            continue
+        score, doi = _candidate_rank(title=title, year=year, item=item)
+        candidate_year = _candidate_year(item)
+        if doi is None or score < STRICT_TITLE_MATCH_THRESHOLD:
+            continue
+        if score > best_score:
+            best_score = score
+            best_doi = doi
+            best_year = candidate_year
+            continue
+        if abs(score - best_score) > 0.02:
+            continue
+        if best_year is None or candidate_year is None:
+            continue
+        if candidate_year < best_year:
+            best_doi = doi
+            best_year = candidate_year
+    return best_doi
+
+
+def _best_candidate_doi_relaxed(
+    *,
+    title: str,
+    year: int | None,
+    items: list[dict],
+    author_surname: str | None,
+) -> str | None:
+    best_score = 0.0
+    best_doi: str | None = None
+    best_author_match = False
+    best_delta: int | None = None
+    best_year: int | None = None
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        score, doi = _candidate_rank_relaxed(
+            title=title,
+            year=year,
+            item=item,
+            author_surname=author_surname,
+        )
+        if doi is None or score < RELAXED_TITLE_MATCH_THRESHOLD:
+            continue
+        candidate_year = _candidate_year(item)
+        candidate_author_match = _candidate_author_match(item, author_surname)
+        candidate_delta = _year_delta(year, candidate_year)
+        if score > best_score:
+            best_score = score
+            best_doi = doi
+            best_author_match = candidate_author_match
+            best_delta = candidate_delta
+            best_year = candidate_year
+            continue
+        if abs(score - best_score) > 0.02:
+            continue
+        if candidate_author_match and not best_author_match:
+            best_doi = doi
+            best_author_match = True
+            best_delta = candidate_delta
+            best_year = candidate_year
+            continue
+        if best_delta is None and candidate_delta is not None:
+            best_doi = doi
+            best_author_match = candidate_author_match
+            best_delta = candidate_delta
+            best_year = candidate_year
+            continue
+        if best_delta is not None and candidate_delta is not None and candidate_delta < best_delta:
+            best_doi = doi
+            best_author_match = candidate_author_match
+            best_delta = candidate_delta
+            best_year = candidate_year
+            continue
+        if best_year is None or candidate_year is None:
+            continue
+        if candidate_year < best_year:
+            best_doi = doi
+            best_author_match = candidate_author_match
+            best_delta = candidate_delta
+            best_year = candidate_year
+    return best_doi
+
+
+def _best_candidate_doi(
+    *,
+    title: str,
+    year: int | None,
+    items: list[dict],
+    author_surname: str | None,
+) -> str | None:
+    strict_match = _best_candidate_doi_strict(
+        title=title,
+        year=year,
+        items=items,
+        author_surname=author_surname,
+    )
+    if strict_match:
+        return strict_match
+    return _best_candidate_doi_relaxed(
+        title=title,
+        year=year,
+        items=items,
+        author_surname=author_surname,
+    )
+
+
+def _works_client(email: str | None) -> Works:
+    if email:
+        etiquette = Etiquette(settings.app_name, _APP_VERSION, "https://scholarr.local", email)
+        return Works(etiquette=etiquette)
+    return Works()
+
+
+def _fetch_items_sync(
+    *,
+    query: str,
+    author: str | None,
+    date_range: tuple[str, str] | None,
+    max_rows: int,
+    email: str | None,
+    min_interval_seconds: float,
+) -> list[dict]:
+    _rate_limit_wait(min_interval_seconds)
+    works = _works_client(email)
+    params = {"bibliographic": query}
+    if author:
+        params["author"] = author
+    request = works.query(**params)
+    if date_range is not None:
+        from_date, until_date = date_range
+        request = request.filter(from_pub_date=from_date, until_pub_date=until_date)
+    request = request.select(["DOI", "title", "issued", "score", "author"])
+    items: list[dict] = []
+    for entry in request:
+        if isinstance(entry, dict):
+            items.append(entry)
+        if len(items) >= max(max_rows, 1):
+            break
+    return items
+
+
+async def _fetch_items(
+    *,
+    query: str,
+    author: str | None,
+    date_range: tuple[str, str] | None,
+    max_rows: int,
+    email: str | None,
+) -> list[dict]:
+    timeout = max(float(settings.crossref_timeout_seconds), 0.5)
+    try:
+        return await asyncio.wait_for(
+            asyncio.to_thread(
+                _fetch_items_sync,
+                query=query,
+                author=author,
+                date_range=date_range,
+                max_rows=max_rows,
+                email=email,
+                min_interval_seconds=settings.crossref_min_interval_seconds,
+            ),
+            timeout=timeout,
+        )
+    except Exception as exc:
+        structured_log(logger, "warning", "crossref.fetch_failed", error=str(exc))
+        return []
+
+
+async def discover_doi_for_publication(
+    *,
+    item: PublicationListItem | UnreadPublicationItem,
+    max_rows: int = 10,
+    email: str | None = None,
+) -> str | None:
+    title = (item.title or "").strip()
+    query = _normalized_query(title)
+    if not query:
+        return None
+    author = _query_author(item.scholar_label)
+    author_surname = _author_surname(item.scholar_label)
+    for date_range in _query_filters(item.year):
+        items = await _fetch_items(
+            query=query,
+            author=author,
+            date_range=date_range,
+            max_rows=max_rows,
+            email=email,
+        )
+        doi = _best_candidate_doi(
+            title=title,
+            year=item.year,
+            items=items,
+            author_surname=author_surname,
+        )
+        if doi:
+            structured_log(logger, "debug", "crossref.doi_discovered")
+            return doi
+    return None

app/services/dbops/__init__.py

@@ -0,0 +1,13 @@
+from app.services.dbops.application import run_publication_link_repair
+from app.services.dbops.integrity import collect_integrity_report
+from app.services.dbops.near_duplicate_repair import (
+    run_publication_near_duplicate_repair,
+)
+from app.services.dbops.query import list_repair_jobs
+
+__all__ = [
+    "collect_integrity_report",
+    "list_repair_jobs",
+    "run_publication_link_repair",
+    "run_publication_near_duplicate_repair",
+]

app/services/dbops/application.py

@@ -0,0 +1,353 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+from sqlalchemy import CursorResult, delete, exists, func, select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import DataRepairJob, IngestionQueueItem, Publication, ScholarProfile, ScholarPublication
+
+REPAIR_STATUS_PLANNED = "planned"
+REPAIR_STATUS_RUNNING = "running"
+REPAIR_STATUS_COMPLETED = "completed"
+REPAIR_STATUS_FAILED = "failed"
+SCOPE_MODE_SINGLE_USER = "single_user"
+SCOPE_MODE_ALL_USERS = "all_users"
+
+
+def _utcnow() -> datetime:
+    return datetime.now(UTC)
+
+
+def _normalize_scope_mode(scope_mode: str) -> str:
+    normalized = scope_mode.strip().lower()
+    if normalized in {SCOPE_MODE_SINGLE_USER, SCOPE_MODE_ALL_USERS}:
+        return normalized
+    raise ValueError("Unknown scope mode.")
+
+
+def _scope_user_id(*, scope_mode: str, user_id: int | None) -> int | None:
+    if scope_mode == SCOPE_MODE_SINGLE_USER:
+        if user_id is None:
+            raise ValueError("user_id is required when scope_mode=single_user.")
+        return int(user_id)
+    if user_id is not None:
+        raise ValueError("user_id must be omitted when scope_mode=all_users.")
+    return None
+
+
+def _scope_payload(
+    *,
+    scope_mode: str,
+    user_id: int | None,
+    target_scholar_profile_ids: list[int],
+    orphan_gc: bool,
+) -> dict[str, Any]:
+    payload: dict[str, Any] = {
+        "scope_mode": scope_mode,
+        "scholar_profile_ids": [int(value) for value in target_scholar_profile_ids],
+        "gc_orphan_publications": bool(orphan_gc),
+    }
+    if user_id is not None:
+        payload["user_id"] = int(user_id)
+    return payload
+
+
+async def _target_scholar_profile_ids(
+    db_session: AsyncSession,
+    *,
+    scope_mode: str,
+    user_id: int | None,
+    scholar_profile_ids: list[int] | None,
+) -> list[int]:
+    stmt = select(ScholarProfile.id)
+    if scope_mode == SCOPE_MODE_SINGLE_USER:
+        stmt = stmt.where(ScholarProfile.user_id == user_id)
+    if scholar_profile_ids:
+        normalized_ids = [int(value) for value in scholar_profile_ids]
+        stmt = stmt.where(ScholarProfile.id.in_(normalized_ids))
+    result = await db_session.execute(stmt.order_by(ScholarProfile.id.asc()))
+    ids = [int(row[0]) for row in result.all()]
+    if not ids:
+        raise ValueError("No target scholar profiles found for the requested scope.")
+    return ids
+
+
+async def _count_scope(
+    db_session: AsyncSession,
+    *,
+    user_id: int | None,
+    target_scholar_profile_ids: list[int],
+) -> dict[str, int]:
+    links_result = await db_session.execute(
+        select(func.count())
+        .select_from(ScholarPublication)
+        .where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids))
+    )
+    queue_stmt = (
+        select(func.count())
+        .select_from(IngestionQueueItem)
+        .where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids))
+    )
+    if user_id is not None:
+        queue_stmt = queue_stmt.where(IngestionQueueItem.user_id == user_id)
+    queue_result = await db_session.execute(queue_stmt)
+    return {
+        "target_scholar_count": len(target_scholar_profile_ids),
+        "links_in_scope": int(links_result.scalar_one() or 0),
+        "queue_items_in_scope": int(queue_result.scalar_one() or 0),
+    }
+
+
+async def _count_orphan_publications(db_session: AsyncSession) -> int:
+    stmt = (
+        select(func.count())
+        .select_from(Publication)
+        .where(~exists(select(1).where(ScholarPublication.publication_id == Publication.id)))
+    )
+    result = await db_session.execute(stmt)
+    return int(result.scalar_one() or 0)
+
+
+async def _create_job(
+    db_session: AsyncSession,
+    *,
+    requested_by: str | None,
+    scope: dict[str, Any],
+    dry_run: bool,
+) -> DataRepairJob:
+    job = DataRepairJob(
+        job_name="repair_publication_links",
+        requested_by=(requested_by or "").strip() or None,
+        scope=scope,
+        dry_run=dry_run,
+        status=REPAIR_STATUS_PLANNED,
+        summary={},
+    )
+    db_session.add(job)
+    await db_session.flush()
+    return job
+
+
+def _job_summary(
+    *,
+    counts: dict[str, int],
+    dry_run: bool,
+    links_deleted: int,
+    queue_items_deleted: int,
+    scholars_reset: int,
+    orphan_publications_before: int,
+    orphan_publications_deleted: int,
+) -> dict[str, Any]:
+    return {
+        **counts,
+        "dry_run": bool(dry_run),
+        "links_deleted": int(links_deleted),
+        "queue_items_deleted": int(queue_items_deleted),
+        "scholars_reset": int(scholars_reset),
+        "orphan_publications_before": int(orphan_publications_before),
+        "orphan_publications_deleted": int(orphan_publications_deleted),
+    }
+
+
+async def _delete_links_for_targets(db_session: AsyncSession, *, target_scholar_profile_ids: list[int]) -> int:
+    result: CursorResult[Any] = await db_session.execute(  # type: ignore[assignment]
+        delete(ScholarPublication).where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids))
+    )
+    return int(result.rowcount or 0)
+
+
+async def _delete_queue_for_targets(
+    db_session: AsyncSession,
+    *,
+    user_id: int | None,
+    target_scholar_profile_ids: list[int],
+) -> int:
+    stmt = delete(IngestionQueueItem).where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids))
+    if user_id is not None:
+        stmt = stmt.where(IngestionQueueItem.user_id == user_id)
+    result: CursorResult[Any] = await db_session.execute(stmt)  # type: ignore[assignment]
+    return int(result.rowcount or 0)
+
+
+async def _reset_scholar_tracking_state(
+    db_session: AsyncSession,
+    *,
+    user_id: int | None,
+    target_scholar_profile_ids: list[int],
+) -> int:
+    stmt = update(ScholarProfile).where(ScholarProfile.id.in_(target_scholar_profile_ids))
+    if user_id is not None:
+        stmt = stmt.where(ScholarProfile.user_id == user_id)
+    result: CursorResult[Any] = await db_session.execute(  # type: ignore[assignment]
+        stmt.values(
+            baseline_completed=False,
+            last_initial_page_fingerprint_sha256=None,
+            last_initial_page_checked_at=None,
+            last_run_dt=None,
+            last_run_status=None,
+        )
+    )
+    return int(result.rowcount or 0)
+
+
+async def _delete_orphan_publications(db_session: AsyncSession) -> int:
+    result: CursorResult[Any] = await db_session.execute(  # type: ignore[assignment]
+        delete(Publication).where(~exists(select(1).where(ScholarPublication.publication_id == Publication.id)))
+    )
+    return int(result.rowcount or 0)
+
+
+async def _mutation_counts(
+    db_session: AsyncSession,
+    *,
+    user_id: int | None,
+    target_ids: list[int],
+    dry_run: bool,
+    gc_orphan_publications: bool,
+) -> tuple[int, int, int, int]:
+    if dry_run:
+        return 0, 0, 0, 0
+    links_deleted = await _delete_links_for_targets(
+        db_session,
+        target_scholar_profile_ids=target_ids,
+    )
+    queue_deleted = await _delete_queue_for_targets(
+        db_session,
+        user_id=user_id,
+        target_scholar_profile_ids=target_ids,
+    )
+    scholars_reset = await _reset_scholar_tracking_state(
+        db_session,
+        user_id=user_id,
+        target_scholar_profile_ids=target_ids,
+    )
+    orphan_deleted = 0
+    if gc_orphan_publications:
+        orphan_deleted = await _delete_orphan_publications(db_session)
+    return links_deleted, queue_deleted, scholars_reset, orphan_deleted
+
+
+def _result_payload(*, job: DataRepairJob, scope: dict[str, Any], summary: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "job_id": int(job.id),
+        "status": job.status,
+        "scope": scope,
+        "summary": summary,
+    }
+
+
+async def _complete_job(
+    db_session: AsyncSession,
+    *,
+    job: DataRepairJob,
+    summary: dict[str, Any],
+    scope: dict[str, Any],
+) -> dict[str, Any]:
+    job.summary = summary
+    job.status = REPAIR_STATUS_COMPLETED
+    job.finished_at = _utcnow()
+    await db_session.commit()
+    return _result_payload(job=job, scope=scope, summary=summary)
+
+
+async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None:
+    await db_session.rollback()
+    job.status = REPAIR_STATUS_FAILED
+    job.error_text = str(error)
+    job.finished_at = _utcnow()
+    db_session.add(job)
+    await db_session.commit()
+
+
+async def _prepare_repair_job(
+    db_session: AsyncSession,
+    *,
+    scope_mode: str,
+    user_id: int | None,
+    scholar_profile_ids: list[int] | None,
+    dry_run: bool,
+    gc_orphan_publications: bool,
+    requested_by: str | None,
+) -> tuple[int | None, list[int], dict[str, Any], DataRepairJob]:
+    normalized_scope = _normalize_scope_mode(scope_mode)
+    scope_user_id = _scope_user_id(scope_mode=normalized_scope, user_id=user_id)
+    target_ids = await _target_scholar_profile_ids(
+        db_session,
+        scope_mode=normalized_scope,
+        user_id=scope_user_id,
+        scholar_profile_ids=scholar_profile_ids,
+    )
+    scope = _scope_payload(
+        scope_mode=normalized_scope,
+        user_id=scope_user_id,
+        target_scholar_profile_ids=target_ids,
+        orphan_gc=gc_orphan_publications,
+    )
+    job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run)
+    job.status = REPAIR_STATUS_RUNNING
+    job.started_at = _utcnow()
+    return scope_user_id, target_ids, scope, job
+
+
+async def _build_repair_summary(
+    db_session: AsyncSession,
+    *,
+    scope_user_id: int | None,
+    target_ids: list[int],
+    dry_run: bool,
+    gc_orphan_publications: bool,
+) -> dict[str, Any]:
+    counts = await _count_scope(db_session, user_id=scope_user_id, target_scholar_profile_ids=target_ids)
+    orphan_before = await _count_orphan_publications(db_session)
+    links_deleted, queue_deleted, scholars_reset, orphan_deleted = await _mutation_counts(
+        db_session,
+        user_id=scope_user_id,
+        target_ids=target_ids,
+        dry_run=dry_run,
+        gc_orphan_publications=gc_orphan_publications,
+    )
+    return _job_summary(
+        counts=counts,
+        dry_run=dry_run,
+        links_deleted=links_deleted,
+        queue_items_deleted=queue_deleted,
+        scholars_reset=scholars_reset,
+        orphan_publications_before=orphan_before,
+        orphan_publications_deleted=orphan_deleted,
+    )
+
+
+async def run_publication_link_repair(
+    db_session: AsyncSession,
+    *,
+    scope_mode: str = SCOPE_MODE_SINGLE_USER,
+    user_id: int | None = None,
+    scholar_profile_ids: list[int] | None = None,
+    dry_run: bool = True,
+    gc_orphan_publications: bool = False,
+    requested_by: str | None = None,
+) -> dict[str, Any]:
+    scope_user_id, target_ids, scope, job = await _prepare_repair_job(
+        db_session,
+        scope_mode=scope_mode,
+        user_id=user_id,
+        scholar_profile_ids=scholar_profile_ids,
+        dry_run=dry_run,
+        gc_orphan_publications=gc_orphan_publications,
+        requested_by=requested_by,
+    )
+
+    try:
+        summary = await _build_repair_summary(
+            db_session,
+            scope_user_id=scope_user_id,
+            target_ids=target_ids,
+            dry_run=dry_run,
+            gc_orphan_publications=gc_orphan_publications,
+        )
+        return await _complete_job(db_session, job=job, summary=summary, scope=scope)
+    except Exception as exc:
+        await _fail_job(db_session, job=job, error=exc)
+        raise