Merge pull request #19 from JustinZeus/feat/decomposition

Feat/decomposition

.claude/settings.json

@@ -0,0 +1,17 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(wc -l /home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src/features/settings/SettingsAdminPanel.vue /home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src/features/settings/components/Admin*.vue)",
+      "Read(//home/jv/src/personal/playground/scholar_scraper/scholar-scraper/**)",
+      "Bash(while read line rest)",
+      "Bash(do echo \"$line $rest\")",
+      "Bash(done)",
+      "Bash(grep -c 'def test_' tests/unit/*.py tests/unit/**/*.py tests/integration/*.py)",
+      "Bash(sort -t: -k2 -rn)",
+      "Bash(wc -l frontend/src/pages/*.vue)"
+    ],
+    "additionalDirectories": [
+      "/home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src"
+    ]
+  }
+}

.env.example

@@ -79,12 +79,15 @@ LOG_REDACT_FIELDS=
 SCHEDULER_ENABLED=1
 SCHEDULER_TICK_SECONDS=60
 SCHEDULER_QUEUE_BATCH_SIZE=10
+SCHEDULER_PDF_QUEUE_BATCH_SIZE=15
 INGESTION_AUTOMATION_ALLOWED=1
 INGESTION_MANUAL_RUN_ALLOWED=1
 INGESTION_MIN_RUN_INTERVAL_MINUTES=15
 INGESTION_MIN_REQUEST_DELAY_SECONDS=2
 INGESTION_NETWORK_ERROR_RETRIES=1
 INGESTION_RETRY_BACKOFF_SECONDS=1.0
+INGESTION_RATE_LIMIT_RETRIES=3
+INGESTION_RATE_LIMIT_BACKOFF_SECONDS=30.0
 INGESTION_MAX_PAGES_PER_SCHOLAR=30
 INGESTION_PAGE_SIZE=100
 INGESTION_ALERT_BLOCKED_FAILURE_THRESHOLD=1
@@ -112,6 +115,10 @@ SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD=1
 SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS=1800
 SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD=2
 SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD=3
+SCHOLAR_HTTP_USER_AGENT=
+SCHOLAR_HTTP_ROTATE_USER_AGENT=0
+SCHOLAR_HTTP_ACCEPT_LANGUAGE=en-US,en;q=0.9
+SCHOLAR_HTTP_COOKIE=
 
 # ------------------------------
 # OA Enrichment + PDF Resolution
@@ -125,6 +132,14 @@ UNPAYWALL_RETRY_COOLDOWN_SECONDS=1800
 UNPAYWALL_PDF_DISCOVERY_ENABLED=1
 UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES=5
 UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES=500000
+ARXIV_ENABLED=1
+ARXIV_TIMEOUT_SECONDS=3.0
+ARXIV_MIN_INTERVAL_SECONDS=4.0
+ARXIV_RATE_LIMIT_COOLDOWN_SECONDS=60.0
+ARXIV_DEFAULT_MAX_RESULTS=3
+ARXIV_CACHE_TTL_SECONDS=900
+ARXIV_CACHE_MAX_ENTRIES=512
+ARXIV_MAILTO=
 PDF_AUTO_RETRY_INTERVAL_SECONDS=86400
 PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS=3600
 PDF_AUTO_RETRY_MAX_ATTEMPTS=3
@@ -133,6 +148,9 @@ CROSSREF_MAX_ROWS=10
 CROSSREF_TIMEOUT_SECONDS=8.0
 CROSSREF_MIN_INTERVAL_SECONDS=0.6
 CROSSREF_MAX_LOOKUPS_PER_REQUEST=8
+OPENALEX_API_KEY=
+CROSSREF_API_TOKEN=
+CROSSREF_API_MAILTO=
 
 # ------------------------------
 # Startup Bootstrap + DB Wait

.github/FUNDING.yml

@@ -0,0 +1 @@
+buy_me_a_coffee: justinzeus

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: npm
+    directory: /frontend
+    schedule:
+      interval: weekly
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly

.github/workflows/ci.yml

@@ -20,10 +20,28 @@ jobs:
       - name: Enforce env contract parity
         run: python3 scripts/check_env_contract.py
 
+  lint:
+    runs-on: ubuntu-latest
+    needs: [repo-hygiene]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: astral-sh/setup-uv@v4
+      - run: uv sync --extra dev
+      - name: Ruff check
+        run: uv run ruff check .
+      - name: Ruff format check
+        run: uv run ruff format --check .
+      - name: Mypy
+        run: uv run mypy app/ --ignore-missing-imports
+
   test:
     runs-on: ubuntu-latest
     needs:
       - repo-hygiene
+      - lint
     services:
       postgres:
         image: postgres:15
@@ -91,7 +109,7 @@ jobs:
 
       - name: Install frontend dependencies
         working-directory: frontend
-        run: npm install
+        run: npm ci
 
       - name: Frontend theme token policy
         working-directory: frontend
@@ -124,7 +142,7 @@ jobs:
           node-version: "20"
 
       - name: Install docs dependencies
-        run: npm install --prefix docs/website
+        run: npm ci --prefix docs/website
 
       - name: Docs build
         run: npm --prefix docs/website run build

.github/workflows/codeql.yml

@@ -0,0 +1,26 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "30 5 * * 1"
+
+jobs:
+  analyze:
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      contents: read
+    strategy:
+      matrix:
+        language: [python, javascript]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+      - uses: github/codeql-action/autobuild@v3
+      - uses: github/codeql-action/analyze@v3

.github/workflows/docs-pages.yml

@@ -32,7 +32,7 @@ jobs:
           node-version: "20"
 
       - name: Install docs dependencies
-        run: npm install --prefix docs/website
+        run: npm ci --prefix docs/website
 
       - name: Build docs site
         run: npm --prefix docs/website run build

.github/workflows/release.yml

@@ -0,0 +1,32 @@
+name: Release
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    if: github.repository == 'JustinZeus/scholarr'
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: astral-sh/setup-uv@v4
+      - run: uv sync --extra dev
+      - name: Lint
+        run: uv run ruff check
+      - name: Test
+        run: uv run pytest
+      - name: Semantic Release
+        id: release
+        run: uv run semantic-release publish
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -12,9 +12,9 @@ htmlcov/
 *.log
 .DS_Store
 planning/
+build/
 frontend/node_modules/
 frontend/dist/
 frontend/.vite/
 docs/website/node_modules/
 docs/website/build/
-AGENTS.MD

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 JustinZeus
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,69 +1,149 @@
-# scholarr
-
 <div align="center">
 
-Self-hosted scholar tracking with a single app image (API + frontend).
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset=".github/logo-dark.png" />
+  <source media="(prefers-color-scheme: light)" srcset="frontend/public/scholar_logo.png" />
+  <img src="frontend/public/scholar_logo.png" alt="Scholarr" width="120" />
+</picture>
+
+# Scholarr
+
+**Self-hosted academic publication tracker.**
+
+Track Google Scholar profiles, discover new papers automatically,
+resolve open-access PDFs, and stay on top of the literature you care about.
 
 [![CI](https://img.shields.io/github/actions/workflow/status/justinzeus/scholarr/ci.yml?style=for-the-badge)](https://github.com/JustinZeus/scholarr/actions/workflows/ci.yml)
+[![CodeQL](https://img.shields.io/github/actions/workflow/status/justinzeus/scholarr/codeql.yml?style=for-the-badge&label=CodeQL)](https://github.com/JustinZeus/scholarr/actions/workflows/codeql.yml)
+[![Release](https://img.shields.io/github/v/release/justinzeus/scholarr?style=for-the-badge)](https://github.com/JustinZeus/scholarr/releases)
 [![Docker Pulls](https://img.shields.io/docker/pulls/justinzeus/scholarr?style=for-the-badge&logo=docker)](https://hub.docker.com/r/justinzeus/scholarr)
-[![Docker Image](https://img.shields.io/badge/docker-justinzeus%2Fscholarr-2496ED?style=for-the-badge&logo=docker&logoColor=white)](https://hub.docker.com/r/justinzeus/scholarr)
+[![Python](https://img.shields.io/badge/python-3.12+-3776AB?style=for-the-badge&logo=python&logoColor=white)](https://www.python.org/)
+[![License](https://img.shields.io/github/license/justinzeus/scholarr?style=for-the-badge)](LICENSE)
+[![Docs](https://img.shields.io/badge/docs-scholarr-2e8555?style=for-the-badge)](https://justinzeus.github.io/scholarr/)
 
 </div>
 
+---
+
+<!-- TODO: Replace these placeholders with actual screenshots.
+     Drop PNG/JPEG files into docs/assets/ and update the paths below.
+
+     Recommended screenshots:
+       1. Dashboard / publications list (light or dark mode)
+       2. Scholar profile view
+       3. Run detail with progress
+
+     Ideal dimensions: 1200-1400px wide, PNG or WebP.
+-->
+
+<p align="center">
+  <img src="docs/assets/screenshot-dashboard.png" alt="Publications dashboard" width="720" />
+</p>
+
+<!--
+<p align="center">
+  <img src="docs/assets/screenshot-scholars.png" alt="Scholar profiles" width="360" />
+  <img src="docs/assets/screenshot-run.png" alt="Ingestion run" width="360" />
+</p>
+-->
+
+## Why Scholarr?
+
+Most researchers track new papers by manually checking Google Scholar, setting up email alerts, or juggling RSS feeds. Scholarr replaces all of that with a single self-hosted service:
+
+- **Add scholars once** -- by profile URL, Scholar ID, or name search
+- **Publications appear automatically** -- a background scheduler scrapes profiles on a configurable interval
+- **Open-access PDFs are resolved for you** -- Unpaywall and arXiv are queried automatically when a DOI is found
+- **Everything is deduplicated** -- publications are global records; no duplicates across scholars
+- **Your data stays yours** -- fully self-hosted, export/import your entire library at any time
+
+## Features
+
+| | |
+|---|---|
+| **Automated Ingestion** | Background scheduler with configurable intervals, continuation queue, and multi-page pagination |
+| **Identifier Resolution** | Cross-references arXiv, Crossref, and OpenAlex to gather DOIs, arXiv IDs, PMIDs |
+| **PDF Discovery** | Resolves open-access PDFs via Unpaywall API and arXiv, with automatic retry queue |
+| **Scrape Safety** | Rate limiting, cooldowns, and backoff strategies that prevent IP bans -- these are safety floors, not optional |
+| **Multi-User** | Session-based auth, admin user management, user-scoped scholar tracking |
+| **Theming** | 7 color presets with light/dark mode, tokenized component system |
+| **Import / Export** | Portable scholar data with full publication and read-state preservation |
+| **Single Container** | FastAPI backend + Vue 3 frontend ship as one Docker image |
+
 ## Quick Start
 
-1. Copy env template:
-
 ```bash
+# 1. Clone and configure
+git clone https://github.com/JustinZeus/scholarr.git
+cd scholarr
 cp .env.example .env
+
+# 2. Set required secrets in .env
+#    POSTGRES_PASSWORD=<secure-password>
+#    SESSION_SECRET_KEY=<random-32-char-string>
+
+# 3. Start
+docker compose up -d
+
+# 4. Open http://localhost:8000
 ```
 
-2. Set required values in `.env`:
+To bootstrap an admin account on first run, add to `.env`:
-- `POSTGRES_PASSWORD`
-- `SESSION_SECRET_KEY`
-
-3. Start stack:
 
 ```bash
-docker compose pull
+BOOTSTRAP_ADMIN_ON_START=1
-docker compose up -d
+BOOTSTRAP_ADMIN_EMAIL=admin@example.com
+BOOTSTRAP_ADMIN_PASSWORD=<secure-password>
 ```
 
-Open:
+## How It Works
-- App/API: `http://localhost:8000`
+
-- Health: `http://localhost:8000/healthz`
+```mermaid
+graph LR
+    UI[Vue 3 Dashboard] <-->|REST + SSE| API[FastAPI]
+    API --> Scheduler[Scheduler]
+    Scheduler -->|Scrape HTML| Scholar[Google Scholar]
+    Scholar -->|Parse & Deduplicate| DB[(PostgreSQL)]
+    Scholar -.->|Identify| Ext[arXiv / Crossref / OpenAlex]
+    Ext --> DB
+    DB -->|DOIs| PDF[PDF Resolution]
+    PDF -->|Unpaywall / arXiv| DB
+    API <--> DB
+```
+
+## Tech Stack
+
+| Layer | Technology |
+|-------|------------|
+| Backend | Python 3.12, FastAPI, SQLAlchemy 2.0 (async), Alembic |
+| Frontend | TypeScript, Vue 3, Vite, Tailwind CSS |
+| Database | PostgreSQL 15 |
+| Infrastructure | Multi-stage Docker, Docker Compose |
 
 ## Documentation
 
-Complete documentation is published at:
+Full documentation: **[justinzeus.github.io/scholarr](https://justinzeus.github.io/scholarr/)**
 
-- https://justinzeus.github.io/scholarr/
+| Section | Covers |
+|---------|--------|
+| [User Guide](docs/user/overview.md) | Installation, configuration, all environment variables |
+| [Developer Guide](docs/developer/overview.md) | Architecture, local dev, contributing, testing |
+| [Operations](docs/operations/overview.md) | Deployment, database runbook, scrape safety |
+| [API Reference](docs/reference/api.md) | Envelope spec, all endpoints, DTO contracts |
 
-Source markdown and docs tooling live under `docs/`.
+## Contributing
 
-## Quality Gates
+Scholarr uses [conventional commits](https://www.conventionalcommits.org/) and [semantic versioning](https://semver.org/). See the [contributing guide](docs/developer/contributing.md) for PR process and code standards.
-
-Backend:
 
 ```bash
-docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest tests/unit
+# Dev environment
-docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest -m integration
+docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build
+
+# Run tests (always in containers)
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app \
+  python -m pytest
 ```
 
-Frontend:
+## License
 
-```bash
+See [LICENSE](LICENSE) for details.
-cd frontend
-npm install
-npm run typecheck
-npm run test:run
-npm run build
-```
-
-Contract and env checks:
-
-```bash
-python3 scripts/check_frontend_api_contract.py
-python3 scripts/check_env_contract.py
-./scripts/check_no_generated_artifacts.sh
-```

agents.md

@@ -1,43 +1,123 @@
 # AI Agent Instructions: Scholarr
 
-Adhere strictly to these constraints.
+Adhere strictly to these constraints when working on this codebase.
 
-## 1. Coding Standards (Strict Enforcement)
+## 1. Code Quality
-* **Function Length:** Maximum 50 lines of code per function. Break down complex logic into small, testable, single-responsibility functions.
-* **DRY (Don't Repeat Yourself):** Abstract repetitive logic immediately. No duplicate boilerplate for database queries, API responses, or error handling.
-* **Negative Space Programming:** Utilize explicit assertions and constraints to define invalid states. Fail fast and early. Do not allow silent failures or cascading malformed data, especially in DOM parsing.
-* **Cyclomatic Complexity:** Flatten logic. Use early returns and guard clauses instead of deep nesting.
 
-## 2. Domain Architecture & Data Model
+- **Function length:** 50 lines max. Extract helpers ruthlessly.
-* **Data Isolation:** Scholar tracking is **user-scoped**. Validate mapping/join tables; never assume global links between users and Scholar IDs.
+- **File length:** 400 lines target, 600 lines hard ceiling. Files above this must be decomposed before adding more code.
-* **Data Deduplication:** Publications are **global records**. Deduplicate via Scholar cluster ID and normalized fingerprinting prior to database insertion.
+- **DRY:** Abstract repeated logic immediately. No duplicate boilerplate for queries, responses, or error handling.
-* **State Management:** Visibility and "read/unread" states exist exclusively on the scholar-publication link table, not the global publication table.
+- **Negative space programming:** Fail fast with explicit assertions and guard clauses. No silent failures, especially in DOM parsing.
-* **API Contract:** Exact envelope format required:
+- **Cyclomatic complexity:** Flatten with early returns. No deep nesting. No magic numbers.
-    * Success: `{"data": ..., "meta": {"request_id": "..."}}`
+- **No dead code:** Do not leave commented-out code, unused imports, or backward-compatibility shims. Delete cleanly.
-    * Error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}`
+- **Variable naming:** Any variable outside of a lambda functions or iterators should be descriptive. Code should be readable for maintainers.
 
-## 3. Scrape Safety & Rate Limiting (Immutable)
+## 2. Architecture
-These limits prevent IP bans and are not to be optimized away.
-* **Minimum Delay:** Enforce `INGESTION_MIN_REQUEST_DELAY_SECONDS` (default 2s) between all external requests.
-* **Anti-Detection:** Default to direct ID or profile URL ingestion. Name searches trigger CAPTCHAs.
-* **Cooldowns:** Respect `INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS` (1800s) and `INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS` (900s) upon threshold breaches.
 
-## 4. Current Environment & Stack
+### Data Model
-* **Backend:** Python 3.12+, FastAPI, SQLAlchemy (Async/asyncpg), Alembic.
-* **Frontend:** TypeScript, Vue 3, Vite.
-* **Infrastructure:** Multi-stage Docker.
 
-## 5. Domain Service Boundaries
+- Scholar tracking is **user-scoped**. Never assume global links between users and Scholar IDs.
-* **Strict Modularity:** Flat files in the `app/services/` root are strictly prohibited. All business logic and routing must reside exclusively within `app/services/domains/`.
+- Publications are **global, deduplicated records**. Deduplicate via cluster ID and normalized fingerprinting.
-* **`app/services/domains/scholar/*`:** Parser contract is fail-fast. Layout drift must emit explicit exceptions and `layout_*` reasons/warnings. Never allow silent partial success.
+- Read/unread, favorites, and visibility state live on the **scholar-publication link**, not the publication.
-* **`app/services/domains/ingestion/application.py`:** Orchestrates ingestion runs; validate parser outputs before persistence; enforce publication candidate constraints before upsert.
-* **`app/services/domains/publications/*`:** Publication list/read-state query layer. Includes `doi` + `pdf_url` fields for UI consumption, enforces non-blocking lazy OA enrichment scheduling on list reads, and exposes per-publication PDF retry behavior.
-* **`app/services/domains/crossref/*`:** DOI discovery fallback module. Must use bounded/paced lookups to avoid burst traffic and 429 responses.
-* **`app/services/domains/unpaywall/*`:** OA resolver by DOI only (best OA location + PDF URL extraction). Do not use Google Scholar for PDF resolution to avoid N+1 scrape amplification.
-* **`app/services/domains/portability/*`:** Handles JSON import/export for user-scoped scholars and scholar-publication link state while preserving global publication dedup rules.
-* **`app/services/domains/ingestion/scheduler.py`:** Owns automatic runs and continuation queue retries/drops; do not bypass safety gate or cooldown logic.
 
+### Service Boundaries
 
-## 6. UI rules
+All business logic lives in `app/services/<domain>/`. No flat files in `app/services/` root. Each domain owns its application service, types, and helpers.
-Make sure to properly integrate tailwind in combination with the preset theming
+
-Clarity through both styling and language are a priority. all UI elements need to have a proper reason for existing.
+### API Envelope
+
+All `/api/v1` responses use this exact envelope:
+
+```
+Success: {"data": ..., "meta": {"request_id": "..."}}
+Error:   {"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}
+```
+
+Use the Pydantic envelope schemas in `app/api/schemas.py`. Do not construct raw dicts.
+
+## 3. Scrape Safety (Immutable)
+
+These constraints prevent IP bans. They are not tunable to zero and must not be optimized away.
+
+- Enforce `INGESTION_MIN_REQUEST_DELAY_SECONDS` (default 2s) between all external requests.
+- Default to direct ID or profile URL ingestion. Name searches trigger CAPTCHAs.
+- Respect `INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS` (1800s) and `INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS` (900s) upon threshold breaches.
+
+## 4. Logging
+
+Use `structured_log()` from `app/logging_utils.py` for all domain logging. Do not use raw `logger.info()` / `logger.warning()` calls.
+
+```python
+from app.logging_utils import structured_log
+
+structured_log(logger, "info", "ingestion.run_started", user_id=user_id, scholar_count=count)
+```
+
+Every event name should be dot-namespaced to its domain (e.g., `arxiv.cache_hit`, `ingestion.safety_cooldown_entered`).
+
+## 5. Stack & Tooling
+
+- **Backend:** Python 3.12+, FastAPI, SQLAlchemy 2.0 (async/asyncpg), Alembic
+- **Frontend:** TypeScript, Vue 3, Vite, Tailwind CSS
+- **Infrastructure:** Multi-stage Docker, Docker Compose
+- **Package manager:** `uv` (used in Dockerfile and CI; `uv run` prefix for all commands)
+- **Linting:** `ruff check .` and `ruff format --check .` (config in `pyproject.toml`)
+- **Type checking:** `mypy app/`
+- **Versioning:** python-semantic-release with conventional commits
+
+## 6. Commits
+
+Follow [Conventional Commits](https://www.conventionalcommits.org/):
+
+```
+<type>(<scope>): <description>
+```
+
+Types: `feat`, `fix`, `docs`, `ci`, `refactor`, `test`, `chore`, `perf`.
+
+## 7. Testing & Quality Gates
+
+All **local development** commands run inside containers. Do not run bare `npm`, `pytest`, or `ruff` on the host. CI workflows (`.github/workflows/`) use bare runners with `uv` and `npm` directly — this is intentional since CI has no Docker-in-Docker dependency.
+
+### Backend
+
+```bash
+# Unit tests (default, excludes integration)
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app python -m pytest
+
+# Integration tests
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app python -m pytest -m integration
+
+# Linting
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app ruff check .
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app ruff format --check .
+
+# Type checking
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app mypy app/
+```
+
+Pytest markers: `integration`, `db`, `migrations`, `schema`, `smoke`.
+
+### Frontend
+
+```bash
+# Type checking
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run typecheck
+
+# Unit tests
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run test:run
+
+# Lint
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run lint
+
+# Production build
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run build
+```
+
+All four gates must pass before a PR is considered ready.
+
+## 8. Frontend
+
+- Use the tokenized theme system (`frontend/src/theme/presets/`). Do not hardcode colors.
+- Integrate Tailwind with preset theme tokens. Reference `frontend/scripts/check_theme_tokens.mjs` for enforcement.
+- Every UI element must have a clear purpose. Clarity through styling and language.

alembic/env.py

@@ -1,13 +1,13 @@
-from logging.config import fileConfig
 import os
+from logging.config import fileConfig
 
-from alembic import context
 from sqlalchemy import pool
 from sqlalchemy.engine import Connection
 from sqlalchemy.ext.asyncio import async_engine_from_config
 
-from app.db.base import metadata
+from alembic import context
 from app.db import models  # noqa: F401
+from app.db.base import metadata
 
 config = context.config
 

alembic/versions/20260216_0001_multi_user_schema.py

@@ -8,10 +8,11 @@ Create Date: 2026-02-16 17:10:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision: str = "20260216_0001"
 down_revision: str | Sequence[str] | None = None

alembic/versions/20260216_0002_add_user_admin_flag.py

@@ -7,9 +7,10 @@ Create Date: 2026-02-16 17:40:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision: str = "20260216_0002"
 down_revision: str | Sequence[str] | None = "20260216_0001"
@@ -31,4 +32,3 @@ def upgrade() -> None:
 
 def downgrade() -> None:
     op.drop_column("users", "is_admin")
-

alembic/versions/20260216_0003_add_ingestion_queue.py

@@ -7,9 +7,10 @@ Create Date: 2026-02-16 23:45:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision: str = "20260216_0003"
 down_revision: str | Sequence[str] | None = "20260216_0002"

alembic/versions/20260217_0004_queue_status_fields.py

@@ -7,9 +7,10 @@ Create Date: 2026-02-17 10:15:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision: str = "20260217_0004"
 down_revision: str | Sequence[str] | None = "20260216_0003"
@@ -22,9 +23,7 @@ def upgrade() -> None:
     inspector = sa.inspect(bind)
     columns = {column["name"] for column in inspector.get_columns("ingestion_queue_items")}
     indexes = {index["name"] for index in inspector.get_indexes("ingestion_queue_items")}
-    checks = {
+    checks = {constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items")}
-        constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items")
-    }
 
     if "status" not in columns:
         op.add_column(
@@ -80,9 +79,7 @@ def downgrade() -> None:
     inspector = sa.inspect(bind)
     columns = {column["name"] for column in inspector.get_columns("ingestion_queue_items")}
     indexes = {index["name"] for index in inspector.get_indexes("ingestion_queue_items")}
-    checks = {
+    checks = {constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items")}
-        constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items")
-    }
 
     if "ix_ingestion_queue_status_next_attempt" in indexes:
         op.drop_index("ix_ingestion_queue_status_next_attempt", table_name="ingestion_queue_items")

alembic/versions/20260217_0005_manual_run_idempotency.py

@@ -7,9 +7,10 @@ Create Date: 2026-02-17 16:20:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision: str = "20260217_0005"
 down_revision: str | Sequence[str] | None = "20260217_0004"
@@ -70,9 +71,7 @@ def upgrade() -> None:
             "crawl_runs",
             ["user_id", "idempotency_key"],
             unique=True,
-            postgresql_where=sa.text(
+            postgresql_where=sa.text("idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"),
-                "idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"
-            ),
         )
 
 

alembic/versions/20260217_0006_scholar_profile_images.py

@@ -7,9 +7,10 @@ Create Date: 2026-02-17 18:30:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision: str = "20260217_0006"
 down_revision: str | Sequence[str] | None = "20260217_0005"

alembic/versions/20260217_0007_scholar_initial_page_fingerprint.py

@@ -7,9 +7,10 @@ Create Date: 2026-02-17 18:55:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
+
 # revision identifiers, used by Alembic.
 revision: str = "20260217_0007"
 down_revision: str | Sequence[str] | None = "20260217_0006"

alembic/versions/20260219_0008_user_settings_nav_visible_pages.py

@@ -7,10 +7,10 @@ Create Date: 2026-02-19 14:58:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "20260219_0008"
@@ -21,7 +21,7 @@ depends_on: str | Sequence[str] | None = None
 
 TABLE_NAME = "user_settings"
 DEFAULT_NAV_VISIBLE_PAGES_SQL = (
-    "'[\"dashboard\",\"scholars\",\"publications\",\"settings\",\"style-guide\",\"runs\",\"users\"]'::jsonb"
+    '\'["dashboard","scholars","publications","settings","style-guide","runs","users"]\'::jsonb'
 )
 
 

alembic/versions/20260219_0009_user_settings_scrape_safety_state.py

@@ -7,10 +7,10 @@ Create Date: 2026-02-19 21:20:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "20260219_0009"

alembic/versions/20260219_0010_author_search_shared_state.py

@@ -7,10 +7,10 @@ Create Date: 2026-02-19 22:20:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "20260219_0010"

alembic/versions/20260220_0011_add_publication_doi.py

@@ -7,9 +7,9 @@ Create Date: 2026-02-20 13:35:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
 
 revision: str = "20260220_0011"
 down_revision: str | Sequence[str] | None = "20260219_0010"

alembic/versions/20260220_0012_add_data_repair_jobs.py

@@ -7,10 +7,10 @@ Create Date: 2026-02-20 23:35:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+from alembic import op
 
 revision: str = "20260220_0012"
 down_revision: str | Sequence[str] | None = "20260220_0011"

alembic/versions/20260221_0013_add_publication_pdf_jobs.py

@@ -7,9 +7,9 @@ Create Date: 2026-02-21 12:25:00.000000
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
 
 revision: str = "20260221_0013"
 down_revision: str | Sequence[str] | None = "20260220_0012"

alembic/versions/20260221_0014_add_scholar_publication_favorite_state.py

@@ -7,9 +7,9 @@ Create Date: 2026-02-21 14:35:00
 
 from __future__ import annotations
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = "20260221_0014"

alembic/versions/20260221_0015_enforce_request_delay_floor.py

@@ -9,9 +9,9 @@ from __future__ import annotations
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 
+from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "20260221_0015"
@@ -30,11 +30,7 @@ LEGACY_CHECK_NAME_MIN_2 = "ck_user_settings_request_delay_seconds_min_2"
 def _check_constraints() -> set[str]:
     bind = op.get_bind()
     inspector = sa.inspect(bind)
-    return {
+    return {str(item.get("name")) for item in inspector.get_check_constraints(TABLE_NAME) if item.get("name")}
-        str(item.get("name"))
-        for item in inspector.get_check_constraints(TABLE_NAME)
-        if item.get("name")
-    }
 
 
 def _drop_check_if_exists(name: str) -> None:
@@ -48,12 +44,7 @@ def _drop_check_if_exists(name: str) -> None:
 
 
 def upgrade() -> None:
-    op.execute(
+    op.execute(sa.text("UPDATE user_settings SET request_delay_seconds = 2 WHERE request_delay_seconds < 2"))
-        sa.text(
-            "UPDATE user_settings SET request_delay_seconds = 2 "
-            "WHERE request_delay_seconds < 2"
-        )
-    )
 
     _drop_check_if_exists(CHECK_NAME_MIN_1)
     _drop_check_if_exists(LEGACY_CHECK_NAME_MIN_1)

alembic/versions/20260222_0016_add_publication_identifiers.py

@@ -0,0 +1,104 @@
+"""Add publication identifiers table.
+
+Revision ID: 20260222_0016
+Revises: 20260221_0015
+Create Date: 2026-02-22 12:45:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision: str = "20260222_0016"
+down_revision: str | Sequence[str] | None = "20260221_0015"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _table_names() -> set[str]:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    return set(inspector.get_table_names())
+
+
+def _create_publication_identifiers_table() -> None:
+    op.create_table(
+        "publication_identifiers",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("publication_id", sa.Integer(), nullable=False),
+        sa.Column("kind", sa.String(length=32), nullable=False),
+        sa.Column("value_raw", sa.Text(), nullable=False),
+        sa.Column("value_normalized", sa.String(length=255), nullable=False),
+        sa.Column("source", sa.String(length=64), nullable=False),
+        sa.Column(
+            "confidence_score",
+            sa.Float(),
+            nullable=False,
+            server_default=sa.text("0"),
+        ),
+        sa.Column("evidence_url", sa.Text(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.CheckConstraint(
+            "confidence_score >= 0 AND confidence_score <= 1",
+            name="publication_identifiers_confidence_score_range",
+        ),
+        sa.ForeignKeyConstraint(
+            ["publication_id"],
+            ["publications.id"],
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_publication_identifiers")),
+        sa.UniqueConstraint(
+            "publication_id",
+            "kind",
+            "value_normalized",
+            name="uq_publication_identifiers_publication_kind_value",
+        ),
+    )
+
+
+def _create_publication_identifiers_indexes() -> None:
+    op.create_index(
+        "ix_publication_identifiers_kind_value",
+        "publication_identifiers",
+        ["kind", "value_normalized"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_publication_identifiers_publication_id",
+        "publication_identifiers",
+        ["publication_id"],
+        unique=False,
+    )
+
+
+def _drop_publication_identifiers_indexes() -> None:
+    op.drop_index("ix_publication_identifiers_publication_id", table_name="publication_identifiers")
+    op.drop_index("ix_publication_identifiers_kind_value", table_name="publication_identifiers")
+
+
+def upgrade() -> None:
+    if "publication_identifiers" in _table_names():
+        return
+    _create_publication_identifiers_table()
+    _create_publication_identifiers_indexes()
+
+
+def downgrade() -> None:
+    if "publication_identifiers" not in _table_names():
+        return
+    _drop_publication_identifiers_indexes()
+    op.drop_table("publication_identifiers")

alembic/versions/20260224_0018_add_openalex_enriched_to_publications.py

@@ -0,0 +1,33 @@
+"""Add openalex_enriched to publications
+
+Revision ID: 20260224_0018
+Revises: fa0c5e3262d5
+Create Date: 2026-02-24 19:10:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260224_0018"
+down_revision: str | Sequence[str] | None = "fa0c5e3262d5"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "publications", sa.Column("openalex_enriched", sa.Boolean(), server_default=sa.text("false"), nullable=False)
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("publications", "openalex_enriched")
+    # ### end Alembic commands ###

alembic/versions/20260224_0019_add_resolving_to_runstatus.py

@@ -0,0 +1,31 @@
+"""Add resolving to RunStatus enum
+
+Revision ID: 20260224_0019
+Revises: 20260224_0018
+Create Date: 2026-02-24 22:20:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260224_0019"
+down_revision: str | Sequence[str] | None = "20260224_0018"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # Use COMMIT to break out of the transaction block that Alembic creates in env.py.
+    # Postgres restricts ALTER TYPE ... ADD VALUE inside transactions when the type is in use.
+    # This is the robust, non-patchwork way to ensure the schema evolves correctly.
+    op.execute("COMMIT")
+    op.execute("ALTER TYPE run_status ADD VALUE IF NOT EXISTS 'resolving' AFTER 'running'")
+
+
+def downgrade() -> None:
+    # Enum values cannot be easily removed in Postgres without recreating the type,
+    # which is risky and usually avoided in simple migrations.
+    pass

alembic/versions/20260224_0020_add_openalex_last_attempt_at.py

@@ -0,0 +1,27 @@
+"""Add openalex_last_attempt_at to publications
+
+Revision ID: 20260224_0020
+Revises: 20260224_0019
+Create Date: 2026-02-24 22:50:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260224_0020"
+down_revision: str | Sequence[str] | None = "20260224_0019"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.add_column("publications", sa.Column("openalex_last_attempt_at", sa.DateTime(timezone=True), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("publications", "openalex_last_attempt_at")

alembic/versions/20260225_0021_enforce_single_active_run_per_user.py

@@ -0,0 +1,68 @@
+"""Enforce one active crawl run per user.
+
+Revision ID: 20260225_0021
+Revises: 20260224_0020
+Create Date: 2026-02-25 09:10:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260225_0021"
+down_revision: str | Sequence[str] | None = "20260224_0020"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+INDEX_NAME = "uq_crawl_runs_user_active"
+ACTIVE_STATUSES = ("running", "resolving")
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    indexes = {index["name"] for index in inspector.get_indexes("crawl_runs")}
+
+    op.execute("ALTER TYPE run_status ADD VALUE IF NOT EXISTS 'canceled'")
+
+    op.execute(
+        """
+        WITH ranked AS (
+            SELECT
+                id,
+                ROW_NUMBER() OVER (
+                    PARTITION BY user_id
+                    ORDER BY start_dt DESC, id DESC
+                ) AS rn
+            FROM crawl_runs
+            WHERE status IN ('running', 'resolving')
+        )
+        UPDATE crawl_runs AS runs
+        SET
+            status = 'failed',
+            end_dt = COALESCE(runs.end_dt, NOW())
+        FROM ranked
+        WHERE runs.id = ranked.id
+          AND ranked.rn > 1
+        """
+    )
+
+    if INDEX_NAME not in indexes:
+        op.create_index(
+            INDEX_NAME,
+            "crawl_runs",
+            ["user_id"],
+            unique=True,
+            postgresql_where=sa.text("status IN ('running'::run_status, 'resolving'::run_status)"),
+        )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    indexes = {index["name"] for index in inspector.get_indexes("crawl_runs")}
+    if INDEX_NAME in indexes:
+        op.drop_index(INDEX_NAME, table_name="crawl_runs")

alembic/versions/20260225_0022_add_canonical_title_hash_to_publications.py

@@ -0,0 +1,35 @@
+"""Add canonical_title_hash to publications for cross-scholar dedup.
+
+Revision ID: 20260225_0022
+Revises: 20260225_0021
+Create Date: 2026-02-25 10:00:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260225_0022"
+down_revision: str | Sequence[str] | None = "20260225_0021"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "publications",
+        sa.Column("canonical_title_hash", sa.String(64), nullable=True),
+    )
+    op.create_index(
+        "ix_publications_canonical_title_hash",
+        "publications",
+        ["canonical_title_hash"],
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_publications_canonical_title_hash", table_name="publications")
+    op.drop_column("publications", "canonical_title_hash")

alembic/versions/20260226_0023_add_arxiv_runtime_state.py

@@ -0,0 +1,55 @@
+"""Add shared arXiv runtime state table.
+
+Revision ID: 20260226_0023
+Revises: 20260225_0022
+Create Date: 2026-02-26 12:40:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260226_0023"
+down_revision: str | Sequence[str] | None = "20260225_0022"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    table_names = set(inspector.get_table_names())
+
+    if "arxiv_runtime_state" in table_names:
+        return
+
+    op.create_table(
+        "arxiv_runtime_state",
+        sa.Column("state_key", sa.String(length=64), nullable=False),
+        sa.Column("next_allowed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("cooldown_until", sa.DateTime(timezone=True), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.PrimaryKeyConstraint("state_key", name=op.f("pk_arxiv_runtime_state")),
+    )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    table_names = set(inspector.get_table_names())
+    if "arxiv_runtime_state" in table_names:
+        op.drop_table("arxiv_runtime_state")

alembic/versions/20260226_0024_add_arxiv_query_cache_entries.py

@@ -0,0 +1,70 @@
+"""Add arXiv query cache table.
+
+Revision ID: 20260226_0024
+Revises: 20260226_0023
+Create Date: 2026-02-26 14:20:00.000000
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "20260226_0024"
+down_revision: str | Sequence[str] | None = "20260226_0023"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    table_names = set(inspector.get_table_names())
+    if "arxiv_query_cache_entries" in table_names:
+        return
+
+    op.create_table(
+        "arxiv_query_cache_entries",
+        sa.Column("query_fingerprint", sa.String(length=64), nullable=False),
+        sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+        sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column(
+            "cached_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.PrimaryKeyConstraint(
+            "query_fingerprint",
+            name=op.f("pk_arxiv_query_cache_entries"),
+        ),
+    )
+    op.create_index(
+        "ix_arxiv_query_cache_expires_at",
+        "arxiv_query_cache_entries",
+        ["expires_at"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_arxiv_query_cache_cached_at",
+        "arxiv_query_cache_entries",
+        ["cached_at"],
+        unique=False,
+    )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    table_names = set(inspector.get_table_names())
+    if "arxiv_query_cache_entries" in table_names:
+        op.drop_table("arxiv_query_cache_entries")

alembic/versions/44f7e10ef777_remove_doi_from_publications_and_.py

@@ -0,0 +1,27 @@
+"""remove_doi_from_publications_and_migrate_identifiers
+
+Revision ID: 44f7e10ef777
+Revises: 20260222_0016
+Create Date: 2026-02-22 17:03:56.261936
+
+"""
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '44f7e10ef777'
+down_revision: str | Sequence[str] | None = '20260222_0016'
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    pass
+
+
+def downgrade() -> None:
+    pass
+

alembic/versions/fa0c5e3262d5_add_api_integration_keys_to_usersetting.py

@@ -0,0 +1,39 @@
+"""Add API integration keys to UserSetting
+
+Revision ID: fa0c5e3262d5
+Revises: 44f7e10ef777
+Create Date: 2026-02-23 14:25:45.021512
+
+"""
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'fa0c5e3262d5'
+down_revision: str | Sequence[str] | None = '44f7e10ef777'
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_publications_doi'), table_name='publications')
+    op.drop_column('publications', 'doi')
+    op.add_column('user_settings', sa.Column('openalex_api_key', sa.String(length=255), nullable=True))
+    op.add_column('user_settings', sa.Column('crossref_api_token', sa.String(length=255), nullable=True))
+    op.add_column('user_settings', sa.Column('crossref_api_mailto', sa.String(length=255), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('user_settings', 'crossref_api_mailto')
+    op.drop_column('user_settings', 'crossref_api_token')
+    op.drop_column('user_settings', 'openalex_api_key')
+    op.add_column('publications', sa.Column('doi', sa.VARCHAR(length=255), autoincrement=False, nullable=True))
+    op.create_index(op.f('ix_publications_doi'), 'publications', ['doi'], unique=False)
+    # ### end Alembic commands ###
+

app/api/__init__.py

@@ -1,2 +1 @@
 from __future__ import annotations
-

app/api/errors.py

@@ -82,4 +82,3 @@ def register_api_exception_handlers(app: FastAPI) -> None:
             message="Request validation failed.",
             details=exc.errors(),
         )
-

app/api/media.py

@@ -10,7 +10,8 @@ from app.api.deps import get_api_current_user
 from app.api.errors import ApiException
 from app.db.models import User
 from app.db.session import get_db_session
-from app.services.domains.scholars import application as scholar_service
+from app.services.scholars import application as scholar_service
+from app.services.scholars import uploads as scholar_uploads
 from app.settings import settings
 
 router = APIRouter(tags=["media"])
@@ -41,7 +42,7 @@ async def get_uploaded_scholar_image(
         )
 
     try:
-        image_path = scholar_service.resolve_upload_file_path(
+        image_path = scholar_uploads.resolve_upload_file_path(
             upload_dir=settings.scholar_image_upload_dir,
             relative_path=profile.profile_image_upload_path,
         )

app/api/router.py

@@ -2,11 +2,12 @@ from __future__ import annotations
 
 from fastapi import APIRouter
 
-from app.api.routers import admin, admin_dbops, auth, publications, runs, scholars, settings
+from app.api.routers import admin, admin_dbops, admin_settings, auth, publications, runs, scholars, settings
 
 router = APIRouter(prefix="/api/v1")
 router.include_router(auth.router)
 router.include_router(admin.router)
+router.include_router(admin_settings.router)
 router.include_router(admin_dbops.router)
 router.include_router(scholars.router)
 router.include_router(settings.router)

app/api/routers/__init__.py

@@ -1,2 +1 @@
 from __future__ import annotations
-

app/api/routers/admin.py

@@ -20,7 +20,8 @@ from app.auth.deps import get_auth_service
 from app.auth.service import AuthService
 from app.db.models import User
 from app.db.session import get_db_session
-from app.services.domains.users import application as user_service
+from app.logging_utils import structured_log
+from app.services.users import application as user_service
 
 logger = logging.getLogger(__name__)
 
@@ -48,14 +49,7 @@ async def list_users(
     admin_user: User = Depends(get_api_admin_user),
 ):
     users = await user_service.list_users(db_session)
-    logger.info(
+    structured_log(logger, "info", "api.admin.users_listed", admin_user_id=int(admin_user.id), user_count=len(users))
-        "api.admin.users_listed",
-        extra={
-            "event": "api.admin.users_listed",
-            "admin_user_id": int(admin_user.id),
-            "user_count": len(users),
-        },
-    )
     return success_payload(
         request,
         data={
@@ -92,14 +86,13 @@ async def create_user(
             message=str(exc),
         ) from exc
 
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.admin.user_created",
-        extra={
+        admin_user_id=int(admin_user.id),
-            "event": "api.admin.user_created",
+        target_user_id=int(created_user.id),
-            "admin_user_id": int(admin_user.id),
+        target_is_admin=bool(created_user.is_admin),
-            "target_user_id": int(created_user.id),
-            "target_is_admin": bool(created_user.is_admin),
-        },
     )
     return success_payload(
         request,
@@ -125,11 +118,7 @@ async def set_user_active(
             code="user_not_found",
             message="User not found.",
         )
-    if (
+    if int(target_user.id) == int(admin_user.id) and bool(target_user.is_active) and not bool(payload.is_active):
-        int(target_user.id) == int(admin_user.id)
-        and bool(target_user.is_active)
-        and not bool(payload.is_active)
-    ):
         raise ApiException(
             status_code=400,
             code="cannot_deactivate_self",
@@ -140,14 +129,13 @@ async def set_user_active(
         user=target_user,
         is_active=bool(payload.is_active),
     )
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.admin.user_active_updated",
-        extra={
+        admin_user_id=int(admin_user.id),
-            "event": "api.admin.user_active_updated",
+        target_user_id=int(updated_user.id),
-            "admin_user_id": int(admin_user.id),
+        is_active=bool(updated_user.is_active),
-            "target_user_id": int(updated_user.id),
-            "is_active": bool(updated_user.is_active),
-        },
     )
     return success_payload(
         request,
@@ -188,13 +176,12 @@ async def reset_user_password(
         user=target_user,
         password_hash=auth_service.hash_password(validated_password),
     )
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.admin.user_password_reset",
-        extra={
+        admin_user_id=int(admin_user.id),
-            "event": "api.admin.user_password_reset",
+        target_user_id=int(target_user.id),
-            "admin_user_id": int(admin_user.id),
-            "target_user_id": int(target_user.id),
-        },
     )
     return success_payload(
         request,

app/api/routers/admin_dbops.py

@@ -10,21 +10,25 @@ from app.api.errors import ApiException
 from app.api.responses import success_payload
 from app.api.schemas import (
     AdminDbIntegrityEnvelope,
-    AdminPdfQueueBulkEnqueueEnvelope,
-    AdminPdfQueueRequeueEnvelope,
-    AdminPdfQueueEnvelope,
     AdminDbRepairJobsEnvelope,
+    AdminPdfQueueBulkEnqueueEnvelope,
+    AdminPdfQueueEnvelope,
+    AdminPdfQueueRequeueEnvelope,
     AdminRepairPublicationLinksEnvelope,
     AdminRepairPublicationLinksRequest,
+    AdminRepairPublicationNearDuplicatesEnvelope,
+    AdminRepairPublicationNearDuplicatesRequest,
 )
 from app.db.models import DataRepairJob, User
 from app.db.session import get_db_session
-from app.services.domains.dbops import (
+from app.logging_utils import structured_log
+from app.services.dbops import (
     collect_integrity_report,
     list_repair_jobs,
     run_publication_link_repair,
+    run_publication_near_duplicate_repair,
 )
-from app.services.domains.publications import application as publication_service
+from app.services.publications import application as publication_service
 
 logger = logging.getLogger(__name__)
 
@@ -48,7 +52,7 @@ def _serialize_repair_job(job: DataRepairJob) -> dict[str, object]:
     }
 
 
-def _requested_by_value(*, payload: AdminRepairPublicationLinksRequest, admin_user: User) -> str:
+def _requested_by_value(*, payload, admin_user: User) -> str:
     from_payload = (payload.requested_by or "").strip()
     return from_payload or admin_user.email
 
@@ -57,7 +61,7 @@ def _serialize_pdf_queue_item(item) -> dict[str, object]:
     return {
         "publication_id": item.publication_id,
         "title": item.title,
-        "doi": item.doi,
+        "display_identifier": _serialize_display_identifier(item.display_identifier),
         "pdf_url": item.pdf_url,
         "status": item.status,
         "attempt_count": item.attempt_count,
@@ -73,6 +77,18 @@ def _serialize_pdf_queue_item(item) -> dict[str, object]:
     }
 
 
+def _serialize_display_identifier(value) -> dict[str, object] | None:
+    if value is None:
+        return None
+    return {
+        "kind": value.kind,
+        "value": value.value,
+        "label": value.label,
+        "url": value.url,
+        "confidence_score": float(value.confidence_score),
+    }
+
+
 def _requeue_response_state(*, queued: bool) -> tuple[str, str]:
     if queued:
         return "queued", "PDF lookup queued."
@@ -120,15 +136,14 @@ async def get_integrity_report(
     admin_user: User = Depends(get_api_admin_user),
 ):
     report = await collect_integrity_report(db_session)
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.admin.db.integrity_checked",
-        extra={
+        admin_user_id=int(admin_user.id),
-            "event": "api.admin.db.integrity_checked",
+        status=report.get("status"),
-            "admin_user_id": int(admin_user.id),
+        failure_count=len(report.get("failures", [])),
-            "status": report.get("status"),
+        warning_count=len(report.get("warnings", [])),
-            "failure_count": len(report.get("failures", [])),
-            "warning_count": len(report.get("warnings", [])),
-        },
     )
     return success_payload(request, data=report)
 
@@ -144,14 +159,13 @@ async def get_repair_jobs(
     admin_user: User = Depends(get_api_admin_user),
 ):
     jobs = await list_repair_jobs(db_session, limit=limit)
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.admin.db.repair_jobs_listed",
-        extra={
+        admin_user_id=int(admin_user.id),
-            "event": "api.admin.db.repair_jobs_listed",
+        limit=int(limit),
-            "admin_user_id": int(admin_user.id),
+        job_count=len(jobs),
-            "limit": int(limit),
-            "job_count": len(jobs),
-        },
     )
     return success_payload(
         request,
@@ -186,18 +200,17 @@ async def get_pdf_queue(
         offset=resolved_offset,
         status=normalized_status,
     )
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.admin.db.pdf_queue_listed",
-        extra={
+        admin_user_id=int(admin_user.id),
-            "event": "api.admin.db.pdf_queue_listed",
+        page=int(resolved_page),
-            "admin_user_id": int(admin_user.id),
+        page_size=int(resolved_limit),
-            "page": int(resolved_page),
+        offset=int(resolved_offset),
-            "page_size": int(resolved_limit),
+        status=normalized_status,
-            "offset": int(resolved_offset),
+        item_count=len(queue_page.items),
-            "status": normalized_status,
+        total_count=int(queue_page.total_count),
-            "item_count": len(queue_page.items),
-            "total_count": int(queue_page.total_count),
-        },
     )
     return success_payload(
         request,
@@ -236,14 +249,13 @@ async def requeue_pdf_lookup(
             message="Publication not found.",
         )
     status, message = _requeue_response_state(queued=result.queued)
-    logger.info(
+    structured_log(
-        "api.admin.db.pdf_queue_requeue_requested",
+        logger,
-        extra={
+        "info",
-            "event": "api.admin.db.pdf_queue_requeue_requested",
+        "api.admin.db.pdf_requeued",
-            "admin_user_id": int(admin_user.id),
+        admin_user_id=int(admin_user.id),
-            "publication_id": int(publication_id),
+        publication_id=int(publication_id),
-            "queued": bool(result.queued),
+        queued=bool(result.queued),
-        },
     )
     return success_payload(
         request,
@@ -272,15 +284,14 @@ async def requeue_all_missing_pdfs(
         request_email=admin_user.email,
         limit=limit,
     )
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.admin.db.pdf_queue_requeue_all",
-        extra={
+        admin_user_id=int(admin_user.id),
-            "event": "api.admin.db.pdf_queue_requeue_all",
+        limit=int(limit),
-            "admin_user_id": int(admin_user.id),
+        requested_count=int(result.requested_count),
-            "limit": int(limit),
+        queued_count=int(result.queued_count),
-            "requested_count": int(result.requested_count),
-            "queued_count": int(result.queued_count),
-        },
     )
     return success_payload(
         request,
@@ -321,17 +332,114 @@ async def trigger_publication_link_repair(
             code="invalid_repair_scope",
             message=str(exc),
         ) from exc
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.admin.db.publication_link_repair_triggered",
-        extra={
+        admin_user_id=int(admin_user.id),
-            "event": "api.admin.db.publication_link_repair_triggered",
+        scope_mode=payload.scope_mode,
-            "admin_user_id": int(admin_user.id),
+        target_user_id=int(payload.user_id) if payload.user_id is not None else None,
-            "scope_mode": payload.scope_mode,
+        dry_run=bool(payload.dry_run),
-            "target_user_id": int(payload.user_id) if payload.user_id is not None else None,
+        gc_orphan_publications=bool(payload.gc_orphan_publications),
-            "dry_run": bool(payload.dry_run),
+        job_id=int(result["job_id"]),
-            "gc_orphan_publications": bool(payload.gc_orphan_publications),
+        status=result["status"],
-            "job_id": int(result["job_id"]),
-            "status": result["status"],
-        },
     )
     return success_payload(request, data=result)
+
+
+@router.post(
+    "/repairs/publication-near-duplicates",
+    response_model=AdminRepairPublicationNearDuplicatesEnvelope,
+)
+async def trigger_publication_near_duplicate_repair(
+    payload: AdminRepairPublicationNearDuplicatesRequest,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    admin_user: User = Depends(get_api_admin_user),
+):
+    try:
+        result = await run_publication_near_duplicate_repair(
+            db_session,
+            dry_run=bool(payload.dry_run),
+            similarity_threshold=float(payload.similarity_threshold),
+            min_shared_tokens=int(payload.min_shared_tokens),
+            max_year_delta=int(payload.max_year_delta),
+            max_clusters=int(payload.max_clusters),
+            selected_cluster_keys=list(payload.selected_cluster_keys),
+            requested_by=_requested_by_value(payload=payload, admin_user=admin_user),
+        )
+    except ValueError as exc:
+        raise ApiException(
+            status_code=400,
+            code="invalid_near_duplicate_repair_request",
+            message=str(exc),
+        ) from exc
+    structured_log(
+        logger,
+        "info",
+        "api.admin.db.dedup_repair_triggered",
+        admin_user_id=int(admin_user.id),
+        dry_run=bool(payload.dry_run),
+        selected_cluster_count=len(payload.selected_cluster_keys),
+        job_id=int(result["job_id"]),
+        status=result["status"],
+    )
+    return success_payload(request, data=result)
+
+
+DROP_PUBLICATIONS_CONFIRMATION = "DROP ALL PUBLICATIONS"
+
+
+@router.post("/drop-all-publications")
+async def drop_all_publications(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    admin_user: User = Depends(get_api_admin_user),
+):
+    body = await request.json()
+    confirmation_text = (body.get("confirmation_text") or "").strip()
+    if confirmation_text != DROP_PUBLICATIONS_CONFIRMATION:
+        raise ApiException(
+            status_code=400,
+            code="confirmation_required",
+            message=f"Type '{DROP_PUBLICATIONS_CONFIRMATION}' to confirm this destructive action.",
+        )
+
+    from sqlalchemy import delete, func, select, update
+
+    from app.db.models import (
+        Publication,
+        PublicationIdentifier,
+        PublicationPdfJob,
+        PublicationPdfJobEvent,
+        ScholarProfile,
+        ScholarPublication,
+    )
+
+    count_result = await db_session.execute(select(func.count()).select_from(Publication))
+    total_publications = count_result.scalar_one()
+
+    await db_session.execute(delete(ScholarPublication))
+    await db_session.execute(delete(PublicationIdentifier))
+    await db_session.execute(delete(PublicationPdfJobEvent))
+    await db_session.execute(delete(PublicationPdfJob))
+    await db_session.execute(delete(Publication))
+    await db_session.execute(update(ScholarProfile).values(baseline_completed=False))
+    await db_session.commit()
+
+    structured_log(
+        logger,
+        "warning",
+        "api.admin.db.all_publications_dropped",
+        admin_user_id=int(admin_user.id),
+        admin_email=admin_user.email,
+        deleted_count=int(total_publications),
+    )
+    return success_payload(
+        request,
+        data={
+            "deleted_count": int(total_publications),
+            "message": f"Dropped {total_publications} publication(s) and all related data. "
+            "Scholar baselines have been reset; the next run will re-discover all publications.",
+        },
+    )

app/api/routers/admin_settings.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import logging
+
+from fastapi import APIRouter, Depends, Request
+
+from app.api.deps import get_api_admin_user
+from app.api.errors import ApiException
+from app.api.responses import success_payload
+from app.api.schemas import (
+    AdminScholarHttpSettingsEnvelope,
+    AdminScholarHttpSettingsUpdateRequest,
+)
+from app.db.models import User
+from app.logging_utils import structured_log
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/admin/settings", tags=["api-admin-settings"])
+_MAX_USER_AGENT_LENGTH = 512
+_MAX_ACCEPT_LANGUAGE_LENGTH = 128
+_MAX_COOKIE_LENGTH = 8192
+
+
+def _normalize_header_value(value: str, *, field_name: str, max_length: int) -> str:
+    normalized = value.strip()
+    if len(normalized) > max_length:
+        raise ApiException(
+            status_code=400,
+            code="invalid_admin_setting",
+            message=f"{field_name} must be {max_length} characters or fewer.",
+        )
+    return normalized
+
+
+def _serialize_scholar_http_settings() -> dict[str, object]:
+    return {
+        "user_agent": settings.scholar_http_user_agent,
+        "rotate_user_agent": bool(settings.scholar_http_rotate_user_agent),
+        "accept_language": settings.scholar_http_accept_language,
+        "cookie": settings.scholar_http_cookie,
+    }
+
+
+def _apply_scholar_http_settings(payload: AdminScholarHttpSettingsUpdateRequest) -> None:
+    user_agent = _normalize_header_value(
+        payload.user_agent,
+        field_name="user_agent",
+        max_length=_MAX_USER_AGENT_LENGTH,
+    )
+    accept_language = _normalize_header_value(
+        payload.accept_language,
+        field_name="accept_language",
+        max_length=_MAX_ACCEPT_LANGUAGE_LENGTH,
+    )
+    cookie = _normalize_header_value(
+        payload.cookie,
+        field_name="cookie",
+        max_length=_MAX_COOKIE_LENGTH,
+    )
+    object.__setattr__(settings, "scholar_http_user_agent", user_agent)
+    object.__setattr__(settings, "scholar_http_rotate_user_agent", bool(payload.rotate_user_agent))
+    object.__setattr__(settings, "scholar_http_accept_language", accept_language)
+    object.__setattr__(settings, "scholar_http_cookie", cookie)
+
+
+@router.get(
+    "/scholar-http",
+    response_model=AdminScholarHttpSettingsEnvelope,
+)
+async def get_scholar_http_settings(
+    request: Request,
+    admin_user: User = Depends(get_api_admin_user),
+):
+    structured_log(
+        logger,
+        "info",
+        "api.admin.settings.scholar_http_read",
+        admin_user_id=int(admin_user.id),
+    )
+    return success_payload(request, data=_serialize_scholar_http_settings())
+
+
+@router.put(
+    "/scholar-http",
+    response_model=AdminScholarHttpSettingsEnvelope,
+)
+async def update_scholar_http_settings(
+    payload: AdminScholarHttpSettingsUpdateRequest,
+    request: Request,
+    admin_user: User = Depends(get_api_admin_user),
+):
+    _apply_scholar_http_settings(payload)
+    structured_log(
+        logger,
+        "info",
+        "api.admin.settings.scholar_http_updated",
+        admin_user_id=int(admin_user.id),
+        rotate_user_agent=bool(settings.scholar_http_rotate_user_agent),
+        user_agent_set=bool(settings.scholar_http_user_agent.strip()),
+        cookie_set=bool(settings.scholar_http_cookie.strip()),
+    )
+    return success_payload(request, data=_serialize_scholar_http_settings())

app/api/routers/auth.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
 import logging
+from typing import NoReturn
 
 from fastapi import APIRouter, Depends, Request
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.api.errors import ApiException
 from app.api.deps import get_api_current_user
+from app.api.errors import ApiException
 from app.api.responses import success_payload
 from app.api.schemas import (
     AuthMeEnvelope,
@@ -16,15 +17,16 @@ from app.api.schemas import (
     LoginRequest,
     MessageEnvelope,
 )
+from app.auth import runtime as auth_runtime
 from app.auth.deps import get_auth_service, get_login_rate_limiter
 from app.auth.rate_limit import SlidingWindowRateLimiter
-from app.auth import runtime as auth_runtime
 from app.auth.service import AuthService
 from app.auth.session import set_session_user
 from app.db.models import User
 from app.db.session import get_db_session
+from app.logging_utils import structured_log
 from app.security.csrf import ensure_csrf_token
-from app.services.domains.users import application as user_service
+from app.services.users import application as user_service
 
 logger = logging.getLogger(__name__)
 
@@ -36,13 +38,12 @@ def _login_limiter_key_and_email(request: Request, payload: LoginRequest) -> tup
 
 
 def _raise_rate_limited(normalized_email: str, retry_after_seconds: int) -> None:
-    logger.warning(
+    structured_log(
+        logger,
+        "warning",
         "api.auth.login_rate_limited",
-        extra={
+        email=normalized_email,
-            "event": "api.auth.login_rate_limited",
+        retry_after_seconds=retry_after_seconds,
-            "email": normalized_email,
-            "retry_after_seconds": retry_after_seconds,
-        },
     )
     raise ApiException(
         status_code=429,
@@ -61,11 +62,8 @@ def _serialize_user_payload(user: User) -> dict[str, object]:
     }
 
 
-def _raise_invalid_credentials(*, normalized_email: str) -> None:
+def _raise_invalid_credentials(*, normalized_email: str) -> NoReturn:
-    logger.info(
+    structured_log(logger, "info", "api.auth.login_failed", email=normalized_email)
-        "api.auth.login_failed",
-        extra={"event": "api.auth.login_failed", "email": normalized_email},
-    )
     raise ApiException(
         status_code=401,
         code="invalid_credentials",
@@ -105,14 +103,7 @@ async def login(
         email=user.email,
         is_admin=user.is_admin,
     )
-    logger.info(
+    structured_log(logger, "info", "api.auth.login_succeeded", user_id=user.id, is_admin=user.is_admin)
-        "api.auth.login_succeeded",
-        extra={
-            "event": "api.auth.login_succeeded",
-            "user_id": user.id,
-            "is_admin": user.is_admin,
-        },
-    )
     return success_payload(
         request,
         data={
@@ -199,13 +190,7 @@ async def change_password(
         user=current_user,
         password_hash=auth_service.hash_password(validated_password),
     )
-    logger.info(
+    structured_log(logger, "info", "api.auth.password_changed", user_id=int(current_user.id))
-        "api.auth.password_changed",
-        extra={
-            "event": "api.auth.password_changed",
-            "user_id": int(current_user.id),
-        },
-    )
     return success_payload(
         request,
         data={"message": "Password updated successfully."},
@@ -222,12 +207,8 @@ async def logout(
 ):
     current_user = await auth_runtime.get_authenticated_user(request, db_session)
     auth_runtime.invalidate_session(request)
-    logger.info(
+    structured_log(
-        "api.auth.logout",
+        logger, "info", "api.auth.logout", user_id=int(current_user.id) if current_user is not None else None
-        extra={
-            "event": "api.auth.logout",
-            "user_id": int(current_user.id) if current_user is not None else None,
-        },
     )
     return success_payload(
         request,

app/api/routers/publications.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+from datetime import UTC, datetime
 from typing import Literal
 
 from fastapi import APIRouter, Depends, Path, Query, Request
@@ -21,8 +22,10 @@ from app.api.schemas import (
 )
 from app.db.models import User
 from app.db.session import get_db_session
-from app.services.domains.publications import application as publication_service
+from app.logging_utils import structured_log
-from app.services.domains.scholars import application as scholar_service
+from app.services.publication_identifiers import application as identifier_service
+from app.services.publications import application as publication_service
+from app.services.scholars import application as scholar_service
 from app.settings import settings
 
 logger = logging.getLogger(__name__)
@@ -61,7 +64,7 @@ def _serialize_publication_item(item) -> dict[str, object]:
         "citation_count": item.citation_count,
         "venue_text": item.venue_text,
         "pub_url": item.pub_url,
-        "doi": item.doi,
+        "display_identifier": _serialize_display_identifier(item.display_identifier),
         "pdf_url": item.pdf_url,
         "pdf_status": item.pdf_status,
         "pdf_attempt_count": item.pdf_attempt_count,
@@ -74,29 +77,46 @@ def _serialize_publication_item(item) -> dict[str, object]:
     }
 
 
+def _serialize_display_identifier(value) -> dict[str, object] | None:
+    if value is None:
+        return None
+    return {
+        "kind": value.kind,
+        "value": value.value,
+        "label": value.label,
+        "url": value.url,
+        "confidence_score": float(value.confidence_score),
+    }
+
+
 async def _publication_counts(
     db_session: AsyncSession,
     *,
     user_id: int,
     selected_scholar_id: int | None,
     favorite_only: bool,
+    search: str | None,
+    snapshot_before: datetime | None,
 ) -> tuple[int, int, int, int]:
     unread_count = await publication_service.count_unread_for_user(
         db_session,
         user_id=user_id,
         scholar_profile_id=selected_scholar_id,
         favorite_only=favorite_only,
+        snapshot_before=snapshot_before,
     )
     favorites_count = await publication_service.count_favorite_for_user(
         db_session,
         user_id=user_id,
         scholar_profile_id=selected_scholar_id,
+        snapshot_before=snapshot_before,
     )
     latest_count = await publication_service.count_latest_for_user(
         db_session,
         user_id=user_id,
         scholar_profile_id=selected_scholar_id,
         favorite_only=favorite_only,
+        snapshot_before=snapshot_before,
     )
     total_count = await publication_service.count_for_user(
         db_session,
@@ -104,6 +124,8 @@ async def _publication_counts(
         mode=publication_service.MODE_ALL,
         scholar_profile_id=selected_scholar_id,
         favorite_only=favorite_only,
+        search=search,
+        snapshot_before=snapshot_before,
     )
     return unread_count, favorites_count, latest_count, total_count
 
@@ -115,8 +137,12 @@ async def _list_publications_for_request(
     mode: Literal["all", "unread", "latest", "new"] | None,
     favorite_only: bool,
     scholar_profile_id: int | None,
+    search: str | None,
+    sort_by: str,
+    sort_dir: str,
     limit: int,
     offset: int,
+    snapshot_before: datetime | None,
 ) -> tuple[str, int | None, list]:
     resolved_mode = publication_service.resolve_publication_view_mode(mode)
     selected_scholar_id = scholar_profile_id
@@ -131,8 +157,12 @@ async def _list_publications_for_request(
         mode=resolved_mode,
         scholar_profile_id=selected_scholar_id,
         favorite_only=favorite_only,
+        search=search,
+        sort_by=sort_by,
+        sort_dir=sort_dir,
         limit=limit,
         offset=offset,
+        snapshot_before=snapshot_before,
     )
     await publication_service.schedule_missing_pdf_enrichment_for_user(
         db_session,
@@ -148,6 +178,27 @@ async def _list_publications_for_request(
     return resolved_mode, selected_scholar_id, hydrated
 
 
+def _resolve_publications_snapshot(
+    *,
+    snapshot: str | None,
+) -> tuple[datetime, str]:
+    if snapshot is None:
+        now_utc = datetime.now(UTC)
+        return now_utc, now_utc.isoformat()
+    try:
+        parsed = datetime.fromisoformat(snapshot)
+    except ValueError as exc:
+        raise ApiException(
+            status_code=400,
+            code="invalid_snapshot",
+            message="Invalid publications snapshot cursor.",
+        ) from exc
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=UTC)
+    normalized = parsed.astimezone(UTC)
+    return normalized, normalized.isoformat()
+
+
 def _resolve_publications_paging(
     *,
     page: int,
@@ -174,6 +225,7 @@ def _publications_list_data(
     page: int,
     page_size: int,
     offset: int,
+    snapshot: str,
 ) -> dict[str, object]:
     return {
         "mode": mode,
@@ -186,6 +238,7 @@ def _publications_list_data(
         "total_count": total_count,
         "page": int(page),
         "page_size": int(page_size),
+        "snapshot": snapshot,
         "has_prev": int(offset) > 0,
         "has_next": int(offset) + int(page_size) < int(total_count),
         "publications": [_serialize_publication_item(item) for item in publications],
@@ -280,32 +333,12 @@ async def _favorite_publication_state(
         db_session,
         items=[publication],
     )
-    return hydrated[0] if hydrated else publication
+    current = hydrated[0] if hydrated else publication
-
+    identifiers = await identifier_service.overlay_publication_items_with_display_identifiers(
-
+        db_session,
-def _log_retry_pdf_result(
+        items=[current],
-    *,
-    current_user: User,
-    scholar_profile_id: int,
-    publication_id: int,
-    queued: bool,
-    resolved_pdf: bool,
-    pdf_status: str,
-    doi: str | None,
-) -> None:
-    logger.info(
-        "api.publications.retry_pdf",
-        extra={
-            "event": "api.publications.retry_pdf",
-            "user_id": current_user.id,
-            "scholar_profile_id": scholar_profile_id,
-            "publication_id": publication_id,
-            "queued": queued,
-            "resolved_pdf": resolved_pdf,
-            "pdf_status": pdf_status,
-            "has_doi": bool(doi),
-        },
     )
+    return identifiers[0] if identifiers else current
 
 
 @router.get(
@@ -317,10 +350,14 @@ async def list_publications(
     mode: Literal["all", "unread", "latest", "new"] | None = Query(default=None),
     favorite_only: bool = Query(default=False),
     scholar_profile_id: int | None = Query(default=None, ge=1),
+    search: str | None = Query(default=None, min_length=1, max_length=200),
+    sort_by: Literal["first_seen", "title", "year", "citations", "scholar", "pdf_status"] = Query(default="first_seen"),
+    sort_dir: Literal["asc", "desc"] = Query(default="desc"),
     page: int = Query(default=1, ge=1),
     page_size: int = Query(default=100, ge=1, le=500),
     limit: int | None = Query(default=None, ge=1, le=1000),
     offset: int | None = Query(default=None, ge=0),
+    snapshot: str | None = Query(default=None, min_length=1, max_length=64),
     db_session: AsyncSession = Depends(get_db_session),
     current_user: User = Depends(get_api_current_user),
 ):
@@ -330,20 +367,28 @@ async def list_publications(
         limit=limit,
         offset=offset,
     )
+    snapshot_before, snapshot_cursor = _resolve_publications_snapshot(snapshot=snapshot)
+    normalized_search = (search or "").strip() or None
     resolved_mode, selected_scholar_id, publications = await _list_publications_for_request(
         db_session,
         current_user=current_user,
         mode=mode,
         favorite_only=favorite_only,
         scholar_profile_id=scholar_profile_id,
+        search=normalized_search,
+        sort_by=sort_by,
+        sort_dir=sort_dir,
         limit=resolved_limit,
         offset=resolved_offset,
+        snapshot_before=snapshot_before,
     )
     unread_count, favorites_count, latest_count, total_count = await _publication_counts(
         db_session,
         user_id=current_user.id,
         selected_scholar_id=selected_scholar_id,
         favorite_only=favorite_only,
+        search=normalized_search,
+        snapshot_before=snapshot_before,
     )
     data = _publications_list_data(
         mode=resolved_mode,
@@ -357,6 +402,7 @@ async def list_publications(
         page=resolved_page,
         page_size=resolved_limit,
         offset=resolved_offset,
+        snapshot=snapshot_cursor,
     )
     return success_payload(request, data=data)
 
@@ -374,13 +420,8 @@ async def mark_all_publications_read(
         db_session,
         user_id=current_user.id,
     )
-    logger.info(
+    structured_log(
-        "api.publications.mark_all_read",
+        logger, "info", "api.publications.mark_all_read", user_id=current_user.id, updated_count=updated_count
-        extra={
-            "event": "api.publications.mark_all_read",
-            "user_id": current_user.id,
-            "updated_count": updated_count,
-        },
     )
     return success_payload(
         request,
@@ -401,25 +442,19 @@ async def mark_selected_publications_read(
     db_session: AsyncSession = Depends(get_db_session),
     current_user: User = Depends(get_api_current_user),
 ):
-    selection_pairs = sorted(
+    selection_pairs = sorted({(int(item.scholar_profile_id), int(item.publication_id)) for item in payload.selections})
-        {
-            (int(item.scholar_profile_id), int(item.publication_id))
-            for item in payload.selections
-        }
-    )
     updated_count = await publication_service.mark_selected_as_read_for_user(
         db_session,
         user_id=current_user.id,
         selections=selection_pairs,
     )
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.publications.mark_selected_read",
-        extra={
+        user_id=current_user.id,
-            "event": "api.publications.mark_selected_read",
+        requested_count=len(selection_pairs),
-            "user_id": current_user.id,
+        updated_count=updated_count,
-            "requested_count": len(selection_pairs),
-            "updated_count": updated_count,
-        },
     )
     return success_payload(
         request,
@@ -454,14 +489,16 @@ async def retry_publication_pdf(
         resolved_pdf=resolved_pdf,
         pdf_status=current.pdf_status,
     )
-    _log_retry_pdf_result(
+    structured_log(
-        current_user=current_user,
+        logger,
+        "info",
+        "api.publications.retry_pdf",
+        user_id=current_user.id,
         scholar_profile_id=payload.scholar_profile_id,
         publication_id=publication_id,
         queued=queued,
         resolved_pdf=resolved_pdf,
         pdf_status=current.pdf_status,
-        doi=current.doi,
     )
     return success_payload(
         request,
@@ -492,15 +529,14 @@ async def toggle_publication_favorite(
         publication_id=publication_id,
         is_favorite=payload.is_favorite,
     )
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.publications.favorite",
-        extra={
+        user_id=current_user.id,
-            "event": "api.publications.favorite",
+        scholar_profile_id=payload.scholar_profile_id,
-            "user_id": current_user.id,
+        publication_id=publication_id,
-            "scholar_profile_id": payload.scholar_profile_id,
+        is_favorite=bool(payload.is_favorite),
-            "publication_id": publication_id,
-            "is_favorite": bool(payload.is_favorite),
-        },
     )
     return success_payload(
         request,

app/api/routers/run_manual.py

@@ -0,0 +1,243 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, NoReturn
+
+from fastapi import Request
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import ApiException
+from app.api.responses import success_payload
+from app.api.routers.run_serializers import manual_run_payload_from_run
+from app.db.models import RunStatus, RunTriggerType
+from app.logging_utils import structured_log
+from app.services.ingestion import application as ingestion_service
+from app.services.ingestion import safety as run_safety_service
+from app.services.runs import application as run_service
+from app.services.settings import application as user_settings_service
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+async def load_safety_state(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> dict[str, Any]:
+    user_settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=user_id,
+    )
+    previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
+    if run_safety_service.clear_expired_cooldown(user_settings):
+        await db_session.commit()
+        await db_session.refresh(user_settings)
+        structured_log(
+            logger,
+            "info",
+            "api.runs.safety_cooldown_cleared",
+            user_id=user_id,
+            reason=previous_safety_state.get("cooldown_reason"),
+            cooldown_until=previous_safety_state.get("cooldown_until"),
+        )
+    return run_safety_service.get_safety_state_payload(user_settings)
+
+
+def raise_manual_runs_disabled(*, user_id: int, safety_state: dict[str, Any]) -> None:
+    structured_log(
+        logger,
+        "warning",
+        "api.runs.manual_blocked_policy",
+        user_id=user_id,
+        policy={"manual_run_allowed": False},
+        safety_state=safety_state,
+    )
+    raise ApiException(
+        status_code=403,
+        code="manual_runs_disabled",
+        message="Manual checks are disabled by server policy.",
+        details={
+            "policy": {"manual_run_allowed": False},
+            "safety_state": safety_state,
+        },
+    )
+
+
+async def reused_manual_run_payload(
+    db_session: AsyncSession,
+    *,
+    request: Request,
+    user_id: int,
+    idempotency_key: str | None,
+    safety_state: dict[str, Any],
+) -> dict[str, Any] | None:
+    if idempotency_key is None:
+        return None
+    previous_run = await run_service.get_manual_run_by_idempotency_key(
+        db_session,
+        user_id=user_id,
+        idempotency_key=idempotency_key,
+    )
+    if previous_run is None:
+        return None
+    if previous_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING):
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run with this idempotency key is still in progress.",
+            details={"run_id": int(previous_run.id), "idempotency_key": idempotency_key},
+        )
+    return success_payload(
+        request,
+        data=manual_run_payload_from_run(
+            previous_run,
+            idempotency_key=idempotency_key,
+            reused_existing_run=True,
+            safety_state=safety_state,
+        ),
+    )
+
+
+async def run_ingestion_for_manual(
+    db_session: AsyncSession,
+    *,
+    ingest_service: ingestion_service.ScholarIngestionService,
+    user_id: int,
+    idempotency_key: str | None,
+):
+    user_settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=user_id,
+    )
+    return await ingest_service.run_for_user(
+        db_session,
+        user_id=user_id,
+        trigger_type=RunTriggerType.MANUAL,
+        request_delay_seconds=user_settings.request_delay_seconds,
+        network_error_retries=settings.ingestion_network_error_retries,
+        retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+        max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
+        page_size=settings.ingestion_page_size,
+        auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
+        queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
+        idempotency_key=idempotency_key,
+        alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
+        alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
+        alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
+    )
+
+
+async def recover_integrity_error(
+    db_session: AsyncSession,
+    *,
+    request: Request,
+    user_id: int,
+    idempotency_key: str | None,
+    original_exc: IntegrityError,
+) -> dict[str, Any]:
+    if idempotency_key is None:
+        structured_log(
+            logger,
+            "exception",
+            "api.runs.manual_integrity_error",
+            user_id=user_id,
+        )
+        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
+    existing_run = await run_service.get_manual_run_by_idempotency_key(
+        db_session,
+        user_id=user_id,
+        idempotency_key=idempotency_key,
+    )
+    if existing_run is None:
+        structured_log(
+            logger,
+            "exception",
+            "api.runs.manual_integrity_error",
+            user_id=user_id,
+        )
+        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
+    if existing_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING):
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run with this idempotency key is still in progress.",
+            details={"run_id": int(existing_run.id), "idempotency_key": idempotency_key},
+        ) from original_exc
+    return success_payload(
+        request,
+        data=manual_run_payload_from_run(
+            existing_run,
+            idempotency_key=idempotency_key,
+            reused_existing_run=True,
+            safety_state=await load_safety_state(db_session, user_id=user_id),
+        ),
+    )
+
+
+async def execute_manual_run(
+    db_session: AsyncSession,
+    *,
+    request: Request,
+    ingest_service: ingestion_service.ScholarIngestionService,
+    user_id: int,
+    idempotency_key: str | None,
+):
+    try:
+        return await run_ingestion_for_manual(
+            db_session,
+            ingest_service=ingest_service,
+            user_id=user_id,
+            idempotency_key=idempotency_key,
+        )
+    except ingestion_service.RunAlreadyInProgressError as exc:
+        await db_session.rollback()
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run is already in progress for this account.",
+        ) from exc
+    except ingestion_service.RunBlockedBySafetyPolicyError as exc:
+        await db_session.rollback()
+        raise_manual_blocked_safety(exc=exc, user_id=user_id)
+    except IntegrityError as exc:
+        await db_session.rollback()
+        return await recover_integrity_error(
+            db_session,
+            request=request,
+            user_id=user_id,
+            idempotency_key=idempotency_key,
+            original_exc=exc,
+        )
+    except Exception as exc:
+        await db_session.rollback()
+        raise_manual_failed(exc=exc, user_id=user_id)
+
+
+def raise_manual_blocked_safety(*, exc, user_id: int) -> NoReturn:
+    structured_log(
+        logger,
+        "info",
+        "api.runs.manual_blocked_safety",
+        user_id=user_id,
+        reason=exc.safety_state.get("cooldown_reason"),
+        cooldown_until=exc.safety_state.get("cooldown_until"),
+        cooldown_remaining_seconds=exc.safety_state.get("cooldown_remaining_seconds"),
+    )
+    raise ApiException(
+        status_code=429,
+        code=exc.code,
+        message=exc.message,
+        details={"safety_state": exc.safety_state},
+    ) from exc
+
+
+def raise_manual_failed(*, exc: Exception, user_id: int) -> NoReturn:
+    structured_log(
+        logger,
+        "exception",
+        "api.runs.manual_failed",
+        user_id=user_id,
+    )
+    raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from exc

app/api/routers/run_serializers.py

@@ -0,0 +1,238 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from app.api.errors import ApiException
+from app.services.runs import application as run_service
+
+IDEMPOTENCY_HEADER = "Idempotency-Key"
+IDEMPOTENCY_MAX_LENGTH = 128
+IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
+
+
+def _int_value(value: Any, default: int = 0) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _str_value(value: Any) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text if text else None
+
+
+def _bool_value(value: Any, default: bool = False) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        lowered = value.strip().lower()
+        if lowered in {"1", "true", "yes", "on"}:
+            return True
+        if lowered in {"0", "false", "no", "off"}:
+            return False
+    return default
+
+
+def normalize_idempotency_key(raw_value: str | None) -> str | None:
+    if raw_value is None:
+        return None
+    candidate = raw_value.strip()
+    if not candidate:
+        return None
+    if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate):
+        raise ApiException(
+            status_code=400,
+            code="invalid_idempotency_key",
+            message=("Invalid Idempotency-Key. Use 8-128 characters from: A-Z a-z 0-9 . _ : -"),
+        )
+    return candidate
+
+
+def serialize_run(run) -> dict[str, Any]:
+    summary = run_service.extract_run_summary(run.error_log)
+    return {
+        "id": int(run.id),
+        "trigger_type": run.trigger_type.value,
+        "status": run.status.value,
+        "start_dt": run.start_dt,
+        "end_dt": run.end_dt,
+        "scholar_count": int(run.scholar_count or 0),
+        "new_publication_count": int(run.new_pub_count or 0),
+        "failed_count": int(summary["failed_count"]),
+        "partial_count": int(summary["partial_count"]),
+    }
+
+
+def serialize_queue_item(item) -> dict[str, Any]:
+    return {
+        "id": int(item.id),
+        "scholar_profile_id": int(item.scholar_profile_id),
+        "scholar_label": item.scholar_label,
+        "status": item.status,
+        "reason": item.reason,
+        "dropped_reason": item.dropped_reason,
+        "attempt_count": int(item.attempt_count),
+        "resume_cstart": int(item.resume_cstart),
+        "next_attempt_dt": item.next_attempt_dt,
+        "updated_at": item.updated_at,
+        "last_error": item.last_error,
+        "last_run_id": item.last_run_id,
+    }
+
+
+def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]:
+    if not isinstance(value, list):
+        return []
+    normalized: list[dict[str, Any]] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        normalized.append(
+            {
+                "attempt": _int_value(item.get("attempt"), 0),
+                "cstart": _int_value(item.get("cstart"), 0),
+                "state": _str_value(item.get("state")),
+                "state_reason": _str_value(item.get("state_reason")),
+                "status_code": (_int_value(item.get("status_code")) if item.get("status_code") is not None else None),
+                "fetch_error": _str_value(item.get("fetch_error")),
+            }
+        )
+    return normalized
+
+
+def _normalize_page_logs(value: Any) -> list[dict[str, Any]]:
+    if not isinstance(value, list):
+        return []
+    normalized: list[dict[str, Any]] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        warning_codes = item.get("warning_codes")
+        normalized.append(
+            {
+                "page": _int_value(item.get("page"), 0),
+                "cstart": _int_value(item.get("cstart"), 0),
+                "state": _str_value(item.get("state")) or "unknown",
+                "state_reason": _str_value(item.get("state_reason")),
+                "status_code": (_int_value(item.get("status_code")) if item.get("status_code") is not None else None),
+                "publication_count": _int_value(item.get("publication_count"), 0),
+                "attempt_count": _int_value(item.get("attempt_count"), 0),
+                "has_show_more_button": _bool_value(item.get("has_show_more_button"), False),
+                "articles_range": _str_value(item.get("articles_range")),
+                "warning_codes": [str(code) for code in (warning_codes if isinstance(warning_codes, list) else [])],
+            }
+        )
+    return normalized
+
+
+def _normalize_debug(value: Any) -> dict[str, Any] | None:
+    if not isinstance(value, dict):
+        return None
+    marker_counts = value.get("marker_counts_nonzero")
+    warning_codes = value.get("warning_codes")
+    return {
+        "status_code": (_int_value(value.get("status_code")) if value.get("status_code") is not None else None),
+        "final_url": _str_value(value.get("final_url")),
+        "fetch_error": _str_value(value.get("fetch_error")),
+        "body_sha256": _str_value(value.get("body_sha256")),
+        "body_length": (_int_value(value.get("body_length")) if value.get("body_length") is not None else None),
+        "has_show_more_button": (
+            _bool_value(value.get("has_show_more_button"), False)
+            if value.get("has_show_more_button") is not None
+            else None
+        ),
+        "articles_range": _str_value(value.get("articles_range")),
+        "state_reason": _str_value(value.get("state_reason")),
+        "warning_codes": [str(code) for code in (warning_codes if isinstance(warning_codes, list) else [])],
+        "marker_counts_nonzero": {
+            str(key): _int_value(count, 0)
+            for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else [])
+        },
+        "page_logs": _normalize_page_logs(value.get("page_logs")),
+        "attempt_log": _normalize_attempt_log(value.get("attempt_log")),
+    }
+
+
+def normalize_scholar_result(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {
+            "scholar_profile_id": 0,
+            "scholar_id": "unknown",
+            "state": "unknown",
+            "state_reason": None,
+            "outcome": "failed",
+            "attempt_count": 0,
+            "publication_count": 0,
+            "start_cstart": 0,
+            "continuation_cstart": None,
+            "continuation_enqueued": False,
+            "continuation_cleared": False,
+            "warnings": [],
+            "error": None,
+            "debug": None,
+        }
+    warnings = value.get("warnings")
+    return {
+        "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0),
+        "scholar_id": _str_value(value.get("scholar_id")) or "unknown",
+        "state": _str_value(value.get("state")) or "unknown",
+        "state_reason": _str_value(value.get("state_reason")),
+        "outcome": _str_value(value.get("outcome")) or "failed",
+        "attempt_count": _int_value(value.get("attempt_count"), 0),
+        "publication_count": _int_value(value.get("publication_count"), 0),
+        "start_cstart": _int_value(value.get("start_cstart"), 0),
+        "continuation_cstart": (
+            _int_value(value.get("continuation_cstart")) if value.get("continuation_cstart") is not None else None
+        ),
+        "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False),
+        "continuation_cleared": _bool_value(value.get("continuation_cleared"), False),
+        "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])],
+        "error": _str_value(value.get("error")),
+        "debug": _normalize_debug(value.get("debug")),
+    }
+
+
+def manual_run_payload_from_run(
+    run,
+    *,
+    idempotency_key: str | None,
+    reused_existing_run: bool,
+    safety_state: dict[str, Any],
+) -> dict[str, Any]:
+    summary = run_service.extract_run_summary(run.error_log)
+    return {
+        "run_id": int(run.id),
+        "status": run.status.value,
+        "scholar_count": int(run.scholar_count or 0),
+        "succeeded_count": int(summary["succeeded_count"]),
+        "failed_count": int(summary["failed_count"]),
+        "partial_count": int(summary["partial_count"]),
+        "new_publication_count": int(run.new_pub_count or 0),
+        "reused_existing_run": reused_existing_run,
+        "idempotency_key": idempotency_key,
+        "safety_state": safety_state,
+    }
+
+
+def manual_run_success_payload(
+    *,
+    run_summary,
+    idempotency_key: str | None,
+    safety_state: dict[str, Any],
+) -> dict[str, Any]:
+    return {
+        "run_id": run_summary.crawl_run_id,
+        "status": run_summary.status.value,
+        "scholar_count": run_summary.scholar_count,
+        "succeeded_count": run_summary.succeeded_count,
+        "failed_count": run_summary.failed_count,
+        "partial_count": run_summary.partial_count,
+        "new_publication_count": run_summary.new_publication_count,
+        "reused_existing_run": False,
+        "idempotency_key": idempotency_key,
+        "safety_state": safety_state,
+    }

app/api/routers/runs.py

@@ -1,16 +1,33 @@
 from __future__ import annotations
 
+import asyncio
 import logging
-import re
 from typing import Any
 
 from fastapi import APIRouter, Depends, Query, Request
+from fastapi.responses import StreamingResponse
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.api.deps import get_api_current_user
 from app.api.errors import ApiException
 from app.api.responses import success_payload
+from app.api.routers.run_manual import (
+    load_safety_state,
+    raise_manual_blocked_safety,
+    raise_manual_failed,
+    raise_manual_runs_disabled,
+    recover_integrity_error,
+    reused_manual_run_payload,
+)
+from app.api.routers.run_serializers import (
+    IDEMPOTENCY_HEADER,
+    normalize_idempotency_key,
+    normalize_scholar_result,
+    serialize_queue_item,
+    serialize_run,
+)
+from app.api.runtime_deps import get_ingestion_service
 from app.api.schemas import (
     ManualRunEnvelope,
     QueueClearEnvelope,
@@ -20,500 +37,27 @@ from app.api.schemas import (
     RunsListEnvelope,
 )
 from app.db.models import RunStatus, RunTriggerType, User
-from app.db.session import get_db_session
+from app.db.session import get_db_session, get_session_factory
-from app.services.domains.ingestion import application as ingestion_service
+from app.logging_utils import structured_log
-from app.services.domains.ingestion import safety as run_safety_service
+from app.services.ingestion import application as ingestion_service
-from app.services.domains.runs import application as run_service
+from app.services.runs import application as run_service
-from app.services.domains.settings import application as user_settings_service
+from app.services.runs.events import event_generator
+from app.services.settings import application as user_settings_service
 from app.settings import settings
-from app.api.runtime_deps import get_ingestion_service
 
 logger = logging.getLogger(__name__)
+_background_tasks: set[asyncio.Task[Any]] = set()
+
+
+def _drop_finished_task(task: asyncio.Task[Any]) -> None:
+    _background_tasks.discard(task)
+    try:
+        task.result()
+    except Exception:
+        structured_log(logger, "exception", "runs.background_task_failed")
 
 router = APIRouter(prefix="/runs", tags=["api-runs"])
-
+ACTIVE_RUN_STATUSES = {RunStatus.RUNNING, RunStatus.RESOLVING}
-IDEMPOTENCY_HEADER = "Idempotency-Key"
-IDEMPOTENCY_MAX_LENGTH = 128
-IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
-
-
-def _int_value(value: Any, default: int = 0) -> int:
-    try:
-        return int(value)
-    except (TypeError, ValueError):
-        return default
-
-
-def _str_value(value: Any) -> str | None:
-    if value is None:
-        return None
-    text = str(value).strip()
-    return text if text else None
-
-
-def _bool_value(value: Any, default: bool = False) -> bool:
-    if isinstance(value, bool):
-        return value
-    if isinstance(value, str):
-        lowered = value.strip().lower()
-        if lowered in {"1", "true", "yes", "on"}:
-            return True
-        if lowered in {"0", "false", "no", "off"}:
-            return False
-    return default
-
-
-def _normalize_idempotency_key(raw_value: str | None) -> str | None:
-    if raw_value is None:
-        return None
-    candidate = raw_value.strip()
-    if not candidate:
-        return None
-    if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate):
-        raise ApiException(
-            status_code=400,
-            code="invalid_idempotency_key",
-            message=(
-                "Invalid Idempotency-Key. Use 8-128 characters from: "
-                "A-Z a-z 0-9 . _ : -"
-            ),
-        )
-    return candidate
-
-
-def _serialize_run(run) -> dict[str, Any]:
-    summary = run_service.extract_run_summary(run.error_log)
-    return {
-        "id": int(run.id),
-        "trigger_type": run.trigger_type.value,
-        "status": run.status.value,
-        "start_dt": run.start_dt,
-        "end_dt": run.end_dt,
-        "scholar_count": int(run.scholar_count or 0),
-        "new_publication_count": int(run.new_pub_count or 0),
-        "failed_count": int(summary["failed_count"]),
-        "partial_count": int(summary["partial_count"]),
-    }
-
-
-def _serialize_queue_item(item) -> dict[str, Any]:
-    return {
-        "id": int(item.id),
-        "scholar_profile_id": int(item.scholar_profile_id),
-        "scholar_label": item.scholar_label,
-        "status": item.status,
-        "reason": item.reason,
-        "dropped_reason": item.dropped_reason,
-        "attempt_count": int(item.attempt_count),
-        "resume_cstart": int(item.resume_cstart),
-        "next_attempt_dt": item.next_attempt_dt,
-        "updated_at": item.updated_at,
-        "last_error": item.last_error,
-        "last_run_id": item.last_run_id,
-    }
-
-
-def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]:
-    if not isinstance(value, list):
-        return []
-    normalized: list[dict[str, Any]] = []
-    for item in value:
-        if not isinstance(item, dict):
-            continue
-        normalized.append(
-            {
-                "attempt": _int_value(item.get("attempt"), 0),
-                "cstart": _int_value(item.get("cstart"), 0),
-                "state": _str_value(item.get("state")),
-                "state_reason": _str_value(item.get("state_reason")),
-                "status_code": (
-                    _int_value(item.get("status_code"))
-                    if item.get("status_code") is not None
-                    else None
-                ),
-                "fetch_error": _str_value(item.get("fetch_error")),
-            }
-        )
-    return normalized
-
-
-def _normalize_page_logs(value: Any) -> list[dict[str, Any]]:
-    if not isinstance(value, list):
-        return []
-    normalized: list[dict[str, Any]] = []
-    for item in value:
-        if not isinstance(item, dict):
-            continue
-        warning_codes = item.get("warning_codes")
-        normalized.append(
-            {
-                "page": _int_value(item.get("page"), 0),
-                "cstart": _int_value(item.get("cstart"), 0),
-                "state": _str_value(item.get("state")) or "unknown",
-                "state_reason": _str_value(item.get("state_reason")),
-                "status_code": (
-                    _int_value(item.get("status_code"))
-                    if item.get("status_code") is not None
-                    else None
-                ),
-                "publication_count": _int_value(item.get("publication_count"), 0),
-                "attempt_count": _int_value(item.get("attempt_count"), 0),
-                "has_show_more_button": _bool_value(item.get("has_show_more_button"), False),
-                "articles_range": _str_value(item.get("articles_range")),
-                "warning_codes": [
-                    str(code)
-                    for code in (warning_codes if isinstance(warning_codes, list) else [])
-                ],
-            }
-        )
-    return normalized
-
-
-def _normalize_debug(value: Any) -> dict[str, Any] | None:
-    if not isinstance(value, dict):
-        return None
-    marker_counts = value.get("marker_counts_nonzero")
-    warning_codes = value.get("warning_codes")
-    return {
-        "status_code": (
-            _int_value(value.get("status_code"))
-            if value.get("status_code") is not None
-            else None
-        ),
-        "final_url": _str_value(value.get("final_url")),
-        "fetch_error": _str_value(value.get("fetch_error")),
-        "body_sha256": _str_value(value.get("body_sha256")),
-        "body_length": (
-            _int_value(value.get("body_length"))
-            if value.get("body_length") is not None
-            else None
-        ),
-        "has_show_more_button": (
-            _bool_value(value.get("has_show_more_button"), False)
-            if value.get("has_show_more_button") is not None
-            else None
-        ),
-        "articles_range": _str_value(value.get("articles_range")),
-        "state_reason": _str_value(value.get("state_reason")),
-        "warning_codes": [
-            str(code)
-            for code in (warning_codes if isinstance(warning_codes, list) else [])
-        ],
-        "marker_counts_nonzero": {
-            str(key): _int_value(count, 0)
-            for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else [])
-        },
-        "page_logs": _normalize_page_logs(value.get("page_logs")),
-        "attempt_log": _normalize_attempt_log(value.get("attempt_log")),
-    }
-
-
-def _normalize_scholar_result(value: Any) -> dict[str, Any]:
-    if not isinstance(value, dict):
-        return {
-            "scholar_profile_id": 0,
-            "scholar_id": "unknown",
-            "state": "unknown",
-            "state_reason": None,
-            "outcome": "failed",
-            "attempt_count": 0,
-            "publication_count": 0,
-            "start_cstart": 0,
-            "continuation_cstart": None,
-            "continuation_enqueued": False,
-            "continuation_cleared": False,
-            "warnings": [],
-            "error": None,
-            "debug": None,
-        }
-    warnings = value.get("warnings")
-    return {
-        "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0),
-        "scholar_id": _str_value(value.get("scholar_id")) or "unknown",
-        "state": _str_value(value.get("state")) or "unknown",
-        "state_reason": _str_value(value.get("state_reason")),
-        "outcome": _str_value(value.get("outcome")) or "failed",
-        "attempt_count": _int_value(value.get("attempt_count"), 0),
-        "publication_count": _int_value(value.get("publication_count"), 0),
-        "start_cstart": _int_value(value.get("start_cstart"), 0),
-        "continuation_cstart": (
-            _int_value(value.get("continuation_cstart"))
-            if value.get("continuation_cstart") is not None
-            else None
-        ),
-        "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False),
-        "continuation_cleared": _bool_value(value.get("continuation_cleared"), False),
-        "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])],
-        "error": _str_value(value.get("error")),
-        "debug": _normalize_debug(value.get("debug")),
-    }
-
-
-def _manual_run_payload_from_run(
-    run,
-    *,
-    idempotency_key: str | None,
-    reused_existing_run: bool,
-    safety_state: dict[str, Any],
-) -> dict[str, Any]:
-    summary = run_service.extract_run_summary(run.error_log)
-    return {
-        "run_id": int(run.id),
-        "status": run.status.value,
-        "scholar_count": int(run.scholar_count or 0),
-        "succeeded_count": int(summary["succeeded_count"]),
-        "failed_count": int(summary["failed_count"]),
-        "partial_count": int(summary["partial_count"]),
-        "new_publication_count": int(run.new_pub_count or 0),
-        "reused_existing_run": reused_existing_run,
-        "idempotency_key": idempotency_key,
-        "safety_state": safety_state,
-    }
-
-
-async def _load_safety_state(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-) -> dict[str, Any]:
-    user_settings = await user_settings_service.get_or_create_settings(
-        db_session,
-        user_id=user_id,
-    )
-    previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
-    if run_safety_service.clear_expired_cooldown(user_settings):
-        await db_session.commit()
-        await db_session.refresh(user_settings)
-        logger.info(
-            "api.runs.safety_cooldown_cleared",
-            extra={
-                "event": "api.runs.safety_cooldown_cleared",
-                "user_id": user_id,
-                "reason": previous_safety_state.get("cooldown_reason"),
-                "cooldown_until": previous_safety_state.get("cooldown_until"),
-                "metric_name": "api_runs_safety_cooldown_cleared_total",
-                "metric_value": 1,
-            },
-    )
-    return run_safety_service.get_safety_state_payload(user_settings)
-
-
-def _raise_manual_runs_disabled(*, user_id: int, safety_state: dict[str, Any]) -> None:
-    logger.warning(
-        "api.runs.manual_blocked_policy",
-        extra={
-            "event": "api.runs.manual_blocked_policy",
-            "user_id": user_id,
-            "policy": {"manual_run_allowed": False},
-            "safety_state": safety_state,
-            "metric_name": "api_runs_manual_blocked_policy_total",
-            "metric_value": 1,
-        },
-    )
-    raise ApiException(
-        status_code=403,
-        code="manual_runs_disabled",
-        message="Manual checks are disabled by server policy.",
-        details={
-            "policy": {"manual_run_allowed": False},
-            "safety_state": safety_state,
-        },
-    )
-
-
-async def _reused_manual_run_payload(
-    db_session: AsyncSession,
-    *,
-    request: Request,
-    user_id: int,
-    idempotency_key: str | None,
-    safety_state: dict[str, Any],
-) -> dict[str, Any] | None:
-    if idempotency_key is None:
-        return None
-    previous_run = await run_service.get_manual_run_by_idempotency_key(
-        db_session,
-        user_id=user_id,
-        idempotency_key=idempotency_key,
-    )
-    if previous_run is None:
-        return None
-    if previous_run.status == RunStatus.RUNNING:
-        raise ApiException(
-            status_code=409,
-            code="run_in_progress",
-            message="A run with this idempotency key is still in progress.",
-            details={"run_id": int(previous_run.id), "idempotency_key": idempotency_key},
-        )
-    return success_payload(
-        request,
-        data=_manual_run_payload_from_run(
-            previous_run,
-            idempotency_key=idempotency_key,
-            reused_existing_run=True,
-            safety_state=safety_state,
-        ),
-    )
-
-
-async def _run_ingestion_for_manual(
-    db_session: AsyncSession,
-    *,
-    ingest_service: ingestion_service.ScholarIngestionService,
-    user_id: int,
-    idempotency_key: str | None,
-):
-    user_settings = await user_settings_service.get_or_create_settings(
-        db_session,
-        user_id=user_id,
-    )
-    return await ingest_service.run_for_user(
-        db_session,
-        user_id=user_id,
-        trigger_type=RunTriggerType.MANUAL,
-        request_delay_seconds=user_settings.request_delay_seconds,
-        network_error_retries=settings.ingestion_network_error_retries,
-        retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
-        max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
-        page_size=settings.ingestion_page_size,
-        auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
-        queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
-        idempotency_key=idempotency_key,
-        alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
-        alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
-        alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
-    )
-
-
-async def _recover_integrity_error(
-    db_session: AsyncSession,
-    *,
-    request: Request,
-    user_id: int,
-    idempotency_key: str | None,
-    original_exc: IntegrityError,
-) -> dict[str, Any]:
-    if idempotency_key is None:
-        logger.exception(
-            "api.runs.manual_integrity_error",
-            extra={"event": "api.runs.manual_integrity_error", "user_id": user_id},
-        )
-        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
-    existing_run = await run_service.get_manual_run_by_idempotency_key(
-        db_session,
-        user_id=user_id,
-        idempotency_key=idempotency_key,
-    )
-    if existing_run is None:
-        logger.exception(
-            "api.runs.manual_integrity_error",
-            extra={"event": "api.runs.manual_integrity_error", "user_id": user_id},
-        )
-        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
-    if existing_run.status == RunStatus.RUNNING:
-        raise ApiException(
-            status_code=409,
-            code="run_in_progress",
-            message="A run with this idempotency key is still in progress.",
-            details={"run_id": int(existing_run.id), "idempotency_key": idempotency_key},
-        ) from original_exc
-    return success_payload(
-        request,
-        data=_manual_run_payload_from_run(
-            existing_run,
-            idempotency_key=idempotency_key,
-            reused_existing_run=True,
-            safety_state=await _load_safety_state(db_session, user_id=user_id),
-        ),
-    )
-
-
-async def _execute_manual_run(
-    db_session: AsyncSession,
-    *,
-    request: Request,
-    ingest_service: ingestion_service.ScholarIngestionService,
-    user_id: int,
-    idempotency_key: str | None,
-):
-    try:
-        return await _run_ingestion_for_manual(
-            db_session,
-            ingest_service=ingest_service,
-            user_id=user_id,
-            idempotency_key=idempotency_key,
-        )
-    except ingestion_service.RunAlreadyInProgressError as exc:
-        await db_session.rollback()
-        raise ApiException(
-            status_code=409,
-            code="run_in_progress",
-            message="A run is already in progress for this account.",
-        ) from exc
-    except ingestion_service.RunBlockedBySafetyPolicyError as exc:
-        await db_session.rollback()
-        _raise_manual_blocked_safety(exc=exc, user_id=user_id)
-    except IntegrityError as exc:
-        await db_session.rollback()
-        return await _recover_integrity_error(
-            db_session,
-            request=request,
-            user_id=user_id,
-            idempotency_key=idempotency_key,
-            original_exc=exc,
-        )
-    except Exception as exc:
-        await db_session.rollback()
-        _raise_manual_failed(exc=exc, user_id=user_id)
-
-
-def _raise_manual_blocked_safety(*, exc, user_id: int) -> None:
-    logger.info(
-        "api.runs.manual_blocked_safety",
-        extra={
-            "event": "api.runs.manual_blocked_safety",
-            "user_id": user_id,
-            "reason": exc.safety_state.get("cooldown_reason"),
-            "cooldown_until": exc.safety_state.get("cooldown_until"),
-            "cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"),
-            "metric_name": "api_runs_manual_blocked_safety_total",
-            "metric_value": 1,
-        },
-    )
-    raise ApiException(
-        status_code=429,
-        code=exc.code,
-        message=exc.message,
-        details={"safety_state": exc.safety_state},
-    ) from exc
-
-
-def _raise_manual_failed(*, exc: Exception, user_id: int) -> None:
-    logger.exception(
-        "api.runs.manual_failed",
-        extra={"event": "api.runs.manual_failed", "user_id": user_id},
-    )
-    raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from exc
-
-
-def _manual_run_success_payload(
-    *,
-    run_summary,
-    idempotency_key: str | None,
-    safety_state: dict[str, Any],
-) -> dict[str, Any]:
-    return {
-        "run_id": run_summary.crawl_run_id,
-        "status": run_summary.status.value,
-        "scholar_count": run_summary.scholar_count,
-        "succeeded_count": run_summary.succeeded_count,
-        "failed_count": run_summary.failed_count,
-        "partial_count": run_summary.partial_count,
-        "new_publication_count": run_summary.new_publication_count,
-        "reused_existing_run": False,
-        "idempotency_key": idempotency_key,
-        "safety_state": safety_state,
-    }
 
 
 @router.get(
@@ -533,14 +77,14 @@ async def list_runs(
         limit=limit,
         failed_only=failed_only,
     )
-    safety_state = await _load_safety_state(
+    safety_state = await load_safety_state(
         db_session,
         user_id=current_user.id,
     )
     return success_payload(
         request,
         data={
-            "runs": [_serialize_run(run) for run in runs],
+            "runs": [serialize_run(run) for run in runs],
             "safety_state": safety_state,
         },
     )
@@ -571,21 +115,137 @@ async def get_run(
     scholar_results = error_log.get("scholar_results")
     if not isinstance(scholar_results, list):
         scholar_results = []
-    safety_state = await _load_safety_state(
+    safety_state = await load_safety_state(
         db_session,
         user_id=current_user.id,
     )
     return success_payload(
         request,
         data={
-            "run": _serialize_run(run),
+            "run": serialize_run(run),
             "summary": run_service.extract_run_summary(error_log),
-            "scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
+            "scholar_results": [normalize_scholar_result(item) for item in scholar_results],
             "safety_state": safety_state,
         },
     )
 
 
+@router.post(
+    "/{run_id}/cancel",
+    response_model=RunDetailEnvelope,
+)
+async def cancel_run(
+    run_id: int,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    run = await run_service.get_run_for_user(
+        db_session,
+        user_id=current_user.id,
+        run_id=run_id,
+    )
+    if run is None:
+        raise ApiException(
+            status_code=404,
+            code="run_not_found",
+            message="Run not found.",
+        )
+
+    if run.status in ACTIVE_RUN_STATUSES:
+        run.status = RunStatus.CANCELED
+        await db_session.commit()
+        await db_session.refresh(run)
+    else:
+        raise ApiException(
+            status_code=409,
+            code="run_not_cancelable",
+            message="Run is already terminal and cannot be canceled.",
+            details={"run_id": int(run.id), "status": run.status.value},
+        )
+
+    error_log = run.error_log if isinstance(run.error_log, dict) else {}
+    scholar_results = error_log.get("scholar_results")
+    if not isinstance(scholar_results, list):
+        scholar_results = []
+
+    safety_state = await load_safety_state(
+        db_session,
+        user_id=current_user.id,
+    )
+
+    return success_payload(
+        request,
+        data={
+            "run": serialize_run(run),
+            "summary": run_service.extract_run_summary(error_log),
+            "scholar_results": [normalize_scholar_result(item) for item in scholar_results],
+            "safety_state": safety_state,
+        },
+    )
+
+
+async def _start_manual_run(
+    db_session: AsyncSession,
+    *,
+    current_user: User,
+    ingest_service: ingestion_service.ScholarIngestionService,
+    idempotency_key: str | None,
+) -> tuple[Any, Any, list, dict]:
+    user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=current_user.id)
+    run, scholars, start_cstart_map = await ingest_service.initialize_run(
+        db_session,
+        user_id=current_user.id,
+        trigger_type=RunTriggerType.MANUAL,
+        request_delay_seconds=user_settings.request_delay_seconds,
+        network_error_retries=settings.ingestion_network_error_retries,
+        retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+        max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
+        page_size=settings.ingestion_page_size,
+        idempotency_key=idempotency_key,
+        alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
+        alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
+        alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
+    )
+    return user_settings, run, scholars, start_cstart_map
+
+
+def _spawn_background_execution(
+    ingest_service: ingestion_service.ScholarIngestionService,
+    *,
+    run: Any,
+    current_user: User,
+    scholars: list,
+    start_cstart_map: dict,
+    user_settings: Any,
+    idempotency_key: str | None,
+) -> None:
+    from app.db.session import get_session_factory
+
+    task = asyncio.create_task(
+        ingest_service.execute_run(
+            session_factory=get_session_factory(),
+            run_id=run.id,
+            user_id=current_user.id,
+            scholars=scholars,
+            start_cstart_map=start_cstart_map,
+            request_delay_seconds=user_settings.request_delay_seconds,
+            network_error_retries=settings.ingestion_network_error_retries,
+            retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+            max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
+            page_size=settings.ingestion_page_size,
+            auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
+            queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
+            alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
+            alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
+            alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
+            idempotency_key=idempotency_key,
+        )
+    )
+    _background_tasks.add(task)
+    task.add_done_callback(_drop_finished_task)
+
+
 @router.post(
     "/manual",
     response_model=ManualRunEnvelope,
@@ -596,46 +256,76 @@ async def run_manual(
     current_user: User = Depends(get_api_current_user),
     ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
 ):
-    safety_state = await _load_safety_state(
+    user_id = int(current_user.id)
-        db_session,
+    safety_state = await load_safety_state(db_session, user_id=user_id)
-        user_id=current_user.id,
-    )
     if not settings.ingestion_manual_run_allowed:
-        _raise_manual_runs_disabled(user_id=current_user.id, safety_state=safety_state)
+        raise_manual_runs_disabled(user_id=user_id, safety_state=safety_state)
 
-    idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
+    idempotency_key = normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
-    reused_payload = await _reused_manual_run_payload(
+    reused_payload = await reused_manual_run_payload(
         db_session,
         request=request,
-        user_id=current_user.id,
+        user_id=user_id,
         idempotency_key=idempotency_key,
         safety_state=safety_state,
     )
     if reused_payload is not None:
         return reused_payload
 
-    run_summary = await _execute_manual_run(
+    try:
+        user_settings, run, scholars, start_cstart_map = await _start_manual_run(
             db_session,
-        request=request,
+            current_user=current_user,
             ingest_service=ingest_service,
-        user_id=current_user.id,
             idempotency_key=idempotency_key,
         )
-    if isinstance(run_summary, dict):
+        await db_session.commit()
-        return run_summary
+        _spawn_background_execution(
-
+            ingest_service,
-    current_safety_state = await _load_safety_state(
+            run=run,
-        db_session,
+            current_user=current_user,
-        user_id=current_user.id,
+            scholars=scholars,
+            start_cstart_map=start_cstart_map,
+            user_settings=user_settings,
+            idempotency_key=idempotency_key,
         )
         return success_payload(
             request,
-        data=_manual_run_success_payload(
+            data={
-            run_summary=run_summary,
+                "run_id": int(run.id),
-            idempotency_key=idempotency_key,
+                "status": run.status.value,
-            safety_state=current_safety_state,
+                "scholar_count": int(run.scholar_count or 0),
-        ),
+                "succeeded_count": 0,
+                "failed_count": 0,
+                "partial_count": 0,
+                "new_publication_count": 0,
+                "reused_existing_run": False,
+                "idempotency_key": idempotency_key,
+                "safety_state": await load_safety_state(db_session, user_id=user_id),
+            },
         )
+    except ingestion_service.RunBlockedBySafetyPolicyError as exc:
+        await db_session.rollback()
+        raise_manual_blocked_safety(exc=exc, user_id=user_id)
+    except ingestion_service.RunAlreadyInProgressError as exc:
+        await db_session.rollback()
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run is already in progress for this account.",
+        ) from exc
+    except IntegrityError as exc:
+        await db_session.rollback()
+        return await recover_integrity_error(
+            db_session,
+            request=request,
+            user_id=user_id,
+            idempotency_key=idempotency_key,
+            original_exc=exc,
+        )
+    except Exception as exc:
+        await db_session.rollback()
+        raise_manual_failed(exc=exc, user_id=user_id)
 
 
 @router.get(
@@ -656,7 +346,7 @@ async def list_queue_items(
     return success_payload(
         request,
         data={
-            "queue_items": [_serialize_queue_item(item) for item in items],
+            "queue_items": [serialize_queue_item(item) for item in items],
         },
     )
 
@@ -692,7 +382,7 @@ async def retry_queue_item(
         )
     return success_payload(
         request,
-        data=_serialize_queue_item(queue_item),
+        data=serialize_queue_item(queue_item),
     )
 
 
@@ -727,7 +417,7 @@ async def drop_queue_item(
         )
     return success_payload(
         request,
-        data=_serialize_queue_item(dropped),
+        data=serialize_queue_item(dropped),
     )
 
 
@@ -769,3 +459,24 @@ async def clear_queue_item(
             "message": "Queue item cleared.",
         },
     )
+
+
+@router.get("/{run_id}/stream")
+async def stream_run_events(
+    run_id: int,
+    current_user: User = Depends(get_api_current_user),
+):
+    session_factory = get_session_factory()
+    async with session_factory() as db_session:
+        run = await run_service.get_run_for_user(
+            db_session,
+            user_id=current_user.id,
+            run_id=run_id,
+        )
+    if run is None:
+        raise ApiException(
+            status_code=404,
+            code="run_not_found",
+            message="Run not found.",
+        )
+    return StreamingResponse(event_generator(run_id), media_type="text/event-stream")

app/api/routers/scholar_helpers.py

@@ -0,0 +1,234 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+from fastapi import UploadFile
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import ApiException
+from app.logging_utils import structured_log
+from app.services.ingestion import queue as ingestion_queue_service
+from app.services.scholar import rate_limit as scholar_rate_limit
+from app.services.scholar.source import ScholarSource
+from app.services.scholars import application as scholar_service
+from app.services.scholars import search_hints as scholar_search_hints
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+CREATE_METADATA_HYDRATION_TIMEOUT_SECONDS = 5.0
+INITIAL_SCHOLAR_SCRAPE_QUEUE_DELAY_SECONDS = 0
+INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON = "scholar_added_initial_scrape"
+
+
+def needs_metadata_hydration(profile) -> bool:
+    if not profile.profile_image_url:
+        return True
+    return not (profile.display_name or "").strip()
+
+
+def is_create_hydration_rate_limited() -> tuple[bool, float]:
+    remaining_seconds = scholar_rate_limit.remaining_scholar_slot_seconds(
+        min_interval_seconds=float(settings.ingestion_min_request_delay_seconds),
+    )
+    return remaining_seconds > 0, remaining_seconds
+
+
+def auto_enqueue_new_scholar_enabled() -> bool:
+    if not settings.scheduler_enabled:
+        return False
+    if not settings.ingestion_automation_allowed:
+        return False
+    return bool(settings.ingestion_continuation_queue_enabled)
+
+
+async def enqueue_initial_scrape_job_for_scholar(
+    db_session: AsyncSession,
+    *,
+    profile,
+    user_id: int,
+) -> bool:
+    if not auto_enqueue_new_scholar_enabled():
+        return False
+    try:
+        await ingestion_queue_service.upsert_job(
+            db_session,
+            user_id=user_id,
+            scholar_profile_id=int(profile.id),
+            resume_cstart=0,
+            reason=INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON,
+            run_id=None,
+            delay_seconds=INITIAL_SCHOLAR_SCRAPE_QUEUE_DELAY_SECONDS,
+        )
+        await db_session.commit()
+    except Exception:
+        await db_session.rollback()
+        structured_log(
+            logger,
+            "warning",
+            "api.scholars.initial_scrape_enqueue_failed",
+            user_id=user_id,
+            scholar_profile_id=profile.id,
+        )
+        return False
+
+    structured_log(
+        logger,
+        "info",
+        "api.scholars.initial_scrape_enqueued",
+        user_id=user_id,
+        scholar_profile_id=profile.id,
+        reason=INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON,
+    )
+    return True
+
+
+def uploaded_image_media_path(scholar_profile_id: int) -> str:
+    return f"/scholar-images/{scholar_profile_id}/upload"
+
+
+def serialize_scholar(profile) -> dict[str, object]:
+    uploaded_image_url = None
+    if profile.profile_image_upload_path:
+        uploaded_image_url = uploaded_image_media_path(int(profile.id))
+
+    profile_image_url, profile_image_source = scholar_search_hints.resolve_profile_image(
+        profile,
+        uploaded_image_url=uploaded_image_url,
+    )
+
+    return {
+        "id": int(profile.id),
+        "scholar_id": profile.scholar_id,
+        "display_name": profile.display_name,
+        "profile_image_url": profile_image_url,
+        "profile_image_source": profile_image_source,
+        "is_enabled": bool(profile.is_enabled),
+        "baseline_completed": bool(profile.baseline_completed),
+        "last_run_dt": profile.last_run_dt,
+        "last_run_status": (profile.last_run_status.value if profile.last_run_status is not None else None),
+    }
+
+
+async def hydrate_scholar_metadata_if_needed(
+    db_session: AsyncSession,
+    *,
+    profile,
+    source: ScholarSource,
+    user_id: int,
+):
+    if not needs_metadata_hydration(profile):
+        return profile
+
+    should_skip, remaining_seconds = is_create_hydration_rate_limited()
+    if should_skip:
+        structured_log(
+            logger,
+            "info",
+            "api.scholars.create_metadata_hydration_skipped",
+            reason="scholar_request_throttle_active",
+            user_id=user_id,
+            scholar_profile_id=profile.id,
+            retry_after_seconds=round(remaining_seconds, 3),
+        )
+        return profile
+
+    try:
+        return await asyncio.wait_for(
+            scholar_service.hydrate_profile_metadata(
+                db_session,
+                profile=profile,
+                source=source,
+            ),
+            timeout=CREATE_METADATA_HYDRATION_TIMEOUT_SECONDS,
+        )
+    except TimeoutError:
+        structured_log(
+            logger,
+            "info",
+            "api.scholars.create_metadata_hydration_skipped",
+            reason="create_timeout",
+            user_id=user_id,
+            scholar_profile_id=profile.id,
+        )
+    except Exception:
+        structured_log(
+            logger,
+            "warning",
+            "api.scholars.create_metadata_hydration_failed",
+            user_id=user_id,
+            scholar_profile_id=profile.id,
+        )
+    return profile
+
+
+def search_kwargs() -> dict[str, Any]:
+    return {
+        "network_error_retries": settings.ingestion_network_error_retries,
+        "retry_backoff_seconds": settings.ingestion_retry_backoff_seconds,
+        "search_enabled": settings.scholar_name_search_enabled,
+        "cache_ttl_seconds": settings.scholar_name_search_cache_ttl_seconds,
+        "blocked_cache_ttl_seconds": settings.scholar_name_search_blocked_cache_ttl_seconds,
+        "cache_max_entries": settings.scholar_name_search_cache_max_entries,
+        "min_interval_seconds": settings.scholar_name_search_min_interval_seconds,
+        "interval_jitter_seconds": settings.scholar_name_search_interval_jitter_seconds,
+        "cooldown_block_threshold": settings.scholar_name_search_cooldown_block_threshold,
+        "cooldown_seconds": settings.scholar_name_search_cooldown_seconds,
+        "retry_alert_threshold": settings.scholar_name_search_alert_retry_count_threshold,
+        "cooldown_rejection_alert_threshold": (settings.scholar_name_search_alert_cooldown_rejections_threshold),
+    }
+
+
+def search_response_data(query: str, parsed) -> dict[str, object]:
+    return {
+        "query": query.strip(),
+        "state": parsed.state.value,
+        "state_reason": parsed.state_reason,
+        "action_hint": scholar_search_hints.scrape_state_hint(
+            state=parsed.state,
+            state_reason=parsed.state_reason,
+        ),
+        "candidates": [
+            {
+                "scholar_id": item.scholar_id,
+                "display_name": item.display_name,
+                "affiliation": item.affiliation,
+                "email_domain": item.email_domain,
+                "cited_by_count": item.cited_by_count,
+                "interests": item.interests,
+                "profile_url": item.profile_url,
+                "profile_image_url": item.profile_image_url,
+            }
+            for item in parsed.candidates
+        ],
+        "warnings": parsed.warnings,
+    }
+
+
+async def read_uploaded_image(image: UploadFile) -> bytes:
+    try:
+        return await image.read()
+    finally:
+        await image.close()
+
+
+async def require_user_profile(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+):
+    profile = await scholar_service.get_user_scholar_by_id(
+        db_session,
+        user_id=user_id,
+        scholar_profile_id=scholar_profile_id,
+    )
+    if profile is None:
+        raise ApiException(
+            status_code=404,
+            code="scholar_not_found",
+            message="Scholar not found.",
+        )
+    return profile

app/api/routers/scholars.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import asyncio
 import logging
 
 from fastapi import APIRouter, Depends, File, Query, Request, UploadFile
@@ -9,6 +8,15 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from app.api.deps import get_api_current_user
 from app.api.errors import ApiException
 from app.api.responses import success_payload
+from app.api.routers.scholar_helpers import (
+    enqueue_initial_scrape_job_for_scholar,
+    hydrate_scholar_metadata_if_needed,
+    read_uploaded_image,
+    require_user_profile,
+    search_kwargs,
+    search_response_data,
+    serialize_scholar,
+)
 from app.api.runtime_deps import get_scholar_source
 from app.api.schemas import (
     DataExportEnvelope,
@@ -23,9 +31,10 @@ from app.api.schemas import (
 )
 from app.db.models import User
 from app.db.session import get_db_session
-from app.services.domains.portability import application as import_export_service
+from app.logging_utils import structured_log
-from app.services.domains.scholars import application as scholar_service
+from app.services.portability import application as import_export_service
-from app.services.domains.scholar.source import ScholarSource
+from app.services.scholar.source import ScholarSource
+from app.services.scholars import application as scholar_service
 from app.settings import settings
 
 logger = logging.getLogger(__name__)
@@ -33,136 +42,6 @@ logger = logging.getLogger(__name__)
 router = APIRouter(prefix="/scholars", tags=["api-scholars"])
 
 
-def _uploaded_image_media_path(scholar_profile_id: int) -> str:
-    return f"/scholar-images/{scholar_profile_id}/upload"
-
-
-def _serialize_scholar(profile) -> dict[str, object]:
-    uploaded_image_url = None
-    if profile.profile_image_upload_path:
-        uploaded_image_url = _uploaded_image_media_path(int(profile.id))
-
-    profile_image_url, profile_image_source = scholar_service.resolve_profile_image(
-        profile,
-        uploaded_image_url=uploaded_image_url,
-    )
-
-    return {
-        "id": int(profile.id),
-        "scholar_id": profile.scholar_id,
-        "display_name": profile.display_name,
-        "profile_image_url": profile_image_url,
-        "profile_image_source": profile_image_source,
-        "is_enabled": bool(profile.is_enabled),
-        "baseline_completed": bool(profile.baseline_completed),
-        "last_run_dt": profile.last_run_dt,
-        "last_run_status": (
-            profile.last_run_status.value if profile.last_run_status is not None else None
-        ),
-    }
-
-
-async def _hydrate_scholar_metadata_if_needed(
-    db_session: AsyncSession,
-    *,
-    profile,
-    source: ScholarSource,
-    user_id: int,
-):
-    try:
-        if not profile.profile_image_url or not (profile.display_name or "").strip():
-            return await asyncio.wait_for(
-                scholar_service.hydrate_profile_metadata(
-                    db_session,
-                    profile=profile,
-                    source=source,
-                ),
-                timeout=5.0,
-            )
-    except Exception:
-        logger.warning(
-            "api.scholars.create_metadata_hydration_failed",
-            extra={
-                "event": "api.scholars.create_metadata_hydration_failed",
-                "user_id": user_id,
-                "scholar_profile_id": profile.id,
-            },
-        )
-    return profile
-
-
-def _search_kwargs() -> dict[str, object]:
-    return {
-        "network_error_retries": settings.ingestion_network_error_retries,
-        "retry_backoff_seconds": settings.ingestion_retry_backoff_seconds,
-        "search_enabled": settings.scholar_name_search_enabled,
-        "cache_ttl_seconds": settings.scholar_name_search_cache_ttl_seconds,
-        "blocked_cache_ttl_seconds": settings.scholar_name_search_blocked_cache_ttl_seconds,
-        "cache_max_entries": settings.scholar_name_search_cache_max_entries,
-        "min_interval_seconds": settings.scholar_name_search_min_interval_seconds,
-        "interval_jitter_seconds": settings.scholar_name_search_interval_jitter_seconds,
-        "cooldown_block_threshold": settings.scholar_name_search_cooldown_block_threshold,
-        "cooldown_seconds": settings.scholar_name_search_cooldown_seconds,
-        "retry_alert_threshold": settings.scholar_name_search_alert_retry_count_threshold,
-        "cooldown_rejection_alert_threshold": (
-            settings.scholar_name_search_alert_cooldown_rejections_threshold
-        ),
-    }
-
-
-def _search_response_data(query: str, parsed) -> dict[str, object]:
-    return {
-        "query": query.strip(),
-        "state": parsed.state.value,
-        "state_reason": parsed.state_reason,
-        "action_hint": scholar_service.scrape_state_hint(
-            state=parsed.state,
-            state_reason=parsed.state_reason,
-        ),
-        "candidates": [
-            {
-                "scholar_id": item.scholar_id,
-                "display_name": item.display_name,
-                "affiliation": item.affiliation,
-                "email_domain": item.email_domain,
-                "cited_by_count": item.cited_by_count,
-                "interests": item.interests,
-                "profile_url": item.profile_url,
-                "profile_image_url": item.profile_image_url,
-            }
-            for item in parsed.candidates
-        ],
-        "warnings": parsed.warnings,
-    }
-
-
-async def _read_uploaded_image(image: UploadFile) -> bytes:
-    try:
-        return await image.read()
-    finally:
-        await image.close()
-
-
-async def _require_user_profile(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholar_profile_id: int,
-):
-    profile = await scholar_service.get_user_scholar_by_id(
-        db_session,
-        user_id=user_id,
-        scholar_profile_id=scholar_profile_id,
-    )
-    if profile is None:
-        raise ApiException(
-            status_code=404,
-            code="scholar_not_found",
-            message="Scholar not found.",
-        )
-    return profile
-
-
 @router.get(
     "",
     response_model=ScholarsListEnvelope,
@@ -179,7 +58,7 @@ async def list_scholars(
     return success_payload(
         request,
         data={
-            "scholars": [_serialize_scholar(profile) for profile in scholars],
+            "scholars": [serialize_scholar(profile) for profile in scholars],
         },
     )
 
@@ -218,8 +97,7 @@ async def import_scholars_and_publications(
             status_code=400,
             code="invalid_import_schema_version",
             message=(
-                "Import schema version is not supported. "
+                f"Import schema version is not supported. Expected {import_export_service.EXPORT_SCHEMA_VERSION}."
-                f"Expected {import_export_service.EXPORT_SCHEMA_VERSION}."
             ),
         )
     try:
@@ -235,14 +113,7 @@ async def import_scholars_and_publications(
             code="invalid_import_payload",
             message=str(exc),
         ) from exc
-    logger.info(
+    structured_log(logger, "info", "api.scholars.imported", user_id=current_user.id, **result)
-        "api.scholars.imported",
-        extra={
-            "event": "api.scholars.imported",
-            "user_id": current_user.id,
-            **result,
-        },
-    )
     return success_payload(request, data=result)
 
 
@@ -272,15 +143,14 @@ async def create_scholar(
             code="invalid_scholar",
             message=str(exc),
         ) from exc
-    logger.info(
+    structured_log(logger, "info", "api.scholars.created", user_id=current_user.id, scholar_profile_id=created.id)
-        "api.scholars.created",
+    did_queue_initial_scrape = await enqueue_initial_scrape_job_for_scholar(
-        extra={
+        db_session,
-            "event": "api.scholars.created",
+        profile=created,
-            "user_id": current_user.id,
+        user_id=current_user.id,
-            "scholar_profile_id": created.id,
-        },
     )
-    created = await _hydrate_scholar_metadata_if_needed(
+    if not did_queue_initial_scrape:
+        created = await hydrate_scholar_metadata_if_needed(
             db_session,
             profile=created,
             source=source,
@@ -289,7 +159,7 @@ async def create_scholar(
 
     return success_payload(
         request,
-        data=_serialize_scholar(created),
+        data=serialize_scholar(created),
     )
 
 
@@ -311,7 +181,7 @@ async def search_scholars(
             db_session=db_session,
             query=query,
             limit=limit,
-            **_search_kwargs(),
+            **search_kwargs(),
         )
     except scholar_service.ScholarServiceError as exc:
         raise ApiException(
@@ -320,19 +190,18 @@ async def search_scholars(
             message=str(exc),
         ) from exc
 
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.scholars.search.completed",
-        extra={
+        user_id=current_user.id,
-            "event": "api.scholars.search.completed",
+        query=query.strip(),
-            "user_id": current_user.id,
+        candidate_count=len(parsed.candidates),
-            "query": query.strip(),
+        state=parsed.state.value,
-            "candidate_count": len(parsed.candidates),
-            "state": parsed.state.value,
-        },
     )
     return success_payload(
         request,
-        data=_search_response_data(query, parsed),
+        data=search_response_data(query, parsed),
     )
 
 
@@ -358,18 +227,17 @@ async def toggle_scholar(
             message="Scholar not found.",
         )
     updated = await scholar_service.toggle_scholar_enabled(db_session, profile=profile)
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.scholars.toggled",
-        extra={
+        user_id=current_user.id,
-            "event": "api.scholars.toggled",
+        scholar_profile_id=updated.id,
-            "user_id": current_user.id,
+        is_enabled=updated.is_enabled,
-            "scholar_profile_id": updated.id,
-            "is_enabled": updated.is_enabled,
-        },
     )
     return success_payload(
         request,
-        data=_serialize_scholar(updated),
+        data=serialize_scholar(updated),
     )
 
 
@@ -399,13 +267,8 @@ async def delete_scholar(
         profile=profile,
         upload_dir=settings.scholar_image_upload_dir,
     )
-    logger.info(
+    structured_log(
-        "api.scholars.deleted",
+        logger, "info", "api.scholars.deleted", user_id=current_user.id, scholar_profile_id=scholar_profile_id
-        extra={
-            "event": "api.scholars.deleted",
-            "user_id": current_user.id,
-            "scholar_profile_id": scholar_profile_id,
-        },
     )
     return success_payload(
         request,
@@ -450,17 +313,12 @@ async def update_scholar_image_url(
             message=str(exc),
         ) from exc
 
-    logger.info(
+    structured_log(
-        "api.scholars.image_url_updated",
+        logger, "info", "api.scholars.image_url_updated", user_id=current_user.id, scholar_profile_id=updated.id
-        extra={
-            "event": "api.scholars.image_url_updated",
-            "user_id": current_user.id,
-            "scholar_profile_id": updated.id,
-        },
     )
     return success_payload(
         request,
-        data=_serialize_scholar(updated),
+        data=serialize_scholar(updated),
     )
 
 
@@ -475,13 +333,13 @@ async def upload_scholar_image(
     db_session: AsyncSession = Depends(get_db_session),
     current_user: User = Depends(get_api_current_user),
 ):
-    profile = await _require_user_profile(
+    profile = await require_user_profile(
         db_session,
         user_id=current_user.id,
         scholar_profile_id=scholar_profile_id,
     )
 
-    image_bytes = await _read_uploaded_image(image)
+    image_bytes = await read_uploaded_image(image)
     try:
         updated = await scholar_service.set_profile_image_upload(
             db_session,
@@ -499,19 +357,18 @@ async def upload_scholar_image(
         ) from exc
 
     image_size = len(image_bytes)
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.scholars.image_uploaded",
-        extra={
+        user_id=current_user.id,
-            "event": "api.scholars.image_uploaded",
+        scholar_profile_id=updated.id,
-            "user_id": current_user.id,
+        content_type=image.content_type,
-            "scholar_profile_id": updated.id,
+        size_bytes=image_size,
-            "content_type": image.content_type,
-            "size_bytes": image_size,
-        },
     )
     return success_payload(
         request,
-        data=_serialize_scholar(updated),
+        data=serialize_scholar(updated),
     )
 
 
@@ -542,16 +399,8 @@ async def clear_scholar_image_customization(
         profile=profile,
         upload_dir=settings.scholar_image_upload_dir,
     )
-    logger.info(
+    structured_log(logger, "info", "api.scholars.image_cleared", user_id=current_user.id, scholar_profile_id=updated.id)
-        "api.scholars.image_customization_cleared",
-        extra={
-            "event": "api.scholars.image_customization_cleared",
-            "user_id": current_user.id,
-            "scholar_profile_id": updated.id,
-        },
-    )
     return success_payload(
         request,
-        data=_serialize_scholar(updated),
+        data=serialize_scholar(updated),
     )
-

app/api/routers/settings.py

@@ -11,8 +11,9 @@ from app.api.responses import success_payload
 from app.api.schemas import SettingsEnvelope, SettingsUpdateRequest
 from app.db.models import User
 from app.db.session import get_db_session
-from app.services.domains.ingestion import safety as run_safety_service
+from app.logging_utils import structured_log
-from app.services.domains.settings import application as user_settings_service
+from app.services.ingestion import safety as run_safety_service
+from app.services.settings import application as user_settings_service
 from app.settings import settings as settings_module
 
 logger = logging.getLogger(__name__)
@@ -38,6 +39,9 @@ def _serialize_settings(user_settings) -> dict[str, object]:
             "cooldown_network_seconds": max(60, int(settings_module.ingestion_safety_cooldown_network_seconds)),
         },
         "safety_state": run_safety_service.get_safety_state_payload(user_settings),
+        "openalex_api_key": user_settings.openalex_api_key,
+        "crossref_api_token": user_settings.crossref_api_token,
+        "crossref_api_mailto": user_settings.crossref_api_mailto,
     }
 
 
@@ -80,16 +84,13 @@ async def _clear_expired_cooldown_with_log(
         return
     await db_session.commit()
     await db_session.refresh(user_settings)
-    logger.info(
+    structured_log(
+        logger,
+        "info",
         "api.settings.safety_cooldown_cleared",
-        extra={
+        user_id=user_id,
-            "event": "api.settings.safety_cooldown_cleared",
+        reason=previous_safety_state.get("cooldown_reason"),
-            "user_id": user_id,
+        cooldown_until=previous_safety_state.get("cooldown_until"),
-            "reason": previous_safety_state.get("cooldown_reason"),
-            "cooldown_until": previous_safety_state.get("cooldown_until"),
-            "metric_name": "api_settings_safety_cooldown_cleared_total",
-            "metric_value": 1,
-        },
     )
 
 
@@ -97,20 +98,6 @@ def _effective_auto_run_enabled(payload: SettingsUpdateRequest) -> bool:
     return bool(payload.auto_run_enabled) and settings_module.ingestion_automation_allowed
 
 
-def _log_settings_update(*, user_id: int, updated) -> None:
-    logger.info(
-        "api.settings.updated",
-        extra={
-            "event": "api.settings.updated",
-            "user_id": user_id,
-            "auto_run_enabled": updated.auto_run_enabled,
-            "run_interval_minutes": updated.run_interval_minutes,
-            "request_delay_seconds": updated.request_delay_seconds,
-            "nav_visible_pages": updated.nav_visible_pages,
-        },
-    )
-
-
 @router.get(
     "",
     response_model=SettingsEnvelope,
@@ -169,13 +156,26 @@ async def update_settings(
         run_interval_minutes=parsed_interval,
         request_delay_seconds=parsed_delay,
         nav_visible_pages=parsed_nav_visible_pages,
+        openalex_api_key=payload.openalex_api_key,
+        crossref_api_token=payload.crossref_api_token,
+        crossref_api_mailto=payload.crossref_api_mailto,
     )
     await _clear_expired_cooldown_with_log(
         db_session,
         user_id=current_user.id,
         user_settings=updated,
     )
-    _log_settings_update(user_id=current_user.id, updated=updated)
+    structured_log(
+        logger,
+        "info",
+        "api.settings.updated",
+        user_id=current_user.id,
+        auto_run_enabled=updated.auto_run_enabled,
+        run_interval_minutes=updated.run_interval_minutes,
+        request_delay_seconds=updated.request_delay_seconds,
+        nav_visible_pages=updated.nav_visible_pages,
+        openalex_api_key="SET" if updated.openalex_api_key else "UNSET",
+    )
     return success_payload(
         request,
         data=_serialize_settings(updated),

app/api/runtime_deps.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 from fastapi import Depends
 
-from app.services.domains.ingestion import application as ingestion_service
+from app.services.ingestion import application as ingestion_service
-from app.services.domains.scholar import source as scholar_source_service
+from app.services.scholar import source as scholar_source_service
 
 
 def get_scholar_source() -> scholar_source_service.ScholarSource:

app/api/schemas.py

@@ -1,867 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Any, Literal
-
-from pydantic import BaseModel, ConfigDict, Field, model_validator
-
-
-class ApiMeta(BaseModel):
-    request_id: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ApiErrorData(BaseModel):
-    code: str
-    message: str
-    details: Any | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ApiErrorEnvelope(BaseModel):
-    error: ApiErrorData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessageData(BaseModel):
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessageEnvelope(BaseModel):
-    data: MessageData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SessionUserData(BaseModel):
-    id: int
-    email: str
-    is_admin: bool
-    is_active: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AuthMeData(BaseModel):
-    authenticated: bool
-    csrf_token: str
-    user: SessionUserData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AuthMeEnvelope(BaseModel):
-    data: AuthMeData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class CsrfBootstrapData(BaseModel):
-    csrf_token: str
-    authenticated: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class CsrfBootstrapEnvelope(BaseModel):
-    data: CsrfBootstrapData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginRequest(BaseModel):
-    email: str
-    password: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginData(BaseModel):
-    authenticated: bool
-    csrf_token: str
-    user: SessionUserData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginEnvelope(BaseModel):
-    data: LoginData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ChangePasswordRequest(BaseModel):
-    current_password: str
-    new_password: str
-    confirm_password: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarItemData(BaseModel):
-    id: int
-    scholar_id: str
-    display_name: str | None
-    profile_image_url: str | None
-    profile_image_source: str
-    is_enabled: bool
-    baseline_completed: bool
-    last_run_dt: datetime | None
-    last_run_status: str | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarsListData(BaseModel):
-    scholars: list[ScholarItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarsListEnvelope(BaseModel):
-    data: ScholarsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarCreateRequest(BaseModel):
-    scholar_id: str
-    profile_image_url: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarEnvelope(BaseModel):
-    data: ScholarItemData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchCandidateData(BaseModel):
-    scholar_id: str
-    display_name: str
-    affiliation: str | None
-    email_domain: str | None
-    cited_by_count: int | None
-    interests: list[str] = Field(default_factory=list)
-    profile_url: str
-    profile_image_url: str | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchData(BaseModel):
-    query: str
-    state: str
-    state_reason: str
-    action_hint: str | None = None
-    candidates: list[ScholarSearchCandidateData]
-    warnings: list[str] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchEnvelope(BaseModel):
-    data: ScholarSearchData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarImageUrlUpdateRequest(BaseModel):
-    image_url: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarExportItemData(BaseModel):
-    scholar_id: str
-    display_name: str | None = None
-    is_enabled: bool = True
-    profile_image_override_url: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationExportItemData(BaseModel):
-    scholar_id: str
-    cluster_id: str | None = None
-    fingerprint_sha256: str | None = None
-    title: str
-    year: int | None = None
-    citation_count: int = 0
-    author_text: str | None = None
-    venue_text: str | None = None
-    pub_url: str | None = None
-    doi: str | None = None
-    pdf_url: str | None = None
-    is_read: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataExportData(BaseModel):
-    schema_version: int
-    exported_at: str
-    scholars: list[ScholarExportItemData]
-    publications: list[PublicationExportItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataExportEnvelope(BaseModel):
-    data: DataExportData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataImportRequest(BaseModel):
-    schema_version: int | None = None
-    scholars: list[ScholarExportItemData] = Field(default_factory=list)
-    publications: list[PublicationExportItemData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataImportResultData(BaseModel):
-    scholars_created: int
-    scholars_updated: int
-    publications_created: int
-    publications_updated: int
-    links_created: int
-    links_updated: int
-    skipped_records: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataImportEnvelope(BaseModel):
-    data: DataImportResultData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunListItemData(BaseModel):
-    id: int
-    trigger_type: str
-    status: str
-    start_dt: datetime
-    end_dt: datetime | None
-    scholar_count: int
-    new_publication_count: int
-    failed_count: int
-    partial_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunsListData(BaseModel):
-    runs: list[RunListItemData]
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunsListEnvelope(BaseModel):
-    data: RunsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunSummaryData(BaseModel):
-    succeeded_count: int
-    failed_count: int
-    partial_count: int
-    failed_state_counts: dict[str, int] = Field(default_factory=dict)
-    failed_reason_counts: dict[str, int] = Field(default_factory=dict)
-    scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
-    retry_counts: dict[str, int] = Field(default_factory=dict)
-    alert_thresholds: dict[str, int] = Field(default_factory=dict)
-    alert_flags: dict[str, bool] = Field(default_factory=dict)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScrapeSafetyCountersData(BaseModel):
-    consecutive_blocked_runs: int = 0
-    consecutive_network_runs: int = 0
-    cooldown_entry_count: int = 0
-    blocked_start_count: int = 0
-    last_blocked_failure_count: int = 0
-    last_network_failure_count: int = 0
-    last_evaluated_run_id: int | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScrapeSafetyStateData(BaseModel):
-    cooldown_active: bool
-    cooldown_reason: str | None = None
-    cooldown_reason_label: str | None = None
-    cooldown_until: datetime | None = None
-    cooldown_remaining_seconds: int = 0
-    recommended_action: str | None = None
-    counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunAttemptLogData(BaseModel):
-    attempt: int
-    cstart: int
-    state: str | None = None
-    state_reason: str | None = None
-    status_code: int | None = None
-    fetch_error: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunPageLogData(BaseModel):
-    page: int
-    cstart: int
-    state: str
-    state_reason: str | None = None
-    status_code: int | None = None
-    publication_count: int = 0
-    attempt_count: int = 0
-    has_show_more_button: bool = False
-    articles_range: str | None = None
-    warning_codes: list[str] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDebugData(BaseModel):
-    status_code: int | None = None
-    final_url: str | None = None
-    fetch_error: str | None = None
-    body_sha256: str | None = None
-    body_length: int | None = None
-    has_show_more_button: bool | None = None
-    articles_range: str | None = None
-    state_reason: str | None = None
-    warning_codes: list[str] = Field(default_factory=list)
-    marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
-    page_logs: list[RunPageLogData] = Field(default_factory=list)
-    attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunScholarResultData(BaseModel):
-    scholar_profile_id: int
-    scholar_id: str
-    state: str
-    state_reason: str | None = None
-    outcome: str
-    attempt_count: int = 0
-    publication_count: int = 0
-    start_cstart: int = 0
-    continuation_cstart: int | None = None
-    continuation_enqueued: bool = False
-    continuation_cleared: bool = False
-    warnings: list[str] = Field(default_factory=list)
-    error: str | None = None
-    debug: RunDebugData | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDetailData(BaseModel):
-    run: RunListItemData
-    summary: RunSummaryData
-    scholar_results: list[RunScholarResultData]
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDetailEnvelope(BaseModel):
-    data: RunDetailData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ManualRunData(BaseModel):
-    run_id: int
-    status: str
-    scholar_count: int
-    succeeded_count: int
-    failed_count: int
-    partial_count: int
-    new_publication_count: int
-    reused_existing_run: bool
-    idempotency_key: str | None = None
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ManualRunEnvelope(BaseModel):
-    data: ManualRunData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueItemData(BaseModel):
-    id: int
-    scholar_profile_id: int
-    scholar_label: str
-    status: str
-    reason: str
-    dropped_reason: str | None
-    attempt_count: int
-    resume_cstart: int
-    next_attempt_dt: datetime | None
-    updated_at: datetime
-    last_error: str | None
-    last_run_id: int | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueListData(BaseModel):
-    queue_items: list[QueueItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueListEnvelope(BaseModel):
-    data: QueueListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueItemEnvelope(BaseModel):
-    data: QueueItemData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueClearData(BaseModel):
-    queue_item_id: int
-    previous_status: str
-    status: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueClearEnvelope(BaseModel):
-    data: QueueClearData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserData(BaseModel):
-    id: int
-    email: str
-    is_active: bool
-    is_admin: bool
-    created_at: datetime
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUsersListData(BaseModel):
-    users: list[AdminUserData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUsersListEnvelope(BaseModel):
-    data: AdminUsersListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserCreateRequest(BaseModel):
-    email: str
-    password: str
-    is_admin: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserActiveUpdateRequest(BaseModel):
-    is_active: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserEnvelope(BaseModel):
-    data: AdminUserData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminResetPasswordRequest(BaseModel):
-    new_password: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityCheckData(BaseModel):
-    name: str
-    count: int
-    severity: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityData(BaseModel):
-    status: str
-    checked_at: datetime
-    failures: list[str] = Field(default_factory=list)
-    warnings: list[str] = Field(default_factory=list)
-    checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityEnvelope(BaseModel):
-    data: AdminDbIntegrityData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobData(BaseModel):
-    id: int
-    job_name: str
-    requested_by: str | None
-    dry_run: bool
-    status: str
-    scope: dict[str, Any] = Field(default_factory=dict)
-    summary: dict[str, Any] = Field(default_factory=dict)
-    error_text: str | None
-    started_at: datetime | None
-    finished_at: datetime | None
-    created_at: datetime
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobsData(BaseModel):
-    jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobsEnvelope(BaseModel):
-    data: AdminDbRepairJobsData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueItemData(BaseModel):
-    publication_id: int
-    title: str
-    doi: str | None
-    pdf_url: str | None
-    status: str
-    attempt_count: int
-    last_failure_reason: str | None
-    last_failure_detail: str | None
-    last_source: str | None
-    requested_by_user_id: int | None
-    requested_by_email: str | None
-    queued_at: datetime | None
-    last_attempt_at: datetime | None
-    resolved_at: datetime | None
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueData(BaseModel):
-    items: list[AdminPdfQueueItemData] = Field(default_factory=list)
-    total_count: int = 0
-    page: int = 1
-    page_size: int = 100
-    has_next: bool = False
-    has_prev: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueEnvelope(BaseModel):
-    data: AdminPdfQueueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueRequeueData(BaseModel):
-    publication_id: int
-    queued: bool
-    status: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueRequeueEnvelope(BaseModel):
-    data: AdminPdfQueueRequeueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueBulkEnqueueData(BaseModel):
-    requested_count: int
-    queued_count: int
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
-    data: AdminPdfQueueBulkEnqueueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationLinksRequest(BaseModel):
-    scope_mode: Literal["single_user", "all_users"] = "single_user"
-    user_id: int | None = Field(default=None, ge=1)
-    scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
-    dry_run: bool = True
-    gc_orphan_publications: bool = False
-    requested_by: str | None = None
-    confirmation_text: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-    @model_validator(mode="after")
-    def validate_scope(self) -> "AdminRepairPublicationLinksRequest":
-        if self.scope_mode == "single_user" and self.user_id is None:
-            raise ValueError("user_id is required when scope_mode=single_user.")
-        if self.scope_mode == "all_users" and self.user_id is not None:
-            raise ValueError("user_id must be omitted when scope_mode=all_users.")
-        if not self.dry_run and self.scope_mode == "all_users":
-            expected = "REPAIR ALL USERS"
-            provided = (self.confirmation_text or "").strip()
-            if provided != expected:
-                raise ValueError(
-                    "confirmation_text must equal 'REPAIR ALL USERS' "
-                    "when applying a repair to all users."
-                )
-        return self
-
-
-class AdminRepairPublicationLinksResultData(BaseModel):
-    job_id: int
-    status: str
-    scope: dict[str, Any] = Field(default_factory=dict)
-    summary: dict[str, Any] = Field(default_factory=dict)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationLinksEnvelope(BaseModel):
-    data: AdminRepairPublicationLinksResultData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsPolicyData(BaseModel):
-    min_run_interval_minutes: int
-    min_request_delay_seconds: int
-    automation_allowed: bool
-    manual_run_allowed: bool
-    blocked_failure_threshold: int
-    network_failure_threshold: int
-    cooldown_blocked_seconds: int
-    cooldown_network_seconds: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsData(BaseModel):
-    auto_run_enabled: bool
-    run_interval_minutes: int
-    request_delay_seconds: int
-    nav_visible_pages: list[str]
-    policy: SettingsPolicyData
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsEnvelope(BaseModel):
-    data: SettingsData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsUpdateRequest(BaseModel):
-    auto_run_enabled: bool
-    run_interval_minutes: int
-    request_delay_seconds: int
-    nav_visible_pages: list[str] | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationItemData(BaseModel):
-    publication_id: int
-    scholar_profile_id: int
-    scholar_label: str
-    title: str
-    year: int | None
-    citation_count: int
-    venue_text: str | None
-    pub_url: str | None
-    doi: str | None
-    pdf_url: str | None
-    pdf_status: str = "untracked"
-    pdf_attempt_count: int = 0
-    pdf_failure_reason: str | None = None
-    pdf_failure_detail: str | None = None
-    is_read: bool
-    is_favorite: bool = False
-    first_seen_at: datetime
-    is_new_in_latest_run: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationsListData(BaseModel):
-    mode: str
-    favorite_only: bool = False
-    selected_scholar_profile_id: int | None
-    unread_count: int
-    favorites_count: int
-    latest_count: int
-    new_count: int
-    total_count: int
-    page: int
-    page_size: int
-    has_next: bool = False
-    has_prev: bool = False
-    publications: list[PublicationItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationsListEnvelope(BaseModel):
-    data: PublicationsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkAllReadData(BaseModel):
-    message: str
-    updated_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkAllReadEnvelope(BaseModel):
-    data: MarkAllReadData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationSelectionItem(BaseModel):
-    scholar_profile_id: int
-    publication_id: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadRequest(BaseModel):
-    selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadData(BaseModel):
-    message: str
-    requested_count: int
-    updated_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadEnvelope(BaseModel):
-    data: MarkSelectedReadData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfRequest(BaseModel):
-    scholar_profile_id: int = Field(ge=1)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfData(BaseModel):
-    message: str
-    queued: bool
-    resolved_pdf: bool
-    publication: PublicationItemData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfEnvelope(BaseModel):
-    data: RetryPublicationPdfData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteRequest(BaseModel):
-    scholar_profile_id: int = Field(ge=1)
-    is_favorite: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteData(BaseModel):
-    message: str
-    publication: PublicationItemData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteEnvelope(BaseModel):
-    data: TogglePublicationFavoriteData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/__init__.py

@@ -0,0 +1,7 @@
+from app.api.schemas.admin import *  # noqa: F403
+from app.api.schemas.auth import *  # noqa: F403
+from app.api.schemas.common import *  # noqa: F403
+from app.api.schemas.publications import *  # noqa: F403
+from app.api.schemas.runs import *  # noqa: F403
+from app.api.schemas.scholars import *  # noqa: F403
+from app.api.schemas.settings import *  # noqa: F403

app/api/schemas/admin.py

@@ -0,0 +1,314 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from app.api.schemas.common import ApiMeta
+from app.api.schemas.publications import DisplayIdentifierData
+
+
+class AdminUserData(BaseModel):
+    id: int
+    email: str
+    is_active: bool
+    is_admin: bool
+    created_at: datetime
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUsersListData(BaseModel):
+    users: list[AdminUserData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUsersListEnvelope(BaseModel):
+    data: AdminUsersListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserCreateRequest(BaseModel):
+    email: str = Field(max_length=254)
+    password: str = Field(min_length=8, max_length=128)
+    is_admin: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserActiveUpdateRequest(BaseModel):
+    is_active: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserEnvelope(BaseModel):
+    data: AdminUserData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminResetPasswordRequest(BaseModel):
+    new_password: str = Field(min_length=8, max_length=128)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminScholarHttpSettingsData(BaseModel):
+    user_agent: str
+    rotate_user_agent: bool
+    accept_language: str
+    cookie: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminScholarHttpSettingsEnvelope(BaseModel):
+    data: AdminScholarHttpSettingsData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminScholarHttpSettingsUpdateRequest(BaseModel):
+    user_agent: str
+    rotate_user_agent: bool
+    accept_language: str
+    cookie: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityCheckData(BaseModel):
+    name: str
+    count: int
+    severity: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityData(BaseModel):
+    status: str
+    checked_at: datetime
+    failures: list[str] = Field(default_factory=list)
+    warnings: list[str] = Field(default_factory=list)
+    checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityEnvelope(BaseModel):
+    data: AdminDbIntegrityData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobData(BaseModel):
+    id: int
+    job_name: str
+    requested_by: str | None
+    dry_run: bool
+    status: str
+    scope: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any] = Field(default_factory=dict)
+    error_text: str | None
+    started_at: datetime | None
+    finished_at: datetime | None
+    created_at: datetime
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobsData(BaseModel):
+    jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobsEnvelope(BaseModel):
+    data: AdminDbRepairJobsData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueItemData(BaseModel):
+    publication_id: int
+    title: str
+    display_identifier: DisplayIdentifierData | None = None
+    pdf_url: str | None
+    status: str
+    attempt_count: int
+    last_failure_reason: str | None
+    last_failure_detail: str | None
+    last_source: str | None
+    requested_by_user_id: int | None
+    requested_by_email: str | None
+    queued_at: datetime | None
+    last_attempt_at: datetime | None
+    resolved_at: datetime | None
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueData(BaseModel):
+    items: list[AdminPdfQueueItemData] = Field(default_factory=list)
+    total_count: int = 0
+    page: int = 1
+    page_size: int = 100
+    has_next: bool = False
+    has_prev: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueEnvelope(BaseModel):
+    data: AdminPdfQueueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueRequeueData(BaseModel):
+    publication_id: int
+    queued: bool
+    status: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueRequeueEnvelope(BaseModel):
+    data: AdminPdfQueueRequeueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueBulkEnqueueData(BaseModel):
+    requested_count: int
+    queued_count: int
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
+    data: AdminPdfQueueBulkEnqueueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationLinksRequest(BaseModel):
+    scope_mode: Literal["single_user", "all_users"] = "single_user"
+    user_id: int | None = Field(default=None, ge=1)
+    scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
+    dry_run: bool = True
+    gc_orphan_publications: bool = False
+    requested_by: str | None = None
+    confirmation_text: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")
+    def validate_scope(self) -> AdminRepairPublicationLinksRequest:
+        if self.scope_mode == "single_user" and self.user_id is None:
+            raise ValueError("user_id is required when scope_mode=single_user.")
+        if self.scope_mode == "all_users" and self.user_id is not None:
+            raise ValueError("user_id must be omitted when scope_mode=all_users.")
+        if not self.dry_run and self.scope_mode == "all_users":
+            expected = "REPAIR ALL USERS"
+            provided = (self.confirmation_text or "").strip()
+            if provided != expected:
+                raise ValueError("confirmation_text must equal 'REPAIR ALL USERS' when applying a repair to all users.")
+        return self
+
+
+class AdminRepairPublicationLinksResultData(BaseModel):
+    job_id: int
+    status: str
+    scope: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationLinksEnvelope(BaseModel):
+    data: AdminRepairPublicationLinksResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminNearDuplicateClusterMemberData(BaseModel):
+    publication_id: int
+    title: str
+    year: int | None
+    citation_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminNearDuplicateClusterData(BaseModel):
+    cluster_key: str
+    winner_publication_id: int
+    member_count: int
+    similarity_score: float = Field(ge=0.0, le=1.0)
+    members: list[AdminNearDuplicateClusterMemberData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationNearDuplicatesRequest(BaseModel):
+    dry_run: bool = True
+    similarity_threshold: float = Field(default=0.78, ge=0.5, le=1.0)
+    min_shared_tokens: int = Field(default=3, ge=1, le=8)
+    max_year_delta: int = Field(default=1, ge=0, le=5)
+    max_clusters: int = Field(default=25, ge=1, le=200)
+    selected_cluster_keys: list[str] = Field(default_factory=list, max_length=200)
+    requested_by: str | None = None
+    confirmation_text: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")
+    def validate_apply_mode(self) -> AdminRepairPublicationNearDuplicatesRequest:
+        if self.dry_run:
+            return self
+        if not self.selected_cluster_keys:
+            raise ValueError("selected_cluster_keys is required when dry_run=false.")
+        expected = "MERGE SELECTED DUPLICATES"
+        provided = (self.confirmation_text or "").strip()
+        if provided != expected:
+            raise ValueError(
+                "confirmation_text must equal 'MERGE SELECTED DUPLICATES' when applying near-duplicate merges."
+            )
+        return self
+
+
+class AdminRepairPublicationNearDuplicatesResultData(BaseModel):
+    job_id: int
+    status: str
+    scope: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any] = Field(default_factory=dict)
+    clusters: list[AdminNearDuplicateClusterData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationNearDuplicatesEnvelope(BaseModel):
+    data: AdminRepairPublicationNearDuplicatesResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/auth.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.api.schemas.common import ApiMeta
+
+
+class SessionUserData(BaseModel):
+    id: int
+    email: str
+    is_admin: bool
+    is_active: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AuthMeData(BaseModel):
+    authenticated: bool
+    csrf_token: str
+    user: SessionUserData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AuthMeEnvelope(BaseModel):
+    data: AuthMeData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class CsrfBootstrapData(BaseModel):
+    csrf_token: str
+    authenticated: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class CsrfBootstrapEnvelope(BaseModel):
+    data: CsrfBootstrapData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginRequest(BaseModel):
+    email: str = Field(max_length=254)
+    password: str = Field(min_length=8, max_length=128)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginData(BaseModel):
+    authenticated: bool
+    csrf_token: str
+    user: SessionUserData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginEnvelope(BaseModel):
+    data: LoginData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ChangePasswordRequest(BaseModel):
+    current_password: str = Field(min_length=8, max_length=128)
+    new_password: str = Field(min_length=8, max_length=128)
+    confirm_password: str = Field(min_length=8, max_length=128)
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/common.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict
+
+
+class ApiMeta(BaseModel):
+    request_id: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ApiErrorData(BaseModel):
+    code: str
+    message: str
+    details: Any | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ApiErrorEnvelope(BaseModel):
+    error: ApiErrorData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessageData(BaseModel):
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessageEnvelope(BaseModel):
+    data: MessageData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/publications.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.api.schemas.common import ApiMeta
+
+
+class DisplayIdentifierData(BaseModel):
+    kind: str
+    value: str
+    label: str
+    url: str | None
+    confidence_score: float = Field(ge=0.0, le=1.0)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationItemData(BaseModel):
+    publication_id: int
+    scholar_profile_id: int
+    scholar_label: str
+    title: str
+    year: int | None
+    citation_count: int
+    venue_text: str | None
+    pub_url: str | None
+    display_identifier: DisplayIdentifierData | None = None
+    pdf_url: str | None
+    pdf_status: str = "untracked"
+    pdf_attempt_count: int = 0
+    pdf_failure_reason: str | None = None
+    pdf_failure_detail: str | None = None
+    is_read: bool
+    is_favorite: bool = False
+    first_seen_at: datetime
+    is_new_in_latest_run: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationsListData(BaseModel):
+    mode: str
+    favorite_only: bool = False
+    selected_scholar_profile_id: int | None
+    unread_count: int
+    favorites_count: int
+    latest_count: int
+    new_count: int
+    total_count: int
+    page: int
+    page_size: int
+    snapshot: str
+    has_next: bool = False
+    has_prev: bool = False
+    publications: list[PublicationItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationsListEnvelope(BaseModel):
+    data: PublicationsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkAllReadData(BaseModel):
+    message: str
+    updated_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkAllReadEnvelope(BaseModel):
+    data: MarkAllReadData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationSelectionItem(BaseModel):
+    scholar_profile_id: int
+    publication_id: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadRequest(BaseModel):
+    selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadData(BaseModel):
+    message: str
+    requested_count: int
+    updated_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadEnvelope(BaseModel):
+    data: MarkSelectedReadData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfRequest(BaseModel):
+    scholar_profile_id: int = Field(ge=1)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfData(BaseModel):
+    message: str
+    queued: bool
+    resolved_pdf: bool
+    publication: PublicationItemData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfEnvelope(BaseModel):
+    data: RetryPublicationPdfData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteRequest(BaseModel):
+    scholar_profile_id: int = Field(ge=1)
+    is_favorite: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteData(BaseModel):
+    message: str
+    publication: PublicationItemData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteEnvelope(BaseModel):
+    data: TogglePublicationFavoriteData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/runs.py

@@ -0,0 +1,226 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.api.schemas.common import ApiMeta
+
+
+class RunListItemData(BaseModel):
+    id: int
+    trigger_type: str
+    status: str
+    start_dt: datetime
+    end_dt: datetime | None
+    scholar_count: int
+    new_publication_count: int
+    failed_count: int
+    partial_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScrapeSafetyCountersData(BaseModel):
+    consecutive_blocked_runs: int = 0
+    consecutive_network_runs: int = 0
+    cooldown_entry_count: int = 0
+    blocked_start_count: int = 0
+    last_blocked_failure_count: int = 0
+    last_network_failure_count: int = 0
+    last_evaluated_run_id: int | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScrapeSafetyStateData(BaseModel):
+    cooldown_active: bool
+    cooldown_reason: str | None = None
+    cooldown_reason_label: str | None = None
+    cooldown_until: datetime | None = None
+    cooldown_remaining_seconds: int = 0
+    recommended_action: str | None = None
+    counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunsListData(BaseModel):
+    runs: list[RunListItemData]
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunsListEnvelope(BaseModel):
+    data: RunsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunSummaryData(BaseModel):
+    succeeded_count: int
+    failed_count: int
+    partial_count: int
+    failed_state_counts: dict[str, int] = Field(default_factory=dict)
+    failed_reason_counts: dict[str, int] = Field(default_factory=dict)
+    scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
+    retry_counts: dict[str, int] = Field(default_factory=dict)
+    alert_thresholds: dict[str, int] = Field(default_factory=dict)
+    alert_flags: dict[str, bool] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunAttemptLogData(BaseModel):
+    attempt: int
+    cstart: int
+    state: str | None = None
+    state_reason: str | None = None
+    status_code: int | None = None
+    fetch_error: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunPageLogData(BaseModel):
+    page: int
+    cstart: int
+    state: str
+    state_reason: str | None = None
+    status_code: int | None = None
+    publication_count: int = 0
+    attempt_count: int = 0
+    has_show_more_button: bool = False
+    articles_range: str | None = None
+    warning_codes: list[str] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDebugData(BaseModel):
+    status_code: int | None = None
+    final_url: str | None = None
+    fetch_error: str | None = None
+    body_sha256: str | None = None
+    body_length: int | None = None
+    has_show_more_button: bool | None = None
+    articles_range: str | None = None
+    state_reason: str | None = None
+    warning_codes: list[str] = Field(default_factory=list)
+    marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
+    page_logs: list[RunPageLogData] = Field(default_factory=list)
+    attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunScholarResultData(BaseModel):
+    scholar_profile_id: int
+    scholar_id: str
+    state: str
+    state_reason: str | None = None
+    outcome: str
+    attempt_count: int = 0
+    publication_count: int = 0
+    start_cstart: int = 0
+    continuation_cstart: int | None = None
+    continuation_enqueued: bool = False
+    continuation_cleared: bool = False
+    warnings: list[str] = Field(default_factory=list)
+    error: str | None = None
+    debug: RunDebugData | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDetailData(BaseModel):
+    run: RunListItemData
+    summary: RunSummaryData
+    scholar_results: list[RunScholarResultData]
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDetailEnvelope(BaseModel):
+    data: RunDetailData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ManualRunData(BaseModel):
+    run_id: int
+    status: str
+    scholar_count: int
+    succeeded_count: int
+    failed_count: int
+    partial_count: int
+    new_publication_count: int
+    reused_existing_run: bool
+    idempotency_key: str | None = None
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ManualRunEnvelope(BaseModel):
+    data: ManualRunData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueItemData(BaseModel):
+    id: int
+    scholar_profile_id: int
+    scholar_label: str
+    status: str
+    reason: str
+    dropped_reason: str | None
+    attempt_count: int
+    resume_cstart: int
+    next_attempt_dt: datetime | None
+    updated_at: datetime
+    last_error: str | None
+    last_run_id: int | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueListData(BaseModel):
+    queue_items: list[QueueItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueListEnvelope(BaseModel):
+    data: QueueListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueItemEnvelope(BaseModel):
+    data: QueueItemData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueClearData(BaseModel):
+    queue_item_id: int
+    previous_status: str
+    status: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueClearEnvelope(BaseModel):
+    data: QueueClearData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/scholars.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.api.schemas.common import ApiMeta
+
+
+class ScholarItemData(BaseModel):
+    id: int
+    scholar_id: str
+    display_name: str | None
+    profile_image_url: str | None
+    profile_image_source: str
+    is_enabled: bool
+    baseline_completed: bool
+    last_run_dt: datetime | None
+    last_run_status: str | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarsListData(BaseModel):
+    scholars: list[ScholarItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarsListEnvelope(BaseModel):
+    data: ScholarsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarCreateRequest(BaseModel):
+    scholar_id: str
+    profile_image_url: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarEnvelope(BaseModel):
+    data: ScholarItemData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchCandidateData(BaseModel):
+    scholar_id: str
+    display_name: str
+    affiliation: str | None
+    email_domain: str | None
+    cited_by_count: int | None
+    interests: list[str] = Field(default_factory=list)
+    profile_url: str
+    profile_image_url: str | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchData(BaseModel):
+    query: str
+    state: str
+    state_reason: str
+    action_hint: str | None = None
+    candidates: list[ScholarSearchCandidateData]
+    warnings: list[str] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchEnvelope(BaseModel):
+    data: ScholarSearchData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarImageUrlUpdateRequest(BaseModel):
+    image_url: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarExportItemData(BaseModel):
+    scholar_id: str
+    display_name: str | None = None
+    is_enabled: bool = True
+    profile_image_override_url: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationExportItemData(BaseModel):
+    scholar_id: str
+    cluster_id: str | None = None
+    fingerprint_sha256: str | None = None
+    title: str
+    year: int | None = None
+    citation_count: int = 0
+    author_text: str | None = None
+    venue_text: str | None = None
+    pub_url: str | None = None
+    pdf_url: str | None = None
+    is_read: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataExportData(BaseModel):
+    schema_version: int
+    exported_at: str
+    scholars: list[ScholarExportItemData]
+    publications: list[PublicationExportItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataExportEnvelope(BaseModel):
+    data: DataExportData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportRequest(BaseModel):
+    schema_version: int | None = None
+    scholars: list[ScholarExportItemData] = Field(default_factory=list)
+    publications: list[PublicationExportItemData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportResultData(BaseModel):
+    scholars_created: int
+    scholars_updated: int
+    publications_created: int
+    publications_updated: int
+    links_created: int
+    links_updated: int
+    skipped_records: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportEnvelope(BaseModel):
+    data: DataImportResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/settings.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict
+
+from app.api.schemas.common import ApiMeta
+from app.api.schemas.runs import ScrapeSafetyStateData
+
+
+class SettingsPolicyData(BaseModel):
+    min_run_interval_minutes: int
+    min_request_delay_seconds: int
+    automation_allowed: bool
+    manual_run_allowed: bool
+    blocked_failure_threshold: int
+    network_failure_threshold: int
+    cooldown_blocked_seconds: int
+    cooldown_network_seconds: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsData(BaseModel):
+    auto_run_enabled: bool
+    run_interval_minutes: int
+    request_delay_seconds: int
+    nav_visible_pages: list[str]
+    policy: SettingsPolicyData
+    safety_state: ScrapeSafetyStateData
+
+    openalex_api_key: str | None = None
+    crossref_api_token: str | None = None
+    crossref_api_mailto: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsEnvelope(BaseModel):
+    data: SettingsData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsUpdateRequest(BaseModel):
+    auto_run_enabled: bool
+    run_interval_minutes: int
+    request_delay_seconds: int
+    nav_visible_pages: list[str] | None = None
+
+    openalex_api_key: str | None = None
+    crossref_api_token: str | None = None
+    crossref_api_mailto: str | None = None
+
+    model_config = ConfigDict(extra="forbid")

app/auth/__init__.py

@@ -1,2 +1 @@
 """Authentication package for scholarr."""
-

app/auth/deps.py

@@ -24,4 +24,3 @@ def get_login_rate_limiter() -> SlidingWindowRateLimiter:
         max_attempts=settings.login_rate_limit_attempts,
         window_seconds=settings.login_rate_limit_window_seconds,
     )
-

app/auth/runtime.py

@@ -7,8 +7,9 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.auth.session import clear_session_user, get_session_user, set_session_user
 from app.db.models import User
+from app.logging_utils import structured_log
 from app.security.csrf import CSRF_SESSION_KEY
-from app.services.domains.users import application as user_service
+from app.services.users import application as user_service
 
 logger = logging.getLogger(__name__)
 
@@ -28,13 +29,7 @@ async def get_authenticated_user(
 
     user = await user_service.get_user_by_id(db_session, session_user.id)
     if user is None or not user.is_active:
-        logger.info(
+        structured_log(logger, "info", "auth.session_invalidated", session_user_id=session_user.id)
-            "auth.session_invalidated",
-            extra={
-                "event": "auth.session_invalidated",
-                "session_user_id": session_user.id,
-            },
-        )
         invalidate_session(request)
         return None
 

app/auth/security.py

@@ -16,4 +16,3 @@ class PasswordService:
             return bool(self._hasher.verify(password_hash, password))
         except (InvalidHashError, VerificationError):
             return False
-

app/auth/service.py

@@ -21,9 +21,7 @@ class AuthService:
         normalized_email = email.strip().lower()
         if not normalized_email or not password:
             return None
-        result = await db_session.execute(
+        result = await db_session.execute(select(User).where(User.email == normalized_email))
-            select(User).where(User.email == normalized_email)
-        )
         user = result.scalar_one_or_none()
         if user is None or not user.is_active:
             return None

app/auth/session.py

@@ -4,7 +4,6 @@ from dataclasses import dataclass
 
 from starlette.requests import Request
 
-
 SESSION_USER_ID_KEY = "auth_user_id"
 SESSION_USER_EMAIL_KEY = "auth_user_email"
 SESSION_USER_IS_ADMIN_KEY = "auth_user_is_admin"

app/db/base.py

@@ -1,9 +1,8 @@
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 
 from sqlalchemy import MetaData, event
 from sqlalchemy.orm import DeclarativeBase
 
-
 NAMING_CONVENTION = {
     "ix": "ix_%(column_0_label)s",
     "uq": "uq_%(table_name)s_%(column_0_name)s",
@@ -21,7 +20,7 @@ class Base(DeclarativeBase):
 def _set_updated_at_before_update(_mapper, _connection, target) -> None:
     # Keep audit timestamps current for ORM-managed updates.
     if hasattr(target, "updated_at"):
-        target.updated_at = datetime.now(timezone.utc)
+        target.updated_at = datetime.now(UTC)
 
 
 metadata = Base.metadata

app/db/models.py

@@ -8,6 +8,7 @@ from sqlalchemy import (
     CheckConstraint,
     DateTime,
     Enum,
+    Float,
     ForeignKey,
     Index,
     Integer,
@@ -30,9 +31,11 @@ class RunTriggerType(StrEnum):
 
 class RunStatus(StrEnum):
     RUNNING = "running"
+    RESOLVING = "resolving"
     SUCCESS = "success"
     PARTIAL_FAILURE = "partial_failure"
     FAILED = "failed"
+    CANCELED = "canceled"
 
 
 class QueueItemStatus(StrEnum):
@@ -59,18 +62,10 @@ class User(Base):
     id: Mapped[int] = mapped_column(primary_key=True)
     email: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
     password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
-    is_active: Mapped[bool] = mapped_column(
+    is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("true"))
-        Boolean, nullable=False, server_default=text("true")
+    is_admin: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
-    )
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    is_admin: Mapped[bool] = mapped_column(
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-        Boolean, nullable=False, server_default=text("false")
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
 
 
 class UserSetting(Base):
@@ -86,18 +81,10 @@ class UserSetting(Base):
         ),
     )
 
-    user_id: Mapped[int] = mapped_column(
+    user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), primary_key=True)
-        ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    auto_run_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
-    )
+    run_interval_minutes: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("1440"))
-    auto_run_enabled: Mapped[bool] = mapped_column(
+    request_delay_seconds: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("10"))
-        Boolean, nullable=False, server_default=text("false")
-    )
-    run_interval_minutes: Mapped[int] = mapped_column(
-        Integer, nullable=False, server_default=text("1440")
-    )
-    request_delay_seconds: Mapped[int] = mapped_column(
-        Integer, nullable=False, server_default=text("10")
-    )
     nav_visible_pages: Mapped[list[str]] = mapped_column(
         JSONB,
         nullable=False,
@@ -112,12 +99,13 @@ class UserSetting(Base):
     )
     scrape_cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     scrape_cooldown_reason: Mapped[str | None] = mapped_column(String(64))
-    created_at: Mapped[datetime] = mapped_column(
+
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+    openalex_api_key: Mapped[str | None] = mapped_column(String(255))
-    )
+    crossref_api_token: Mapped[str | None] = mapped_column(String(255))
-    updated_at: Mapped[datetime] = mapped_column(
+    crossref_api_mailto: Mapped[str | None] = mapped_column(String(255))
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+
-    )
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
 
 
 class ScholarProfile(Base):
@@ -128,9 +116,7 @@ class ScholarProfile(Base):
     )
 
     id: Mapped[int] = mapped_column(primary_key=True)
-    user_id: Mapped[int] = mapped_column(
+    user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
-        ForeignKey("users.id", ondelete="CASCADE"), nullable=False
-    )
     scholar_id: Mapped[str] = mapped_column(String(64), nullable=False)
     display_name: Mapped[str | None] = mapped_column(String(255))
     profile_image_url: Mapped[str | None] = mapped_column(Text)
@@ -138,69 +124,47 @@ class ScholarProfile(Base):
     profile_image_upload_path: Mapped[str | None] = mapped_column(Text)
     last_initial_page_fingerprint_sha256: Mapped[str | None] = mapped_column(String(64))
     last_initial_page_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    is_enabled: Mapped[bool] = mapped_column(
+    is_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("true"))
-        Boolean, nullable=False, server_default=text("true")
+    baseline_completed: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
-    )
-    baseline_completed: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, server_default=text("false")
-    )
     last_run_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     last_run_status: Mapped[RunStatus | None] = mapped_column(
         RUN_STATUS_DB_ENUM,
     )
-    created_at: Mapped[datetime] = mapped_column(
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
 
 
 class CrawlRun(Base):
     __tablename__ = "crawl_runs"
     __table_args__ = (
         Index("ix_crawl_runs_user_start", "user_id", "start_dt"),
+        Index(
+            "uq_crawl_runs_user_active",
+            "user_id",
+            unique=True,
+            postgresql_where=text("status IN ('running'::run_status, 'resolving'::run_status)"),
+        ),
         Index(
             "uq_crawl_runs_user_manual_idempotency_key",
             "user_id",
             "idempotency_key",
             unique=True,
-            postgresql_where=text(
+            postgresql_where=text("idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"),
-                "idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"
-            ),
         ),
     )
 
     id: Mapped[int] = mapped_column(primary_key=True)
-    user_id: Mapped[int] = mapped_column(
+    user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
-        ForeignKey("users.id", ondelete="CASCADE"), nullable=False
+    trigger_type: Mapped[RunTriggerType] = mapped_column(RUN_TRIGGER_TYPE_DB_ENUM, nullable=False)
-    )
+    status: Mapped[RunStatus] = mapped_column(RUN_STATUS_DB_ENUM, nullable=False)
-    trigger_type: Mapped[RunTriggerType] = mapped_column(
+    start_dt: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-        RUN_TRIGGER_TYPE_DB_ENUM, nullable=False
-    )
-    status: Mapped[RunStatus] = mapped_column(
-        RUN_STATUS_DB_ENUM, nullable=False
-    )
-    start_dt: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
     end_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    scholar_count: Mapped[int] = mapped_column(
+    scholar_count: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
-        Integer, nullable=False, server_default=text("0")
+    new_pub_count: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
-    )
-    new_pub_count: Mapped[int] = mapped_column(
-        Integer, nullable=False, server_default=text("0")
-    )
     idempotency_key: Mapped[str | None] = mapped_column(String(128))
-    error_log: Mapped[dict] = mapped_column(
+    error_log: Mapped[dict] = mapped_column(JSONB, nullable=False, server_default=text("'{}'::jsonb"))
-        JSONB, nullable=False, server_default=text("'{}'::jsonb")
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    )
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
 
 
 class Publication(Base):
@@ -221,20 +185,59 @@ class Publication(Base):
     title_raw: Mapped[str] = mapped_column(Text, nullable=False)
     title_normalized: Mapped[str] = mapped_column(Text, nullable=False)
     year: Mapped[int | None] = mapped_column(Integer)
-    citation_count: Mapped[int] = mapped_column(
+    citation_count: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
-        Integer, nullable=False, server_default=text("0")
-    )
     author_text: Mapped[str | None] = mapped_column(Text)
     venue_text: Mapped[str | None] = mapped_column(Text)
     pub_url: Mapped[str | None] = mapped_column(Text)
-    doi: Mapped[str | None] = mapped_column(String(255))
     pdf_url: Mapped[str | None] = mapped_column(Text)
-    created_at: Mapped[datetime] = mapped_column(
+    canonical_title_hash: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+    openalex_enriched: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
+    openalex_last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+
+
+class PublicationIdentifier(Base):
+    __tablename__ = "publication_identifiers"
+    __table_args__ = (
+        UniqueConstraint(
+            "publication_id",
+            "kind",
+            "value_normalized",
+            name="uq_publication_identifiers_publication_kind_value",
+        ),
+        CheckConstraint(
+            "confidence_score >= 0 AND confidence_score <= 1",
+            name="publication_identifiers_confidence_score_range",
+        ),
+        Index(
+            "ix_publication_identifiers_kind_value",
+            "kind",
+            "value_normalized",
+        ),
+        Index(
+            "ix_publication_identifiers_publication_id",
+            "publication_id",
+        ),
     )
-    updated_at: Mapped[datetime] = mapped_column(
+
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+    id: Mapped[int] = mapped_column(primary_key=True)
+    publication_id: Mapped[int] = mapped_column(
+        ForeignKey("publications.id", ondelete="CASCADE"),
+        nullable=False,
     )
+    kind: Mapped[str] = mapped_column(String(32), nullable=False)
+    value_raw: Mapped[str] = mapped_column(Text, nullable=False)
+    value_normalized: Mapped[str] = mapped_column(String(255), nullable=False)
+    source: Mapped[str] = mapped_column(String(64), nullable=False)
+    confidence_score: Mapped[float] = mapped_column(
+        Float,
+        nullable=False,
+        server_default=text("0"),
+    )
+    evidence_url: Mapped[str | None] = mapped_column(Text)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
 
 
 class PublicationPdfJob(Base):
@@ -272,12 +275,8 @@ class PublicationPdfJob(Base):
     last_requested_by_user_id: Mapped[int | None] = mapped_column(
         ForeignKey("users.id", ondelete="SET NULL"),
     )
-    created_at: Mapped[datetime] = mapped_column(
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
 
 
 class PublicationPdfJobEvent(Base):
@@ -301,9 +300,7 @@ class PublicationPdfJobEvent(Base):
     source: Mapped[str | None] = mapped_column(String(32))
     failure_reason: Mapped[str | None] = mapped_column(String(64))
     message: Mapped[str | None] = mapped_column(Text)
-    created_at: Mapped[datetime] = mapped_column(
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
 
 
 class ScholarPublication(Base):
@@ -321,18 +318,10 @@ class ScholarPublication(Base):
         ForeignKey("publications.id", ondelete="CASCADE"),
         primary_key=True,
     )
-    is_read: Mapped[bool] = mapped_column(
+    is_read: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
-        Boolean, nullable=False, server_default=text("false")
+    is_favorite: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
-    )
+    first_seen_run_id: Mapped[int | None] = mapped_column(ForeignKey("crawl_runs.id", ondelete="SET NULL"))
-    is_favorite: Mapped[bool] = mapped_column(
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-        Boolean, nullable=False, server_default=text("false")
-    )
-    first_seen_run_id: Mapped[int | None] = mapped_column(
-        ForeignKey("crawl_runs.id", ondelete="SET NULL")
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
 
 
 class IngestionQueueItem(Base):
@@ -387,12 +376,8 @@ class IngestionQueueItem(Base):
     last_error: Mapped[str | None] = mapped_column(Text)
     dropped_reason: Mapped[str | None] = mapped_column(String(128))
     dropped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    created_at: Mapped[datetime] = mapped_column(
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
 
 
 class DataRepairJob(Base):
@@ -432,12 +417,8 @@ class DataRepairJob(Base):
     error_text: Mapped[str | None] = mapped_column(Text)
     started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    created_at: Mapped[datetime] = mapped_column(
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
 
 
 class AuthorSearchRuntimeState(Base):
@@ -461,12 +442,38 @@ class AuthorSearchRuntimeState(Base):
         nullable=False,
         server_default=text("false"),
     )
-    created_at: Mapped[datetime] = mapped_column(
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+
+
+class ArxivRuntimeState(Base):
+    __tablename__ = "arxiv_runtime_state"
+
+    state_key: Mapped[str] = mapped_column(String(64), primary_key=True)
+    next_allowed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+
+
+class ArxivQueryCacheEntry(Base):
+    __tablename__ = "arxiv_query_cache_entries"
+    __table_args__ = (
+        Index("ix_arxiv_query_cache_expires_at", "expires_at"),
+        Index("ix_arxiv_query_cache_cached_at", "cached_at"),
     )
-    updated_at: Mapped[datetime] = mapped_column(
+
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+    query_fingerprint: Mapped[str] = mapped_column(String(64), primary_key=True)
+    payload: Mapped[dict] = mapped_column(
+        JSONB,
+        nullable=False,
     )
+    expires_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        nullable=False,
+    )
+    cached_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
 
 
 class AuthorSearchCacheEntry(Base):
@@ -485,9 +492,5 @@ class AuthorSearchCacheEntry(Base):
         DateTime(timezone=True),
         nullable=False,
     )
-    cached_at: Mapped[datetime] = mapped_column(
+    cached_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-        DateTime(timezone=True), nullable=False, server_default=func.now()
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )

app/db/session.py

@@ -1,6 +1,6 @@
-from collections.abc import AsyncIterator
 import logging
 import os
+from collections.abc import AsyncIterator
 
 from sqlalchemy import text
 from sqlalchemy.ext.asyncio import (
@@ -11,6 +11,7 @@ from sqlalchemy.ext.asyncio import (
 )
 from sqlalchemy.pool import NullPool
 
+from app.logging_utils import structured_log
 from app.settings import settings
 
 logger = logging.getLogger(__name__)
@@ -30,13 +31,12 @@ def _normalized_pool_mode(raw_mode: str) -> str:
         return "queue"
     if mode in {"null", "queue"}:
         return mode
-    logger.warning(
+    structured_log(
+        logger,
+        "warning",
         "db.invalid_pool_mode_fallback",
-        extra={
+        database_pool_mode=raw_mode,
-            "event": "db.invalid_pool_mode_fallback",
+        fallback_mode="queue",
-            "database_pool_mode": raw_mode,
-            "fallback_mode": "queue",
-        },
     )
     return "queue"
 
@@ -56,12 +56,11 @@ def get_engine() -> AsyncEngine:
             engine_kwargs["pool_timeout"] = max(1, int(settings.database_pool_timeout_seconds))
 
         _engine = create_async_engine(settings.database_url, **engine_kwargs)
-        logger.info(
+        structured_log(
+            logger,
+            "info",
             "db.engine_initialized",
-            extra={
+            pool_mode=pool_mode,
-                "event": "db.engine_initialized",
-                "pool_mode": pool_mode,
-            },
         )
     return _engine
 
@@ -86,7 +85,7 @@ async def check_database() -> bool:
             result = await conn.execute(text("SELECT 1"))
             return result.scalar_one() == 1
     except Exception:
-        logger.exception("db.healthcheck_failed", extra={"event": "db.healthcheck_failed"})
+        structured_log(logger, "exception", "db.healthcheck_failed")
         return False
 
 
@@ -94,6 +93,6 @@ async def close_engine() -> None:
     global _engine, _session_factory
     if _engine is not None:
         await _engine.dispose()
-        logger.info("db.engine_disposed", extra={"event": "db.engine_disposed"})
+        structured_log(logger, "info", "db.engine_disposed")
         _engine = None
         _session_factory = None

app/http/middleware.py

@@ -1,14 +1,15 @@
 from __future__ import annotations
 
-from secrets import token_urlsafe
 import logging
 import time
+from secrets import token_urlsafe
 
 from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.requests import Request
 from starlette.responses import Response
 
 from app.logging_context import set_request_id
+from app.logging_utils import structured_log
 
 REQUEST_ID_HEADER = "X-Request-ID"
 DEFAULT_PERMISSIONS_POLICY = (
@@ -61,42 +62,39 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
         start = time.perf_counter()
         should_log = self._log_requests and not self._is_skipped_path(request.url.path)
         if should_log:
-            logger.debug(
+            structured_log(
+                logger,
+                "debug",
                 "request.started",
-                extra={
+                method=request.method,
-                    "event": "request.started",
+                path=request.url.path,
-                    "method": request.method,
-                    "path": request.url.path,
-                },
             )
 
         try:
             response = await call_next(request)
         except Exception:
             duration_ms = int((time.perf_counter() - start) * 1000)
-            logger.exception(
+            structured_log(
+                logger,
+                "exception",
                 "request.failed",
-                extra={
+                method=request.method,
-                    "event": "request.failed",
+                path=request.url.path,
-                    "method": request.method,
+                duration_ms=duration_ms,
-                    "path": request.url.path,
-                    "duration_ms": duration_ms,
-                },
             )
             raise
         else:
             duration_ms = int((time.perf_counter() - start) * 1000)
             response.headers[REQUEST_ID_HEADER] = request_id
             if should_log:
-                logger.debug(
+                structured_log(
+                    logger,
+                    "debug",
                     "request.completed",
-                    extra={
+                    method=request.method,
-                        "event": "request.completed",
+                    path=request.url.path,
-                        "method": request.method,
+                    status_code=response.status_code,
-                        "path": request.url.path,
+                    duration_ms=duration_ms,
-                        "status_code": response.status_code,
-                        "duration_ms": duration_ms,
-                    },
                 )
             return response
         finally:
@@ -170,11 +168,7 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
 
         csp_policy = self._csp_policy_for_path(request.url.path)
         if self._csp_enabled and csp_policy:
-            csp_header = (
+            csp_header = "Content-Security-Policy-Report-Only" if self._csp_report_only else "Content-Security-Policy"
-                "Content-Security-Policy-Report-Only"
-                if self._csp_report_only
-                else "Content-Security-Policy"
-            )
             response.headers.setdefault(csp_header, csp_policy)
 
         hsts = self._strict_transport_security_value()

app/logging_config.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 
-from datetime import datetime, timezone
 import json
 import logging
 import sys
+from datetime import UTC, datetime
 from typing import Any
 
 from app.logging_context import get_request_id
@@ -102,12 +102,22 @@ class JsonLogFormatter(logging.Formatter):
         if key.lower() in self._redact_fields:
             return "[REDACTED]"
         if isinstance(value, dict):
-            return {nested_key: self._redact_value(nested_key, nested_value) for nested_key, nested_value in value.items()}
+            return {
+                nested_key: self._redact_value(nested_key, nested_value) for nested_key, nested_value in value.items()
+            }
         if isinstance(value, (list, tuple)):
             return [self._redact_value(key, item) for item in value]
         return value
 
 
+_CONSOLE_SHORT_KEYS = {
+    "user_id": "user",
+    "scholar_id": "scholar",
+    "crawl_run_id": "run",
+    "run_id": "run",
+}
+
+
 class ConsoleLogFormatter(logging.Formatter):
     def __init__(self, *, redact_fields: set[str]) -> None:
         super().__init__()
@@ -140,7 +150,8 @@ class ConsoleLogFormatter(logging.Formatter):
         for key in sorted(payload.keys()):
             if key in {"timestamp", "level", "logger", "event", "exception"}:
                 continue
-            parts.append(f"{key}={payload[key]}")
+            display_key = _CONSOLE_SHORT_KEYS.get(key, key)
+            parts.append(f"{display_key}={payload[key]}")
 
         if "exception" in payload:
             parts.append(f"exception={payload['exception']}")
@@ -168,7 +179,7 @@ def _normalize_level(level: str) -> int:
 
 
 def _format_timestamp(created_ts: float) -> str:
-    dt = datetime.fromtimestamp(created_ts, tz=timezone.utc)
+    dt = datetime.fromtimestamp(created_ts, tz=UTC)
     return dt.strftime("%Y-%m-%d %H:%M:%SZ")
 
 

app/logging_context.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 from contextvars import ContextVar
 
-
 _request_id_ctx: ContextVar[str | None] = ContextVar("request_id", default=None)
 
 
@@ -12,4 +11,3 @@ def get_request_id() -> str | None:
 
 def set_request_id(value: str | None) -> None:
     _request_id_ctx.set(value)
-

app/logging_utils.py

@@ -0,0 +1,29 @@
+"""Structured logging utility — eliminates boilerplate across domain services."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+
+def structured_log(
+    logger: logging.Logger,
+    level: str,
+    event: str,
+    /,
+    **fields: Any,
+) -> None:
+    """Emit a structured log entry.
+
+    The event name is passed as the log message. The JsonLogFormatter in
+    logging_config.py extracts it via record.getMessage() when no explicit
+    'event' key exists in extra — so we do NOT duplicate it.
+
+    Usage:
+        structured_log(logger, "info", "ingestion.run_started", user_id=1, scholar_count=5)
+    """
+    fields.pop("metric_name", None)
+    fields.pop("metric_value", None)
+
+    log_method = getattr(logger, level.lower())
+    log_method(event, extra=fields)

app/main.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-from contextlib import asynccontextmanager
 import logging
+from contextlib import asynccontextmanager
 from pathlib import Path
 
 from fastapi import FastAPI, HTTPException
@@ -12,17 +12,16 @@ from starlette.middleware.sessions import SessionMiddleware
 from app.api.errors import register_api_exception_handlers
 from app.api.media import router as media_router
 from app.api.router import router as api_router
-from app.api.runtime_deps import get_ingestion_service, get_scholar_source
+from app.db.session import check_database, close_engine
-from app.db.session import check_database
-from app.db.session import close_engine
 from app.http.middleware import (
     RequestLoggingMiddleware,
     SecurityHeadersMiddleware,
     parse_skip_paths,
 )
 from app.logging_config import configure_logging, parse_redact_fields
+from app.logging_utils import structured_log
 from app.security.csrf import CSRFMiddleware
-from app.services.domains.ingestion.scheduler import SchedulerService
+from app.services.ingestion.scheduler import SchedulerService
 from app.settings import settings
 
 logger = logging.getLogger(__name__)
@@ -50,22 +49,58 @@ scheduler_service = SchedulerService(
 )
 
 
-def _log_startup_build_marker() -> None:
-    logger.info(
-        "app.startup_build_marker",
-        extra={
-            "event": "app.startup_build_marker",
-            "build_marker": BUILD_MARKER,
-            "frontend_enabled": settings.frontend_enabled,
-            "scheduler_enabled": settings.scheduler_enabled,
-            "log_format": settings.log_format,
-        },
-    )
-
-
 @asynccontextmanager
 async def lifespan(_: FastAPI):
-    _log_startup_build_marker()
+    structured_log(
+        logger,
+        "info",
+        "app.startup_build_marker",
+        build_marker=BUILD_MARKER,
+        frontend_enabled=settings.frontend_enabled,
+        scheduler_enabled=settings.scheduler_enabled,
+        log_format=settings.log_format,
+    )
+
+    from sqlalchemy import text
+
+    from app.db.session import get_session_factory
+
+    try:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            await session.execute(
+                text("UPDATE crawl_runs SET status = 'failed' WHERE status::text IN ('running', 'resolving')")
+            )
+            await session.commit()
+            structured_log(logger, "info", "app.startup_orphaned_runs_cleaned")
+    except Exception as exc:
+        structured_log(
+            logger,
+            "error",
+            "app.startup_orphaned_runs_cleanup_failed",
+            error=str(exc),
+        )
+
+    try:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            await session.execute(
+                text(
+                    "UPDATE publication_pdf_jobs SET status = 'queued'"
+                    " WHERE status = 'running'"
+                    " AND (last_attempt_at IS NULL OR last_attempt_at < NOW() - INTERVAL '10 minutes')"
+                )
+            )
+            await session.commit()
+            structured_log(logger, "info", "app.startup_stuck_pdf_jobs_recovered")
+    except Exception as exc:
+        structured_log(
+            logger,
+            "error",
+            "app.startup_stuck_pdf_jobs_recovery_failed",
+            error=str(exc),
+        )
+
     await scheduler_service.start()
     yield
     await scheduler_service.stop()
@@ -101,9 +136,7 @@ app.add_middleware(
     content_security_policy_report_only=settings.security_csp_report_only,
     strict_transport_security_enabled=settings.security_strict_transport_security_enabled,
     strict_transport_security_max_age=settings.security_strict_transport_security_max_age,
-    strict_transport_security_include_subdomains=(
+    strict_transport_security_include_subdomains=(settings.security_strict_transport_security_include_subdomains),
-        settings.security_strict_transport_security_include_subdomains
-    ),
     strict_transport_security_preload=settings.security_strict_transport_security_preload,
 )
 app.include_router(api_router)

app/security/__init__.py

@@ -1,2 +1 @@
 """Security middleware and helpers for scholarr."""
-

app/security/csrf.py

@@ -6,9 +6,11 @@ from urllib.parse import parse_qs
 
 from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.requests import Request
-from starlette.responses import JSONResponse, PlainTextResponse, Response
+from starlette.responses import PlainTextResponse, Response
 from starlette.types import Message
 
+from app.api.responses import error_response
+from app.logging_utils import structured_log
 
 CSRF_SESSION_KEY = "csrf_token"
 CSRF_FORM_FIELD = "csrf_token"
@@ -36,13 +38,12 @@ class CSRFMiddleware(BaseHTTPMiddleware):
 
         session_token = request.session.get(CSRF_SESSION_KEY)
         if not session_token:
-            logger.warning(
+            structured_log(
+                logger,
+                "warning",
                 "csrf.missing_session_token",
-                extra={
+                method=request.method,
-                    "event": "csrf.missing_session_token",
+                path=request.url.path,
-                    "method": request.method,
-                    "path": request.url.path,
-                },
             )
             return self._csrf_error_response(
                 request,
@@ -54,16 +55,15 @@ class CSRFMiddleware(BaseHTTPMiddleware):
         if request_token is None and self._is_form_payload(request):
             body = await request.body()
             request_token = self._token_from_form_body(request, body)
-            request._receive = self._build_receive(body)  # type: ignore[attr-defined]
+            request._receive = self._build_receive(body)
 
         if not request_token or not compare_digest(str(session_token), str(request_token)):
-            logger.warning(
+            structured_log(
+                logger,
+                "warning",
                 "csrf.invalid_token",
-                extra={
+                method=request.method,
-                    "event": "csrf.invalid_token",
+                path=request.url.path,
-                    "method": request.method,
-                    "path": request.url.path,
-                },
             )
             return self._csrf_error_response(
                 request,
@@ -89,7 +89,10 @@ class CSRFMiddleware(BaseHTTPMiddleware):
         content_type = request.headers.get("content-type", "")
         if not content_type.startswith("application/x-www-form-urlencoded"):
             return None
+        try:
             parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True)
+        except (UnicodeDecodeError, ValueError):
+            return None
         values = parsed.get(CSRF_FORM_FIELD)
         if not values:
             return None
@@ -115,19 +118,11 @@ class CSRFMiddleware(BaseHTTPMiddleware):
         message: str,
     ) -> Response:
         if request.url.path.startswith("/api/"):
-            request_state = getattr(request, "state", None)
+            return error_response(
-            request_id = getattr(request_state, "request_id", None) if request_state else None
+                request,
-            return JSONResponse(
-                {
-                    "error": {
-                        "code": code,
-                        "message": message,
-                        "details": None,
-                    },
-                    "meta": {
-                        "request_id": request_id,
-                    },
-                },
                 status_code=403,
+                code=code,
+                message=message,
+                details=None,
             )
         return PlainTextResponse(message, status_code=403)

app/services/__init__.py

@@ -1,2 +1 @@
 """Service layer for scholarr application workflows."""
-

app/services/arxiv/application.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from app.services.arxiv.gateway import (
+    build_arxiv_query,
+    get_arxiv_gateway,
+)
+
+if TYPE_CHECKING:
+    from app.services.publications.types import PublicationListItem, UnreadPublicationItem
+
+
+def _build_arxiv_query(title: str, author_surname: str | None) -> str | None:
+    return build_arxiv_query(title, author_surname)
+
+
+async def discover_arxiv_id_for_publication(
+    *,
+    item: PublicationListItem | UnreadPublicationItem,
+    request_email: str | None = None,
+    timeout_seconds: float | None = None,
+) -> str | None:
+    gateway = get_arxiv_gateway()
+    return await gateway.discover_arxiv_id_for_publication(
+        item=item,
+        request_email=request_email,
+        timeout_seconds=timeout_seconds,
+    )

app/services/arxiv/cache.py

@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+from collections.abc import Awaitable, Callable, Mapping
+from dataclasses import asdict
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from sqlalchemy import delete, func, select
+
+from app.db.models import ArxivQueryCacheEntry
+from app.db.session import get_session_factory
+from app.services.arxiv.constants import ARXIV_CACHE_FINGERPRINT_VERSION
+from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
+
+_INFLIGHT_LOCK = asyncio.Lock()
+_INFLIGHT_FEEDS: dict[str, asyncio.Future[ArxivFeed]] = {}
+
+
+def build_query_fingerprint(*, params: Mapping[str, object]) -> str:
+    canonical = _canonical_cache_payload(params=params)
+    encoded = json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
+    payload = f"{ARXIV_CACHE_FINGERPRINT_VERSION}:{encoded}"
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+async def get_cached_feed(
+    *,
+    query_fingerprint: str,
+    now_utc: datetime | None = None,
+) -> ArxivFeed | None:
+    timestamp = _as_utc(now_utc)
+    session_factory = get_session_factory()
+    async with session_factory() as db_session, db_session.begin():
+        result = await db_session.execute(
+            select(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint == query_fingerprint)
+        )
+        entry = result.scalar_one_or_none()
+        return await _validate_cached_entry(db_session, entry=entry, now_utc=timestamp)
+
+
+async def set_cached_feed(
+    *,
+    query_fingerprint: str,
+    feed: ArxivFeed,
+    ttl_seconds: float,
+    max_entries: int,
+    now_utc: datetime | None = None,
+) -> None:
+    timestamp = _as_utc(now_utc)
+    session_factory = get_session_factory()
+    async with session_factory() as db_session, db_session.begin():
+        await _write_cached_entry(
+            db_session,
+            query_fingerprint=query_fingerprint,
+            feed=feed,
+            ttl_seconds=ttl_seconds,
+            max_entries=max_entries,
+            now_utc=timestamp,
+        )
+
+
+async def run_with_inflight_dedupe(
+    *,
+    query_fingerprint: str,
+    fetch_feed: Callable[[], Awaitable[ArxivFeed]],
+) -> ArxivFeed:
+    future, is_owner = await _reserve_inflight_future(query_fingerprint=query_fingerprint)
+    if not is_owner:
+        return await asyncio.shield(future)
+    try:
+        result = await fetch_feed()
+    except Exception as exc:
+        _complete_future(future, error=exc)
+        raise
+    finally:
+        await _release_inflight_future(query_fingerprint=query_fingerprint, future=future)
+    _complete_future(future, result=result)
+    return result
+
+
+def _canonical_cache_payload(*, params: Mapping[str, object]) -> dict[str, object]:
+    payload: dict[str, object] = {}
+    for key in sorted(params.keys()):
+        payload[str(key)] = _normalize_param_value(str(key), params[key])
+    return payload
+
+
+def _normalize_param_value(key: str, value: object) -> object:
+    if key == "search_query":
+        return _normalize_search_query(str(value or ""))
+    if key == "id_list":
+        return _normalize_id_list(str(value or ""))
+    if isinstance(value, str):
+        return " ".join(value.strip().split())
+    if isinstance(value, (int, float, bool)) or value is None:
+        return value
+    return str(value).strip()
+
+
+def _normalize_search_query(value: str) -> str:
+    return " ".join(value.strip().lower().split())
+
+
+def _normalize_id_list(value: str) -> str:
+    normalized = [item.strip().lower() for item in value.split(",") if item.strip()]
+    return ",".join(sorted(normalized))
+
+
+async def _validate_cached_entry(
+    db_session,
+    *,
+    entry: ArxivQueryCacheEntry | None,
+    now_utc: datetime,
+) -> ArxivFeed | None:
+    if entry is None:
+        return None
+    if _as_utc(entry.expires_at) <= now_utc:
+        await db_session.delete(entry)
+        return None
+    parsed = _deserialize_feed(entry.payload)
+    if parsed is None:
+        await db_session.delete(entry)
+        return None
+    return parsed
+
+
+async def _write_cached_entry(
+    db_session,
+    *,
+    query_fingerprint: str,
+    feed: ArxivFeed,
+    ttl_seconds: float,
+    max_entries: int,
+    now_utc: datetime,
+) -> None:
+    ttl = max(float(ttl_seconds), 0.0)
+    result = await db_session.execute(
+        select(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint == query_fingerprint)
+    )
+    existing = result.scalar_one_or_none()
+    if ttl <= 0.0:
+        if existing is not None:
+            await db_session.delete(existing)
+        return
+    expires_at = now_utc + timedelta(seconds=ttl)
+    payload = _serialize_feed(feed)
+    if existing is None:
+        db_session.add(
+            ArxivQueryCacheEntry(
+                query_fingerprint=query_fingerprint,
+                payload=payload,
+                expires_at=expires_at,
+                cached_at=now_utc,
+                updated_at=now_utc,
+            )
+        )
+    else:
+        existing.payload = payload
+        existing.expires_at = expires_at
+        existing.cached_at = now_utc
+        existing.updated_at = now_utc
+    await _prune_cache_entries(db_session, now_utc=now_utc, max_entries=max_entries)
+
+
+async def _prune_cache_entries(
+    db_session,
+    *,
+    now_utc: datetime,
+    max_entries: int,
+) -> None:
+    await db_session.execute(delete(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.expires_at <= now_utc))
+    bounded_max_entries = int(max_entries)
+    if bounded_max_entries <= 0:
+        return
+    count_result = await db_session.execute(select(func.count()).select_from(ArxivQueryCacheEntry))
+    entry_count = int(count_result.scalar_one() or 0)
+    overflow = max(0, entry_count - bounded_max_entries)
+    if overflow <= 0:
+        return
+    stale_result = await db_session.execute(
+        select(ArxivQueryCacheEntry.query_fingerprint).order_by(ArxivQueryCacheEntry.cached_at.asc()).limit(overflow)
+    )
+    stale_keys = [str(row[0]) for row in stale_result.all()]
+    if stale_keys:
+        await db_session.execute(
+            delete(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint.in_(stale_keys))
+        )
+
+
+def _serialize_feed(feed: ArxivFeed) -> dict[str, Any]:
+    return asdict(feed)
+
+
+def _deserialize_feed(payload: object) -> ArxivFeed | None:
+    if not isinstance(payload, dict):
+        return None
+    entries_payload = payload.get("entries")
+    opensearch_payload = payload.get("opensearch")
+    if not isinstance(entries_payload, list):
+        return None
+    entries: list[ArxivEntry] = []
+    for value in entries_payload:
+        entry = _deserialize_entry(value)
+        if entry is None:
+            return None
+        entries.append(entry)
+    opensearch = _deserialize_opensearch(opensearch_payload)
+    if opensearch is None:
+        return None
+    return ArxivFeed(entries=entries, opensearch=opensearch)
+
+
+def _deserialize_entry(value: object) -> ArxivEntry | None:
+    if not isinstance(value, dict):
+        return None
+    try:
+        return ArxivEntry(
+            entry_id_url=str(value["entry_id_url"]),
+            arxiv_id=_as_optional_string(value.get("arxiv_id")),
+            title=str(value["title"]),
+            summary=str(value["summary"]),
+            published=_as_optional_string(value.get("published")),
+            updated=_as_optional_string(value.get("updated")),
+            authors=_as_string_list(value.get("authors")),
+            links=_as_string_list(value.get("links")),
+            categories=_as_string_list(value.get("categories")),
+            primary_category=_as_optional_string(value.get("primary_category")),
+        )
+    except KeyError:
+        return None
+
+
+def _deserialize_opensearch(value: object) -> ArxivOpenSearchMeta | None:
+    if not isinstance(value, dict):
+        return None
+    try:
+        return ArxivOpenSearchMeta(
+            total_results=int(value.get("total_results", 0)),
+            start_index=int(value.get("start_index", 0)),
+            items_per_page=int(value.get("items_per_page", 0)),
+        )
+    except (TypeError, ValueError):
+        return None
+
+
+def _as_optional_string(value: object) -> str | None:
+    if value is None:
+        return None
+    normalized = str(value).strip()
+    return normalized or None
+
+
+def _as_string_list(value: object) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(item) for item in value]
+
+
+def _as_utc(value: datetime | None) -> datetime:
+    if value is None:
+        return datetime.now(UTC)
+    if value.tzinfo is None:
+        return value.replace(tzinfo=UTC)
+    return value
+
+
+async def _reserve_inflight_future(
+    *,
+    query_fingerprint: str,
+) -> tuple[asyncio.Future[ArxivFeed], bool]:
+    async with _INFLIGHT_LOCK:
+        existing = _INFLIGHT_FEEDS.get(query_fingerprint)
+        if existing is not None:
+            return existing, False
+        loop = asyncio.get_running_loop()
+        created = loop.create_future()
+        created.add_done_callback(_consume_unretrieved_future_exception)
+        _INFLIGHT_FEEDS[query_fingerprint] = created
+        return created, True
+
+
+async def _release_inflight_future(
+    *,
+    query_fingerprint: str,
+    future: asyncio.Future[ArxivFeed],
+) -> None:
+    async with _INFLIGHT_LOCK:
+        current = _INFLIGHT_FEEDS.get(query_fingerprint)
+        if current is future:
+            _INFLIGHT_FEEDS.pop(query_fingerprint, None)
+
+
+def _complete_future(
+    future: asyncio.Future[ArxivFeed],
+    *,
+    result: ArxivFeed | None = None,
+    error: Exception | None = None,
+) -> None:
+    if future.done():
+        return
+    if error is not None:
+        future.set_exception(error)
+        return
+    if result is None:
+        raise RuntimeError("in-flight future completion requires result or error")
+    future.set_result(result)
+
+
+def _consume_unretrieved_future_exception(future: asyncio.Future[ArxivFeed]) -> None:
+    if future.cancelled():
+        return
+    try:
+        _ = future.exception()
+    except Exception:
+        return

app/services/arxiv/client.py

@@ -0,0 +1,278 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import Awaitable, Callable
+
+import httpx
+
+from app.logging_utils import structured_log
+from app.services.arxiv.cache import (
+    build_query_fingerprint,
+    get_cached_feed,
+    run_with_inflight_dedupe,
+    set_cached_feed,
+)
+from app.services.arxiv.constants import (
+    ARXIV_SOURCE_PATH_LOOKUP_IDS,
+    ARXIV_SOURCE_PATH_SEARCH,
+    ARXIV_SOURCE_PATH_UNKNOWN,
+)
+from app.services.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
+from app.services.arxiv.parser import parse_arxiv_feed
+from app.services.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
+from app.services.arxiv.types import ArxivFeed
+from app.settings import settings
+
+_ARXIV_API_URL = "https://export.arxiv.org/api/query"
+_ARXIV_QUERY_START = 0
+_ARXIV_MAX_RESULTS_LIMIT = 30_000
+_ARXIV_SORT_BY_ALLOWED = {"relevance", "lastUpdatedDate", "submittedDate"}
+_ARXIV_SORT_ORDER_ALLOWED = {"ascending", "descending"}
+_FALLBACK_CONTACT_EMAIL = "unknown@example.com"
+
+ArxivRequestFn = Callable[..., Awaitable[httpx.Response]]
+logger = logging.getLogger(__name__)
+
+
+class ArxivClient:
+    def __init__(
+        self,
+        *,
+        request_fn: ArxivRequestFn | None = None,
+        cache_enabled: bool | None = None,
+    ) -> None:
+        self._request_fn = request_fn or _request_arxiv_feed
+        self._cache_enabled = _resolve_cache_enabled(
+            cache_enabled=cache_enabled,
+            request_fn=request_fn,
+        )
+        self._cache_ttl_seconds = _cache_ttl_seconds()
+        self._cache_max_entries = _cache_max_entries()
+
+    async def search(
+        self,
+        *,
+        query: str,
+        start: int = _ARXIV_QUERY_START,
+        max_results: int | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        request_email: str | None = None,
+        timeout_seconds: float | None = None,
+    ) -> ArxivFeed:
+        params = _search_params(
+            query=query,
+            start=start,
+            max_results=max_results,
+            sort_by=sort_by,
+            sort_order=sort_order,
+        )
+        return await self._fetch_feed(
+            params=params,
+            request_email=request_email,
+            timeout_seconds=timeout_seconds,
+            source_path=ARXIV_SOURCE_PATH_SEARCH,
+        )
+
+    async def lookup_ids(
+        self,
+        *,
+        id_list: list[str],
+        start: int = _ARXIV_QUERY_START,
+        max_results: int | None = None,
+        request_email: str | None = None,
+        timeout_seconds: float | None = None,
+    ) -> ArxivFeed:
+        params = _lookup_params(id_list=id_list, start=start, max_results=max_results)
+        return await self._fetch_feed(
+            params=params,
+            request_email=request_email,
+            timeout_seconds=timeout_seconds,
+            source_path=ARXIV_SOURCE_PATH_LOOKUP_IDS,
+        )
+
+    async def _fetch_feed(
+        self,
+        *,
+        params: dict[str, object],
+        request_email: str | None,
+        timeout_seconds: float | None,
+        source_path: str,
+    ) -> ArxivFeed:
+        query_fingerprint = build_query_fingerprint(params=params)
+        if self._cache_enabled:
+            cached = await get_cached_feed(query_fingerprint=query_fingerprint)
+            if cached is not None:
+                structured_log(
+                    logger, "info", "arxiv.cache_hit", query_fingerprint=query_fingerprint, source_path=source_path
+                )
+                return cached
+            structured_log(
+                logger, "info", "arxiv.cache_miss", query_fingerprint=query_fingerprint, source_path=source_path
+            )
+        return await run_with_inflight_dedupe(
+            query_fingerprint=query_fingerprint,
+            fetch_feed=lambda: self._fetch_live_feed(
+                params=params,
+                request_email=request_email,
+                timeout_seconds=timeout_seconds,
+                query_fingerprint=query_fingerprint,
+            ),
+        )
+
+    async def _fetch_live_feed(
+        self,
+        *,
+        params: dict[str, object],
+        request_email: str | None,
+        timeout_seconds: float | None,
+        query_fingerprint: str,
+    ) -> ArxivFeed:
+        response = await self._request_fn(
+            params=params,
+            request_email=request_email,
+            timeout_seconds=timeout_seconds,
+        )
+        response.raise_for_status()
+        feed = parse_arxiv_feed(response.text)
+        if self._cache_enabled:
+            await set_cached_feed(
+                query_fingerprint=query_fingerprint,
+                feed=feed,
+                ttl_seconds=self._cache_ttl_seconds,
+                max_entries=self._cache_max_entries,
+            )
+        return feed
+
+
+def _search_params(
+    *,
+    query: str,
+    start: int,
+    max_results: int | None,
+    sort_by: str | None,
+    sort_order: str | None,
+) -> dict[str, object]:
+    clean_query = query.strip()
+    if not clean_query:
+        raise ArxivClientValidationError("search query must not be empty")
+    params: dict[str, object] = {
+        "search_query": clean_query,
+        "start": _validate_start(start),
+        "max_results": _validate_max_results(max_results),
+    }
+    if sort_by is not None:
+        params["sortBy"] = _validate_sort_by(sort_by)
+    if sort_order is not None:
+        params["sortOrder"] = _validate_sort_order(sort_order)
+    return params
+
+
+def _lookup_params(*, id_list: list[str], start: int, max_results: int | None) -> dict[str, object]:
+    normalized_ids = [value.strip() for value in id_list if value and value.strip()]
+    if not normalized_ids:
+        raise ArxivClientValidationError("id_list must include at least one id")
+    return {
+        "id_list": ",".join(normalized_ids),
+        "start": _validate_start(start),
+        "max_results": _validate_max_results(max_results),
+    }
+
+
+def _validate_start(value: int) -> int:
+    start = int(value)
+    if start < 0:
+        raise ArxivClientValidationError("start must be >= 0")
+    return start
+
+
+def _validate_max_results(value: int | None) -> int:
+    if value is None:
+        default_value = int(settings.arxiv_default_max_results)
+        return max(default_value, 1)
+    parsed = int(value)
+    if parsed < 1:
+        raise ArxivClientValidationError("max_results must be >= 1")
+    if parsed > _ARXIV_MAX_RESULTS_LIMIT:
+        raise ArxivClientValidationError(f"max_results must be <= {_ARXIV_MAX_RESULTS_LIMIT}")
+    return parsed
+
+
+def _validate_sort_by(value: str) -> str:
+    if value not in _ARXIV_SORT_BY_ALLOWED:
+        raise ArxivClientValidationError(f"sort_by must be one of: {sorted(_ARXIV_SORT_BY_ALLOWED)!r}")
+    return value
+
+
+def _validate_sort_order(value: str) -> str:
+    if value not in _ARXIV_SORT_ORDER_ALLOWED:
+        raise ArxivClientValidationError(f"sort_order must be one of: {sorted(_ARXIV_SORT_ORDER_ALLOWED)!r}")
+    return value
+
+
+async def _request_arxiv_feed(
+    *,
+    params: dict[str, object],
+    request_email: str | None,
+    timeout_seconds: float | None,
+) -> httpx.Response:
+    source_path = _source_path_from_params(params)
+    cooldown_status = await get_arxiv_cooldown_status()
+    if cooldown_status.is_active:
+        structured_log(
+            logger,
+            "warning",
+            "arxiv.request_skipped_cooldown",
+            source_path=source_path,
+            cooldown_remaining_seconds=float(cooldown_status.remaining_seconds),
+        )
+        raise ArxivRateLimitError(f"arXiv global cooldown active ({cooldown_status.remaining_seconds:.0f}s remaining)")
+
+    async def _fetch() -> httpx.Response:
+        timeout_value = _timeout_seconds(timeout_seconds)
+        headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{_contact_email(request_email)})"}
+        async with httpx.AsyncClient(timeout=timeout_value, follow_redirects=True, headers=headers) as client:
+            return await client.get(_ARXIV_API_URL, params=params)  # type: ignore[arg-type]
+
+    return await run_with_global_arxiv_limit(
+        fetch=_fetch,
+        source_path=source_path,
+    )
+
+
+def _timeout_seconds(timeout_seconds: float | None) -> float:
+    if timeout_seconds is not None:
+        return max(float(timeout_seconds), 0.5)
+    return max(float(settings.arxiv_timeout_seconds), 0.5)
+
+
+def _contact_email(request_email: str | None) -> str:
+    return request_email or settings.arxiv_mailto or settings.crossref_api_mailto or _FALLBACK_CONTACT_EMAIL
+
+
+def _resolve_cache_enabled(
+    *,
+    cache_enabled: bool | None,
+    request_fn: ArxivRequestFn | None,
+) -> bool:
+    if cache_enabled is not None:
+        return bool(cache_enabled)
+    if request_fn is not None:
+        return False
+    return _cache_ttl_seconds() > 0.0
+
+
+def _cache_ttl_seconds() -> float:
+    return max(float(settings.arxiv_cache_ttl_seconds), 0.0)
+
+
+def _cache_max_entries() -> int:
+    return max(int(settings.arxiv_cache_max_entries), 0)
+
+
+def _source_path_from_params(params: dict[str, object]) -> str:
+    if "search_query" in params:
+        return ARXIV_SOURCE_PATH_SEARCH
+    if "id_list" in params:
+        return ARXIV_SOURCE_PATH_LOOKUP_IDS
+    return ARXIV_SOURCE_PATH_UNKNOWN

app/services/arxiv/constants.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+ARXIV_RUNTIME_STATE_KEY = "global"
+ARXIV_RATE_LIMIT_LOCK_NAMESPACE = 91_100
+ARXIV_RATE_LIMIT_LOCK_KEY = 1
+ARXIV_SOURCE_PATH_SEARCH = "search"
+ARXIV_SOURCE_PATH_LOOKUP_IDS = "lookup_ids"
+ARXIV_SOURCE_PATH_UNKNOWN = "unknown"
+ARXIV_CACHE_FINGERPRINT_VERSION = "v1"
+ARXIV_TITLE_TOKEN_MIN_LENGTH = 3
+ARXIV_TITLE_MIN_TOKENS = 3
+ARXIV_TITLE_MIN_ALPHA_TOKENS = 2
+ARXIV_STRONG_IDENTIFIER_CONFIDENCE = 0.9

app/services/arxiv/errors.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+
+class ArxivRateLimitError(Exception):
+    """arXiv returned 429 or cooldown is active."""
+
+
+class ArxivClientValidationError(ValueError):
+    """arXiv client inputs are invalid."""
+
+
+class ArxivParseError(ValueError):
+    """arXiv API payload could not be parsed."""

app/services/arxiv/gateway.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import logging
+import re
+import unicodedata
+from typing import TYPE_CHECKING, Protocol
+
+from app.logging_utils import structured_log
+from app.services.arxiv.client import ArxivClient
+from app.services.arxiv.errors import ArxivRateLimitError
+from app.services.arxiv.types import ArxivFeed
+from app.settings import settings
+
+if TYPE_CHECKING:
+    from app.services.publications.types import PublicationListItem, UnreadPublicationItem
+
+logger = logging.getLogger(__name__)
+
+_default_gateway: ArxivGateway | None = None
+_MOJIBAKE_HINT_RE = re.compile(r"[ÃÂâ]")
+_NON_ALNUM_RE = re.compile(r"[^\w\s]+", re.UNICODE)
+_WHITESPACE_RE = re.compile(r"\s+")
+
+
+class ArxivGateway(Protocol):
+    async def discover_arxiv_id_for_publication(
+        self,
+        *,
+        item: PublicationListItem | UnreadPublicationItem,
+        request_email: str | None = None,
+        timeout_seconds: float | None = None,
+        max_results: int | None = None,
+    ) -> str | None: ...
+
+
+def build_arxiv_query(title: str, author_surname: str | None) -> str | None:
+    parts: list[str] = []
+    if title:
+        clean_title = _normalize_query_title(title)
+        if clean_title:
+            parts.append(f'ti:"{clean_title}"')
+    if author_surname:
+        clean_author = _normalize_query_title(author_surname)
+        if clean_author:
+            parts.append(f'au:"{clean_author}"')
+    if not parts:
+        return None
+    return " AND ".join(parts)
+
+
+def _normalize_query_title(value: str) -> str:
+    repaired = _repair_mojibake(value.strip())
+    normalized = unicodedata.normalize("NFKC", repaired)
+    stripped = _NON_ALNUM_RE.sub(" ", _MOJIBAKE_HINT_RE.sub(" ", normalized))
+    return _WHITESPACE_RE.sub(" ", stripped).strip()
+
+
+def _repair_mojibake(value: str) -> str:
+    if not value or not _MOJIBAKE_HINT_RE.search(value):
+        return value
+    try:
+        repaired = value.encode("latin1").decode("utf-8")
+    except UnicodeError:
+        return value
+    return repaired if _mojibake_score(repaired) < _mojibake_score(value) else value
+
+
+def _mojibake_score(value: str) -> int:
+    return len(_MOJIBAKE_HINT_RE.findall(value))
+
+
+def get_arxiv_gateway() -> ArxivGateway:
+    global _default_gateway
+    if _default_gateway is None:
+        _default_gateway = HttpArxivGateway()
+    return _default_gateway
+
+
+def set_arxiv_gateway(gateway: ArxivGateway | None) -> ArxivGateway | None:
+    global _default_gateway
+    previous = _default_gateway
+    _default_gateway = gateway
+    return previous
+
+
+class HttpArxivGateway:
+    def __init__(self, *, client: ArxivClient | None = None) -> None:
+        self._client = client or ArxivClient()
+
+    async def discover_arxiv_id_for_publication(
+        self,
+        *,
+        item: PublicationListItem | UnreadPublicationItem,
+        request_email: str | None = None,
+        timeout_seconds: float | None = None,
+        max_results: int | None = None,
+    ) -> str | None:
+        if not settings.arxiv_enabled:
+            return None
+        query = _query_for_item(item)
+        if query is None:
+            return None
+
+        try:
+            result = await self._client.search(
+                query=query,
+                start=0,
+                request_email=request_email,
+                timeout_seconds=timeout_seconds,
+                max_results=max_results,
+            )
+            return _first_discovered_id(result)
+        except ArxivRateLimitError:
+            raise
+        except Exception as exc:
+            structured_log(logger, "debug", "arxiv.query_failed", error=str(exc))
+            return None
+
+
+def _query_for_item(item: PublicationListItem | UnreadPublicationItem) -> str | None:
+    title = (item.title or "").strip()
+    if not title:
+        return None
+    author_surname = _author_surname(item.scholar_label)
+    return build_arxiv_query(title, author_surname)
+
+
+def _author_surname(scholar_label: str | None) -> str | None:
+    if not scholar_label:
+        return None
+    tokens = [token for token in scholar_label.strip().split() if token]
+    if not tokens:
+        return None
+    return tokens[-1].lower()
+
+
+def _first_discovered_id(result: ArxivFeed) -> str | None:
+    for entry in result.entries:
+        if entry.arxiv_id:
+            structured_log(logger, "debug", "arxiv.id_discovered", arxiv_id=entry.arxiv_id)
+            return entry.arxiv_id
+    return None

app/services/arxiv/guards.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+from app.services.arxiv.constants import (
+    ARXIV_STRONG_IDENTIFIER_CONFIDENCE,
+    ARXIV_TITLE_MIN_ALPHA_TOKENS,
+    ARXIV_TITLE_MIN_TOKENS,
+    ARXIV_TITLE_TOKEN_MIN_LENGTH,
+)
+from app.services.doi.normalize import normalize_doi
+from app.services.publication_identifiers.normalize import normalize_arxiv_id
+
+if TYPE_CHECKING:
+    from app.services.publications.types import PublicationListItem, UnreadPublicationItem
+
+_TITLE_TOKEN_RE = re.compile(r"[a-z0-9]+")
+
+
+def arxiv_skip_reason_for_item(
+    *,
+    item: PublicationListItem | UnreadPublicationItem,
+    has_strong_doi: bool = False,
+    has_existing_arxiv: bool = False,
+) -> str | None:
+    if has_existing_arxiv or _has_arxiv_identifier_evidence(item):
+        return "arxiv_identifier_present"
+    if has_strong_doi or _has_strong_doi_evidence(item):
+        return "strong_doi_present"
+    if not _title_passes_quality_guard(item.title):
+        return "title_quality_below_threshold"
+    return None
+
+
+def _has_arxiv_identifier_evidence(item: PublicationListItem | UnreadPublicationItem) -> bool:
+    if _display_identifier_matches(item, expected_kind="arxiv"):
+        return True
+    return _has_normalized_identifier(item, normalizer=normalize_arxiv_id)
+
+
+def _has_strong_doi_evidence(item: PublicationListItem | UnreadPublicationItem) -> bool:
+    if _display_identifier_matches(item, expected_kind="doi"):
+        return True
+    return _has_normalized_identifier(item, normalizer=normalize_doi)
+
+
+def _display_identifier_matches(
+    item: PublicationListItem | UnreadPublicationItem,
+    *,
+    expected_kind: str,
+) -> bool:
+    display = getattr(item, "display_identifier", None)
+    if display is None:
+        return False
+    if str(display.kind).lower() != expected_kind:
+        return False
+    return float(display.confidence_score) >= ARXIV_STRONG_IDENTIFIER_CONFIDENCE
+
+
+def _has_normalized_identifier(
+    item: PublicationListItem | UnreadPublicationItem,
+    *,
+    normalizer,
+) -> bool:
+    if normalizer(item.pub_url):
+        return True
+    return normalizer(item.pdf_url) is not None
+
+
+def _title_passes_quality_guard(title: str | None) -> bool:
+    tokens = _normalized_tokens(title or "")
+    if len(tokens) < ARXIV_TITLE_MIN_TOKENS:
+        return False
+    alpha_tokens = [token for token in tokens if _is_alpha_token(token)]
+    return len(alpha_tokens) >= ARXIV_TITLE_MIN_ALPHA_TOKENS
+
+
+def _normalized_tokens(value: str) -> list[str]:
+    return [token for token in _TITLE_TOKEN_RE.findall(value.lower()) if token]
+
+
+def _is_alpha_token(token: str) -> bool:
+    if len(token) < ARXIV_TITLE_TOKEN_MIN_LENGTH:
+        return False
+    return any(char.isalpha() for char in token)

app/services/arxiv/parser.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+
+from app.services.arxiv.errors import ArxivParseError
+from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
+from app.services.publication_identifiers.normalize import normalize_arxiv_id
+
+_NAMESPACES = {
+    "atom": "http://www.w3.org/2005/Atom",
+    "opensearch": "http://a9.com/-/spec/opensearch/1.1/",
+    "arxiv": "http://arxiv.org/schemas/atom",
+}
+
+
+def parse_arxiv_feed(payload: str) -> ArxivFeed:
+    root = _parse_xml_root(payload)
+    opensearch = ArxivOpenSearchMeta(
+        total_results=_opensearch_int(root, "opensearch:totalResults"),
+        start_index=_opensearch_int(root, "opensearch:startIndex"),
+        items_per_page=_opensearch_int(root, "opensearch:itemsPerPage"),
+    )
+    entries = [_parse_entry(entry_elem) for entry_elem in root.findall("atom:entry", _NAMESPACES)]
+    return ArxivFeed(entries=entries, opensearch=opensearch)
+
+
+def _parse_xml_root(payload: str) -> ET.Element:
+    try:
+        return ET.fromstring(payload)
+    except ET.ParseError as exc:
+        raise ArxivParseError(f"Invalid arXiv XML payload: {exc}") from exc
+
+
+def _opensearch_int(root: ET.Element, path: str) -> int:
+    text = _optional_text(root, path)
+    if text is None:
+        return 0
+    try:
+        return int(text.strip())
+    except ValueError as exc:
+        raise ArxivParseError(f"Invalid integer value at {path}: {text!r}") from exc
+
+
+def _parse_entry(entry_elem: ET.Element) -> ArxivEntry:
+    entry_id_url = _required_text(entry_elem, "atom:id").strip()
+    arxiv_id = normalize_arxiv_id(entry_id_url)
+    title = _required_text(entry_elem, "atom:title").strip()
+    summary = (_optional_text(entry_elem, "atom:summary") or "").strip()
+    published = _optional_text(entry_elem, "atom:published")
+    updated = _optional_text(entry_elem, "atom:updated")
+    return ArxivEntry(
+        entry_id_url=entry_id_url,
+        arxiv_id=arxiv_id,
+        title=title,
+        summary=summary,
+        published=published,
+        updated=updated,
+        authors=_authors(entry_elem),
+        links=_links(entry_elem),
+        categories=_categories(entry_elem),
+        primary_category=_primary_category(entry_elem),
+    )
+
+
+def _required_text(elem: ET.Element, path: str) -> str:
+    text = _optional_text(elem, path)
+    if text is None or not text.strip():
+        raise ArxivParseError(f"Missing required field: {path}")
+    return text
+
+
+def _optional_text(elem: ET.Element, path: str) -> str | None:
+    node = elem.find(path, _NAMESPACES)
+    if node is None or node.text is None:
+        return None
+    return str(node.text)
+
+
+def _authors(entry_elem: ET.Element) -> list[str]:
+    authors: list[str] = []
+    for author in entry_elem.findall("atom:author", _NAMESPACES):
+        name = _optional_text(author, "atom:name")
+        if name:
+            authors.append(name.strip())
+    return authors
+
+
+def _links(entry_elem: ET.Element) -> list[str]:
+    values: list[str] = []
+    for link in entry_elem.findall("atom:link", _NAMESPACES):
+        href = str(link.attrib.get("href") or "").strip()
+        if href:
+            values.append(href)
+    return values
+
+
+def _categories(entry_elem: ET.Element) -> list[str]:
+    values: list[str] = []
+    for cat in entry_elem.findall("atom:category", _NAMESPACES):
+        term = str(cat.attrib.get("term") or "").strip()
+        if term:
+            values.append(term)
+    return values
+
+
+def _primary_category(entry_elem: ET.Element) -> str | None:
+    node = entry_elem.find("arxiv:primary_category", _NAMESPACES)
+    if node is None:
+        return None
+    value = str(node.attrib.get("term") or "").strip()
+    return value or None

app/services/arxiv/rate_limit.py

@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+
+import httpx
+from sqlalchemy import select, text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import ArxivRuntimeState
+from app.db.session import get_session_factory
+from app.logging_utils import structured_log
+from app.services.arxiv.constants import (
+    ARXIV_RATE_LIMIT_LOCK_KEY,
+    ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
+    ARXIV_RUNTIME_STATE_KEY,
+    ARXIV_SOURCE_PATH_UNKNOWN,
+)
+from app.services.arxiv.errors import ArxivRateLimitError
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class ArxivCooldownStatus:
+    is_active: bool
+    remaining_seconds: float
+    cooldown_until: datetime | None
+
+
+async def run_with_global_arxiv_limit(
+    *,
+    fetch: Callable[[], Awaitable[httpx.Response]],
+    source_path: str = ARXIV_SOURCE_PATH_UNKNOWN,
+) -> httpx.Response:
+    response, hit_rate_limit = await _run_serialized_fetch(
+        fetch=fetch,
+        source_path=source_path,
+    )
+    if hit_rate_limit:
+        raise ArxivRateLimitError("arXiv rate limit hit (429) — stopping batch")
+    return response
+
+
+async def get_arxiv_cooldown_status(*, now_utc: datetime | None = None) -> ArxivCooldownStatus:
+    timestamp = _normalize_datetime(now_utc) or datetime.now(UTC)
+    session_factory = get_session_factory()
+    async with session_factory() as db_session:
+        result = await db_session.execute(
+            select(ArxivRuntimeState.cooldown_until).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY)
+        )
+        cooldown_until = _normalize_datetime(result.scalar_one_or_none())
+    remaining_seconds = _cooldown_remaining_seconds(cooldown_until, now_utc=timestamp)
+    return ArxivCooldownStatus(
+        is_active=remaining_seconds > 0.0,
+        remaining_seconds=float(remaining_seconds),
+        cooldown_until=cooldown_until,
+    )
+
+
+async def _run_serialized_fetch(
+    *,
+    fetch: Callable[[], Awaitable[httpx.Response]],
+    source_path: str,
+) -> tuple[httpx.Response, bool]:
+    session_factory = get_session_factory()
+    async with session_factory() as lock_session:
+        await _acquire_arxiv_lock(lock_session)
+        try:
+            async with session_factory() as db_session, db_session.begin():
+                runtime_state = await _load_runtime_state_for_update(db_session)
+                wait_seconds = await _wait_for_allowed_slot_or_raise(
+                    runtime_state,
+                    source_path=source_path,
+                )
+                runtime_state.next_allowed_at = datetime.now(UTC) + timedelta(seconds=_min_interval_seconds())
+
+            if wait_seconds > 0:
+                await asyncio.sleep(wait_seconds)
+            response = await fetch()
+
+            async with session_factory() as db_session, db_session.begin():
+                runtime_state = await _load_runtime_state_for_update(db_session)
+                hit_rate_limit = _record_post_response_state(
+                    runtime_state,
+                    response_status=int(response.status_code),
+                    source_path=source_path,
+                )
+                cooldown_until_value = runtime_state.cooldown_until
+        finally:
+            await _release_arxiv_lock(lock_session)
+    structured_log(
+        logger,
+        "info",
+        "arxiv.request_completed",
+        status_code=int(response.status_code),
+        wait_seconds=wait_seconds,
+        cooldown_remaining_seconds=_cooldown_remaining_seconds(
+            cooldown_until_value, now_utc=datetime.now(UTC)
+        ),
+        source_path=source_path,
+    )
+    return response, hit_rate_limit
+
+
+async def _acquire_arxiv_lock(db_session: AsyncSession) -> None:
+    await db_session.execute(
+        text("SELECT pg_advisory_lock(:namespace, :lock_key)"),
+        {
+            "namespace": ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
+            "lock_key": ARXIV_RATE_LIMIT_LOCK_KEY,
+        },
+    )
+
+
+async def _release_arxiv_lock(db_session: AsyncSession) -> None:
+    await db_session.execute(
+        text("SELECT pg_advisory_unlock(:namespace, :lock_key)"),
+        {
+            "namespace": ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
+            "lock_key": ARXIV_RATE_LIMIT_LOCK_KEY,
+        },
+    )
+
+
+async def _load_runtime_state_for_update(db_session: AsyncSession) -> ArxivRuntimeState:
+    result = await db_session.execute(
+        select(ArxivRuntimeState).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY).with_for_update()
+    )
+    state = result.scalar_one_or_none()
+    if state is not None:
+        return state
+    state = ArxivRuntimeState(state_key=ARXIV_RUNTIME_STATE_KEY)
+    db_session.add(state)
+    await db_session.flush()
+    return state
+
+
+async def _wait_for_allowed_slot_or_raise(
+    runtime_state: ArxivRuntimeState,
+    *,
+    source_path: str,
+) -> float:
+    now_utc = datetime.now(UTC)
+    cooldown_seconds = _cooldown_remaining_seconds(runtime_state.cooldown_until, now_utc=now_utc)
+    if cooldown_seconds > 0:
+        structured_log(
+            logger,
+            "info",
+            "arxiv.request_scheduled",
+            wait_seconds=0.0,
+            source_path=source_path,
+            cooldown_remaining_seconds=cooldown_seconds,
+        )
+        raise ArxivRateLimitError(f"arXiv global cooldown active ({cooldown_seconds:.0f}s remaining)")
+    wait_seconds = _next_allowed_wait_seconds(runtime_state.next_allowed_at, now_utc=now_utc)
+    structured_log(
+        logger,
+        "info",
+        "arxiv.request_scheduled",
+        wait_seconds=wait_seconds,
+        source_path=source_path,
+        cooldown_remaining_seconds=0.0,
+    )
+    return wait_seconds
+
+
+def _record_post_response_state(
+    runtime_state: ArxivRuntimeState,
+    *,
+    response_status: int,
+    source_path: str,
+) -> bool:
+    now_utc = datetime.now(UTC)
+    runtime_state.next_allowed_at = now_utc + timedelta(seconds=_min_interval_seconds())
+    if response_status == 429:
+        cooldown_seconds = _cooldown_seconds()
+        runtime_state.cooldown_until = now_utc + timedelta(seconds=cooldown_seconds)
+        structured_log(
+            logger,
+            "warning",
+            "arxiv.cooldown_activated",
+            cooldown_remaining_seconds=cooldown_seconds,
+            source_path=source_path,
+        )
+        return True
+    if _cooldown_remaining_seconds(runtime_state.cooldown_until, now_utc=now_utc) <= 0:
+        runtime_state.cooldown_until = None
+    return False
+
+
+def _cooldown_remaining_seconds(cooldown_until: datetime | None, *, now_utc: datetime) -> float:
+    bounded = _normalize_datetime(cooldown_until)
+    if bounded is None:
+        return 0.0
+    return max((bounded - now_utc).total_seconds(), 0.0)
+
+
+def _next_allowed_wait_seconds(next_allowed_at: datetime | None, *, now_utc: datetime) -> float:
+    bounded = _normalize_datetime(next_allowed_at)
+    if bounded is None:
+        return 0.0
+    return max((bounded - now_utc).total_seconds(), 0.0)
+
+
+def _normalize_datetime(value: datetime | None) -> datetime | None:
+    if value is None:
+        return None
+    if value.tzinfo is None:
+        return value.replace(tzinfo=UTC)
+    return value
+
+
+def _min_interval_seconds() -> float:
+    return max(float(settings.arxiv_min_interval_seconds), 0.0)
+
+
+def _cooldown_seconds() -> float:
+    return max(float(settings.arxiv_rate_limit_cooldown_seconds), 0.0)

app/services/arxiv/types.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Literal
+
+ArxivSortBy = Literal["relevance", "lastUpdatedDate", "submittedDate"]
+ArxivSortOrder = Literal["ascending", "descending"]
+
+
+@dataclass(frozen=True)
+class ArxivOpenSearchMeta:
+    total_results: int = 0
+    start_index: int = 0
+    items_per_page: int = 0
+
+
+@dataclass(frozen=True)
+class ArxivEntry:
+    entry_id_url: str
+    arxiv_id: str | None
+    title: str
+    summary: str
+    published: str | None
+    updated: str | None
+    authors: list[str] = field(default_factory=list)
+    links: list[str] = field(default_factory=list)
+    categories: list[str] = field(default_factory=list)
+    primary_category: str | None = None
+
+
+@dataclass(frozen=True)
+class ArxivFeed:
+    entries: list[ArxivEntry] = field(default_factory=list)
+    opensearch: ArxivOpenSearchMeta = field(default_factory=ArxivOpenSearchMeta)

app/services/crossref/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from app.services.crossref.application import discover_doi_for_publication
+
+__all__ = ["discover_doi_for_publication"]

app/services/crossref/application.py

@@ -5,22 +5,28 @@ import logging
 import re
 import threading
 import time
+from importlib.metadata import version as pkg_version
 from typing import TYPE_CHECKING
 
 from crossref.restful import Etiquette, Works
 
-from app.services.domains.doi.normalize import normalize_doi
+from app.logging_utils import structured_log
+from app.services.doi.normalize import normalize_doi
 from app.settings import settings
 
+_APP_VERSION = pkg_version("scholarr")
+
 if TYPE_CHECKING:
-    from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
+    from app.services.publications.types import PublicationListItem, UnreadPublicationItem
 
 TOKEN_RE = re.compile(r"[a-z0-9]+")
-NON_ALNUM_RE = re.compile(r"[^a-z0-9\\s]+")
+NON_ALNUM_RE = re.compile(r"[^a-z0-9\s]+")
 STOP_WORDS = {"the", "and", "for", "with", "from", "method", "study", "analysis"}
 _RATE_LOCK = threading.Lock()
 _LAST_REQUEST_AT = 0.0
 logger = logging.getLogger(__name__)
+STRICT_TITLE_MATCH_THRESHOLD = 0.75
+RELAXED_TITLE_MATCH_THRESHOLD = 0.85
 
 
 def _rate_limit_wait(min_interval_seconds: float) -> None:
@@ -31,11 +37,12 @@ def _rate_limit_wait(min_interval_seconds: float) -> None:
         remaining = interval - elapsed
     if remaining > 0:
         time.sleep(remaining)
+    with _RATE_LOCK:
         _LAST_REQUEST_AT = time.monotonic()
 
 
 def _normalized_tokens(value: str) -> list[str]:
-    lowered = value.lower().replace("’", "'").replace("“", "\"").replace("”", "\"")
+    lowered = value.lower().replace("’", "'").replace("“", '"').replace("”", '"')
     lowered = NON_ALNUM_RE.sub(" ", lowered)
     return [token for token in TOKEN_RE.findall(lowered) if len(token) >= 3]
 
@@ -132,7 +139,42 @@ def _candidate_rank(*, title: str, year: int | None, item: dict) -> tuple[float,
     return score, doi
 
 
-def _best_candidate_doi(
+def _year_delta(source_year: int | None, candidate_year: int | None) -> int | None:
+    if source_year is None or candidate_year is None:
+        return None
+    return abs(int(source_year) - int(candidate_year))
+
+
+def _candidate_rank_relaxed(
+    *,
+    title: str,
+    year: int | None,
+    item: dict,
+    author_surname: str | None,
+) -> tuple[float, str | None]:
+    doi = normalize_doi(str(item.get("DOI") or ""))
+    if doi is None:
+        return 0.0, None
+    score = _title_match_score(title, _candidate_title(item))
+    if score <= 0:
+        return 0.0, None
+    candidate_year = _candidate_year(item)
+    delta = _year_delta(year, candidate_year)
+    if delta is not None:
+        if delta <= 1:
+            score += 0.05
+        elif delta <= 3:
+            score += 0.0
+        elif delta <= 5:
+            score -= 0.03
+        else:
+            score -= 0.08
+    if _candidate_author_match(item, author_surname):
+        score += 0.03
+    return score, doi
+
+
+def _best_candidate_doi_strict(
     *,
     title: str,
     year: int | None,
@@ -149,7 +191,7 @@ def _best_candidate_doi(
             continue
         score, doi = _candidate_rank(title=title, year=year, item=item)
         candidate_year = _candidate_year(item)
-        if doi is None or score < 0.75:
+        if doi is None or score < STRICT_TITLE_MATCH_THRESHOLD:
             continue
         if score > best_score:
             best_score = score
@@ -166,9 +208,95 @@ def _best_candidate_doi(
     return best_doi
 
 
+def _best_candidate_doi_relaxed(
+    *,
+    title: str,
+    year: int | None,
+    items: list[dict],
+    author_surname: str | None,
+) -> str | None:
+    best_score = 0.0
+    best_doi: str | None = None
+    best_author_match = False
+    best_delta: int | None = None
+    best_year: int | None = None
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        score, doi = _candidate_rank_relaxed(
+            title=title,
+            year=year,
+            item=item,
+            author_surname=author_surname,
+        )
+        if doi is None or score < RELAXED_TITLE_MATCH_THRESHOLD:
+            continue
+        candidate_year = _candidate_year(item)
+        candidate_author_match = _candidate_author_match(item, author_surname)
+        candidate_delta = _year_delta(year, candidate_year)
+        if score > best_score:
+            best_score = score
+            best_doi = doi
+            best_author_match = candidate_author_match
+            best_delta = candidate_delta
+            best_year = candidate_year
+            continue
+        if abs(score - best_score) > 0.02:
+            continue
+        if candidate_author_match and not best_author_match:
+            best_doi = doi
+            best_author_match = True
+            best_delta = candidate_delta
+            best_year = candidate_year
+            continue
+        if best_delta is None and candidate_delta is not None:
+            best_doi = doi
+            best_author_match = candidate_author_match
+            best_delta = candidate_delta
+            best_year = candidate_year
+            continue
+        if best_delta is not None and candidate_delta is not None and candidate_delta < best_delta:
+            best_doi = doi
+            best_author_match = candidate_author_match
+            best_delta = candidate_delta
+            best_year = candidate_year
+            continue
+        if best_year is None or candidate_year is None:
+            continue
+        if candidate_year < best_year:
+            best_doi = doi
+            best_author_match = candidate_author_match
+            best_delta = candidate_delta
+            best_year = candidate_year
+    return best_doi
+
+
+def _best_candidate_doi(
+    *,
+    title: str,
+    year: int | None,
+    items: list[dict],
+    author_surname: str | None,
+) -> str | None:
+    strict_match = _best_candidate_doi_strict(
+        title=title,
+        year=year,
+        items=items,
+        author_surname=author_surname,
+    )
+    if strict_match:
+        return strict_match
+    return _best_candidate_doi_relaxed(
+        title=title,
+        year=year,
+        items=items,
+        author_surname=author_surname,
+    )
+
+
 def _works_client(email: str | None) -> Works:
     if email:
-        etiquette = Etiquette(settings.app_name, "0.1.0", "https://scholarr.local", email)
+        etiquette = Etiquette(settings.app_name, _APP_VERSION, "https://scholarr.local", email)
         return Works(etiquette=etiquette)
     return Works()
 
@@ -223,7 +351,8 @@ async def _fetch_items(
             ),
             timeout=timeout,
         )
-    except Exception:
+    except Exception as exc:
+        structured_log(logger, "warning", "crossref.fetch_failed", error=str(exc))
         return []
 
 
@@ -254,6 +383,6 @@ async def discover_doi_for_publication(
             author_surname=author_surname,
         )
         if doi:
-            logger.debug("crossref.doi_discovered", extra={"event": "crossref.doi_discovered"})
+            structured_log(logger, "debug", "crossref.doi_discovered")
             return doi
     return None

app/services/dbops/__init__.py

@@ -0,0 +1,13 @@
+from app.services.dbops.application import run_publication_link_repair
+from app.services.dbops.integrity import collect_integrity_report
+from app.services.dbops.near_duplicate_repair import (
+    run_publication_near_duplicate_repair,
+)
+from app.services.dbops.query import list_repair_jobs
+
+__all__ = [
+    "collect_integrity_report",
+    "list_repair_jobs",
+    "run_publication_link_repair",
+    "run_publication_near_duplicate_repair",
+]

app/services/dbops/application.py

@@ -1,14 +1,13 @@
 from __future__ import annotations
 
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from typing import Any
 
-from sqlalchemy import delete, exists, func, select, update
+from sqlalchemy import CursorResult, delete, exists, func, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db.models import DataRepairJob, IngestionQueueItem, Publication, ScholarProfile, ScholarPublication
 
-
 REPAIR_STATUS_PLANNED = "planned"
 REPAIR_STATUS_RUNNING = "running"
 REPAIR_STATUS_COMPLETED = "completed"
@@ -18,7 +17,7 @@ SCOPE_MODE_ALL_USERS = "all_users"
 
 
 def _utcnow() -> datetime:
-    return datetime.now(timezone.utc)
+    return datetime.now(UTC)
 
 
 def _normalize_scope_mode(scope_mode: str) -> str:
@@ -105,11 +104,7 @@ async def _count_orphan_publications(db_session: AsyncSession) -> int:
     stmt = (
         select(func.count())
         .select_from(Publication)
-        .where(
+        .where(~exists(select(1).where(ScholarPublication.publication_id == Publication.id)))
-            ~exists(
-                select(1).where(ScholarPublication.publication_id == Publication.id)
-            )
-        )
     )
     result = await db_session.execute(stmt)
     return int(result.scalar_one() or 0)
@@ -157,10 +152,8 @@ def _job_summary(
 
 
 async def _delete_links_for_targets(db_session: AsyncSession, *, target_scholar_profile_ids: list[int]) -> int:
-    result = await db_session.execute(
+    result: CursorResult[Any] = await db_session.execute(  # type: ignore[assignment]
-        delete(ScholarPublication).where(
+        delete(ScholarPublication).where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids))
-            ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids)
-        )
     )
     return int(result.rowcount or 0)
 
@@ -171,12 +164,10 @@ async def _delete_queue_for_targets(
     user_id: int | None,
     target_scholar_profile_ids: list[int],
 ) -> int:
-    stmt = delete(IngestionQueueItem).where(
+    stmt = delete(IngestionQueueItem).where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids))
-        IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids)
-    )
     if user_id is not None:
         stmt = stmt.where(IngestionQueueItem.user_id == user_id)
-    result = await db_session.execute(stmt)
+    result: CursorResult[Any] = await db_session.execute(stmt)  # type: ignore[assignment]
     return int(result.rowcount or 0)
 
 
@@ -189,7 +180,7 @@ async def _reset_scholar_tracking_state(
     stmt = update(ScholarProfile).where(ScholarProfile.id.in_(target_scholar_profile_ids))
     if user_id is not None:
         stmt = stmt.where(ScholarProfile.user_id == user_id)
-    result = await db_session.execute(
+    result: CursorResult[Any] = await db_session.execute(  # type: ignore[assignment]
         stmt.values(
             baseline_completed=False,
             last_initial_page_fingerprint_sha256=None,
@@ -202,10 +193,8 @@ async def _reset_scholar_tracking_state(
 
 
 async def _delete_orphan_publications(db_session: AsyncSession) -> int:
-    result = await db_session.execute(
+    result: CursorResult[Any] = await db_session.execute(  # type: ignore[assignment]
-        delete(Publication).where(
+        delete(Publication).where(~exists(select(1).where(ScholarPublication.publication_id == Publication.id)))
-            ~exists(select(1).where(ScholarPublication.publication_id == Publication.id))
-        )
     )
     return int(result.rowcount or 0)
 

app/services/dbops/integrity.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from typing import Any
 
 from sqlalchemy import func, select, text
@@ -168,7 +168,7 @@ async def collect_integrity_report(db_session: AsyncSession) -> dict[str, Any]:
 
     return {
         "status": _status_from_issues(failures=failures, warnings=warnings),
-        "checked_at": datetime.now(timezone.utc).isoformat(),
+        "checked_at": datetime.now(UTC).isoformat(),
         "failures": failures,
         "warnings": warnings,
         "checks": checks,

app/services/dbops/near_duplicate_repair.py

@@ -0,0 +1,228 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import DataRepairJob
+from app.services.dbops.application import (
+    REPAIR_STATUS_COMPLETED,
+    REPAIR_STATUS_FAILED,
+    REPAIR_STATUS_PLANNED,
+    REPAIR_STATUS_RUNNING,
+)
+from app.services.publications import dedup as dedup_service
+
+NEAR_DUP_JOB_NAME = "repair_publication_near_duplicates"
+NEAR_DUP_DEFAULT_MAX_CLUSTERS = 25
+
+
+def _utcnow() -> datetime:
+    return datetime.now(UTC)
+
+
+def _normalized_cluster_keys(values: list[str] | None) -> list[str]:
+    if not values:
+        return []
+    seen: set[str] = set()
+    normalized: list[str] = []
+    for value in values:
+        key = str(value or "").strip().lower()
+        if not key or key in seen:
+            continue
+        seen.add(key)
+        normalized.append(key)
+    return normalized
+
+
+def _normalized_max_clusters(value: int) -> int:
+    return max(1, min(int(value), 200))
+
+
+def _scope_payload(
+    *,
+    similarity_threshold: float,
+    min_shared_tokens: int,
+    max_year_delta: int,
+    max_clusters: int,
+    selected_cluster_keys: list[str],
+) -> dict[str, Any]:
+    return {
+        "similarity_threshold": float(similarity_threshold),
+        "min_shared_tokens": int(min_shared_tokens),
+        "max_year_delta": int(max_year_delta),
+        "max_clusters": int(max_clusters),
+        "selected_cluster_keys": selected_cluster_keys,
+    }
+
+
+async def _create_job(
+    db_session: AsyncSession,
+    *,
+    requested_by: str | None,
+    scope: dict[str, Any],
+    dry_run: bool,
+) -> DataRepairJob:
+    job = DataRepairJob(
+        job_name=NEAR_DUP_JOB_NAME,
+        requested_by=(requested_by or "").strip() or None,
+        scope=scope,
+        dry_run=bool(dry_run),
+        status=REPAIR_STATUS_PLANNED,
+        summary={},
+    )
+    db_session.add(job)
+    await db_session.flush()
+    job.status = REPAIR_STATUS_RUNNING
+    job.started_at = _utcnow()
+    return job
+
+
+def _selected_clusters(
+    *,
+    clusters: list[dedup_service.NearDuplicateCluster],
+    selected_cluster_keys: list[str],
+) -> tuple[list[dedup_service.NearDuplicateCluster], list[str]]:
+    if not selected_cluster_keys:
+        return [], []
+    by_key = {cluster.cluster_key.lower(): cluster for cluster in clusters}
+    selected: list[dedup_service.NearDuplicateCluster] = []
+    missing: list[str] = []
+    for key in selected_cluster_keys:
+        cluster = by_key.get(key)
+        if cluster is None:
+            missing.append(key)
+            continue
+        selected.append(cluster)
+    return selected, missing
+
+
+async def _merge_selected_clusters(
+    db_session: AsyncSession,
+    *,
+    selected_clusters: list[dedup_service.NearDuplicateCluster],
+) -> int:
+    merged_publications = 0
+    for cluster in selected_clusters:
+        merged_publications += await dedup_service.merge_near_duplicate_cluster(
+            db_session,
+            cluster=cluster,
+        )
+    return merged_publications
+
+
+def _clusters_payload(
+    *,
+    clusters: list[dedup_service.NearDuplicateCluster],
+    max_clusters: int,
+) -> list[dict[str, object]]:
+    preview = clusters[:max_clusters]
+    return [dedup_service.near_duplicate_cluster_payload(cluster) for cluster in preview]
+
+
+def _summary_payload(
+    *,
+    dry_run: bool,
+    cluster_count: int,
+    selected_count: int,
+    missing_count: int,
+    merged_publications: int,
+    max_clusters: int,
+) -> dict[str, Any]:
+    return {
+        "dry_run": bool(dry_run),
+        "candidate_cluster_count": int(cluster_count),
+        "selected_cluster_count": int(selected_count),
+        "missing_selected_cluster_count": int(missing_count),
+        "merged_publications": int(merged_publications),
+        "preview_cluster_count": int(min(cluster_count, max_clusters)),
+    }
+
+
+async def _complete_job(
+    db_session: AsyncSession,
+    *,
+    job: DataRepairJob,
+    scope: dict[str, Any],
+    summary: dict[str, Any],
+    clusters: list[dict[str, object]],
+) -> dict[str, Any]:
+    job.status = REPAIR_STATUS_COMPLETED
+    job.finished_at = _utcnow()
+    job.summary = summary
+    await db_session.commit()
+    return {
+        "job_id": int(job.id),
+        "status": job.status,
+        "scope": scope,
+        "summary": summary,
+        "clusters": clusters,
+    }
+
+
+async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None:
+    from sqlalchemy.orm import make_transient
+
+    await db_session.rollback()
+    make_transient(job)
+    job.status = REPAIR_STATUS_FAILED
+    job.error_text = str(error)
+    job.finished_at = _utcnow()
+    db_session.add(job)
+    await db_session.commit()
+
+
+async def run_publication_near_duplicate_repair(
+    db_session: AsyncSession,
+    *,
+    dry_run: bool = True,
+    similarity_threshold: float = dedup_service.NEAR_DUP_DEFAULT_SIMILARITY_THRESHOLD,
+    min_shared_tokens: int = dedup_service.NEAR_DUP_DEFAULT_MIN_SHARED_TOKENS,
+    max_year_delta: int = dedup_service.NEAR_DUP_DEFAULT_MAX_YEAR_DELTA,
+    max_clusters: int = NEAR_DUP_DEFAULT_MAX_CLUSTERS,
+    selected_cluster_keys: list[str] | None = None,
+    requested_by: str | None = None,
+) -> dict[str, Any]:
+    normalized_keys = _normalized_cluster_keys(selected_cluster_keys)
+    bounded_clusters = _normalized_max_clusters(max_clusters)
+    scope = _scope_payload(
+        similarity_threshold=similarity_threshold,
+        min_shared_tokens=min_shared_tokens,
+        max_year_delta=max_year_delta,
+        max_clusters=bounded_clusters,
+        selected_cluster_keys=normalized_keys,
+    )
+    job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run)
+    try:
+        clusters = await dedup_service.find_near_duplicate_clusters(
+            db_session,
+            similarity_threshold=similarity_threshold,
+            min_shared_tokens=min_shared_tokens,
+            max_year_delta=max_year_delta,
+        )
+        selected, missing = _selected_clusters(clusters=clusters, selected_cluster_keys=normalized_keys)
+        merged_publications = 0
+        if not dry_run:
+            if not selected:
+                raise ValueError("No selected near-duplicate clusters matched current data.")
+            merged_publications = await _merge_selected_clusters(db_session, selected_clusters=selected)
+        preview = _clusters_payload(clusters=clusters, max_clusters=bounded_clusters)
+        summary = _summary_payload(
+            dry_run=dry_run,
+            cluster_count=len(clusters),
+            selected_count=len(selected),
+            missing_count=len(missing),
+            merged_publications=merged_publications,
+            max_clusters=bounded_clusters,
+        )
+        return await _complete_job(
+            db_session,
+            job=job,
+            scope=scope,
+            summary=summary,
+            clusters=preview,
+        )
+    except Exception as exc:
+        await _fail_job(db_session, job=job, error=exc)
+        raise

app/services/dbops/query.py

@@ -16,7 +16,5 @@ async def list_repair_jobs(
     limit: int = 50,
 ) -> list[DataRepairJob]:
     bounded = _bounded_limit(limit)
-    result = await db_session.execute(
+    result = await db_session.execute(select(DataRepairJob).order_by(DataRepairJob.created_at.desc()).limit(bounded))
-        select(DataRepairJob).order_by(DataRepairJob.created_at.desc()).limit(bounded)
-    )
     return list(result.scalars())