.claude/settings.json

@@ -1,33 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(wc -l /home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src/features/settings/SettingsAdminPanel.vue /home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src/features/settings/components/Admin*.vue)",
-      "Read(//home/jv/src/personal/playground/scholar_scraper/scholar-scraper/**)",
-      "Bash(while read line rest)",
-      "Bash(do echo \"$line $rest\")",
-      "Bash(done)",
-      "Bash(grep -c 'def test_' tests/unit/*.py tests/unit/**/*.py tests/integration/*.py)",
-      "Bash(sort -t: -k2 -rn)",
-      "Bash(wc -l frontend/src/pages/*.vue)",
-      "Bash(uv run ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
-      "Bash(ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
-      "Bash(npm install --package-lock-only)",
-      "Bash(python -m ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
-      "Bash(pip show ruff)",
-      "Bash(pip install ruff)",
-      "Bash(uvx ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
-      "Bash(git add app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py docs/website/package-lock.json)",
-      "Bash(git push)",
-      "Bash(uvx ruff format --check app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
-      "Bash(test -f docs/website/package-lock.json)",
-      "Bash(gh run list --branch openalex-size-reduction-scholar-hydration --limit 3 --json databaseId,status,conclusion,name,event)",
-      "Bash(gh run list --limit 10 --json databaseId,status,conclusion,name,event,headBranch)",
-      "Bash(gh run view 22554585841 --log-failed)",
-      "Bash(gh run rerun 22554585841 --failed)",
-      "Bash(bash scripts/premerge.sh)"
-    ],
-    "additionalDirectories": [
-      "/home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src"
-    ]
-  }
-}

.env.example

@@ -28,7 +28,6 @@ DATABASE_POOL_MODE=auto
 DATABASE_POOL_SIZE=5
 DATABASE_POOL_MAX_OVERFLOW=10
 DATABASE_POOL_TIMEOUT_SECONDS=30
-DATABASE_RESERVED_API_CONNECTIONS=3
 
 # ------------------------------
 # Frontend Dev Overrides
@@ -80,15 +79,12 @@ LOG_REDACT_FIELDS=
 SCHEDULER_ENABLED=1
 SCHEDULER_TICK_SECONDS=60
 SCHEDULER_QUEUE_BATCH_SIZE=10
-SCHEDULER_PDF_QUEUE_BATCH_SIZE=15
 INGESTION_AUTOMATION_ALLOWED=1
 INGESTION_MANUAL_RUN_ALLOWED=1
 INGESTION_MIN_RUN_INTERVAL_MINUTES=15
 INGESTION_MIN_REQUEST_DELAY_SECONDS=2
 INGESTION_NETWORK_ERROR_RETRIES=1
 INGESTION_RETRY_BACKOFF_SECONDS=1.0
-INGESTION_RATE_LIMIT_RETRIES=3
-INGESTION_RATE_LIMIT_BACKOFF_SECONDS=30.0
 INGESTION_MAX_PAGES_PER_SCHOLAR=30
 INGESTION_PAGE_SIZE=100
 INGESTION_ALERT_BLOCKED_FAILURE_THRESHOLD=1
@@ -116,10 +112,6 @@ SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD=1
 SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS=1800
 SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD=2
 SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD=3
-SCHOLAR_HTTP_USER_AGENT=
-SCHOLAR_HTTP_ROTATE_USER_AGENT=0
-SCHOLAR_HTTP_ACCEPT_LANGUAGE=en-US,en;q=0.9
-SCHOLAR_HTTP_COOKIE=
 
 # ------------------------------
 # OA Enrichment + PDF Resolution
@@ -133,14 +125,6 @@ UNPAYWALL_RETRY_COOLDOWN_SECONDS=1800
 UNPAYWALL_PDF_DISCOVERY_ENABLED=1
 UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES=5
 UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES=500000
-ARXIV_ENABLED=1
-ARXIV_TIMEOUT_SECONDS=3.0
-ARXIV_MIN_INTERVAL_SECONDS=4.0
-ARXIV_RATE_LIMIT_COOLDOWN_SECONDS=60.0
-ARXIV_DEFAULT_MAX_RESULTS=3
-ARXIV_CACHE_TTL_SECONDS=900
-ARXIV_CACHE_MAX_ENTRIES=512
-ARXIV_MAILTO=
 PDF_AUTO_RETRY_INTERVAL_SECONDS=86400
 PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS=3600
 PDF_AUTO_RETRY_MAX_ATTEMPTS=3
@@ -149,9 +133,6 @@ CROSSREF_MAX_ROWS=10
 CROSSREF_TIMEOUT_SECONDS=8.0
 CROSSREF_MIN_INTERVAL_SECONDS=0.6
 CROSSREF_MAX_LOOKUPS_PER_REQUEST=8
-OPENALEX_API_KEY=
-CROSSREF_API_TOKEN=
-CROSSREF_API_MAILTO=
 
 # ------------------------------
 # Startup Bootstrap + DB Wait

.github/FUNDING.yml

@@ -1 +0,0 @@
-buy_me_a_coffee: justinzeus

.github/dependabot.yml

@@ -1,14 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: pip
-    directory: /
-    schedule:
-      interval: weekly
-  - package-ecosystem: npm
-    directory: /frontend
-    schedule:
-      interval: weekly
-  - package-ecosystem: github-actions
-    directory: /
-    schedule:
-      interval: weekly

.github/workflows/ci.yml

@@ -20,28 +20,10 @@ jobs:
       - name: Enforce env contract parity
         run: python3 scripts/check_env_contract.py
 
-  lint:
-    runs-on: ubuntu-latest
-    needs: [repo-hygiene]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-      - uses: astral-sh/setup-uv@v4
-      - run: uv sync --extra dev
-      - name: Ruff check
-        run: uv run ruff check .
-      - name: Ruff format check
-        run: uv run ruff format --check .
-      - name: Mypy
-        run: uv run mypy app/ --ignore-missing-imports
-
   test:
     runs-on: ubuntu-latest
     needs:
       - repo-hygiene
-      - lint
     services:
       postgres:
         image: postgres:15
@@ -109,7 +91,7 @@ jobs:
 
       - name: Install frontend dependencies
         working-directory: frontend
-        run: npm ci
+        run: npm install
 
       - name: Frontend theme token policy
         working-directory: frontend
@@ -142,7 +124,7 @@ jobs:
           node-version: "20"
 
       - name: Install docs dependencies
-        run: npm ci --prefix docs/website
+        run: npm install --prefix docs/website
 
       - name: Docs build
         run: npm --prefix docs/website run build

.github/workflows/codeql.yml

@@ -1,26 +0,0 @@
-name: CodeQL
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-  schedule:
-    - cron: "30 5 * * 1"
-
-jobs:
-  analyze:
-    runs-on: ubuntu-latest
-    permissions:
-      security-events: write
-      contents: read
-    strategy:
-      matrix:
-        language: [python, javascript]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: github/codeql-action/init@v3
-        with:
-          languages: ${{ matrix.language }}
-      - uses: github/codeql-action/autobuild@v3
-      - uses: github/codeql-action/analyze@v3

.github/workflows/docs-pages.yml

@@ -32,7 +32,7 @@ jobs:
           node-version: "20"
 
       - name: Install docs dependencies
-        run: npm ci --prefix docs/website
+        run: npm install --prefix docs/website
 
       - name: Build docs site
         run: npm --prefix docs/website run build

.github/workflows/release.yml

@@ -1,32 +0,0 @@
-name: Release
-
-on:
-  push:
-    branches: [main]
-
-jobs:
-  release:
-    runs-on: ubuntu-latest
-    if: github.repository == 'JustinZeus/scholarr'
-    permissions:
-      contents: write
-      id-token: write
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-      - uses: astral-sh/setup-uv@v4
-      - run: uv sync --extra dev
-      - name: Lint
-        run: uv run ruff check
-      - name: Test
-        run: uv run pytest
-      - name: Semantic Release
-        id: release
-        run: uv run semantic-release publish
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -12,11 +12,9 @@ htmlcov/
 *.log
 .DS_Store
 planning/
-build/
 frontend/node_modules/
 frontend/dist/
 frontend/.vite/
 docs/website/node_modules/
 docs/website/build/
-docs/website/.docusaurus/
-.claude/
+AGENTS.MD

LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2026 JustinZeus
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

README.md

@@ -1,149 +1,69 @@
+# scholarr
+
 <div align="center">
 
-<picture>
-  <source media="(prefers-color-scheme: dark)" srcset=".github/logo-dark.png" />
-  <source media="(prefers-color-scheme: light)" srcset="frontend/public/scholar_logo.png" />
-  <img src="frontend/public/scholar_logo.png" alt="Scholarr" width="120" />
-</picture>
-
-# Scholarr
-
-**Self-hosted academic publication tracker.**
-
-Track Google Scholar profiles, discover new papers automatically,
-resolve open-access PDFs, and stay on top of the literature you care about.
+Self-hosted scholar tracking with a single app image (API + frontend).
 
 [![CI](https://img.shields.io/github/actions/workflow/status/justinzeus/scholarr/ci.yml?style=for-the-badge)](https://github.com/JustinZeus/scholarr/actions/workflows/ci.yml)
-[![CodeQL](https://img.shields.io/github/actions/workflow/status/justinzeus/scholarr/codeql.yml?style=for-the-badge&label=CodeQL)](https://github.com/JustinZeus/scholarr/actions/workflows/codeql.yml)
-[![Release](https://img.shields.io/github/v/release/justinzeus/scholarr?style=for-the-badge)](https://github.com/JustinZeus/scholarr/releases)
 [![Docker Pulls](https://img.shields.io/docker/pulls/justinzeus/scholarr?style=for-the-badge&logo=docker)](https://hub.docker.com/r/justinzeus/scholarr)
-[![Python](https://img.shields.io/badge/python-3.12+-3776AB?style=for-the-badge&logo=python&logoColor=white)](https://www.python.org/)
-[![License](https://img.shields.io/github/license/justinzeus/scholarr?style=for-the-badge)](LICENSE)
-[![Docs](https://img.shields.io/badge/docs-scholarr-2e8555?style=for-the-badge)](https://justinzeus.github.io/scholarr/)
+[![Docker Image](https://img.shields.io/badge/docker-justinzeus%2Fscholarr-2496ED?style=for-the-badge&logo=docker&logoColor=white)](https://hub.docker.com/r/justinzeus/scholarr)
 
 </div>
 
----
-
-<!-- TODO: Replace these placeholders with actual screenshots.
-     Drop PNG/JPEG files into docs/assets/ and update the paths below.
-
-     Recommended screenshots:
-       1. Dashboard / publications list (light or dark mode)
-       2. Scholar profile view
-       3. Run detail with progress
-
-     Ideal dimensions: 1200-1400px wide, PNG or WebP.
--->
-
-<p align="center">
-  <img src="docs/assets/screenshot-dashboard.png" alt="Publications dashboard" width="720" />
-</p>
-
-<!--
-<p align="center">
-  <img src="docs/assets/screenshot-scholars.png" alt="Scholar profiles" width="360" />
-  <img src="docs/assets/screenshot-run.png" alt="Ingestion run" width="360" />
-</p>
--->
-
-## Why Scholarr?
-
-Most researchers track new papers by manually checking Google Scholar, setting up email alerts, or juggling RSS feeds. Scholarr replaces all of that with a single self-hosted service:
-
-- **Add scholars once** -- by profile URL, Scholar ID, or name search
-- **Publications appear automatically** -- a background scheduler scrapes profiles on a configurable interval
-- **Open-access PDFs are resolved for you** -- Unpaywall and arXiv are queried automatically when a DOI is found
-- **Everything is deduplicated** -- publications are global records; no duplicates across scholars
-- **Your data stays yours** -- fully self-hosted, export/import your entire library at any time
-
-## Features
-
-| | |
-|---|---|
-| **Automated Ingestion** | Background scheduler with configurable intervals, continuation queue, and multi-page pagination |
-| **Identifier Resolution** | Cross-references arXiv, Crossref, and OpenAlex to gather DOIs, arXiv IDs, PMIDs |
-| **PDF Discovery** | Resolves open-access PDFs via Unpaywall API and arXiv, with automatic retry queue |
-| **Scrape Safety** | Rate limiting, cooldowns, and backoff strategies that prevent IP bans -- these are safety floors, not optional |
-| **Multi-User** | Session-based auth, admin user management, user-scoped scholar tracking |
-| **Theming** | 7 color presets with light/dark mode, tokenized component system |
-| **Import / Export** | Portable scholar data with full publication and read-state preservation |
-| **Single Container** | FastAPI backend + Vue 3 frontend ship as one Docker image |
-
 ## Quick Start
 
+1. Copy env template:
+
 ```bash
-# 1. Clone and configure
-git clone https://github.com/JustinZeus/scholarr.git
-cd scholarr
 cp .env.example .env
-
-# 2. Set required secrets in .env
-#    POSTGRES_PASSWORD=<secure-password>
-#    SESSION_SECRET_KEY=<random-32-char-string>
-
-# 3. Start
-docker compose up -d
-
-# 4. Open http://localhost:8000
 ```
 
-To bootstrap an admin account on first run, add to `.env`:
+2. Set required values in `.env`:
+- `POSTGRES_PASSWORD`
+- `SESSION_SECRET_KEY`
+
+3. Start stack:
 
 ```bash
-BOOTSTRAP_ADMIN_ON_START=1
-BOOTSTRAP_ADMIN_EMAIL=admin@example.com
-BOOTSTRAP_ADMIN_PASSWORD=<secure-password>
+docker compose pull
+docker compose up -d
 ```
 
-## How It Works
-
-```mermaid
-graph LR
-    UI[Vue 3 Dashboard] <-->|REST + SSE| API[FastAPI]
-    API --> Scheduler[Scheduler]
-    Scheduler -->|Scrape HTML| Scholar[Google Scholar]
-    Scholar -->|Parse & Deduplicate| DB[(PostgreSQL)]
-    Scholar -.->|Identify| Ext[arXiv / Crossref / OpenAlex]
-    Ext --> DB
-    DB -->|DOIs| PDF[PDF Resolution]
-    PDF -->|Unpaywall / arXiv| DB
-    API <--> DB
-```
-
-## Tech Stack
-
-| Layer | Technology |
-|-------|------------|
-| Backend | Python 3.12, FastAPI, SQLAlchemy 2.0 (async), Alembic |
-| Frontend | TypeScript, Vue 3, Vite, Tailwind CSS |
-| Database | PostgreSQL 15 |
-| Infrastructure | Multi-stage Docker, Docker Compose |
+Open:
+- App/API: `http://localhost:8000`
+- Health: `http://localhost:8000/healthz`
 
 ## Documentation
 
-Full documentation: **[justinzeus.github.io/scholarr](https://justinzeus.github.io/scholarr/)**
+Complete documentation is published at:
 
-| Section | Covers |
-|---------|--------|
-| [User Guide](docs/user/overview.md) | Installation, configuration, all environment variables |
-| [Developer Guide](docs/developer/overview.md) | Architecture, local dev, contributing, testing |
-| [Operations](docs/operations/overview.md) | Deployment, database runbook, scrape safety |
-| [API Reference](docs/reference/api.md) | Envelope spec, all endpoints, DTO contracts |
+- https://justinzeus.github.io/scholarr/
 
-## Contributing
+Source markdown and docs tooling live under `docs/`.
 
-Scholarr uses [conventional commits](https://www.conventionalcommits.org/) and [semantic versioning](https://semver.org/). See the [contributing guide](docs/developer/contributing.md) for PR process and code standards.
+## Quality Gates
+
+Backend:
 
 ```bash
-# Dev environment
-docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build
-
-# Run tests (always in containers)
-docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app \
-  python -m pytest
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest tests/unit
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest -m integration
 ```
 
-## License
+Frontend:
 
-See [LICENSE](LICENSE) for details.
+```bash
+cd frontend
+npm install
+npm run typecheck
+npm run test:run
+npm run build
+```
+
+Contract and env checks:
+
+```bash
+python3 scripts/check_frontend_api_contract.py
+python3 scripts/check_env_contract.py
+./scripts/check_no_generated_artifacts.sh
+```

agents.md

@@ -1,123 +1,43 @@
 # AI Agent Instructions: Scholarr
 
-Adhere strictly to these constraints when working on this codebase.
+Adhere strictly to these constraints.
 
-## 1. Code Quality
+## 1. Coding Standards (Strict Enforcement)
+* **Function Length:** Maximum 50 lines of code per function. Break down complex logic into small, testable, single-responsibility functions.
+* **DRY (Don't Repeat Yourself):** Abstract repetitive logic immediately. No duplicate boilerplate for database queries, API responses, or error handling.
+* **Negative Space Programming:** Utilize explicit assertions and constraints to define invalid states. Fail fast and early. Do not allow silent failures or cascading malformed data, especially in DOM parsing.
+* **Cyclomatic Complexity:** Flatten logic. Use early returns and guard clauses instead of deep nesting.
 
-- **Function length:** 50 lines max. Extract helpers ruthlessly.
-- **File length:** 400 lines target, 600 lines hard ceiling. Files above this must be decomposed before adding more code.
-- **DRY:** Abstract repeated logic immediately. No duplicate boilerplate for queries, responses, or error handling.
-- **Negative space programming:** Fail fast with explicit assertions and guard clauses. No silent failures, especially in DOM parsing.
-- **Cyclomatic complexity:** Flatten with early returns. No deep nesting. No magic numbers.
-- **No dead code:** Do not leave commented-out code, unused imports, or backward-compatibility shims. Delete cleanly.
-- **Variable naming:** Any variable outside of a lambda functions or iterators should be descriptive. Code should be readable for maintainers.
+## 2. Domain Architecture & Data Model
+* **Data Isolation:** Scholar tracking is **user-scoped**. Validate mapping/join tables; never assume global links between users and Scholar IDs.
+* **Data Deduplication:** Publications are **global records**. Deduplicate via Scholar cluster ID and normalized fingerprinting prior to database insertion.
+* **State Management:** Visibility and "read/unread" states exist exclusively on the scholar-publication link table, not the global publication table.
+* **API Contract:** Exact envelope format required:
+    * Success: `{"data": ..., "meta": {"request_id": "..."}}`
+    * Error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}`
 
-## 2. Architecture
+## 3. Scrape Safety & Rate Limiting (Immutable)
+These limits prevent IP bans and are not to be optimized away.
+* **Minimum Delay:** Enforce `INGESTION_MIN_REQUEST_DELAY_SECONDS` (default 2s) between all external requests.
+* **Anti-Detection:** Default to direct ID or profile URL ingestion. Name searches trigger CAPTCHAs.
+* **Cooldowns:** Respect `INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS` (1800s) and `INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS` (900s) upon threshold breaches.
 
-### Data Model
+## 4. Current Environment & Stack
+* **Backend:** Python 3.12+, FastAPI, SQLAlchemy (Async/asyncpg), Alembic.
+* **Frontend:** TypeScript, Vue 3, Vite.
+* **Infrastructure:** Multi-stage Docker.
 
-- Scholar tracking is **user-scoped**. Never assume global links between users and Scholar IDs.
-- Publications are **global, deduplicated records**. Deduplicate via cluster ID and normalized fingerprinting.
-- Read/unread, favorites, and visibility state live on the **scholar-publication link**, not the publication.
+## 5. Domain Service Boundaries
+* **Strict Modularity:** Flat files in the `app/services/` root are strictly prohibited. All business logic and routing must reside exclusively within `app/services/domains/`.
+* **`app/services/domains/scholar/*`:** Parser contract is fail-fast. Layout drift must emit explicit exceptions and `layout_*` reasons/warnings. Never allow silent partial success.
+* **`app/services/domains/ingestion/application.py`:** Orchestrates ingestion runs; validate parser outputs before persistence; enforce publication candidate constraints before upsert.
+* **`app/services/domains/publications/*`:** Publication list/read-state query layer. Includes `doi` + `pdf_url` fields for UI consumption, enforces non-blocking lazy OA enrichment scheduling on list reads, and exposes per-publication PDF retry behavior.
+* **`app/services/domains/crossref/*`:** DOI discovery fallback module. Must use bounded/paced lookups to avoid burst traffic and 429 responses.
+* **`app/services/domains/unpaywall/*`:** OA resolver by DOI only (best OA location + PDF URL extraction). Do not use Google Scholar for PDF resolution to avoid N+1 scrape amplification.
+* **`app/services/domains/portability/*`:** Handles JSON import/export for user-scoped scholars and scholar-publication link state while preserving global publication dedup rules.
+* **`app/services/domains/ingestion/scheduler.py`:** Owns automatic runs and continuation queue retries/drops; do not bypass safety gate or cooldown logic.
 
-### Service Boundaries
 
-All business logic lives in `app/services/<domain>/`. No flat files in `app/services/` root. Each domain owns its application service, types, and helpers.
-
-### API Envelope
-
-All `/api/v1` responses use this exact envelope:
-
-```
-Success: {"data": ..., "meta": {"request_id": "..."}}
-Error:   {"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}
-```
-
-Use the Pydantic envelope schemas in `app/api/schemas.py`. Do not construct raw dicts.
-
-## 3. Scrape Safety (Immutable)
-
-These constraints prevent IP bans. They are not tunable to zero and must not be optimized away.
-
-- Enforce `INGESTION_MIN_REQUEST_DELAY_SECONDS` (default 2s) between all external requests.
-- Default to direct ID or profile URL ingestion. Name searches trigger CAPTCHAs.
-- Respect `INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS` (1800s) and `INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS` (900s) upon threshold breaches.
-
-## 4. Logging
-
-Use `structured_log()` from `app/logging_utils.py` for all domain logging. Do not use raw `logger.info()` / `logger.warning()` calls.
-
-```python
-from app.logging_utils import structured_log
-
-structured_log(logger, "info", "ingestion.run_started", user_id=user_id, scholar_count=count)
-```
-
-Every event name should be dot-namespaced to its domain (e.g., `arxiv.cache_hit`, `ingestion.safety_cooldown_entered`).
-
-## 5. Stack & Tooling
-
-- **Backend:** Python 3.12+, FastAPI, SQLAlchemy 2.0 (async/asyncpg), Alembic
-- **Frontend:** TypeScript, Vue 3, Vite, Tailwind CSS
-- **Infrastructure:** Multi-stage Docker, Docker Compose
-- **Package manager:** `uv` (used in Dockerfile and CI; `uv run` prefix for all commands)
-- **Linting:** `ruff check .` and `ruff format --check .` (config in `pyproject.toml`)
-- **Type checking:** `mypy app/`
-- **Versioning:** python-semantic-release with conventional commits
-
-## 6. Commits
-
-Follow [Conventional Commits](https://www.conventionalcommits.org/):
-
-```
-<type>(<scope>): <description>
-```
-
-Types: `feat`, `fix`, `docs`, `ci`, `refactor`, `test`, `chore`, `perf`.
-
-## 7. Testing & Quality Gates
-
-All **local development** commands run inside containers. Do not run bare `npm`, `pytest`, or `ruff` on the host. CI workflows (`.github/workflows/`) use bare runners with `uv` and `npm` directly — this is intentional since CI has no Docker-in-Docker dependency.
-
-### Backend
-
-```bash
-# Unit tests (default, excludes integration)
-docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app python -m pytest
-
-# Integration tests
-docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app python -m pytest -m integration
-
-# Linting
-docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app ruff check .
-docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app ruff format --check .
-
-# Type checking
-docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app mypy app/
-```
-
-Pytest markers: `integration`, `db`, `migrations`, `schema`, `smoke`.
-
-### Frontend
-
-```bash
-# Type checking
-docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run typecheck
-
-# Unit tests
-docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run test:run
-
-# Lint
-docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run lint
-
-# Production build
-docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm frontend npm run build
-```
-
-All four gates must pass before a PR is considered ready.
-
-## 8. Frontend
-
-- Use the tokenized theme system (`frontend/src/theme/presets/`). Do not hardcode colors.
-- Integrate Tailwind with preset theme tokens. Reference `frontend/scripts/check_theme_tokens.mjs` for enforcement.
-- Every UI element must have a clear purpose. Clarity through styling and language.
+## 6. UI rules
+Make sure to properly integrate tailwind in combination with the preset theming
+Clarity through both styling and language are a priority. all UI elements need to have a proper reason for existing.

alembic/env.py

@@ -1,13 +1,13 @@
-import os
 from logging.config import fileConfig
+import os
 
+from alembic import context
 from sqlalchemy import pool
 from sqlalchemy.engine import Connection
 from sqlalchemy.ext.asyncio import async_engine_from_config
 
-from alembic import context
-from app.db import models  # noqa: F401
 from app.db.base import metadata
+from app.db import models  # noqa: F401
 
 config = context.config
 

alembic/versions/20260216_0001_multi_user_schema.py

@@ -8,11 +8,10 @@ Create Date: 2026-02-16 17:10:00.000000
 
 from collections.abc import Sequence
 
+from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
-from alembic import op
-
 # revision identifiers, used by Alembic.
 revision: str = "20260216_0001"
 down_revision: str | Sequence[str] | None = None

alembic/versions/20260216_0002_add_user_admin_flag.py

@@ -7,9 +7,8 @@ Create Date: 2026-02-16 17:40:00.000000
 
 from collections.abc import Sequence
 
-import sqlalchemy as sa
-
 from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision: str = "20260216_0002"
@@ -32,3 +31,4 @@ def upgrade() -> None:
 
 def downgrade() -> None:
     op.drop_column("users", "is_admin")
+

alembic/versions/20260216_0003_add_ingestion_queue.py

@@ -7,9 +7,8 @@ Create Date: 2026-02-16 23:45:00.000000
 
 from collections.abc import Sequence
 
-import sqlalchemy as sa
-
 from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision: str = "20260216_0003"

alembic/versions/20260217_0004_queue_status_fields.py

@@ -7,9 +7,8 @@ Create Date: 2026-02-17 10:15:00.000000
 
 from collections.abc import Sequence
 
-import sqlalchemy as sa
-
 from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision: str = "20260217_0004"
@@ -23,7 +22,9 @@ def upgrade() -> None:
     inspector = sa.inspect(bind)
     columns = {column["name"] for column in inspector.get_columns("ingestion_queue_items")}
     indexes = {index["name"] for index in inspector.get_indexes("ingestion_queue_items")}
-    checks = {constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items")}
+    checks = {
+        constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items")
+    }
 
     if "status" not in columns:
         op.add_column(
@@ -79,7 +80,9 @@ def downgrade() -> None:
     inspector = sa.inspect(bind)
     columns = {column["name"] for column in inspector.get_columns("ingestion_queue_items")}
     indexes = {index["name"] for index in inspector.get_indexes("ingestion_queue_items")}
-    checks = {constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items")}
+    checks = {
+        constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items")
+    }
 
     if "ix_ingestion_queue_status_next_attempt" in indexes:
         op.drop_index("ix_ingestion_queue_status_next_attempt", table_name="ingestion_queue_items")

alembic/versions/20260217_0005_manual_run_idempotency.py

@@ -7,9 +7,8 @@ Create Date: 2026-02-17 16:20:00.000000
 
 from collections.abc import Sequence
 
-import sqlalchemy as sa
-
 from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision: str = "20260217_0005"
@@ -71,7 +70,9 @@ def upgrade() -> None:
             "crawl_runs",
             ["user_id", "idempotency_key"],
             unique=True,
-            postgresql_where=sa.text("idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"),
+            postgresql_where=sa.text(
+                "idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"
+            ),
         )
 
 

alembic/versions/20260217_0006_scholar_profile_images.py

@@ -7,9 +7,8 @@ Create Date: 2026-02-17 18:30:00.000000
 
 from collections.abc import Sequence
 
-import sqlalchemy as sa
-
 from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision: str = "20260217_0006"

alembic/versions/20260217_0007_scholar_initial_page_fingerprint.py

@@ -7,9 +7,8 @@ Create Date: 2026-02-17 18:55:00.000000
 
 from collections.abc import Sequence
 
-import sqlalchemy as sa
-
 from alembic import op
+import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
 revision: str = "20260217_0007"

alembic/versions/20260219_0008_user_settings_nav_visible_pages.py

@@ -7,10 +7,10 @@ Create Date: 2026-02-19 14:58:00.000000
 
 from collections.abc import Sequence
 
+from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
-from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "20260219_0008"
@@ -21,7 +21,7 @@ depends_on: str | Sequence[str] | None = None
 
 TABLE_NAME = "user_settings"
 DEFAULT_NAV_VISIBLE_PAGES_SQL = (
-    '\'["dashboard","scholars","publications","settings","style-guide","runs","users"]\'::jsonb'
+    "'[\"dashboard\",\"scholars\",\"publications\",\"settings\",\"style-guide\",\"runs\",\"users\"]'::jsonb"
 )
 
 

alembic/versions/20260219_0009_user_settings_scrape_safety_state.py

@@ -7,10 +7,10 @@ Create Date: 2026-02-19 21:20:00.000000
 
 from collections.abc import Sequence
 
+from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
-from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "20260219_0009"

alembic/versions/20260219_0010_author_search_shared_state.py

@@ -7,10 +7,10 @@ Create Date: 2026-02-19 22:20:00.000000
 
 from collections.abc import Sequence
 
+from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
-from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "20260219_0010"

alembic/versions/20260220_0011_add_publication_doi.py

@@ -7,9 +7,9 @@ Create Date: 2026-02-20 13:35:00.000000
 
 from collections.abc import Sequence
 
+from alembic import op
 import sqlalchemy as sa
 
-from alembic import op
 
 revision: str = "20260220_0011"
 down_revision: str | Sequence[str] | None = "20260219_0010"

alembic/versions/20260220_0012_add_data_repair_jobs.py

@@ -7,10 +7,10 @@ Create Date: 2026-02-20 23:35:00.000000
 
 from collections.abc import Sequence
 
+from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
-from alembic import op
 
 revision: str = "20260220_0012"
 down_revision: str | Sequence[str] | None = "20260220_0011"

alembic/versions/20260221_0013_add_publication_pdf_jobs.py

@@ -7,9 +7,9 @@ Create Date: 2026-02-21 12:25:00.000000
 
 from collections.abc import Sequence
 
+from alembic import op
 import sqlalchemy as sa
 
-from alembic import op
 
 revision: str = "20260221_0013"
 down_revision: str | Sequence[str] | None = "20260220_0012"

alembic/versions/20260221_0014_add_scholar_publication_favorite_state.py

@@ -7,9 +7,9 @@ Create Date: 2026-02-21 14:35:00
 
 from __future__ import annotations
 
+from alembic import op
 import sqlalchemy as sa
 
-from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = "20260221_0014"

alembic/versions/20260221_0015_enforce_request_delay_floor.py

@@ -9,9 +9,9 @@ from __future__ import annotations
 
 from collections.abc import Sequence
 
+from alembic import op
 import sqlalchemy as sa
 
-from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "20260221_0015"
@@ -30,7 +30,11 @@ LEGACY_CHECK_NAME_MIN_2 = "ck_user_settings_request_delay_seconds_min_2"
 def _check_constraints() -> set[str]:
     bind = op.get_bind()
     inspector = sa.inspect(bind)
-    return {str(item.get("name")) for item in inspector.get_check_constraints(TABLE_NAME) if item.get("name")}
+    return {
+        str(item.get("name"))
+        for item in inspector.get_check_constraints(TABLE_NAME)
+        if item.get("name")
+    }
 
 
 def _drop_check_if_exists(name: str) -> None:
@@ -44,7 +48,12 @@ def _drop_check_if_exists(name: str) -> None:
 
 
 def upgrade() -> None:
-    op.execute(sa.text("UPDATE user_settings SET request_delay_seconds = 2 WHERE request_delay_seconds < 2"))
+    op.execute(
+        sa.text(
+            "UPDATE user_settings SET request_delay_seconds = 2 "
+            "WHERE request_delay_seconds < 2"
+        )
+    )
 
     _drop_check_if_exists(CHECK_NAME_MIN_1)
     _drop_check_if_exists(LEGACY_CHECK_NAME_MIN_1)

alembic/versions/20260222_0016_add_publication_identifiers.py

@@ -1,104 +0,0 @@
-"""Add publication identifiers table.
-
-Revision ID: 20260222_0016
-Revises: 20260221_0015
-Create Date: 2026-02-22 12:45:00.000000
-"""
-
-from collections.abc import Sequence
-
-import sqlalchemy as sa
-
-from alembic import op
-
-revision: str = "20260222_0016"
-down_revision: str | Sequence[str] | None = "20260221_0015"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def _table_names() -> set[str]:
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-    return set(inspector.get_table_names())
-
-
-def _create_publication_identifiers_table() -> None:
-    op.create_table(
-        "publication_identifiers",
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("publication_id", sa.Integer(), nullable=False),
-        sa.Column("kind", sa.String(length=32), nullable=False),
-        sa.Column("value_raw", sa.Text(), nullable=False),
-        sa.Column("value_normalized", sa.String(length=255), nullable=False),
-        sa.Column("source", sa.String(length=64), nullable=False),
-        sa.Column(
-            "confidence_score",
-            sa.Float(),
-            nullable=False,
-            server_default=sa.text("0"),
-        ),
-        sa.Column("evidence_url", sa.Text(), nullable=True),
-        sa.Column(
-            "created_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column(
-            "updated_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.CheckConstraint(
-            "confidence_score >= 0 AND confidence_score <= 1",
-            name="publication_identifiers_confidence_score_range",
-        ),
-        sa.ForeignKeyConstraint(
-            ["publication_id"],
-            ["publications.id"],
-            ondelete="CASCADE",
-        ),
-        sa.PrimaryKeyConstraint("id", name=op.f("pk_publication_identifiers")),
-        sa.UniqueConstraint(
-            "publication_id",
-            "kind",
-            "value_normalized",
-            name="uq_publication_identifiers_publication_kind_value",
-        ),
-    )
-
-
-def _create_publication_identifiers_indexes() -> None:
-    op.create_index(
-        "ix_publication_identifiers_kind_value",
-        "publication_identifiers",
-        ["kind", "value_normalized"],
-        unique=False,
-    )
-    op.create_index(
-        "ix_publication_identifiers_publication_id",
-        "publication_identifiers",
-        ["publication_id"],
-        unique=False,
-    )
-
-
-def _drop_publication_identifiers_indexes() -> None:
-    op.drop_index("ix_publication_identifiers_publication_id", table_name="publication_identifiers")
-    op.drop_index("ix_publication_identifiers_kind_value", table_name="publication_identifiers")
-
-
-def upgrade() -> None:
-    if "publication_identifiers" in _table_names():
-        return
-    _create_publication_identifiers_table()
-    _create_publication_identifiers_indexes()
-
-
-def downgrade() -> None:
-    if "publication_identifiers" not in _table_names():
-        return
-    _drop_publication_identifiers_indexes()
-    op.drop_table("publication_identifiers")

alembic/versions/20260224_0018_add_openalex_enriched_to_publications.py

@@ -1,33 +0,0 @@
-"""Add openalex_enriched to publications
-
-Revision ID: 20260224_0018
-Revises: fa0c5e3262d5
-Create Date: 2026-02-24 19:10:00.000000
-
-"""
-
-from collections.abc import Sequence
-
-import sqlalchemy as sa
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision: str = "20260224_0018"
-down_revision: str | Sequence[str] | None = "fa0c5e3262d5"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column(
-        "publications", sa.Column("openalex_enriched", sa.Boolean(), server_default=sa.text("false"), nullable=False)
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_column("publications", "openalex_enriched")
-    # ### end Alembic commands ###

alembic/versions/20260224_0019_add_resolving_to_runstatus.py

@@ -1,31 +0,0 @@
-"""Add resolving to RunStatus enum
-
-Revision ID: 20260224_0019
-Revises: 20260224_0018
-Create Date: 2026-02-24 22:20:00.000000
-
-"""
-
-from collections.abc import Sequence
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision: str = "20260224_0019"
-down_revision: str | Sequence[str] | None = "20260224_0018"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def upgrade() -> None:
-    # Use COMMIT to break out of the transaction block that Alembic creates in env.py.
-    # Postgres restricts ALTER TYPE ... ADD VALUE inside transactions when the type is in use.
-    # This is the robust, non-patchwork way to ensure the schema evolves correctly.
-    op.execute("COMMIT")
-    op.execute("ALTER TYPE run_status ADD VALUE IF NOT EXISTS 'resolving' AFTER 'running'")
-
-
-def downgrade() -> None:
-    # Enum values cannot be easily removed in Postgres without recreating the type,
-    # which is risky and usually avoided in simple migrations.
-    pass

alembic/versions/20260224_0020_add_openalex_last_attempt_at.py

@@ -1,27 +0,0 @@
-"""Add openalex_last_attempt_at to publications
-
-Revision ID: 20260224_0020
-Revises: 20260224_0019
-Create Date: 2026-02-24 22:50:00.000000
-
-"""
-
-from collections.abc import Sequence
-
-import sqlalchemy as sa
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision: str = "20260224_0020"
-down_revision: str | Sequence[str] | None = "20260224_0019"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def upgrade() -> None:
-    op.add_column("publications", sa.Column("openalex_last_attempt_at", sa.DateTime(timezone=True), nullable=True))
-
-
-def downgrade() -> None:
-    op.drop_column("publications", "openalex_last_attempt_at")

alembic/versions/20260225_0021_enforce_single_active_run_per_user.py

@@ -1,68 +0,0 @@
-"""Enforce one active crawl run per user.
-
-Revision ID: 20260225_0021
-Revises: 20260224_0020
-Create Date: 2026-02-25 09:10:00.000000
-"""
-
-from collections.abc import Sequence
-
-import sqlalchemy as sa
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision: str = "20260225_0021"
-down_revision: str | Sequence[str] | None = "20260224_0020"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-INDEX_NAME = "uq_crawl_runs_user_active"
-ACTIVE_STATUSES = ("running", "resolving")
-
-
-def upgrade() -> None:
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-    indexes = {index["name"] for index in inspector.get_indexes("crawl_runs")}
-
-    op.execute("ALTER TYPE run_status ADD VALUE IF NOT EXISTS 'canceled'")
-
-    op.execute(
-        """
-        WITH ranked AS (
-            SELECT
-                id,
-                ROW_NUMBER() OVER (
-                    PARTITION BY user_id
-                    ORDER BY start_dt DESC, id DESC
-                ) AS rn
-            FROM crawl_runs
-            WHERE status IN ('running', 'resolving')
-        )
-        UPDATE crawl_runs AS runs
-        SET
-            status = 'failed',
-            end_dt = COALESCE(runs.end_dt, NOW())
-        FROM ranked
-        WHERE runs.id = ranked.id
-          AND ranked.rn > 1
-        """
-    )
-
-    if INDEX_NAME not in indexes:
-        op.create_index(
-            INDEX_NAME,
-            "crawl_runs",
-            ["user_id"],
-            unique=True,
-            postgresql_where=sa.text("status IN ('running'::run_status, 'resolving'::run_status)"),
-        )
-
-
-def downgrade() -> None:
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-    indexes = {index["name"] for index in inspector.get_indexes("crawl_runs")}
-    if INDEX_NAME in indexes:
-        op.drop_index(INDEX_NAME, table_name="crawl_runs")

alembic/versions/20260225_0022_add_canonical_title_hash_to_publications.py

@@ -1,35 +0,0 @@
-"""Add canonical_title_hash to publications for cross-scholar dedup.
-
-Revision ID: 20260225_0022
-Revises: 20260225_0021
-Create Date: 2026-02-25 10:00:00.000000
-"""
-
-from collections.abc import Sequence
-
-import sqlalchemy as sa
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision: str = "20260225_0022"
-down_revision: str | Sequence[str] | None = "20260225_0021"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "publications",
-        sa.Column("canonical_title_hash", sa.String(64), nullable=True),
-    )
-    op.create_index(
-        "ix_publications_canonical_title_hash",
-        "publications",
-        ["canonical_title_hash"],
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_publications_canonical_title_hash", table_name="publications")
-    op.drop_column("publications", "canonical_title_hash")

alembic/versions/20260226_0023_add_arxiv_runtime_state.py

@@ -1,55 +0,0 @@
-"""Add shared arXiv runtime state table.
-
-Revision ID: 20260226_0023
-Revises: 20260225_0022
-Create Date: 2026-02-26 12:40:00.000000
-"""
-
-from collections.abc import Sequence
-
-import sqlalchemy as sa
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision: str = "20260226_0023"
-down_revision: str | Sequence[str] | None = "20260225_0022"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def upgrade() -> None:
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-    table_names = set(inspector.get_table_names())
-
-    if "arxiv_runtime_state" in table_names:
-        return
-
-    op.create_table(
-        "arxiv_runtime_state",
-        sa.Column("state_key", sa.String(length=64), nullable=False),
-        sa.Column("next_allowed_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("cooldown_until", sa.DateTime(timezone=True), nullable=True),
-        sa.Column(
-            "created_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column(
-            "updated_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.PrimaryKeyConstraint("state_key", name=op.f("pk_arxiv_runtime_state")),
-    )
-
-
-def downgrade() -> None:
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-    table_names = set(inspector.get_table_names())
-    if "arxiv_runtime_state" in table_names:
-        op.drop_table("arxiv_runtime_state")

alembic/versions/20260226_0024_add_arxiv_query_cache_entries.py

@@ -1,70 +0,0 @@
-"""Add arXiv query cache table.
-
-Revision ID: 20260226_0024
-Revises: 20260226_0023
-Create Date: 2026-02-26 14:20:00.000000
-"""
-
-from collections.abc import Sequence
-
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision: str = "20260226_0024"
-down_revision: str | Sequence[str] | None = "20260226_0023"
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def upgrade() -> None:
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-    table_names = set(inspector.get_table_names())
-    if "arxiv_query_cache_entries" in table_names:
-        return
-
-    op.create_table(
-        "arxiv_query_cache_entries",
-        sa.Column("query_fingerprint", sa.String(length=64), nullable=False),
-        sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
-        sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
-        sa.Column(
-            "cached_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.Column(
-            "updated_at",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.func.now(),
-        ),
-        sa.PrimaryKeyConstraint(
-            "query_fingerprint",
-            name=op.f("pk_arxiv_query_cache_entries"),
-        ),
-    )
-    op.create_index(
-        "ix_arxiv_query_cache_expires_at",
-        "arxiv_query_cache_entries",
-        ["expires_at"],
-        unique=False,
-    )
-    op.create_index(
-        "ix_arxiv_query_cache_cached_at",
-        "arxiv_query_cache_entries",
-        ["cached_at"],
-        unique=False,
-    )
-
-
-def downgrade() -> None:
-    bind = op.get_bind()
-    inspector = sa.inspect(bind)
-    table_names = set(inspector.get_table_names())
-    if "arxiv_query_cache_entries" in table_names:
-        op.drop_table("arxiv_query_cache_entries")

alembic/versions/44f7e10ef777_remove_doi_from_publications_and_.py

@@ -1,27 +0,0 @@
-"""remove_doi_from_publications_and_migrate_identifiers
-
-Revision ID: 44f7e10ef777
-Revises: 20260222_0016
-Create Date: 2026-02-22 17:03:56.261936
-
-"""
-from collections.abc import Sequence
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision: str = '44f7e10ef777'
-down_revision: str | Sequence[str] | None = '20260222_0016'
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def upgrade() -> None:
-    pass
-
-
-def downgrade() -> None:
-    pass
-

alembic/versions/fa0c5e3262d5_add_api_integration_keys_to_usersetting.py

@@ -1,39 +0,0 @@
-"""Add API integration keys to UserSetting
-
-Revision ID: fa0c5e3262d5
-Revises: 44f7e10ef777
-Create Date: 2026-02-23 14:25:45.021512
-
-"""
-from collections.abc import Sequence
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision: str = 'fa0c5e3262d5'
-down_revision: str | Sequence[str] | None = '44f7e10ef777'
-branch_labels: str | Sequence[str] | None = None
-depends_on: str | Sequence[str] | None = None
-
-
-def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_index(op.f('ix_publications_doi'), table_name='publications')
-    op.drop_column('publications', 'doi')
-    op.add_column('user_settings', sa.Column('openalex_api_key', sa.String(length=255), nullable=True))
-    op.add_column('user_settings', sa.Column('crossref_api_token', sa.String(length=255), nullable=True))
-    op.add_column('user_settings', sa.Column('crossref_api_mailto', sa.String(length=255), nullable=True))
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_column('user_settings', 'crossref_api_mailto')
-    op.drop_column('user_settings', 'crossref_api_token')
-    op.drop_column('user_settings', 'openalex_api_key')
-    op.add_column('publications', sa.Column('doi', sa.VARCHAR(length=255), autoincrement=False, nullable=True))
-    op.create_index(op.f('ix_publications_doi'), 'publications', ['doi'], unique=False)
-    # ### end Alembic commands ###
-

app/api/__init__.py

@@ -1 +1,2 @@
 from __future__ import annotations
+

app/api/errors.py

@@ -82,3 +82,4 @@ def register_api_exception_handlers(app: FastAPI) -> None:
             message="Request validation failed.",
             details=exc.errors(),
         )
+

app/api/media.py

@@ -10,8 +10,7 @@ from app.api.deps import get_api_current_user
 from app.api.errors import ApiException
 from app.db.models import User
 from app.db.session import get_db_session
-from app.services.scholars import application as scholar_service
-from app.services.scholars import uploads as scholar_uploads
+from app.services.domains.scholars import application as scholar_service
 from app.settings import settings
 
 router = APIRouter(tags=["media"])
@@ -42,7 +41,7 @@ async def get_uploaded_scholar_image(
         )
 
     try:
-        image_path = scholar_uploads.resolve_upload_file_path(
+        image_path = scholar_service.resolve_upload_file_path(
             upload_dir=settings.scholar_image_upload_dir,
             relative_path=profile.profile_image_upload_path,
         )

app/api/router.py

@@ -2,12 +2,11 @@ from __future__ import annotations
 
 from fastapi import APIRouter
 
-from app.api.routers import admin, admin_dbops, admin_settings, auth, publications, runs, scholars, settings
+from app.api.routers import admin, admin_dbops, auth, publications, runs, scholars, settings
 
 router = APIRouter(prefix="/api/v1")
 router.include_router(auth.router)
 router.include_router(admin.router)
-router.include_router(admin_settings.router)
 router.include_router(admin_dbops.router)
 router.include_router(scholars.router)
 router.include_router(settings.router)

app/api/routers/__init__.py

@@ -1 +1,2 @@
 from __future__ import annotations
+

app/api/routers/admin.py

@@ -20,8 +20,7 @@ from app.auth.deps import get_auth_service
 from app.auth.service import AuthService
 from app.db.models import User
 from app.db.session import get_db_session
-from app.logging_utils import structured_log
-from app.services.users import application as user_service
+from app.services.domains.users import application as user_service
 
 logger = logging.getLogger(__name__)
 
@@ -49,7 +48,14 @@ async def list_users(
     admin_user: User = Depends(get_api_admin_user),
 ):
     users = await user_service.list_users(db_session)
-    structured_log(logger, "info", "api.admin.users_listed", admin_user_id=int(admin_user.id), user_count=len(users))
+    logger.info(
+        "api.admin.users_listed",
+        extra={
+            "event": "api.admin.users_listed",
+            "admin_user_id": int(admin_user.id),
+            "user_count": len(users),
+        },
+    )
     return success_payload(
         request,
         data={
@@ -86,13 +92,14 @@ async def create_user(
             message=str(exc),
         ) from exc
 
-    structured_log(
-        logger,
-        "info",
+    logger.info(
         "api.admin.user_created",
-        admin_user_id=int(admin_user.id),
-        target_user_id=int(created_user.id),
-        target_is_admin=bool(created_user.is_admin),
+        extra={
+            "event": "api.admin.user_created",
+            "admin_user_id": int(admin_user.id),
+            "target_user_id": int(created_user.id),
+            "target_is_admin": bool(created_user.is_admin),
+        },
     )
     return success_payload(
         request,
@@ -118,7 +125,11 @@ async def set_user_active(
             code="user_not_found",
             message="User not found.",
         )
-    if int(target_user.id) == int(admin_user.id) and bool(target_user.is_active) and not bool(payload.is_active):
+    if (
+        int(target_user.id) == int(admin_user.id)
+        and bool(target_user.is_active)
+        and not bool(payload.is_active)
+    ):
         raise ApiException(
             status_code=400,
             code="cannot_deactivate_self",
@@ -129,13 +140,14 @@ async def set_user_active(
         user=target_user,
         is_active=bool(payload.is_active),
     )
-    structured_log(
-        logger,
-        "info",
+    logger.info(
         "api.admin.user_active_updated",
-        admin_user_id=int(admin_user.id),
-        target_user_id=int(updated_user.id),
-        is_active=bool(updated_user.is_active),
+        extra={
+            "event": "api.admin.user_active_updated",
+            "admin_user_id": int(admin_user.id),
+            "target_user_id": int(updated_user.id),
+            "is_active": bool(updated_user.is_active),
+        },
     )
     return success_payload(
         request,
@@ -176,12 +188,13 @@ async def reset_user_password(
         user=target_user,
         password_hash=auth_service.hash_password(validated_password),
     )
-    structured_log(
-        logger,
-        "info",
+    logger.info(
         "api.admin.user_password_reset",
-        admin_user_id=int(admin_user.id),
-        target_user_id=int(target_user.id),
+        extra={
+            "event": "api.admin.user_password_reset",
+            "admin_user_id": int(admin_user.id),
+            "target_user_id": int(target_user.id),
+        },
     )
     return success_payload(
         request,

app/api/routers/admin_dbops.py

@@ -5,30 +5,26 @@ import logging
 from fastapi import APIRouter, Depends, Path, Query, Request
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.api.deps import get_api_admin_user, get_api_current_user
+from app.api.deps import get_api_admin_user
 from app.api.errors import ApiException
 from app.api.responses import success_payload
 from app.api.schemas import (
     AdminDbIntegrityEnvelope,
-    AdminDbRepairJobsEnvelope,
     AdminPdfQueueBulkEnqueueEnvelope,
-    AdminPdfQueueEnvelope,
     AdminPdfQueueRequeueEnvelope,
+    AdminPdfQueueEnvelope,
+    AdminDbRepairJobsEnvelope,
     AdminRepairPublicationLinksEnvelope,
     AdminRepairPublicationLinksRequest,
-    AdminRepairPublicationNearDuplicatesEnvelope,
-    AdminRepairPublicationNearDuplicatesRequest,
 )
 from app.db.models import DataRepairJob, User
 from app.db.session import get_db_session
-from app.logging_utils import structured_log
-from app.services.dbops import (
+from app.services.domains.dbops import (
     collect_integrity_report,
     list_repair_jobs,
     run_publication_link_repair,
-    run_publication_near_duplicate_repair,
 )
-from app.services.publications import application as publication_service
+from app.services.domains.publications import application as publication_service
 
 logger = logging.getLogger(__name__)
 
@@ -52,16 +48,16 @@ def _serialize_repair_job(job: DataRepairJob) -> dict[str, object]:
     }
 
 
-def _requested_by_value(*, payload, admin_user: User) -> str:
+def _requested_by_value(*, payload: AdminRepairPublicationLinksRequest, admin_user: User) -> str:
     from_payload = (payload.requested_by or "").strip()
     return from_payload or admin_user.email
 
 
-def _serialize_pdf_queue_item(item, *, is_admin: bool = False) -> dict[str, object]:
+def _serialize_pdf_queue_item(item) -> dict[str, object]:
     return {
         "publication_id": item.publication_id,
         "title": item.title,
-        "display_identifier": _serialize_display_identifier(item.display_identifier),
+        "doi": item.doi,
         "pdf_url": item.pdf_url,
         "status": item.status,
         "attempt_count": item.attempt_count,
@@ -69,7 +65,7 @@ def _serialize_pdf_queue_item(item, *, is_admin: bool = False) -> dict[str, obje
         "last_failure_detail": item.last_failure_detail,
         "last_source": item.last_source,
         "requested_by_user_id": item.requested_by_user_id,
-        "requested_by_email": item.requested_by_email if is_admin else None,
+        "requested_by_email": item.requested_by_email,
         "queued_at": item.queued_at,
         "last_attempt_at": item.last_attempt_at,
         "resolved_at": item.resolved_at,
@@ -77,18 +73,6 @@ def _serialize_pdf_queue_item(item, *, is_admin: bool = False) -> dict[str, obje
     }
 
 
-def _serialize_display_identifier(value) -> dict[str, object] | None:
-    if value is None:
-        return None
-    return {
-        "kind": value.kind,
-        "value": value.value,
-        "label": value.label,
-        "url": value.url,
-        "confidence_score": float(value.confidence_score),
-    }
-
-
 def _requeue_response_state(*, queued: bool) -> tuple[str, str]:
     if queued:
         return "queued", "PDF lookup queued."
@@ -136,14 +120,15 @@ async def get_integrity_report(
     admin_user: User = Depends(get_api_admin_user),
 ):
     report = await collect_integrity_report(db_session)
-    structured_log(
-        logger,
-        "info",
+    logger.info(
         "api.admin.db.integrity_checked",
-        admin_user_id=int(admin_user.id),
-        status=report.get("status"),
-        failure_count=len(report.get("failures", [])),
-        warning_count=len(report.get("warnings", [])),
+        extra={
+            "event": "api.admin.db.integrity_checked",
+            "admin_user_id": int(admin_user.id),
+            "status": report.get("status"),
+            "failure_count": len(report.get("failures", [])),
+            "warning_count": len(report.get("warnings", [])),
+        },
     )
     return success_payload(request, data=report)
 
@@ -159,13 +144,14 @@ async def get_repair_jobs(
     admin_user: User = Depends(get_api_admin_user),
 ):
     jobs = await list_repair_jobs(db_session, limit=limit)
-    structured_log(
-        logger,
-        "info",
+    logger.info(
         "api.admin.db.repair_jobs_listed",
-        admin_user_id=int(admin_user.id),
-        limit=int(limit),
-        job_count=len(jobs),
+        extra={
+            "event": "api.admin.db.repair_jobs_listed",
+            "admin_user_id": int(admin_user.id),
+            "limit": int(limit),
+            "job_count": len(jobs),
+        },
     )
     return success_payload(
         request,
@@ -185,7 +171,7 @@ async def get_pdf_queue(
     offset: int | None = Query(default=None, ge=0),
     status: str | None = Query(default=None),
     db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
+    admin_user: User = Depends(get_api_admin_user),
 ):
     normalized_status = (status or "").strip().lower() or None
     resolved_page, resolved_limit, resolved_offset = _resolve_pdf_queue_paging(
@@ -200,22 +186,23 @@ async def get_pdf_queue(
         offset=resolved_offset,
         status=normalized_status,
     )
-    structured_log(
-        logger,
-        "info",
+    logger.info(
         "api.admin.db.pdf_queue_listed",
-        user_id=int(current_user.id),
-        page=int(resolved_page),
-        page_size=int(resolved_limit),
-        offset=int(resolved_offset),
-        status=normalized_status,
-        item_count=len(queue_page.items),
-        total_count=int(queue_page.total_count),
+        extra={
+            "event": "api.admin.db.pdf_queue_listed",
+            "admin_user_id": int(admin_user.id),
+            "page": int(resolved_page),
+            "page_size": int(resolved_limit),
+            "offset": int(resolved_offset),
+            "status": normalized_status,
+            "item_count": len(queue_page.items),
+            "total_count": int(queue_page.total_count),
+        },
     )
     return success_payload(
         request,
         data={
-            "items": [_serialize_pdf_queue_item(item, is_admin=current_user.is_admin) for item in queue_page.items],
+            "items": [_serialize_pdf_queue_item(item) for item in queue_page.items],
             **_pdf_queue_page_data(
                 total_count=queue_page.total_count,
                 page=resolved_page,
@@ -249,13 +236,14 @@ async def requeue_pdf_lookup(
             message="Publication not found.",
         )
     status, message = _requeue_response_state(queued=result.queued)
-    structured_log(
-        logger,
-        "info",
-        "api.admin.db.pdf_requeued",
-        admin_user_id=int(admin_user.id),
-        publication_id=int(publication_id),
-        queued=bool(result.queued),
+    logger.info(
+        "api.admin.db.pdf_queue_requeue_requested",
+        extra={
+            "event": "api.admin.db.pdf_queue_requeue_requested",
+            "admin_user_id": int(admin_user.id),
+            "publication_id": int(publication_id),
+            "queued": bool(result.queued),
+        },
     )
     return success_payload(
         request,
@@ -284,14 +272,15 @@ async def requeue_all_missing_pdfs(
         request_email=admin_user.email,
         limit=limit,
     )
-    structured_log(
-        logger,
-        "info",
+    logger.info(
         "api.admin.db.pdf_queue_requeue_all",
-        admin_user_id=int(admin_user.id),
-        limit=int(limit),
-        requested_count=int(result.requested_count),
-        queued_count=int(result.queued_count),
+        extra={
+            "event": "api.admin.db.pdf_queue_requeue_all",
+            "admin_user_id": int(admin_user.id),
+            "limit": int(limit),
+            "requested_count": int(result.requested_count),
+            "queued_count": int(result.queued_count),
+        },
     )
     return success_payload(
         request,
@@ -332,114 +321,17 @@ async def trigger_publication_link_repair(
             code="invalid_repair_scope",
             message=str(exc),
         ) from exc
-    structured_log(
-        logger,
-        "info",
+    logger.info(
         "api.admin.db.publication_link_repair_triggered",
-        admin_user_id=int(admin_user.id),
-        scope_mode=payload.scope_mode,
-        target_user_id=int(payload.user_id) if payload.user_id is not None else None,
-        dry_run=bool(payload.dry_run),
-        gc_orphan_publications=bool(payload.gc_orphan_publications),
-        job_id=int(result["job_id"]),
-        status=result["status"],
-    )
-    return success_payload(request, data=result)
-
-
-@router.post(
-    "/repairs/publication-near-duplicates",
-    response_model=AdminRepairPublicationNearDuplicatesEnvelope,
-)
-async def trigger_publication_near_duplicate_repair(
-    payload: AdminRepairPublicationNearDuplicatesRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    admin_user: User = Depends(get_api_admin_user),
-):
-    try:
-        result = await run_publication_near_duplicate_repair(
-            db_session,
-            dry_run=bool(payload.dry_run),
-            similarity_threshold=float(payload.similarity_threshold),
-            min_shared_tokens=int(payload.min_shared_tokens),
-            max_year_delta=int(payload.max_year_delta),
-            max_clusters=int(payload.max_clusters),
-            selected_cluster_keys=list(payload.selected_cluster_keys),
-            requested_by=_requested_by_value(payload=payload, admin_user=admin_user),
-        )
-    except ValueError as exc:
-        raise ApiException(
-            status_code=400,
-            code="invalid_near_duplicate_repair_request",
-            message=str(exc),
-        ) from exc
-    structured_log(
-        logger,
-        "info",
-        "api.admin.db.dedup_repair_triggered",
-        admin_user_id=int(admin_user.id),
-        dry_run=bool(payload.dry_run),
-        selected_cluster_count=len(payload.selected_cluster_keys),
-        job_id=int(result["job_id"]),
-        status=result["status"],
-    )
-    return success_payload(request, data=result)
-
-
-DROP_PUBLICATIONS_CONFIRMATION = "DROP ALL PUBLICATIONS"
-
-
-@router.post("/drop-all-publications")
-async def drop_all_publications(
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    admin_user: User = Depends(get_api_admin_user),
-):
-    body = await request.json()
-    confirmation_text = (body.get("confirmation_text") or "").strip()
-    if confirmation_text != DROP_PUBLICATIONS_CONFIRMATION:
-        raise ApiException(
-            status_code=400,
-            code="confirmation_required",
-            message=f"Type '{DROP_PUBLICATIONS_CONFIRMATION}' to confirm this destructive action.",
-        )
-
-    from sqlalchemy import delete, func, select, update
-
-    from app.db.models import (
-        Publication,
-        PublicationIdentifier,
-        PublicationPdfJob,
-        PublicationPdfJobEvent,
-        ScholarProfile,
-        ScholarPublication,
-    )
-
-    count_result = await db_session.execute(select(func.count()).select_from(Publication))
-    total_publications = count_result.scalar_one()
-
-    await db_session.execute(delete(ScholarPublication))
-    await db_session.execute(delete(PublicationIdentifier))
-    await db_session.execute(delete(PublicationPdfJobEvent))
-    await db_session.execute(delete(PublicationPdfJob))
-    await db_session.execute(delete(Publication))
-    await db_session.execute(update(ScholarProfile).values(baseline_completed=False))
-    await db_session.commit()
-
-    structured_log(
-        logger,
-        "warning",
-        "api.admin.db.all_publications_dropped",
-        admin_user_id=int(admin_user.id),
-        admin_email=admin_user.email,
-        deleted_count=int(total_publications),
-    )
-    return success_payload(
-        request,
-        data={
-            "deleted_count": int(total_publications),
-            "message": f"Dropped {total_publications} publication(s) and all related data. "
-            "Scholar baselines have been reset; the next run will re-discover all publications.",
+        extra={
+            "event": "api.admin.db.publication_link_repair_triggered",
+            "admin_user_id": int(admin_user.id),
+            "scope_mode": payload.scope_mode,
+            "target_user_id": int(payload.user_id) if payload.user_id is not None else None,
+            "dry_run": bool(payload.dry_run),
+            "gc_orphan_publications": bool(payload.gc_orphan_publications),
+            "job_id": int(result["job_id"]),
+            "status": result["status"],
         },
     )
+    return success_payload(request, data=result)

app/api/routers/admin_settings.py

@@ -1,104 +0,0 @@
-from __future__ import annotations
-
-import logging
-
-from fastapi import APIRouter, Depends, Request
-
-from app.api.deps import get_api_admin_user
-from app.api.errors import ApiException
-from app.api.responses import success_payload
-from app.api.schemas import (
-    AdminScholarHttpSettingsEnvelope,
-    AdminScholarHttpSettingsUpdateRequest,
-)
-from app.db.models import User
-from app.logging_utils import structured_log
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/admin/settings", tags=["api-admin-settings"])
-_MAX_USER_AGENT_LENGTH = 512
-_MAX_ACCEPT_LANGUAGE_LENGTH = 128
-_MAX_COOKIE_LENGTH = 8192
-
-
-def _normalize_header_value(value: str, *, field_name: str, max_length: int) -> str:
-    normalized = value.strip()
-    if len(normalized) > max_length:
-        raise ApiException(
-            status_code=400,
-            code="invalid_admin_setting",
-            message=f"{field_name} must be {max_length} characters or fewer.",
-        )
-    return normalized
-
-
-def _serialize_scholar_http_settings() -> dict[str, object]:
-    return {
-        "user_agent": settings.scholar_http_user_agent,
-        "rotate_user_agent": bool(settings.scholar_http_rotate_user_agent),
-        "accept_language": settings.scholar_http_accept_language,
-        "cookie": settings.scholar_http_cookie,
-    }
-
-
-def _apply_scholar_http_settings(payload: AdminScholarHttpSettingsUpdateRequest) -> None:
-    user_agent = _normalize_header_value(
-        payload.user_agent,
-        field_name="user_agent",
-        max_length=_MAX_USER_AGENT_LENGTH,
-    )
-    accept_language = _normalize_header_value(
-        payload.accept_language,
-        field_name="accept_language",
-        max_length=_MAX_ACCEPT_LANGUAGE_LENGTH,
-    )
-    cookie = _normalize_header_value(
-        payload.cookie or "",
-        field_name="cookie",
-        max_length=_MAX_COOKIE_LENGTH,
-    )
-    object.__setattr__(settings, "scholar_http_user_agent", user_agent)
-    object.__setattr__(settings, "scholar_http_rotate_user_agent", bool(payload.rotate_user_agent))
-    object.__setattr__(settings, "scholar_http_accept_language", accept_language)
-    object.__setattr__(settings, "scholar_http_cookie", cookie)
-
-
-@router.get(
-    "/scholar-http",
-    response_model=AdminScholarHttpSettingsEnvelope,
-)
-async def get_scholar_http_settings(
-    request: Request,
-    admin_user: User = Depends(get_api_admin_user),
-):
-    structured_log(
-        logger,
-        "info",
-        "api.admin.settings.scholar_http_read",
-        admin_user_id=int(admin_user.id),
-    )
-    return success_payload(request, data=_serialize_scholar_http_settings())
-
-
-@router.put(
-    "/scholar-http",
-    response_model=AdminScholarHttpSettingsEnvelope,
-)
-async def update_scholar_http_settings(
-    payload: AdminScholarHttpSettingsUpdateRequest,
-    request: Request,
-    admin_user: User = Depends(get_api_admin_user),
-):
-    _apply_scholar_http_settings(payload)
-    structured_log(
-        logger,
-        "info",
-        "api.admin.settings.scholar_http_updated",
-        admin_user_id=int(admin_user.id),
-        rotate_user_agent=bool(settings.scholar_http_rotate_user_agent),
-        user_agent_set=bool(settings.scholar_http_user_agent.strip()),
-        cookie_set=bool(settings.scholar_http_cookie.strip()),
-    )
-    return success_payload(request, data=_serialize_scholar_http_settings())

app/api/routers/auth.py

@@ -1,13 +1,12 @@
 from __future__ import annotations
 
 import logging
-from typing import NoReturn
 
 from fastapi import APIRouter, Depends, Request
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.api.deps import get_api_current_user
 from app.api.errors import ApiException
+from app.api.deps import get_api_current_user
 from app.api.responses import success_payload
 from app.api.schemas import (
     AuthMeEnvelope,
@@ -17,16 +16,15 @@ from app.api.schemas import (
     LoginRequest,
     MessageEnvelope,
 )
-from app.auth import runtime as auth_runtime
 from app.auth.deps import get_auth_service, get_login_rate_limiter
 from app.auth.rate_limit import SlidingWindowRateLimiter
+from app.auth import runtime as auth_runtime
 from app.auth.service import AuthService
 from app.auth.session import set_session_user
 from app.db.models import User
 from app.db.session import get_db_session
-from app.logging_utils import structured_log
 from app.security.csrf import ensure_csrf_token
-from app.services.users import application as user_service
+from app.services.domains.users import application as user_service
 
 logger = logging.getLogger(__name__)
 
@@ -38,12 +36,13 @@ def _login_limiter_key_and_email(request: Request, payload: LoginRequest) -> tup
 
 
 def _raise_rate_limited(normalized_email: str, retry_after_seconds: int) -> None:
-    structured_log(
-        logger,
-        "warning",
+    logger.warning(
         "api.auth.login_rate_limited",
-        email=normalized_email,
-        retry_after_seconds=retry_after_seconds,
+        extra={
+            "event": "api.auth.login_rate_limited",
+            "email": normalized_email,
+            "retry_after_seconds": retry_after_seconds,
+        },
     )
     raise ApiException(
         status_code=429,
@@ -62,8 +61,11 @@ def _serialize_user_payload(user: User) -> dict[str, object]:
     }
 
 
-def _raise_invalid_credentials(*, normalized_email: str) -> NoReturn:
-    structured_log(logger, "info", "api.auth.login_failed", email=normalized_email)
+def _raise_invalid_credentials(*, normalized_email: str) -> None:
+    logger.info(
+        "api.auth.login_failed",
+        extra={"event": "api.auth.login_failed", "email": normalized_email},
+    )
     raise ApiException(
         status_code=401,
         code="invalid_credentials",
@@ -103,7 +105,14 @@ async def login(
         email=user.email,
         is_admin=user.is_admin,
     )
-    structured_log(logger, "info", "api.auth.login_succeeded", user_id=user.id, is_admin=user.is_admin)
+    logger.info(
+        "api.auth.login_succeeded",
+        extra={
+            "event": "api.auth.login_succeeded",
+            "user_id": user.id,
+            "is_admin": user.is_admin,
+        },
+    )
     return success_payload(
         request,
         data={
@@ -190,7 +199,13 @@ async def change_password(
         user=current_user,
         password_hash=auth_service.hash_password(validated_password),
     )
-    structured_log(logger, "info", "api.auth.password_changed", user_id=int(current_user.id))
+    logger.info(
+        "api.auth.password_changed",
+        extra={
+            "event": "api.auth.password_changed",
+            "user_id": int(current_user.id),
+        },
+    )
     return success_payload(
         request,
         data={"message": "Password updated successfully."},
@@ -207,8 +222,12 @@ async def logout(
 ):
     current_user = await auth_runtime.get_authenticated_user(request, db_session)
     auth_runtime.invalidate_session(request)
-    structured_log(
-        logger, "info", "api.auth.logout", user_id=int(current_user.id) if current_user is not None else None
+    logger.info(
+        "api.auth.logout",
+        extra={
+            "event": "api.auth.logout",
+            "user_id": int(current_user.id) if current_user is not None else None,
+        },
     )
     return success_payload(
         request,

app/api/routers/publications.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import logging
-from datetime import UTC, datetime
 from typing import Literal
 
 from fastapi import APIRouter, Depends, Path, Query, Request
@@ -22,10 +21,8 @@ from app.api.schemas import (
 )
 from app.db.models import User
 from app.db.session import get_db_session
-from app.logging_utils import structured_log
-from app.services.publication_identifiers import application as identifier_service
-from app.services.publications import application as publication_service
-from app.services.scholars import application as scholar_service
+from app.services.domains.publications import application as publication_service
+from app.services.domains.scholars import application as scholar_service
 from app.settings import settings
 
 logger = logging.getLogger(__name__)
@@ -64,7 +61,7 @@ def _serialize_publication_item(item) -> dict[str, object]:
         "citation_count": item.citation_count,
         "venue_text": item.venue_text,
         "pub_url": item.pub_url,
-        "display_identifier": _serialize_display_identifier(item.display_identifier),
+        "doi": item.doi,
         "pdf_url": item.pdf_url,
         "pdf_status": item.pdf_status,
         "pdf_attempt_count": item.pdf_attempt_count,
@@ -77,46 +74,29 @@ def _serialize_publication_item(item) -> dict[str, object]:
     }
 
 
-def _serialize_display_identifier(value) -> dict[str, object] | None:
-    if value is None:
-        return None
-    return {
-        "kind": value.kind,
-        "value": value.value,
-        "label": value.label,
-        "url": value.url,
-        "confidence_score": float(value.confidence_score),
-    }
-
-
 async def _publication_counts(
     db_session: AsyncSession,
     *,
     user_id: int,
     selected_scholar_id: int | None,
     favorite_only: bool,
-    search: str | None,
-    snapshot_before: datetime | None,
 ) -> tuple[int, int, int, int]:
     unread_count = await publication_service.count_unread_for_user(
         db_session,
         user_id=user_id,
         scholar_profile_id=selected_scholar_id,
         favorite_only=favorite_only,
-        snapshot_before=snapshot_before,
     )
     favorites_count = await publication_service.count_favorite_for_user(
         db_session,
         user_id=user_id,
         scholar_profile_id=selected_scholar_id,
-        snapshot_before=snapshot_before,
     )
     latest_count = await publication_service.count_latest_for_user(
         db_session,
         user_id=user_id,
         scholar_profile_id=selected_scholar_id,
         favorite_only=favorite_only,
-        snapshot_before=snapshot_before,
     )
     total_count = await publication_service.count_for_user(
         db_session,
@@ -124,8 +104,6 @@ async def _publication_counts(
         mode=publication_service.MODE_ALL,
         scholar_profile_id=selected_scholar_id,
         favorite_only=favorite_only,
-        search=search,
-        snapshot_before=snapshot_before,
     )
     return unread_count, favorites_count, latest_count, total_count
 
@@ -137,12 +115,8 @@ async def _list_publications_for_request(
     mode: Literal["all", "unread", "latest", "new"] | None,
     favorite_only: bool,
     scholar_profile_id: int | None,
-    search: str | None,
-    sort_by: str,
-    sort_dir: str,
     limit: int,
     offset: int,
-    snapshot_before: datetime | None,
 ) -> tuple[str, int | None, list]:
     resolved_mode = publication_service.resolve_publication_view_mode(mode)
     selected_scholar_id = scholar_profile_id
@@ -157,12 +131,8 @@ async def _list_publications_for_request(
         mode=resolved_mode,
         scholar_profile_id=selected_scholar_id,
         favorite_only=favorite_only,
-        search=search,
-        sort_by=sort_by,
-        sort_dir=sort_dir,
         limit=limit,
         offset=offset,
-        snapshot_before=snapshot_before,
     )
     await publication_service.schedule_missing_pdf_enrichment_for_user(
         db_session,
@@ -178,27 +148,6 @@ async def _list_publications_for_request(
     return resolved_mode, selected_scholar_id, hydrated
 
 
-def _resolve_publications_snapshot(
-    *,
-    snapshot: str | None,
-) -> tuple[datetime, str]:
-    if snapshot is None:
-        now_utc = datetime.now(UTC)
-        return now_utc, now_utc.isoformat()
-    try:
-        parsed = datetime.fromisoformat(snapshot)
-    except ValueError as exc:
-        raise ApiException(
-            status_code=400,
-            code="invalid_snapshot",
-            message="Invalid publications snapshot cursor.",
-        ) from exc
-    if parsed.tzinfo is None:
-        parsed = parsed.replace(tzinfo=UTC)
-    normalized = parsed.astimezone(UTC)
-    return normalized, normalized.isoformat()
-
-
 def _resolve_publications_paging(
     *,
     page: int,
@@ -225,7 +174,6 @@ def _publications_list_data(
     page: int,
     page_size: int,
     offset: int,
-    snapshot: str,
 ) -> dict[str, object]:
     return {
         "mode": mode,
@@ -238,7 +186,6 @@ def _publications_list_data(
         "total_count": total_count,
         "page": int(page),
         "page_size": int(page_size),
-        "snapshot": snapshot,
         "has_prev": int(offset) > 0,
         "has_next": int(offset) + int(page_size) < int(total_count),
         "publications": [_serialize_publication_item(item) for item in publications],
@@ -333,12 +280,32 @@ async def _favorite_publication_state(
         db_session,
         items=[publication],
     )
-    current = hydrated[0] if hydrated else publication
-    identifiers = await identifier_service.overlay_publication_items_with_display_identifiers(
-        db_session,
-        items=[current],
+    return hydrated[0] if hydrated else publication
+
+
+def _log_retry_pdf_result(
+    *,
+    current_user: User,
+    scholar_profile_id: int,
+    publication_id: int,
+    queued: bool,
+    resolved_pdf: bool,
+    pdf_status: str,
+    doi: str | None,
+) -> None:
+    logger.info(
+        "api.publications.retry_pdf",
+        extra={
+            "event": "api.publications.retry_pdf",
+            "user_id": current_user.id,
+            "scholar_profile_id": scholar_profile_id,
+            "publication_id": publication_id,
+            "queued": queued,
+            "resolved_pdf": resolved_pdf,
+            "pdf_status": pdf_status,
+            "has_doi": bool(doi),
+        },
     )
-    return identifiers[0] if identifiers else current
 
 
 @router.get(
@@ -350,14 +317,10 @@ async def list_publications(
     mode: Literal["all", "unread", "latest", "new"] | None = Query(default=None),
     favorite_only: bool = Query(default=False),
     scholar_profile_id: int | None = Query(default=None, ge=1),
-    search: str | None = Query(default=None, min_length=1, max_length=200),
-    sort_by: Literal["first_seen", "title", "year", "citations", "scholar", "pdf_status"] = Query(default="first_seen"),
-    sort_dir: Literal["asc", "desc"] = Query(default="desc"),
     page: int = Query(default=1, ge=1),
     page_size: int = Query(default=100, ge=1, le=500),
     limit: int | None = Query(default=None, ge=1, le=1000),
     offset: int | None = Query(default=None, ge=0),
-    snapshot: str | None = Query(default=None, min_length=1, max_length=64),
     db_session: AsyncSession = Depends(get_db_session),
     current_user: User = Depends(get_api_current_user),
 ):
@@ -367,28 +330,20 @@ async def list_publications(
         limit=limit,
         offset=offset,
     )
-    snapshot_before, snapshot_cursor = _resolve_publications_snapshot(snapshot=snapshot)
-    normalized_search = (search or "").strip() or None
     resolved_mode, selected_scholar_id, publications = await _list_publications_for_request(
         db_session,
         current_user=current_user,
         mode=mode,
         favorite_only=favorite_only,
         scholar_profile_id=scholar_profile_id,
-        search=normalized_search,
-        sort_by=sort_by,
-        sort_dir=sort_dir,
         limit=resolved_limit,
         offset=resolved_offset,
-        snapshot_before=snapshot_before,
     )
     unread_count, favorites_count, latest_count, total_count = await _publication_counts(
         db_session,
         user_id=current_user.id,
         selected_scholar_id=selected_scholar_id,
         favorite_only=favorite_only,
-        search=normalized_search,
-        snapshot_before=snapshot_before,
     )
     data = _publications_list_data(
         mode=resolved_mode,
@@ -402,7 +357,6 @@ async def list_publications(
         page=resolved_page,
         page_size=resolved_limit,
         offset=resolved_offset,
-        snapshot=snapshot_cursor,
     )
     return success_payload(request, data=data)
 
@@ -420,8 +374,13 @@ async def mark_all_publications_read(
         db_session,
         user_id=current_user.id,
     )
-    structured_log(
-        logger, "info", "api.publications.mark_all_read", user_id=current_user.id, updated_count=updated_count
+    logger.info(
+        "api.publications.mark_all_read",
+        extra={
+            "event": "api.publications.mark_all_read",
+            "user_id": current_user.id,
+            "updated_count": updated_count,
+        },
     )
     return success_payload(
         request,
@@ -442,19 +401,25 @@ async def mark_selected_publications_read(
     db_session: AsyncSession = Depends(get_db_session),
     current_user: User = Depends(get_api_current_user),
 ):
-    selection_pairs = sorted({(int(item.scholar_profile_id), int(item.publication_id)) for item in payload.selections})
+    selection_pairs = sorted(
+        {
+            (int(item.scholar_profile_id), int(item.publication_id))
+            for item in payload.selections
+        }
+    )
     updated_count = await publication_service.mark_selected_as_read_for_user(
         db_session,
         user_id=current_user.id,
         selections=selection_pairs,
     )
-    structured_log(
-        logger,
-        "info",
+    logger.info(
         "api.publications.mark_selected_read",
-        user_id=current_user.id,
-        requested_count=len(selection_pairs),
-        updated_count=updated_count,
+        extra={
+            "event": "api.publications.mark_selected_read",
+            "user_id": current_user.id,
+            "requested_count": len(selection_pairs),
+            "updated_count": updated_count,
+        },
     )
     return success_payload(
         request,
@@ -489,16 +454,14 @@ async def retry_publication_pdf(
         resolved_pdf=resolved_pdf,
         pdf_status=current.pdf_status,
     )
-    structured_log(
-        logger,
-        "info",
-        "api.publications.retry_pdf",
-        user_id=current_user.id,
+    _log_retry_pdf_result(
+        current_user=current_user,
         scholar_profile_id=payload.scholar_profile_id,
         publication_id=publication_id,
         queued=queued,
         resolved_pdf=resolved_pdf,
         pdf_status=current.pdf_status,
+        doi=current.doi,
     )
     return success_payload(
         request,
@@ -529,14 +492,15 @@ async def toggle_publication_favorite(
         publication_id=publication_id,
         is_favorite=payload.is_favorite,
     )
-    structured_log(
-        logger,
-        "info",
+    logger.info(
         "api.publications.favorite",
-        user_id=current_user.id,
-        scholar_profile_id=payload.scholar_profile_id,
-        publication_id=publication_id,
-        is_favorite=bool(payload.is_favorite),
+        extra={
+            "event": "api.publications.favorite",
+            "user_id": current_user.id,
+            "scholar_profile_id": payload.scholar_profile_id,
+            "publication_id": publication_id,
+            "is_favorite": bool(payload.is_favorite),
+        },
     )
     return success_payload(
         request,

app/api/routers/run_manual.py

@@ -1,243 +0,0 @@
-from __future__ import annotations
-
-import logging
-from typing import Any, NoReturn
-
-from fastapi import Request
-from sqlalchemy.exc import IntegrityError
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.api.errors import ApiException
-from app.api.responses import success_payload
-from app.api.routers.run_serializers import manual_run_payload_from_run
-from app.db.models import RunStatus, RunTriggerType
-from app.logging_utils import structured_log
-from app.services.ingestion import application as ingestion_service
-from app.services.ingestion import safety as run_safety_service
-from app.services.runs import application as run_service
-from app.services.settings import application as user_settings_service
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-
-async def load_safety_state(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-) -> dict[str, Any]:
-    user_settings = await user_settings_service.get_or_create_settings(
-        db_session,
-        user_id=user_id,
-    )
-    previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
-    if run_safety_service.clear_expired_cooldown(user_settings):
-        await db_session.commit()
-        await db_session.refresh(user_settings)
-        structured_log(
-            logger,
-            "info",
-            "api.runs.safety_cooldown_cleared",
-            user_id=user_id,
-            reason=previous_safety_state.get("cooldown_reason"),
-            cooldown_until=previous_safety_state.get("cooldown_until"),
-        )
-    return run_safety_service.get_safety_state_payload(user_settings)
-
-
-def raise_manual_runs_disabled(*, user_id: int, safety_state: dict[str, Any]) -> None:
-    structured_log(
-        logger,
-        "warning",
-        "api.runs.manual_blocked_policy",
-        user_id=user_id,
-        policy={"manual_run_allowed": False},
-        safety_state=safety_state,
-    )
-    raise ApiException(
-        status_code=403,
-        code="manual_runs_disabled",
-        message="Manual checks are disabled by server policy.",
-        details={
-            "policy": {"manual_run_allowed": False},
-            "safety_state": safety_state,
-        },
-    )
-
-
-async def reused_manual_run_payload(
-    db_session: AsyncSession,
-    *,
-    request: Request,
-    user_id: int,
-    idempotency_key: str | None,
-    safety_state: dict[str, Any],
-) -> dict[str, Any] | None:
-    if idempotency_key is None:
-        return None
-    previous_run = await run_service.get_manual_run_by_idempotency_key(
-        db_session,
-        user_id=user_id,
-        idempotency_key=idempotency_key,
-    )
-    if previous_run is None:
-        return None
-    if previous_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING):
-        raise ApiException(
-            status_code=409,
-            code="run_in_progress",
-            message="A run with this idempotency key is still in progress.",
-            details={"run_id": int(previous_run.id), "idempotency_key": idempotency_key},
-        )
-    return success_payload(
-        request,
-        data=manual_run_payload_from_run(
-            previous_run,
-            idempotency_key=idempotency_key,
-            reused_existing_run=True,
-            safety_state=safety_state,
-        ),
-    )
-
-
-async def run_ingestion_for_manual(
-    db_session: AsyncSession,
-    *,
-    ingest_service: ingestion_service.ScholarIngestionService,
-    user_id: int,
-    idempotency_key: str | None,
-):
-    user_settings = await user_settings_service.get_or_create_settings(
-        db_session,
-        user_id=user_id,
-    )
-    return await ingest_service.run_for_user(
-        db_session,
-        user_id=user_id,
-        trigger_type=RunTriggerType.MANUAL,
-        request_delay_seconds=user_settings.request_delay_seconds,
-        network_error_retries=settings.ingestion_network_error_retries,
-        retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
-        max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
-        page_size=settings.ingestion_page_size,
-        auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
-        queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
-        idempotency_key=idempotency_key,
-        alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
-        alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
-        alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
-    )
-
-
-async def recover_integrity_error(
-    db_session: AsyncSession,
-    *,
-    request: Request,
-    user_id: int,
-    idempotency_key: str | None,
-    original_exc: IntegrityError,
-) -> dict[str, Any]:
-    if idempotency_key is None:
-        structured_log(
-            logger,
-            "exception",
-            "api.runs.manual_integrity_error",
-            user_id=user_id,
-        )
-        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
-    existing_run = await run_service.get_manual_run_by_idempotency_key(
-        db_session,
-        user_id=user_id,
-        idempotency_key=idempotency_key,
-    )
-    if existing_run is None:
-        structured_log(
-            logger,
-            "exception",
-            "api.runs.manual_integrity_error",
-            user_id=user_id,
-        )
-        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
-    if existing_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING):
-        raise ApiException(
-            status_code=409,
-            code="run_in_progress",
-            message="A run with this idempotency key is still in progress.",
-            details={"run_id": int(existing_run.id), "idempotency_key": idempotency_key},
-        ) from original_exc
-    return success_payload(
-        request,
-        data=manual_run_payload_from_run(
-            existing_run,
-            idempotency_key=idempotency_key,
-            reused_existing_run=True,
-            safety_state=await load_safety_state(db_session, user_id=user_id),
-        ),
-    )
-
-
-async def execute_manual_run(
-    db_session: AsyncSession,
-    *,
-    request: Request,
-    ingest_service: ingestion_service.ScholarIngestionService,
-    user_id: int,
-    idempotency_key: str | None,
-):
-    try:
-        return await run_ingestion_for_manual(
-            db_session,
-            ingest_service=ingest_service,
-            user_id=user_id,
-            idempotency_key=idempotency_key,
-        )
-    except ingestion_service.RunAlreadyInProgressError as exc:
-        await db_session.rollback()
-        raise ApiException(
-            status_code=409,
-            code="run_in_progress",
-            message="A run is already in progress for this account.",
-        ) from exc
-    except ingestion_service.RunBlockedBySafetyPolicyError as exc:
-        await db_session.rollback()
-        raise_manual_blocked_safety(exc=exc, user_id=user_id)
-    except IntegrityError as exc:
-        await db_session.rollback()
-        return await recover_integrity_error(
-            db_session,
-            request=request,
-            user_id=user_id,
-            idempotency_key=idempotency_key,
-            original_exc=exc,
-        )
-    except Exception as exc:
-        await db_session.rollback()
-        raise_manual_failed(exc=exc, user_id=user_id)
-
-
-def raise_manual_blocked_safety(*, exc, user_id: int) -> NoReturn:
-    structured_log(
-        logger,
-        "info",
-        "api.runs.manual_blocked_safety",
-        user_id=user_id,
-        reason=exc.safety_state.get("cooldown_reason"),
-        cooldown_until=exc.safety_state.get("cooldown_until"),
-        cooldown_remaining_seconds=exc.safety_state.get("cooldown_remaining_seconds"),
-    )
-    raise ApiException(
-        status_code=429,
-        code=exc.code,
-        message=exc.message,
-        details={"safety_state": exc.safety_state},
-    ) from exc
-
-
-def raise_manual_failed(*, exc: Exception, user_id: int) -> NoReturn:
-    structured_log(
-        logger,
-        "exception",
-        "api.runs.manual_failed",
-        user_id=user_id,
-    )
-    raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from exc

app/api/routers/run_serializers.py

@@ -1,238 +0,0 @@
-from __future__ import annotations
-
-import re
-from typing import Any
-
-from app.api.errors import ApiException
-from app.services.runs import application as run_service
-
-IDEMPOTENCY_HEADER = "Idempotency-Key"
-IDEMPOTENCY_MAX_LENGTH = 128
-IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
-
-
-def _int_value(value: Any, default: int = 0) -> int:
-    try:
-        return int(value)
-    except (TypeError, ValueError):
-        return default
-
-
-def _str_value(value: Any) -> str | None:
-    if value is None:
-        return None
-    text = str(value).strip()
-    return text if text else None
-
-
-def _bool_value(value: Any, default: bool = False) -> bool:
-    if isinstance(value, bool):
-        return value
-    if isinstance(value, str):
-        lowered = value.strip().lower()
-        if lowered in {"1", "true", "yes", "on"}:
-            return True
-        if lowered in {"0", "false", "no", "off"}:
-            return False
-    return default
-
-
-def normalize_idempotency_key(raw_value: str | None) -> str | None:
-    if raw_value is None:
-        return None
-    candidate = raw_value.strip()
-    if not candidate:
-        return None
-    if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate):
-        raise ApiException(
-            status_code=400,
-            code="invalid_idempotency_key",
-            message=("Invalid Idempotency-Key. Use 8-128 characters from: A-Z a-z 0-9 . _ : -"),
-        )
-    return candidate
-
-
-def serialize_run(run) -> dict[str, Any]:
-    summary = run_service.extract_run_summary(run.error_log)
-    return {
-        "id": int(run.id),
-        "trigger_type": run.trigger_type.value,
-        "status": run.status.value,
-        "start_dt": run.start_dt,
-        "end_dt": run.end_dt,
-        "scholar_count": int(run.scholar_count or 0),
-        "new_publication_count": int(run.new_pub_count or 0),
-        "failed_count": int(summary["failed_count"]),
-        "partial_count": int(summary["partial_count"]),
-    }
-
-
-def serialize_queue_item(item) -> dict[str, Any]:
-    return {
-        "id": int(item.id),
-        "scholar_profile_id": int(item.scholar_profile_id),
-        "scholar_label": item.scholar_label,
-        "status": item.status,
-        "reason": item.reason,
-        "dropped_reason": item.dropped_reason,
-        "attempt_count": int(item.attempt_count),
-        "resume_cstart": int(item.resume_cstart),
-        "next_attempt_dt": item.next_attempt_dt,
-        "updated_at": item.updated_at,
-        "last_error": item.last_error,
-        "last_run_id": item.last_run_id,
-    }
-
-
-def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]:
-    if not isinstance(value, list):
-        return []
-    normalized: list[dict[str, Any]] = []
-    for item in value:
-        if not isinstance(item, dict):
-            continue
-        normalized.append(
-            {
-                "attempt": _int_value(item.get("attempt"), 0),
-                "cstart": _int_value(item.get("cstart"), 0),
-                "state": _str_value(item.get("state")),
-                "state_reason": _str_value(item.get("state_reason")),
-                "status_code": (_int_value(item.get("status_code")) if item.get("status_code") is not None else None),
-                "fetch_error": _str_value(item.get("fetch_error")),
-            }
-        )
-    return normalized
-
-
-def _normalize_page_logs(value: Any) -> list[dict[str, Any]]:
-    if not isinstance(value, list):
-        return []
-    normalized: list[dict[str, Any]] = []
-    for item in value:
-        if not isinstance(item, dict):
-            continue
-        warning_codes = item.get("warning_codes")
-        normalized.append(
-            {
-                "page": _int_value(item.get("page"), 0),
-                "cstart": _int_value(item.get("cstart"), 0),
-                "state": _str_value(item.get("state")) or "unknown",
-                "state_reason": _str_value(item.get("state_reason")),
-                "status_code": (_int_value(item.get("status_code")) if item.get("status_code") is not None else None),
-                "publication_count": _int_value(item.get("publication_count"), 0),
-                "attempt_count": _int_value(item.get("attempt_count"), 0),
-                "has_show_more_button": _bool_value(item.get("has_show_more_button"), False),
-                "articles_range": _str_value(item.get("articles_range")),
-                "warning_codes": [str(code) for code in (warning_codes if isinstance(warning_codes, list) else [])],
-            }
-        )
-    return normalized
-
-
-def _normalize_debug(value: Any) -> dict[str, Any] | None:
-    if not isinstance(value, dict):
-        return None
-    marker_counts = value.get("marker_counts_nonzero")
-    warning_codes = value.get("warning_codes")
-    return {
-        "status_code": (_int_value(value.get("status_code")) if value.get("status_code") is not None else None),
-        "final_url": _str_value(value.get("final_url")),
-        "fetch_error": _str_value(value.get("fetch_error")),
-        "body_sha256": _str_value(value.get("body_sha256")),
-        "body_length": (_int_value(value.get("body_length")) if value.get("body_length") is not None else None),
-        "has_show_more_button": (
-            _bool_value(value.get("has_show_more_button"), False)
-            if value.get("has_show_more_button") is not None
-            else None
-        ),
-        "articles_range": _str_value(value.get("articles_range")),
-        "state_reason": _str_value(value.get("state_reason")),
-        "warning_codes": [str(code) for code in (warning_codes if isinstance(warning_codes, list) else [])],
-        "marker_counts_nonzero": {
-            str(key): _int_value(count, 0)
-            for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else [])
-        },
-        "page_logs": _normalize_page_logs(value.get("page_logs")),
-        "attempt_log": _normalize_attempt_log(value.get("attempt_log")),
-    }
-
-
-def normalize_scholar_result(value: Any) -> dict[str, Any]:
-    if not isinstance(value, dict):
-        return {
-            "scholar_profile_id": 0,
-            "scholar_id": "unknown",
-            "state": "unknown",
-            "state_reason": None,
-            "outcome": "failed",
-            "attempt_count": 0,
-            "publication_count": 0,
-            "start_cstart": 0,
-            "continuation_cstart": None,
-            "continuation_enqueued": False,
-            "continuation_cleared": False,
-            "warnings": [],
-            "error": None,
-            "debug": None,
-        }
-    warnings = value.get("warnings")
-    return {
-        "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0),
-        "scholar_id": _str_value(value.get("scholar_id")) or "unknown",
-        "state": _str_value(value.get("state")) or "unknown",
-        "state_reason": _str_value(value.get("state_reason")),
-        "outcome": _str_value(value.get("outcome")) or "failed",
-        "attempt_count": _int_value(value.get("attempt_count"), 0),
-        "publication_count": _int_value(value.get("publication_count"), 0),
-        "start_cstart": _int_value(value.get("start_cstart"), 0),
-        "continuation_cstart": (
-            _int_value(value.get("continuation_cstart")) if value.get("continuation_cstart") is not None else None
-        ),
-        "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False),
-        "continuation_cleared": _bool_value(value.get("continuation_cleared"), False),
-        "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])],
-        "error": _str_value(value.get("error")),
-        "debug": _normalize_debug(value.get("debug")),
-    }
-
-
-def manual_run_payload_from_run(
-    run,
-    *,
-    idempotency_key: str | None,
-    reused_existing_run: bool,
-    safety_state: dict[str, Any],
-) -> dict[str, Any]:
-    summary = run_service.extract_run_summary(run.error_log)
-    return {
-        "run_id": int(run.id),
-        "status": run.status.value,
-        "scholar_count": int(run.scholar_count or 0),
-        "succeeded_count": int(summary["succeeded_count"]),
-        "failed_count": int(summary["failed_count"]),
-        "partial_count": int(summary["partial_count"]),
-        "new_publication_count": int(run.new_pub_count or 0),
-        "reused_existing_run": reused_existing_run,
-        "idempotency_key": idempotency_key,
-        "safety_state": safety_state,
-    }
-
-
-def manual_run_success_payload(
-    *,
-    run_summary,
-    idempotency_key: str | None,
-    safety_state: dict[str, Any],
-) -> dict[str, Any]:
-    return {
-        "run_id": run_summary.crawl_run_id,
-        "status": run_summary.status.value,
-        "scholar_count": run_summary.scholar_count,
-        "succeeded_count": run_summary.succeeded_count,
-        "failed_count": run_summary.failed_count,
-        "partial_count": run_summary.partial_count,
-        "new_publication_count": run_summary.new_publication_count,
-        "reused_existing_run": False,
-        "idempotency_key": idempotency_key,
-        "safety_state": safety_state,
-    }

app/api/routers/runs.py

@@ -1,33 +1,16 @@
 from __future__ import annotations
 
-import asyncio
 import logging
+import re
 from typing import Any
 
 from fastapi import APIRouter, Depends, Query, Request
-from fastapi.responses import StreamingResponse
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.api.deps import get_api_current_user
 from app.api.errors import ApiException
 from app.api.responses import success_payload
-from app.api.routers.run_manual import (
-    load_safety_state,
-    raise_manual_blocked_safety,
-    raise_manual_failed,
-    raise_manual_runs_disabled,
-    recover_integrity_error,
-    reused_manual_run_payload,
-)
-from app.api.routers.run_serializers import (
-    IDEMPOTENCY_HEADER,
-    normalize_idempotency_key,
-    normalize_scholar_result,
-    serialize_queue_item,
-    serialize_run,
-)
-from app.api.runtime_deps import get_ingestion_service
 from app.api.schemas import (
     ManualRunEnvelope,
     QueueClearEnvelope,
@@ -37,28 +20,500 @@ from app.api.schemas import (
     RunsListEnvelope,
 )
 from app.db.models import RunStatus, RunTriggerType, User
-from app.db.session import get_db_session, get_session_factory
-from app.logging_utils import structured_log
-from app.services.ingestion import application as ingestion_service
-from app.services.runs import application as run_service
-from app.services.runs.events import event_generator
-from app.services.settings import application as user_settings_service
+from app.db.session import get_db_session
+from app.services.domains.ingestion import application as ingestion_service
+from app.services.domains.ingestion import safety as run_safety_service
+from app.services.domains.runs import application as run_service
+from app.services.domains.settings import application as user_settings_service
 from app.settings import settings
+from app.api.runtime_deps import get_ingestion_service
 
 logger = logging.getLogger(__name__)
-_background_tasks: set[asyncio.Task[Any]] = set()
-
-
-def _drop_finished_task(task: asyncio.Task[Any]) -> None:
-    _background_tasks.discard(task)
-    try:
-        task.result()
-    except Exception:
-        structured_log(logger, "exception", "runs.background_task_failed")
-
 
 router = APIRouter(prefix="/runs", tags=["api-runs"])
-ACTIVE_RUN_STATUSES = {RunStatus.RUNNING, RunStatus.RESOLVING}
+
+IDEMPOTENCY_HEADER = "Idempotency-Key"
+IDEMPOTENCY_MAX_LENGTH = 128
+IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
+
+
+def _int_value(value: Any, default: int = 0) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _str_value(value: Any) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text if text else None
+
+
+def _bool_value(value: Any, default: bool = False) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        lowered = value.strip().lower()
+        if lowered in {"1", "true", "yes", "on"}:
+            return True
+        if lowered in {"0", "false", "no", "off"}:
+            return False
+    return default
+
+
+def _normalize_idempotency_key(raw_value: str | None) -> str | None:
+    if raw_value is None:
+        return None
+    candidate = raw_value.strip()
+    if not candidate:
+        return None
+    if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate):
+        raise ApiException(
+            status_code=400,
+            code="invalid_idempotency_key",
+            message=(
+                "Invalid Idempotency-Key. Use 8-128 characters from: "
+                "A-Z a-z 0-9 . _ : -"
+            ),
+        )
+    return candidate
+
+
+def _serialize_run(run) -> dict[str, Any]:
+    summary = run_service.extract_run_summary(run.error_log)
+    return {
+        "id": int(run.id),
+        "trigger_type": run.trigger_type.value,
+        "status": run.status.value,
+        "start_dt": run.start_dt,
+        "end_dt": run.end_dt,
+        "scholar_count": int(run.scholar_count or 0),
+        "new_publication_count": int(run.new_pub_count or 0),
+        "failed_count": int(summary["failed_count"]),
+        "partial_count": int(summary["partial_count"]),
+    }
+
+
+def _serialize_queue_item(item) -> dict[str, Any]:
+    return {
+        "id": int(item.id),
+        "scholar_profile_id": int(item.scholar_profile_id),
+        "scholar_label": item.scholar_label,
+        "status": item.status,
+        "reason": item.reason,
+        "dropped_reason": item.dropped_reason,
+        "attempt_count": int(item.attempt_count),
+        "resume_cstart": int(item.resume_cstart),
+        "next_attempt_dt": item.next_attempt_dt,
+        "updated_at": item.updated_at,
+        "last_error": item.last_error,
+        "last_run_id": item.last_run_id,
+    }
+
+
+def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]:
+    if not isinstance(value, list):
+        return []
+    normalized: list[dict[str, Any]] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        normalized.append(
+            {
+                "attempt": _int_value(item.get("attempt"), 0),
+                "cstart": _int_value(item.get("cstart"), 0),
+                "state": _str_value(item.get("state")),
+                "state_reason": _str_value(item.get("state_reason")),
+                "status_code": (
+                    _int_value(item.get("status_code"))
+                    if item.get("status_code") is not None
+                    else None
+                ),
+                "fetch_error": _str_value(item.get("fetch_error")),
+            }
+        )
+    return normalized
+
+
+def _normalize_page_logs(value: Any) -> list[dict[str, Any]]:
+    if not isinstance(value, list):
+        return []
+    normalized: list[dict[str, Any]] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        warning_codes = item.get("warning_codes")
+        normalized.append(
+            {
+                "page": _int_value(item.get("page"), 0),
+                "cstart": _int_value(item.get("cstart"), 0),
+                "state": _str_value(item.get("state")) or "unknown",
+                "state_reason": _str_value(item.get("state_reason")),
+                "status_code": (
+                    _int_value(item.get("status_code"))
+                    if item.get("status_code") is not None
+                    else None
+                ),
+                "publication_count": _int_value(item.get("publication_count"), 0),
+                "attempt_count": _int_value(item.get("attempt_count"), 0),
+                "has_show_more_button": _bool_value(item.get("has_show_more_button"), False),
+                "articles_range": _str_value(item.get("articles_range")),
+                "warning_codes": [
+                    str(code)
+                    for code in (warning_codes if isinstance(warning_codes, list) else [])
+                ],
+            }
+        )
+    return normalized
+
+
+def _normalize_debug(value: Any) -> dict[str, Any] | None:
+    if not isinstance(value, dict):
+        return None
+    marker_counts = value.get("marker_counts_nonzero")
+    warning_codes = value.get("warning_codes")
+    return {
+        "status_code": (
+            _int_value(value.get("status_code"))
+            if value.get("status_code") is not None
+            else None
+        ),
+        "final_url": _str_value(value.get("final_url")),
+        "fetch_error": _str_value(value.get("fetch_error")),
+        "body_sha256": _str_value(value.get("body_sha256")),
+        "body_length": (
+            _int_value(value.get("body_length"))
+            if value.get("body_length") is not None
+            else None
+        ),
+        "has_show_more_button": (
+            _bool_value(value.get("has_show_more_button"), False)
+            if value.get("has_show_more_button") is not None
+            else None
+        ),
+        "articles_range": _str_value(value.get("articles_range")),
+        "state_reason": _str_value(value.get("state_reason")),
+        "warning_codes": [
+            str(code)
+            for code in (warning_codes if isinstance(warning_codes, list) else [])
+        ],
+        "marker_counts_nonzero": {
+            str(key): _int_value(count, 0)
+            for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else [])
+        },
+        "page_logs": _normalize_page_logs(value.get("page_logs")),
+        "attempt_log": _normalize_attempt_log(value.get("attempt_log")),
+    }
+
+
+def _normalize_scholar_result(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {
+            "scholar_profile_id": 0,
+            "scholar_id": "unknown",
+            "state": "unknown",
+            "state_reason": None,
+            "outcome": "failed",
+            "attempt_count": 0,
+            "publication_count": 0,
+            "start_cstart": 0,
+            "continuation_cstart": None,
+            "continuation_enqueued": False,
+            "continuation_cleared": False,
+            "warnings": [],
+            "error": None,
+            "debug": None,
+        }
+    warnings = value.get("warnings")
+    return {
+        "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0),
+        "scholar_id": _str_value(value.get("scholar_id")) or "unknown",
+        "state": _str_value(value.get("state")) or "unknown",
+        "state_reason": _str_value(value.get("state_reason")),
+        "outcome": _str_value(value.get("outcome")) or "failed",
+        "attempt_count": _int_value(value.get("attempt_count"), 0),
+        "publication_count": _int_value(value.get("publication_count"), 0),
+        "start_cstart": _int_value(value.get("start_cstart"), 0),
+        "continuation_cstart": (
+            _int_value(value.get("continuation_cstart"))
+            if value.get("continuation_cstart") is not None
+            else None
+        ),
+        "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False),
+        "continuation_cleared": _bool_value(value.get("continuation_cleared"), False),
+        "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])],
+        "error": _str_value(value.get("error")),
+        "debug": _normalize_debug(value.get("debug")),
+    }
+
+
+def _manual_run_payload_from_run(
+    run,
+    *,
+    idempotency_key: str | None,
+    reused_existing_run: bool,
+    safety_state: dict[str, Any],
+) -> dict[str, Any]:
+    summary = run_service.extract_run_summary(run.error_log)
+    return {
+        "run_id": int(run.id),
+        "status": run.status.value,
+        "scholar_count": int(run.scholar_count or 0),
+        "succeeded_count": int(summary["succeeded_count"]),
+        "failed_count": int(summary["failed_count"]),
+        "partial_count": int(summary["partial_count"]),
+        "new_publication_count": int(run.new_pub_count or 0),
+        "reused_existing_run": reused_existing_run,
+        "idempotency_key": idempotency_key,
+        "safety_state": safety_state,
+    }
+
+
+async def _load_safety_state(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> dict[str, Any]:
+    user_settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=user_id,
+    )
+    previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
+    if run_safety_service.clear_expired_cooldown(user_settings):
+        await db_session.commit()
+        await db_session.refresh(user_settings)
+        logger.info(
+            "api.runs.safety_cooldown_cleared",
+            extra={
+                "event": "api.runs.safety_cooldown_cleared",
+                "user_id": user_id,
+                "reason": previous_safety_state.get("cooldown_reason"),
+                "cooldown_until": previous_safety_state.get("cooldown_until"),
+                "metric_name": "api_runs_safety_cooldown_cleared_total",
+                "metric_value": 1,
+            },
+    )
+    return run_safety_service.get_safety_state_payload(user_settings)
+
+
+def _raise_manual_runs_disabled(*, user_id: int, safety_state: dict[str, Any]) -> None:
+    logger.warning(
+        "api.runs.manual_blocked_policy",
+        extra={
+            "event": "api.runs.manual_blocked_policy",
+            "user_id": user_id,
+            "policy": {"manual_run_allowed": False},
+            "safety_state": safety_state,
+            "metric_name": "api_runs_manual_blocked_policy_total",
+            "metric_value": 1,
+        },
+    )
+    raise ApiException(
+        status_code=403,
+        code="manual_runs_disabled",
+        message="Manual checks are disabled by server policy.",
+        details={
+            "policy": {"manual_run_allowed": False},
+            "safety_state": safety_state,
+        },
+    )
+
+
+async def _reused_manual_run_payload(
+    db_session: AsyncSession,
+    *,
+    request: Request,
+    user_id: int,
+    idempotency_key: str | None,
+    safety_state: dict[str, Any],
+) -> dict[str, Any] | None:
+    if idempotency_key is None:
+        return None
+    previous_run = await run_service.get_manual_run_by_idempotency_key(
+        db_session,
+        user_id=user_id,
+        idempotency_key=idempotency_key,
+    )
+    if previous_run is None:
+        return None
+    if previous_run.status == RunStatus.RUNNING:
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run with this idempotency key is still in progress.",
+            details={"run_id": int(previous_run.id), "idempotency_key": idempotency_key},
+        )
+    return success_payload(
+        request,
+        data=_manual_run_payload_from_run(
+            previous_run,
+            idempotency_key=idempotency_key,
+            reused_existing_run=True,
+            safety_state=safety_state,
+        ),
+    )
+
+
+async def _run_ingestion_for_manual(
+    db_session: AsyncSession,
+    *,
+    ingest_service: ingestion_service.ScholarIngestionService,
+    user_id: int,
+    idempotency_key: str | None,
+):
+    user_settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=user_id,
+    )
+    return await ingest_service.run_for_user(
+        db_session,
+        user_id=user_id,
+        trigger_type=RunTriggerType.MANUAL,
+        request_delay_seconds=user_settings.request_delay_seconds,
+        network_error_retries=settings.ingestion_network_error_retries,
+        retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+        max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
+        page_size=settings.ingestion_page_size,
+        auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
+        queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
+        idempotency_key=idempotency_key,
+        alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
+        alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
+        alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
+    )
+
+
+async def _recover_integrity_error(
+    db_session: AsyncSession,
+    *,
+    request: Request,
+    user_id: int,
+    idempotency_key: str | None,
+    original_exc: IntegrityError,
+) -> dict[str, Any]:
+    if idempotency_key is None:
+        logger.exception(
+            "api.runs.manual_integrity_error",
+            extra={"event": "api.runs.manual_integrity_error", "user_id": user_id},
+        )
+        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
+    existing_run = await run_service.get_manual_run_by_idempotency_key(
+        db_session,
+        user_id=user_id,
+        idempotency_key=idempotency_key,
+    )
+    if existing_run is None:
+        logger.exception(
+            "api.runs.manual_integrity_error",
+            extra={"event": "api.runs.manual_integrity_error", "user_id": user_id},
+        )
+        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
+    if existing_run.status == RunStatus.RUNNING:
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run with this idempotency key is still in progress.",
+            details={"run_id": int(existing_run.id), "idempotency_key": idempotency_key},
+        ) from original_exc
+    return success_payload(
+        request,
+        data=_manual_run_payload_from_run(
+            existing_run,
+            idempotency_key=idempotency_key,
+            reused_existing_run=True,
+            safety_state=await _load_safety_state(db_session, user_id=user_id),
+        ),
+    )
+
+
+async def _execute_manual_run(
+    db_session: AsyncSession,
+    *,
+    request: Request,
+    ingest_service: ingestion_service.ScholarIngestionService,
+    user_id: int,
+    idempotency_key: str | None,
+):
+    try:
+        return await _run_ingestion_for_manual(
+            db_session,
+            ingest_service=ingest_service,
+            user_id=user_id,
+            idempotency_key=idempotency_key,
+        )
+    except ingestion_service.RunAlreadyInProgressError as exc:
+        await db_session.rollback()
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run is already in progress for this account.",
+        ) from exc
+    except ingestion_service.RunBlockedBySafetyPolicyError as exc:
+        await db_session.rollback()
+        _raise_manual_blocked_safety(exc=exc, user_id=user_id)
+    except IntegrityError as exc:
+        await db_session.rollback()
+        return await _recover_integrity_error(
+            db_session,
+            request=request,
+            user_id=user_id,
+            idempotency_key=idempotency_key,
+            original_exc=exc,
+        )
+    except Exception as exc:
+        await db_session.rollback()
+        _raise_manual_failed(exc=exc, user_id=user_id)
+
+
+def _raise_manual_blocked_safety(*, exc, user_id: int) -> None:
+    logger.info(
+        "api.runs.manual_blocked_safety",
+        extra={
+            "event": "api.runs.manual_blocked_safety",
+            "user_id": user_id,
+            "reason": exc.safety_state.get("cooldown_reason"),
+            "cooldown_until": exc.safety_state.get("cooldown_until"),
+            "cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"),
+            "metric_name": "api_runs_manual_blocked_safety_total",
+            "metric_value": 1,
+        },
+    )
+    raise ApiException(
+        status_code=429,
+        code=exc.code,
+        message=exc.message,
+        details={"safety_state": exc.safety_state},
+    ) from exc
+
+
+def _raise_manual_failed(*, exc: Exception, user_id: int) -> None:
+    logger.exception(
+        "api.runs.manual_failed",
+        extra={"event": "api.runs.manual_failed", "user_id": user_id},
+    )
+    raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from exc
+
+
+def _manual_run_success_payload(
+    *,
+    run_summary,
+    idempotency_key: str | None,
+    safety_state: dict[str, Any],
+) -> dict[str, Any]:
+    return {
+        "run_id": run_summary.crawl_run_id,
+        "status": run_summary.status.value,
+        "scholar_count": run_summary.scholar_count,
+        "succeeded_count": run_summary.succeeded_count,
+        "failed_count": run_summary.failed_count,
+        "partial_count": run_summary.partial_count,
+        "new_publication_count": run_summary.new_publication_count,
+        "reused_existing_run": False,
+        "idempotency_key": idempotency_key,
+        "safety_state": safety_state,
+    }
 
 
 @router.get(
@@ -78,14 +533,14 @@ async def list_runs(
         limit=limit,
         failed_only=failed_only,
     )
-    safety_state = await load_safety_state(
+    safety_state = await _load_safety_state(
         db_session,
         user_id=current_user.id,
     )
     return success_payload(
         request,
         data={
-            "runs": [serialize_run(run) for run in runs],
+            "runs": [_serialize_run(run) for run in runs],
             "safety_state": safety_state,
         },
     )
@@ -116,137 +571,21 @@ async def get_run(
     scholar_results = error_log.get("scholar_results")
     if not isinstance(scholar_results, list):
         scholar_results = []
-    safety_state = await load_safety_state(
+    safety_state = await _load_safety_state(
         db_session,
         user_id=current_user.id,
     )
     return success_payload(
         request,
         data={
-            "run": serialize_run(run),
+            "run": _serialize_run(run),
             "summary": run_service.extract_run_summary(error_log),
-            "scholar_results": [normalize_scholar_result(item) for item in scholar_results],
+            "scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
             "safety_state": safety_state,
         },
     )
 
 
-@router.post(
-    "/{run_id}/cancel",
-    response_model=RunDetailEnvelope,
-)
-async def cancel_run(
-    run_id: int,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    run = await run_service.get_run_for_user(
-        db_session,
-        user_id=current_user.id,
-        run_id=run_id,
-    )
-    if run is None:
-        raise ApiException(
-            status_code=404,
-            code="run_not_found",
-            message="Run not found.",
-        )
-
-    if run.status in ACTIVE_RUN_STATUSES:
-        run.status = RunStatus.CANCELED
-        await db_session.commit()
-        await db_session.refresh(run)
-    else:
-        raise ApiException(
-            status_code=409,
-            code="run_not_cancelable",
-            message="Run is already terminal and cannot be canceled.",
-            details={"run_id": int(run.id), "status": run.status.value},
-        )
-
-    error_log = run.error_log if isinstance(run.error_log, dict) else {}
-    scholar_results = error_log.get("scholar_results")
-    if not isinstance(scholar_results, list):
-        scholar_results = []
-
-    safety_state = await load_safety_state(
-        db_session,
-        user_id=current_user.id,
-    )
-
-    return success_payload(
-        request,
-        data={
-            "run": serialize_run(run),
-            "summary": run_service.extract_run_summary(error_log),
-            "scholar_results": [normalize_scholar_result(item) for item in scholar_results],
-            "safety_state": safety_state,
-        },
-    )
-
-
-async def _start_manual_run(
-    db_session: AsyncSession,
-    *,
-    current_user: User,
-    ingest_service: ingestion_service.ScholarIngestionService,
-    idempotency_key: str | None,
-) -> tuple[Any, Any, list, dict]:
-    user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=current_user.id)
-    run, scholars, start_cstart_map = await ingest_service.initialize_run(
-        db_session,
-        user_id=current_user.id,
-        trigger_type=RunTriggerType.MANUAL,
-        request_delay_seconds=user_settings.request_delay_seconds,
-        network_error_retries=settings.ingestion_network_error_retries,
-        retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
-        max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
-        page_size=settings.ingestion_page_size,
-        idempotency_key=idempotency_key,
-        alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
-        alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
-        alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
-    )
-    return user_settings, run, scholars, start_cstart_map
-
-
-def _spawn_background_execution(
-    ingest_service: ingestion_service.ScholarIngestionService,
-    *,
-    run: Any,
-    current_user: User,
-    scholars: list,
-    start_cstart_map: dict,
-    user_settings: Any,
-    idempotency_key: str | None,
-) -> None:
-    from app.db.background_session import background_session
-
-    task = asyncio.create_task(
-        ingest_service.execute_run(
-            session_factory=background_session,
-            run_id=run.id,
-            user_id=current_user.id,
-            scholars=scholars,
-            start_cstart_map=start_cstart_map,
-            request_delay_seconds=user_settings.request_delay_seconds,
-            network_error_retries=settings.ingestion_network_error_retries,
-            retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
-            max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
-            page_size=settings.ingestion_page_size,
-            auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
-            queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
-            alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
-            alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
-            alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
-            idempotency_key=idempotency_key,
-        )
-    )
-    _background_tasks.add(task)
-    task.add_done_callback(_drop_finished_task)
-
-
 @router.post(
     "/manual",
     response_model=ManualRunEnvelope,
@@ -257,76 +596,46 @@ async def run_manual(
     current_user: User = Depends(get_api_current_user),
     ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
 ):
-    user_id = int(current_user.id)
-    safety_state = await load_safety_state(db_session, user_id=user_id)
+    safety_state = await _load_safety_state(
+        db_session,
+        user_id=current_user.id,
+    )
     if not settings.ingestion_manual_run_allowed:
-        raise_manual_runs_disabled(user_id=user_id, safety_state=safety_state)
+        _raise_manual_runs_disabled(user_id=current_user.id, safety_state=safety_state)
 
-    idempotency_key = normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
-    reused_payload = await reused_manual_run_payload(
+    idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
+    reused_payload = await _reused_manual_run_payload(
         db_session,
         request=request,
-        user_id=user_id,
+        user_id=current_user.id,
         idempotency_key=idempotency_key,
         safety_state=safety_state,
     )
     if reused_payload is not None:
         return reused_payload
 
-    try:
-        user_settings, run, scholars, start_cstart_map = await _start_manual_run(
-            db_session,
-            current_user=current_user,
-            ingest_service=ingest_service,
+    run_summary = await _execute_manual_run(
+        db_session,
+        request=request,
+        ingest_service=ingest_service,
+        user_id=current_user.id,
+        idempotency_key=idempotency_key,
+    )
+    if isinstance(run_summary, dict):
+        return run_summary
+
+    current_safety_state = await _load_safety_state(
+        db_session,
+        user_id=current_user.id,
+    )
+    return success_payload(
+        request,
+        data=_manual_run_success_payload(
+            run_summary=run_summary,
             idempotency_key=idempotency_key,
-        )
-        await db_session.commit()
-        _spawn_background_execution(
-            ingest_service,
-            run=run,
-            current_user=current_user,
-            scholars=scholars,
-            start_cstart_map=start_cstart_map,
-            user_settings=user_settings,
-            idempotency_key=idempotency_key,
-        )
-        return success_payload(
-            request,
-            data={
-                "run_id": int(run.id),
-                "status": run.status.value,
-                "scholar_count": int(run.scholar_count or 0),
-                "succeeded_count": 0,
-                "failed_count": 0,
-                "partial_count": 0,
-                "new_publication_count": 0,
-                "reused_existing_run": False,
-                "idempotency_key": idempotency_key,
-                "safety_state": await load_safety_state(db_session, user_id=user_id),
-            },
-        )
-    except ingestion_service.RunBlockedBySafetyPolicyError as exc:
-        await db_session.rollback()
-        raise_manual_blocked_safety(exc=exc, user_id=user_id)
-    except ingestion_service.RunAlreadyInProgressError as exc:
-        await db_session.rollback()
-        raise ApiException(
-            status_code=409,
-            code="run_in_progress",
-            message="A run is already in progress for this account.",
-        ) from exc
-    except IntegrityError as exc:
-        await db_session.rollback()
-        return await recover_integrity_error(
-            db_session,
-            request=request,
-            user_id=user_id,
-            idempotency_key=idempotency_key,
-            original_exc=exc,
-        )
-    except Exception as exc:
-        await db_session.rollback()
-        raise_manual_failed(exc=exc, user_id=user_id)
+            safety_state=current_safety_state,
+        ),
+    )
 
 
 @router.get(
@@ -347,7 +656,7 @@ async def list_queue_items(
     return success_payload(
         request,
         data={
-            "queue_items": [serialize_queue_item(item) for item in items],
+            "queue_items": [_serialize_queue_item(item) for item in items],
         },
     )
 
@@ -383,7 +692,7 @@ async def retry_queue_item(
         )
     return success_payload(
         request,
-        data=serialize_queue_item(queue_item),
+        data=_serialize_queue_item(queue_item),
     )
 
 
@@ -418,7 +727,7 @@ async def drop_queue_item(
         )
     return success_payload(
         request,
-        data=serialize_queue_item(dropped),
+        data=_serialize_queue_item(dropped),
     )
 
 
@@ -460,24 +769,3 @@ async def clear_queue_item(
             "message": "Queue item cleared.",
         },
     )
-
-
-@router.get("/{run_id}/stream")
-async def stream_run_events(
-    run_id: int,
-    current_user: User = Depends(get_api_current_user),
-):
-    session_factory = get_session_factory()
-    async with session_factory() as db_session:
-        run = await run_service.get_run_for_user(
-            db_session,
-            user_id=current_user.id,
-            run_id=run_id,
-        )
-    if run is None:
-        raise ApiException(
-            status_code=404,
-            code="run_not_found",
-            message="Run not found.",
-        )
-    return StreamingResponse(event_generator(run_id), media_type="text/event-stream")

app/api/routers/scholar_helpers.py

@@ -1,234 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-from typing import Any
-
-from fastapi import UploadFile
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.api.errors import ApiException
-from app.logging_utils import structured_log
-from app.services.ingestion import queue as ingestion_queue_service
-from app.services.scholar import rate_limit as scholar_rate_limit
-from app.services.scholar.source import ScholarSource
-from app.services.scholars import application as scholar_service
-from app.services.scholars import search_hints as scholar_search_hints
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-CREATE_METADATA_HYDRATION_TIMEOUT_SECONDS = 5.0
-INITIAL_SCHOLAR_SCRAPE_QUEUE_DELAY_SECONDS = 0
-INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON = "scholar_added_initial_scrape"
-
-
-def needs_metadata_hydration(profile) -> bool:
-    if not profile.profile_image_url:
-        return True
-    return not (profile.display_name or "").strip()
-
-
-def is_create_hydration_rate_limited() -> tuple[bool, float]:
-    remaining_seconds = scholar_rate_limit.remaining_scholar_slot_seconds(
-        min_interval_seconds=float(settings.ingestion_min_request_delay_seconds),
-    )
-    return remaining_seconds > 0, remaining_seconds
-
-
-def auto_enqueue_new_scholar_enabled() -> bool:
-    if not settings.scheduler_enabled:
-        return False
-    if not settings.ingestion_automation_allowed:
-        return False
-    return bool(settings.ingestion_continuation_queue_enabled)
-
-
-async def enqueue_initial_scrape_job_for_scholar(
-    db_session: AsyncSession,
-    *,
-    profile,
-    user_id: int,
-) -> bool:
-    if not auto_enqueue_new_scholar_enabled():
-        return False
-    try:
-        await ingestion_queue_service.upsert_job(
-            db_session,
-            user_id=user_id,
-            scholar_profile_id=int(profile.id),
-            resume_cstart=0,
-            reason=INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON,
-            run_id=None,
-            delay_seconds=INITIAL_SCHOLAR_SCRAPE_QUEUE_DELAY_SECONDS,
-        )
-        await db_session.commit()
-    except Exception:
-        await db_session.rollback()
-        structured_log(
-            logger,
-            "warning",
-            "api.scholars.initial_scrape_enqueue_failed",
-            user_id=user_id,
-            scholar_profile_id=profile.id,
-        )
-        return False
-
-    structured_log(
-        logger,
-        "info",
-        "api.scholars.initial_scrape_enqueued",
-        user_id=user_id,
-        scholar_profile_id=profile.id,
-        reason=INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON,
-    )
-    return True
-
-
-def uploaded_image_media_path(scholar_profile_id: int) -> str:
-    return f"/scholar-images/{scholar_profile_id}/upload"
-
-
-def serialize_scholar(profile) -> dict[str, object]:
-    uploaded_image_url = None
-    if profile.profile_image_upload_path:
-        uploaded_image_url = uploaded_image_media_path(int(profile.id))
-
-    profile_image_url, profile_image_source = scholar_search_hints.resolve_profile_image(
-        profile,
-        uploaded_image_url=uploaded_image_url,
-    )
-
-    return {
-        "id": int(profile.id),
-        "scholar_id": profile.scholar_id,
-        "display_name": profile.display_name,
-        "profile_image_url": profile_image_url,
-        "profile_image_source": profile_image_source,
-        "is_enabled": bool(profile.is_enabled),
-        "baseline_completed": bool(profile.baseline_completed),
-        "last_run_dt": profile.last_run_dt,
-        "last_run_status": (profile.last_run_status.value if profile.last_run_status is not None else None),
-    }
-
-
-async def hydrate_scholar_metadata_if_needed(
-    db_session: AsyncSession,
-    *,
-    profile,
-    source: ScholarSource,
-    user_id: int,
-):
-    if not needs_metadata_hydration(profile):
-        return profile
-
-    should_skip, remaining_seconds = is_create_hydration_rate_limited()
-    if should_skip:
-        structured_log(
-            logger,
-            "info",
-            "api.scholars.create_metadata_hydration_skipped",
-            reason="scholar_request_throttle_active",
-            user_id=user_id,
-            scholar_profile_id=profile.id,
-            retry_after_seconds=round(remaining_seconds, 3),
-        )
-        return profile
-
-    try:
-        return await asyncio.wait_for(
-            scholar_service.hydrate_profile_metadata(
-                db_session,
-                profile=profile,
-                source=source,
-            ),
-            timeout=CREATE_METADATA_HYDRATION_TIMEOUT_SECONDS,
-        )
-    except TimeoutError:
-        structured_log(
-            logger,
-            "info",
-            "api.scholars.create_metadata_hydration_skipped",
-            reason="create_timeout",
-            user_id=user_id,
-            scholar_profile_id=profile.id,
-        )
-    except Exception:
-        structured_log(
-            logger,
-            "warning",
-            "api.scholars.create_metadata_hydration_failed",
-            user_id=user_id,
-            scholar_profile_id=profile.id,
-        )
-    return profile
-
-
-def search_kwargs() -> dict[str, Any]:
-    return {
-        "network_error_retries": settings.ingestion_network_error_retries,
-        "retry_backoff_seconds": settings.ingestion_retry_backoff_seconds,
-        "search_enabled": settings.scholar_name_search_enabled,
-        "cache_ttl_seconds": settings.scholar_name_search_cache_ttl_seconds,
-        "blocked_cache_ttl_seconds": settings.scholar_name_search_blocked_cache_ttl_seconds,
-        "cache_max_entries": settings.scholar_name_search_cache_max_entries,
-        "min_interval_seconds": settings.scholar_name_search_min_interval_seconds,
-        "interval_jitter_seconds": settings.scholar_name_search_interval_jitter_seconds,
-        "cooldown_block_threshold": settings.scholar_name_search_cooldown_block_threshold,
-        "cooldown_seconds": settings.scholar_name_search_cooldown_seconds,
-        "retry_alert_threshold": settings.scholar_name_search_alert_retry_count_threshold,
-        "cooldown_rejection_alert_threshold": (settings.scholar_name_search_alert_cooldown_rejections_threshold),
-    }
-
-
-def search_response_data(query: str, parsed) -> dict[str, object]:
-    return {
-        "query": query.strip(),
-        "state": parsed.state.value,
-        "state_reason": parsed.state_reason,
-        "action_hint": scholar_search_hints.scrape_state_hint(
-            state=parsed.state,
-            state_reason=parsed.state_reason,
-        ),
-        "candidates": [
-            {
-                "scholar_id": item.scholar_id,
-                "display_name": item.display_name,
-                "affiliation": item.affiliation,
-                "email_domain": item.email_domain,
-                "cited_by_count": item.cited_by_count,
-                "interests": item.interests,
-                "profile_url": item.profile_url,
-                "profile_image_url": item.profile_image_url,
-            }
-            for item in parsed.candidates
-        ],
-        "warnings": parsed.warnings,
-    }
-
-
-async def read_uploaded_image(image: UploadFile) -> bytes:
-    try:
-        return await image.read()
-    finally:
-        await image.close()
-
-
-async def require_user_profile(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholar_profile_id: int,
-):
-    profile = await scholar_service.get_user_scholar_by_id(
-        db_session,
-        user_id=user_id,
-        scholar_profile_id=scholar_profile_id,
-    )
-    if profile is None:
-        raise ApiException(
-            status_code=404,
-            code="scholar_not_found",
-            message="Scholar not found.",
-        )
-    return profile

app/api/routers/scholars.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import logging
 
 from fastapi import APIRouter, Depends, File, Query, Request, UploadFile
@@ -8,24 +9,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from app.api.deps import get_api_current_user
 from app.api.errors import ApiException
 from app.api.responses import success_payload
-from app.api.routers.scholar_helpers import (
-    enqueue_initial_scrape_job_for_scholar,
-    hydrate_scholar_metadata_if_needed,
-    read_uploaded_image,
-    require_user_profile,
-    search_kwargs,
-    search_response_data,
-    serialize_scholar,
-)
 from app.api.runtime_deps import get_scholar_source
 from app.api.schemas import (
     DataExportEnvelope,
     DataImportEnvelope,
     DataImportRequest,
     MessageEnvelope,
-    ScholarBulkCountEnvelope,
-    ScholarBulkIdsRequest,
-    ScholarBulkToggleRequest,
     ScholarCreateRequest,
     ScholarEnvelope,
     ScholarImageUrlUpdateRequest,
@@ -34,10 +23,9 @@ from app.api.schemas import (
 )
 from app.db.models import User
 from app.db.session import get_db_session
-from app.logging_utils import structured_log
-from app.services.portability import application as import_export_service
-from app.services.scholar.source import ScholarSource
-from app.services.scholars import application as scholar_service
+from app.services.domains.portability import application as import_export_service
+from app.services.domains.scholars import application as scholar_service
+from app.services.domains.scholar.source import ScholarSource
 from app.settings import settings
 
 logger = logging.getLogger(__name__)
@@ -45,20 +33,134 @@ logger = logging.getLogger(__name__)
 router = APIRouter(prefix="/scholars", tags=["api-scholars"])
 
 
-def _parse_ids_param(ids: str | None) -> list[int] | None:
-    if not ids:
-        return None
-    parts = [p.strip() for p in ids.split(",") if p.strip()]
-    if not parts:
-        return None
+def _uploaded_image_media_path(scholar_profile_id: int) -> str:
+    return f"/scholar-images/{scholar_profile_id}/upload"
+
+
+def _serialize_scholar(profile) -> dict[str, object]:
+    uploaded_image_url = None
+    if profile.profile_image_upload_path:
+        uploaded_image_url = _uploaded_image_media_path(int(profile.id))
+
+    profile_image_url, profile_image_source = scholar_service.resolve_profile_image(
+        profile,
+        uploaded_image_url=uploaded_image_url,
+    )
+
+    return {
+        "id": int(profile.id),
+        "scholar_id": profile.scholar_id,
+        "display_name": profile.display_name,
+        "profile_image_url": profile_image_url,
+        "profile_image_source": profile_image_source,
+        "is_enabled": bool(profile.is_enabled),
+        "baseline_completed": bool(profile.baseline_completed),
+        "last_run_dt": profile.last_run_dt,
+        "last_run_status": (
+            profile.last_run_status.value if profile.last_run_status is not None else None
+        ),
+    }
+
+
+async def _hydrate_scholar_metadata_if_needed(
+    db_session: AsyncSession,
+    *,
+    profile,
+    source: ScholarSource,
+    user_id: int,
+):
     try:
-        return [int(p) for p in parts]
-    except ValueError as exc:
+        if not profile.profile_image_url or not (profile.display_name or "").strip():
+            return await asyncio.wait_for(
+                scholar_service.hydrate_profile_metadata(
+                    db_session,
+                    profile=profile,
+                    source=source,
+                ),
+                timeout=5.0,
+            )
+    except Exception:
+        logger.warning(
+            "api.scholars.create_metadata_hydration_failed",
+            extra={
+                "event": "api.scholars.create_metadata_hydration_failed",
+                "user_id": user_id,
+                "scholar_profile_id": profile.id,
+            },
+        )
+    return profile
+
+
+def _search_kwargs() -> dict[str, object]:
+    return {
+        "network_error_retries": settings.ingestion_network_error_retries,
+        "retry_backoff_seconds": settings.ingestion_retry_backoff_seconds,
+        "search_enabled": settings.scholar_name_search_enabled,
+        "cache_ttl_seconds": settings.scholar_name_search_cache_ttl_seconds,
+        "blocked_cache_ttl_seconds": settings.scholar_name_search_blocked_cache_ttl_seconds,
+        "cache_max_entries": settings.scholar_name_search_cache_max_entries,
+        "min_interval_seconds": settings.scholar_name_search_min_interval_seconds,
+        "interval_jitter_seconds": settings.scholar_name_search_interval_jitter_seconds,
+        "cooldown_block_threshold": settings.scholar_name_search_cooldown_block_threshold,
+        "cooldown_seconds": settings.scholar_name_search_cooldown_seconds,
+        "retry_alert_threshold": settings.scholar_name_search_alert_retry_count_threshold,
+        "cooldown_rejection_alert_threshold": (
+            settings.scholar_name_search_alert_cooldown_rejections_threshold
+        ),
+    }
+
+
+def _search_response_data(query: str, parsed) -> dict[str, object]:
+    return {
+        "query": query.strip(),
+        "state": parsed.state.value,
+        "state_reason": parsed.state_reason,
+        "action_hint": scholar_service.scrape_state_hint(
+            state=parsed.state,
+            state_reason=parsed.state_reason,
+        ),
+        "candidates": [
+            {
+                "scholar_id": item.scholar_id,
+                "display_name": item.display_name,
+                "affiliation": item.affiliation,
+                "email_domain": item.email_domain,
+                "cited_by_count": item.cited_by_count,
+                "interests": item.interests,
+                "profile_url": item.profile_url,
+                "profile_image_url": item.profile_image_url,
+            }
+            for item in parsed.candidates
+        ],
+        "warnings": parsed.warnings,
+    }
+
+
+async def _read_uploaded_image(image: UploadFile) -> bytes:
+    try:
+        return await image.read()
+    finally:
+        await image.close()
+
+
+async def _require_user_profile(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+):
+    profile = await scholar_service.get_user_scholar_by_id(
+        db_session,
+        user_id=user_id,
+        scholar_profile_id=scholar_profile_id,
+    )
+    if profile is None:
         raise ApiException(
-            status_code=400,
-            code="invalid_ids_param",
-            message="The 'ids' parameter must be a comma-separated list of integers.",
-        ) from exc
+            status_code=404,
+            code="scholar_not_found",
+            message="Scholar not found.",
+        )
+    return profile
 
 
 @router.get(
@@ -77,7 +179,7 @@ async def list_scholars(
     return success_payload(
         request,
         data={
-            "scholars": [serialize_scholar(profile) for profile in scholars],
+            "scholars": [_serialize_scholar(profile) for profile in scholars],
         },
     )
 
@@ -88,81 +190,16 @@ async def list_scholars(
 )
 async def export_scholars_and_publications(
     request: Request,
-    ids: str | None = Query(None, description="Comma-separated scholar profile IDs to export"),
     db_session: AsyncSession = Depends(get_db_session),
     current_user: User = Depends(get_api_current_user),
 ):
-    scholar_profile_ids = _parse_ids_param(ids)
     data = await import_export_service.export_user_data(
         db_session,
         user_id=current_user.id,
-        scholar_profile_ids=scholar_profile_ids,
     )
     return success_payload(request, data=data)
 
 
-@router.post(
-    "/bulk-delete",
-    response_model=ScholarBulkCountEnvelope,
-)
-async def bulk_delete_scholars(
-    payload: ScholarBulkIdsRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    try:
-        deleted_count = await scholar_service.bulk_delete_scholars(
-            db_session,
-            user_id=current_user.id,
-            scholar_profile_ids=payload.scholar_profile_ids,
-            upload_dir=settings.scholar_image_upload_dir,
-        )
-    except scholar_service.ScholarServiceError as exc:
-        raise ApiException(
-            status_code=409,
-            code="scholar_bulk_delete_failed",
-            message=str(exc),
-        ) from exc
-    structured_log(
-        logger,
-        "info",
-        "scholars.bulk_delete",
-        user_id=current_user.id,
-        requested_ids=payload.scholar_profile_ids,
-        deleted_count=deleted_count,
-    )
-    return success_payload(request, data={"deleted_count": deleted_count, "updated_count": 0})
-
-
-@router.post(
-    "/bulk-toggle",
-    response_model=ScholarBulkCountEnvelope,
-)
-async def bulk_toggle_scholars(
-    payload: ScholarBulkToggleRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    updated_count = await scholar_service.bulk_toggle_scholars(
-        db_session,
-        user_id=current_user.id,
-        scholar_profile_ids=payload.scholar_profile_ids,
-        is_enabled=payload.is_enabled,
-    )
-    structured_log(
-        logger,
-        "info",
-        "scholars.bulk_toggle",
-        user_id=current_user.id,
-        requested_ids=payload.scholar_profile_ids,
-        is_enabled=payload.is_enabled,
-        updated_count=updated_count,
-    )
-    return success_payload(request, data={"deleted_count": 0, "updated_count": updated_count})
-
-
 @router.post(
     "/import",
     response_model=DataImportEnvelope,
@@ -181,7 +218,8 @@ async def import_scholars_and_publications(
             status_code=400,
             code="invalid_import_schema_version",
             message=(
-                f"Import schema version is not supported. Expected {import_export_service.EXPORT_SCHEMA_VERSION}."
+                "Import schema version is not supported. "
+                f"Expected {import_export_service.EXPORT_SCHEMA_VERSION}."
             ),
         )
     try:
@@ -197,7 +235,14 @@ async def import_scholars_and_publications(
             code="invalid_import_payload",
             message=str(exc),
         ) from exc
-    structured_log(logger, "info", "api.scholars.imported", user_id=current_user.id, **result)
+    logger.info(
+        "api.scholars.imported",
+        extra={
+            "event": "api.scholars.imported",
+            "user_id": current_user.id,
+            **result,
+        },
+    )
     return success_payload(request, data=result)
 
 
@@ -227,13 +272,15 @@ async def create_scholar(
             code="invalid_scholar",
             message=str(exc),
         ) from exc
-    structured_log(logger, "info", "api.scholars.created", user_id=current_user.id, scholar_profile_id=created.id)
-    await enqueue_initial_scrape_job_for_scholar(
-        db_session,
-        profile=created,
-        user_id=current_user.id,
+    logger.info(
+        "api.scholars.created",
+        extra={
+            "event": "api.scholars.created",
+            "user_id": current_user.id,
+            "scholar_profile_id": created.id,
+        },
     )
-    created = await hydrate_scholar_metadata_if_needed(
+    created = await _hydrate_scholar_metadata_if_needed(
         db_session,
         profile=created,
         source=source,
@@ -242,7 +289,7 @@ async def create_scholar(
 
     return success_payload(
         request,
-        data=serialize_scholar(created),
+        data=_serialize_scholar(created),
     )
 
 
@@ -264,7 +311,7 @@ async def search_scholars(
             db_session=db_session,
             query=query,
             limit=limit,
-            **search_kwargs(),
+            **_search_kwargs(),
         )
     except scholar_service.ScholarServiceError as exc:
         raise ApiException(
@@ -273,18 +320,19 @@ async def search_scholars(
             message=str(exc),
         ) from exc
 
-    structured_log(
-        logger,
-        "info",
+    logger.info(
         "api.scholars.search.completed",
-        user_id=current_user.id,
-        query=query.strip(),
-        candidate_count=len(parsed.candidates),
-        state=parsed.state.value,
+        extra={
+            "event": "api.scholars.search.completed",
+            "user_id": current_user.id,
+            "query": query.strip(),
+            "candidate_count": len(parsed.candidates),
+            "state": parsed.state.value,
+        },
     )
     return success_payload(
         request,
-        data=search_response_data(query, parsed),
+        data=_search_response_data(query, parsed),
     )
 
 
@@ -310,17 +358,18 @@ async def toggle_scholar(
             message="Scholar not found.",
         )
     updated = await scholar_service.toggle_scholar_enabled(db_session, profile=profile)
-    structured_log(
-        logger,
-        "info",
+    logger.info(
         "api.scholars.toggled",
-        user_id=current_user.id,
-        scholar_profile_id=updated.id,
-        is_enabled=updated.is_enabled,
+        extra={
+            "event": "api.scholars.toggled",
+            "user_id": current_user.id,
+            "scholar_profile_id": updated.id,
+            "is_enabled": updated.is_enabled,
+        },
     )
     return success_payload(
         request,
-        data=serialize_scholar(updated),
+        data=_serialize_scholar(updated),
     )
 
 
@@ -345,20 +394,18 @@ async def delete_scholar(
             code="scholar_not_found",
             message="Scholar not found.",
         )
-    try:
-        await scholar_service.delete_scholar(
-            db_session,
-            profile=profile,
-            upload_dir=settings.scholar_image_upload_dir,
-        )
-    except scholar_service.ScholarServiceError as exc:
-        raise ApiException(
-            status_code=409,
-            code="scholar_delete_failed",
-            message=str(exc),
-        ) from exc
-    structured_log(
-        logger, "info", "api.scholars.deleted", user_id=current_user.id, scholar_profile_id=scholar_profile_id
+    await scholar_service.delete_scholar(
+        db_session,
+        profile=profile,
+        upload_dir=settings.scholar_image_upload_dir,
+    )
+    logger.info(
+        "api.scholars.deleted",
+        extra={
+            "event": "api.scholars.deleted",
+            "user_id": current_user.id,
+            "scholar_profile_id": scholar_profile_id,
+        },
     )
     return success_payload(
         request,
@@ -403,12 +450,17 @@ async def update_scholar_image_url(
             message=str(exc),
         ) from exc
 
-    structured_log(
-        logger, "info", "api.scholars.image_url_updated", user_id=current_user.id, scholar_profile_id=updated.id
+    logger.info(
+        "api.scholars.image_url_updated",
+        extra={
+            "event": "api.scholars.image_url_updated",
+            "user_id": current_user.id,
+            "scholar_profile_id": updated.id,
+        },
     )
     return success_payload(
         request,
-        data=serialize_scholar(updated),
+        data=_serialize_scholar(updated),
     )
 
 
@@ -423,13 +475,13 @@ async def upload_scholar_image(
     db_session: AsyncSession = Depends(get_db_session),
     current_user: User = Depends(get_api_current_user),
 ):
-    profile = await require_user_profile(
+    profile = await _require_user_profile(
         db_session,
         user_id=current_user.id,
         scholar_profile_id=scholar_profile_id,
     )
 
-    image_bytes = await read_uploaded_image(image)
+    image_bytes = await _read_uploaded_image(image)
     try:
         updated = await scholar_service.set_profile_image_upload(
             db_session,
@@ -447,18 +499,19 @@ async def upload_scholar_image(
         ) from exc
 
     image_size = len(image_bytes)
-    structured_log(
-        logger,
-        "info",
+    logger.info(
         "api.scholars.image_uploaded",
-        user_id=current_user.id,
-        scholar_profile_id=updated.id,
-        content_type=image.content_type,
-        size_bytes=image_size,
+        extra={
+            "event": "api.scholars.image_uploaded",
+            "user_id": current_user.id,
+            "scholar_profile_id": updated.id,
+            "content_type": image.content_type,
+            "size_bytes": image_size,
+        },
     )
     return success_payload(
         request,
-        data=serialize_scholar(updated),
+        data=_serialize_scholar(updated),
     )
 
 
@@ -489,8 +542,16 @@ async def clear_scholar_image_customization(
         profile=profile,
         upload_dir=settings.scholar_image_upload_dir,
     )
-    structured_log(logger, "info", "api.scholars.image_cleared", user_id=current_user.id, scholar_profile_id=updated.id)
+    logger.info(
+        "api.scholars.image_customization_cleared",
+        extra={
+            "event": "api.scholars.image_customization_cleared",
+            "user_id": current_user.id,
+            "scholar_profile_id": updated.id,
+        },
+    )
     return success_payload(
         request,
-        data=serialize_scholar(updated),
+        data=_serialize_scholar(updated),
     )
+

app/api/routers/settings.py

@@ -11,9 +11,8 @@ from app.api.responses import success_payload
 from app.api.schemas import SettingsEnvelope, SettingsUpdateRequest
 from app.db.models import User
 from app.db.session import get_db_session
-from app.logging_utils import structured_log
-from app.services.ingestion import safety as run_safety_service
-from app.services.settings import application as user_settings_service
+from app.services.domains.ingestion import safety as run_safety_service
+from app.services.domains.settings import application as user_settings_service
 from app.settings import settings as settings_module
 
 logger = logging.getLogger(__name__)
@@ -39,9 +38,6 @@ def _serialize_settings(user_settings) -> dict[str, object]:
             "cooldown_network_seconds": max(60, int(settings_module.ingestion_safety_cooldown_network_seconds)),
         },
         "safety_state": run_safety_service.get_safety_state_payload(user_settings),
-        "openalex_api_key": user_settings.openalex_api_key,
-        "crossref_api_token": user_settings.crossref_api_token,
-        "crossref_api_mailto": user_settings.crossref_api_mailto,
     }
 
 
@@ -84,13 +80,16 @@ async def _clear_expired_cooldown_with_log(
         return
     await db_session.commit()
     await db_session.refresh(user_settings)
-    structured_log(
-        logger,
-        "info",
+    logger.info(
         "api.settings.safety_cooldown_cleared",
-        user_id=user_id,
-        reason=previous_safety_state.get("cooldown_reason"),
-        cooldown_until=previous_safety_state.get("cooldown_until"),
+        extra={
+            "event": "api.settings.safety_cooldown_cleared",
+            "user_id": user_id,
+            "reason": previous_safety_state.get("cooldown_reason"),
+            "cooldown_until": previous_safety_state.get("cooldown_until"),
+            "metric_name": "api_settings_safety_cooldown_cleared_total",
+            "metric_value": 1,
+        },
     )
 
 
@@ -98,6 +97,20 @@ def _effective_auto_run_enabled(payload: SettingsUpdateRequest) -> bool:
     return bool(payload.auto_run_enabled) and settings_module.ingestion_automation_allowed
 
 
+def _log_settings_update(*, user_id: int, updated) -> None:
+    logger.info(
+        "api.settings.updated",
+        extra={
+            "event": "api.settings.updated",
+            "user_id": user_id,
+            "auto_run_enabled": updated.auto_run_enabled,
+            "run_interval_minutes": updated.run_interval_minutes,
+            "request_delay_seconds": updated.request_delay_seconds,
+            "nav_visible_pages": updated.nav_visible_pages,
+        },
+    )
+
+
 @router.get(
     "",
     response_model=SettingsEnvelope,
@@ -156,26 +169,13 @@ async def update_settings(
         run_interval_minutes=parsed_interval,
         request_delay_seconds=parsed_delay,
         nav_visible_pages=parsed_nav_visible_pages,
-        openalex_api_key=payload.openalex_api_key,
-        crossref_api_token=payload.crossref_api_token,
-        crossref_api_mailto=payload.crossref_api_mailto,
     )
     await _clear_expired_cooldown_with_log(
         db_session,
         user_id=current_user.id,
         user_settings=updated,
     )
-    structured_log(
-        logger,
-        "info",
-        "api.settings.updated",
-        user_id=current_user.id,
-        auto_run_enabled=updated.auto_run_enabled,
-        run_interval_minutes=updated.run_interval_minutes,
-        request_delay_seconds=updated.request_delay_seconds,
-        nav_visible_pages=updated.nav_visible_pages,
-        openalex_api_key="SET" if updated.openalex_api_key else "UNSET",
-    )
+    _log_settings_update(user_id=current_user.id, updated=updated)
     return success_payload(
         request,
         data=_serialize_settings(updated),

app/api/runtime_deps.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 from fastapi import Depends
 
-from app.services.ingestion import application as ingestion_service
-from app.services.scholar import source as scholar_source_service
+from app.services.domains.ingestion import application as ingestion_service
+from app.services.domains.scholar import source as scholar_source_service
 
 
 def get_scholar_source() -> scholar_source_service.ScholarSource:

app/api/schemas.py

@@ -0,0 +1,867 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class ApiMeta(BaseModel):
+    request_id: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ApiErrorData(BaseModel):
+    code: str
+    message: str
+    details: Any | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ApiErrorEnvelope(BaseModel):
+    error: ApiErrorData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessageData(BaseModel):
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessageEnvelope(BaseModel):
+    data: MessageData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SessionUserData(BaseModel):
+    id: int
+    email: str
+    is_admin: bool
+    is_active: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AuthMeData(BaseModel):
+    authenticated: bool
+    csrf_token: str
+    user: SessionUserData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AuthMeEnvelope(BaseModel):
+    data: AuthMeData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class CsrfBootstrapData(BaseModel):
+    csrf_token: str
+    authenticated: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class CsrfBootstrapEnvelope(BaseModel):
+    data: CsrfBootstrapData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginRequest(BaseModel):
+    email: str
+    password: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginData(BaseModel):
+    authenticated: bool
+    csrf_token: str
+    user: SessionUserData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginEnvelope(BaseModel):
+    data: LoginData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ChangePasswordRequest(BaseModel):
+    current_password: str
+    new_password: str
+    confirm_password: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarItemData(BaseModel):
+    id: int
+    scholar_id: str
+    display_name: str | None
+    profile_image_url: str | None
+    profile_image_source: str
+    is_enabled: bool
+    baseline_completed: bool
+    last_run_dt: datetime | None
+    last_run_status: str | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarsListData(BaseModel):
+    scholars: list[ScholarItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarsListEnvelope(BaseModel):
+    data: ScholarsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarCreateRequest(BaseModel):
+    scholar_id: str
+    profile_image_url: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarEnvelope(BaseModel):
+    data: ScholarItemData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchCandidateData(BaseModel):
+    scholar_id: str
+    display_name: str
+    affiliation: str | None
+    email_domain: str | None
+    cited_by_count: int | None
+    interests: list[str] = Field(default_factory=list)
+    profile_url: str
+    profile_image_url: str | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchData(BaseModel):
+    query: str
+    state: str
+    state_reason: str
+    action_hint: str | None = None
+    candidates: list[ScholarSearchCandidateData]
+    warnings: list[str] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchEnvelope(BaseModel):
+    data: ScholarSearchData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarImageUrlUpdateRequest(BaseModel):
+    image_url: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarExportItemData(BaseModel):
+    scholar_id: str
+    display_name: str | None = None
+    is_enabled: bool = True
+    profile_image_override_url: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationExportItemData(BaseModel):
+    scholar_id: str
+    cluster_id: str | None = None
+    fingerprint_sha256: str | None = None
+    title: str
+    year: int | None = None
+    citation_count: int = 0
+    author_text: str | None = None
+    venue_text: str | None = None
+    pub_url: str | None = None
+    doi: str | None = None
+    pdf_url: str | None = None
+    is_read: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataExportData(BaseModel):
+    schema_version: int
+    exported_at: str
+    scholars: list[ScholarExportItemData]
+    publications: list[PublicationExportItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataExportEnvelope(BaseModel):
+    data: DataExportData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportRequest(BaseModel):
+    schema_version: int | None = None
+    scholars: list[ScholarExportItemData] = Field(default_factory=list)
+    publications: list[PublicationExportItemData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportResultData(BaseModel):
+    scholars_created: int
+    scholars_updated: int
+    publications_created: int
+    publications_updated: int
+    links_created: int
+    links_updated: int
+    skipped_records: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportEnvelope(BaseModel):
+    data: DataImportResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunListItemData(BaseModel):
+    id: int
+    trigger_type: str
+    status: str
+    start_dt: datetime
+    end_dt: datetime | None
+    scholar_count: int
+    new_publication_count: int
+    failed_count: int
+    partial_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunsListData(BaseModel):
+    runs: list[RunListItemData]
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunsListEnvelope(BaseModel):
+    data: RunsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunSummaryData(BaseModel):
+    succeeded_count: int
+    failed_count: int
+    partial_count: int
+    failed_state_counts: dict[str, int] = Field(default_factory=dict)
+    failed_reason_counts: dict[str, int] = Field(default_factory=dict)
+    scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
+    retry_counts: dict[str, int] = Field(default_factory=dict)
+    alert_thresholds: dict[str, int] = Field(default_factory=dict)
+    alert_flags: dict[str, bool] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScrapeSafetyCountersData(BaseModel):
+    consecutive_blocked_runs: int = 0
+    consecutive_network_runs: int = 0
+    cooldown_entry_count: int = 0
+    blocked_start_count: int = 0
+    last_blocked_failure_count: int = 0
+    last_network_failure_count: int = 0
+    last_evaluated_run_id: int | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScrapeSafetyStateData(BaseModel):
+    cooldown_active: bool
+    cooldown_reason: str | None = None
+    cooldown_reason_label: str | None = None
+    cooldown_until: datetime | None = None
+    cooldown_remaining_seconds: int = 0
+    recommended_action: str | None = None
+    counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunAttemptLogData(BaseModel):
+    attempt: int
+    cstart: int
+    state: str | None = None
+    state_reason: str | None = None
+    status_code: int | None = None
+    fetch_error: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunPageLogData(BaseModel):
+    page: int
+    cstart: int
+    state: str
+    state_reason: str | None = None
+    status_code: int | None = None
+    publication_count: int = 0
+    attempt_count: int = 0
+    has_show_more_button: bool = False
+    articles_range: str | None = None
+    warning_codes: list[str] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDebugData(BaseModel):
+    status_code: int | None = None
+    final_url: str | None = None
+    fetch_error: str | None = None
+    body_sha256: str | None = None
+    body_length: int | None = None
+    has_show_more_button: bool | None = None
+    articles_range: str | None = None
+    state_reason: str | None = None
+    warning_codes: list[str] = Field(default_factory=list)
+    marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
+    page_logs: list[RunPageLogData] = Field(default_factory=list)
+    attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunScholarResultData(BaseModel):
+    scholar_profile_id: int
+    scholar_id: str
+    state: str
+    state_reason: str | None = None
+    outcome: str
+    attempt_count: int = 0
+    publication_count: int = 0
+    start_cstart: int = 0
+    continuation_cstart: int | None = None
+    continuation_enqueued: bool = False
+    continuation_cleared: bool = False
+    warnings: list[str] = Field(default_factory=list)
+    error: str | None = None
+    debug: RunDebugData | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDetailData(BaseModel):
+    run: RunListItemData
+    summary: RunSummaryData
+    scholar_results: list[RunScholarResultData]
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDetailEnvelope(BaseModel):
+    data: RunDetailData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ManualRunData(BaseModel):
+    run_id: int
+    status: str
+    scholar_count: int
+    succeeded_count: int
+    failed_count: int
+    partial_count: int
+    new_publication_count: int
+    reused_existing_run: bool
+    idempotency_key: str | None = None
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ManualRunEnvelope(BaseModel):
+    data: ManualRunData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueItemData(BaseModel):
+    id: int
+    scholar_profile_id: int
+    scholar_label: str
+    status: str
+    reason: str
+    dropped_reason: str | None
+    attempt_count: int
+    resume_cstart: int
+    next_attempt_dt: datetime | None
+    updated_at: datetime
+    last_error: str | None
+    last_run_id: int | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueListData(BaseModel):
+    queue_items: list[QueueItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueListEnvelope(BaseModel):
+    data: QueueListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueItemEnvelope(BaseModel):
+    data: QueueItemData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueClearData(BaseModel):
+    queue_item_id: int
+    previous_status: str
+    status: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueClearEnvelope(BaseModel):
+    data: QueueClearData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserData(BaseModel):
+    id: int
+    email: str
+    is_active: bool
+    is_admin: bool
+    created_at: datetime
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUsersListData(BaseModel):
+    users: list[AdminUserData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUsersListEnvelope(BaseModel):
+    data: AdminUsersListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserCreateRequest(BaseModel):
+    email: str
+    password: str
+    is_admin: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserActiveUpdateRequest(BaseModel):
+    is_active: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserEnvelope(BaseModel):
+    data: AdminUserData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminResetPasswordRequest(BaseModel):
+    new_password: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityCheckData(BaseModel):
+    name: str
+    count: int
+    severity: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityData(BaseModel):
+    status: str
+    checked_at: datetime
+    failures: list[str] = Field(default_factory=list)
+    warnings: list[str] = Field(default_factory=list)
+    checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityEnvelope(BaseModel):
+    data: AdminDbIntegrityData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobData(BaseModel):
+    id: int
+    job_name: str
+    requested_by: str | None
+    dry_run: bool
+    status: str
+    scope: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any] = Field(default_factory=dict)
+    error_text: str | None
+    started_at: datetime | None
+    finished_at: datetime | None
+    created_at: datetime
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobsData(BaseModel):
+    jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobsEnvelope(BaseModel):
+    data: AdminDbRepairJobsData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueItemData(BaseModel):
+    publication_id: int
+    title: str
+    doi: str | None
+    pdf_url: str | None
+    status: str
+    attempt_count: int
+    last_failure_reason: str | None
+    last_failure_detail: str | None
+    last_source: str | None
+    requested_by_user_id: int | None
+    requested_by_email: str | None
+    queued_at: datetime | None
+    last_attempt_at: datetime | None
+    resolved_at: datetime | None
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueData(BaseModel):
+    items: list[AdminPdfQueueItemData] = Field(default_factory=list)
+    total_count: int = 0
+    page: int = 1
+    page_size: int = 100
+    has_next: bool = False
+    has_prev: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueEnvelope(BaseModel):
+    data: AdminPdfQueueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueRequeueData(BaseModel):
+    publication_id: int
+    queued: bool
+    status: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueRequeueEnvelope(BaseModel):
+    data: AdminPdfQueueRequeueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueBulkEnqueueData(BaseModel):
+    requested_count: int
+    queued_count: int
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
+    data: AdminPdfQueueBulkEnqueueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationLinksRequest(BaseModel):
+    scope_mode: Literal["single_user", "all_users"] = "single_user"
+    user_id: int | None = Field(default=None, ge=1)
+    scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
+    dry_run: bool = True
+    gc_orphan_publications: bool = False
+    requested_by: str | None = None
+    confirmation_text: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")
+    def validate_scope(self) -> "AdminRepairPublicationLinksRequest":
+        if self.scope_mode == "single_user" and self.user_id is None:
+            raise ValueError("user_id is required when scope_mode=single_user.")
+        if self.scope_mode == "all_users" and self.user_id is not None:
+            raise ValueError("user_id must be omitted when scope_mode=all_users.")
+        if not self.dry_run and self.scope_mode == "all_users":
+            expected = "REPAIR ALL USERS"
+            provided = (self.confirmation_text or "").strip()
+            if provided != expected:
+                raise ValueError(
+                    "confirmation_text must equal 'REPAIR ALL USERS' "
+                    "when applying a repair to all users."
+                )
+        return self
+
+
+class AdminRepairPublicationLinksResultData(BaseModel):
+    job_id: int
+    status: str
+    scope: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationLinksEnvelope(BaseModel):
+    data: AdminRepairPublicationLinksResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsPolicyData(BaseModel):
+    min_run_interval_minutes: int
+    min_request_delay_seconds: int
+    automation_allowed: bool
+    manual_run_allowed: bool
+    blocked_failure_threshold: int
+    network_failure_threshold: int
+    cooldown_blocked_seconds: int
+    cooldown_network_seconds: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsData(BaseModel):
+    auto_run_enabled: bool
+    run_interval_minutes: int
+    request_delay_seconds: int
+    nav_visible_pages: list[str]
+    policy: SettingsPolicyData
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsEnvelope(BaseModel):
+    data: SettingsData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsUpdateRequest(BaseModel):
+    auto_run_enabled: bool
+    run_interval_minutes: int
+    request_delay_seconds: int
+    nav_visible_pages: list[str] | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationItemData(BaseModel):
+    publication_id: int
+    scholar_profile_id: int
+    scholar_label: str
+    title: str
+    year: int | None
+    citation_count: int
+    venue_text: str | None
+    pub_url: str | None
+    doi: str | None
+    pdf_url: str | None
+    pdf_status: str = "untracked"
+    pdf_attempt_count: int = 0
+    pdf_failure_reason: str | None = None
+    pdf_failure_detail: str | None = None
+    is_read: bool
+    is_favorite: bool = False
+    first_seen_at: datetime
+    is_new_in_latest_run: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationsListData(BaseModel):
+    mode: str
+    favorite_only: bool = False
+    selected_scholar_profile_id: int | None
+    unread_count: int
+    favorites_count: int
+    latest_count: int
+    new_count: int
+    total_count: int
+    page: int
+    page_size: int
+    has_next: bool = False
+    has_prev: bool = False
+    publications: list[PublicationItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationsListEnvelope(BaseModel):
+    data: PublicationsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkAllReadData(BaseModel):
+    message: str
+    updated_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkAllReadEnvelope(BaseModel):
+    data: MarkAllReadData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationSelectionItem(BaseModel):
+    scholar_profile_id: int
+    publication_id: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadRequest(BaseModel):
+    selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadData(BaseModel):
+    message: str
+    requested_count: int
+    updated_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadEnvelope(BaseModel):
+    data: MarkSelectedReadData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfRequest(BaseModel):
+    scholar_profile_id: int = Field(ge=1)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfData(BaseModel):
+    message: str
+    queued: bool
+    resolved_pdf: bool
+    publication: PublicationItemData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfEnvelope(BaseModel):
+    data: RetryPublicationPdfData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteRequest(BaseModel):
+    scholar_profile_id: int = Field(ge=1)
+    is_favorite: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteData(BaseModel):
+    message: str
+    publication: PublicationItemData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteEnvelope(BaseModel):
+    data: TogglePublicationFavoriteData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/__init__.py

@@ -1,7 +0,0 @@
-from app.api.schemas.admin import *  # noqa: F403
-from app.api.schemas.auth import *  # noqa: F403
-from app.api.schemas.common import *  # noqa: F403
-from app.api.schemas.publications import *  # noqa: F403
-from app.api.schemas.runs import *  # noqa: F403
-from app.api.schemas.scholars import *  # noqa: F403
-from app.api.schemas.settings import *  # noqa: F403

app/api/schemas/admin.py

@@ -1,314 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Any, Literal
-
-from pydantic import BaseModel, ConfigDict, Field, model_validator
-
-from app.api.schemas.common import ApiMeta
-from app.api.schemas.publications import DisplayIdentifierData
-
-
-class AdminUserData(BaseModel):
-    id: int
-    email: str
-    is_active: bool
-    is_admin: bool
-    created_at: datetime
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUsersListData(BaseModel):
-    users: list[AdminUserData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUsersListEnvelope(BaseModel):
-    data: AdminUsersListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserCreateRequest(BaseModel):
-    email: str = Field(max_length=254)
-    password: str = Field(min_length=8, max_length=128)
-    is_admin: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserActiveUpdateRequest(BaseModel):
-    is_active: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserEnvelope(BaseModel):
-    data: AdminUserData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminResetPasswordRequest(BaseModel):
-    new_password: str = Field(min_length=8, max_length=128)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminScholarHttpSettingsData(BaseModel):
-    user_agent: str
-    rotate_user_agent: bool
-    accept_language: str
-    cookie: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminScholarHttpSettingsEnvelope(BaseModel):
-    data: AdminScholarHttpSettingsData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminScholarHttpSettingsUpdateRequest(BaseModel):
-    user_agent: str
-    rotate_user_agent: bool
-    accept_language: str
-    cookie: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityCheckData(BaseModel):
-    name: str
-    count: int
-    severity: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityData(BaseModel):
-    status: str
-    checked_at: datetime
-    failures: list[str] = Field(default_factory=list)
-    warnings: list[str] = Field(default_factory=list)
-    checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityEnvelope(BaseModel):
-    data: AdminDbIntegrityData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobData(BaseModel):
-    id: int
-    job_name: str
-    requested_by: str | None
-    dry_run: bool
-    status: str
-    scope: dict[str, Any] = Field(default_factory=dict)
-    summary: dict[str, Any] = Field(default_factory=dict)
-    error_text: str | None
-    started_at: datetime | None
-    finished_at: datetime | None
-    created_at: datetime
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobsData(BaseModel):
-    jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobsEnvelope(BaseModel):
-    data: AdminDbRepairJobsData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueItemData(BaseModel):
-    publication_id: int
-    title: str
-    display_identifier: DisplayIdentifierData | None = None
-    pdf_url: str | None
-    status: str
-    attempt_count: int
-    last_failure_reason: str | None
-    last_failure_detail: str | None
-    last_source: str | None
-    requested_by_user_id: int | None
-    requested_by_email: str | None
-    queued_at: datetime | None
-    last_attempt_at: datetime | None
-    resolved_at: datetime | None
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueData(BaseModel):
-    items: list[AdminPdfQueueItemData] = Field(default_factory=list)
-    total_count: int = 0
-    page: int = 1
-    page_size: int = 100
-    has_next: bool = False
-    has_prev: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueEnvelope(BaseModel):
-    data: AdminPdfQueueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueRequeueData(BaseModel):
-    publication_id: int
-    queued: bool
-    status: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueRequeueEnvelope(BaseModel):
-    data: AdminPdfQueueRequeueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueBulkEnqueueData(BaseModel):
-    requested_count: int
-    queued_count: int
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
-    data: AdminPdfQueueBulkEnqueueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationLinksRequest(BaseModel):
-    scope_mode: Literal["single_user", "all_users"] = "single_user"
-    user_id: int | None = Field(default=None, ge=1)
-    scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
-    dry_run: bool = True
-    gc_orphan_publications: bool = False
-    requested_by: str | None = None
-    confirmation_text: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-    @model_validator(mode="after")
-    def validate_scope(self) -> AdminRepairPublicationLinksRequest:
-        if self.scope_mode == "single_user" and self.user_id is None:
-            raise ValueError("user_id is required when scope_mode=single_user.")
-        if self.scope_mode == "all_users" and self.user_id is not None:
-            raise ValueError("user_id must be omitted when scope_mode=all_users.")
-        if not self.dry_run and self.scope_mode == "all_users":
-            expected = "REPAIR ALL USERS"
-            provided = (self.confirmation_text or "").strip()
-            if provided != expected:
-                raise ValueError("confirmation_text must equal 'REPAIR ALL USERS' when applying a repair to all users.")
-        return self
-
-
-class AdminRepairPublicationLinksResultData(BaseModel):
-    job_id: int
-    status: str
-    scope: dict[str, Any] = Field(default_factory=dict)
-    summary: dict[str, Any] = Field(default_factory=dict)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationLinksEnvelope(BaseModel):
-    data: AdminRepairPublicationLinksResultData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminNearDuplicateClusterMemberData(BaseModel):
-    publication_id: int
-    title: str
-    year: int | None
-    citation_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminNearDuplicateClusterData(BaseModel):
-    cluster_key: str
-    winner_publication_id: int
-    member_count: int
-    similarity_score: float = Field(ge=0.0, le=1.0)
-    members: list[AdminNearDuplicateClusterMemberData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationNearDuplicatesRequest(BaseModel):
-    dry_run: bool = True
-    similarity_threshold: float = Field(default=0.78, ge=0.5, le=1.0)
-    min_shared_tokens: int = Field(default=3, ge=1, le=8)
-    max_year_delta: int = Field(default=1, ge=0, le=5)
-    max_clusters: int = Field(default=25, ge=1, le=200)
-    selected_cluster_keys: list[str] = Field(default_factory=list, max_length=200)
-    requested_by: str | None = None
-    confirmation_text: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-    @model_validator(mode="after")
-    def validate_apply_mode(self) -> AdminRepairPublicationNearDuplicatesRequest:
-        if self.dry_run:
-            return self
-        if not self.selected_cluster_keys:
-            raise ValueError("selected_cluster_keys is required when dry_run=false.")
-        expected = "MERGE SELECTED DUPLICATES"
-        provided = (self.confirmation_text or "").strip()
-        if provided != expected:
-            raise ValueError(
-                "confirmation_text must equal 'MERGE SELECTED DUPLICATES' when applying near-duplicate merges."
-            )
-        return self
-
-
-class AdminRepairPublicationNearDuplicatesResultData(BaseModel):
-    job_id: int
-    status: str
-    scope: dict[str, Any] = Field(default_factory=dict)
-    summary: dict[str, Any] = Field(default_factory=dict)
-    clusters: list[AdminNearDuplicateClusterData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationNearDuplicatesEnvelope(BaseModel):
-    data: AdminRepairPublicationNearDuplicatesResultData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/auth.py

@@ -1,73 +0,0 @@
-from __future__ import annotations
-
-from pydantic import BaseModel, ConfigDict, Field
-
-from app.api.schemas.common import ApiMeta
-
-
-class SessionUserData(BaseModel):
-    id: int
-    email: str
-    is_admin: bool
-    is_active: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AuthMeData(BaseModel):
-    authenticated: bool
-    csrf_token: str
-    user: SessionUserData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AuthMeEnvelope(BaseModel):
-    data: AuthMeData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class CsrfBootstrapData(BaseModel):
-    csrf_token: str
-    authenticated: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class CsrfBootstrapEnvelope(BaseModel):
-    data: CsrfBootstrapData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginRequest(BaseModel):
-    email: str = Field(max_length=254)
-    password: str = Field(min_length=8, max_length=128)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginData(BaseModel):
-    authenticated: bool
-    csrf_token: str
-    user: SessionUserData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginEnvelope(BaseModel):
-    data: LoginData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ChangePasswordRequest(BaseModel):
-    current_password: str = Field(min_length=8, max_length=128)
-    new_password: str = Field(min_length=8, max_length=128)
-    confirm_password: str = Field(min_length=8, max_length=128)
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/common.py

@@ -1,39 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-from pydantic import BaseModel, ConfigDict
-
-
-class ApiMeta(BaseModel):
-    request_id: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ApiErrorData(BaseModel):
-    code: str
-    message: str
-    details: Any | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ApiErrorEnvelope(BaseModel):
-    error: ApiErrorData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessageData(BaseModel):
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessageEnvelope(BaseModel):
-    data: MessageData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/publications.py

@@ -1,151 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime
-
-from pydantic import BaseModel, ConfigDict, Field
-
-from app.api.schemas.common import ApiMeta
-
-
-class DisplayIdentifierData(BaseModel):
-    kind: str
-    value: str
-    label: str
-    url: str | None
-    confidence_score: float = Field(ge=0.0, le=1.0)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationItemData(BaseModel):
-    publication_id: int
-    scholar_profile_id: int
-    scholar_label: str
-    title: str
-    year: int | None
-    citation_count: int
-    venue_text: str | None
-    pub_url: str | None
-    display_identifier: DisplayIdentifierData | None = None
-    pdf_url: str | None
-    pdf_status: str = "untracked"
-    pdf_attempt_count: int = 0
-    pdf_failure_reason: str | None = None
-    pdf_failure_detail: str | None = None
-    is_read: bool
-    is_favorite: bool = False
-    first_seen_at: datetime
-    is_new_in_latest_run: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationsListData(BaseModel):
-    mode: str
-    favorite_only: bool = False
-    selected_scholar_profile_id: int | None
-    unread_count: int
-    favorites_count: int
-    latest_count: int
-    new_count: int
-    total_count: int
-    page: int
-    page_size: int
-    snapshot: str
-    has_next: bool = False
-    has_prev: bool = False
-    publications: list[PublicationItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationsListEnvelope(BaseModel):
-    data: PublicationsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkAllReadData(BaseModel):
-    message: str
-    updated_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkAllReadEnvelope(BaseModel):
-    data: MarkAllReadData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationSelectionItem(BaseModel):
-    scholar_profile_id: int
-    publication_id: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadRequest(BaseModel):
-    selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadData(BaseModel):
-    message: str
-    requested_count: int
-    updated_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadEnvelope(BaseModel):
-    data: MarkSelectedReadData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfRequest(BaseModel):
-    scholar_profile_id: int = Field(ge=1)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfData(BaseModel):
-    message: str
-    queued: bool
-    resolved_pdf: bool
-    publication: PublicationItemData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfEnvelope(BaseModel):
-    data: RetryPublicationPdfData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteRequest(BaseModel):
-    scholar_profile_id: int = Field(ge=1)
-    is_favorite: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteData(BaseModel):
-    message: str
-    publication: PublicationItemData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteEnvelope(BaseModel):
-    data: TogglePublicationFavoriteData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/runs.py

@@ -1,226 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime
-
-from pydantic import BaseModel, ConfigDict, Field
-
-from app.api.schemas.common import ApiMeta
-
-
-class RunListItemData(BaseModel):
-    id: int
-    trigger_type: str
-    status: str
-    start_dt: datetime
-    end_dt: datetime | None
-    scholar_count: int
-    new_publication_count: int
-    failed_count: int
-    partial_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScrapeSafetyCountersData(BaseModel):
-    consecutive_blocked_runs: int = 0
-    consecutive_network_runs: int = 0
-    cooldown_entry_count: int = 0
-    blocked_start_count: int = 0
-    last_blocked_failure_count: int = 0
-    last_network_failure_count: int = 0
-    last_evaluated_run_id: int | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScrapeSafetyStateData(BaseModel):
-    cooldown_active: bool
-    cooldown_reason: str | None = None
-    cooldown_reason_label: str | None = None
-    cooldown_until: datetime | None = None
-    cooldown_remaining_seconds: int = 0
-    recommended_action: str | None = None
-    counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunsListData(BaseModel):
-    runs: list[RunListItemData]
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunsListEnvelope(BaseModel):
-    data: RunsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunSummaryData(BaseModel):
-    succeeded_count: int
-    failed_count: int
-    partial_count: int
-    failed_state_counts: dict[str, int] = Field(default_factory=dict)
-    failed_reason_counts: dict[str, int] = Field(default_factory=dict)
-    scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
-    retry_counts: dict[str, int] = Field(default_factory=dict)
-    alert_thresholds: dict[str, int] = Field(default_factory=dict)
-    alert_flags: dict[str, bool] = Field(default_factory=dict)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunAttemptLogData(BaseModel):
-    attempt: int
-    cstart: int
-    state: str | None = None
-    state_reason: str | None = None
-    status_code: int | None = None
-    fetch_error: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunPageLogData(BaseModel):
-    page: int
-    cstart: int
-    state: str
-    state_reason: str | None = None
-    status_code: int | None = None
-    publication_count: int = 0
-    attempt_count: int = 0
-    has_show_more_button: bool = False
-    articles_range: str | None = None
-    warning_codes: list[str] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDebugData(BaseModel):
-    status_code: int | None = None
-    final_url: str | None = None
-    fetch_error: str | None = None
-    body_sha256: str | None = None
-    body_length: int | None = None
-    has_show_more_button: bool | None = None
-    articles_range: str | None = None
-    state_reason: str | None = None
-    warning_codes: list[str] = Field(default_factory=list)
-    marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
-    page_logs: list[RunPageLogData] = Field(default_factory=list)
-    attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunScholarResultData(BaseModel):
-    scholar_profile_id: int
-    scholar_id: str
-    state: str
-    state_reason: str | None = None
-    outcome: str
-    attempt_count: int = 0
-    publication_count: int = 0
-    start_cstart: int = 0
-    continuation_cstart: int | None = None
-    continuation_enqueued: bool = False
-    continuation_cleared: bool = False
-    warnings: list[str] = Field(default_factory=list)
-    error: str | None = None
-    debug: RunDebugData | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDetailData(BaseModel):
-    run: RunListItemData
-    summary: RunSummaryData
-    scholar_results: list[RunScholarResultData]
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDetailEnvelope(BaseModel):
-    data: RunDetailData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ManualRunData(BaseModel):
-    run_id: int
-    status: str
-    scholar_count: int
-    succeeded_count: int
-    failed_count: int
-    partial_count: int
-    new_publication_count: int
-    reused_existing_run: bool
-    idempotency_key: str | None = None
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ManualRunEnvelope(BaseModel):
-    data: ManualRunData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueItemData(BaseModel):
-    id: int
-    scholar_profile_id: int
-    scholar_label: str
-    status: str
-    reason: str
-    dropped_reason: str | None
-    attempt_count: int
-    resume_cstart: int
-    next_attempt_dt: datetime | None
-    updated_at: datetime
-    last_error: str | None
-    last_run_id: int | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueListData(BaseModel):
-    queue_items: list[QueueItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueListEnvelope(BaseModel):
-    data: QueueListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueItemEnvelope(BaseModel):
-    data: QueueItemData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueClearData(BaseModel):
-    queue_item_id: int
-    previous_status: str
-    status: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueClearEnvelope(BaseModel):
-    data: QueueClearData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/scholars.py

@@ -1,189 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime
-
-from pydantic import BaseModel, ConfigDict, Field, model_validator
-
-from app.api.schemas.common import ApiMeta
-
-
-class ScholarItemData(BaseModel):
-    id: int
-    scholar_id: str
-    display_name: str | None
-    profile_image_url: str | None
-    profile_image_source: str
-    is_enabled: bool
-    baseline_completed: bool
-    last_run_dt: datetime | None
-    last_run_status: str | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarsListData(BaseModel):
-    scholars: list[ScholarItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarsListEnvelope(BaseModel):
-    data: ScholarsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarCreateRequest(BaseModel):
-    scholar_id: str
-    profile_image_url: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarEnvelope(BaseModel):
-    data: ScholarItemData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchCandidateData(BaseModel):
-    scholar_id: str
-    display_name: str
-    affiliation: str | None
-    email_domain: str | None
-    cited_by_count: int | None
-    interests: list[str] = Field(default_factory=list)
-    profile_url: str
-    profile_image_url: str | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchData(BaseModel):
-    query: str
-    state: str
-    state_reason: str
-    action_hint: str | None = None
-    candidates: list[ScholarSearchCandidateData]
-    warnings: list[str] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchEnvelope(BaseModel):
-    data: ScholarSearchData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarImageUrlUpdateRequest(BaseModel):
-    image_url: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarExportItemData(BaseModel):
-    scholar_id: str
-    display_name: str | None = None
-    is_enabled: bool = True
-    profile_image_override_url: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationExportItemData(BaseModel):
-    scholar_id: str
-    cluster_id: str | None = None
-    fingerprint_sha256: str | None = None
-    title: str
-    year: int | None = None
-    citation_count: int = 0
-    author_text: str | None = None
-    venue_text: str | None = None
-    pub_url: str | None = None
-    pdf_url: str | None = None
-    is_read: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataExportData(BaseModel):
-    schema_version: int
-    exported_at: str
-    scholars: list[ScholarExportItemData]
-    publications: list[PublicationExportItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataExportEnvelope(BaseModel):
-    data: DataExportData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarBulkIdsRequest(BaseModel):
-    scholar_profile_ids: list[int] = Field(..., min_length=1, max_length=500)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarBulkToggleRequest(BaseModel):
-    scholar_profile_ids: list[int] = Field(..., min_length=1, max_length=500)
-    is_enabled: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarBulkCountData(BaseModel):
-    deleted_count: int = 0
-    updated_count: int = 0
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarBulkCountEnvelope(BaseModel):
-    data: ScholarBulkCountData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataImportRequest(BaseModel):
-    schema_version: int | None = None
-    exported_at: str | None = None
-    scholars: list[ScholarExportItemData] = Field(default_factory=list)
-    publications: list[PublicationExportItemData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-    @model_validator(mode="before")
-    @classmethod
-    def unwrap_export_envelope(cls, values: dict) -> dict:
-        """Accept the full export envelope format (with data/meta wrapper)."""
-        if isinstance(values, dict) and "data" in values and isinstance(values["data"], dict):
-            return values["data"]
-        return values
-
-
-class DataImportResultData(BaseModel):
-    scholars_created: int
-    scholars_updated: int
-    publications_created: int
-    publications_updated: int
-    links_created: int
-    links_updated: int
-    skipped_records: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataImportEnvelope(BaseModel):
-    data: DataImportResultData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/settings.py

@@ -1,54 +0,0 @@
-from __future__ import annotations
-
-from pydantic import BaseModel, ConfigDict
-
-from app.api.schemas.common import ApiMeta
-from app.api.schemas.runs import ScrapeSafetyStateData
-
-
-class SettingsPolicyData(BaseModel):
-    min_run_interval_minutes: int
-    min_request_delay_seconds: int
-    automation_allowed: bool
-    manual_run_allowed: bool
-    blocked_failure_threshold: int
-    network_failure_threshold: int
-    cooldown_blocked_seconds: int
-    cooldown_network_seconds: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsData(BaseModel):
-    auto_run_enabled: bool
-    run_interval_minutes: int
-    request_delay_seconds: int
-    nav_visible_pages: list[str]
-    policy: SettingsPolicyData
-    safety_state: ScrapeSafetyStateData
-
-    openalex_api_key: str | None = None
-    crossref_api_token: str | None = None
-    crossref_api_mailto: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsEnvelope(BaseModel):
-    data: SettingsData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsUpdateRequest(BaseModel):
-    auto_run_enabled: bool
-    run_interval_minutes: int
-    request_delay_seconds: int
-    nav_visible_pages: list[str] | None = None
-
-    openalex_api_key: str | None = None
-    crossref_api_token: str | None = None
-    crossref_api_mailto: str | None = None
-
-    model_config = ConfigDict(extra="forbid")

app/auth/__init__.py

@@ -1 +1,2 @@
 """Authentication package for scholarr."""
+

app/auth/deps.py

@@ -24,3 +24,4 @@ def get_login_rate_limiter() -> SlidingWindowRateLimiter:
         max_attempts=settings.login_rate_limit_attempts,
         window_seconds=settings.login_rate_limit_window_seconds,
     )
+

app/auth/runtime.py

@@ -7,9 +7,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.auth.session import clear_session_user, get_session_user, set_session_user
 from app.db.models import User
-from app.logging_utils import structured_log
 from app.security.csrf import CSRF_SESSION_KEY
-from app.services.users import application as user_service
+from app.services.domains.users import application as user_service
 
 logger = logging.getLogger(__name__)
 
@@ -29,7 +28,13 @@ async def get_authenticated_user(
 
     user = await user_service.get_user_by_id(db_session, session_user.id)
     if user is None or not user.is_active:
-        structured_log(logger, "info", "auth.session_invalidated", session_user_id=session_user.id)
+        logger.info(
+            "auth.session_invalidated",
+            extra={
+                "event": "auth.session_invalidated",
+                "session_user_id": session_user.id,
+            },
+        )
         invalidate_session(request)
         return None
 

app/auth/security.py

@@ -16,3 +16,4 @@ class PasswordService:
             return bool(self._hasher.verify(password_hash, password))
         except (InvalidHashError, VerificationError):
             return False
+

app/auth/service.py

@@ -21,7 +21,9 @@ class AuthService:
         normalized_email = email.strip().lower()
         if not normalized_email or not password:
             return None
-        result = await db_session.execute(select(User).where(User.email == normalized_email))
+        result = await db_session.execute(
+            select(User).where(User.email == normalized_email)
+        )
         user = result.scalar_one_or_none()
         if user is None or not user.is_active:
             return None

app/auth/session.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 
 from starlette.requests import Request
 
+
 SESSION_USER_ID_KEY = "auth_user_id"
 SESSION_USER_EMAIL_KEY = "auth_user_email"
 SESSION_USER_IS_ADMIN_KEY = "auth_user_is_admin"

app/db/background_session.py

@@ -1,57 +0,0 @@
-"""Semaphore-gated DB sessions for background tasks.
-
-Background tasks (ingestion, PDF resolution, scheduler) acquire a semaphore
-permit before checking out a database connection.  The semaphore cap is
-``pool_size + max_overflow - reserved_for_api``, which guarantees that at
-least *reserved_for_api* connections remain available for API request
-handlers at all times.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-from collections.abc import AsyncIterator
-from contextlib import asynccontextmanager
-
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.session import get_session_factory
-from app.logging_utils import structured_log
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-_semaphore: asyncio.Semaphore | None = None
-
-
-def _build_semaphore() -> asyncio.Semaphore:
-    pool_capacity = max(1, settings.database_pool_size) + max(0, settings.database_pool_max_overflow)
-    reserved = max(0, settings.database_reserved_api_connections)
-    limit = max(1, pool_capacity - reserved)
-    structured_log(
-        logger,
-        "info",
-        "db.background_semaphore_initialized",
-        pool_capacity=pool_capacity,
-        reserved_for_api=reserved,
-        background_limit=limit,
-    )
-    return asyncio.Semaphore(limit)
-
-
-def get_background_semaphore() -> asyncio.Semaphore:
-    global _semaphore
-    if _semaphore is None:
-        _semaphore = _build_semaphore()
-    return _semaphore
-
-
-@asynccontextmanager
-async def background_session() -> AsyncIterator[AsyncSession]:
-    """Yield a DB session after acquiring the background semaphore."""
-    semaphore = get_background_semaphore()
-    async with semaphore:
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            yield session

app/db/base.py

@@ -1,8 +1,9 @@
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 
 from sqlalchemy import MetaData, event
 from sqlalchemy.orm import DeclarativeBase
 
+
 NAMING_CONVENTION = {
     "ix": "ix_%(column_0_label)s",
     "uq": "uq_%(table_name)s_%(column_0_name)s",
@@ -20,7 +21,7 @@ class Base(DeclarativeBase):
 def _set_updated_at_before_update(_mapper, _connection, target) -> None:
     # Keep audit timestamps current for ORM-managed updates.
     if hasattr(target, "updated_at"):
-        target.updated_at = datetime.now(UTC)
+        target.updated_at = datetime.now(timezone.utc)
 
 
 metadata = Base.metadata

app/db/models.py

@@ -8,7 +8,6 @@ from sqlalchemy import (
     CheckConstraint,
     DateTime,
     Enum,
-    Float,
     ForeignKey,
     Index,
     Integer,
@@ -31,11 +30,9 @@ class RunTriggerType(StrEnum):
 
 class RunStatus(StrEnum):
     RUNNING = "running"
-    RESOLVING = "resolving"
     SUCCESS = "success"
     PARTIAL_FAILURE = "partial_failure"
     FAILED = "failed"
-    CANCELED = "canceled"
 
 
 class QueueItemStatus(StrEnum):
@@ -62,10 +59,18 @@ class User(Base):
     id: Mapped[int] = mapped_column(primary_key=True)
     email: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
     password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
-    is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("true"))
-    is_admin: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
-    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    is_active: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, server_default=text("true")
+    )
+    is_admin: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, server_default=text("false")
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
 
 
 class UserSetting(Base):
@@ -81,10 +86,18 @@ class UserSetting(Base):
         ),
     )
 
-    user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), primary_key=True)
-    auto_run_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
-    run_interval_minutes: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("1440"))
-    request_delay_seconds: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("10"))
+    user_id: Mapped[int] = mapped_column(
+        ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    auto_run_enabled: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, server_default=text("false")
+    )
+    run_interval_minutes: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default=text("1440")
+    )
+    request_delay_seconds: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default=text("10")
+    )
     nav_visible_pages: Mapped[list[str]] = mapped_column(
         JSONB,
         nullable=False,
@@ -99,13 +112,12 @@ class UserSetting(Base):
     )
     scrape_cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     scrape_cooldown_reason: Mapped[str | None] = mapped_column(String(64))
-
-    openalex_api_key: Mapped[str | None] = mapped_column(String(255))
-    crossref_api_token: Mapped[str | None] = mapped_column(String(255))
-    crossref_api_mailto: Mapped[str | None] = mapped_column(String(255))
-
-    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
 
 
 class ScholarProfile(Base):
@@ -116,7 +128,9 @@ class ScholarProfile(Base):
     )
 
     id: Mapped[int] = mapped_column(primary_key=True)
-    user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
+    user_id: Mapped[int] = mapped_column(
+        ForeignKey("users.id", ondelete="CASCADE"), nullable=False
+    )
     scholar_id: Mapped[str] = mapped_column(String(64), nullable=False)
     display_name: Mapped[str | None] = mapped_column(String(255))
     profile_image_url: Mapped[str | None] = mapped_column(Text)
@@ -124,47 +138,69 @@ class ScholarProfile(Base):
     profile_image_upload_path: Mapped[str | None] = mapped_column(Text)
     last_initial_page_fingerprint_sha256: Mapped[str | None] = mapped_column(String(64))
     last_initial_page_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    is_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("true"))
-    baseline_completed: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
+    is_enabled: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, server_default=text("true")
+    )
+    baseline_completed: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, server_default=text("false")
+    )
     last_run_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     last_run_status: Mapped[RunStatus | None] = mapped_column(
         RUN_STATUS_DB_ENUM,
     )
-    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
 
 
 class CrawlRun(Base):
     __tablename__ = "crawl_runs"
     __table_args__ = (
         Index("ix_crawl_runs_user_start", "user_id", "start_dt"),
-        Index(
-            "uq_crawl_runs_user_active",
-            "user_id",
-            unique=True,
-            postgresql_where=text("status IN ('running'::run_status, 'resolving'::run_status)"),
-        ),
         Index(
             "uq_crawl_runs_user_manual_idempotency_key",
             "user_id",
             "idempotency_key",
             unique=True,
-            postgresql_where=text("idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"),
+            postgresql_where=text(
+                "idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"
+            ),
         ),
     )
 
     id: Mapped[int] = mapped_column(primary_key=True)
-    user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
-    trigger_type: Mapped[RunTriggerType] = mapped_column(RUN_TRIGGER_TYPE_DB_ENUM, nullable=False)
-    status: Mapped[RunStatus] = mapped_column(RUN_STATUS_DB_ENUM, nullable=False)
-    start_dt: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    user_id: Mapped[int] = mapped_column(
+        ForeignKey("users.id", ondelete="CASCADE"), nullable=False
+    )
+    trigger_type: Mapped[RunTriggerType] = mapped_column(
+        RUN_TRIGGER_TYPE_DB_ENUM, nullable=False
+    )
+    status: Mapped[RunStatus] = mapped_column(
+        RUN_STATUS_DB_ENUM, nullable=False
+    )
+    start_dt: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
     end_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    scholar_count: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
-    new_pub_count: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
+    scholar_count: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default=text("0")
+    )
+    new_pub_count: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default=text("0")
+    )
     idempotency_key: Mapped[str | None] = mapped_column(String(128))
-    error_log: Mapped[dict] = mapped_column(JSONB, nullable=False, server_default=text("'{}'::jsonb"))
-    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    error_log: Mapped[dict] = mapped_column(
+        JSONB, nullable=False, server_default=text("'{}'::jsonb")
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
 
 
 class Publication(Base):
@@ -185,59 +221,20 @@ class Publication(Base):
     title_raw: Mapped[str] = mapped_column(Text, nullable=False)
     title_normalized: Mapped[str] = mapped_column(Text, nullable=False)
     year: Mapped[int | None] = mapped_column(Integer)
-    citation_count: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0"))
+    citation_count: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default=text("0")
+    )
     author_text: Mapped[str | None] = mapped_column(Text)
     venue_text: Mapped[str | None] = mapped_column(Text)
     pub_url: Mapped[str | None] = mapped_column(Text)
+    doi: Mapped[str | None] = mapped_column(String(255))
     pdf_url: Mapped[str | None] = mapped_column(Text)
-    canonical_title_hash: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
-    openalex_enriched: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
-    openalex_last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
-    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-
-
-class PublicationIdentifier(Base):
-    __tablename__ = "publication_identifiers"
-    __table_args__ = (
-        UniqueConstraint(
-            "publication_id",
-            "kind",
-            "value_normalized",
-            name="uq_publication_identifiers_publication_kind_value",
-        ),
-        CheckConstraint(
-            "confidence_score >= 0 AND confidence_score <= 1",
-            name="publication_identifiers_confidence_score_range",
-        ),
-        Index(
-            "ix_publication_identifiers_kind_value",
-            "kind",
-            "value_normalized",
-        ),
-        Index(
-            "ix_publication_identifiers_publication_id",
-            "publication_id",
-        ),
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
     )
-
-    id: Mapped[int] = mapped_column(primary_key=True)
-    publication_id: Mapped[int] = mapped_column(
-        ForeignKey("publications.id", ondelete="CASCADE"),
-        nullable=False,
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
     )
-    kind: Mapped[str] = mapped_column(String(32), nullable=False)
-    value_raw: Mapped[str] = mapped_column(Text, nullable=False)
-    value_normalized: Mapped[str] = mapped_column(String(255), nullable=False)
-    source: Mapped[str] = mapped_column(String(64), nullable=False)
-    confidence_score: Mapped[float] = mapped_column(
-        Float,
-        nullable=False,
-        server_default=text("0"),
-    )
-    evidence_url: Mapped[str | None] = mapped_column(Text)
-    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
 
 
 class PublicationPdfJob(Base):
@@ -275,8 +272,12 @@ class PublicationPdfJob(Base):
     last_requested_by_user_id: Mapped[int | None] = mapped_column(
         ForeignKey("users.id", ondelete="SET NULL"),
     )
-    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
 
 
 class PublicationPdfJobEvent(Base):
@@ -300,7 +301,9 @@ class PublicationPdfJobEvent(Base):
     source: Mapped[str | None] = mapped_column(String(32))
     failure_reason: Mapped[str | None] = mapped_column(String(64))
     message: Mapped[str | None] = mapped_column(Text)
-    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
 
 
 class ScholarPublication(Base):
@@ -318,10 +321,18 @@ class ScholarPublication(Base):
         ForeignKey("publications.id", ondelete="CASCADE"),
         primary_key=True,
     )
-    is_read: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
-    is_favorite: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false"))
-    first_seen_run_id: Mapped[int | None] = mapped_column(ForeignKey("crawl_runs.id", ondelete="SET NULL"))
-    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    is_read: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, server_default=text("false")
+    )
+    is_favorite: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, server_default=text("false")
+    )
+    first_seen_run_id: Mapped[int | None] = mapped_column(
+        ForeignKey("crawl_runs.id", ondelete="SET NULL")
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
 
 
 class IngestionQueueItem(Base):
@@ -376,8 +387,12 @@ class IngestionQueueItem(Base):
     last_error: Mapped[str | None] = mapped_column(Text)
     dropped_reason: Mapped[str | None] = mapped_column(String(128))
     dropped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
 
 
 class DataRepairJob(Base):
@@ -417,8 +432,12 @@ class DataRepairJob(Base):
     error_text: Mapped[str | None] = mapped_column(Text)
     started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
 
 
 class AuthorSearchRuntimeState(Base):
@@ -442,38 +461,12 @@ class AuthorSearchRuntimeState(Base):
         nullable=False,
         server_default=text("false"),
     )
-    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-
-
-class ArxivRuntimeState(Base):
-    __tablename__ = "arxiv_runtime_state"
-
-    state_key: Mapped[str] = mapped_column(String(64), primary_key=True)
-    next_allowed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-
-
-class ArxivQueryCacheEntry(Base):
-    __tablename__ = "arxiv_query_cache_entries"
-    __table_args__ = (
-        Index("ix_arxiv_query_cache_expires_at", "expires_at"),
-        Index("ix_arxiv_query_cache_cached_at", "cached_at"),
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
     )
-
-    query_fingerprint: Mapped[str] = mapped_column(String(64), primary_key=True)
-    payload: Mapped[dict] = mapped_column(
-        JSONB,
-        nullable=False,
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
     )
-    expires_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True),
-        nullable=False,
-    )
-    cached_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
 
 
 class AuthorSearchCacheEntry(Base):
@@ -492,5 +485,9 @@ class AuthorSearchCacheEntry(Base):
         DateTime(timezone=True),
         nullable=False,
     )
-    cached_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
-    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    cached_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

app/db/session.py

@@ -1,6 +1,6 @@
+from collections.abc import AsyncIterator
 import logging
 import os
-from collections.abc import AsyncIterator
 
 from sqlalchemy import text
 from sqlalchemy.ext.asyncio import (
@@ -11,7 +11,6 @@ from sqlalchemy.ext.asyncio import (
 )
 from sqlalchemy.pool import NullPool
 
-from app.logging_utils import structured_log
 from app.settings import settings
 
 logger = logging.getLogger(__name__)
@@ -31,12 +30,13 @@ def _normalized_pool_mode(raw_mode: str) -> str:
         return "queue"
     if mode in {"null", "queue"}:
         return mode
-    structured_log(
-        logger,
-        "warning",
+    logger.warning(
         "db.invalid_pool_mode_fallback",
-        database_pool_mode=raw_mode,
-        fallback_mode="queue",
+        extra={
+            "event": "db.invalid_pool_mode_fallback",
+            "database_pool_mode": raw_mode,
+            "fallback_mode": "queue",
+        },
     )
     return "queue"
 
@@ -56,11 +56,12 @@ def get_engine() -> AsyncEngine:
             engine_kwargs["pool_timeout"] = max(1, int(settings.database_pool_timeout_seconds))
 
         _engine = create_async_engine(settings.database_url, **engine_kwargs)
-        structured_log(
-            logger,
-            "info",
+        logger.info(
             "db.engine_initialized",
-            pool_mode=pool_mode,
+            extra={
+                "event": "db.engine_initialized",
+                "pool_mode": pool_mode,
+            },
         )
     return _engine
 
@@ -85,7 +86,7 @@ async def check_database() -> bool:
             result = await conn.execute(text("SELECT 1"))
             return result.scalar_one() == 1
     except Exception:
-        structured_log(logger, "exception", "db.healthcheck_failed")
+        logger.exception("db.healthcheck_failed", extra={"event": "db.healthcheck_failed"})
         return False
 
 
@@ -93,6 +94,6 @@ async def close_engine() -> None:
     global _engine, _session_factory
     if _engine is not None:
         await _engine.dispose()
-        structured_log(logger, "info", "db.engine_disposed")
+        logger.info("db.engine_disposed", extra={"event": "db.engine_disposed"})
         _engine = None
         _session_factory = None

app/http/middleware.py

@@ -1,15 +1,14 @@
 from __future__ import annotations
 
+from secrets import token_urlsafe
 import logging
 import time
-from secrets import token_urlsafe
 
 from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.requests import Request
 from starlette.responses import Response
 
 from app.logging_context import set_request_id
-from app.logging_utils import structured_log
 
 REQUEST_ID_HEADER = "X-Request-ID"
 DEFAULT_PERMISSIONS_POLICY = (
@@ -62,39 +61,42 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
         start = time.perf_counter()
         should_log = self._log_requests and not self._is_skipped_path(request.url.path)
         if should_log:
-            structured_log(
-                logger,
-                "debug",
+            logger.debug(
                 "request.started",
-                method=request.method,
-                path=request.url.path,
+                extra={
+                    "event": "request.started",
+                    "method": request.method,
+                    "path": request.url.path,
+                },
             )
 
         try:
             response = await call_next(request)
         except Exception:
             duration_ms = int((time.perf_counter() - start) * 1000)
-            structured_log(
-                logger,
-                "exception",
+            logger.exception(
                 "request.failed",
-                method=request.method,
-                path=request.url.path,
-                duration_ms=duration_ms,
+                extra={
+                    "event": "request.failed",
+                    "method": request.method,
+                    "path": request.url.path,
+                    "duration_ms": duration_ms,
+                },
             )
             raise
         else:
             duration_ms = int((time.perf_counter() - start) * 1000)
             response.headers[REQUEST_ID_HEADER] = request_id
             if should_log:
-                structured_log(
-                    logger,
-                    "debug",
+                logger.debug(
                     "request.completed",
-                    method=request.method,
-                    path=request.url.path,
-                    status_code=response.status_code,
-                    duration_ms=duration_ms,
+                    extra={
+                        "event": "request.completed",
+                        "method": request.method,
+                        "path": request.url.path,
+                        "status_code": response.status_code,
+                        "duration_ms": duration_ms,
+                    },
                 )
             return response
         finally:
@@ -168,7 +170,11 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
 
         csp_policy = self._csp_policy_for_path(request.url.path)
         if self._csp_enabled and csp_policy:
-            csp_header = "Content-Security-Policy-Report-Only" if self._csp_report_only else "Content-Security-Policy"
+            csp_header = (
+                "Content-Security-Policy-Report-Only"
+                if self._csp_report_only
+                else "Content-Security-Policy"
+            )
             response.headers.setdefault(csp_header, csp_policy)
 
         hsts = self._strict_transport_security_value()

app/logging_config.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 
+from datetime import datetime, timezone
 import json
 import logging
 import sys
-from datetime import UTC, datetime
 from typing import Any
 
 from app.logging_context import get_request_id
@@ -102,22 +102,12 @@ class JsonLogFormatter(logging.Formatter):
         if key.lower() in self._redact_fields:
             return "[REDACTED]"
         if isinstance(value, dict):
-            return {
-                nested_key: self._redact_value(nested_key, nested_value) for nested_key, nested_value in value.items()
-            }
+            return {nested_key: self._redact_value(nested_key, nested_value) for nested_key, nested_value in value.items()}
         if isinstance(value, (list, tuple)):
             return [self._redact_value(key, item) for item in value]
         return value
 
 
-_CONSOLE_SHORT_KEYS = {
-    "user_id": "user",
-    "scholar_id": "scholar",
-    "crawl_run_id": "run",
-    "run_id": "run",
-}
-
-
 class ConsoleLogFormatter(logging.Formatter):
     def __init__(self, *, redact_fields: set[str]) -> None:
         super().__init__()
@@ -150,8 +140,7 @@ class ConsoleLogFormatter(logging.Formatter):
         for key in sorted(payload.keys()):
             if key in {"timestamp", "level", "logger", "event", "exception"}:
                 continue
-            display_key = _CONSOLE_SHORT_KEYS.get(key, key)
-            parts.append(f"{display_key}={payload[key]}")
+            parts.append(f"{key}={payload[key]}")
 
         if "exception" in payload:
             parts.append(f"exception={payload['exception']}")
@@ -179,7 +168,7 @@ def _normalize_level(level: str) -> int:
 
 
 def _format_timestamp(created_ts: float) -> str:
-    dt = datetime.fromtimestamp(created_ts, tz=UTC)
+    dt = datetime.fromtimestamp(created_ts, tz=timezone.utc)
     return dt.strftime("%Y-%m-%d %H:%M:%SZ")
 
 

app/logging_context.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from contextvars import ContextVar
 
+
 _request_id_ctx: ContextVar[str | None] = ContextVar("request_id", default=None)
 
 
@@ -11,3 +12,4 @@ def get_request_id() -> str | None:
 
 def set_request_id(value: str | None) -> None:
     _request_id_ctx.set(value)
+

app/logging_utils.py

@@ -1,29 +0,0 @@
-"""Structured logging utility — eliminates boilerplate across domain services."""
-
-from __future__ import annotations
-
-import logging
-from typing import Any
-
-
-def structured_log(
-    logger: logging.Logger,
-    level: str,
-    event: str,
-    /,
-    **fields: Any,
-) -> None:
-    """Emit a structured log entry.
-
-    The event name is passed as the log message. The JsonLogFormatter in
-    logging_config.py extracts it via record.getMessage() when no explicit
-    'event' key exists in extra — so we do NOT duplicate it.
-
-    Usage:
-        structured_log(logger, "info", "ingestion.run_started", user_id=1, scholar_count=5)
-    """
-    fields.pop("metric_name", None)
-    fields.pop("metric_value", None)
-
-    log_method = getattr(logger, level.lower())
-    log_method(event, extra=fields)

app/main.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-import logging
 from contextlib import asynccontextmanager
+import logging
 from pathlib import Path
 
 from fastapi import FastAPI, HTTPException
@@ -12,16 +12,17 @@ from starlette.middleware.sessions import SessionMiddleware
 from app.api.errors import register_api_exception_handlers
 from app.api.media import router as media_router
 from app.api.router import router as api_router
-from app.db.session import check_database, close_engine
+from app.api.runtime_deps import get_ingestion_service, get_scholar_source
+from app.db.session import check_database
+from app.db.session import close_engine
 from app.http.middleware import (
     RequestLoggingMiddleware,
     SecurityHeadersMiddleware,
     parse_skip_paths,
 )
 from app.logging_config import configure_logging, parse_redact_fields
-from app.logging_utils import structured_log
 from app.security.csrf import CSRFMiddleware
-from app.services.ingestion.scheduler import SchedulerService
+from app.services.domains.ingestion.scheduler import SchedulerService
 from app.settings import settings
 
 logger = logging.getLogger(__name__)
@@ -49,58 +50,22 @@ scheduler_service = SchedulerService(
 )
 
 
-@asynccontextmanager
-async def lifespan(_: FastAPI):
-    structured_log(
-        logger,
-        "info",
+def _log_startup_build_marker() -> None:
+    logger.info(
         "app.startup_build_marker",
-        build_marker=BUILD_MARKER,
-        frontend_enabled=settings.frontend_enabled,
-        scheduler_enabled=settings.scheduler_enabled,
-        log_format=settings.log_format,
+        extra={
+            "event": "app.startup_build_marker",
+            "build_marker": BUILD_MARKER,
+            "frontend_enabled": settings.frontend_enabled,
+            "scheduler_enabled": settings.scheduler_enabled,
+            "log_format": settings.log_format,
+        },
     )
 
-    from sqlalchemy import text
-
-    from app.db.session import get_session_factory
-
-    try:
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            await session.execute(
-                text("UPDATE crawl_runs SET status = 'failed' WHERE status::text IN ('running', 'resolving')")
-            )
-            await session.commit()
-            structured_log(logger, "info", "app.startup_orphaned_runs_cleaned")
-    except Exception as exc:
-        structured_log(
-            logger,
-            "error",
-            "app.startup_orphaned_runs_cleanup_failed",
-            error=str(exc),
-        )
-
-    try:
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            await session.execute(
-                text(
-                    "UPDATE publication_pdf_jobs SET status = 'queued'"
-                    " WHERE status = 'running'"
-                    " AND (last_attempt_at IS NULL OR last_attempt_at < NOW() - INTERVAL '10 minutes')"
-                )
-            )
-            await session.commit()
-            structured_log(logger, "info", "app.startup_stuck_pdf_jobs_recovered")
-    except Exception as exc:
-        structured_log(
-            logger,
-            "error",
-            "app.startup_stuck_pdf_jobs_recovery_failed",
-            error=str(exc),
-        )
 
+@asynccontextmanager
+async def lifespan(_: FastAPI):
+    _log_startup_build_marker()
     await scheduler_service.start()
     yield
     await scheduler_service.stop()
@@ -136,7 +101,9 @@ app.add_middleware(
     content_security_policy_report_only=settings.security_csp_report_only,
     strict_transport_security_enabled=settings.security_strict_transport_security_enabled,
     strict_transport_security_max_age=settings.security_strict_transport_security_max_age,
-    strict_transport_security_include_subdomains=(settings.security_strict_transport_security_include_subdomains),
+    strict_transport_security_include_subdomains=(
+        settings.security_strict_transport_security_include_subdomains
+    ),
     strict_transport_security_preload=settings.security_strict_transport_security_preload,
 )
 app.include_router(api_router)

app/security/__init__.py

@@ -1 +1,2 @@
 """Security middleware and helpers for scholarr."""
+

app/security/csrf.py

@@ -6,11 +6,9 @@ from urllib.parse import parse_qs
 
 from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.requests import Request
-from starlette.responses import PlainTextResponse, Response
+from starlette.responses import JSONResponse, PlainTextResponse, Response
 from starlette.types import Message
 
-from app.api.responses import error_response
-from app.logging_utils import structured_log
 
 CSRF_SESSION_KEY = "csrf_token"
 CSRF_FORM_FIELD = "csrf_token"
@@ -38,12 +36,13 @@ class CSRFMiddleware(BaseHTTPMiddleware):
 
         session_token = request.session.get(CSRF_SESSION_KEY)
         if not session_token:
-            structured_log(
-                logger,
-                "warning",
+            logger.warning(
                 "csrf.missing_session_token",
-                method=request.method,
-                path=request.url.path,
+                extra={
+                    "event": "csrf.missing_session_token",
+                    "method": request.method,
+                    "path": request.url.path,
+                },
             )
             return self._csrf_error_response(
                 request,
@@ -55,15 +54,16 @@ class CSRFMiddleware(BaseHTTPMiddleware):
         if request_token is None and self._is_form_payload(request):
             body = await request.body()
             request_token = self._token_from_form_body(request, body)
-            request._receive = self._build_receive(body)
+            request._receive = self._build_receive(body)  # type: ignore[attr-defined]
 
         if not request_token or not compare_digest(str(session_token), str(request_token)):
-            structured_log(
-                logger,
-                "warning",
+            logger.warning(
                 "csrf.invalid_token",
-                method=request.method,
-                path=request.url.path,
+                extra={
+                    "event": "csrf.invalid_token",
+                    "method": request.method,
+                    "path": request.url.path,
+                },
             )
             return self._csrf_error_response(
                 request,
@@ -89,10 +89,7 @@ class CSRFMiddleware(BaseHTTPMiddleware):
         content_type = request.headers.get("content-type", "")
         if not content_type.startswith("application/x-www-form-urlencoded"):
             return None
-        try:
-            parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True)
-        except (UnicodeDecodeError, ValueError):
-            return None
+        parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True)
         values = parsed.get(CSRF_FORM_FIELD)
         if not values:
             return None
@@ -118,11 +115,19 @@ class CSRFMiddleware(BaseHTTPMiddleware):
         message: str,
     ) -> Response:
         if request.url.path.startswith("/api/"):
-            return error_response(
-                request,
+            request_state = getattr(request, "state", None)
+            request_id = getattr(request_state, "request_id", None) if request_state else None
+            return JSONResponse(
+                {
+                    "error": {
+                        "code": code,
+                        "message": message,
+                        "details": None,
+                    },
+                    "meta": {
+                        "request_id": request_id,
+                    },
+                },
                 status_code=403,
-                code=code,
-                message=message,
-                details=None,
             )
         return PlainTextResponse(message, status_code=403)

app/services/__init__.py

@@ -1 +1,2 @@
 """Service layer for scholarr application workflows."""
+

app/services/arxiv/application.py

@@ -1,29 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from app.services.arxiv.gateway import (
-    build_arxiv_query,
-    get_arxiv_gateway,
-)
-
-if TYPE_CHECKING:
-    from app.services.publications.types import PublicationListItem, UnreadPublicationItem
-
-
-def _build_arxiv_query(title: str, author_surname: str | None) -> str | None:
-    return build_arxiv_query(title, author_surname)
-
-
-async def discover_arxiv_id_for_publication(
-    *,
-    item: PublicationListItem | UnreadPublicationItem,
-    request_email: str | None = None,
-    timeout_seconds: float | None = None,
-) -> str | None:
-    gateway = get_arxiv_gateway()
-    return await gateway.discover_arxiv_id_for_publication(
-        item=item,
-        request_email=request_email,
-        timeout_seconds=timeout_seconds,
-    )

app/services/arxiv/cache.py

@@ -1,318 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import hashlib
-import json
-from collections.abc import Awaitable, Callable, Mapping
-from dataclasses import asdict
-from datetime import UTC, datetime, timedelta
-from typing import Any
-
-from sqlalchemy import delete, func, select
-
-from app.db.models import ArxivQueryCacheEntry
-from app.db.session import get_session_factory
-from app.services.arxiv.constants import ARXIV_CACHE_FINGERPRINT_VERSION
-from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
-
-_INFLIGHT_LOCK = asyncio.Lock()
-_INFLIGHT_FEEDS: dict[str, asyncio.Future[ArxivFeed]] = {}
-
-
-def build_query_fingerprint(*, params: Mapping[str, object]) -> str:
-    canonical = _canonical_cache_payload(params=params)
-    encoded = json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
-    payload = f"{ARXIV_CACHE_FINGERPRINT_VERSION}:{encoded}"
-    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
-
-
-async def get_cached_feed(
-    *,
-    query_fingerprint: str,
-    now_utc: datetime | None = None,
-) -> ArxivFeed | None:
-    timestamp = _as_utc(now_utc)
-    session_factory = get_session_factory()
-    async with session_factory() as db_session, db_session.begin():
-        result = await db_session.execute(
-            select(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint == query_fingerprint)
-        )
-        entry = result.scalar_one_or_none()
-        return await _validate_cached_entry(db_session, entry=entry, now_utc=timestamp)
-
-
-async def set_cached_feed(
-    *,
-    query_fingerprint: str,
-    feed: ArxivFeed,
-    ttl_seconds: float,
-    max_entries: int,
-    now_utc: datetime | None = None,
-) -> None:
-    timestamp = _as_utc(now_utc)
-    session_factory = get_session_factory()
-    async with session_factory() as db_session, db_session.begin():
-        await _write_cached_entry(
-            db_session,
-            query_fingerprint=query_fingerprint,
-            feed=feed,
-            ttl_seconds=ttl_seconds,
-            max_entries=max_entries,
-            now_utc=timestamp,
-        )
-
-
-async def run_with_inflight_dedupe(
-    *,
-    query_fingerprint: str,
-    fetch_feed: Callable[[], Awaitable[ArxivFeed]],
-) -> ArxivFeed:
-    future, is_owner = await _reserve_inflight_future(query_fingerprint=query_fingerprint)
-    if not is_owner:
-        return await asyncio.shield(future)
-    try:
-        result = await fetch_feed()
-    except Exception as exc:
-        _complete_future(future, error=exc)
-        raise
-    finally:
-        await _release_inflight_future(query_fingerprint=query_fingerprint, future=future)
-    _complete_future(future, result=result)
-    return result
-
-
-def _canonical_cache_payload(*, params: Mapping[str, object]) -> dict[str, object]:
-    payload: dict[str, object] = {}
-    for key in sorted(params.keys()):
-        payload[str(key)] = _normalize_param_value(str(key), params[key])
-    return payload
-
-
-def _normalize_param_value(key: str, value: object) -> object:
-    if key == "search_query":
-        return _normalize_search_query(str(value or ""))
-    if key == "id_list":
-        return _normalize_id_list(str(value or ""))
-    if isinstance(value, str):
-        return " ".join(value.strip().split())
-    if isinstance(value, (int, float, bool)) or value is None:
-        return value
-    return str(value).strip()
-
-
-def _normalize_search_query(value: str) -> str:
-    return " ".join(value.strip().lower().split())
-
-
-def _normalize_id_list(value: str) -> str:
-    normalized = [item.strip().lower() for item in value.split(",") if item.strip()]
-    return ",".join(sorted(normalized))
-
-
-async def _validate_cached_entry(
-    db_session,
-    *,
-    entry: ArxivQueryCacheEntry | None,
-    now_utc: datetime,
-) -> ArxivFeed | None:
-    if entry is None:
-        return None
-    if _as_utc(entry.expires_at) <= now_utc:
-        await db_session.delete(entry)
-        return None
-    parsed = _deserialize_feed(entry.payload)
-    if parsed is None:
-        await db_session.delete(entry)
-        return None
-    return parsed
-
-
-async def _write_cached_entry(
-    db_session,
-    *,
-    query_fingerprint: str,
-    feed: ArxivFeed,
-    ttl_seconds: float,
-    max_entries: int,
-    now_utc: datetime,
-) -> None:
-    ttl = max(float(ttl_seconds), 0.0)
-    result = await db_session.execute(
-        select(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint == query_fingerprint)
-    )
-    existing = result.scalar_one_or_none()
-    if ttl <= 0.0:
-        if existing is not None:
-            await db_session.delete(existing)
-        return
-    expires_at = now_utc + timedelta(seconds=ttl)
-    payload = _serialize_feed(feed)
-    if existing is None:
-        db_session.add(
-            ArxivQueryCacheEntry(
-                query_fingerprint=query_fingerprint,
-                payload=payload,
-                expires_at=expires_at,
-                cached_at=now_utc,
-                updated_at=now_utc,
-            )
-        )
-    else:
-        existing.payload = payload
-        existing.expires_at = expires_at
-        existing.cached_at = now_utc
-        existing.updated_at = now_utc
-    await _prune_cache_entries(db_session, now_utc=now_utc, max_entries=max_entries)
-
-
-async def _prune_cache_entries(
-    db_session,
-    *,
-    now_utc: datetime,
-    max_entries: int,
-) -> None:
-    await db_session.execute(delete(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.expires_at <= now_utc))
-    bounded_max_entries = int(max_entries)
-    if bounded_max_entries <= 0:
-        return
-    count_result = await db_session.execute(select(func.count()).select_from(ArxivQueryCacheEntry))
-    entry_count = int(count_result.scalar_one() or 0)
-    overflow = max(0, entry_count - bounded_max_entries)
-    if overflow <= 0:
-        return
-    stale_result = await db_session.execute(
-        select(ArxivQueryCacheEntry.query_fingerprint).order_by(ArxivQueryCacheEntry.cached_at.asc()).limit(overflow)
-    )
-    stale_keys = [str(row[0]) for row in stale_result.all()]
-    if stale_keys:
-        await db_session.execute(
-            delete(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint.in_(stale_keys))
-        )
-
-
-def _serialize_feed(feed: ArxivFeed) -> dict[str, Any]:
-    return asdict(feed)
-
-
-def _deserialize_feed(payload: object) -> ArxivFeed | None:
-    if not isinstance(payload, dict):
-        return None
-    entries_payload = payload.get("entries")
-    opensearch_payload = payload.get("opensearch")
-    if not isinstance(entries_payload, list):
-        return None
-    entries: list[ArxivEntry] = []
-    for value in entries_payload:
-        entry = _deserialize_entry(value)
-        if entry is None:
-            return None
-        entries.append(entry)
-    opensearch = _deserialize_opensearch(opensearch_payload)
-    if opensearch is None:
-        return None
-    return ArxivFeed(entries=entries, opensearch=opensearch)
-
-
-def _deserialize_entry(value: object) -> ArxivEntry | None:
-    if not isinstance(value, dict):
-        return None
-    try:
-        return ArxivEntry(
-            entry_id_url=str(value["entry_id_url"]),
-            arxiv_id=_as_optional_string(value.get("arxiv_id")),
-            title=str(value["title"]),
-            summary=str(value["summary"]),
-            published=_as_optional_string(value.get("published")),
-            updated=_as_optional_string(value.get("updated")),
-            authors=_as_string_list(value.get("authors")),
-            links=_as_string_list(value.get("links")),
-            categories=_as_string_list(value.get("categories")),
-            primary_category=_as_optional_string(value.get("primary_category")),
-        )
-    except KeyError:
-        return None
-
-
-def _deserialize_opensearch(value: object) -> ArxivOpenSearchMeta | None:
-    if not isinstance(value, dict):
-        return None
-    try:
-        return ArxivOpenSearchMeta(
-            total_results=int(value.get("total_results", 0)),
-            start_index=int(value.get("start_index", 0)),
-            items_per_page=int(value.get("items_per_page", 0)),
-        )
-    except (TypeError, ValueError):
-        return None
-
-
-def _as_optional_string(value: object) -> str | None:
-    if value is None:
-        return None
-    normalized = str(value).strip()
-    return normalized or None
-
-
-def _as_string_list(value: object) -> list[str]:
-    if not isinstance(value, list):
-        return []
-    return [str(item) for item in value]
-
-
-def _as_utc(value: datetime | None) -> datetime:
-    if value is None:
-        return datetime.now(UTC)
-    if value.tzinfo is None:
-        return value.replace(tzinfo=UTC)
-    return value
-
-
-async def _reserve_inflight_future(
-    *,
-    query_fingerprint: str,
-) -> tuple[asyncio.Future[ArxivFeed], bool]:
-    async with _INFLIGHT_LOCK:
-        existing = _INFLIGHT_FEEDS.get(query_fingerprint)
-        if existing is not None:
-            return existing, False
-        loop = asyncio.get_running_loop()
-        created = loop.create_future()
-        created.add_done_callback(_consume_unretrieved_future_exception)
-        _INFLIGHT_FEEDS[query_fingerprint] = created
-        return created, True
-
-
-async def _release_inflight_future(
-    *,
-    query_fingerprint: str,
-    future: asyncio.Future[ArxivFeed],
-) -> None:
-    async with _INFLIGHT_LOCK:
-        current = _INFLIGHT_FEEDS.get(query_fingerprint)
-        if current is future:
-            _INFLIGHT_FEEDS.pop(query_fingerprint, None)
-
-
-def _complete_future(
-    future: asyncio.Future[ArxivFeed],
-    *,
-    result: ArxivFeed | None = None,
-    error: Exception | None = None,
-) -> None:
-    if future.done():
-        return
-    if error is not None:
-        future.set_exception(error)
-        return
-    if result is None:
-        raise RuntimeError("in-flight future completion requires result or error")
-    future.set_result(result)
-
-
-def _consume_unretrieved_future_exception(future: asyncio.Future[ArxivFeed]) -> None:
-    if future.cancelled():
-        return
-    try:
-        _ = future.exception()
-    except Exception:
-        return

app/services/arxiv/client.py

@@ -1,278 +0,0 @@
-from __future__ import annotations
-
-import logging
-from collections.abc import Awaitable, Callable
-
-import httpx
-
-from app.logging_utils import structured_log
-from app.services.arxiv.cache import (
-    build_query_fingerprint,
-    get_cached_feed,
-    run_with_inflight_dedupe,
-    set_cached_feed,
-)
-from app.services.arxiv.constants import (
-    ARXIV_SOURCE_PATH_LOOKUP_IDS,
-    ARXIV_SOURCE_PATH_SEARCH,
-    ARXIV_SOURCE_PATH_UNKNOWN,
-)
-from app.services.arxiv.errors import ArxivClientValidationError, ArxivRateLimitError
-from app.services.arxiv.parser import parse_arxiv_feed
-from app.services.arxiv.rate_limit import get_arxiv_cooldown_status, run_with_global_arxiv_limit
-from app.services.arxiv.types import ArxivFeed
-from app.settings import settings
-
-_ARXIV_API_URL = "https://export.arxiv.org/api/query"
-_ARXIV_QUERY_START = 0
-_ARXIV_MAX_RESULTS_LIMIT = 30_000
-_ARXIV_SORT_BY_ALLOWED = {"relevance", "lastUpdatedDate", "submittedDate"}
-_ARXIV_SORT_ORDER_ALLOWED = {"ascending", "descending"}
-_FALLBACK_CONTACT_EMAIL = "unknown@example.com"
-
-ArxivRequestFn = Callable[..., Awaitable[httpx.Response]]
-logger = logging.getLogger(__name__)
-
-
-class ArxivClient:
-    def __init__(
-        self,
-        *,
-        request_fn: ArxivRequestFn | None = None,
-        cache_enabled: bool | None = None,
-    ) -> None:
-        self._request_fn = request_fn or _request_arxiv_feed
-        self._cache_enabled = _resolve_cache_enabled(
-            cache_enabled=cache_enabled,
-            request_fn=request_fn,
-        )
-        self._cache_ttl_seconds = _cache_ttl_seconds()
-        self._cache_max_entries = _cache_max_entries()
-
-    async def search(
-        self,
-        *,
-        query: str,
-        start: int = _ARXIV_QUERY_START,
-        max_results: int | None = None,
-        sort_by: str | None = None,
-        sort_order: str | None = None,
-        request_email: str | None = None,
-        timeout_seconds: float | None = None,
-    ) -> ArxivFeed:
-        params = _search_params(
-            query=query,
-            start=start,
-            max_results=max_results,
-            sort_by=sort_by,
-            sort_order=sort_order,
-        )
-        return await self._fetch_feed(
-            params=params,
-            request_email=request_email,
-            timeout_seconds=timeout_seconds,
-            source_path=ARXIV_SOURCE_PATH_SEARCH,
-        )
-
-    async def lookup_ids(
-        self,
-        *,
-        id_list: list[str],
-        start: int = _ARXIV_QUERY_START,
-        max_results: int | None = None,
-        request_email: str | None = None,
-        timeout_seconds: float | None = None,
-    ) -> ArxivFeed:
-        params = _lookup_params(id_list=id_list, start=start, max_results=max_results)
-        return await self._fetch_feed(
-            params=params,
-            request_email=request_email,
-            timeout_seconds=timeout_seconds,
-            source_path=ARXIV_SOURCE_PATH_LOOKUP_IDS,
-        )
-
-    async def _fetch_feed(
-        self,
-        *,
-        params: dict[str, object],
-        request_email: str | None,
-        timeout_seconds: float | None,
-        source_path: str,
-    ) -> ArxivFeed:
-        query_fingerprint = build_query_fingerprint(params=params)
-        if self._cache_enabled:
-            cached = await get_cached_feed(query_fingerprint=query_fingerprint)
-            if cached is not None:
-                structured_log(
-                    logger, "info", "arxiv.cache_hit", query_fingerprint=query_fingerprint, source_path=source_path
-                )
-                return cached
-            structured_log(
-                logger, "info", "arxiv.cache_miss", query_fingerprint=query_fingerprint, source_path=source_path
-            )
-        return await run_with_inflight_dedupe(
-            query_fingerprint=query_fingerprint,
-            fetch_feed=lambda: self._fetch_live_feed(
-                params=params,
-                request_email=request_email,
-                timeout_seconds=timeout_seconds,
-                query_fingerprint=query_fingerprint,
-            ),
-        )
-
-    async def _fetch_live_feed(
-        self,
-        *,
-        params: dict[str, object],
-        request_email: str | None,
-        timeout_seconds: float | None,
-        query_fingerprint: str,
-    ) -> ArxivFeed:
-        response = await self._request_fn(
-            params=params,
-            request_email=request_email,
-            timeout_seconds=timeout_seconds,
-        )
-        response.raise_for_status()
-        feed = parse_arxiv_feed(response.text)
-        if self._cache_enabled:
-            await set_cached_feed(
-                query_fingerprint=query_fingerprint,
-                feed=feed,
-                ttl_seconds=self._cache_ttl_seconds,
-                max_entries=self._cache_max_entries,
-            )
-        return feed
-
-
-def _search_params(
-    *,
-    query: str,
-    start: int,
-    max_results: int | None,
-    sort_by: str | None,
-    sort_order: str | None,
-) -> dict[str, object]:
-    clean_query = query.strip()
-    if not clean_query:
-        raise ArxivClientValidationError("search query must not be empty")
-    params: dict[str, object] = {
-        "search_query": clean_query,
-        "start": _validate_start(start),
-        "max_results": _validate_max_results(max_results),
-    }
-    if sort_by is not None:
-        params["sortBy"] = _validate_sort_by(sort_by)
-    if sort_order is not None:
-        params["sortOrder"] = _validate_sort_order(sort_order)
-    return params
-
-
-def _lookup_params(*, id_list: list[str], start: int, max_results: int | None) -> dict[str, object]:
-    normalized_ids = [value.strip() for value in id_list if value and value.strip()]
-    if not normalized_ids:
-        raise ArxivClientValidationError("id_list must include at least one id")
-    return {
-        "id_list": ",".join(normalized_ids),
-        "start": _validate_start(start),
-        "max_results": _validate_max_results(max_results),
-    }
-
-
-def _validate_start(value: int) -> int:
-    start = int(value)
-    if start < 0:
-        raise ArxivClientValidationError("start must be >= 0")
-    return start
-
-
-def _validate_max_results(value: int | None) -> int:
-    if value is None:
-        default_value = int(settings.arxiv_default_max_results)
-        return max(default_value, 1)
-    parsed = int(value)
-    if parsed < 1:
-        raise ArxivClientValidationError("max_results must be >= 1")
-    if parsed > _ARXIV_MAX_RESULTS_LIMIT:
-        raise ArxivClientValidationError(f"max_results must be <= {_ARXIV_MAX_RESULTS_LIMIT}")
-    return parsed
-
-
-def _validate_sort_by(value: str) -> str:
-    if value not in _ARXIV_SORT_BY_ALLOWED:
-        raise ArxivClientValidationError(f"sort_by must be one of: {sorted(_ARXIV_SORT_BY_ALLOWED)!r}")
-    return value
-
-
-def _validate_sort_order(value: str) -> str:
-    if value not in _ARXIV_SORT_ORDER_ALLOWED:
-        raise ArxivClientValidationError(f"sort_order must be one of: {sorted(_ARXIV_SORT_ORDER_ALLOWED)!r}")
-    return value
-
-
-async def _request_arxiv_feed(
-    *,
-    params: dict[str, object],
-    request_email: str | None,
-    timeout_seconds: float | None,
-) -> httpx.Response:
-    source_path = _source_path_from_params(params)
-    cooldown_status = await get_arxiv_cooldown_status()
-    if cooldown_status.is_active:
-        structured_log(
-            logger,
-            "warning",
-            "arxiv.request_skipped_cooldown",
-            source_path=source_path,
-            cooldown_remaining_seconds=float(cooldown_status.remaining_seconds),
-        )
-        raise ArxivRateLimitError(f"arXiv global cooldown active ({cooldown_status.remaining_seconds:.0f}s remaining)")
-
-    async def _fetch() -> httpx.Response:
-        timeout_value = _timeout_seconds(timeout_seconds)
-        headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{_contact_email(request_email)})"}
-        async with httpx.AsyncClient(timeout=timeout_value, follow_redirects=True, headers=headers) as client:
-            return await client.get(_ARXIV_API_URL, params=params)  # type: ignore[arg-type]
-
-    return await run_with_global_arxiv_limit(
-        fetch=_fetch,
-        source_path=source_path,
-    )
-
-
-def _timeout_seconds(timeout_seconds: float | None) -> float:
-    if timeout_seconds is not None:
-        return max(float(timeout_seconds), 0.5)
-    return max(float(settings.arxiv_timeout_seconds), 0.5)
-
-
-def _contact_email(request_email: str | None) -> str:
-    return request_email or settings.arxiv_mailto or settings.crossref_api_mailto or _FALLBACK_CONTACT_EMAIL
-
-
-def _resolve_cache_enabled(
-    *,
-    cache_enabled: bool | None,
-    request_fn: ArxivRequestFn | None,
-) -> bool:
-    if cache_enabled is not None:
-        return bool(cache_enabled)
-    if request_fn is not None:
-        return False
-    return _cache_ttl_seconds() > 0.0
-
-
-def _cache_ttl_seconds() -> float:
-    return max(float(settings.arxiv_cache_ttl_seconds), 0.0)
-
-
-def _cache_max_entries() -> int:
-    return max(int(settings.arxiv_cache_max_entries), 0)
-
-
-def _source_path_from_params(params: dict[str, object]) -> str:
-    if "search_query" in params:
-        return ARXIV_SOURCE_PATH_SEARCH
-    if "id_list" in params:
-        return ARXIV_SOURCE_PATH_LOOKUP_IDS
-    return ARXIV_SOURCE_PATH_UNKNOWN

app/services/arxiv/constants.py

@@ -1,13 +0,0 @@
-from __future__ import annotations
-
-ARXIV_RUNTIME_STATE_KEY = "global"
-ARXIV_RATE_LIMIT_LOCK_NAMESPACE = 91_100
-ARXIV_RATE_LIMIT_LOCK_KEY = 1
-ARXIV_SOURCE_PATH_SEARCH = "search"
-ARXIV_SOURCE_PATH_LOOKUP_IDS = "lookup_ids"
-ARXIV_SOURCE_PATH_UNKNOWN = "unknown"
-ARXIV_CACHE_FINGERPRINT_VERSION = "v1"
-ARXIV_TITLE_TOKEN_MIN_LENGTH = 3
-ARXIV_TITLE_MIN_TOKENS = 3
-ARXIV_TITLE_MIN_ALPHA_TOKENS = 2
-ARXIV_STRONG_IDENTIFIER_CONFIDENCE = 0.9

app/services/arxiv/errors.py

@@ -1,13 +0,0 @@
-from __future__ import annotations
-
-
-class ArxivRateLimitError(Exception):
-    """arXiv returned 429 or cooldown is active."""
-
-
-class ArxivClientValidationError(ValueError):
-    """arXiv client inputs are invalid."""
-
-
-class ArxivParseError(ValueError):
-    """arXiv API payload could not be parsed."""

app/services/arxiv/gateway.py

@@ -1,142 +0,0 @@
-from __future__ import annotations
-
-import logging
-import re
-import unicodedata
-from typing import TYPE_CHECKING, Protocol
-
-from app.logging_utils import structured_log
-from app.services.arxiv.client import ArxivClient
-from app.services.arxiv.errors import ArxivRateLimitError
-from app.services.arxiv.types import ArxivFeed
-from app.settings import settings
-
-if TYPE_CHECKING:
-    from app.services.publications.types import PublicationListItem, UnreadPublicationItem
-
-logger = logging.getLogger(__name__)
-
-_default_gateway: ArxivGateway | None = None
-_MOJIBAKE_HINT_RE = re.compile(r"[ÃÂâ]")
-_NON_ALNUM_RE = re.compile(r"[^\w\s]+", re.UNICODE)
-_WHITESPACE_RE = re.compile(r"\s+")
-
-
-class ArxivGateway(Protocol):
-    async def discover_arxiv_id_for_publication(
-        self,
-        *,
-        item: PublicationListItem | UnreadPublicationItem,
-        request_email: str | None = None,
-        timeout_seconds: float | None = None,
-        max_results: int | None = None,
-    ) -> str | None: ...
-
-
-def build_arxiv_query(title: str, author_surname: str | None) -> str | None:
-    parts: list[str] = []
-    if title:
-        clean_title = _normalize_query_title(title)
-        if clean_title:
-            parts.append(f'ti:"{clean_title}"')
-    if author_surname:
-        clean_author = _normalize_query_title(author_surname)
-        if clean_author:
-            parts.append(f'au:"{clean_author}"')
-    if not parts:
-        return None
-    return " AND ".join(parts)
-
-
-def _normalize_query_title(value: str) -> str:
-    repaired = _repair_mojibake(value.strip())
-    normalized = unicodedata.normalize("NFKC", repaired)
-    stripped = _NON_ALNUM_RE.sub(" ", _MOJIBAKE_HINT_RE.sub(" ", normalized))
-    return _WHITESPACE_RE.sub(" ", stripped).strip()
-
-
-def _repair_mojibake(value: str) -> str:
-    if not value or not _MOJIBAKE_HINT_RE.search(value):
-        return value
-    try:
-        repaired = value.encode("latin1").decode("utf-8")
-    except UnicodeError:
-        return value
-    return repaired if _mojibake_score(repaired) < _mojibake_score(value) else value
-
-
-def _mojibake_score(value: str) -> int:
-    return len(_MOJIBAKE_HINT_RE.findall(value))
-
-
-def get_arxiv_gateway() -> ArxivGateway:
-    global _default_gateway
-    if _default_gateway is None:
-        _default_gateway = HttpArxivGateway()
-    return _default_gateway
-
-
-def set_arxiv_gateway(gateway: ArxivGateway | None) -> ArxivGateway | None:
-    global _default_gateway
-    previous = _default_gateway
-    _default_gateway = gateway
-    return previous
-
-
-class HttpArxivGateway:
-    def __init__(self, *, client: ArxivClient | None = None) -> None:
-        self._client = client or ArxivClient()
-
-    async def discover_arxiv_id_for_publication(
-        self,
-        *,
-        item: PublicationListItem | UnreadPublicationItem,
-        request_email: str | None = None,
-        timeout_seconds: float | None = None,
-        max_results: int | None = None,
-    ) -> str | None:
-        if not settings.arxiv_enabled:
-            return None
-        query = _query_for_item(item)
-        if query is None:
-            return None
-
-        try:
-            result = await self._client.search(
-                query=query,
-                start=0,
-                request_email=request_email,
-                timeout_seconds=timeout_seconds,
-                max_results=max_results,
-            )
-            return _first_discovered_id(result)
-        except ArxivRateLimitError:
-            raise
-        except Exception as exc:
-            structured_log(logger, "debug", "arxiv.query_failed", error=str(exc))
-            return None
-
-
-def _query_for_item(item: PublicationListItem | UnreadPublicationItem) -> str | None:
-    title = (item.title or "").strip()
-    if not title:
-        return None
-    author_surname = _author_surname(item.scholar_label)
-    return build_arxiv_query(title, author_surname)
-
-
-def _author_surname(scholar_label: str | None) -> str | None:
-    if not scholar_label:
-        return None
-    tokens = [token for token in scholar_label.strip().split() if token]
-    if not tokens:
-        return None
-    return tokens[-1].lower()
-
-
-def _first_discovered_id(result: ArxivFeed) -> str | None:
-    for entry in result.entries:
-        if entry.arxiv_id:
-            structured_log(logger, "debug", "arxiv.id_discovered", arxiv_id=entry.arxiv_id)
-            return entry.arxiv_id
-    return None

app/services/arxiv/guards.py

@@ -1,86 +0,0 @@
-from __future__ import annotations
-
-import re
-from typing import TYPE_CHECKING
-
-from app.services.arxiv.constants import (
-    ARXIV_STRONG_IDENTIFIER_CONFIDENCE,
-    ARXIV_TITLE_MIN_ALPHA_TOKENS,
-    ARXIV_TITLE_MIN_TOKENS,
-    ARXIV_TITLE_TOKEN_MIN_LENGTH,
-)
-from app.services.doi.normalize import normalize_doi
-from app.services.publication_identifiers.normalize import normalize_arxiv_id
-
-if TYPE_CHECKING:
-    from app.services.publications.types import PublicationListItem, UnreadPublicationItem
-
-_TITLE_TOKEN_RE = re.compile(r"[a-z0-9]+")
-
-
-def arxiv_skip_reason_for_item(
-    *,
-    item: PublicationListItem | UnreadPublicationItem,
-    has_strong_doi: bool = False,
-    has_existing_arxiv: bool = False,
-) -> str | None:
-    if has_existing_arxiv or _has_arxiv_identifier_evidence(item):
-        return "arxiv_identifier_present"
-    if has_strong_doi or _has_strong_doi_evidence(item):
-        return "strong_doi_present"
-    if not _title_passes_quality_guard(item.title):
-        return "title_quality_below_threshold"
-    return None
-
-
-def _has_arxiv_identifier_evidence(item: PublicationListItem | UnreadPublicationItem) -> bool:
-    if _display_identifier_matches(item, expected_kind="arxiv"):
-        return True
-    return _has_normalized_identifier(item, normalizer=normalize_arxiv_id)
-
-
-def _has_strong_doi_evidence(item: PublicationListItem | UnreadPublicationItem) -> bool:
-    if _display_identifier_matches(item, expected_kind="doi"):
-        return True
-    return _has_normalized_identifier(item, normalizer=normalize_doi)
-
-
-def _display_identifier_matches(
-    item: PublicationListItem | UnreadPublicationItem,
-    *,
-    expected_kind: str,
-) -> bool:
-    display = getattr(item, "display_identifier", None)
-    if display is None:
-        return False
-    if str(display.kind).lower() != expected_kind:
-        return False
-    return float(display.confidence_score) >= ARXIV_STRONG_IDENTIFIER_CONFIDENCE
-
-
-def _has_normalized_identifier(
-    item: PublicationListItem | UnreadPublicationItem,
-    *,
-    normalizer,
-) -> bool:
-    if normalizer(item.pub_url):
-        return True
-    return normalizer(item.pdf_url) is not None
-
-
-def _title_passes_quality_guard(title: str | None) -> bool:
-    tokens = _normalized_tokens(title or "")
-    if len(tokens) < ARXIV_TITLE_MIN_TOKENS:
-        return False
-    alpha_tokens = [token for token in tokens if _is_alpha_token(token)]
-    return len(alpha_tokens) >= ARXIV_TITLE_MIN_ALPHA_TOKENS
-
-
-def _normalized_tokens(value: str) -> list[str]:
-    return [token for token in _TITLE_TOKEN_RE.findall(value.lower()) if token]
-
-
-def _is_alpha_token(token: str) -> bool:
-    if len(token) < ARXIV_TITLE_TOKEN_MIN_LENGTH:
-        return False
-    return any(char.isalpha() for char in token)

app/services/arxiv/parser.py

@@ -1,111 +0,0 @@
-from __future__ import annotations
-
-import xml.etree.ElementTree as ET
-
-from app.services.arxiv.errors import ArxivParseError
-from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
-from app.services.publication_identifiers.normalize import normalize_arxiv_id
-
-_NAMESPACES = {
-    "atom": "http://www.w3.org/2005/Atom",
-    "opensearch": "http://a9.com/-/spec/opensearch/1.1/",
-    "arxiv": "http://arxiv.org/schemas/atom",
-}
-
-
-def parse_arxiv_feed(payload: str) -> ArxivFeed:
-    root = _parse_xml_root(payload)
-    opensearch = ArxivOpenSearchMeta(
-        total_results=_opensearch_int(root, "opensearch:totalResults"),
-        start_index=_opensearch_int(root, "opensearch:startIndex"),
-        items_per_page=_opensearch_int(root, "opensearch:itemsPerPage"),
-    )
-    entries = [_parse_entry(entry_elem) for entry_elem in root.findall("atom:entry", _NAMESPACES)]
-    return ArxivFeed(entries=entries, opensearch=opensearch)
-
-
-def _parse_xml_root(payload: str) -> ET.Element:
-    try:
-        return ET.fromstring(payload)
-    except ET.ParseError as exc:
-        raise ArxivParseError(f"Invalid arXiv XML payload: {exc}") from exc
-
-
-def _opensearch_int(root: ET.Element, path: str) -> int:
-    text = _optional_text(root, path)
-    if text is None:
-        return 0
-    try:
-        return int(text.strip())
-    except ValueError as exc:
-        raise ArxivParseError(f"Invalid integer value at {path}: {text!r}") from exc
-
-
-def _parse_entry(entry_elem: ET.Element) -> ArxivEntry:
-    entry_id_url = _required_text(entry_elem, "atom:id").strip()
-    arxiv_id = normalize_arxiv_id(entry_id_url)
-    title = _required_text(entry_elem, "atom:title").strip()
-    summary = (_optional_text(entry_elem, "atom:summary") or "").strip()
-    published = _optional_text(entry_elem, "atom:published")
-    updated = _optional_text(entry_elem, "atom:updated")
-    return ArxivEntry(
-        entry_id_url=entry_id_url,
-        arxiv_id=arxiv_id,
-        title=title,
-        summary=summary,
-        published=published,
-        updated=updated,
-        authors=_authors(entry_elem),
-        links=_links(entry_elem),
-        categories=_categories(entry_elem),
-        primary_category=_primary_category(entry_elem),
-    )
-
-
-def _required_text(elem: ET.Element, path: str) -> str:
-    text = _optional_text(elem, path)
-    if text is None or not text.strip():
-        raise ArxivParseError(f"Missing required field: {path}")
-    return text
-
-
-def _optional_text(elem: ET.Element, path: str) -> str | None:
-    node = elem.find(path, _NAMESPACES)
-    if node is None or node.text is None:
-        return None
-    return str(node.text)
-
-
-def _authors(entry_elem: ET.Element) -> list[str]:
-    authors: list[str] = []
-    for author in entry_elem.findall("atom:author", _NAMESPACES):
-        name = _optional_text(author, "atom:name")
-        if name:
-            authors.append(name.strip())
-    return authors
-
-
-def _links(entry_elem: ET.Element) -> list[str]:
-    values: list[str] = []
-    for link in entry_elem.findall("atom:link", _NAMESPACES):
-        href = str(link.attrib.get("href") or "").strip()
-        if href:
-            values.append(href)
-    return values
-
-
-def _categories(entry_elem: ET.Element) -> list[str]:
-    values: list[str] = []
-    for cat in entry_elem.findall("atom:category", _NAMESPACES):
-        term = str(cat.attrib.get("term") or "").strip()
-        if term:
-            values.append(term)
-    return values
-
-
-def _primary_category(entry_elem: ET.Element) -> str | None:
-    node = entry_elem.find("arxiv:primary_category", _NAMESPACES)
-    if node is None:
-        return None
-    value = str(node.attrib.get("term") or "").strip()
-    return value or None

app/services/arxiv/rate_limit.py

@@ -1,221 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-from collections.abc import Awaitable, Callable
-from dataclasses import dataclass
-from datetime import UTC, datetime, timedelta
-
-import httpx
-from sqlalchemy import select, text
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import ArxivRuntimeState
-from app.db.session import get_session_factory
-from app.logging_utils import structured_log
-from app.services.arxiv.constants import (
-    ARXIV_RATE_LIMIT_LOCK_KEY,
-    ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
-    ARXIV_RUNTIME_STATE_KEY,
-    ARXIV_SOURCE_PATH_UNKNOWN,
-)
-from app.services.arxiv.errors import ArxivRateLimitError
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass(frozen=True)
-class ArxivCooldownStatus:
-    is_active: bool
-    remaining_seconds: float
-    cooldown_until: datetime | None
-
-
-async def run_with_global_arxiv_limit(
-    *,
-    fetch: Callable[[], Awaitable[httpx.Response]],
-    source_path: str = ARXIV_SOURCE_PATH_UNKNOWN,
-) -> httpx.Response:
-    response, hit_rate_limit = await _run_serialized_fetch(
-        fetch=fetch,
-        source_path=source_path,
-    )
-    if hit_rate_limit:
-        raise ArxivRateLimitError("arXiv rate limit hit (429) — stopping batch")
-    return response
-
-
-async def get_arxiv_cooldown_status(*, now_utc: datetime | None = None) -> ArxivCooldownStatus:
-    timestamp = _normalize_datetime(now_utc) or datetime.now(UTC)
-    session_factory = get_session_factory()
-    async with session_factory() as db_session:
-        result = await db_session.execute(
-            select(ArxivRuntimeState.cooldown_until).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY)
-        )
-        cooldown_until = _normalize_datetime(result.scalar_one_or_none())
-    remaining_seconds = _cooldown_remaining_seconds(cooldown_until, now_utc=timestamp)
-    return ArxivCooldownStatus(
-        is_active=remaining_seconds > 0.0,
-        remaining_seconds=float(remaining_seconds),
-        cooldown_until=cooldown_until,
-    )
-
-
-async def _run_serialized_fetch(
-    *,
-    fetch: Callable[[], Awaitable[httpx.Response]],
-    source_path: str,
-) -> tuple[httpx.Response, bool]:
-    session_factory = get_session_factory()
-    async with session_factory() as lock_session:
-        await _acquire_arxiv_lock(lock_session)
-        try:
-            async with session_factory() as db_session, db_session.begin():
-                runtime_state = await _load_runtime_state_for_update(db_session)
-                wait_seconds = await _wait_for_allowed_slot_or_raise(
-                    runtime_state,
-                    source_path=source_path,
-                )
-                runtime_state.next_allowed_at = datetime.now(UTC) + timedelta(seconds=_min_interval_seconds())
-
-            if wait_seconds > 0:
-                await asyncio.sleep(wait_seconds)
-            response = await fetch()
-
-            async with session_factory() as db_session, db_session.begin():
-                runtime_state = await _load_runtime_state_for_update(db_session)
-                hit_rate_limit = _record_post_response_state(
-                    runtime_state,
-                    response_status=int(response.status_code),
-                    source_path=source_path,
-                )
-                cooldown_until_value = runtime_state.cooldown_until
-        finally:
-            await _release_arxiv_lock(lock_session)
-    structured_log(
-        logger,
-        "info",
-        "arxiv.request_completed",
-        status_code=int(response.status_code),
-        wait_seconds=wait_seconds,
-        cooldown_remaining_seconds=_cooldown_remaining_seconds(cooldown_until_value, now_utc=datetime.now(UTC)),
-        source_path=source_path,
-    )
-    return response, hit_rate_limit
-
-
-async def _acquire_arxiv_lock(db_session: AsyncSession) -> None:
-    await db_session.execute(
-        text("SELECT pg_advisory_lock(:namespace, :lock_key)"),
-        {
-            "namespace": ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
-            "lock_key": ARXIV_RATE_LIMIT_LOCK_KEY,
-        },
-    )
-
-
-async def _release_arxiv_lock(db_session: AsyncSession) -> None:
-    await db_session.execute(
-        text("SELECT pg_advisory_unlock(:namespace, :lock_key)"),
-        {
-            "namespace": ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
-            "lock_key": ARXIV_RATE_LIMIT_LOCK_KEY,
-        },
-    )
-
-
-async def _load_runtime_state_for_update(db_session: AsyncSession) -> ArxivRuntimeState:
-    result = await db_session.execute(
-        select(ArxivRuntimeState).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY).with_for_update()
-    )
-    state = result.scalar_one_or_none()
-    if state is not None:
-        return state
-    state = ArxivRuntimeState(state_key=ARXIV_RUNTIME_STATE_KEY)
-    db_session.add(state)
-    await db_session.flush()
-    return state
-
-
-async def _wait_for_allowed_slot_or_raise(
-    runtime_state: ArxivRuntimeState,
-    *,
-    source_path: str,
-) -> float:
-    now_utc = datetime.now(UTC)
-    cooldown_seconds = _cooldown_remaining_seconds(runtime_state.cooldown_until, now_utc=now_utc)
-    if cooldown_seconds > 0:
-        structured_log(
-            logger,
-            "info",
-            "arxiv.request_scheduled",
-            wait_seconds=0.0,
-            source_path=source_path,
-            cooldown_remaining_seconds=cooldown_seconds,
-        )
-        raise ArxivRateLimitError(f"arXiv global cooldown active ({cooldown_seconds:.0f}s remaining)")
-    wait_seconds = _next_allowed_wait_seconds(runtime_state.next_allowed_at, now_utc=now_utc)
-    structured_log(
-        logger,
-        "info",
-        "arxiv.request_scheduled",
-        wait_seconds=wait_seconds,
-        source_path=source_path,
-        cooldown_remaining_seconds=0.0,
-    )
-    return wait_seconds
-
-
-def _record_post_response_state(
-    runtime_state: ArxivRuntimeState,
-    *,
-    response_status: int,
-    source_path: str,
-) -> bool:
-    now_utc = datetime.now(UTC)
-    runtime_state.next_allowed_at = now_utc + timedelta(seconds=_min_interval_seconds())
-    if response_status == 429:
-        cooldown_seconds = _cooldown_seconds()
-        runtime_state.cooldown_until = now_utc + timedelta(seconds=cooldown_seconds)
-        structured_log(
-            logger,
-            "warning",
-            "arxiv.cooldown_activated",
-            cooldown_remaining_seconds=cooldown_seconds,
-            source_path=source_path,
-        )
-        return True
-    if _cooldown_remaining_seconds(runtime_state.cooldown_until, now_utc=now_utc) <= 0:
-        runtime_state.cooldown_until = None
-    return False
-
-
-def _cooldown_remaining_seconds(cooldown_until: datetime | None, *, now_utc: datetime) -> float:
-    bounded = _normalize_datetime(cooldown_until)
-    if bounded is None:
-        return 0.0
-    return max((bounded - now_utc).total_seconds(), 0.0)
-
-
-def _next_allowed_wait_seconds(next_allowed_at: datetime | None, *, now_utc: datetime) -> float:
-    bounded = _normalize_datetime(next_allowed_at)
-    if bounded is None:
-        return 0.0
-    return max((bounded - now_utc).total_seconds(), 0.0)
-
-
-def _normalize_datetime(value: datetime | None) -> datetime | None:
-    if value is None:
-        return None
-    if value.tzinfo is None:
-        return value.replace(tzinfo=UTC)
-    return value
-
-
-def _min_interval_seconds() -> float:
-    return max(float(settings.arxiv_min_interval_seconds), 0.0)
-
-
-def _cooldown_seconds() -> float:
-    return max(float(settings.arxiv_rate_limit_cooldown_seconds), 0.0)

app/services/arxiv/types.py

@@ -1,34 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import Literal
-
-ArxivSortBy = Literal["relevance", "lastUpdatedDate", "submittedDate"]
-ArxivSortOrder = Literal["ascending", "descending"]
-
-
-@dataclass(frozen=True)
-class ArxivOpenSearchMeta:
-    total_results: int = 0
-    start_index: int = 0
-    items_per_page: int = 0
-
-
-@dataclass(frozen=True)
-class ArxivEntry:
-    entry_id_url: str
-    arxiv_id: str | None
-    title: str
-    summary: str
-    published: str | None
-    updated: str | None
-    authors: list[str] = field(default_factory=list)
-    links: list[str] = field(default_factory=list)
-    categories: list[str] = field(default_factory=list)
-    primary_category: str | None = None
-
-
-@dataclass(frozen=True)
-class ArxivFeed:
-    entries: list[ArxivEntry] = field(default_factory=list)
-    opensearch: ArxivOpenSearchMeta = field(default_factory=ArxivOpenSearchMeta)

app/services/crossref/__init__.py

@@ -1,5 +0,0 @@
-from __future__ import annotations
-
-from app.services.crossref.application import discover_doi_for_publication
-
-__all__ = ["discover_doi_for_publication"]

app/services/dbops/__init__.py

@@ -1,13 +0,0 @@
-from app.services.dbops.application import run_publication_link_repair
-from app.services.dbops.integrity import collect_integrity_report
-from app.services.dbops.near_duplicate_repair import (
-    run_publication_near_duplicate_repair,
-)
-from app.services.dbops.query import list_repair_jobs
-
-__all__ = [
-    "collect_integrity_report",
-    "list_repair_jobs",
-    "run_publication_link_repair",
-    "run_publication_near_duplicate_repair",
-]

app/services/dbops/near_duplicate_repair.py

@@ -1,228 +0,0 @@
-from __future__ import annotations
-
-from datetime import UTC, datetime
-from typing import Any
-
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import DataRepairJob
-from app.services.dbops.application import (
-    REPAIR_STATUS_COMPLETED,
-    REPAIR_STATUS_FAILED,
-    REPAIR_STATUS_PLANNED,
-    REPAIR_STATUS_RUNNING,
-)
-from app.services.publications import dedup as dedup_service
-
-NEAR_DUP_JOB_NAME = "repair_publication_near_duplicates"
-NEAR_DUP_DEFAULT_MAX_CLUSTERS = 25
-
-
-def _utcnow() -> datetime:
-    return datetime.now(UTC)
-
-
-def _normalized_cluster_keys(values: list[str] | None) -> list[str]:
-    if not values:
-        return []
-    seen: set[str] = set()
-    normalized: list[str] = []
-    for value in values:
-        key = str(value or "").strip().lower()
-        if not key or key in seen:
-            continue
-        seen.add(key)
-        normalized.append(key)
-    return normalized
-
-
-def _normalized_max_clusters(value: int) -> int:
-    return max(1, min(int(value), 200))
-
-
-def _scope_payload(
-    *,
-    similarity_threshold: float,
-    min_shared_tokens: int,
-    max_year_delta: int,
-    max_clusters: int,
-    selected_cluster_keys: list[str],
-) -> dict[str, Any]:
-    return {
-        "similarity_threshold": float(similarity_threshold),
-        "min_shared_tokens": int(min_shared_tokens),
-        "max_year_delta": int(max_year_delta),
-        "max_clusters": int(max_clusters),
-        "selected_cluster_keys": selected_cluster_keys,
-    }
-
-
-async def _create_job(
-    db_session: AsyncSession,
-    *,
-    requested_by: str | None,
-    scope: dict[str, Any],
-    dry_run: bool,
-) -> DataRepairJob:
-    job = DataRepairJob(
-        job_name=NEAR_DUP_JOB_NAME,
-        requested_by=(requested_by or "").strip() or None,
-        scope=scope,
-        dry_run=bool(dry_run),
-        status=REPAIR_STATUS_PLANNED,
-        summary={},
-    )
-    db_session.add(job)
-    await db_session.flush()
-    job.status = REPAIR_STATUS_RUNNING
-    job.started_at = _utcnow()
-    return job
-
-
-def _selected_clusters(
-    *,
-    clusters: list[dedup_service.NearDuplicateCluster],
-    selected_cluster_keys: list[str],
-) -> tuple[list[dedup_service.NearDuplicateCluster], list[str]]:
-    if not selected_cluster_keys:
-        return [], []
-    by_key = {cluster.cluster_key.lower(): cluster for cluster in clusters}
-    selected: list[dedup_service.NearDuplicateCluster] = []
-    missing: list[str] = []
-    for key in selected_cluster_keys:
-        cluster = by_key.get(key)
-        if cluster is None:
-            missing.append(key)
-            continue
-        selected.append(cluster)
-    return selected, missing
-
-
-async def _merge_selected_clusters(
-    db_session: AsyncSession,
-    *,
-    selected_clusters: list[dedup_service.NearDuplicateCluster],
-) -> int:
-    merged_publications = 0
-    for cluster in selected_clusters:
-        merged_publications += await dedup_service.merge_near_duplicate_cluster(
-            db_session,
-            cluster=cluster,
-        )
-    return merged_publications
-
-
-def _clusters_payload(
-    *,
-    clusters: list[dedup_service.NearDuplicateCluster],
-    max_clusters: int,
-) -> list[dict[str, object]]:
-    preview = clusters[:max_clusters]
-    return [dedup_service.near_duplicate_cluster_payload(cluster) for cluster in preview]
-
-
-def _summary_payload(
-    *,
-    dry_run: bool,
-    cluster_count: int,
-    selected_count: int,
-    missing_count: int,
-    merged_publications: int,
-    max_clusters: int,
-) -> dict[str, Any]:
-    return {
-        "dry_run": bool(dry_run),
-        "candidate_cluster_count": int(cluster_count),
-        "selected_cluster_count": int(selected_count),
-        "missing_selected_cluster_count": int(missing_count),
-        "merged_publications": int(merged_publications),
-        "preview_cluster_count": int(min(cluster_count, max_clusters)),
-    }
-
-
-async def _complete_job(
-    db_session: AsyncSession,
-    *,
-    job: DataRepairJob,
-    scope: dict[str, Any],
-    summary: dict[str, Any],
-    clusters: list[dict[str, object]],
-) -> dict[str, Any]:
-    job.status = REPAIR_STATUS_COMPLETED
-    job.finished_at = _utcnow()
-    job.summary = summary
-    await db_session.commit()
-    return {
-        "job_id": int(job.id),
-        "status": job.status,
-        "scope": scope,
-        "summary": summary,
-        "clusters": clusters,
-    }
-
-
-async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None:
-    from sqlalchemy.orm import make_transient
-
-    await db_session.rollback()
-    make_transient(job)
-    job.status = REPAIR_STATUS_FAILED
-    job.error_text = str(error)
-    job.finished_at = _utcnow()
-    db_session.add(job)
-    await db_session.commit()
-
-
-async def run_publication_near_duplicate_repair(
-    db_session: AsyncSession,
-    *,
-    dry_run: bool = True,
-    similarity_threshold: float = dedup_service.NEAR_DUP_DEFAULT_SIMILARITY_THRESHOLD,
-    min_shared_tokens: int = dedup_service.NEAR_DUP_DEFAULT_MIN_SHARED_TOKENS,
-    max_year_delta: int = dedup_service.NEAR_DUP_DEFAULT_MAX_YEAR_DELTA,
-    max_clusters: int = NEAR_DUP_DEFAULT_MAX_CLUSTERS,
-    selected_cluster_keys: list[str] | None = None,
-    requested_by: str | None = None,
-) -> dict[str, Any]:
-    normalized_keys = _normalized_cluster_keys(selected_cluster_keys)
-    bounded_clusters = _normalized_max_clusters(max_clusters)
-    scope = _scope_payload(
-        similarity_threshold=similarity_threshold,
-        min_shared_tokens=min_shared_tokens,
-        max_year_delta=max_year_delta,
-        max_clusters=bounded_clusters,
-        selected_cluster_keys=normalized_keys,
-    )
-    job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run)
-    try:
-        clusters = await dedup_service.find_near_duplicate_clusters(
-            db_session,
-            similarity_threshold=similarity_threshold,
-            min_shared_tokens=min_shared_tokens,
-            max_year_delta=max_year_delta,
-        )
-        selected, missing = _selected_clusters(clusters=clusters, selected_cluster_keys=normalized_keys)
-        merged_publications = 0
-        if not dry_run:
-            if not selected:
-                raise ValueError("No selected near-duplicate clusters matched current data.")
-            merged_publications = await _merge_selected_clusters(db_session, selected_clusters=selected)
-        preview = _clusters_payload(clusters=clusters, max_clusters=bounded_clusters)
-        summary = _summary_payload(
-            dry_run=dry_run,
-            cluster_count=len(clusters),
-            selected_count=len(selected),
-            missing_count=len(missing),
-            merged_publications=merged_publications,
-            max_clusters=bounded_clusters,
-        )
-        return await _complete_job(
-            db_session,
-            job=job,
-            scope=scope,
-            summary=summary,
-            clusters=preview,
-        )
-    except Exception as exc:
-        await _fail_job(db_session, job=job, error=exc)
-        raise

app/services/domains/crossref/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from app.services.domains.crossref.application import discover_doi_for_publication
+
+__all__ = ["discover_doi_for_publication"]

app/services/domains/crossref/application.py

@@ -5,28 +5,22 @@ import logging
 import re
 import threading
 import time
-from importlib.metadata import version as pkg_version
 from typing import TYPE_CHECKING
 
 from crossref.restful import Etiquette, Works
 
-from app.logging_utils import structured_log
-from app.services.doi.normalize import normalize_doi
+from app.services.domains.doi.normalize import normalize_doi
 from app.settings import settings
 
-_APP_VERSION = pkg_version("scholarr")
-
 if TYPE_CHECKING:
-    from app.services.publications.types import PublicationListItem, UnreadPublicationItem
+    from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
 
 TOKEN_RE = re.compile(r"[a-z0-9]+")
-NON_ALNUM_RE = re.compile(r"[^a-z0-9\s]+")
+NON_ALNUM_RE = re.compile(r"[^a-z0-9\\s]+")
 STOP_WORDS = {"the", "and", "for", "with", "from", "method", "study", "analysis"}
 _RATE_LOCK = threading.Lock()
 _LAST_REQUEST_AT = 0.0
 logger = logging.getLogger(__name__)
-STRICT_TITLE_MATCH_THRESHOLD = 0.75
-RELAXED_TITLE_MATCH_THRESHOLD = 0.85
 
 
 def _rate_limit_wait(min_interval_seconds: float) -> None:
@@ -35,14 +29,13 @@ def _rate_limit_wait(min_interval_seconds: float) -> None:
     with _RATE_LOCK:
         elapsed = time.monotonic() - _LAST_REQUEST_AT
         remaining = interval - elapsed
-    if remaining > 0:
-        time.sleep(remaining)
-    with _RATE_LOCK:
+        if remaining > 0:
+            time.sleep(remaining)
         _LAST_REQUEST_AT = time.monotonic()
 
 
 def _normalized_tokens(value: str) -> list[str]:
-    lowered = value.lower().replace("’", "'").replace("“", '"').replace("”", '"')
+    lowered = value.lower().replace("’", "'").replace("“", "\"").replace("”", "\"")
     lowered = NON_ALNUM_RE.sub(" ", lowered)
     return [token for token in TOKEN_RE.findall(lowered) if len(token) >= 3]
 
@@ -139,42 +132,7 @@ def _candidate_rank(*, title: str, year: int | None, item: dict) -> tuple[float,
     return score, doi
 
 
-def _year_delta(source_year: int | None, candidate_year: int | None) -> int | None:
-    if source_year is None or candidate_year is None:
-        return None
-    return abs(int(source_year) - int(candidate_year))
-
-
-def _candidate_rank_relaxed(
-    *,
-    title: str,
-    year: int | None,
-    item: dict,
-    author_surname: str | None,
-) -> tuple[float, str | None]:
-    doi = normalize_doi(str(item.get("DOI") or ""))
-    if doi is None:
-        return 0.0, None
-    score = _title_match_score(title, _candidate_title(item))
-    if score <= 0:
-        return 0.0, None
-    candidate_year = _candidate_year(item)
-    delta = _year_delta(year, candidate_year)
-    if delta is not None:
-        if delta <= 1:
-            score += 0.05
-        elif delta <= 3:
-            score += 0.0
-        elif delta <= 5:
-            score -= 0.03
-        else:
-            score -= 0.08
-    if _candidate_author_match(item, author_surname):
-        score += 0.03
-    return score, doi
-
-
-def _best_candidate_doi_strict(
+def _best_candidate_doi(
     *,
     title: str,
     year: int | None,
@@ -191,7 +149,7 @@ def _best_candidate_doi_strict(
             continue
         score, doi = _candidate_rank(title=title, year=year, item=item)
         candidate_year = _candidate_year(item)
-        if doi is None or score < STRICT_TITLE_MATCH_THRESHOLD:
+        if doi is None or score < 0.75:
             continue
         if score > best_score:
             best_score = score
@@ -208,95 +166,9 @@ def _best_candidate_doi_strict(
     return best_doi
 
 
-def _best_candidate_doi_relaxed(
-    *,
-    title: str,
-    year: int | None,
-    items: list[dict],
-    author_surname: str | None,
-) -> str | None:
-    best_score = 0.0
-    best_doi: str | None = None
-    best_author_match = False
-    best_delta: int | None = None
-    best_year: int | None = None
-    for item in items:
-        if not isinstance(item, dict):
-            continue
-        score, doi = _candidate_rank_relaxed(
-            title=title,
-            year=year,
-            item=item,
-            author_surname=author_surname,
-        )
-        if doi is None or score < RELAXED_TITLE_MATCH_THRESHOLD:
-            continue
-        candidate_year = _candidate_year(item)
-        candidate_author_match = _candidate_author_match(item, author_surname)
-        candidate_delta = _year_delta(year, candidate_year)
-        if score > best_score:
-            best_score = score
-            best_doi = doi
-            best_author_match = candidate_author_match
-            best_delta = candidate_delta
-            best_year = candidate_year
-            continue
-        if abs(score - best_score) > 0.02:
-            continue
-        if candidate_author_match and not best_author_match:
-            best_doi = doi
-            best_author_match = True
-            best_delta = candidate_delta
-            best_year = candidate_year
-            continue
-        if best_delta is None and candidate_delta is not None:
-            best_doi = doi
-            best_author_match = candidate_author_match
-            best_delta = candidate_delta
-            best_year = candidate_year
-            continue
-        if best_delta is not None and candidate_delta is not None and candidate_delta < best_delta:
-            best_doi = doi
-            best_author_match = candidate_author_match
-            best_delta = candidate_delta
-            best_year = candidate_year
-            continue
-        if best_year is None or candidate_year is None:
-            continue
-        if candidate_year < best_year:
-            best_doi = doi
-            best_author_match = candidate_author_match
-            best_delta = candidate_delta
-            best_year = candidate_year
-    return best_doi
-
-
-def _best_candidate_doi(
-    *,
-    title: str,
-    year: int | None,
-    items: list[dict],
-    author_surname: str | None,
-) -> str | None:
-    strict_match = _best_candidate_doi_strict(
-        title=title,
-        year=year,
-        items=items,
-        author_surname=author_surname,
-    )
-    if strict_match:
-        return strict_match
-    return _best_candidate_doi_relaxed(
-        title=title,
-        year=year,
-        items=items,
-        author_surname=author_surname,
-    )
-
-
 def _works_client(email: str | None) -> Works:
     if email:
-        etiquette = Etiquette(settings.app_name, _APP_VERSION, "https://scholarr.local", email)
+        etiquette = Etiquette(settings.app_name, "0.1.0", "https://scholarr.local", email)
         return Works(etiquette=etiquette)
     return Works()
 
@@ -351,8 +223,7 @@ async def _fetch_items(
             ),
             timeout=timeout,
         )
-    except Exception as exc:
-        structured_log(logger, "warning", "crossref.fetch_failed", error=str(exc))
+    except Exception:
         return []
 
 
@@ -383,6 +254,6 @@ async def discover_doi_for_publication(
             author_surname=author_surname,
         )
         if doi:
-            structured_log(logger, "debug", "crossref.doi_discovered")
+            logger.debug("crossref.doi_discovered", extra={"event": "crossref.doi_discovered"})
             return doi
     return None

app/services/domains/dbops/__init__.py

@@ -0,0 +1,5 @@
+from app.services.domains.dbops.application import run_publication_link_repair
+from app.services.domains.dbops.integrity import collect_integrity_report
+from app.services.domains.dbops.query import list_repair_jobs
+
+__all__ = ["collect_integrity_report", "list_repair_jobs", "run_publication_link_repair"]