.claude/settings.json

@@ -1,33 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(wc -l /home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src/features/settings/SettingsAdminPanel.vue /home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src/features/settings/components/Admin*.vue)",
-      "Read(//home/jv/src/personal/playground/scholar_scraper/scholar-scraper/**)",
-      "Bash(while read line rest)",
-      "Bash(do echo \"$line $rest\")",
-      "Bash(done)",
-      "Bash(grep -c 'def test_' tests/unit/*.py tests/unit/**/*.py tests/integration/*.py)",
-      "Bash(sort -t: -k2 -rn)",
-      "Bash(wc -l frontend/src/pages/*.vue)",
-      "Bash(uv run ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
-      "Bash(ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
-      "Bash(npm install --package-lock-only)",
-      "Bash(python -m ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
-      "Bash(pip show ruff)",
-      "Bash(pip install ruff)",
-      "Bash(uvx ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
-      "Bash(git add app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py docs/website/package-lock.json)",
-      "Bash(git push)",
-      "Bash(uvx ruff format --check app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
-      "Bash(test -f docs/website/package-lock.json)",
-      "Bash(gh run list --branch openalex-size-reduction-scholar-hydration --limit 3 --json databaseId,status,conclusion,name,event)",
-      "Bash(gh run list --limit 10 --json databaseId,status,conclusion,name,event,headBranch)",
-      "Bash(gh run view 22554585841 --log-failed)",
-      "Bash(gh run rerun 22554585841 --failed)",
-      "Bash(bash scripts/premerge.sh)"
-    ],
-    "additionalDirectories": [
-      "/home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src"
-    ]
-  }
-}

.env.example

@@ -28,7 +28,6 @@ DATABASE_POOL_MODE=auto
 DATABASE_POOL_SIZE=5
 DATABASE_POOL_MAX_OVERFLOW=10
 DATABASE_POOL_TIMEOUT_SECONDS=30
-DATABASE_RESERVED_API_CONNECTIONS=3
 
 # ------------------------------
 # Frontend Dev Overrides
@@ -116,10 +115,6 @@ SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD=1
 SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS=1800
 SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD=2
 SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD=3
-SCHOLAR_HTTP_USER_AGENT=
-SCHOLAR_HTTP_ROTATE_USER_AGENT=0
-SCHOLAR_HTTP_ACCEPT_LANGUAGE=en-US,en;q=0.9
-SCHOLAR_HTTP_COOKIE=
 
 # ------------------------------
 # OA Enrichment + PDF Resolution

.github/workflows/release.yml

@@ -19,14 +19,9 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
-      - uses: astral-sh/setup-uv@v4
+      - run: pip install python-semantic-release
-      - run: uv sync --extra dev
-      - name: Lint
-        run: uv run ruff check
-      - name: Test
-        run: uv run pytest
       - name: Semantic Release
         id: release
-        run: uv run semantic-release publish
+        run: semantic-release publish
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -18,5 +18,3 @@ frontend/dist/
 frontend/.vite/
 docs/website/node_modules/
 docs/website/build/
-docs/website/.docusaurus/
-.claude/

MODERNIZATION_PLAN.md

@@ -0,0 +1,628 @@
+# Scholarr Modernization Plan — Slices 2-10
+
+> **Slice 1 is complete** (commit `7736cab`): removed 114 stale `build/` files, fixed `.gitignore`, committed pending deletions, replaced `npm install` with `npm ci` in CI.
+>
+> Each slice below is a self-contained prompt designed for an LLM executor. Execute in order — each slice depends on the prior ones being complete.
+
+---
+
+## Slice 2: Create `structured_log()` Utility
+
+**Why:** The codebase has 19 `_log_*` helper functions (422 lines) and 138 instances of duplicated `"event"` keys in logging calls. A single utility eliminates all of it.
+
+**Files to create:**
+- `app/logging_utils.py`
+
+**Files to modify:**
+- `tests/unit/test_logging.py`
+
+**Context files (read but don't modify):**
+- `app/logging_config.py` — line 88 has `"event": getattr(record, "event", record.getMessage())` which means when no `event` key is in `extra`, it falls back to `getMessage()`. This is the mechanism that makes `structured_log()` work without duplicating the event name.
+- `app/logging_context.py`
+
+**Prompt:**
+```
+You are working on the scholarr repository. The current logging pattern has a DRY problem:
+
+1. Every log call duplicates the event name as both the message and in extra["event"]:
+   `logger.info("event.name", extra={"event": "event.name", "key": value})`
+
+2. There are 19 `_log_*` helper functions (422 lines total) that just wrap logger calls with typed signatures to build extra dicts.
+
+3. Metrics data (metric_name/metric_value) is mixed into log extra dicts but has no consumer.
+
+Create `app/logging_utils.py`:
+
+```python
+"""Structured logging utility — eliminates boilerplate across domain services."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+
+def structured_log(
+    logger: logging.Logger,
+    level: str,
+    event: str,
+    /,
+    **fields: Any,
+) -> None:
+    """Emit a structured log entry.
+
+    The event name is passed as the log message. The JsonLogFormatter in
+    logging_config.py extracts it via record.getMessage() when no explicit
+    'event' key exists in extra — so we do NOT duplicate it.
+
+    Usage:
+        structured_log(logger, "info", "ingestion.run_started", user_id=1, scholar_count=5)
+    """
+    fields.pop("metric_name", None)
+    fields.pop("metric_value", None)
+
+    log_method = getattr(logger, level.lower())
+    log_method(event, extra=fields)
+```
+
+Verify that `app/logging_config.py` line 88 already handles this correctly — when no `event` key is in extra, it falls back to getMessage() which returns the event string passed as the first argument. No changes to logging_config.py should be needed.
+
+Add tests in `tests/unit/test_logging.py` (append to existing file):
+- Test that `structured_log()` produces a log record where the JsonLogFormatter outputs the event name correctly
+- Test that `structured_log()` works with ConsoleLogFormatter
+- Test that metric_name/metric_value fields are stripped from output
+- Test that extra fields (user_id, scholar_id, etc.) appear in the formatted output
+
+Commit message: "refactor: add structured_log utility to eliminate logging boilerplate"
+```
+
+---
+
+## Slice 3: Migrate Ingestion Logging to `structured_log()`
+
+**Why:** `app/services/ingestion/application.py` is 3,089 lines. 8 `_log_*` helpers consume ~239 lines. 34 inline logger calls duplicate event names and include dead metric fields. This slice removes ~300 lines.
+
+**Files to modify:**
+- `app/services/ingestion/application.py`
+
+**Context files:**
+- `app/logging_utils.py` (created in Slice 2)
+
+**Prompt:**
+```
+You are working on the scholarr repository. The file `app/services/ingestion/application.py` (3,089 lines) has 8 `_log_*` helper functions consuming ~239 lines, plus 34 inline logger calls with duplicated `"event":` keys and mixed `metric_name`/`metric_value` fields.
+
+Migrate ALL logging in this file to use `structured_log()` from `app.logging_utils`.
+
+1. Add `from app.logging_utils import structured_log` at the top.
+
+2. Delete these 8 helper methods entirely:
+   - `_log_request_delay_coercion` (line ~123, 21 lines)
+   - `_log_run_started` (line ~310, 36 lines)
+   - `_log_scholar_parsed` (line ~383, 28 lines)
+   - `_log_alert_thresholds` (line ~1013, 47 lines)
+   - `_log_safety_transition` (line ~1093, 42 lines)
+   - `_log_run_completed` (line ~1178, 28 lines)
+   - `_attempt_log_entry` (line ~1837, 16 lines)
+   - `_page_log_entry` (line ~1999, 21 lines)
+
+3. Replace every call to these deleted helpers with an inline `structured_log()` call. Example:
+
+   Before:
+   ```python
+   self._log_run_started(
+       user_id=user_id,
+       trigger_type=trigger_type,
+       scholar_count=scholar_count,
+       ...
+   )
+   ```
+
+   After:
+   ```python
+   structured_log(
+       logger, "info", "ingestion.run_started",
+       user_id=user_id,
+       trigger_type=trigger_type.value,
+       scholar_count=scholar_count,
+       ...
+   )
+   ```
+
+4. For ALL remaining inline `logger.info/warning/debug/exception()` calls:
+   - Replace with `structured_log()` where the call uses `extra={"event": ..., ...}` pattern
+   - Do NOT convert `logger.exception()` calls — keep them but remove the duplicated `"event"` key from their extra dict
+   - Remove `metric_name` and `metric_value` from all calls
+
+5. Do NOT change any business logic. Only logging call sites.
+
+Expected outcome: ~300 lines removed. All existing tests must pass unchanged.
+
+Commit message: "refactor: migrate ingestion service logging to structured_log"
+```
+
+---
+
+## Slice 4: Migrate All Remaining Services to `structured_log()`
+
+**Why:** 11 more `_log_*` helpers remain across 8 files (183 lines total), plus inline logger calls with the same duplication pattern in ~20 files.
+
+**Files to modify (all have `_log_*` helpers to delete):**
+- `app/services/ingestion/scheduler.py` — `_log_queue_item_resolved` (line ~518, 21 lines)
+- `app/services/arxiv/rate_limit.py` — `_log_request_scheduled` (199), `_log_request_completed` (216), `_log_cooldown_activated` (235)
+- `app/services/arxiv/client.py` — `_log_cache_event` (283), `_log_request_skipped_for_cooldown` (299)
+- `app/services/publications/pdf_resolution_pipeline.py` — `_log_arxiv_skip` (148)
+- `app/services/unpaywall/application.py` — `_log_resolution_summary` (194)
+- `app/api/routers/publications.py` — `_log_retry_pdf_result` (343)
+- `app/api/routers/settings.py` — `_log_settings_update` (103)
+- `app/main.py` — `_log_startup_build_marker` (53)
+
+**Files to modify (inline `extra={"event":...}` pattern only — no helpers to delete):**
+- `app/http/middleware.py`
+- `app/db/session.py`
+- `app/auth/runtime.py`
+- `app/security/csrf.py`
+- `app/api/routers/scholars.py`
+- `app/api/routers/runs.py`
+- `app/api/routers/admin_dbops.py`
+- `app/api/routers/admin.py`
+- `app/api/routers/auth.py`
+- `app/api/routers/publications.py`
+- `app/services/scholars/application.py`
+- `app/services/runs/events.py`
+- `app/services/openalex/client.py`
+- `app/services/openalex/matching.py`
+- `app/services/crossref/application.py`
+- `app/services/publications/dedup.py`
+- `app/services/publications/enrichment.py`
+- `app/services/publications/pdf_queue.py`
+- `app/services/scholar/source.py`
+- `app/services/arxiv/gateway.py`
+
+**Prompt:**
+```
+You are working on the scholarr repository. Slice 3 migrated `ingestion/application.py` to `structured_log()`. Now do the same for ALL remaining files.
+
+1. Delete these `_log_*` helper functions and replace their call sites with inline `structured_log()`:
+
+   - `app/services/ingestion/scheduler.py:~518` — `_log_queue_item_resolved` (21 lines)
+   - `app/services/arxiv/rate_limit.py:~199` — `_log_request_scheduled` (17 lines)
+   - `app/services/arxiv/rate_limit.py:~216` — `_log_request_completed` (19 lines)
+   - `app/services/arxiv/rate_limit.py:~235` — `_log_cooldown_activated` (13 lines)
+   - `app/services/arxiv/client.py:~283` — `_log_cache_event` (16 lines)
+   - `app/services/arxiv/client.py:~299` — `_log_request_skipped_for_cooldown` (13 lines)
+   - `app/services/publications/pdf_resolution_pipeline.py:~148` — `_log_arxiv_skip` (11 lines)
+   - `app/services/unpaywall/application.py:~194` — `_log_resolution_summary` (21 lines)
+   - `app/api/routers/publications.py:~343` — `_log_retry_pdf_result` (24 lines)
+   - `app/api/routers/settings.py:~103` — `_log_settings_update` (15 lines)
+   - `app/main.py:~53` — `_log_startup_build_marker` (13 lines)
+
+2. In ALL files under `app/` that have inline `logger.info/warning/debug("event.name", extra={"event": "event.name", ...})` calls:
+   - Replace with `structured_log(logger, "level", "event.name", key=value, ...)`
+   - Add `from app.logging_utils import structured_log` import
+   - Remove `metric_name`/`metric_value` from all calls
+   - Keep `logger.exception()` calls but remove the duplicated `"event"` key from their extra dicts
+
+3. In `app/services/openalex/client.py`, the lines that log `response.text` (lines ~87, ~137):
+   - Truncate: `response.text[:500]`
+
+4. Do NOT change any business logic. Only logging call sites.
+
+Verify: `grep -rn '"event":' app/ --include="*.py" | grep -v __pycache__` should return 0 matches.
+Verify: `grep -rn 'def _log_' app/ --include="*.py"` should return 0 matches.
+
+Commit message: "refactor: migrate all remaining logging to structured_log, remove boilerplate helpers"
+```
+
+---
+
+## Slice 5: Logging Relevance Audit — Reduce Noise, Improve Readability
+
+**Why:** Self-hosted users need clear, actionable logs. Current logging has verbose debug noise (per-HTTP-request, per-publication), overly long event names, and no console display optimization.
+
+**Files to modify:**
+- `app/services/scholar/source.py`
+- `app/services/ingestion/application.py`
+- `app/logging_config.py`
+- Any files with event names longer than ~40 chars
+
+**Prompt:**
+```
+You are working on the scholarr repository. The logging has been migrated to `structured_log()` (slices 2-4). Now audit log RELEVANCE and READABILITY for self-hosted users running this in Docker.
+
+1. **Remove redundant debug logs:**
+   - `app/services/scholar/source.py`: Remove the 3 debug-level HTTP fetch events (fetch_started, search_fetch_started, publication_fetch_started). Keep only `fetch_succeeded` at DEBUG. The HTTP middleware already logs request.started/completed.
+   - `app/services/ingestion/application.py`: Remove per-publication `publication.discovered` and `publication.created` DEBUG logs. The run_completed summary already reports totals.
+
+2. **Shorten overly long event names** (search all structured_log calls in app/):
+   - `ingestion.request_delay_coerced_to_policy_floor` → `ingestion.delay_coerced`
+   - `ingestion.safety_cooldown_cleared` → `ingestion.cooldown_cleared`
+   - Any other names longer than ~40 chars — shorten while preserving meaning
+
+3. **Improve console readability** in `app/logging_config.py` `ConsoleLogFormatter.format()`:
+   - Add a short-name mapping for common context fields in console output ONLY (JSON keeps full names):
+     ```python
+     _CONSOLE_SHORT_KEYS = {
+         "user_id": "user",
+         "scholar_id": "scholar",
+         "crawl_run_id": "run",
+         "run_id": "run",
+     }
+     ```
+   - Apply in the `for key in sorted(payload.keys())` loop
+
+4. Do NOT remove any WARNING or ERROR level logs. Only remove/demote DEBUG/INFO that are redundant.
+5. Do NOT change business logic.
+
+Commit message: "refactor: reduce log noise, improve event naming and console readability"
+```
+
+---
+
+## Slice 6: Add Ruff + Mypy to CI
+
+**Why:** No Python linting or type checking in CI. Code quality regressions go undetected.
+
+**Files to modify:**
+- `pyproject.toml`
+- `.github/workflows/ci.yml`
+
+**Prompt:**
+```
+You are working on the scholarr repository. Add Python linting and type checking to CI.
+
+1. In `pyproject.toml`, add ruff configuration:
+   ```toml
+   [tool.ruff]
+   target-version = "py312"
+   line-length = 120
+
+   [tool.ruff.lint]
+   select = ["E", "F", "W", "I", "UP", "B", "SIM", "RUF"]
+   ignore = ["E501"]
+
+   [tool.ruff.lint.isort]
+   known-first-party = ["app"]
+   ```
+
+2. In `pyproject.toml`, add mypy configuration:
+   ```toml
+   [tool.mypy]
+   python_version = "3.12"
+   ignore_missing_imports = true
+   check_untyped_defs = false
+   warn_unused_ignores = true
+   ```
+
+3. In `.github/workflows/ci.yml`, add a `lint` job after `repo-hygiene` and before `test`:
+   ```yaml
+   lint:
+     runs-on: ubuntu-latest
+     needs: [repo-hygiene]
+     steps:
+       - uses: actions/checkout@v4
+       - uses: actions/setup-python@v5
+         with:
+           python-version: "3.12"
+       - uses: astral-sh/setup-uv@v4
+       - run: uv sync --extra dev
+       - name: Ruff check
+         run: uv run ruff check .
+       - name: Ruff format check
+         run: uv run ruff format --check .
+       - name: Mypy
+         run: uv run mypy app/ --ignore-missing-imports
+   ```
+
+4. Update the `test` job's `needs` to include `lint`.
+
+5. Add `ruff` and `mypy` to dev dependencies in `pyproject.toml` under `[project.optional-dependencies]`.
+
+6. Run `ruff check .` locally and fix auto-fixable issues with `ruff check --fix .`. For unfixable issues, add targeted `noqa` comments only if fixing would change behavior.
+
+Commit message: "ci: add ruff linting and mypy type checking"
+```
+
+---
+
+## Slice 7: Add CodeQL + Dependabot
+
+**Why:** No security scanning. Missed vulnerabilities, no automated dependency updates.
+
+**Files to create:**
+- `.github/workflows/codeql.yml`
+- `.github/dependabot.yml`
+
+**Prompt:**
+```
+You are working on the scholarr repository. Add security scanning.
+
+1. Create `.github/workflows/codeql.yml`:
+   ```yaml
+   name: CodeQL
+
+   on:
+     push:
+       branches: [main]
+     pull_request:
+       branches: [main]
+     schedule:
+       - cron: "30 5 * * 1"
+
+   jobs:
+     analyze:
+       runs-on: ubuntu-latest
+       permissions:
+         security-events: write
+         contents: read
+       strategy:
+         matrix:
+           language: [python, javascript]
+       steps:
+         - uses: actions/checkout@v4
+         - uses: github/codeql-action/init@v3
+           with:
+             languages: ${{ matrix.language }}
+         - uses: github/codeql-action/autobuild@v3
+         - uses: github/codeql-action/analyze@v3
+   ```
+
+2. Create `.github/dependabot.yml`:
+   ```yaml
+   version: 2
+   updates:
+     - package-ecosystem: pip
+       directory: /
+       schedule:
+         interval: weekly
+     - package-ecosystem: npm
+       directory: /frontend
+       schedule:
+         interval: weekly
+     - package-ecosystem: github-actions
+       directory: /
+       schedule:
+         interval: weekly
+   ```
+
+Commit message: "ci: add CodeQL security scanning and Dependabot"
+```
+
+---
+
+## Slice 8: Fix Version Remnants + Adopt Semantic Release
+
+**Why:** Version `0.1.0` is hardcoded in 5 places (including `crossref/application.py:294`). `0.0.1` exists in `docs/website/package.json`. No CHANGELOG, no GitHub Releases.
+
+**Files to modify:**
+- `pyproject.toml`
+- `app/services/crossref/application.py`
+
+**Files to create:**
+- `.github/workflows/release.yml`
+
+**Prompt:**
+```
+You are working on the scholarr repository. Set up automated versioning with python-semantic-release.
+
+1. Fix hardcoded version in `app/services/crossref/application.py` (line ~294):
+   ```python
+   # Before:
+   etiquette = Etiquette(settings.app_name, "0.1.0", "https://scholarr.local", email)
+
+   # After:
+   from importlib.metadata import version as pkg_version
+   _APP_VERSION = pkg_version("scholarr")
+   # then in function:
+   etiquette = Etiquette(settings.app_name, _APP_VERSION, "https://scholarr.local", email)
+   ```
+
+2. In `pyproject.toml`, add semantic-release config:
+   ```toml
+   [tool.semantic_release]
+   version_toml = ["pyproject.toml:project.version"]
+   version_variables = ["frontend/package.json:version"]
+   branch = "main"
+   build_command = ""
+   commit_message = "chore(release): v{version}"
+   ```
+
+3. Create `.github/workflows/release.yml`:
+   ```yaml
+   name: Release
+
+   on:
+     push:
+       branches: [main]
+
+   jobs:
+     release:
+       runs-on: ubuntu-latest
+       if: github.repository == 'JustinZeus/scholarr'
+       permissions:
+         contents: write
+         id-token: write
+       steps:
+         - uses: actions/checkout@v4
+           with:
+             fetch-depth: 0
+             token: ${{ secrets.GITHUB_TOKEN }}
+         - uses: actions/setup-python@v5
+           with:
+             python-version: "3.12"
+         - run: pip install python-semantic-release
+         - name: Semantic Release
+           id: release
+           run: semantic-release publish
+           env:
+             GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+   ```
+
+4. Add `python-semantic-release` to dev dependencies in `pyproject.toml`.
+
+Note: `docs/website/package.json` version (`0.0.1`) is addressed in Slice 9 (docs rebuild). `frontend/package-lock.json` and `uv.lock` versions auto-update on next `npm install`/`uv sync`.
+
+Commit message: "ci: adopt python-semantic-release, fix hardcoded version strings"
+```
+
+---
+
+## Slice 9: Docs — Delete and Rebuild from Scratch
+
+**Why:** Current docs have split authority (two api-contract.md files), orphaned pages, stale version (`0.0.1`), and missing critical docs (testing, deployment, configuration reference). Clean slate is faster than migration.
+
+**Files to delete:**
+- `docs/` (entire directory)
+
+**Files to create:**
+- Complete new `docs/` tree
+
+**Prompt:**
+```
+You are working on the scholarr repository. Delete the entire `docs/` directory and rebuild from scratch.
+
+Target IA:
+
+docs/
+├── index.md                          # Landing page: what is scholarr, quick links
+├── user/
+│   ├── overview.md                   # What scholarr does, key concepts
+│   ├── getting-started.md            # Install (docker compose), first run, add first scholar
+│   └── configuration.md              # ALL env vars from .env.example, organized by category
+├── developer/
+│   ├── overview.md                   # Dev quickstart
+│   ├── architecture.md               # FastAPI + SQLAlchemy + Vue 3, domain service boundaries
+│   ├── local-development.md          # docker-compose.dev.yml, hot reload, running tests
+│   ├── contributing.md               # PR process, conventional commits, code standards from agents.md
+│   ├── ingestion.md                  # Ingestion pipeline: parsing, rate limiting, safety gates
+│   ├── frontend-theme-inventory.md   # Theme tokens, Tailwind integration
+│   └── testing.md                    # Test tiers (unit/integration/smoke), markers, fixtures, how to run
+├── operations/
+│   ├── overview.md                   # Ops quickstart
+│   ├── deployment.md                 # Production Docker, scaling, health checks
+│   ├── database-runbook.md           # Backup, restore, integrity checks, migration procedures
+│   ├── scrape-safety-runbook.md      # Rate limiting, cooldowns, CAPTCHA handling
+│   └── arxiv-runbook.md              # ArXiv rate limits, cache, query patterns
+├── reference/
+│   ├── overview.md                   # Reference index
+│   ├── api.md                        # CANONICAL: envelope spec + endpoints + DTO contract
+│   ├── environment.md                # Env var quick-reference (links to user/configuration.md)
+│   └── changelog.md                  # Placeholder: "auto-generated by semantic-release"
+└── website/
+    ├── docusaurus.config.js
+    ├── sidebars.js                   # ALL pages listed, no orphans
+    ├── package.json                  # version: "0.1.0"
+    └── src/css/custom.css
+
+Content guidelines:
+- Self-contained, scannable docs (headers, bullets, tables)
+- Frontmatter: `title`, `sidebar_position`
+- `configuration.md`: read `.env.example`, organize every variable with description, type, default, example
+- `testing.md`: pytest markers from pyproject.toml (`integration`, `db`, `migrations`, `schema`, `smoke`), container-based runner from agents.md, fixture org under tests/fixtures/
+- `api.md`: merge old `developer/api-contract.md` (DTO structure) and `reference/api-contract.md` (envelope spec). Include exact envelope shapes:
+  - Success: `{"data": ..., "meta": {"request_id": "..."}}`
+  - Error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}`
+  - Publication semantics (modes, pagination, identifiers)
+  - Scholar portability endpoints
+- `deployment.md`: practical Docker commands from docker-compose.yml
+- `database-runbook.md`: reference scripts/db/ (backup_full.sh, restore_dump.sh, check_integrity.py, repair_publication_links.py)
+
+Docusaurus config:
+- url: "https://justinzeus.github.io"
+- baseUrl: "/scholarr/"
+- organizationName: "JustinZeus"
+- projectName: "scholarr"
+- onBrokenLinks: "throw"
+- docs path: ".." (reads from docs/ parent)
+- Exclude: "website/**", "README.md"
+
+Commit message: "docs: rebuild documentation from scratch with clean IA"
+```
+
+---
+
+## Slice 10: Begin `ingestion/application.py` Decomposition
+
+**Why:** After logging cleanup (slices 3+5), the file is ~2,700-2,800 lines. Still the largest source file by 3x. This begins decomposition with the safest extract: safety gate logic.
+
+**Files to modify:**
+- `app/services/ingestion/application.py`
+
+**Files to create:**
+- `app/services/ingestion/safety.py`
+
+**Prompt:**
+```
+You are working on the scholarr repository. `app/services/ingestion/application.py` is ~2,700-2,800 lines (after logging cleanup). It contains the entire ingestion orchestration in a single class.
+
+Extract the FIRST logical chunk: safety gate logic.
+
+1. Read `app/services/ingestion/application.py` fully.
+
+2. Identify safety/cooldown methods:
+   - `_enforce_safety_gate`
+   - `_raise_safety_blocked_start`
+   - Cooldown activation/clearing methods
+   - Alert threshold evaluation methods
+
+3. Create `app/services/ingestion/safety.py`:
+   - Move identified methods to standalone functions or a small class
+   - Accept explicit parameters (no implicit self.xxx state)
+   - Keep same function signatures where possible
+
+4. Update `application.py` to import from `safety.py`.
+
+5. Run full test suite: `docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app pytest tests/`
+   All tests must pass unchanged.
+
+Constraints from `agents.md`:
+- Max 50 lines per function
+- No flat files in `app/services/` root — all within `app/services/ingestion/`
+- Fail fast, early returns, guard clauses
+
+Commit message: "refactor: extract ingestion safety gate logic to dedicated module"
+```
+
+---
+
+## Verification (after all slices)
+
+Run these checks to confirm everything is clean:
+
+```bash
+# 1. No tracked build artifacts
+git ls-files build/ | wc -l  # expect: 0
+
+# 2. No duplicated event keys in logging
+grep -rn '"event":' app/ --include="*.py" | grep -v __pycache__ | wc -l  # expect: 0
+
+# 3. No _log_ helper functions remain
+grep -rn 'def _log_' app/ --include="*.py" | wc -l  # expect: 0
+
+# 4. No stale version strings
+grep -rn '0\.0\.1' . --include="*.py" --include="*.json" --include="*.toml" \
+  | grep -v node_modules | grep -v .venv | grep -v uv.lock | wc -l  # expect: 0
+
+# 5. Docs build
+npm --prefix docs/website run build  # expect: success, no broken links
+
+# 6. All tests pass
+docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app pytest tests/
+```
+
+---
+
+## Slice Summary
+
+| # | Slice | Files Changed | Estimated Impact |
+|---|-------|---------------|------------------|
+| ~~1~~ | ~~Repo hygiene~~ | ~~120 files~~ | ~~-18,380 lines~~ **DONE** |
+| 2 | `structured_log()` utility | 2 files | +50 lines |
+| 3 | Migrate ingestion logging | 1 file | -300 lines |
+| 4 | Migrate remaining logging | ~20 files | -180 lines |
+| 5 | Logging noise/readability | ~5 files | -30 lines, better output |
+| 6 | Ruff + mypy in CI | 2 files | +50 lines config |
+| 7 | CodeQL + Dependabot | 2 files | +40 lines config |
+| 8 | Version fix + semantic-release | 3 files | +40 lines config |
+| 9 | Docs rebuild | full replacement | ~20 new files |
+| 10 | Ingestion decomposition | 2 files | net zero (move) |

agents.md

@@ -77,7 +77,7 @@ Types: `feat`, `fix`, `docs`, `ci`, `refactor`, `test`, `chore`, `perf`.
 
 ## 7. Testing & Quality Gates
 
-All **local development** commands run inside containers. Do not run bare `npm`, `pytest`, or `ruff` on the host. CI workflows (`.github/workflows/`) use bare runners with `uv` and `npm` directly — this is intentional since CI has no Docker-in-Docker dependency.
+All commands run inside containers. Do not run bare `npm`, `pytest`, or `ruff` on the host.
 
 ### Backend
 

app/api/router.py

@@ -2,12 +2,11 @@ from __future__ import annotations
 
 from fastapi import APIRouter
 
-from app.api.routers import admin, admin_dbops, admin_settings, auth, publications, runs, scholars, settings
+from app.api.routers import admin, admin_dbops, auth, publications, runs, scholars, settings
 
 router = APIRouter(prefix="/api/v1")
 router.include_router(auth.router)
 router.include_router(admin.router)
-router.include_router(admin_settings.router)
 router.include_router(admin_dbops.router)
 router.include_router(scholars.router)
 router.include_router(settings.router)

app/api/routers/admin_dbops.py

@@ -5,7 +5,7 @@ import logging
 from fastapi import APIRouter, Depends, Path, Query, Request
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.api.deps import get_api_admin_user, get_api_current_user
+from app.api.deps import get_api_admin_user
 from app.api.errors import ApiException
 from app.api.responses import success_payload
 from app.api.schemas import (
@@ -57,7 +57,7 @@ def _requested_by_value(*, payload, admin_user: User) -> str:
     return from_payload or admin_user.email
 
 
-def _serialize_pdf_queue_item(item, *, is_admin: bool = False) -> dict[str, object]:
+def _serialize_pdf_queue_item(item) -> dict[str, object]:
     return {
         "publication_id": item.publication_id,
         "title": item.title,
@@ -69,7 +69,7 @@ def _serialize_pdf_queue_item(item, *, is_admin: bool = False) -> dict[str, obje
         "last_failure_detail": item.last_failure_detail,
         "last_source": item.last_source,
         "requested_by_user_id": item.requested_by_user_id,
-        "requested_by_email": item.requested_by_email if is_admin else None,
+        "requested_by_email": item.requested_by_email,
         "queued_at": item.queued_at,
         "last_attempt_at": item.last_attempt_at,
         "resolved_at": item.resolved_at,
@@ -185,7 +185,7 @@ async def get_pdf_queue(
     offset: int | None = Query(default=None, ge=0),
     status: str | None = Query(default=None),
     db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
+    admin_user: User = Depends(get_api_admin_user),
 ):
     normalized_status = (status or "").strip().lower() or None
     resolved_page, resolved_limit, resolved_offset = _resolve_pdf_queue_paging(
@@ -204,7 +204,7 @@ async def get_pdf_queue(
         logger,
         "info",
         "api.admin.db.pdf_queue_listed",
-        user_id=int(current_user.id),
+        admin_user_id=int(admin_user.id),
         page=int(resolved_page),
         page_size=int(resolved_limit),
         offset=int(resolved_offset),
@@ -215,7 +215,7 @@ async def get_pdf_queue(
     return success_payload(
         request,
         data={
-            "items": [_serialize_pdf_queue_item(item, is_admin=current_user.is_admin) for item in queue_page.items],
+            "items": [_serialize_pdf_queue_item(item) for item in queue_page.items],
             **_pdf_queue_page_data(
                 total_count=queue_page.total_count,
                 page=resolved_page,

app/api/routers/admin_settings.py

@@ -1,104 +0,0 @@
-from __future__ import annotations
-
-import logging
-
-from fastapi import APIRouter, Depends, Request
-
-from app.api.deps import get_api_admin_user
-from app.api.errors import ApiException
-from app.api.responses import success_payload
-from app.api.schemas import (
-    AdminScholarHttpSettingsEnvelope,
-    AdminScholarHttpSettingsUpdateRequest,
-)
-from app.db.models import User
-from app.logging_utils import structured_log
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/admin/settings", tags=["api-admin-settings"])
-_MAX_USER_AGENT_LENGTH = 512
-_MAX_ACCEPT_LANGUAGE_LENGTH = 128
-_MAX_COOKIE_LENGTH = 8192
-
-
-def _normalize_header_value(value: str, *, field_name: str, max_length: int) -> str:
-    normalized = value.strip()
-    if len(normalized) > max_length:
-        raise ApiException(
-            status_code=400,
-            code="invalid_admin_setting",
-            message=f"{field_name} must be {max_length} characters or fewer.",
-        )
-    return normalized
-
-
-def _serialize_scholar_http_settings() -> dict[str, object]:
-    return {
-        "user_agent": settings.scholar_http_user_agent,
-        "rotate_user_agent": bool(settings.scholar_http_rotate_user_agent),
-        "accept_language": settings.scholar_http_accept_language,
-        "cookie": settings.scholar_http_cookie,
-    }
-
-
-def _apply_scholar_http_settings(payload: AdminScholarHttpSettingsUpdateRequest) -> None:
-    user_agent = _normalize_header_value(
-        payload.user_agent,
-        field_name="user_agent",
-        max_length=_MAX_USER_AGENT_LENGTH,
-    )
-    accept_language = _normalize_header_value(
-        payload.accept_language,
-        field_name="accept_language",
-        max_length=_MAX_ACCEPT_LANGUAGE_LENGTH,
-    )
-    cookie = _normalize_header_value(
-        payload.cookie or "",
-        field_name="cookie",
-        max_length=_MAX_COOKIE_LENGTH,
-    )
-    object.__setattr__(settings, "scholar_http_user_agent", user_agent)
-    object.__setattr__(settings, "scholar_http_rotate_user_agent", bool(payload.rotate_user_agent))
-    object.__setattr__(settings, "scholar_http_accept_language", accept_language)
-    object.__setattr__(settings, "scholar_http_cookie", cookie)
-
-
-@router.get(
-    "/scholar-http",
-    response_model=AdminScholarHttpSettingsEnvelope,
-)
-async def get_scholar_http_settings(
-    request: Request,
-    admin_user: User = Depends(get_api_admin_user),
-):
-    structured_log(
-        logger,
-        "info",
-        "api.admin.settings.scholar_http_read",
-        admin_user_id=int(admin_user.id),
-    )
-    return success_payload(request, data=_serialize_scholar_http_settings())
-
-
-@router.put(
-    "/scholar-http",
-    response_model=AdminScholarHttpSettingsEnvelope,
-)
-async def update_scholar_http_settings(
-    payload: AdminScholarHttpSettingsUpdateRequest,
-    request: Request,
-    admin_user: User = Depends(get_api_admin_user),
-):
-    _apply_scholar_http_settings(payload)
-    structured_log(
-        logger,
-        "info",
-        "api.admin.settings.scholar_http_updated",
-        admin_user_id=int(admin_user.id),
-        rotate_user_agent=bool(settings.scholar_http_rotate_user_agent),
-        user_agent_set=bool(settings.scholar_http_user_agent.strip()),
-        cookie_set=bool(settings.scholar_http_cookie.strip()),
-    )
-    return success_payload(request, data=_serialize_scholar_http_settings())

app/api/routers/run_manual.py

@@ -1,243 +0,0 @@
-from __future__ import annotations
-
-import logging
-from typing import Any, NoReturn
-
-from fastapi import Request
-from sqlalchemy.exc import IntegrityError
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.api.errors import ApiException
-from app.api.responses import success_payload
-from app.api.routers.run_serializers import manual_run_payload_from_run
-from app.db.models import RunStatus, RunTriggerType
-from app.logging_utils import structured_log
-from app.services.ingestion import application as ingestion_service
-from app.services.ingestion import safety as run_safety_service
-from app.services.runs import application as run_service
-from app.services.settings import application as user_settings_service
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-
-async def load_safety_state(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-) -> dict[str, Any]:
-    user_settings = await user_settings_service.get_or_create_settings(
-        db_session,
-        user_id=user_id,
-    )
-    previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
-    if run_safety_service.clear_expired_cooldown(user_settings):
-        await db_session.commit()
-        await db_session.refresh(user_settings)
-        structured_log(
-            logger,
-            "info",
-            "api.runs.safety_cooldown_cleared",
-            user_id=user_id,
-            reason=previous_safety_state.get("cooldown_reason"),
-            cooldown_until=previous_safety_state.get("cooldown_until"),
-        )
-    return run_safety_service.get_safety_state_payload(user_settings)
-
-
-def raise_manual_runs_disabled(*, user_id: int, safety_state: dict[str, Any]) -> None:
-    structured_log(
-        logger,
-        "warning",
-        "api.runs.manual_blocked_policy",
-        user_id=user_id,
-        policy={"manual_run_allowed": False},
-        safety_state=safety_state,
-    )
-    raise ApiException(
-        status_code=403,
-        code="manual_runs_disabled",
-        message="Manual checks are disabled by server policy.",
-        details={
-            "policy": {"manual_run_allowed": False},
-            "safety_state": safety_state,
-        },
-    )
-
-
-async def reused_manual_run_payload(
-    db_session: AsyncSession,
-    *,
-    request: Request,
-    user_id: int,
-    idempotency_key: str | None,
-    safety_state: dict[str, Any],
-) -> dict[str, Any] | None:
-    if idempotency_key is None:
-        return None
-    previous_run = await run_service.get_manual_run_by_idempotency_key(
-        db_session,
-        user_id=user_id,
-        idempotency_key=idempotency_key,
-    )
-    if previous_run is None:
-        return None
-    if previous_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING):
-        raise ApiException(
-            status_code=409,
-            code="run_in_progress",
-            message="A run with this idempotency key is still in progress.",
-            details={"run_id": int(previous_run.id), "idempotency_key": idempotency_key},
-        )
-    return success_payload(
-        request,
-        data=manual_run_payload_from_run(
-            previous_run,
-            idempotency_key=idempotency_key,
-            reused_existing_run=True,
-            safety_state=safety_state,
-        ),
-    )
-
-
-async def run_ingestion_for_manual(
-    db_session: AsyncSession,
-    *,
-    ingest_service: ingestion_service.ScholarIngestionService,
-    user_id: int,
-    idempotency_key: str | None,
-):
-    user_settings = await user_settings_service.get_or_create_settings(
-        db_session,
-        user_id=user_id,
-    )
-    return await ingest_service.run_for_user(
-        db_session,
-        user_id=user_id,
-        trigger_type=RunTriggerType.MANUAL,
-        request_delay_seconds=user_settings.request_delay_seconds,
-        network_error_retries=settings.ingestion_network_error_retries,
-        retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
-        max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
-        page_size=settings.ingestion_page_size,
-        auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
-        queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
-        idempotency_key=idempotency_key,
-        alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
-        alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
-        alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
-    )
-
-
-async def recover_integrity_error(
-    db_session: AsyncSession,
-    *,
-    request: Request,
-    user_id: int,
-    idempotency_key: str | None,
-    original_exc: IntegrityError,
-) -> dict[str, Any]:
-    if idempotency_key is None:
-        structured_log(
-            logger,
-            "exception",
-            "api.runs.manual_integrity_error",
-            user_id=user_id,
-        )
-        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
-    existing_run = await run_service.get_manual_run_by_idempotency_key(
-        db_session,
-        user_id=user_id,
-        idempotency_key=idempotency_key,
-    )
-    if existing_run is None:
-        structured_log(
-            logger,
-            "exception",
-            "api.runs.manual_integrity_error",
-            user_id=user_id,
-        )
-        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
-    if existing_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING):
-        raise ApiException(
-            status_code=409,
-            code="run_in_progress",
-            message="A run with this idempotency key is still in progress.",
-            details={"run_id": int(existing_run.id), "idempotency_key": idempotency_key},
-        ) from original_exc
-    return success_payload(
-        request,
-        data=manual_run_payload_from_run(
-            existing_run,
-            idempotency_key=idempotency_key,
-            reused_existing_run=True,
-            safety_state=await load_safety_state(db_session, user_id=user_id),
-        ),
-    )
-
-
-async def execute_manual_run(
-    db_session: AsyncSession,
-    *,
-    request: Request,
-    ingest_service: ingestion_service.ScholarIngestionService,
-    user_id: int,
-    idempotency_key: str | None,
-):
-    try:
-        return await run_ingestion_for_manual(
-            db_session,
-            ingest_service=ingest_service,
-            user_id=user_id,
-            idempotency_key=idempotency_key,
-        )
-    except ingestion_service.RunAlreadyInProgressError as exc:
-        await db_session.rollback()
-        raise ApiException(
-            status_code=409,
-            code="run_in_progress",
-            message="A run is already in progress for this account.",
-        ) from exc
-    except ingestion_service.RunBlockedBySafetyPolicyError as exc:
-        await db_session.rollback()
-        raise_manual_blocked_safety(exc=exc, user_id=user_id)
-    except IntegrityError as exc:
-        await db_session.rollback()
-        return await recover_integrity_error(
-            db_session,
-            request=request,
-            user_id=user_id,
-            idempotency_key=idempotency_key,
-            original_exc=exc,
-        )
-    except Exception as exc:
-        await db_session.rollback()
-        raise_manual_failed(exc=exc, user_id=user_id)
-
-
-def raise_manual_blocked_safety(*, exc, user_id: int) -> NoReturn:
-    structured_log(
-        logger,
-        "info",
-        "api.runs.manual_blocked_safety",
-        user_id=user_id,
-        reason=exc.safety_state.get("cooldown_reason"),
-        cooldown_until=exc.safety_state.get("cooldown_until"),
-        cooldown_remaining_seconds=exc.safety_state.get("cooldown_remaining_seconds"),
-    )
-    raise ApiException(
-        status_code=429,
-        code=exc.code,
-        message=exc.message,
-        details={"safety_state": exc.safety_state},
-    ) from exc
-
-
-def raise_manual_failed(*, exc: Exception, user_id: int) -> NoReturn:
-    structured_log(
-        logger,
-        "exception",
-        "api.runs.manual_failed",
-        user_id=user_id,
-    )
-    raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from exc

app/api/routers/run_serializers.py

@@ -1,238 +0,0 @@
-from __future__ import annotations
-
-import re
-from typing import Any
-
-from app.api.errors import ApiException
-from app.services.runs import application as run_service
-
-IDEMPOTENCY_HEADER = "Idempotency-Key"
-IDEMPOTENCY_MAX_LENGTH = 128
-IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
-
-
-def _int_value(value: Any, default: int = 0) -> int:
-    try:
-        return int(value)
-    except (TypeError, ValueError):
-        return default
-
-
-def _str_value(value: Any) -> str | None:
-    if value is None:
-        return None
-    text = str(value).strip()
-    return text if text else None
-
-
-def _bool_value(value: Any, default: bool = False) -> bool:
-    if isinstance(value, bool):
-        return value
-    if isinstance(value, str):
-        lowered = value.strip().lower()
-        if lowered in {"1", "true", "yes", "on"}:
-            return True
-        if lowered in {"0", "false", "no", "off"}:
-            return False
-    return default
-
-
-def normalize_idempotency_key(raw_value: str | None) -> str | None:
-    if raw_value is None:
-        return None
-    candidate = raw_value.strip()
-    if not candidate:
-        return None
-    if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate):
-        raise ApiException(
-            status_code=400,
-            code="invalid_idempotency_key",
-            message=("Invalid Idempotency-Key. Use 8-128 characters from: A-Z a-z 0-9 . _ : -"),
-        )
-    return candidate
-
-
-def serialize_run(run) -> dict[str, Any]:
-    summary = run_service.extract_run_summary(run.error_log)
-    return {
-        "id": int(run.id),
-        "trigger_type": run.trigger_type.value,
-        "status": run.status.value,
-        "start_dt": run.start_dt,
-        "end_dt": run.end_dt,
-        "scholar_count": int(run.scholar_count or 0),
-        "new_publication_count": int(run.new_pub_count or 0),
-        "failed_count": int(summary["failed_count"]),
-        "partial_count": int(summary["partial_count"]),
-    }
-
-
-def serialize_queue_item(item) -> dict[str, Any]:
-    return {
-        "id": int(item.id),
-        "scholar_profile_id": int(item.scholar_profile_id),
-        "scholar_label": item.scholar_label,
-        "status": item.status,
-        "reason": item.reason,
-        "dropped_reason": item.dropped_reason,
-        "attempt_count": int(item.attempt_count),
-        "resume_cstart": int(item.resume_cstart),
-        "next_attempt_dt": item.next_attempt_dt,
-        "updated_at": item.updated_at,
-        "last_error": item.last_error,
-        "last_run_id": item.last_run_id,
-    }
-
-
-def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]:
-    if not isinstance(value, list):
-        return []
-    normalized: list[dict[str, Any]] = []
-    for item in value:
-        if not isinstance(item, dict):
-            continue
-        normalized.append(
-            {
-                "attempt": _int_value(item.get("attempt"), 0),
-                "cstart": _int_value(item.get("cstart"), 0),
-                "state": _str_value(item.get("state")),
-                "state_reason": _str_value(item.get("state_reason")),
-                "status_code": (_int_value(item.get("status_code")) if item.get("status_code") is not None else None),
-                "fetch_error": _str_value(item.get("fetch_error")),
-            }
-        )
-    return normalized
-
-
-def _normalize_page_logs(value: Any) -> list[dict[str, Any]]:
-    if not isinstance(value, list):
-        return []
-    normalized: list[dict[str, Any]] = []
-    for item in value:
-        if not isinstance(item, dict):
-            continue
-        warning_codes = item.get("warning_codes")
-        normalized.append(
-            {
-                "page": _int_value(item.get("page"), 0),
-                "cstart": _int_value(item.get("cstart"), 0),
-                "state": _str_value(item.get("state")) or "unknown",
-                "state_reason": _str_value(item.get("state_reason")),
-                "status_code": (_int_value(item.get("status_code")) if item.get("status_code") is not None else None),
-                "publication_count": _int_value(item.get("publication_count"), 0),
-                "attempt_count": _int_value(item.get("attempt_count"), 0),
-                "has_show_more_button": _bool_value(item.get("has_show_more_button"), False),
-                "articles_range": _str_value(item.get("articles_range")),
-                "warning_codes": [str(code) for code in (warning_codes if isinstance(warning_codes, list) else [])],
-            }
-        )
-    return normalized
-
-
-def _normalize_debug(value: Any) -> dict[str, Any] | None:
-    if not isinstance(value, dict):
-        return None
-    marker_counts = value.get("marker_counts_nonzero")
-    warning_codes = value.get("warning_codes")
-    return {
-        "status_code": (_int_value(value.get("status_code")) if value.get("status_code") is not None else None),
-        "final_url": _str_value(value.get("final_url")),
-        "fetch_error": _str_value(value.get("fetch_error")),
-        "body_sha256": _str_value(value.get("body_sha256")),
-        "body_length": (_int_value(value.get("body_length")) if value.get("body_length") is not None else None),
-        "has_show_more_button": (
-            _bool_value(value.get("has_show_more_button"), False)
-            if value.get("has_show_more_button") is not None
-            else None
-        ),
-        "articles_range": _str_value(value.get("articles_range")),
-        "state_reason": _str_value(value.get("state_reason")),
-        "warning_codes": [str(code) for code in (warning_codes if isinstance(warning_codes, list) else [])],
-        "marker_counts_nonzero": {
-            str(key): _int_value(count, 0)
-            for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else [])
-        },
-        "page_logs": _normalize_page_logs(value.get("page_logs")),
-        "attempt_log": _normalize_attempt_log(value.get("attempt_log")),
-    }
-
-
-def normalize_scholar_result(value: Any) -> dict[str, Any]:
-    if not isinstance(value, dict):
-        return {
-            "scholar_profile_id": 0,
-            "scholar_id": "unknown",
-            "state": "unknown",
-            "state_reason": None,
-            "outcome": "failed",
-            "attempt_count": 0,
-            "publication_count": 0,
-            "start_cstart": 0,
-            "continuation_cstart": None,
-            "continuation_enqueued": False,
-            "continuation_cleared": False,
-            "warnings": [],
-            "error": None,
-            "debug": None,
-        }
-    warnings = value.get("warnings")
-    return {
-        "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0),
-        "scholar_id": _str_value(value.get("scholar_id")) or "unknown",
-        "state": _str_value(value.get("state")) or "unknown",
-        "state_reason": _str_value(value.get("state_reason")),
-        "outcome": _str_value(value.get("outcome")) or "failed",
-        "attempt_count": _int_value(value.get("attempt_count"), 0),
-        "publication_count": _int_value(value.get("publication_count"), 0),
-        "start_cstart": _int_value(value.get("start_cstart"), 0),
-        "continuation_cstart": (
-            _int_value(value.get("continuation_cstart")) if value.get("continuation_cstart") is not None else None
-        ),
-        "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False),
-        "continuation_cleared": _bool_value(value.get("continuation_cleared"), False),
-        "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])],
-        "error": _str_value(value.get("error")),
-        "debug": _normalize_debug(value.get("debug")),
-    }
-
-
-def manual_run_payload_from_run(
-    run,
-    *,
-    idempotency_key: str | None,
-    reused_existing_run: bool,
-    safety_state: dict[str, Any],
-) -> dict[str, Any]:
-    summary = run_service.extract_run_summary(run.error_log)
-    return {
-        "run_id": int(run.id),
-        "status": run.status.value,
-        "scholar_count": int(run.scholar_count or 0),
-        "succeeded_count": int(summary["succeeded_count"]),
-        "failed_count": int(summary["failed_count"]),
-        "partial_count": int(summary["partial_count"]),
-        "new_publication_count": int(run.new_pub_count or 0),
-        "reused_existing_run": reused_existing_run,
-        "idempotency_key": idempotency_key,
-        "safety_state": safety_state,
-    }
-
-
-def manual_run_success_payload(
-    *,
-    run_summary,
-    idempotency_key: str | None,
-    safety_state: dict[str, Any],
-) -> dict[str, Any]:
-    return {
-        "run_id": run_summary.crawl_run_id,
-        "status": run_summary.status.value,
-        "scholar_count": run_summary.scholar_count,
-        "succeeded_count": run_summary.succeeded_count,
-        "failed_count": run_summary.failed_count,
-        "partial_count": run_summary.partial_count,
-        "new_publication_count": run_summary.new_publication_count,
-        "reused_existing_run": False,
-        "idempotency_key": idempotency_key,
-        "safety_state": safety_state,
-    }

app/api/routers/runs.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import re
 from typing import Any
 
 from fastapi import APIRouter, Depends, Query, Request
@@ -12,21 +13,6 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from app.api.deps import get_api_current_user
 from app.api.errors import ApiException
 from app.api.responses import success_payload
-from app.api.routers.run_manual import (
-    load_safety_state,
-    raise_manual_blocked_safety,
-    raise_manual_failed,
-    raise_manual_runs_disabled,
-    recover_integrity_error,
-    reused_manual_run_payload,
-)
-from app.api.routers.run_serializers import (
-    IDEMPOTENCY_HEADER,
-    normalize_idempotency_key,
-    normalize_scholar_result,
-    serialize_queue_item,
-    serialize_run,
-)
 from app.api.runtime_deps import get_ingestion_service
 from app.api.schemas import (
     ManualRunEnvelope,
@@ -37,9 +23,10 @@ from app.api.schemas import (
     RunsListEnvelope,
 )
 from app.db.models import RunStatus, RunTriggerType, User
-from app.db.session import get_db_session, get_session_factory
+from app.db.session import get_db_session
 from app.logging_utils import structured_log
 from app.services.ingestion import application as ingestion_service
+from app.services.ingestion import safety as run_safety_service
 from app.services.runs import application as run_service
 from app.services.runs.events import event_generator
 from app.services.settings import application as user_settings_service
@@ -48,18 +35,456 @@ from app.settings import settings
 logger = logging.getLogger(__name__)
 _background_tasks: set[asyncio.Task[Any]] = set()
 
-
-def _drop_finished_task(task: asyncio.Task[Any]) -> None:
-    _background_tasks.discard(task)
-    try:
-        task.result()
-    except Exception:
-        structured_log(logger, "exception", "runs.background_task_failed")
-
-
 router = APIRouter(prefix="/runs", tags=["api-runs"])
 ACTIVE_RUN_STATUSES = {RunStatus.RUNNING, RunStatus.RESOLVING}
 
+IDEMPOTENCY_HEADER = "Idempotency-Key"
+IDEMPOTENCY_MAX_LENGTH = 128
+IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
+
+
+def _int_value(value: Any, default: int = 0) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _str_value(value: Any) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text if text else None
+
+
+def _bool_value(value: Any, default: bool = False) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        lowered = value.strip().lower()
+        if lowered in {"1", "true", "yes", "on"}:
+            return True
+        if lowered in {"0", "false", "no", "off"}:
+            return False
+    return default
+
+
+def _normalize_idempotency_key(raw_value: str | None) -> str | None:
+    if raw_value is None:
+        return None
+    candidate = raw_value.strip()
+    if not candidate:
+        return None
+    if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate):
+        raise ApiException(
+            status_code=400,
+            code="invalid_idempotency_key",
+            message=("Invalid Idempotency-Key. Use 8-128 characters from: A-Z a-z 0-9 . _ : -"),
+        )
+    return candidate
+
+
+def _serialize_run(run) -> dict[str, Any]:
+    summary = run_service.extract_run_summary(run.error_log)
+    return {
+        "id": int(run.id),
+        "trigger_type": run.trigger_type.value,
+        "status": run.status.value,
+        "start_dt": run.start_dt,
+        "end_dt": run.end_dt,
+        "scholar_count": int(run.scholar_count or 0),
+        "new_publication_count": int(run.new_pub_count or 0),
+        "failed_count": int(summary["failed_count"]),
+        "partial_count": int(summary["partial_count"]),
+    }
+
+
+def _serialize_queue_item(item) -> dict[str, Any]:
+    return {
+        "id": int(item.id),
+        "scholar_profile_id": int(item.scholar_profile_id),
+        "scholar_label": item.scholar_label,
+        "status": item.status,
+        "reason": item.reason,
+        "dropped_reason": item.dropped_reason,
+        "attempt_count": int(item.attempt_count),
+        "resume_cstart": int(item.resume_cstart),
+        "next_attempt_dt": item.next_attempt_dt,
+        "updated_at": item.updated_at,
+        "last_error": item.last_error,
+        "last_run_id": item.last_run_id,
+    }
+
+
+def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]:
+    if not isinstance(value, list):
+        return []
+    normalized: list[dict[str, Any]] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        normalized.append(
+            {
+                "attempt": _int_value(item.get("attempt"), 0),
+                "cstart": _int_value(item.get("cstart"), 0),
+                "state": _str_value(item.get("state")),
+                "state_reason": _str_value(item.get("state_reason")),
+                "status_code": (_int_value(item.get("status_code")) if item.get("status_code") is not None else None),
+                "fetch_error": _str_value(item.get("fetch_error")),
+            }
+        )
+    return normalized
+
+
+def _normalize_page_logs(value: Any) -> list[dict[str, Any]]:
+    if not isinstance(value, list):
+        return []
+    normalized: list[dict[str, Any]] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        warning_codes = item.get("warning_codes")
+        normalized.append(
+            {
+                "page": _int_value(item.get("page"), 0),
+                "cstart": _int_value(item.get("cstart"), 0),
+                "state": _str_value(item.get("state")) or "unknown",
+                "state_reason": _str_value(item.get("state_reason")),
+                "status_code": (_int_value(item.get("status_code")) if item.get("status_code") is not None else None),
+                "publication_count": _int_value(item.get("publication_count"), 0),
+                "attempt_count": _int_value(item.get("attempt_count"), 0),
+                "has_show_more_button": _bool_value(item.get("has_show_more_button"), False),
+                "articles_range": _str_value(item.get("articles_range")),
+                "warning_codes": [str(code) for code in (warning_codes if isinstance(warning_codes, list) else [])],
+            }
+        )
+    return normalized
+
+
+def _normalize_debug(value: Any) -> dict[str, Any] | None:
+    if not isinstance(value, dict):
+        return None
+    marker_counts = value.get("marker_counts_nonzero")
+    warning_codes = value.get("warning_codes")
+    return {
+        "status_code": (_int_value(value.get("status_code")) if value.get("status_code") is not None else None),
+        "final_url": _str_value(value.get("final_url")),
+        "fetch_error": _str_value(value.get("fetch_error")),
+        "body_sha256": _str_value(value.get("body_sha256")),
+        "body_length": (_int_value(value.get("body_length")) if value.get("body_length") is not None else None),
+        "has_show_more_button": (
+            _bool_value(value.get("has_show_more_button"), False)
+            if value.get("has_show_more_button") is not None
+            else None
+        ),
+        "articles_range": _str_value(value.get("articles_range")),
+        "state_reason": _str_value(value.get("state_reason")),
+        "warning_codes": [str(code) for code in (warning_codes if isinstance(warning_codes, list) else [])],
+        "marker_counts_nonzero": {
+            str(key): _int_value(count, 0)
+            for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else [])
+        },
+        "page_logs": _normalize_page_logs(value.get("page_logs")),
+        "attempt_log": _normalize_attempt_log(value.get("attempt_log")),
+    }
+
+
+def _normalize_scholar_result(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {
+            "scholar_profile_id": 0,
+            "scholar_id": "unknown",
+            "state": "unknown",
+            "state_reason": None,
+            "outcome": "failed",
+            "attempt_count": 0,
+            "publication_count": 0,
+            "start_cstart": 0,
+            "continuation_cstart": None,
+            "continuation_enqueued": False,
+            "continuation_cleared": False,
+            "warnings": [],
+            "error": None,
+            "debug": None,
+        }
+    warnings = value.get("warnings")
+    return {
+        "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0),
+        "scholar_id": _str_value(value.get("scholar_id")) or "unknown",
+        "state": _str_value(value.get("state")) or "unknown",
+        "state_reason": _str_value(value.get("state_reason")),
+        "outcome": _str_value(value.get("outcome")) or "failed",
+        "attempt_count": _int_value(value.get("attempt_count"), 0),
+        "publication_count": _int_value(value.get("publication_count"), 0),
+        "start_cstart": _int_value(value.get("start_cstart"), 0),
+        "continuation_cstart": (
+            _int_value(value.get("continuation_cstart")) if value.get("continuation_cstart") is not None else None
+        ),
+        "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False),
+        "continuation_cleared": _bool_value(value.get("continuation_cleared"), False),
+        "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])],
+        "error": _str_value(value.get("error")),
+        "debug": _normalize_debug(value.get("debug")),
+    }
+
+
+def _manual_run_payload_from_run(
+    run,
+    *,
+    idempotency_key: str | None,
+    reused_existing_run: bool,
+    safety_state: dict[str, Any],
+) -> dict[str, Any]:
+    summary = run_service.extract_run_summary(run.error_log)
+    return {
+        "run_id": int(run.id),
+        "status": run.status.value,
+        "scholar_count": int(run.scholar_count or 0),
+        "succeeded_count": int(summary["succeeded_count"]),
+        "failed_count": int(summary["failed_count"]),
+        "partial_count": int(summary["partial_count"]),
+        "new_publication_count": int(run.new_pub_count or 0),
+        "reused_existing_run": reused_existing_run,
+        "idempotency_key": idempotency_key,
+        "safety_state": safety_state,
+    }
+
+
+async def _load_safety_state(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> dict[str, Any]:
+    user_settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=user_id,
+    )
+    previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
+    if run_safety_service.clear_expired_cooldown(user_settings):
+        await db_session.commit()
+        await db_session.refresh(user_settings)
+        structured_log(
+            logger,
+            "info",
+            "api.runs.safety_cooldown_cleared",
+            user_id=user_id,
+            reason=previous_safety_state.get("cooldown_reason"),
+            cooldown_until=previous_safety_state.get("cooldown_until"),
+        )
+    return run_safety_service.get_safety_state_payload(user_settings)
+
+
+def _raise_manual_runs_disabled(*, user_id: int, safety_state: dict[str, Any]) -> None:
+    structured_log(
+        logger,
+        "warning",
+        "api.runs.manual_blocked_policy",
+        user_id=user_id,
+        policy={"manual_run_allowed": False},
+        safety_state=safety_state,
+    )
+    raise ApiException(
+        status_code=403,
+        code="manual_runs_disabled",
+        message="Manual checks are disabled by server policy.",
+        details={
+            "policy": {"manual_run_allowed": False},
+            "safety_state": safety_state,
+        },
+    )
+
+
+async def _reused_manual_run_payload(
+    db_session: AsyncSession,
+    *,
+    request: Request,
+    user_id: int,
+    idempotency_key: str | None,
+    safety_state: dict[str, Any],
+) -> dict[str, Any] | None:
+    if idempotency_key is None:
+        return None
+    previous_run = await run_service.get_manual_run_by_idempotency_key(
+        db_session,
+        user_id=user_id,
+        idempotency_key=idempotency_key,
+    )
+    if previous_run is None:
+        return None
+    if previous_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING):
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run with this idempotency key is still in progress.",
+            details={"run_id": int(previous_run.id), "idempotency_key": idempotency_key},
+        )
+    return success_payload(
+        request,
+        data=_manual_run_payload_from_run(
+            previous_run,
+            idempotency_key=idempotency_key,
+            reused_existing_run=True,
+            safety_state=safety_state,
+        ),
+    )
+
+
+async def _run_ingestion_for_manual(
+    db_session: AsyncSession,
+    *,
+    ingest_service: ingestion_service.ScholarIngestionService,
+    user_id: int,
+    idempotency_key: str | None,
+):
+    user_settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=user_id,
+    )
+    return await ingest_service.run_for_user(
+        db_session,
+        user_id=user_id,
+        trigger_type=RunTriggerType.MANUAL,
+        request_delay_seconds=user_settings.request_delay_seconds,
+        network_error_retries=settings.ingestion_network_error_retries,
+        retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+        max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
+        page_size=settings.ingestion_page_size,
+        auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
+        queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
+        idempotency_key=idempotency_key,
+        alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
+        alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
+        alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
+    )
+
+
+async def _recover_integrity_error(
+    db_session: AsyncSession,
+    *,
+    request: Request,
+    user_id: int,
+    idempotency_key: str | None,
+    original_exc: IntegrityError,
+) -> dict[str, Any]:
+    if idempotency_key is None:
+        logger.exception(
+            "api.runs.manual_integrity_error",
+            extra={"user_id": user_id},
+        )
+        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
+    existing_run = await run_service.get_manual_run_by_idempotency_key(
+        db_session,
+        user_id=user_id,
+        idempotency_key=idempotency_key,
+    )
+    if existing_run is None:
+        logger.exception(
+            "api.runs.manual_integrity_error",
+            extra={"user_id": user_id},
+        )
+        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
+    if existing_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING):
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run with this idempotency key is still in progress.",
+            details={"run_id": int(existing_run.id), "idempotency_key": idempotency_key},
+        ) from original_exc
+    return success_payload(
+        request,
+        data=_manual_run_payload_from_run(
+            existing_run,
+            idempotency_key=idempotency_key,
+            reused_existing_run=True,
+            safety_state=await _load_safety_state(db_session, user_id=user_id),
+        ),
+    )
+
+
+async def _execute_manual_run(
+    db_session: AsyncSession,
+    *,
+    request: Request,
+    ingest_service: ingestion_service.ScholarIngestionService,
+    user_id: int,
+    idempotency_key: str | None,
+):
+    try:
+        return await _run_ingestion_for_manual(
+            db_session,
+            ingest_service=ingest_service,
+            user_id=user_id,
+            idempotency_key=idempotency_key,
+        )
+    except ingestion_service.RunAlreadyInProgressError as exc:
+        await db_session.rollback()
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run is already in progress for this account.",
+        ) from exc
+    except ingestion_service.RunBlockedBySafetyPolicyError as exc:
+        await db_session.rollback()
+        _raise_manual_blocked_safety(exc=exc, user_id=user_id)
+    except IntegrityError as exc:
+        await db_session.rollback()
+        return await _recover_integrity_error(
+            db_session,
+            request=request,
+            user_id=user_id,
+            idempotency_key=idempotency_key,
+            original_exc=exc,
+        )
+    except Exception as exc:
+        await db_session.rollback()
+        _raise_manual_failed(exc=exc, user_id=user_id)
+
+
+def _raise_manual_blocked_safety(*, exc, user_id: int) -> None:
+    structured_log(
+        logger,
+        "info",
+        "api.runs.manual_blocked_safety",
+        user_id=user_id,
+        reason=exc.safety_state.get("cooldown_reason"),
+        cooldown_until=exc.safety_state.get("cooldown_until"),
+        cooldown_remaining_seconds=exc.safety_state.get("cooldown_remaining_seconds"),
+    )
+    raise ApiException(
+        status_code=429,
+        code=exc.code,
+        message=exc.message,
+        details={"safety_state": exc.safety_state},
+    ) from exc
+
+
+def _raise_manual_failed(*, exc: Exception, user_id: int) -> None:
+    logger.exception(
+        "api.runs.manual_failed",
+        extra={"user_id": user_id},
+    )
+    raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from exc
+
+
+def _manual_run_success_payload(
+    *,
+    run_summary,
+    idempotency_key: str | None,
+    safety_state: dict[str, Any],
+) -> dict[str, Any]:
+    return {
+        "run_id": run_summary.crawl_run_id,
+        "status": run_summary.status.value,
+        "scholar_count": run_summary.scholar_count,
+        "succeeded_count": run_summary.succeeded_count,
+        "failed_count": run_summary.failed_count,
+        "partial_count": run_summary.partial_count,
+        "new_publication_count": run_summary.new_publication_count,
+        "reused_existing_run": False,
+        "idempotency_key": idempotency_key,
+        "safety_state": safety_state,
+    }
+
 
 @router.get(
     "",
@@ -78,14 +503,14 @@ async def list_runs(
         limit=limit,
         failed_only=failed_only,
     )
-    safety_state = await load_safety_state(
+    safety_state = await _load_safety_state(
         db_session,
         user_id=current_user.id,
     )
     return success_payload(
         request,
         data={
-            "runs": [serialize_run(run) for run in runs],
+            "runs": [_serialize_run(run) for run in runs],
             "safety_state": safety_state,
         },
     )
@@ -116,16 +541,16 @@ async def get_run(
     scholar_results = error_log.get("scholar_results")
     if not isinstance(scholar_results, list):
         scholar_results = []
-    safety_state = await load_safety_state(
+    safety_state = await _load_safety_state(
         db_session,
         user_id=current_user.id,
     )
     return success_payload(
         request,
         data={
-            "run": serialize_run(run),
+            "run": _serialize_run(run),
             "summary": run_service.extract_run_summary(error_log),
-            "scholar_results": [normalize_scholar_result(item) for item in scholar_results],
+            "scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
             "safety_state": safety_state,
         },
     )
@@ -170,7 +595,7 @@ async def cancel_run(
     if not isinstance(scholar_results, list):
         scholar_results = []
 
-    safety_state = await load_safety_state(
+    safety_state = await _load_safety_state(
         db_session,
         user_id=current_user.id,
     )
@@ -178,75 +603,14 @@ async def cancel_run(
     return success_payload(
         request,
         data={
-            "run": serialize_run(run),
+            "run": _serialize_run(run),
             "summary": run_service.extract_run_summary(error_log),
-            "scholar_results": [normalize_scholar_result(item) for item in scholar_results],
+            "scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
             "safety_state": safety_state,
         },
     )
 
 
-async def _start_manual_run(
-    db_session: AsyncSession,
-    *,
-    current_user: User,
-    ingest_service: ingestion_service.ScholarIngestionService,
-    idempotency_key: str | None,
-) -> tuple[Any, Any, list, dict]:
-    user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=current_user.id)
-    run, scholars, start_cstart_map = await ingest_service.initialize_run(
-        db_session,
-        user_id=current_user.id,
-        trigger_type=RunTriggerType.MANUAL,
-        request_delay_seconds=user_settings.request_delay_seconds,
-        network_error_retries=settings.ingestion_network_error_retries,
-        retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
-        max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
-        page_size=settings.ingestion_page_size,
-        idempotency_key=idempotency_key,
-        alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
-        alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
-        alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
-    )
-    return user_settings, run, scholars, start_cstart_map
-
-
-def _spawn_background_execution(
-    ingest_service: ingestion_service.ScholarIngestionService,
-    *,
-    run: Any,
-    current_user: User,
-    scholars: list,
-    start_cstart_map: dict,
-    user_settings: Any,
-    idempotency_key: str | None,
-) -> None:
-    from app.db.background_session import background_session
-
-    task = asyncio.create_task(
-        ingest_service.execute_run(
-            session_factory=background_session,
-            run_id=run.id,
-            user_id=current_user.id,
-            scholars=scholars,
-            start_cstart_map=start_cstart_map,
-            request_delay_seconds=user_settings.request_delay_seconds,
-            network_error_retries=settings.ingestion_network_error_retries,
-            retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
-            max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
-            page_size=settings.ingestion_page_size,
-            auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
-            queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
-            alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
-            alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
-            alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
-            idempotency_key=idempotency_key,
-        )
-    )
-    _background_tasks.add(task)
-    task.add_done_callback(_drop_finished_task)
-
-
 @router.post(
     "/manual",
     response_model=ManualRunEnvelope,
@@ -257,16 +621,15 @@ async def run_manual(
     current_user: User = Depends(get_api_current_user),
     ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
 ):
-    user_id = int(current_user.id)
+    safety_state = await _load_safety_state(db_session, user_id=current_user.id)
-    safety_state = await load_safety_state(db_session, user_id=user_id)
     if not settings.ingestion_manual_run_allowed:
-        raise_manual_runs_disabled(user_id=user_id, safety_state=safety_state)
+        _raise_manual_runs_disabled(user_id=current_user.id, safety_state=safety_state)
 
-    idempotency_key = normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
+    idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
-    reused_payload = await reused_manual_run_payload(
+    reused_payload = await _reused_manual_run_payload(
         db_session,
         request=request,
-        user_id=user_id,
+        user_id=current_user.id,
         idempotency_key=idempotency_key,
         safety_state=safety_state,
     )
@@ -274,22 +637,52 @@ async def run_manual(
         return reused_payload
 
     try:
-        user_settings, run, scholars, start_cstart_map = await _start_manual_run(
+        user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=current_user.id)
+
+        # Initialize run (creates the record and performs safety checks)
+        run, scholars, start_cstart_map = await ingest_service.initialize_run(
             db_session,
-            current_user=current_user,
+            user_id=current_user.id,
-            ingest_service=ingest_service,
+            trigger_type=RunTriggerType.MANUAL,
+            request_delay_seconds=user_settings.request_delay_seconds,
+            network_error_retries=settings.ingestion_network_error_retries,
+            retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+            max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
+            page_size=settings.ingestion_page_size,
             idempotency_key=idempotency_key,
+            alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
+            alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
+            alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
         )
+
         await db_session.commit()
-        _spawn_background_execution(
+
-            ingest_service,
+        # Kick off background execution
-            run=run,
+        from app.db.session import get_session_factory
-            current_user=current_user,
+
-            scholars=scholars,
+        task = asyncio.create_task(
-            start_cstart_map=start_cstart_map,
+            ingest_service.execute_run(
-            user_settings=user_settings,
+                session_factory=get_session_factory(),
-            idempotency_key=idempotency_key,
+                run_id=run.id,
+                user_id=current_user.id,
+                scholars=scholars,
+                start_cstart_map=start_cstart_map,
+                request_delay_seconds=user_settings.request_delay_seconds,
+                network_error_retries=settings.ingestion_network_error_retries,
+                retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+                max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
+                page_size=settings.ingestion_page_size,
+                auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
+                queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
+                alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
+                alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
+                alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
+                idempotency_key=idempotency_key,
+            )
         )
+        _background_tasks.add(task)
+        task.add_done_callback(_background_tasks.discard)
+
         return success_payload(
             request,
             data={
@@ -302,12 +695,12 @@ async def run_manual(
                 "new_publication_count": 0,
                 "reused_existing_run": False,
                 "idempotency_key": idempotency_key,
-                "safety_state": await load_safety_state(db_session, user_id=user_id),
+                "safety_state": await _load_safety_state(db_session, user_id=current_user.id),
             },
         )
     except ingestion_service.RunBlockedBySafetyPolicyError as exc:
         await db_session.rollback()
-        raise_manual_blocked_safety(exc=exc, user_id=user_id)
+        _raise_manual_blocked_safety(exc=exc, user_id=current_user.id)
     except ingestion_service.RunAlreadyInProgressError as exc:
         await db_session.rollback()
         raise ApiException(
@@ -317,16 +710,16 @@ async def run_manual(
         ) from exc
     except IntegrityError as exc:
         await db_session.rollback()
-        return await recover_integrity_error(
+        return await _recover_integrity_error(
             db_session,
             request=request,
-            user_id=user_id,
+            user_id=current_user.id,
             idempotency_key=idempotency_key,
             original_exc=exc,
         )
     except Exception as exc:
         await db_session.rollback()
-        raise_manual_failed(exc=exc, user_id=user_id)
+        _raise_manual_failed(exc=exc, user_id=current_user.id)
 
 
 @router.get(
@@ -347,7 +740,7 @@ async def list_queue_items(
     return success_payload(
         request,
         data={
-            "queue_items": [serialize_queue_item(item) for item in items],
+            "queue_items": [_serialize_queue_item(item) for item in items],
         },
     )
 
@@ -383,7 +776,7 @@ async def retry_queue_item(
         )
     return success_payload(
         request,
-        data=serialize_queue_item(queue_item),
+        data=_serialize_queue_item(queue_item),
     )
 
 
@@ -418,7 +811,7 @@ async def drop_queue_item(
         )
     return success_payload(
         request,
-        data=serialize_queue_item(dropped),
+        data=_serialize_queue_item(dropped),
     )
 
 
@@ -465,15 +858,14 @@ async def clear_queue_item(
 @router.get("/{run_id}/stream")
 async def stream_run_events(
     run_id: int,
+    db_session: AsyncSession = Depends(get_db_session),
     current_user: User = Depends(get_api_current_user),
 ):
-    session_factory = get_session_factory()
+    run = await run_service.get_run_for_user(
-    async with session_factory() as db_session:
+        db_session,
-        run = await run_service.get_run_for_user(
+        user_id=current_user.id,
-            db_session,
+        run_id=run_id,
-            user_id=current_user.id,
+    )
-            run_id=run_id,
-        )
     if run is None:
         raise ApiException(
             status_code=404,

app/api/routers/scholar_helpers.py

@@ -1,234 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-from typing import Any
-
-from fastapi import UploadFile
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.api.errors import ApiException
-from app.logging_utils import structured_log
-from app.services.ingestion import queue as ingestion_queue_service
-from app.services.scholar import rate_limit as scholar_rate_limit
-from app.services.scholar.source import ScholarSource
-from app.services.scholars import application as scholar_service
-from app.services.scholars import search_hints as scholar_search_hints
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-CREATE_METADATA_HYDRATION_TIMEOUT_SECONDS = 5.0
-INITIAL_SCHOLAR_SCRAPE_QUEUE_DELAY_SECONDS = 0
-INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON = "scholar_added_initial_scrape"
-
-
-def needs_metadata_hydration(profile) -> bool:
-    if not profile.profile_image_url:
-        return True
-    return not (profile.display_name or "").strip()
-
-
-def is_create_hydration_rate_limited() -> tuple[bool, float]:
-    remaining_seconds = scholar_rate_limit.remaining_scholar_slot_seconds(
-        min_interval_seconds=float(settings.ingestion_min_request_delay_seconds),
-    )
-    return remaining_seconds > 0, remaining_seconds
-
-
-def auto_enqueue_new_scholar_enabled() -> bool:
-    if not settings.scheduler_enabled:
-        return False
-    if not settings.ingestion_automation_allowed:
-        return False
-    return bool(settings.ingestion_continuation_queue_enabled)
-
-
-async def enqueue_initial_scrape_job_for_scholar(
-    db_session: AsyncSession,
-    *,
-    profile,
-    user_id: int,
-) -> bool:
-    if not auto_enqueue_new_scholar_enabled():
-        return False
-    try:
-        await ingestion_queue_service.upsert_job(
-            db_session,
-            user_id=user_id,
-            scholar_profile_id=int(profile.id),
-            resume_cstart=0,
-            reason=INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON,
-            run_id=None,
-            delay_seconds=INITIAL_SCHOLAR_SCRAPE_QUEUE_DELAY_SECONDS,
-        )
-        await db_session.commit()
-    except Exception:
-        await db_session.rollback()
-        structured_log(
-            logger,
-            "warning",
-            "api.scholars.initial_scrape_enqueue_failed",
-            user_id=user_id,
-            scholar_profile_id=profile.id,
-        )
-        return False
-
-    structured_log(
-        logger,
-        "info",
-        "api.scholars.initial_scrape_enqueued",
-        user_id=user_id,
-        scholar_profile_id=profile.id,
-        reason=INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON,
-    )
-    return True
-
-
-def uploaded_image_media_path(scholar_profile_id: int) -> str:
-    return f"/scholar-images/{scholar_profile_id}/upload"
-
-
-def serialize_scholar(profile) -> dict[str, object]:
-    uploaded_image_url = None
-    if profile.profile_image_upload_path:
-        uploaded_image_url = uploaded_image_media_path(int(profile.id))
-
-    profile_image_url, profile_image_source = scholar_search_hints.resolve_profile_image(
-        profile,
-        uploaded_image_url=uploaded_image_url,
-    )
-
-    return {
-        "id": int(profile.id),
-        "scholar_id": profile.scholar_id,
-        "display_name": profile.display_name,
-        "profile_image_url": profile_image_url,
-        "profile_image_source": profile_image_source,
-        "is_enabled": bool(profile.is_enabled),
-        "baseline_completed": bool(profile.baseline_completed),
-        "last_run_dt": profile.last_run_dt,
-        "last_run_status": (profile.last_run_status.value if profile.last_run_status is not None else None),
-    }
-
-
-async def hydrate_scholar_metadata_if_needed(
-    db_session: AsyncSession,
-    *,
-    profile,
-    source: ScholarSource,
-    user_id: int,
-):
-    if not needs_metadata_hydration(profile):
-        return profile
-
-    should_skip, remaining_seconds = is_create_hydration_rate_limited()
-    if should_skip:
-        structured_log(
-            logger,
-            "info",
-            "api.scholars.create_metadata_hydration_skipped",
-            reason="scholar_request_throttle_active",
-            user_id=user_id,
-            scholar_profile_id=profile.id,
-            retry_after_seconds=round(remaining_seconds, 3),
-        )
-        return profile
-
-    try:
-        return await asyncio.wait_for(
-            scholar_service.hydrate_profile_metadata(
-                db_session,
-                profile=profile,
-                source=source,
-            ),
-            timeout=CREATE_METADATA_HYDRATION_TIMEOUT_SECONDS,
-        )
-    except TimeoutError:
-        structured_log(
-            logger,
-            "info",
-            "api.scholars.create_metadata_hydration_skipped",
-            reason="create_timeout",
-            user_id=user_id,
-            scholar_profile_id=profile.id,
-        )
-    except Exception:
-        structured_log(
-            logger,
-            "warning",
-            "api.scholars.create_metadata_hydration_failed",
-            user_id=user_id,
-            scholar_profile_id=profile.id,
-        )
-    return profile
-
-
-def search_kwargs() -> dict[str, Any]:
-    return {
-        "network_error_retries": settings.ingestion_network_error_retries,
-        "retry_backoff_seconds": settings.ingestion_retry_backoff_seconds,
-        "search_enabled": settings.scholar_name_search_enabled,
-        "cache_ttl_seconds": settings.scholar_name_search_cache_ttl_seconds,
-        "blocked_cache_ttl_seconds": settings.scholar_name_search_blocked_cache_ttl_seconds,
-        "cache_max_entries": settings.scholar_name_search_cache_max_entries,
-        "min_interval_seconds": settings.scholar_name_search_min_interval_seconds,
-        "interval_jitter_seconds": settings.scholar_name_search_interval_jitter_seconds,
-        "cooldown_block_threshold": settings.scholar_name_search_cooldown_block_threshold,
-        "cooldown_seconds": settings.scholar_name_search_cooldown_seconds,
-        "retry_alert_threshold": settings.scholar_name_search_alert_retry_count_threshold,
-        "cooldown_rejection_alert_threshold": (settings.scholar_name_search_alert_cooldown_rejections_threshold),
-    }
-
-
-def search_response_data(query: str, parsed) -> dict[str, object]:
-    return {
-        "query": query.strip(),
-        "state": parsed.state.value,
-        "state_reason": parsed.state_reason,
-        "action_hint": scholar_search_hints.scrape_state_hint(
-            state=parsed.state,
-            state_reason=parsed.state_reason,
-        ),
-        "candidates": [
-            {
-                "scholar_id": item.scholar_id,
-                "display_name": item.display_name,
-                "affiliation": item.affiliation,
-                "email_domain": item.email_domain,
-                "cited_by_count": item.cited_by_count,
-                "interests": item.interests,
-                "profile_url": item.profile_url,
-                "profile_image_url": item.profile_image_url,
-            }
-            for item in parsed.candidates
-        ],
-        "warnings": parsed.warnings,
-    }
-
-
-async def read_uploaded_image(image: UploadFile) -> bytes:
-    try:
-        return await image.read()
-    finally:
-        await image.close()
-
-
-async def require_user_profile(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholar_profile_id: int,
-):
-    profile = await scholar_service.get_user_scholar_by_id(
-        db_session,
-        user_id=user_id,
-        scholar_profile_id=scholar_profile_id,
-    )
-    if profile is None:
-        raise ApiException(
-            status_code=404,
-            code="scholar_not_found",
-            message="Scholar not found.",
-        )
-    return profile

app/api/routers/scholars.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
+import asyncio
 import logging
+from typing import Any
 
 from fastapi import APIRouter, Depends, File, Query, Request, UploadFile
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -8,24 +10,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from app.api.deps import get_api_current_user
 from app.api.errors import ApiException
 from app.api.responses import success_payload
-from app.api.routers.scholar_helpers import (
-    enqueue_initial_scrape_job_for_scholar,
-    hydrate_scholar_metadata_if_needed,
-    read_uploaded_image,
-    require_user_profile,
-    search_kwargs,
-    search_response_data,
-    serialize_scholar,
-)
 from app.api.runtime_deps import get_scholar_source
 from app.api.schemas import (
     DataExportEnvelope,
     DataImportEnvelope,
     DataImportRequest,
     MessageEnvelope,
-    ScholarBulkCountEnvelope,
-    ScholarBulkIdsRequest,
-    ScholarBulkToggleRequest,
     ScholarCreateRequest,
     ScholarEnvelope,
     ScholarImageUrlUpdateRequest,
@@ -35,30 +25,231 @@ from app.api.schemas import (
 from app.db.models import User
 from app.db.session import get_db_session
 from app.logging_utils import structured_log
+from app.services.ingestion import queue as ingestion_queue_service
 from app.services.portability import application as import_export_service
+from app.services.scholar import rate_limit as scholar_rate_limit
 from app.services.scholar.source import ScholarSource
 from app.services.scholars import application as scholar_service
+from app.services.scholars import search_hints as scholar_search_hints
 from app.settings import settings
 
 logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/scholars", tags=["api-scholars"])
+CREATE_METADATA_HYDRATION_TIMEOUT_SECONDS = 5.0
+INITIAL_SCHOLAR_SCRAPE_QUEUE_DELAY_SECONDS = 0
+INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON = "scholar_added_initial_scrape"
 
 
-def _parse_ids_param(ids: str | None) -> list[int] | None:
+def _needs_metadata_hydration(profile) -> bool:
-    if not ids:
+    if not profile.profile_image_url:
-        return None
+        return True
-    parts = [p.strip() for p in ids.split(",") if p.strip()]
+    return not (profile.display_name or "").strip()
-    if not parts:
+
-        return None
+
+def _is_create_hydration_rate_limited() -> tuple[bool, float]:
+    remaining_seconds = scholar_rate_limit.remaining_scholar_slot_seconds(
+        min_interval_seconds=float(settings.ingestion_min_request_delay_seconds),
+    )
+    return remaining_seconds > 0, remaining_seconds
+
+
+def _auto_enqueue_new_scholar_enabled() -> bool:
+    if not settings.scheduler_enabled:
+        return False
+    if not settings.ingestion_automation_allowed:
+        return False
+    return bool(settings.ingestion_continuation_queue_enabled)
+
+
+async def _enqueue_initial_scrape_job_for_scholar(
+    db_session: AsyncSession,
+    *,
+    profile,
+    user_id: int,
+) -> bool:
+    if not _auto_enqueue_new_scholar_enabled():
+        return False
     try:
-        return [int(p) for p in parts]
+        await ingestion_queue_service.upsert_job(
-    except ValueError as exc:
+            db_session,
+            user_id=user_id,
+            scholar_profile_id=int(profile.id),
+            resume_cstart=0,
+            reason=INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON,
+            run_id=None,
+            delay_seconds=INITIAL_SCHOLAR_SCRAPE_QUEUE_DELAY_SECONDS,
+        )
+        await db_session.commit()
+    except Exception:
+        await db_session.rollback()
+        structured_log(
+            logger,
+            "warning",
+            "api.scholars.initial_scrape_enqueue_failed",
+            user_id=user_id,
+            scholar_profile_id=profile.id,
+        )
+        return False
+
+    structured_log(
+        logger,
+        "info",
+        "api.scholars.initial_scrape_enqueued",
+        user_id=user_id,
+        scholar_profile_id=profile.id,
+        reason=INITIAL_SCHOLAR_SCRAPE_QUEUE_REASON,
+    )
+    return True
+
+
+def _uploaded_image_media_path(scholar_profile_id: int) -> str:
+    return f"/scholar-images/{scholar_profile_id}/upload"
+
+
+def _serialize_scholar(profile) -> dict[str, object]:
+    uploaded_image_url = None
+    if profile.profile_image_upload_path:
+        uploaded_image_url = _uploaded_image_media_path(int(profile.id))
+
+    profile_image_url, profile_image_source = scholar_search_hints.resolve_profile_image(
+        profile,
+        uploaded_image_url=uploaded_image_url,
+    )
+
+    return {
+        "id": int(profile.id),
+        "scholar_id": profile.scholar_id,
+        "display_name": profile.display_name,
+        "profile_image_url": profile_image_url,
+        "profile_image_source": profile_image_source,
+        "is_enabled": bool(profile.is_enabled),
+        "baseline_completed": bool(profile.baseline_completed),
+        "last_run_dt": profile.last_run_dt,
+        "last_run_status": (profile.last_run_status.value if profile.last_run_status is not None else None),
+    }
+
+
+async def _hydrate_scholar_metadata_if_needed(
+    db_session: AsyncSession,
+    *,
+    profile,
+    source: ScholarSource,
+    user_id: int,
+):
+    if not _needs_metadata_hydration(profile):
+        return profile
+
+    should_skip, remaining_seconds = _is_create_hydration_rate_limited()
+    if should_skip:
+        structured_log(
+            logger,
+            "info",
+            "api.scholars.create_metadata_hydration_skipped",
+            reason="scholar_request_throttle_active",
+            user_id=user_id,
+            scholar_profile_id=profile.id,
+            retry_after_seconds=round(remaining_seconds, 3),
+        )
+        return profile
+
+    try:
+        return await asyncio.wait_for(
+            scholar_service.hydrate_profile_metadata(
+                db_session,
+                profile=profile,
+                source=source,
+            ),
+            timeout=CREATE_METADATA_HYDRATION_TIMEOUT_SECONDS,
+        )
+    except TimeoutError:
+        structured_log(
+            logger,
+            "info",
+            "api.scholars.create_metadata_hydration_skipped",
+            reason="create_timeout",
+            user_id=user_id,
+            scholar_profile_id=profile.id,
+        )
+    except Exception:
+        structured_log(
+            logger,
+            "warning",
+            "api.scholars.create_metadata_hydration_failed",
+            user_id=user_id,
+            scholar_profile_id=profile.id,
+        )
+    return profile
+
+
+def _search_kwargs() -> dict[str, Any]:
+    return {
+        "network_error_retries": settings.ingestion_network_error_retries,
+        "retry_backoff_seconds": settings.ingestion_retry_backoff_seconds,
+        "search_enabled": settings.scholar_name_search_enabled,
+        "cache_ttl_seconds": settings.scholar_name_search_cache_ttl_seconds,
+        "blocked_cache_ttl_seconds": settings.scholar_name_search_blocked_cache_ttl_seconds,
+        "cache_max_entries": settings.scholar_name_search_cache_max_entries,
+        "min_interval_seconds": settings.scholar_name_search_min_interval_seconds,
+        "interval_jitter_seconds": settings.scholar_name_search_interval_jitter_seconds,
+        "cooldown_block_threshold": settings.scholar_name_search_cooldown_block_threshold,
+        "cooldown_seconds": settings.scholar_name_search_cooldown_seconds,
+        "retry_alert_threshold": settings.scholar_name_search_alert_retry_count_threshold,
+        "cooldown_rejection_alert_threshold": (settings.scholar_name_search_alert_cooldown_rejections_threshold),
+    }
+
+
+def _search_response_data(query: str, parsed) -> dict[str, object]:
+    return {
+        "query": query.strip(),
+        "state": parsed.state.value,
+        "state_reason": parsed.state_reason,
+        "action_hint": scholar_search_hints.scrape_state_hint(
+            state=parsed.state,
+            state_reason=parsed.state_reason,
+        ),
+        "candidates": [
+            {
+                "scholar_id": item.scholar_id,
+                "display_name": item.display_name,
+                "affiliation": item.affiliation,
+                "email_domain": item.email_domain,
+                "cited_by_count": item.cited_by_count,
+                "interests": item.interests,
+                "profile_url": item.profile_url,
+                "profile_image_url": item.profile_image_url,
+            }
+            for item in parsed.candidates
+        ],
+        "warnings": parsed.warnings,
+    }
+
+
+async def _read_uploaded_image(image: UploadFile) -> bytes:
+    try:
+        return await image.read()
+    finally:
+        await image.close()
+
+
+async def _require_user_profile(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+):
+    profile = await scholar_service.get_user_scholar_by_id(
+        db_session,
+        user_id=user_id,
+        scholar_profile_id=scholar_profile_id,
+    )
+    if profile is None:
         raise ApiException(
-            status_code=400,
+            status_code=404,
-            code="invalid_ids_param",
+            code="scholar_not_found",
-            message="The 'ids' parameter must be a comma-separated list of integers.",
+            message="Scholar not found.",
-        ) from exc
+        )
+    return profile
 
 
 @router.get(
@@ -77,7 +268,7 @@ async def list_scholars(
     return success_payload(
         request,
         data={
-            "scholars": [serialize_scholar(profile) for profile in scholars],
+            "scholars": [_serialize_scholar(profile) for profile in scholars],
         },
     )
 
@@ -88,81 +279,16 @@ async def list_scholars(
 )
 async def export_scholars_and_publications(
     request: Request,
-    ids: str | None = Query(None, description="Comma-separated scholar profile IDs to export"),
     db_session: AsyncSession = Depends(get_db_session),
     current_user: User = Depends(get_api_current_user),
 ):
-    scholar_profile_ids = _parse_ids_param(ids)
     data = await import_export_service.export_user_data(
         db_session,
         user_id=current_user.id,
-        scholar_profile_ids=scholar_profile_ids,
     )
     return success_payload(request, data=data)
 
 
-@router.post(
-    "/bulk-delete",
-    response_model=ScholarBulkCountEnvelope,
-)
-async def bulk_delete_scholars(
-    payload: ScholarBulkIdsRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    try:
-        deleted_count = await scholar_service.bulk_delete_scholars(
-            db_session,
-            user_id=current_user.id,
-            scholar_profile_ids=payload.scholar_profile_ids,
-            upload_dir=settings.scholar_image_upload_dir,
-        )
-    except scholar_service.ScholarServiceError as exc:
-        raise ApiException(
-            status_code=409,
-            code="scholar_bulk_delete_failed",
-            message=str(exc),
-        ) from exc
-    structured_log(
-        logger,
-        "info",
-        "scholars.bulk_delete",
-        user_id=current_user.id,
-        requested_ids=payload.scholar_profile_ids,
-        deleted_count=deleted_count,
-    )
-    return success_payload(request, data={"deleted_count": deleted_count, "updated_count": 0})
-
-
-@router.post(
-    "/bulk-toggle",
-    response_model=ScholarBulkCountEnvelope,
-)
-async def bulk_toggle_scholars(
-    payload: ScholarBulkToggleRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    updated_count = await scholar_service.bulk_toggle_scholars(
-        db_session,
-        user_id=current_user.id,
-        scholar_profile_ids=payload.scholar_profile_ids,
-        is_enabled=payload.is_enabled,
-    )
-    structured_log(
-        logger,
-        "info",
-        "scholars.bulk_toggle",
-        user_id=current_user.id,
-        requested_ids=payload.scholar_profile_ids,
-        is_enabled=payload.is_enabled,
-        updated_count=updated_count,
-    )
-    return success_payload(request, data={"deleted_count": 0, "updated_count": updated_count})
-
-
 @router.post(
     "/import",
     response_model=DataImportEnvelope,
@@ -228,21 +354,22 @@ async def create_scholar(
             message=str(exc),
         ) from exc
     structured_log(logger, "info", "api.scholars.created", user_id=current_user.id, scholar_profile_id=created.id)
-    await enqueue_initial_scrape_job_for_scholar(
+    did_queue_initial_scrape = await _enqueue_initial_scrape_job_for_scholar(
         db_session,
         profile=created,
         user_id=current_user.id,
     )
-    created = await hydrate_scholar_metadata_if_needed(
+    if not did_queue_initial_scrape:
-        db_session,
+        created = await _hydrate_scholar_metadata_if_needed(
-        profile=created,
+            db_session,
-        source=source,
+            profile=created,
-        user_id=current_user.id,
+            source=source,
-    )
+            user_id=current_user.id,
+        )
 
     return success_payload(
         request,
-        data=serialize_scholar(created),
+        data=_serialize_scholar(created),
     )
 
 
@@ -264,7 +391,7 @@ async def search_scholars(
             db_session=db_session,
             query=query,
             limit=limit,
-            **search_kwargs(),
+            **_search_kwargs(),
         )
     except scholar_service.ScholarServiceError as exc:
         raise ApiException(
@@ -284,7 +411,7 @@ async def search_scholars(
     )
     return success_payload(
         request,
-        data=search_response_data(query, parsed),
+        data=_search_response_data(query, parsed),
     )
 
 
@@ -320,7 +447,7 @@ async def toggle_scholar(
     )
     return success_payload(
         request,
-        data=serialize_scholar(updated),
+        data=_serialize_scholar(updated),
     )
 
 
@@ -345,18 +472,11 @@ async def delete_scholar(
             code="scholar_not_found",
             message="Scholar not found.",
         )
-    try:
+    await scholar_service.delete_scholar(
-        await scholar_service.delete_scholar(
+        db_session,
-            db_session,
+        profile=profile,
-            profile=profile,
+        upload_dir=settings.scholar_image_upload_dir,
-            upload_dir=settings.scholar_image_upload_dir,
+    )
-        )
-    except scholar_service.ScholarServiceError as exc:
-        raise ApiException(
-            status_code=409,
-            code="scholar_delete_failed",
-            message=str(exc),
-        ) from exc
     structured_log(
         logger, "info", "api.scholars.deleted", user_id=current_user.id, scholar_profile_id=scholar_profile_id
     )
@@ -408,7 +528,7 @@ async def update_scholar_image_url(
     )
     return success_payload(
         request,
-        data=serialize_scholar(updated),
+        data=_serialize_scholar(updated),
     )
 
 
@@ -423,13 +543,13 @@ async def upload_scholar_image(
     db_session: AsyncSession = Depends(get_db_session),
     current_user: User = Depends(get_api_current_user),
 ):
-    profile = await require_user_profile(
+    profile = await _require_user_profile(
         db_session,
         user_id=current_user.id,
         scholar_profile_id=scholar_profile_id,
     )
 
-    image_bytes = await read_uploaded_image(image)
+    image_bytes = await _read_uploaded_image(image)
     try:
         updated = await scholar_service.set_profile_image_upload(
             db_session,
@@ -458,7 +578,7 @@ async def upload_scholar_image(
     )
     return success_payload(
         request,
-        data=serialize_scholar(updated),
+        data=_serialize_scholar(updated),
     )
 
 
@@ -492,5 +612,5 @@ async def clear_scholar_image_customization(
     structured_log(logger, "info", "api.scholars.image_cleared", user_id=current_user.id, scholar_profile_id=updated.id)
     return success_payload(
         request,
-        data=serialize_scholar(updated),
+        data=_serialize_scholar(updated),
     )

app/api/schemas.py

@@ -0,0 +1,945 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class ApiMeta(BaseModel):
+    request_id: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ApiErrorData(BaseModel):
+    code: str
+    message: str
+    details: Any | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ApiErrorEnvelope(BaseModel):
+    error: ApiErrorData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessageData(BaseModel):
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessageEnvelope(BaseModel):
+    data: MessageData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SessionUserData(BaseModel):
+    id: int
+    email: str
+    is_admin: bool
+    is_active: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AuthMeData(BaseModel):
+    authenticated: bool
+    csrf_token: str
+    user: SessionUserData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AuthMeEnvelope(BaseModel):
+    data: AuthMeData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class CsrfBootstrapData(BaseModel):
+    csrf_token: str
+    authenticated: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class CsrfBootstrapEnvelope(BaseModel):
+    data: CsrfBootstrapData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginRequest(BaseModel):
+    email: str
+    password: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginData(BaseModel):
+    authenticated: bool
+    csrf_token: str
+    user: SessionUserData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginEnvelope(BaseModel):
+    data: LoginData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ChangePasswordRequest(BaseModel):
+    current_password: str
+    new_password: str
+    confirm_password: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarItemData(BaseModel):
+    id: int
+    scholar_id: str
+    display_name: str | None
+    profile_image_url: str | None
+    profile_image_source: str
+    is_enabled: bool
+    baseline_completed: bool
+    last_run_dt: datetime | None
+    last_run_status: str | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarsListData(BaseModel):
+    scholars: list[ScholarItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarsListEnvelope(BaseModel):
+    data: ScholarsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarCreateRequest(BaseModel):
+    scholar_id: str
+    profile_image_url: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarEnvelope(BaseModel):
+    data: ScholarItemData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchCandidateData(BaseModel):
+    scholar_id: str
+    display_name: str
+    affiliation: str | None
+    email_domain: str | None
+    cited_by_count: int | None
+    interests: list[str] = Field(default_factory=list)
+    profile_url: str
+    profile_image_url: str | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchData(BaseModel):
+    query: str
+    state: str
+    state_reason: str
+    action_hint: str | None = None
+    candidates: list[ScholarSearchCandidateData]
+    warnings: list[str] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarSearchEnvelope(BaseModel):
+    data: ScholarSearchData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarImageUrlUpdateRequest(BaseModel):
+    image_url: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarExportItemData(BaseModel):
+    scholar_id: str
+    display_name: str | None = None
+    is_enabled: bool = True
+    profile_image_override_url: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationExportItemData(BaseModel):
+    scholar_id: str
+    cluster_id: str | None = None
+    fingerprint_sha256: str | None = None
+    title: str
+    year: int | None = None
+    citation_count: int = 0
+    author_text: str | None = None
+    venue_text: str | None = None
+    pub_url: str | None = None
+    pdf_url: str | None = None
+    is_read: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataExportData(BaseModel):
+    schema_version: int
+    exported_at: str
+    scholars: list[ScholarExportItemData]
+    publications: list[PublicationExportItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataExportEnvelope(BaseModel):
+    data: DataExportData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportRequest(BaseModel):
+    schema_version: int | None = None
+    scholars: list[ScholarExportItemData] = Field(default_factory=list)
+    publications: list[PublicationExportItemData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportResultData(BaseModel):
+    scholars_created: int
+    scholars_updated: int
+    publications_created: int
+    publications_updated: int
+    links_created: int
+    links_updated: int
+    skipped_records: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DataImportEnvelope(BaseModel):
+    data: DataImportResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunListItemData(BaseModel):
+    id: int
+    trigger_type: str
+    status: str
+    start_dt: datetime
+    end_dt: datetime | None
+    scholar_count: int
+    new_publication_count: int
+    failed_count: int
+    partial_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunsListData(BaseModel):
+    runs: list[RunListItemData]
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunsListEnvelope(BaseModel):
+    data: RunsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunSummaryData(BaseModel):
+    succeeded_count: int
+    failed_count: int
+    partial_count: int
+    failed_state_counts: dict[str, int] = Field(default_factory=dict)
+    failed_reason_counts: dict[str, int] = Field(default_factory=dict)
+    scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
+    retry_counts: dict[str, int] = Field(default_factory=dict)
+    alert_thresholds: dict[str, int] = Field(default_factory=dict)
+    alert_flags: dict[str, bool] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScrapeSafetyCountersData(BaseModel):
+    consecutive_blocked_runs: int = 0
+    consecutive_network_runs: int = 0
+    cooldown_entry_count: int = 0
+    blocked_start_count: int = 0
+    last_blocked_failure_count: int = 0
+    last_network_failure_count: int = 0
+    last_evaluated_run_id: int | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScrapeSafetyStateData(BaseModel):
+    cooldown_active: bool
+    cooldown_reason: str | None = None
+    cooldown_reason_label: str | None = None
+    cooldown_until: datetime | None = None
+    cooldown_remaining_seconds: int = 0
+    recommended_action: str | None = None
+    counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunAttemptLogData(BaseModel):
+    attempt: int
+    cstart: int
+    state: str | None = None
+    state_reason: str | None = None
+    status_code: int | None = None
+    fetch_error: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunPageLogData(BaseModel):
+    page: int
+    cstart: int
+    state: str
+    state_reason: str | None = None
+    status_code: int | None = None
+    publication_count: int = 0
+    attempt_count: int = 0
+    has_show_more_button: bool = False
+    articles_range: str | None = None
+    warning_codes: list[str] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDebugData(BaseModel):
+    status_code: int | None = None
+    final_url: str | None = None
+    fetch_error: str | None = None
+    body_sha256: str | None = None
+    body_length: int | None = None
+    has_show_more_button: bool | None = None
+    articles_range: str | None = None
+    state_reason: str | None = None
+    warning_codes: list[str] = Field(default_factory=list)
+    marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
+    page_logs: list[RunPageLogData] = Field(default_factory=list)
+    attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunScholarResultData(BaseModel):
+    scholar_profile_id: int
+    scholar_id: str
+    state: str
+    state_reason: str | None = None
+    outcome: str
+    attempt_count: int = 0
+    publication_count: int = 0
+    start_cstart: int = 0
+    continuation_cstart: int | None = None
+    continuation_enqueued: bool = False
+    continuation_cleared: bool = False
+    warnings: list[str] = Field(default_factory=list)
+    error: str | None = None
+    debug: RunDebugData | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDetailData(BaseModel):
+    run: RunListItemData
+    summary: RunSummaryData
+    scholar_results: list[RunScholarResultData]
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDetailEnvelope(BaseModel):
+    data: RunDetailData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ManualRunData(BaseModel):
+    run_id: int
+    status: str
+    scholar_count: int
+    succeeded_count: int
+    failed_count: int
+    partial_count: int
+    new_publication_count: int
+    reused_existing_run: bool
+    idempotency_key: str | None = None
+    safety_state: ScrapeSafetyStateData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ManualRunEnvelope(BaseModel):
+    data: ManualRunData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueItemData(BaseModel):
+    id: int
+    scholar_profile_id: int
+    scholar_label: str
+    status: str
+    reason: str
+    dropped_reason: str | None
+    attempt_count: int
+    resume_cstart: int
+    next_attempt_dt: datetime | None
+    updated_at: datetime
+    last_error: str | None
+    last_run_id: int | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueListData(BaseModel):
+    queue_items: list[QueueItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueListEnvelope(BaseModel):
+    data: QueueListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueItemEnvelope(BaseModel):
+    data: QueueItemData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueClearData(BaseModel):
+    queue_item_id: int
+    previous_status: str
+    status: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueClearEnvelope(BaseModel):
+    data: QueueClearData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserData(BaseModel):
+    id: int
+    email: str
+    is_active: bool
+    is_admin: bool
+    created_at: datetime
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUsersListData(BaseModel):
+    users: list[AdminUserData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUsersListEnvelope(BaseModel):
+    data: AdminUsersListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserCreateRequest(BaseModel):
+    email: str
+    password: str
+    is_admin: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserActiveUpdateRequest(BaseModel):
+    is_active: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserEnvelope(BaseModel):
+    data: AdminUserData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminResetPasswordRequest(BaseModel):
+    new_password: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityCheckData(BaseModel):
+    name: str
+    count: int
+    severity: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityData(BaseModel):
+    status: str
+    checked_at: datetime
+    failures: list[str] = Field(default_factory=list)
+    warnings: list[str] = Field(default_factory=list)
+    checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbIntegrityEnvelope(BaseModel):
+    data: AdminDbIntegrityData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobData(BaseModel):
+    id: int
+    job_name: str
+    requested_by: str | None
+    dry_run: bool
+    status: str
+    scope: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any] = Field(default_factory=dict)
+    error_text: str | None
+    started_at: datetime | None
+    finished_at: datetime | None
+    created_at: datetime
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobsData(BaseModel):
+    jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminDbRepairJobsEnvelope(BaseModel):
+    data: AdminDbRepairJobsData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class DisplayIdentifierData(BaseModel):
+    kind: str
+    value: str
+    label: str
+    url: str | None
+    confidence_score: float = Field(ge=0.0, le=1.0)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueItemData(BaseModel):
+    publication_id: int
+    title: str
+    display_identifier: DisplayIdentifierData | None = None
+    pdf_url: str | None
+    status: str
+    attempt_count: int
+    last_failure_reason: str | None
+    last_failure_detail: str | None
+    last_source: str | None
+    requested_by_user_id: int | None
+    requested_by_email: str | None
+    queued_at: datetime | None
+    last_attempt_at: datetime | None
+    resolved_at: datetime | None
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueData(BaseModel):
+    items: list[AdminPdfQueueItemData] = Field(default_factory=list)
+    total_count: int = 0
+    page: int = 1
+    page_size: int = 100
+    has_next: bool = False
+    has_prev: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueEnvelope(BaseModel):
+    data: AdminPdfQueueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueRequeueData(BaseModel):
+    publication_id: int
+    queued: bool
+    status: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueRequeueEnvelope(BaseModel):
+    data: AdminPdfQueueRequeueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueBulkEnqueueData(BaseModel):
+    requested_count: int
+    queued_count: int
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
+    data: AdminPdfQueueBulkEnqueueData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationLinksRequest(BaseModel):
+    scope_mode: Literal["single_user", "all_users"] = "single_user"
+    user_id: int | None = Field(default=None, ge=1)
+    scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
+    dry_run: bool = True
+    gc_orphan_publications: bool = False
+    requested_by: str | None = None
+    confirmation_text: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")
+    def validate_scope(self) -> AdminRepairPublicationLinksRequest:
+        if self.scope_mode == "single_user" and self.user_id is None:
+            raise ValueError("user_id is required when scope_mode=single_user.")
+        if self.scope_mode == "all_users" and self.user_id is not None:
+            raise ValueError("user_id must be omitted when scope_mode=all_users.")
+        if not self.dry_run and self.scope_mode == "all_users":
+            expected = "REPAIR ALL USERS"
+            provided = (self.confirmation_text or "").strip()
+            if provided != expected:
+                raise ValueError("confirmation_text must equal 'REPAIR ALL USERS' when applying a repair to all users.")
+        return self
+
+
+class AdminRepairPublicationLinksResultData(BaseModel):
+    job_id: int
+    status: str
+    scope: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationLinksEnvelope(BaseModel):
+    data: AdminRepairPublicationLinksResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminNearDuplicateClusterMemberData(BaseModel):
+    publication_id: int
+    title: str
+    year: int | None
+    citation_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminNearDuplicateClusterData(BaseModel):
+    cluster_key: str
+    winner_publication_id: int
+    member_count: int
+    similarity_score: float = Field(ge=0.0, le=1.0)
+    members: list[AdminNearDuplicateClusterMemberData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationNearDuplicatesRequest(BaseModel):
+    dry_run: bool = True
+    similarity_threshold: float = Field(default=0.78, ge=0.5, le=1.0)
+    min_shared_tokens: int = Field(default=3, ge=1, le=8)
+    max_year_delta: int = Field(default=1, ge=0, le=5)
+    max_clusters: int = Field(default=25, ge=1, le=200)
+    selected_cluster_keys: list[str] = Field(default_factory=list, max_length=200)
+    requested_by: str | None = None
+    confirmation_text: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")
+    def validate_apply_mode(self) -> AdminRepairPublicationNearDuplicatesRequest:
+        if self.dry_run:
+            return self
+        if not self.selected_cluster_keys:
+            raise ValueError("selected_cluster_keys is required when dry_run=false.")
+        expected = "MERGE SELECTED DUPLICATES"
+        provided = (self.confirmation_text or "").strip()
+        if provided != expected:
+            raise ValueError(
+                "confirmation_text must equal 'MERGE SELECTED DUPLICATES' when applying near-duplicate merges."
+            )
+        return self
+
+
+class AdminRepairPublicationNearDuplicatesResultData(BaseModel):
+    job_id: int
+    status: str
+    scope: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any] = Field(default_factory=dict)
+    clusters: list[AdminNearDuplicateClusterData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminRepairPublicationNearDuplicatesEnvelope(BaseModel):
+    data: AdminRepairPublicationNearDuplicatesResultData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsPolicyData(BaseModel):
+    min_run_interval_minutes: int
+    min_request_delay_seconds: int
+    automation_allowed: bool
+    manual_run_allowed: bool
+    blocked_failure_threshold: int
+    network_failure_threshold: int
+    cooldown_blocked_seconds: int
+    cooldown_network_seconds: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsData(BaseModel):
+    auto_run_enabled: bool
+    run_interval_minutes: int
+    request_delay_seconds: int
+    nav_visible_pages: list[str]
+    policy: SettingsPolicyData
+    safety_state: ScrapeSafetyStateData
+
+    openalex_api_key: str | None = None
+    crossref_api_token: str | None = None
+    crossref_api_mailto: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsEnvelope(BaseModel):
+    data: SettingsData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsUpdateRequest(BaseModel):
+    auto_run_enabled: bool
+    run_interval_minutes: int
+    request_delay_seconds: int
+    nav_visible_pages: list[str] | None = None
+
+    openalex_api_key: str | None = None
+    crossref_api_token: str | None = None
+    crossref_api_mailto: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationItemData(BaseModel):
+    publication_id: int
+    scholar_profile_id: int
+    scholar_label: str
+    title: str
+    year: int | None
+    citation_count: int
+    venue_text: str | None
+    pub_url: str | None
+    display_identifier: DisplayIdentifierData | None = None
+    pdf_url: str | None
+    pdf_status: str = "untracked"
+    pdf_attempt_count: int = 0
+    pdf_failure_reason: str | None = None
+    pdf_failure_detail: str | None = None
+    is_read: bool
+    is_favorite: bool = False
+    first_seen_at: datetime
+    is_new_in_latest_run: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationsListData(BaseModel):
+    mode: str
+    favorite_only: bool = False
+    selected_scholar_profile_id: int | None
+    unread_count: int
+    favorites_count: int
+    latest_count: int
+    new_count: int
+    total_count: int
+    page: int
+    page_size: int
+    snapshot: str
+    has_next: bool = False
+    has_prev: bool = False
+    publications: list[PublicationItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationsListEnvelope(BaseModel):
+    data: PublicationsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkAllReadData(BaseModel):
+    message: str
+    updated_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkAllReadEnvelope(BaseModel):
+    data: MarkAllReadData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationSelectionItem(BaseModel):
+    scholar_profile_id: int
+    publication_id: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadRequest(BaseModel):
+    selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadData(BaseModel):
+    message: str
+    requested_count: int
+    updated_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkSelectedReadEnvelope(BaseModel):
+    data: MarkSelectedReadData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfRequest(BaseModel):
+    scholar_profile_id: int = Field(ge=1)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfData(BaseModel):
+    message: str
+    queued: bool
+    resolved_pdf: bool
+    publication: PublicationItemData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RetryPublicationPdfEnvelope(BaseModel):
+    data: RetryPublicationPdfData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteRequest(BaseModel):
+    scholar_profile_id: int = Field(ge=1)
+    is_favorite: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteData(BaseModel):
+    message: str
+    publication: PublicationItemData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class TogglePublicationFavoriteEnvelope(BaseModel):
+    data: TogglePublicationFavoriteData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/api/schemas/__init__.py

@@ -1,7 +0,0 @@
-from app.api.schemas.admin import *  # noqa: F403
-from app.api.schemas.auth import *  # noqa: F403
-from app.api.schemas.common import *  # noqa: F403
-from app.api.schemas.publications import *  # noqa: F403
-from app.api.schemas.runs import *  # noqa: F403
-from app.api.schemas.scholars import *  # noqa: F403
-from app.api.schemas.settings import *  # noqa: F403

app/api/schemas/admin.py

@@ -1,314 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Any, Literal
-
-from pydantic import BaseModel, ConfigDict, Field, model_validator
-
-from app.api.schemas.common import ApiMeta
-from app.api.schemas.publications import DisplayIdentifierData
-
-
-class AdminUserData(BaseModel):
-    id: int
-    email: str
-    is_active: bool
-    is_admin: bool
-    created_at: datetime
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUsersListData(BaseModel):
-    users: list[AdminUserData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUsersListEnvelope(BaseModel):
-    data: AdminUsersListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserCreateRequest(BaseModel):
-    email: str = Field(max_length=254)
-    password: str = Field(min_length=8, max_length=128)
-    is_admin: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserActiveUpdateRequest(BaseModel):
-    is_active: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserEnvelope(BaseModel):
-    data: AdminUserData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminResetPasswordRequest(BaseModel):
-    new_password: str = Field(min_length=8, max_length=128)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminScholarHttpSettingsData(BaseModel):
-    user_agent: str
-    rotate_user_agent: bool
-    accept_language: str
-    cookie: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminScholarHttpSettingsEnvelope(BaseModel):
-    data: AdminScholarHttpSettingsData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminScholarHttpSettingsUpdateRequest(BaseModel):
-    user_agent: str
-    rotate_user_agent: bool
-    accept_language: str
-    cookie: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityCheckData(BaseModel):
-    name: str
-    count: int
-    severity: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityData(BaseModel):
-    status: str
-    checked_at: datetime
-    failures: list[str] = Field(default_factory=list)
-    warnings: list[str] = Field(default_factory=list)
-    checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityEnvelope(BaseModel):
-    data: AdminDbIntegrityData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobData(BaseModel):
-    id: int
-    job_name: str
-    requested_by: str | None
-    dry_run: bool
-    status: str
-    scope: dict[str, Any] = Field(default_factory=dict)
-    summary: dict[str, Any] = Field(default_factory=dict)
-    error_text: str | None
-    started_at: datetime | None
-    finished_at: datetime | None
-    created_at: datetime
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobsData(BaseModel):
-    jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobsEnvelope(BaseModel):
-    data: AdminDbRepairJobsData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueItemData(BaseModel):
-    publication_id: int
-    title: str
-    display_identifier: DisplayIdentifierData | None = None
-    pdf_url: str | None
-    status: str
-    attempt_count: int
-    last_failure_reason: str | None
-    last_failure_detail: str | None
-    last_source: str | None
-    requested_by_user_id: int | None
-    requested_by_email: str | None
-    queued_at: datetime | None
-    last_attempt_at: datetime | None
-    resolved_at: datetime | None
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueData(BaseModel):
-    items: list[AdminPdfQueueItemData] = Field(default_factory=list)
-    total_count: int = 0
-    page: int = 1
-    page_size: int = 100
-    has_next: bool = False
-    has_prev: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueEnvelope(BaseModel):
-    data: AdminPdfQueueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueRequeueData(BaseModel):
-    publication_id: int
-    queued: bool
-    status: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueRequeueEnvelope(BaseModel):
-    data: AdminPdfQueueRequeueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueBulkEnqueueData(BaseModel):
-    requested_count: int
-    queued_count: int
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
-    data: AdminPdfQueueBulkEnqueueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationLinksRequest(BaseModel):
-    scope_mode: Literal["single_user", "all_users"] = "single_user"
-    user_id: int | None = Field(default=None, ge=1)
-    scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
-    dry_run: bool = True
-    gc_orphan_publications: bool = False
-    requested_by: str | None = None
-    confirmation_text: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-    @model_validator(mode="after")
-    def validate_scope(self) -> AdminRepairPublicationLinksRequest:
-        if self.scope_mode == "single_user" and self.user_id is None:
-            raise ValueError("user_id is required when scope_mode=single_user.")
-        if self.scope_mode == "all_users" and self.user_id is not None:
-            raise ValueError("user_id must be omitted when scope_mode=all_users.")
-        if not self.dry_run and self.scope_mode == "all_users":
-            expected = "REPAIR ALL USERS"
-            provided = (self.confirmation_text or "").strip()
-            if provided != expected:
-                raise ValueError("confirmation_text must equal 'REPAIR ALL USERS' when applying a repair to all users.")
-        return self
-
-
-class AdminRepairPublicationLinksResultData(BaseModel):
-    job_id: int
-    status: str
-    scope: dict[str, Any] = Field(default_factory=dict)
-    summary: dict[str, Any] = Field(default_factory=dict)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationLinksEnvelope(BaseModel):
-    data: AdminRepairPublicationLinksResultData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminNearDuplicateClusterMemberData(BaseModel):
-    publication_id: int
-    title: str
-    year: int | None
-    citation_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminNearDuplicateClusterData(BaseModel):
-    cluster_key: str
-    winner_publication_id: int
-    member_count: int
-    similarity_score: float = Field(ge=0.0, le=1.0)
-    members: list[AdminNearDuplicateClusterMemberData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationNearDuplicatesRequest(BaseModel):
-    dry_run: bool = True
-    similarity_threshold: float = Field(default=0.78, ge=0.5, le=1.0)
-    min_shared_tokens: int = Field(default=3, ge=1, le=8)
-    max_year_delta: int = Field(default=1, ge=0, le=5)
-    max_clusters: int = Field(default=25, ge=1, le=200)
-    selected_cluster_keys: list[str] = Field(default_factory=list, max_length=200)
-    requested_by: str | None = None
-    confirmation_text: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-    @model_validator(mode="after")
-    def validate_apply_mode(self) -> AdminRepairPublicationNearDuplicatesRequest:
-        if self.dry_run:
-            return self
-        if not self.selected_cluster_keys:
-            raise ValueError("selected_cluster_keys is required when dry_run=false.")
-        expected = "MERGE SELECTED DUPLICATES"
-        provided = (self.confirmation_text or "").strip()
-        if provided != expected:
-            raise ValueError(
-                "confirmation_text must equal 'MERGE SELECTED DUPLICATES' when applying near-duplicate merges."
-            )
-        return self
-
-
-class AdminRepairPublicationNearDuplicatesResultData(BaseModel):
-    job_id: int
-    status: str
-    scope: dict[str, Any] = Field(default_factory=dict)
-    summary: dict[str, Any] = Field(default_factory=dict)
-    clusters: list[AdminNearDuplicateClusterData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationNearDuplicatesEnvelope(BaseModel):
-    data: AdminRepairPublicationNearDuplicatesResultData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/auth.py

@@ -1,73 +0,0 @@
-from __future__ import annotations
-
-from pydantic import BaseModel, ConfigDict, Field
-
-from app.api.schemas.common import ApiMeta
-
-
-class SessionUserData(BaseModel):
-    id: int
-    email: str
-    is_admin: bool
-    is_active: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AuthMeData(BaseModel):
-    authenticated: bool
-    csrf_token: str
-    user: SessionUserData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AuthMeEnvelope(BaseModel):
-    data: AuthMeData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class CsrfBootstrapData(BaseModel):
-    csrf_token: str
-    authenticated: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class CsrfBootstrapEnvelope(BaseModel):
-    data: CsrfBootstrapData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginRequest(BaseModel):
-    email: str = Field(max_length=254)
-    password: str = Field(min_length=8, max_length=128)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginData(BaseModel):
-    authenticated: bool
-    csrf_token: str
-    user: SessionUserData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginEnvelope(BaseModel):
-    data: LoginData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ChangePasswordRequest(BaseModel):
-    current_password: str = Field(min_length=8, max_length=128)
-    new_password: str = Field(min_length=8, max_length=128)
-    confirm_password: str = Field(min_length=8, max_length=128)
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/common.py

@@ -1,39 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-from pydantic import BaseModel, ConfigDict
-
-
-class ApiMeta(BaseModel):
-    request_id: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ApiErrorData(BaseModel):
-    code: str
-    message: str
-    details: Any | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ApiErrorEnvelope(BaseModel):
-    error: ApiErrorData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessageData(BaseModel):
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessageEnvelope(BaseModel):
-    data: MessageData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/publications.py

@@ -1,151 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime
-
-from pydantic import BaseModel, ConfigDict, Field
-
-from app.api.schemas.common import ApiMeta
-
-
-class DisplayIdentifierData(BaseModel):
-    kind: str
-    value: str
-    label: str
-    url: str | None
-    confidence_score: float = Field(ge=0.0, le=1.0)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationItemData(BaseModel):
-    publication_id: int
-    scholar_profile_id: int
-    scholar_label: str
-    title: str
-    year: int | None
-    citation_count: int
-    venue_text: str | None
-    pub_url: str | None
-    display_identifier: DisplayIdentifierData | None = None
-    pdf_url: str | None
-    pdf_status: str = "untracked"
-    pdf_attempt_count: int = 0
-    pdf_failure_reason: str | None = None
-    pdf_failure_detail: str | None = None
-    is_read: bool
-    is_favorite: bool = False
-    first_seen_at: datetime
-    is_new_in_latest_run: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationsListData(BaseModel):
-    mode: str
-    favorite_only: bool = False
-    selected_scholar_profile_id: int | None
-    unread_count: int
-    favorites_count: int
-    latest_count: int
-    new_count: int
-    total_count: int
-    page: int
-    page_size: int
-    snapshot: str
-    has_next: bool = False
-    has_prev: bool = False
-    publications: list[PublicationItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationsListEnvelope(BaseModel):
-    data: PublicationsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkAllReadData(BaseModel):
-    message: str
-    updated_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkAllReadEnvelope(BaseModel):
-    data: MarkAllReadData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationSelectionItem(BaseModel):
-    scholar_profile_id: int
-    publication_id: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadRequest(BaseModel):
-    selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadData(BaseModel):
-    message: str
-    requested_count: int
-    updated_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadEnvelope(BaseModel):
-    data: MarkSelectedReadData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfRequest(BaseModel):
-    scholar_profile_id: int = Field(ge=1)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfData(BaseModel):
-    message: str
-    queued: bool
-    resolved_pdf: bool
-    publication: PublicationItemData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfEnvelope(BaseModel):
-    data: RetryPublicationPdfData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteRequest(BaseModel):
-    scholar_profile_id: int = Field(ge=1)
-    is_favorite: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteData(BaseModel):
-    message: str
-    publication: PublicationItemData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteEnvelope(BaseModel):
-    data: TogglePublicationFavoriteData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/runs.py

@@ -1,226 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime
-
-from pydantic import BaseModel, ConfigDict, Field
-
-from app.api.schemas.common import ApiMeta
-
-
-class RunListItemData(BaseModel):
-    id: int
-    trigger_type: str
-    status: str
-    start_dt: datetime
-    end_dt: datetime | None
-    scholar_count: int
-    new_publication_count: int
-    failed_count: int
-    partial_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScrapeSafetyCountersData(BaseModel):
-    consecutive_blocked_runs: int = 0
-    consecutive_network_runs: int = 0
-    cooldown_entry_count: int = 0
-    blocked_start_count: int = 0
-    last_blocked_failure_count: int = 0
-    last_network_failure_count: int = 0
-    last_evaluated_run_id: int | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScrapeSafetyStateData(BaseModel):
-    cooldown_active: bool
-    cooldown_reason: str | None = None
-    cooldown_reason_label: str | None = None
-    cooldown_until: datetime | None = None
-    cooldown_remaining_seconds: int = 0
-    recommended_action: str | None = None
-    counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunsListData(BaseModel):
-    runs: list[RunListItemData]
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunsListEnvelope(BaseModel):
-    data: RunsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunSummaryData(BaseModel):
-    succeeded_count: int
-    failed_count: int
-    partial_count: int
-    failed_state_counts: dict[str, int] = Field(default_factory=dict)
-    failed_reason_counts: dict[str, int] = Field(default_factory=dict)
-    scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
-    retry_counts: dict[str, int] = Field(default_factory=dict)
-    alert_thresholds: dict[str, int] = Field(default_factory=dict)
-    alert_flags: dict[str, bool] = Field(default_factory=dict)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunAttemptLogData(BaseModel):
-    attempt: int
-    cstart: int
-    state: str | None = None
-    state_reason: str | None = None
-    status_code: int | None = None
-    fetch_error: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunPageLogData(BaseModel):
-    page: int
-    cstart: int
-    state: str
-    state_reason: str | None = None
-    status_code: int | None = None
-    publication_count: int = 0
-    attempt_count: int = 0
-    has_show_more_button: bool = False
-    articles_range: str | None = None
-    warning_codes: list[str] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDebugData(BaseModel):
-    status_code: int | None = None
-    final_url: str | None = None
-    fetch_error: str | None = None
-    body_sha256: str | None = None
-    body_length: int | None = None
-    has_show_more_button: bool | None = None
-    articles_range: str | None = None
-    state_reason: str | None = None
-    warning_codes: list[str] = Field(default_factory=list)
-    marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
-    page_logs: list[RunPageLogData] = Field(default_factory=list)
-    attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunScholarResultData(BaseModel):
-    scholar_profile_id: int
-    scholar_id: str
-    state: str
-    state_reason: str | None = None
-    outcome: str
-    attempt_count: int = 0
-    publication_count: int = 0
-    start_cstart: int = 0
-    continuation_cstart: int | None = None
-    continuation_enqueued: bool = False
-    continuation_cleared: bool = False
-    warnings: list[str] = Field(default_factory=list)
-    error: str | None = None
-    debug: RunDebugData | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDetailData(BaseModel):
-    run: RunListItemData
-    summary: RunSummaryData
-    scholar_results: list[RunScholarResultData]
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDetailEnvelope(BaseModel):
-    data: RunDetailData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ManualRunData(BaseModel):
-    run_id: int
-    status: str
-    scholar_count: int
-    succeeded_count: int
-    failed_count: int
-    partial_count: int
-    new_publication_count: int
-    reused_existing_run: bool
-    idempotency_key: str | None = None
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ManualRunEnvelope(BaseModel):
-    data: ManualRunData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueItemData(BaseModel):
-    id: int
-    scholar_profile_id: int
-    scholar_label: str
-    status: str
-    reason: str
-    dropped_reason: str | None
-    attempt_count: int
-    resume_cstart: int
-    next_attempt_dt: datetime | None
-    updated_at: datetime
-    last_error: str | None
-    last_run_id: int | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueListData(BaseModel):
-    queue_items: list[QueueItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueListEnvelope(BaseModel):
-    data: QueueListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueItemEnvelope(BaseModel):
-    data: QueueItemData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueClearData(BaseModel):
-    queue_item_id: int
-    previous_status: str
-    status: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueClearEnvelope(BaseModel):
-    data: QueueClearData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/scholars.py

@@ -1,189 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime
-
-from pydantic import BaseModel, ConfigDict, Field, model_validator
-
-from app.api.schemas.common import ApiMeta
-
-
-class ScholarItemData(BaseModel):
-    id: int
-    scholar_id: str
-    display_name: str | None
-    profile_image_url: str | None
-    profile_image_source: str
-    is_enabled: bool
-    baseline_completed: bool
-    last_run_dt: datetime | None
-    last_run_status: str | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarsListData(BaseModel):
-    scholars: list[ScholarItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarsListEnvelope(BaseModel):
-    data: ScholarsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarCreateRequest(BaseModel):
-    scholar_id: str
-    profile_image_url: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarEnvelope(BaseModel):
-    data: ScholarItemData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchCandidateData(BaseModel):
-    scholar_id: str
-    display_name: str
-    affiliation: str | None
-    email_domain: str | None
-    cited_by_count: int | None
-    interests: list[str] = Field(default_factory=list)
-    profile_url: str
-    profile_image_url: str | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchData(BaseModel):
-    query: str
-    state: str
-    state_reason: str
-    action_hint: str | None = None
-    candidates: list[ScholarSearchCandidateData]
-    warnings: list[str] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchEnvelope(BaseModel):
-    data: ScholarSearchData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarImageUrlUpdateRequest(BaseModel):
-    image_url: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarExportItemData(BaseModel):
-    scholar_id: str
-    display_name: str | None = None
-    is_enabled: bool = True
-    profile_image_override_url: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationExportItemData(BaseModel):
-    scholar_id: str
-    cluster_id: str | None = None
-    fingerprint_sha256: str | None = None
-    title: str
-    year: int | None = None
-    citation_count: int = 0
-    author_text: str | None = None
-    venue_text: str | None = None
-    pub_url: str | None = None
-    pdf_url: str | None = None
-    is_read: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataExportData(BaseModel):
-    schema_version: int
-    exported_at: str
-    scholars: list[ScholarExportItemData]
-    publications: list[PublicationExportItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataExportEnvelope(BaseModel):
-    data: DataExportData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarBulkIdsRequest(BaseModel):
-    scholar_profile_ids: list[int] = Field(..., min_length=1, max_length=500)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarBulkToggleRequest(BaseModel):
-    scholar_profile_ids: list[int] = Field(..., min_length=1, max_length=500)
-    is_enabled: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarBulkCountData(BaseModel):
-    deleted_count: int = 0
-    updated_count: int = 0
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarBulkCountEnvelope(BaseModel):
-    data: ScholarBulkCountData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataImportRequest(BaseModel):
-    schema_version: int | None = None
-    exported_at: str | None = None
-    scholars: list[ScholarExportItemData] = Field(default_factory=list)
-    publications: list[PublicationExportItemData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-    @model_validator(mode="before")
-    @classmethod
-    def unwrap_export_envelope(cls, values: dict) -> dict:
-        """Accept the full export envelope format (with data/meta wrapper)."""
-        if isinstance(values, dict) and "data" in values and isinstance(values["data"], dict):
-            return values["data"]
-        return values
-
-
-class DataImportResultData(BaseModel):
-    scholars_created: int
-    scholars_updated: int
-    publications_created: int
-    publications_updated: int
-    links_created: int
-    links_updated: int
-    skipped_records: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataImportEnvelope(BaseModel):
-    data: DataImportResultData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")

app/api/schemas/settings.py

@@ -1,54 +0,0 @@
-from __future__ import annotations
-
-from pydantic import BaseModel, ConfigDict
-
-from app.api.schemas.common import ApiMeta
-from app.api.schemas.runs import ScrapeSafetyStateData
-
-
-class SettingsPolicyData(BaseModel):
-    min_run_interval_minutes: int
-    min_request_delay_seconds: int
-    automation_allowed: bool
-    manual_run_allowed: bool
-    blocked_failure_threshold: int
-    network_failure_threshold: int
-    cooldown_blocked_seconds: int
-    cooldown_network_seconds: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsData(BaseModel):
-    auto_run_enabled: bool
-    run_interval_minutes: int
-    request_delay_seconds: int
-    nav_visible_pages: list[str]
-    policy: SettingsPolicyData
-    safety_state: ScrapeSafetyStateData
-
-    openalex_api_key: str | None = None
-    crossref_api_token: str | None = None
-    crossref_api_mailto: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsEnvelope(BaseModel):
-    data: SettingsData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsUpdateRequest(BaseModel):
-    auto_run_enabled: bool
-    run_interval_minutes: int
-    request_delay_seconds: int
-    nav_visible_pages: list[str] | None = None
-
-    openalex_api_key: str | None = None
-    crossref_api_token: str | None = None
-    crossref_api_mailto: str | None = None
-
-    model_config = ConfigDict(extra="forbid")

app/db/background_session.py

@@ -1,57 +0,0 @@
-"""Semaphore-gated DB sessions for background tasks.
-
-Background tasks (ingestion, PDF resolution, scheduler) acquire a semaphore
-permit before checking out a database connection.  The semaphore cap is
-``pool_size + max_overflow - reserved_for_api``, which guarantees that at
-least *reserved_for_api* connections remain available for API request
-handlers at all times.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-from collections.abc import AsyncIterator
-from contextlib import asynccontextmanager
-
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.session import get_session_factory
-from app.logging_utils import structured_log
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-_semaphore: asyncio.Semaphore | None = None
-
-
-def _build_semaphore() -> asyncio.Semaphore:
-    pool_capacity = max(1, settings.database_pool_size) + max(0, settings.database_pool_max_overflow)
-    reserved = max(0, settings.database_reserved_api_connections)
-    limit = max(1, pool_capacity - reserved)
-    structured_log(
-        logger,
-        "info",
-        "db.background_semaphore_initialized",
-        pool_capacity=pool_capacity,
-        reserved_for_api=reserved,
-        background_limit=limit,
-    )
-    return asyncio.Semaphore(limit)
-
-
-def get_background_semaphore() -> asyncio.Semaphore:
-    global _semaphore
-    if _semaphore is None:
-        _semaphore = _build_semaphore()
-    return _semaphore
-
-
-@asynccontextmanager
-async def background_session() -> AsyncIterator[AsyncSession]:
-    """Yield a DB session after acquiring the background semaphore."""
-    semaphore = get_background_semaphore()
-    async with semaphore:
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            yield session

app/db/session.py

@@ -85,7 +85,7 @@ async def check_database() -> bool:
             result = await conn.execute(text("SELECT 1"))
             return result.scalar_one() == 1
     except Exception:
-        structured_log(logger, "exception", "db.healthcheck_failed")
+        logger.exception("db.healthcheck_failed")
         return False
 
 

app/http/middleware.py

@@ -74,13 +74,13 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
             response = await call_next(request)
         except Exception:
             duration_ms = int((time.perf_counter() - start) * 1000)
-            structured_log(
+            logger.exception(
-                logger,
-                "exception",
                 "request.failed",
-                method=request.method,
+                extra={
-                path=request.url.path,
+                    "method": request.method,
-                duration_ms=duration_ms,
+                    "path": request.url.path,
+                    "duration_ms": duration_ms,
+                },
             )
             raise
         else:

app/main.py

@@ -73,33 +73,8 @@ async def lifespan(_: FastAPI):
             )
             await session.commit()
             structured_log(logger, "info", "app.startup_orphaned_runs_cleaned")
-    except Exception as exc:
+    except Exception as e:
-        structured_log(
+        logger.error(f"Failed to clean orphaned runs at startup: {e}")
-            logger,
-            "error",
-            "app.startup_orphaned_runs_cleanup_failed",
-            error=str(exc),
-        )
-
-    try:
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            await session.execute(
-                text(
-                    "UPDATE publication_pdf_jobs SET status = 'queued'"
-                    " WHERE status = 'running'"
-                    " AND (last_attempt_at IS NULL OR last_attempt_at < NOW() - INTERVAL '10 minutes')"
-                )
-            )
-            await session.commit()
-            structured_log(logger, "info", "app.startup_stuck_pdf_jobs_recovered")
-    except Exception as exc:
-        structured_log(
-            logger,
-            "error",
-            "app.startup_stuck_pdf_jobs_recovery_failed",
-            error=str(exc),
-        )
 
     await scheduler_service.start()
     yield

app/security/csrf.py

@@ -6,10 +6,9 @@ from urllib.parse import parse_qs
 
 from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.requests import Request
-from starlette.responses import PlainTextResponse, Response
+from starlette.responses import JSONResponse, PlainTextResponse, Response
 from starlette.types import Message
 
-from app.api.responses import error_response
 from app.logging_utils import structured_log
 
 CSRF_SESSION_KEY = "csrf_token"
@@ -89,10 +88,7 @@ class CSRFMiddleware(BaseHTTPMiddleware):
         content_type = request.headers.get("content-type", "")
         if not content_type.startswith("application/x-www-form-urlencoded"):
             return None
-        try:
+        parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True)
-            parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True)
-        except (UnicodeDecodeError, ValueError):
-            return None
         values = parsed.get(CSRF_FORM_FIELD)
         if not values:
             return None
@@ -118,11 +114,19 @@ class CSRFMiddleware(BaseHTTPMiddleware):
         message: str,
     ) -> Response:
         if request.url.path.startswith("/api/"):
-            return error_response(
+            request_state = getattr(request, "state", None)
-                request,
+            request_id = getattr(request_state, "request_id", None) if request_state else None
+            return JSONResponse(
+                {
+                    "error": {
+                        "code": code,
+                        "message": message,
+                        "details": None,
+                    },
+                    "meta": {
+                        "request_id": request_id,
+                    },
+                },
                 status_code=403,
-                code=code,
-                message=message,
-                details=None,
             )
         return PlainTextResponse(message, status_code=403)

app/services/arxiv/client.py

@@ -232,7 +232,7 @@ async def _request_arxiv_feed(
         timeout_value = _timeout_seconds(timeout_seconds)
         headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{_contact_email(request_email)})"}
         async with httpx.AsyncClient(timeout=timeout_value, follow_redirects=True, headers=headers) as client:
-            return await client.get(_ARXIV_API_URL, params=params)  # type: ignore[arg-type]
+            return await client.get(_ARXIV_API_URL, params=params)
 
     return await run_with_global_arxiv_limit(
         fetch=_fetch,

app/services/arxiv/rate_limit.py

@@ -53,7 +53,7 @@ async def get_arxiv_cooldown_status(*, now_utc: datetime | None = None) -> Arxiv
         result = await db_session.execute(
             select(ArxivRuntimeState.cooldown_until).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY)
         )
-        cooldown_until = _normalize_datetime(result.scalar_one_or_none())
+    cooldown_until = _normalize_datetime(result.scalar_one_or_none())
     remaining_seconds = _cooldown_remaining_seconds(cooldown_until, now_utc=timestamp)
     return ArxivCooldownStatus(
         is_active=remaining_seconds > 0.0,
@@ -68,56 +68,36 @@ async def _run_serialized_fetch(
     source_path: str,
 ) -> tuple[httpx.Response, bool]:
     session_factory = get_session_factory()
-    async with session_factory() as lock_session:
+    async with session_factory() as db_session, db_session.begin():
-        await _acquire_arxiv_lock(lock_session)
+            await _acquire_arxiv_lock(db_session)
-        try:
+            runtime_state = await _load_runtime_state_for_update(db_session)
-            async with session_factory() as db_session, db_session.begin():
+            wait_seconds = await _wait_for_allowed_slot_or_raise(
-                runtime_state = await _load_runtime_state_for_update(db_session)
+                runtime_state,
-                wait_seconds = await _wait_for_allowed_slot_or_raise(
+                source_path=source_path,
-                    runtime_state,
+            )
-                    source_path=source_path,
-                )
-                runtime_state.next_allowed_at = datetime.now(UTC) + timedelta(seconds=_min_interval_seconds())
-
-            if wait_seconds > 0:
-                await asyncio.sleep(wait_seconds)
             response = await fetch()
-
+            hit_rate_limit = _record_post_response_state(
-            async with session_factory() as db_session, db_session.begin():
+                runtime_state,
-                runtime_state = await _load_runtime_state_for_update(db_session)
+                response_status=int(response.status_code),
-                hit_rate_limit = _record_post_response_state(
+                source_path=source_path,
-                    runtime_state,
+            )
-                    response_status=int(response.status_code),
+            structured_log(
-                    source_path=source_path,
+                logger,
-                )
+                "info",
-                cooldown_until_value = runtime_state.cooldown_until
+                "arxiv.request_completed",
-        finally:
+                status_code=int(response.status_code),
-            await _release_arxiv_lock(lock_session)
+                wait_seconds=wait_seconds,
-    structured_log(
+                cooldown_remaining_seconds=_cooldown_remaining_seconds(
-        logger,
+                    runtime_state.cooldown_until, now_utc=datetime.now(UTC)
-        "info",
+                ),
-        "arxiv.request_completed",
+                source_path=source_path,
-        status_code=int(response.status_code),
+            )
-        wait_seconds=wait_seconds,
+            return response, hit_rate_limit
-        cooldown_remaining_seconds=_cooldown_remaining_seconds(cooldown_until_value, now_utc=datetime.now(UTC)),
-        source_path=source_path,
-    )
-    return response, hit_rate_limit
 
 
 async def _acquire_arxiv_lock(db_session: AsyncSession) -> None:
     await db_session.execute(
-        text("SELECT pg_advisory_lock(:namespace, :lock_key)"),
+        text("SELECT pg_advisory_xact_lock(:namespace, :lock_key)"),
-        {
-            "namespace": ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
-            "lock_key": ARXIV_RATE_LIMIT_LOCK_KEY,
-        },
-    )
-
-
-async def _release_arxiv_lock(db_session: AsyncSession) -> None:
-    await db_session.execute(
-        text("SELECT pg_advisory_unlock(:namespace, :lock_key)"),
         {
             "namespace": ARXIV_RATE_LIMIT_LOCK_NAMESPACE,
             "lock_key": ARXIV_RATE_LIMIT_LOCK_KEY,
@@ -164,6 +144,8 @@ async def _wait_for_allowed_slot_or_raise(
         source_path=source_path,
         cooldown_remaining_seconds=0.0,
     )
+    if wait_seconds > 0:
+        await asyncio.sleep(wait_seconds)
     return wait_seconds
 
 

app/services/crossref/application.py

@@ -20,7 +20,7 @@ if TYPE_CHECKING:
     from app.services.publications.types import PublicationListItem, UnreadPublicationItem
 
 TOKEN_RE = re.compile(r"[a-z0-9]+")
-NON_ALNUM_RE = re.compile(r"[^a-z0-9\s]+")
+NON_ALNUM_RE = re.compile(r"[^a-z0-9\\s]+")
 STOP_WORDS = {"the", "and", "for", "with", "from", "method", "study", "analysis"}
 _RATE_LOCK = threading.Lock()
 _LAST_REQUEST_AT = 0.0
@@ -35,9 +35,8 @@ def _rate_limit_wait(min_interval_seconds: float) -> None:
     with _RATE_LOCK:
         elapsed = time.monotonic() - _LAST_REQUEST_AT
         remaining = interval - elapsed
-    if remaining > 0:
+        if remaining > 0:
-        time.sleep(remaining)
+            time.sleep(remaining)
-    with _RATE_LOCK:
         _LAST_REQUEST_AT = time.monotonic()
 
 
@@ -351,8 +350,7 @@ async def _fetch_items(
             ),
             timeout=timeout,
         )
-    except Exception as exc:
+    except Exception:
-        structured_log(logger, "warning", "crossref.fetch_failed", error=str(exc))
         return []
 
 

app/services/dbops/application.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from datetime import UTC, datetime
 from typing import Any
 
-from sqlalchemy import CursorResult, delete, exists, func, select, update
+from sqlalchemy import delete, exists, func, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db.models import DataRepairJob, IngestionQueueItem, Publication, ScholarProfile, ScholarPublication
@@ -152,7 +152,7 @@ def _job_summary(
 
 
 async def _delete_links_for_targets(db_session: AsyncSession, *, target_scholar_profile_ids: list[int]) -> int:
-    result: CursorResult[Any] = await db_session.execute(  # type: ignore[assignment]
+    result = await db_session.execute(
         delete(ScholarPublication).where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids))
     )
     return int(result.rowcount or 0)
@@ -167,7 +167,7 @@ async def _delete_queue_for_targets(
     stmt = delete(IngestionQueueItem).where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids))
     if user_id is not None:
         stmt = stmt.where(IngestionQueueItem.user_id == user_id)
-    result: CursorResult[Any] = await db_session.execute(stmt)  # type: ignore[assignment]
+    result = await db_session.execute(stmt)
     return int(result.rowcount or 0)
 
 
@@ -180,7 +180,7 @@ async def _reset_scholar_tracking_state(
     stmt = update(ScholarProfile).where(ScholarProfile.id.in_(target_scholar_profile_ids))
     if user_id is not None:
         stmt = stmt.where(ScholarProfile.user_id == user_id)
-    result: CursorResult[Any] = await db_session.execute(  # type: ignore[assignment]
+    result = await db_session.execute(
         stmt.values(
             baseline_completed=False,
             last_initial_page_fingerprint_sha256=None,
@@ -193,7 +193,7 @@ async def _reset_scholar_tracking_state(
 
 
 async def _delete_orphan_publications(db_session: AsyncSession) -> int:
-    result: CursorResult[Any] = await db_session.execute(  # type: ignore[assignment]
+    result = await db_session.execute(
         delete(Publication).where(~exists(select(1).where(ScholarPublication.publication_id == Publication.id)))
     )
     return int(result.rowcount or 0)

app/services/dbops/near_duplicate_repair.py

@@ -6,14 +6,12 @@ from typing import Any
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db.models import DataRepairJob
-from app.services.dbops.application import (
-    REPAIR_STATUS_COMPLETED,
-    REPAIR_STATUS_FAILED,
-    REPAIR_STATUS_PLANNED,
-    REPAIR_STATUS_RUNNING,
-)
 from app.services.publications import dedup as dedup_service
 
+REPAIR_STATUS_PLANNED = "planned"
+REPAIR_STATUS_RUNNING = "running"
+REPAIR_STATUS_COMPLETED = "completed"
+REPAIR_STATUS_FAILED = "failed"
 NEAR_DUP_JOB_NAME = "repair_publication_near_duplicates"
 NEAR_DUP_DEFAULT_MAX_CLUSTERS = 25
 
@@ -162,10 +160,7 @@ async def _complete_job(
 
 
 async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None:
-    from sqlalchemy.orm import make_transient
-
     await db_session.rollback()
-    make_transient(job)
     job.status = REPAIR_STATUS_FAILED
     job.error_text = str(error)
     job.finished_at = _utcnow()

app/services/ingestion/enrichment.py

@@ -1,409 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-import re
-from datetime import UTC, datetime, timedelta
-
-from sqlalchemy import or_, select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import (
-    CrawlRun,
-    Publication,
-    RunStatus,
-    ScholarProfile,
-    ScholarPublication,
-)
-from app.logging_utils import structured_log
-from app.services.arxiv.errors import ArxivRateLimitError
-from app.services.publication_identifiers import application as identifier_service
-from app.services.runs.events import run_events
-from app.services.scholar.parser import PublicationCandidate
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-
-def _sanitize_titles(publications: list) -> list[str]:
-    titles = []
-    for p in publications:
-        raw = getattr(p, "title_raw", None) or getattr(p, "title", None)
-        if not raw or not raw.strip():
-            continue
-        safe = " ".join(re.sub(r"[^\w\s]", " ", raw).split())
-        if safe:
-            titles.append(safe)
-    return titles
-
-
-# OpenAlex (and most HTTP servers) struggle with very long query strings.
-# Non-ASCII characters expand significantly when percent-encoded in URLs,
-# so a fixed batch count is unreliable.  Instead we cap the combined
-# byte-length of the pipe-joined title filter value to stay well within
-# safe URL limits (~4 KiB of filter text ≈ ~6 KiB when percent-encoded).
-_MAX_TITLE_FILTER_BYTES = 4_000
-
-
-def _chunk_titles_by_url_length(
-    titles: list[str],
-    max_bytes: int = _MAX_TITLE_FILTER_BYTES,
-) -> list[list[str]]:
-    """Split *titles* into chunks whose pipe-joined UTF-8 size stays under *max_bytes*."""
-    chunks: list[list[str]] = []
-    current_chunk: list[str] = []
-    current_bytes = 0
-
-    for title in titles:
-        title_bytes = len(title.encode("utf-8"))
-        # Account for the pipe separator between titles
-        separator = 3 if current_chunk else 0  # len("|".encode) but url-encoded as %7C = 3
-        if current_chunk and current_bytes + separator + title_bytes > max_bytes:
-            chunks.append(current_chunk)
-            current_chunk = [title]
-            current_bytes = title_bytes
-        else:
-            current_chunk.append(title)
-            current_bytes += separator + title_bytes
-
-    if current_chunk:
-        chunks.append(current_chunk)
-    return chunks
-
-
-class EnrichmentRunner:
-    """Post-run OpenAlex enrichment logic.
-
-    Receives service dependencies at construction so it can be tested
-    independently of ``ScholarIngestionService``.
-    """
-
-    async def run_is_canceled(
-        self,
-        db_session: AsyncSession,
-        *,
-        run_id: int,
-    ) -> bool:
-        check_result = await db_session.execute(select(CrawlRun.status).where(CrawlRun.id == run_id))
-        status = check_result.scalar_one_or_none()
-        if status is None:
-            raise RuntimeError(f"Missing crawl_run for run_id={run_id}.")
-        return status == RunStatus.CANCELED
-
-    async def _load_unenriched_publications(
-        self,
-        db_session: AsyncSession,
-        *,
-        run_id: int,
-    ) -> tuple[int, list[Publication]]:
-        run_result = await db_session.execute(select(CrawlRun.user_id).where(CrawlRun.id == run_id))
-        user_id = run_result.scalar_one()
-        cooldown_threshold = datetime.now(UTC) - timedelta(days=7)
-        stmt = (
-            select(Publication)
-            .join(ScholarPublication)
-            .join(ScholarProfile, ScholarPublication.scholar_profile_id == ScholarProfile.id)
-            .where(
-                ScholarProfile.user_id == user_id,
-                Publication.openalex_enriched.is_(False),
-                or_(
-                    Publication.openalex_last_attempt_at.is_(None),
-                    Publication.openalex_last_attempt_at < cooldown_threshold,
-                ),
-            )
-            .distinct()
-        )
-        result = await db_session.execute(stmt)
-        return user_id, list(result.scalars().all())
-
-    async def _enrich_batch(
-        self,
-        db_session: AsyncSession,
-        *,
-        batch: list[Publication],
-        run_id: int,
-        openalex_works: list,
-        now: datetime,
-        arxiv_lookup_allowed: bool,
-    ) -> tuple[bool, bool]:
-        from app.services.openalex.matching import find_best_match
-
-        for p in batch:
-            if await self.run_is_canceled(db_session, run_id=run_id):
-                structured_log(logger, "info", "ingestion.enrichment_aborted", run_id=run_id)
-                return False, arxiv_lookup_allowed
-            p.openalex_last_attempt_at = now
-            arxiv_lookup_allowed = await self._discover_identifiers_for_enrichment(
-                db_session,
-                publication=p,
-                run_id=run_id,
-                allow_arxiv_lookup=arxiv_lookup_allowed,
-            )
-            match = find_best_match(
-                target_title=p.title_raw,
-                target_year=p.year,
-                target_authors=p.author_text or "",
-                candidates=openalex_works,
-            )
-            if match:
-                p.year = match.publication_year if match.publication_year is not None else p.year
-                p.citation_count = match.cited_by_count if match.cited_by_count is not None else p.citation_count
-                p.pdf_url = match.oa_url if match.oa_url is not None else p.pdf_url
-                p.openalex_enriched = True
-        return True, arxiv_lookup_allowed
-
-    async def _flush_and_sweep_duplicates(
-        self,
-        db_session: AsyncSession,
-        *,
-        run_id: int,
-    ) -> None:
-        await db_session.flush()
-        from app.services.publications.dedup import sweep_identifier_duplicates
-
-        merge_count = await sweep_identifier_duplicates(db_session)
-        if merge_count:
-            structured_log(
-                logger,
-                "info",
-                "ingestion.identifier_dedup_sweep",
-                merged_count=merge_count,
-                run_id=run_id,
-            )
-
-    async def enrich_pending_publications(
-        self,
-        db_session: AsyncSession,
-        *,
-        run_id: int,
-        openalex_api_key: str | None = None,
-    ) -> None:
-        from app.services.openalex.client import (
-            OpenAlexBudgetExhaustedError,
-            OpenAlexClient,
-            OpenAlexRateLimitError,
-        )
-
-        _, publications = await self._load_unenriched_publications(db_session, run_id=run_id)
-        if not publications:
-            return
-
-        resolved_key = openalex_api_key or settings.openalex_api_key
-        client = OpenAlexClient(api_key=resolved_key, mailto=settings.crossref_api_mailto)
-        now = datetime.now(UTC)
-        arxiv_lookup_allowed = True
-
-        all_titles = _sanitize_titles(publications)
-        title_chunks = _chunk_titles_by_url_length(all_titles)
-
-        # Build a mapping from sanitized title → publications for batch association
-        title_to_pubs: dict[str, list[Publication]] = {}
-        for p in publications:
-            raw = getattr(p, "title_raw", None) or getattr(p, "title", None)
-            if not raw or not raw.strip():
-                continue
-            safe = " ".join(re.sub(r"[^\w\s]", " ", raw).split())
-            if safe:
-                title_to_pubs.setdefault(safe, []).append(p)
-
-        for title_chunk in title_chunks:
-            if await self.run_is_canceled(db_session, run_id=run_id):
-                structured_log(logger, "info", "ingestion.enrichment_aborted", run_id=run_id)
-                return
-            batch = []
-            for t in title_chunk:
-                batch.extend(title_to_pubs.get(t, []))
-            if not batch:
-                continue
-            try:
-                openalex_works = await client.get_works_by_filter(
-                    {"title.search": "|".join(title_chunk)}, limit=len(title_chunk) * 3
-                )
-            except OpenAlexBudgetExhaustedError:
-                structured_log(logger, "warning", "ingestion.openalex_budget_exhausted", run_id=run_id)
-                break
-            except OpenAlexRateLimitError:
-                structured_log(logger, "warning", "ingestion.openalex_rate_limited", run_id=run_id)
-                await asyncio.sleep(60)
-                continue
-            except Exception as e:
-                structured_log(logger, "warning", "ingestion.openalex_enrichment_failed", error=str(e), run_id=run_id)
-                for p in batch:
-                    p.openalex_last_attempt_at = now
-                continue
-            should_continue, arxiv_lookup_allowed = await self._enrich_batch(
-                db_session,
-                batch=batch,
-                run_id=run_id,
-                openalex_works=openalex_works,
-                now=now,
-                arxiv_lookup_allowed=arxiv_lookup_allowed,
-            )
-            if not should_continue:
-                return
-
-        await self._flush_and_sweep_duplicates(db_session, run_id=run_id)
-
-    async def _discover_identifiers_for_enrichment(
-        self,
-        db_session: AsyncSession,
-        *,
-        publication: Publication,
-        run_id: int,
-        allow_arxiv_lookup: bool,
-    ) -> bool:
-        if not allow_arxiv_lookup:
-            await identifier_service.sync_identifiers_for_publication_fields(
-                db_session,
-                publication=publication,
-            )
-            await self._publish_identifier_update_event(
-                db_session,
-                run_id=run_id,
-                publication_id=int(publication.id),
-            )
-            return False
-        try:
-            await identifier_service.discover_and_sync_identifiers_for_publication(
-                db_session,
-                publication=publication,
-                scholar_label=publication.author_text or "",
-            )
-            await self._publish_identifier_update_event(
-                db_session,
-                run_id=run_id,
-                publication_id=int(publication.id),
-            )
-            return True
-        except ArxivRateLimitError:
-            structured_log(
-                logger,
-                "warning",
-                "ingestion.arxiv_rate_limited",
-                run_id=run_id,
-                publication_id=int(publication.id),
-                detail="arXiv temporarily disabled for remaining enrichment pass",
-            )
-            await identifier_service.sync_identifiers_for_publication_fields(
-                db_session,
-                publication=publication,
-            )
-            await self._publish_identifier_update_event(
-                db_session,
-                run_id=run_id,
-                publication_id=int(publication.id),
-            )
-            return False
-
-    async def _publish_identifier_update_event(
-        self,
-        db_session: AsyncSession,
-        *,
-        run_id: int,
-        publication_id: int,
-    ) -> None:
-        display = await identifier_service.display_identifier_for_publication_id(
-            db_session,
-            publication_id=publication_id,
-        )
-        if display is None:
-            return
-        await run_events.publish(
-            run_id=run_id,
-            event_type="identifier_updated",
-            data={
-                "publication_id": int(publication_id),
-                "display_identifier": {
-                    "kind": display.kind,
-                    "value": display.value,
-                    "label": display.label,
-                    "url": display.url,
-                    "confidence_score": float(display.confidence_score),
-                },
-            },
-        )
-
-    async def enrich_publications_with_openalex(
-        self,
-        scholar: ScholarProfile,
-        publications: list[PublicationCandidate],
-    ) -> list[PublicationCandidate]:
-        if not publications:
-            return publications
-
-        from app.services.openalex.client import OpenAlexClient
-        from app.services.openalex.matching import find_best_match
-
-        client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto)
-
-        all_titles = _sanitize_titles(publications)
-        title_chunks = _chunk_titles_by_url_length(all_titles)
-
-        # Build title → publication mapping for batch association
-        title_to_pubs: dict[str, list[PublicationCandidate]] = {}
-        for p in publications:
-            raw = getattr(p, "title", None)
-            if not raw or not raw.strip():
-                continue
-            safe = " ".join(re.sub(r"[^\w\s]", " ", raw).split())
-            if safe:
-                title_to_pubs.setdefault(safe, []).append(p)
-
-        # Collect publications that had no sanitizable title — pass through as-is
-        pubs_with_titles = set()
-        for titles in title_to_pubs.values():
-            for p in titles:
-                pubs_with_titles.add(id(p))
-
-        enriched: list[PublicationCandidate] = []
-
-        for title_chunk in title_chunks:
-            batch = []
-            for t in title_chunk:
-                batch.extend(title_to_pubs.get(t, []))
-            if not batch:
-                continue
-
-            try:
-                openalex_works = await client.get_works_by_filter(
-                    {"title.search": "|".join(title_chunk)}, limit=len(title_chunk) * 3
-                )
-            except Exception as e:
-                structured_log(
-                    logger,
-                    "warning",
-                    "ingestion.openalex_enrichment_failed",
-                    error=str(e),
-                    batch_size=len(batch),
-                    scholar_id=scholar.id,
-                )
-                openalex_works = []
-
-            for p in batch:
-                match = find_best_match(
-                    target_title=p.title,
-                    target_year=p.year,
-                    target_authors=p.authors_text or (scholar.display_name or scholar.scholar_id),
-                    candidates=openalex_works,
-                )
-                if match:
-                    new_p = PublicationCandidate(
-                        title=p.title,
-                        title_url=p.title_url,
-                        cluster_id=p.cluster_id,
-                        year=match.publication_year if match.publication_year is not None else p.year,
-                        citation_count=match.cited_by_count if match.cited_by_count is not None else p.citation_count,
-                        authors_text=p.authors_text,
-                        venue_text=p.venue_text,
-                        pdf_url=match.oa_url if match.oa_url is not None else p.pdf_url,
-                    )
-                    enriched.append(new_p)
-                else:
-                    enriched.append(p)
-
-        # Append publications that had no sanitizable title (pass-through)
-        for p in publications:
-            if id(p) not in pubs_with_titles:
-                enriched.append(p)
-
-        return enriched

app/services/ingestion/page_fetch.py

@@ -1,223 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-from typing import Any
-
-from app.logging_utils import structured_log
-from app.services.scholar.parser import (
-    ParsedProfilePage,
-    ParseState,
-    ScholarParserError,
-    parse_profile_page,
-)
-from app.services.scholar.source import FetchResult, ScholarSource
-
-logger = logging.getLogger(__name__)
-
-
-class PageFetcher:
-    """Fetches and parses a single Google Scholar profile page with retry logic."""
-
-    def __init__(self, *, source: ScholarSource) -> None:
-        self._source = source
-
-    async def fetch_profile_page(
-        self,
-        *,
-        scholar_id: str,
-        cstart: int,
-        page_size: int,
-    ) -> FetchResult:
-        try:
-            page_fetcher = getattr(self._source, "fetch_profile_page_html", None)
-            if callable(page_fetcher):
-                return await page_fetcher(
-                    scholar_id,
-                    cstart=cstart,
-                    pagesize=page_size,
-                )
-            if cstart <= 0:
-                return await self._source.fetch_profile_html(scholar_id)
-            return FetchResult(
-                requested_url=(
-                    f"https://scholar.google.com/citations?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}"
-                ),
-                status_code=None,
-                final_url=None,
-                body="",
-                error="source_does_not_support_pagination",
-            )
-        except Exception as exc:
-            structured_log(
-                logger,
-                "exception",
-                "ingestion.fetch_unexpected_error",
-                scholar_id=scholar_id,
-                cstart=cstart,
-                page_size=page_size,
-            )
-            return FetchResult(
-                requested_url=(
-                    f"https://scholar.google.com/citations?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}"
-                ),
-                status_code=None,
-                final_url=None,
-                body="",
-                error=str(exc),
-            )
-
-    # ── Parse helpers ────────────────────────────────────────────────
-
-    def parse_page_or_layout_error(
-        self,
-        *,
-        fetch_result: FetchResult,
-    ) -> ParsedProfilePage:
-        try:
-            return parse_profile_page(fetch_result)
-        except ScholarParserError as exc:
-            return self._parsed_page_from_parser_error(code=exc.code)
-
-    @staticmethod
-    def _parsed_page_from_parser_error(*, code: str) -> ParsedProfilePage:
-        return ParsedProfilePage(
-            state=ParseState.LAYOUT_CHANGED,
-            state_reason=code,
-            profile_name=None,
-            profile_image_url=None,
-            publications=[],
-            marker_counts={},
-            warnings=[code],
-            has_show_more_button=False,
-            has_operation_error_banner=False,
-            articles_range=None,
-        )
-
-    # ── Retry logic ──────────────────────────────────────────────────
-
-    @staticmethod
-    def _should_retry(
-        *,
-        parsed_page: ParsedProfilePage,
-        network_attempt_count: int,
-        rate_limit_attempt_count: int,
-        network_error_retries: int,
-        rate_limit_retries: int,
-    ) -> bool:
-        if parsed_page.state == ParseState.NETWORK_ERROR:
-            return network_attempt_count <= network_error_retries
-        if (
-            parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA
-            and parsed_page.state_reason == "blocked_http_429_rate_limited"
-        ):
-            return rate_limit_attempt_count <= rate_limit_retries
-        return False
-
-    @staticmethod
-    async def _sleep_backoff(
-        *,
-        scholar_id: str,
-        cstart: int,
-        network_attempt_count: int,
-        rate_limit_attempt_count: int,
-        retry_backoff_seconds: float,
-        rate_limit_backoff_seconds: float,
-        state_reason: str,
-    ) -> None:
-        if state_reason == "blocked_http_429_rate_limited":
-            sleep_seconds = rate_limit_backoff_seconds * rate_limit_attempt_count
-            attempt_label = rate_limit_attempt_count
-        else:
-            sleep_seconds = retry_backoff_seconds * (2 ** (network_attempt_count - 1))
-            attempt_label = network_attempt_count
-
-        structured_log(
-            logger,
-            "warning",
-            "ingestion.scholar_retry_scheduled",
-            scholar_id=scholar_id,
-            cstart=cstart,
-            attempt_count=attempt_label,
-            sleep_seconds=sleep_seconds,
-            state_reason=state_reason,
-        )
-        if sleep_seconds > 0:
-            await asyncio.sleep(sleep_seconds)
-
-    @staticmethod
-    def _classify_attempt(
-        parsed_page: ParsedProfilePage,
-        *,
-        network_attempts: int,
-        rate_limit_attempts: int,
-    ) -> tuple[int, int, int]:
-        if parsed_page.state == ParseState.NETWORK_ERROR:
-            network_attempts += 1
-            return network_attempts, rate_limit_attempts, network_attempts
-        if (
-            parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA
-            and parsed_page.state_reason == "blocked_http_429_rate_limited"
-        ):
-            rate_limit_attempts += 1
-            return network_attempts, rate_limit_attempts, rate_limit_attempts
-        return network_attempts, rate_limit_attempts, network_attempts + rate_limit_attempts + 1
-
-    async def fetch_and_parse_with_retry(
-        self,
-        *,
-        scholar_id: str,
-        cstart: int,
-        page_size: int,
-        network_error_retries: int,
-        retry_backoff_seconds: float,
-        rate_limit_retries: int,
-        rate_limit_backoff_seconds: float,
-    ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]:
-        network_attempts = 0
-        rate_limit_attempts = 0
-        attempt_log: list[dict[str, Any]] = []
-        fetch_result: FetchResult | None = None
-        parsed_page: ParsedProfilePage | None = None
-
-        while True:
-            fetch_result = await self.fetch_profile_page(
-                scholar_id=scholar_id,
-                cstart=cstart,
-                page_size=page_size,
-            )
-            parsed_page = self.parse_page_or_layout_error(fetch_result=fetch_result)
-            network_attempts, rate_limit_attempts, total_attempts = self._classify_attempt(
-                parsed_page, network_attempts=network_attempts, rate_limit_attempts=rate_limit_attempts
-            )
-            attempt_log.append(
-                {
-                    "attempt": total_attempts,
-                    "cstart": cstart,
-                    "state": parsed_page.state.value,
-                    "state_reason": parsed_page.state_reason,
-                    "status_code": fetch_result.status_code,
-                    "fetch_error": fetch_result.error,
-                }
-            )
-            if not self._should_retry(
-                parsed_page=parsed_page,
-                network_attempt_count=network_attempts,
-                rate_limit_attempt_count=rate_limit_attempts,
-                network_error_retries=network_error_retries,
-                rate_limit_retries=rate_limit_retries,
-            ):
-                break
-            await self._sleep_backoff(
-                scholar_id=scholar_id,
-                cstart=cstart,
-                network_attempt_count=network_attempts,
-                rate_limit_attempt_count=rate_limit_attempts,
-                retry_backoff_seconds=max(float(retry_backoff_seconds), 0.0),
-                rate_limit_backoff_seconds=max(float(rate_limit_backoff_seconds), 0.0),
-                state_reason=parsed_page.state_reason,
-            )
-
-        if fetch_result is None or parsed_page is None:
-            raise RuntimeError("Fetch-and-parse retry loop produced no result.")
-        return fetch_result, parsed_page, attempt_log

app/services/ingestion/pagination.py

@@ -1,504 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-from typing import Any
-
-from app.db.models import CrawlRun, RunStatus, ScholarProfile
-from app.logging_utils import structured_log
-from app.services.ingestion.fingerprints import (
-    _dedupe_publication_candidates,
-    _next_cstart_value,
-    build_initial_page_fingerprint,
-)
-from app.services.ingestion.page_fetch import PageFetcher
-from app.services.ingestion.types import PagedLoopState, PagedParseResult
-from app.services.scholar.parser import ParsedProfilePage, ParseState
-from app.services.scholar.source import FetchResult, ScholarSource
-
-logger = logging.getLogger(__name__)
-
-
-class PaginationEngine:
-    """Fetches and paginates Google Scholar profile pages.
-
-    Pure HTTP + parsing — no DB writes except via the provided upsert callback.
-    """
-
-    def __init__(self, *, source: ScholarSource) -> None:
-        self._source = source
-        self._fetcher = PageFetcher(source=source)
-
-    # ── No-change / initial failure short-circuits ───────────────────
-
-    @staticmethod
-    def _should_skip_no_change(
-        *,
-        start_cstart: int,
-        first_page_fingerprint_sha256: str | None,
-        previous_initial_page_fingerprint_sha256: str | None,
-        parsed_page: ParsedProfilePage,
-    ) -> bool:
-        return (
-            start_cstart <= 0
-            and first_page_fingerprint_sha256 is not None
-            and previous_initial_page_fingerprint_sha256 is not None
-            and first_page_fingerprint_sha256 == previous_initial_page_fingerprint_sha256
-            and parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS}
-        )
-
-    @staticmethod
-    def _skip_no_change_result(
-        *,
-        fetch_result: FetchResult,
-        parsed_page: ParsedProfilePage,
-        first_page_fingerprint_sha256: str | None,
-        attempt_log: list[dict[str, Any]],
-        page_logs: list[dict[str, Any]],
-    ) -> PagedParseResult:
-        return PagedParseResult(
-            fetch_result=fetch_result,
-            parsed_page=parsed_page,
-            first_page_fetch_result=fetch_result,
-            first_page_parsed_page=parsed_page,
-            first_page_fingerprint_sha256=first_page_fingerprint_sha256,
-            publications=[],
-            attempt_log=attempt_log,
-            page_logs=page_logs,
-            pages_fetched=1,
-            pages_attempted=1,
-            has_more_remaining=False,
-            pagination_truncated_reason=None,
-            continuation_cstart=None,
-            skipped_no_change=True,
-            discovered_publication_count=0,
-        )
-
-    @staticmethod
-    def _initial_failure_result(
-        *,
-        fetch_result: FetchResult,
-        parsed_page: ParsedProfilePage,
-        first_page_fingerprint_sha256: str | None,
-        start_cstart: int,
-        attempt_log: list[dict[str, Any]],
-        page_logs: list[dict[str, Any]],
-    ) -> PagedParseResult:
-        continuation_cstart = start_cstart if parsed_page.state == ParseState.NETWORK_ERROR else None
-        return PagedParseResult(
-            fetch_result=fetch_result,
-            parsed_page=parsed_page,
-            first_page_fetch_result=fetch_result,
-            first_page_parsed_page=parsed_page,
-            first_page_fingerprint_sha256=first_page_fingerprint_sha256,
-            publications=[],
-            attempt_log=attempt_log,
-            page_logs=page_logs,
-            pages_fetched=0,
-            pages_attempted=1,
-            has_more_remaining=False,
-            pagination_truncated_reason=None,
-            continuation_cstart=continuation_cstart,
-            skipped_no_change=False,
-            discovered_publication_count=0,
-        )
-
-    # ── Loop state management ────────────────────────────────────────
-
-    @staticmethod
-    def _build_loop_state(
-        *,
-        start_cstart: int,
-        fetch_result: FetchResult,
-        parsed_page: ParsedProfilePage,
-        attempt_log: list[dict[str, Any]],
-        page_logs: list[dict[str, Any]],
-    ) -> PagedLoopState:
-        next_cstart = _next_cstart_value(
-            articles_range=parsed_page.articles_range,
-            fallback=start_cstart + len(parsed_page.publications),
-        )
-        return PagedLoopState(
-            fetch_result=fetch_result,
-            parsed_page=parsed_page,
-            attempt_log=attempt_log,
-            page_logs=page_logs,
-            publications=list(parsed_page.publications),
-            pages_fetched=1,
-            pages_attempted=1,
-            current_cstart=start_cstart,
-            next_cstart=next_cstart,
-        )
-
-    @staticmethod
-    def _set_truncated_state(
-        *,
-        state: PagedLoopState,
-        reason: str,
-        continuation_cstart: int,
-    ) -> None:
-        state.has_more_remaining = True
-        state.pagination_truncated_reason = reason
-        state.continuation_cstart = continuation_cstart
-
-    def _should_stop_pagination(self, *, state: PagedLoopState, bounded_max_pages: int) -> bool:
-        if state.pages_fetched >= bounded_max_pages:
-            self._set_truncated_state(
-                state=state,
-                reason="max_pages_reached",
-                continuation_cstart=(
-                    state.next_cstart if state.next_cstart > state.current_cstart else state.current_cstart
-                ),
-            )
-            return True
-        if state.next_cstart <= state.current_cstart:
-            self._set_truncated_state(
-                state=state,
-                reason="pagination_cursor_stalled",
-                continuation_cstart=state.current_cstart,
-            )
-            return True
-        return False
-
-    # ── Multi-page fetch helpers ─────────────────────────────────────
-
-    async def _fetch_next_page(
-        self,
-        *,
-        scholar_id: str,
-        state: PagedLoopState,
-        request_delay_seconds: int,
-        bounded_page_size: int,
-        network_error_retries: int,
-        retry_backoff_seconds: float,
-        rate_limit_retries: int,
-        rate_limit_backoff_seconds: float,
-    ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]:
-        if request_delay_seconds > 0:
-            await asyncio.sleep(float(request_delay_seconds))
-        state.current_cstart = state.next_cstart
-        return await self._fetcher.fetch_and_parse_with_retry(
-            scholar_id=scholar_id,
-            cstart=state.current_cstart,
-            page_size=bounded_page_size,
-            network_error_retries=network_error_retries,
-            retry_backoff_seconds=retry_backoff_seconds,
-            rate_limit_retries=rate_limit_retries,
-            rate_limit_backoff_seconds=rate_limit_backoff_seconds,
-        )
-
-    @staticmethod
-    def _record_next_page(
-        *,
-        state: PagedLoopState,
-        fetch_result: FetchResult,
-        parsed_page: ParsedProfilePage,
-        page_attempt_log: list[dict[str, Any]],
-    ) -> None:
-        state.pages_attempted += 1
-        state.attempt_log.extend(page_attempt_log)
-        state.page_logs.append(
-            {
-                "page": state.pages_attempted,
-                "cstart": state.current_cstart,
-                "state": parsed_page.state.value,
-                "state_reason": parsed_page.state_reason,
-                "status_code": fetch_result.status_code,
-                "publication_count": len(parsed_page.publications),
-                "articles_range": parsed_page.articles_range,
-                "has_show_more_button": parsed_page.has_show_more_button,
-                "warning_codes": parsed_page.warnings,
-                "attempt_count": len(page_attempt_log),
-            }
-        )
-        state.fetch_result = fetch_result
-        state.parsed_page = parsed_page
-
-    def _handle_page_state_transition(self, *, state: PagedLoopState) -> bool:
-        if state.parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
-            self._set_truncated_state(
-                state=state,
-                reason=f"page_state_{state.parsed_page.state.value}",
-                continuation_cstart=state.current_cstart,
-            )
-            return True
-        if state.parsed_page.state == ParseState.NO_RESULTS and len(state.parsed_page.publications) == 0:
-            state.pages_fetched += 1
-            return True
-        state.pages_fetched += 1
-        state.publications.extend(state.parsed_page.publications)
-        state.next_cstart = _next_cstart_value(
-            articles_range=state.parsed_page.articles_range,
-            fallback=state.current_cstart + len(state.parsed_page.publications),
-        )
-        return False
-
-    async def _fetch_initial_page_context(
-        self,
-        *,
-        scholar_id: str,
-        start_cstart: int,
-        bounded_page_size: int,
-        network_error_retries: int,
-        retry_backoff_seconds: float,
-        rate_limit_retries: int,
-        rate_limit_backoff_seconds: float,
-    ) -> tuple[FetchResult, ParsedProfilePage, str | None, list[dict[str, Any]], list[dict[str, Any]]]:
-        fetch_result, parsed_page, first_attempt_log = await self._fetcher.fetch_and_parse_with_retry(
-            scholar_id=scholar_id,
-            cstart=start_cstart,
-            page_size=bounded_page_size,
-            network_error_retries=network_error_retries,
-            retry_backoff_seconds=retry_backoff_seconds,
-            rate_limit_retries=rate_limit_retries,
-            rate_limit_backoff_seconds=rate_limit_backoff_seconds,
-        )
-        first_page_fingerprint_sha256 = build_initial_page_fingerprint(parsed_page)
-        attempt_log = list(first_attempt_log)
-        page_logs = [
-            {
-                "page": 1,
-                "cstart": start_cstart,
-                "state": parsed_page.state.value,
-                "state_reason": parsed_page.state_reason,
-                "status_code": fetch_result.status_code,
-                "publication_count": len(parsed_page.publications),
-                "articles_range": parsed_page.articles_range,
-                "has_show_more_button": parsed_page.has_show_more_button,
-                "warning_codes": parsed_page.warnings,
-                "attempt_count": len(first_attempt_log),
-            }
-        ]
-        return fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs
-
-    # ── Pagination loop ──────────────────────────────────────────────
-
-    @staticmethod
-    async def _upsert_page_publications(
-        db_session: Any,
-        *,
-        run: CrawlRun,
-        scholar: ScholarProfile,
-        publications: list,
-        seen_canonical: set[str],
-        state: PagedLoopState,
-        upsert_publications_fn: Any,
-    ) -> None:
-        deduped = _dedupe_publication_candidates(list(publications), seen_canonical=seen_canonical)
-        if deduped:
-            discovered_count = await upsert_publications_fn(db_session, run=run, scholar=scholar, publications=deduped)
-            state.discovered_publication_count += discovered_count
-
-    async def _paginate_loop(
-        self,
-        *,
-        scholar: ScholarProfile,
-        run: CrawlRun,
-        db_session: Any,
-        state: PagedLoopState,
-        bounded_max_pages: int,
-        request_delay_seconds: int,
-        bounded_page_size: int,
-        network_error_retries: int,
-        retry_backoff_seconds: float,
-        rate_limit_retries: int,
-        rate_limit_backoff_seconds: float,
-        upsert_publications_fn: Any,
-    ) -> None:
-        seen_canonical: set[str] = set()
-
-        if state.parsed_page.publications:
-            await self._upsert_page_publications(
-                db_session,
-                run=run,
-                scholar=scholar,
-                publications=state.parsed_page.publications,
-                seen_canonical=seen_canonical,
-                state=state,
-                upsert_publications_fn=upsert_publications_fn,
-            )
-
-        while state.parsed_page.has_show_more_button:
-            await db_session.refresh(run)
-            if run.status == RunStatus.CANCELED:
-                structured_log(
-                    logger,
-                    "info",
-                    "ingestion.pagination_canceled",
-                    run_id=run.id,
-                )
-                self._set_truncated_state(
-                    state=state,
-                    reason="run_canceled",
-                    continuation_cstart=state.current_cstart,
-                )
-                return
-
-            if self._should_stop_pagination(state=state, bounded_max_pages=bounded_max_pages):
-                return
-            next_fetch_result, next_parsed_page, next_attempt_log = await self._fetch_next_page(
-                scholar_id=scholar.scholar_id,
-                state=state,
-                request_delay_seconds=request_delay_seconds,
-                bounded_page_size=bounded_page_size,
-                network_error_retries=network_error_retries,
-                retry_backoff_seconds=retry_backoff_seconds,
-                rate_limit_retries=rate_limit_retries,
-                rate_limit_backoff_seconds=rate_limit_backoff_seconds,
-            )
-            self._record_next_page(
-                state=state,
-                fetch_result=next_fetch_result,
-                parsed_page=next_parsed_page,
-                page_attempt_log=next_attempt_log,
-            )
-
-            if self._handle_page_state_transition(state=state):
-                return
-
-            if next_parsed_page.publications:
-                await self._upsert_page_publications(
-                    db_session,
-                    run=run,
-                    scholar=scholar,
-                    publications=next_parsed_page.publications,
-                    seen_canonical=seen_canonical,
-                    state=state,
-                    upsert_publications_fn=upsert_publications_fn,
-                )
-
-    @staticmethod
-    def _result_from_pagination_state(
-        *,
-        state: PagedLoopState,
-        first_page_fetch_result: FetchResult,
-        first_page_parsed_page: ParsedProfilePage,
-        first_page_fingerprint_sha256: str | None,
-    ) -> PagedParseResult:
-        return PagedParseResult(
-            fetch_result=state.fetch_result,
-            parsed_page=state.parsed_page,
-            first_page_fetch_result=first_page_fetch_result,
-            first_page_parsed_page=first_page_parsed_page,
-            first_page_fingerprint_sha256=first_page_fingerprint_sha256,
-            publications=_dedupe_publication_candidates(state.publications),
-            attempt_log=state.attempt_log,
-            page_logs=state.page_logs,
-            pages_fetched=state.pages_fetched,
-            pages_attempted=state.pages_attempted,
-            has_more_remaining=state.has_more_remaining,
-            pagination_truncated_reason=state.pagination_truncated_reason,
-            continuation_cstart=state.continuation_cstart,
-            skipped_no_change=False,
-            discovered_publication_count=state.discovered_publication_count,
-        )
-
-    def _short_circuit_initial_page(
-        self,
-        *,
-        start_cstart: int,
-        previous_initial_page_fingerprint_sha256: str | None,
-        fetch_result: FetchResult,
-        parsed_page: ParsedProfilePage,
-        first_page_fingerprint_sha256: str | None,
-        attempt_log: list[dict[str, Any]],
-        page_logs: list[dict[str, Any]],
-    ) -> PagedParseResult | None:
-        if self._should_skip_no_change(
-            start_cstart=start_cstart,
-            first_page_fingerprint_sha256=first_page_fingerprint_sha256,
-            previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256,
-            parsed_page=parsed_page,
-        ):
-            return self._skip_no_change_result(
-                fetch_result=fetch_result,
-                parsed_page=parsed_page,
-                first_page_fingerprint_sha256=first_page_fingerprint_sha256,
-                attempt_log=attempt_log,
-                page_logs=page_logs,
-            )
-        if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
-            return self._initial_failure_result(
-                fetch_result=fetch_result,
-                parsed_page=parsed_page,
-                first_page_fingerprint_sha256=first_page_fingerprint_sha256,
-                start_cstart=start_cstart,
-                attempt_log=attempt_log,
-                page_logs=page_logs,
-            )
-        return None
-
-    # ── Public entry point ───────────────────────────────────────────
-
-    async def fetch_and_parse_all_pages(
-        self,
-        *,
-        scholar: ScholarProfile,
-        run: CrawlRun,
-        db_session: Any,
-        start_cstart: int,
-        request_delay_seconds: int,
-        network_error_retries: int,
-        retry_backoff_seconds: float,
-        rate_limit_retries: int,
-        rate_limit_backoff_seconds: float,
-        max_pages: int,
-        page_size: int,
-        previous_initial_page_fingerprint_sha256: str | None = None,
-        upsert_publications_fn: Any = None,
-    ) -> PagedParseResult:
-        bounded_max_pages = max(1, int(max_pages))
-        bounded_page_size = max(1, int(page_size))
-        (
-            fetch_result,
-            parsed_page,
-            first_page_fingerprint_sha256,
-            attempt_log,
-            page_logs,
-        ) = await self._fetch_initial_page_context(
-            scholar_id=scholar.scholar_id,
-            start_cstart=start_cstart,
-            bounded_page_size=bounded_page_size,
-            network_error_retries=network_error_retries,
-            retry_backoff_seconds=retry_backoff_seconds,
-            rate_limit_retries=rate_limit_retries,
-            rate_limit_backoff_seconds=rate_limit_backoff_seconds,
-        )
-        shortcut_result = self._short_circuit_initial_page(
-            start_cstart=start_cstart,
-            previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256,
-            fetch_result=fetch_result,
-            parsed_page=parsed_page,
-            first_page_fingerprint_sha256=first_page_fingerprint_sha256,
-            attempt_log=attempt_log,
-            page_logs=page_logs,
-        )
-        if shortcut_result is not None:
-            return shortcut_result
-        state = self._build_loop_state(
-            start_cstart=start_cstart,
-            fetch_result=fetch_result,
-            parsed_page=parsed_page,
-            attempt_log=attempt_log,
-            page_logs=page_logs,
-        )
-        await self._paginate_loop(
-            scholar=scholar,
-            run=run,
-            db_session=db_session,
-            state=state,
-            bounded_max_pages=bounded_max_pages,
-            request_delay_seconds=request_delay_seconds,
-            bounded_page_size=bounded_page_size,
-            network_error_retries=network_error_retries,
-            retry_backoff_seconds=retry_backoff_seconds,
-            rate_limit_retries=rate_limit_retries,
-            rate_limit_backoff_seconds=rate_limit_backoff_seconds,
-            upsert_publications_fn=upsert_publications_fn,
-        )
-        return self._result_from_pagination_state(
-            state=state,
-            first_page_fetch_result=fetch_result,
-            first_page_parsed_page=parsed_page,
-            first_page_fingerprint_sha256=first_page_fingerprint_sha256,
-        )

app/services/ingestion/preflight.py

@@ -1,70 +0,0 @@
-from __future__ import annotations
-
-import logging
-from dataclasses import dataclass
-
-from app.logging_utils import structured_log
-from app.services.scholar.source import FetchResult, ScholarSource
-from app.services.scholar.state_detection import (
-    classify_block_or_captcha_reason,
-    is_hard_challenge_reason,
-)
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass(frozen=True)
-class PreflightResult:
-    passed: bool
-    block_reason: str | None
-    status_code: int | None
-
-
-def _evaluate_fetch_result(fetch_result: FetchResult) -> PreflightResult:
-    if fetch_result.status_code is None:
-        return PreflightResult(passed=True, block_reason=None, status_code=None)
-    block_reason = classify_block_or_captcha_reason(
-        status_code=fetch_result.status_code,
-        final_url=(fetch_result.final_url or "").lower(),
-        body_lowered=fetch_result.body.lower(),
-    )
-    if block_reason is not None and is_hard_challenge_reason(block_reason):
-        return PreflightResult(
-            passed=False,
-            block_reason=block_reason,
-            status_code=fetch_result.status_code,
-        )
-    return PreflightResult(
-        passed=True,
-        block_reason=None,
-        status_code=fetch_result.status_code,
-    )
-
-
-async def check_scholar_reachable(
-    source: ScholarSource,
-    *,
-    scholar_id: str,
-) -> PreflightResult:
-    """Single-request probe to detect active Scholar blocks before a full run."""
-    structured_log(logger, "info", "ingestion.preflight_started", scholar_id=scholar_id)
-    fetch_result = await source.fetch_profile_html(scholar_id)
-    result = _evaluate_fetch_result(fetch_result)
-    if result.passed:
-        structured_log(
-            logger,
-            "info",
-            "ingestion.preflight_passed",
-            scholar_id=scholar_id,
-            status_code=result.status_code,
-        )
-    else:
-        structured_log(
-            logger,
-            "warning",
-            "ingestion.preflight_failed",
-            scholar_id=scholar_id,
-            block_reason=result.block_reason,
-            status_code=result.status_code,
-        )
-    return result

app/services/ingestion/publication_upsert.py

@@ -1,244 +0,0 @@
-from __future__ import annotations
-
-import hashlib
-import logging
-from datetime import UTC, datetime
-
-from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import CrawlRun, Publication, ScholarProfile, ScholarPublication
-from app.services.ingestion.fingerprints import (
-    build_publication_fingerprint,
-    build_publication_url,
-    canonical_title_for_dedup,
-    normalize_title,
-)
-from app.services.publication_identifiers import application as identifier_service
-from app.services.runs.events import run_events
-from app.services.scholar.parser import PublicationCandidate
-
-logger = logging.getLogger(__name__)
-
-
-def validate_publication_candidate(candidate: PublicationCandidate) -> None:
-    if not candidate.title.strip():
-        raise RuntimeError("Publication candidate is missing title.")
-    if candidate.citation_count is not None and int(candidate.citation_count) < 0:
-        raise RuntimeError("Publication candidate has negative citation_count.")
-
-
-async def find_publication_by_cluster(
-    db_session: AsyncSession,
-    *,
-    cluster_id: str | None,
-) -> Publication | None:
-    if not cluster_id:
-        return None
-    result = await db_session.execute(select(Publication).where(Publication.cluster_id == cluster_id))
-    return result.scalar_one_or_none()
-
-
-async def find_publication_by_fingerprint(
-    db_session: AsyncSession,
-    *,
-    fingerprint: str,
-) -> Publication | None:
-    result = await db_session.execute(select(Publication).where(Publication.fingerprint_sha256 == fingerprint))
-    return result.scalar_one_or_none()
-
-
-def select_existing_publication(
-    *,
-    cluster_publication: Publication | None,
-    fingerprint_publication: Publication | None,
-) -> Publication | None:
-    if cluster_publication is not None:
-        return cluster_publication
-    return fingerprint_publication
-
-
-def compute_canonical_title_hash(title: str) -> str:
-    canonical = canonical_title_for_dedup(title)
-    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
-
-
-async def find_publication_by_canonical_title_hash(
-    db_session: AsyncSession,
-    *,
-    canonical_title_hash: str,
-) -> Publication | None:
-    result = await db_session.execute(
-        select(Publication).where(Publication.canonical_title_hash == canonical_title_hash)
-    )
-    return result.scalar_one_or_none()
-
-
-async def create_publication(
-    db_session: AsyncSession,
-    *,
-    candidate: PublicationCandidate,
-    fingerprint: str,
-) -> Publication:
-    publication = Publication(
-        cluster_id=candidate.cluster_id,
-        fingerprint_sha256=fingerprint,
-        title_raw=candidate.title,
-        title_normalized=normalize_title(candidate.title),
-        canonical_title_hash=compute_canonical_title_hash(candidate.title),
-        year=candidate.year,
-        citation_count=int(candidate.citation_count or 0),
-        author_text=candidate.authors_text,
-        venue_text=candidate.venue_text,
-        pub_url=build_publication_url(candidate.title_url),
-        pdf_url=None,
-    )
-    db_session.add(publication)
-    await db_session.flush()
-    return publication
-
-
-def update_existing_publication(
-    *,
-    publication: Publication,
-    candidate: PublicationCandidate,
-) -> None:
-    if candidate.cluster_id and publication.cluster_id is None:
-        publication.cluster_id = candidate.cluster_id
-    if not publication.title_raw:
-        publication.title_raw = candidate.title
-        publication.title_normalized = normalize_title(candidate.title)
-    if candidate.year is not None:
-        publication.year = candidate.year
-    if candidate.citation_count is not None:
-        publication.citation_count = int(candidate.citation_count)
-    if candidate.authors_text:
-        publication.author_text = candidate.authors_text
-    if candidate.venue_text:
-        publication.venue_text = candidate.venue_text
-    if candidate.title_url:
-        publication.pub_url = build_publication_url(candidate.title_url)
-
-
-async def resolve_publication(
-    db_session: AsyncSession,
-    candidate: PublicationCandidate,
-) -> Publication:
-    validate_publication_candidate(candidate)
-    fingerprint = build_publication_fingerprint(candidate)
-    cluster_publication = await find_publication_by_cluster(
-        db_session,
-        cluster_id=candidate.cluster_id,
-    )
-    fingerprint_publication = await find_publication_by_fingerprint(
-        db_session,
-        fingerprint=fingerprint,
-    )
-    publication = select_existing_publication(
-        cluster_publication=cluster_publication,
-        fingerprint_publication=fingerprint_publication,
-    )
-    if publication is None:
-        canonical_hash = compute_canonical_title_hash(candidate.title)
-        publication = await find_publication_by_canonical_title_hash(
-            db_session,
-            canonical_title_hash=canonical_hash,
-        )
-    if publication is None:
-        created = await create_publication(
-            db_session,
-            candidate=candidate,
-            fingerprint=fingerprint,
-        )
-        await identifier_service.sync_identifiers_for_publication_fields(
-            db_session,
-            publication=created,
-        )
-        return created
-    update_existing_publication(
-        publication=publication,
-        candidate=candidate,
-    )
-    await identifier_service.sync_identifiers_for_publication_fields(
-        db_session,
-        publication=publication,
-    )
-    return publication
-
-
-async def upsert_profile_publications(
-    db_session: AsyncSession,
-    *,
-    run: CrawlRun,
-    scholar: ScholarProfile,
-    publications: list[PublicationCandidate],
-) -> int:
-    seen_publication_ids: set[int] = set()
-    discovered_count = 0
-
-    try:
-        for candidate in publications:
-            publication = await resolve_publication(db_session, candidate)
-            if publication.id in seen_publication_ids:
-                continue
-            seen_publication_ids.add(publication.id)
-
-            link_result = await db_session.execute(
-                select(ScholarPublication).where(
-                    ScholarPublication.scholar_profile_id == scholar.id,
-                    ScholarPublication.publication_id == publication.id,
-                )
-            )
-            link = link_result.scalar_one_or_none()
-            if link is not None:
-                continue
-
-            link = ScholarPublication(
-                scholar_profile_id=scholar.id,
-                publication_id=publication.id,
-                is_read=False,
-                first_seen_run_id=run.id,
-            )
-            db_session.add(link)
-            discovered_count += 1
-
-            await flush_discovered_publication(
-                db_session,
-                run=run,
-                scholar=scholar,
-                publication=publication,
-            )
-
-        if not scholar.baseline_completed:
-            scholar.baseline_completed = True
-
-        await db_session.commit()
-    except Exception:
-        await db_session.rollback()
-        raise
-
-    return discovered_count
-
-
-async def flush_discovered_publication(
-    db_session: AsyncSession,
-    *,
-    run: CrawlRun,
-    scholar: ScholarProfile,
-    publication: Publication,
-) -> None:
-    run.new_pub_count = int(run.new_pub_count or 0) + 1
-    await db_session.flush()
-    await run_events.publish(
-        run_id=run.id,
-        event_type="publication_discovered",
-        data={
-            "publication_id": publication.id,
-            "title": publication.title_raw,
-            "pub_url": publication.pub_url,
-            "scholar_profile_id": scholar.id,
-            "scholar_label": scholar.display_name or scholar.scholar_id,
-            "first_seen_at": datetime.now(UTC).isoformat(),
-            "new_publication_count": int(run.new_pub_count or 0),
-        },
-    )

app/services/ingestion/queue.py

@@ -196,7 +196,6 @@ async def list_due_jobs(
             IngestionQueueItem.id.asc(),
         )
         .limit(limit)
-        .with_for_update(skip_locked=True)
     )
     rows = list(result.scalars().all())
     jobs: list[ContinuationQueueJob] = []

app/services/ingestion/queue_runner.py

@@ -1,381 +0,0 @@
-from __future__ import annotations
-
-import logging
-from datetime import UTC, datetime
-
-from sqlalchemy import select
-
-from app.db.background_session import background_session
-from app.db.models import QueueItemStatus, RunTriggerType, ScholarProfile, UserSetting
-from app.logging_utils import structured_log
-from app.services.ingestion import queue as queue_service
-from app.services.ingestion.application import (
-    RunAlreadyInProgressError,
-    RunBlockedBySafetyPolicyError,
-    ScholarIngestionService,
-)
-from app.services.scholar.source import LiveScholarSource
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-
-def effective_request_delay_seconds(value: int | None, *, floor: int) -> int:
-    try:
-        parsed = int(value) if value is not None else floor
-    except (TypeError, ValueError):
-        parsed = floor
-    return max(floor, parsed)
-
-
-class QueueJobRunner:
-    def __init__(
-        self,
-        *,
-        tick_seconds: int,
-        network_error_retries: int,
-        retry_backoff_seconds: float,
-        max_pages_per_scholar: int,
-        page_size: int,
-        continuation_queue_enabled: bool,
-        continuation_base_delay_seconds: int,
-        continuation_max_delay_seconds: int,
-        continuation_max_attempts: int,
-        queue_batch_size: int,
-    ) -> None:
-        self._tick_seconds = tick_seconds
-        self._network_error_retries = network_error_retries
-        self._retry_backoff_seconds = retry_backoff_seconds
-        self._max_pages_per_scholar = max_pages_per_scholar
-        self._page_size = page_size
-        self._continuation_queue_enabled = continuation_queue_enabled
-        self._continuation_base_delay_seconds = continuation_base_delay_seconds
-        self._continuation_max_delay_seconds = continuation_max_delay_seconds
-        self._continuation_max_attempts = continuation_max_attempts
-        self._queue_batch_size = queue_batch_size
-        self._source = LiveScholarSource()
-
-    async def drain_continuation_queue(self) -> None:
-        now = datetime.now(UTC)
-        async with background_session() as session:
-            jobs = await queue_service.list_due_jobs(
-                session,
-                now=now,
-                limit=self._queue_batch_size,
-            )
-        for job in jobs:
-            await self._run_queue_job(job)
-
-    async def _drop_queue_job_if_max_attempts(
-        self,
-        job: queue_service.ContinuationQueueJob,
-    ) -> bool:
-        if job.attempt_count < self._continuation_max_attempts:
-            return False
-        async with background_session() as session:
-            dropped = await queue_service.mark_dropped(
-                session,
-                job_id=job.id,
-                reason="max_attempts_reached",
-            )
-            await session.commit()
-        if dropped is not None:
-            structured_log(
-                logger,
-                "warning",
-                "scheduler.queue_item_dropped_max_attempts",
-                queue_item_id=job.id,
-                user_id=job.user_id,
-                scholar_profile_id=job.scholar_profile_id,
-                attempt_count=job.attempt_count,
-                max_attempts=self._continuation_max_attempts,
-            )
-        return True
-
-    async def _mark_queue_job_retrying(
-        self,
-        job: queue_service.ContinuationQueueJob,
-    ) -> bool:
-        async with background_session() as session:
-            queue_item = await queue_service.mark_retrying(session, job_id=job.id)
-            await session.commit()
-        if queue_item is None:
-            return False
-        return queue_item.status != QueueItemStatus.DROPPED.value
-
-    async def _queue_job_has_available_scholar(
-        self,
-        job: queue_service.ContinuationQueueJob,
-    ) -> bool:
-        async with background_session() as session:
-            scholar_result = await session.execute(
-                select(ScholarProfile.id).where(
-                    ScholarProfile.user_id == job.user_id,
-                    ScholarProfile.id == job.scholar_profile_id,
-                    ScholarProfile.is_enabled.is_(True),
-                )
-            )
-            scholar_id = scholar_result.scalar_one_or_none()
-            if scholar_id is not None:
-                return True
-            dropped = await queue_service.mark_dropped(
-                session,
-                job_id=job.id,
-                reason="scholar_unavailable",
-            )
-            await session.commit()
-        if dropped is not None:
-            structured_log(
-                logger,
-                "info",
-                "scheduler.queue_item_dropped_scholar_unavailable",
-                queue_item_id=job.id,
-                user_id=job.user_id,
-                scholar_profile_id=job.scholar_profile_id,
-            )
-        return False
-
-    async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
-        async with background_session() as recovery_session:
-            await queue_service.reschedule_job(
-                recovery_session,
-                job_id=job.id,
-                delay_seconds=max(self._tick_seconds, 15),
-                reason="user_run_lock_active",
-                error="run_already_in_progress",
-            )
-            await recovery_session.commit()
-        structured_log(
-            logger,
-            "info",
-            "scheduler.queue_item_deferred_lock",
-            queue_item_id=job.id,
-            user_id=job.user_id,
-        )
-
-    async def _reschedule_queue_job_safety_cooldown(
-        self,
-        job: queue_service.ContinuationQueueJob,
-        exc: RunBlockedBySafetyPolicyError,
-    ) -> None:
-        cooldown_remaining_seconds = max(
-            self._tick_seconds,
-            int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
-        )
-        async with background_session() as recovery_session:
-            await queue_service.reschedule_job(
-                recovery_session,
-                job_id=job.id,
-                delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds),
-                reason="scrape_safety_cooldown",
-                error=str(exc.message),
-            )
-            await recovery_session.commit()
-        structured_log(
-            logger,
-            "info",
-            "scheduler.queue_item_deferred_safety_cooldown",
-            queue_item_id=job.id,
-            user_id=job.user_id,
-            reason=exc.safety_state.get("cooldown_reason"),
-            cooldown_remaining_seconds=cooldown_remaining_seconds,
-        )
-
-    async def _reschedule_queue_job_after_exception(
-        self,
-        job: queue_service.ContinuationQueueJob,
-        *,
-        exc: Exception,
-    ) -> None:
-        async with background_session() as recovery_session:
-            queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
-            if queue_item is None:
-                await recovery_session.commit()
-                return
-            if int(queue_item.attempt_count) >= self._continuation_max_attempts:
-                await queue_service.mark_dropped(
-                    recovery_session,
-                    job_id=job.id,
-                    reason="scheduler_exception_max_attempts",
-                    error=str(exc),
-                )
-                await recovery_session.commit()
-                structured_log(
-                    logger,
-                    "warning",
-                    "scheduler.queue_item_dropped_after_exception",
-                    queue_item_id=job.id,
-                    user_id=job.user_id,
-                    attempt_count=queue_item.attempt_count,
-                )
-                return
-            delay_seconds = queue_service.compute_backoff_seconds(
-                base_seconds=self._continuation_base_delay_seconds,
-                attempt_count=int(queue_item.attempt_count),
-                max_seconds=self._continuation_max_delay_seconds,
-            )
-            await queue_service.reschedule_job(
-                recovery_session,
-                job_id=job.id,
-                delay_seconds=delay_seconds,
-                reason="scheduler_exception",
-                error=str(exc),
-            )
-            await recovery_session.commit()
-        structured_log(
-            logger,
-            "exception",
-            "scheduler.queue_item_run_failed",
-            queue_item_id=job.id,
-            user_id=job.user_id,
-        )
-
-    async def _load_request_delay_for_user(self, user_id: int, *, floor: int) -> int:
-        async with background_session() as session:
-            result = await session.execute(
-                select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
-            )
-            delay = result.scalar_one_or_none()
-        return effective_request_delay_seconds(delay, floor=floor)
-
-    async def _run_ingestion_for_queue_job(
-        self,
-        job: queue_service.ContinuationQueueJob,
-        *,
-        request_delay_floor: int,
-    ):
-        request_delay_seconds = await self._load_request_delay_for_user(job.user_id, floor=request_delay_floor)
-        async with background_session() as session:
-            ingestion = ScholarIngestionService(source=self._source)
-            try:
-                return await ingestion.run_for_user(
-                    session,
-                    user_id=job.user_id,
-                    trigger_type=RunTriggerType.SCHEDULED,
-                    request_delay_seconds=request_delay_seconds,
-                    network_error_retries=self._network_error_retries,
-                    retry_backoff_seconds=self._retry_backoff_seconds,
-                    rate_limit_retries=settings.ingestion_rate_limit_retries,
-                    rate_limit_backoff_seconds=settings.ingestion_rate_limit_backoff_seconds,
-                    max_pages_per_scholar=self._max_pages_per_scholar,
-                    page_size=self._page_size,
-                    scholar_profile_ids={job.scholar_profile_id},
-                    start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart},
-                    auto_queue_continuations=self._continuation_queue_enabled,
-                    queue_delay_seconds=self._continuation_base_delay_seconds,
-                    alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
-                    alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
-                    alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
-                )
-            except RunAlreadyInProgressError:
-                await session.rollback()
-                await self._reschedule_queue_job_lock_active(job)
-            except RunBlockedBySafetyPolicyError as exc:
-                await session.rollback()
-                await self._reschedule_queue_job_safety_cooldown(job, exc)
-            except Exception as exc:
-                await session.rollback()
-                await self._reschedule_queue_job_after_exception(job, exc=exc)
-        return None
-
-    async def _finalize_successful_queue_job(self, session, job, run_summary) -> None:
-        queue_item = await queue_service.reset_attempt_count(session, job_id=job.id)
-        await session.commit()
-        if queue_item is None:
-            structured_log(
-                logger,
-                "info",
-                "scheduler.queue_item_resolved",
-                queue_item_id=job.id,
-                user_id=job.user_id,
-                run_id=run_summary.crawl_run_id,
-                status=run_summary.status.value,
-            )
-            return
-        structured_log(
-            logger,
-            "info",
-            "scheduler.queue_item_progressed",
-            queue_item_id=job.id,
-            user_id=job.user_id,
-            run_id=run_summary.crawl_run_id,
-            status=run_summary.status.value,
-            attempt_count=int(queue_item.attempt_count),
-        )
-
-    async def _finalize_failed_queue_job(self, session, job, run_summary) -> None:
-        queue_item = await queue_service.increment_attempt_count(session, job_id=job.id)
-        if queue_item is None:
-            await session.commit()
-            structured_log(
-                logger,
-                "info",
-                "scheduler.queue_item_resolved",
-                queue_item_id=job.id,
-                user_id=job.user_id,
-                run_id=run_summary.crawl_run_id,
-                status=run_summary.status.value,
-            )
-            return
-        if int(queue_item.attempt_count) >= self._continuation_max_attempts:
-            await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
-            await session.commit()
-            structured_log(
-                logger,
-                "warning",
-                "scheduler.queue_item_dropped_max_attempts_after_run",
-                queue_item_id=job.id,
-                user_id=job.user_id,
-                attempt_count=queue_item.attempt_count,
-                run_id=run_summary.crawl_run_id,
-                status=run_summary.status.value,
-            )
-            return
-        delay_seconds = queue_service.compute_backoff_seconds(
-            base_seconds=self._continuation_base_delay_seconds,
-            attempt_count=int(queue_item.attempt_count),
-            max_seconds=self._continuation_max_delay_seconds,
-        )
-        await queue_service.reschedule_job(
-            session,
-            job_id=job.id,
-            delay_seconds=delay_seconds,
-            reason=queue_item.reason,
-            error=queue_item.last_error,
-        )
-        await session.commit()
-        structured_log(
-            logger,
-            "info",
-            "scheduler.queue_item_rescheduled_failed",
-            queue_item_id=job.id,
-            user_id=job.user_id,
-            run_id=run_summary.crawl_run_id,
-            status=run_summary.status.value,
-            attempt_count=int(queue_item.attempt_count),
-            delay_seconds=delay_seconds,
-        )
-
-    async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
-        async with background_session() as session:
-            if int(run_summary.failed_count) <= 0:
-                await self._finalize_successful_queue_job(session, job, run_summary)
-            else:
-                await self._finalize_failed_queue_job(session, job, run_summary)
-
-    async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
-        if await self._drop_queue_job_if_max_attempts(job):
-            return
-        if not await self._mark_queue_job_retrying(job):
-            return
-        if not await self._queue_job_has_available_scholar(job):
-            return
-        from app.services.settings import application as user_settings_service
-
-        request_delay_floor = user_settings_service.resolve_request_delay_minimum(
-            settings.ingestion_min_request_delay_seconds,
-        )
-        run_summary = await self._run_ingestion_for_queue_job(job, request_delay_floor=request_delay_floor)
-        if run_summary is None:
-            return
-        await self._finalize_queue_job_after_run(job, run_summary)

app/services/ingestion/run_completion.py

@@ -1,457 +0,0 @@
-from __future__ import annotations
-
-import hashlib
-import logging
-from datetime import UTC, datetime
-from typing import Any
-
-from app.db.models import (
-    CrawlRun,
-    RunStatus,
-    ScholarProfile,
-)
-from app.logging_utils import structured_log
-from app.services.ingestion import safety as run_safety_service
-from app.services.ingestion.constants import (
-    FAILED_STATES,
-    FAILURE_BUCKET_BLOCKED,
-    FAILURE_BUCKET_INGESTION,
-    FAILURE_BUCKET_LAYOUT,
-    FAILURE_BUCKET_NETWORK,
-    FAILURE_BUCKET_OTHER,
-)
-from app.services.ingestion.fingerprints import _build_body_excerpt
-from app.services.ingestion.types import (
-    RunAlertSummary,
-    RunExecutionSummary,
-    RunFailureSummary,
-    RunProgress,
-    ScholarProcessingOutcome,
-)
-from app.services.scholar.parser import ParsedProfilePage, ParseState
-from app.services.scholar.source import FetchResult
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-
-def int_or_default(value: Any, default: int = 0) -> int:
-    try:
-        return int(value)
-    except (TypeError, ValueError):
-        return default
-
-
-def classify_failure_bucket(*, state: str, state_reason: str) -> str:
-    reason = state_reason.strip().lower()
-    normalized_state = state.strip().lower()
-
-    if normalized_state == ParseState.BLOCKED_OR_CAPTCHA.value or reason.startswith("blocked_"):
-        return FAILURE_BUCKET_BLOCKED
-    if normalized_state == ParseState.NETWORK_ERROR.value or reason.startswith("network_"):
-        return FAILURE_BUCKET_NETWORK
-    if normalized_state == ParseState.LAYOUT_CHANGED.value:
-        return FAILURE_BUCKET_LAYOUT
-    if normalized_state == "ingestion_error":
-        return FAILURE_BUCKET_INGESTION
-    return FAILURE_BUCKET_OTHER
-
-
-def summarize_failures(
-    *,
-    scholar_results: list[dict[str, Any]],
-) -> RunFailureSummary:
-    failed_state_counts: dict[str, int] = {}
-    failed_reason_counts: dict[str, int] = {}
-    scrape_failure_counts: dict[str, int] = {}
-    retries_scheduled_count = 0
-    scholars_with_retries_count = 0
-    retry_exhausted_count = 0
-    for entry in scholar_results:
-        retries_for_entry = max(0, int_or_default(entry.get("attempt_count"), 0) - 1)
-        if retries_for_entry > 0:
-            retries_scheduled_count += retries_for_entry
-            scholars_with_retries_count += 1
-        if str(entry.get("outcome", "")) != "failed":
-            continue
-        state = str(entry.get("state", "")).strip()
-        if state not in FAILED_STATES:
-            continue
-        failed_state_counts[state] = failed_state_counts.get(state, 0) + 1
-        reason = str(entry.get("state_reason", "")).strip()
-        if reason:
-            failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1
-        bucket = classify_failure_bucket(state=state, state_reason=reason)
-        scrape_failure_counts[bucket] = scrape_failure_counts.get(bucket, 0) + 1
-        if state == ParseState.NETWORK_ERROR.value and retries_for_entry > 0:
-            retry_exhausted_count += 1
-    return RunFailureSummary(
-        failed_state_counts=failed_state_counts,
-        failed_reason_counts=failed_reason_counts,
-        scrape_failure_counts=scrape_failure_counts,
-        retries_scheduled_count=retries_scheduled_count,
-        scholars_with_retries_count=scholars_with_retries_count,
-        retry_exhausted_count=retry_exhausted_count,
-    )
-
-
-def build_alert_summary(
-    *,
-    failure_summary: RunFailureSummary,
-    alert_blocked_failure_threshold: int,
-    alert_network_failure_threshold: int,
-    alert_retry_scheduled_threshold: int,
-) -> RunAlertSummary:
-    blocked_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_BLOCKED, 0))
-    network_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_NETWORK, 0))
-    blocked_threshold = max(1, int(alert_blocked_failure_threshold))
-    network_threshold = max(1, int(alert_network_failure_threshold))
-    retry_threshold = max(1, int(alert_retry_scheduled_threshold))
-    alert_flags = {
-        "blocked_failure_threshold_exceeded": blocked_failure_count >= blocked_threshold,
-        "network_failure_threshold_exceeded": network_failure_count >= network_threshold,
-        "retry_scheduled_threshold_exceeded": failure_summary.retries_scheduled_count >= retry_threshold,
-    }
-    return RunAlertSummary(
-        blocked_failure_count=blocked_failure_count,
-        network_failure_count=network_failure_count,
-        blocked_failure_threshold=blocked_threshold,
-        network_failure_threshold=network_threshold,
-        retry_scheduled_threshold=retry_threshold,
-        alert_flags=alert_flags,
-    )
-
-
-def apply_safety_outcome(
-    *,
-    user_settings: Any,
-    run: CrawlRun,
-    user_id: int,
-    alert_summary: RunAlertSummary,
-) -> None:
-    pre_apply_state = run_safety_service.get_safety_event_context(
-        user_settings,
-        now_utc=datetime.now(UTC),
-    )
-    safety_state, cooldown_trigger_reason = run_safety_service.apply_run_safety_outcome(
-        user_settings,
-        run_id=int(run.id),
-        blocked_failure_count=alert_summary.blocked_failure_count,
-        network_failure_count=alert_summary.network_failure_count,
-        blocked_failure_threshold=alert_summary.blocked_failure_threshold,
-        network_failure_threshold=alert_summary.network_failure_threshold,
-        blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds,
-        network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds,
-        now_utc=datetime.now(UTC),
-    )
-    if cooldown_trigger_reason is not None:
-        structured_log(
-            logger,
-            "warning",
-            "ingestion.safety_cooldown_entered",
-            user_id=user_id,
-            crawl_run_id=int(run.id),
-            reason=cooldown_trigger_reason,
-            blocked_failure_count=alert_summary.blocked_failure_count,
-            network_failure_count=alert_summary.network_failure_count,
-            blocked_failure_threshold=alert_summary.blocked_failure_threshold,
-            network_failure_threshold=alert_summary.network_failure_threshold,
-            cooldown_until=safety_state.get("cooldown_until"),
-            cooldown_remaining_seconds=safety_state.get("cooldown_remaining_seconds"),
-            safety_counters=safety_state.get("counters", {}),
-        )
-    elif pre_apply_state.get("cooldown_active") and not safety_state.get("cooldown_active"):
-        structured_log(
-            logger,
-            "info",
-            "ingestion.cooldown_cleared",
-            user_id=user_id,
-            crawl_run_id=int(run.id),
-            reason=pre_apply_state.get("cooldown_reason"),
-            cooldown_until=pre_apply_state.get("cooldown_until"),
-        )
-
-
-def finalize_run_record(
-    *,
-    run: CrawlRun,
-    scholars: list[ScholarProfile],
-    progress: RunProgress,
-    failure_summary: RunFailureSummary,
-    alert_summary: RunAlertSummary,
-    idempotency_key: str | None,
-    run_status: RunStatus,
-) -> None:
-    run.end_dt = datetime.now(UTC)
-    if run.status != RunStatus.CANCELED:
-        run.status = run_status
-    run.error_log = {
-        "scholar_results": progress.scholar_results,
-        "summary": {
-            "succeeded_count": progress.succeeded_count,
-            "failed_count": progress.failed_count,
-            "partial_count": progress.partial_count,
-            "failed_state_counts": failure_summary.failed_state_counts,
-            "failed_reason_counts": failure_summary.failed_reason_counts,
-            "scrape_failure_counts": failure_summary.scrape_failure_counts,
-            "retry_counts": {
-                "retries_scheduled_count": failure_summary.retries_scheduled_count,
-                "scholars_with_retries_count": failure_summary.scholars_with_retries_count,
-                "retry_exhausted_count": failure_summary.retry_exhausted_count,
-            },
-            "alert_thresholds": {
-                "blocked_failure_threshold": alert_summary.blocked_failure_threshold,
-                "network_failure_threshold": alert_summary.network_failure_threshold,
-                "retry_scheduled_threshold": alert_summary.retry_scheduled_threshold,
-            },
-            "alert_flags": alert_summary.alert_flags,
-        },
-        "meta": {"idempotency_key": idempotency_key} if idempotency_key else {},
-    }
-
-
-def resolve_run_status(
-    *,
-    scholar_count: int,
-    succeeded_count: int,
-    failed_count: int,
-    partial_count: int,
-) -> RunStatus:
-    if scholar_count == 0:
-        return RunStatus.SUCCESS
-    if failed_count == scholar_count:
-        return RunStatus.FAILED
-    if failed_count > 0 or partial_count > 0:
-        return RunStatus.PARTIAL_FAILURE
-    if succeeded_count > 0:
-        return RunStatus.SUCCESS
-    return RunStatus.FAILED
-
-
-def build_failure_debug_context(
-    *,
-    fetch_result: FetchResult,
-    parsed_page: ParsedProfilePage,
-    attempt_log: list[dict[str, Any]],
-    page_logs: list[dict[str, Any]] | None = None,
-    exception: Exception | None = None,
-) -> dict[str, Any]:
-    context: dict[str, Any] = {
-        "requested_url": fetch_result.requested_url,
-        "final_url": fetch_result.final_url,
-        "status_code": fetch_result.status_code,
-        "fetch_error": fetch_result.error,
-        "state_reason": parsed_page.state_reason,
-        "profile_name": parsed_page.profile_name,
-        "profile_image_url": parsed_page.profile_image_url,
-        "articles_range": parsed_page.articles_range,
-        "has_show_more_button": parsed_page.has_show_more_button,
-        "has_operation_error_banner": parsed_page.has_operation_error_banner,
-        "warning_codes": parsed_page.warnings,
-        "marker_counts_nonzero": {key: value for key, value in parsed_page.marker_counts.items() if value > 0},
-        "body_length": len(fetch_result.body),
-        "body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest() if fetch_result.body else None,
-        "body_excerpt": _build_body_excerpt(fetch_result.body),
-        "attempt_log": attempt_log,
-    }
-    if page_logs:
-        context["page_logs"] = page_logs
-    if exception is not None:
-        context["exception_type"] = type(exception).__name__
-        context["exception_message"] = str(exception)
-    return context
-
-
-def _log_alert_threshold_warnings(
-    *,
-    user_id: int,
-    run: CrawlRun,
-    failure_summary: RunFailureSummary,
-    alert_summary: RunAlertSummary,
-) -> None:
-    if alert_summary.alert_flags["blocked_failure_threshold_exceeded"]:
-        structured_log(
-            logger,
-            "warning",
-            "ingestion.alert_blocked_failure_threshold_exceeded",
-            user_id=user_id,
-            crawl_run_id=int(run.id),
-            blocked_failure_count=alert_summary.blocked_failure_count,
-            threshold=alert_summary.blocked_failure_threshold,
-        )
-    if alert_summary.alert_flags["network_failure_threshold_exceeded"]:
-        structured_log(
-            logger,
-            "warning",
-            "ingestion.alert_network_failure_threshold_exceeded",
-            user_id=user_id,
-            crawl_run_id=int(run.id),
-            network_failure_count=alert_summary.network_failure_count,
-            threshold=alert_summary.network_failure_threshold,
-        )
-    if alert_summary.alert_flags["retry_scheduled_threshold_exceeded"]:
-        structured_log(
-            logger,
-            "warning",
-            "ingestion.alert_retry_scheduled_threshold_exceeded",
-            user_id=user_id,
-            crawl_run_id=int(run.id),
-            retries_scheduled_count=failure_summary.retries_scheduled_count,
-            threshold=alert_summary.retry_scheduled_threshold,
-        )
-
-
-def complete_run_for_user(
-    *,
-    user_settings: Any,
-    run: CrawlRun,
-    scholars: list[ScholarProfile],
-    user_id: int,
-    progress: RunProgress,
-    idempotency_key: str | None,
-    alert_blocked_failure_threshold: int,
-    alert_network_failure_threshold: int,
-    alert_retry_scheduled_threshold: int,
-) -> tuple[RunFailureSummary, RunAlertSummary]:
-    failure_summary = summarize_failures(scholar_results=progress.scholar_results)
-    alert_summary = build_alert_summary(
-        failure_summary=failure_summary,
-        alert_blocked_failure_threshold=alert_blocked_failure_threshold,
-        alert_network_failure_threshold=alert_network_failure_threshold,
-        alert_retry_scheduled_threshold=alert_retry_scheduled_threshold,
-    )
-    _log_alert_threshold_warnings(
-        user_id=user_id, run=run, failure_summary=failure_summary, alert_summary=alert_summary
-    )
-    apply_safety_outcome(user_settings=user_settings, run=run, user_id=user_id, alert_summary=alert_summary)
-    run_status = resolve_run_status(
-        scholar_count=len(scholars),
-        succeeded_count=progress.succeeded_count,
-        failed_count=progress.failed_count,
-        partial_count=progress.partial_count,
-    )
-    finalize_run_record(
-        run=run,
-        scholars=scholars,
-        progress=progress,
-        failure_summary=failure_summary,
-        alert_summary=alert_summary,
-        idempotency_key=idempotency_key,
-        run_status=run_status,
-    )
-    return failure_summary, alert_summary
-
-
-def result_counters(result_entry: dict[str, Any]) -> tuple[int, int, int]:
-    outcome = str(result_entry.get("outcome", "")).strip().lower()
-    if outcome == "success":
-        return 1, 0, 0
-    if outcome == "partial":
-        return 1, 0, 1
-    if outcome == "failed":
-        return 0, 1, 0
-    raise RuntimeError(f"Unexpected scholar outcome label: {outcome!r}")
-
-
-def find_scholar_result_index(
-    *,
-    scholar_results: list[dict[str, Any]],
-    scholar_profile_id: int,
-) -> int | None:
-    for index, result_entry in enumerate(scholar_results):
-        current_scholar_id = int_or_default(result_entry.get("scholar_profile_id"), 0)
-        if current_scholar_id == scholar_profile_id:
-            return index
-    return None
-
-
-def adjust_progress_counts(
-    *,
-    progress: RunProgress,
-    succeeded_delta: int,
-    failed_delta: int,
-    partial_delta: int,
-) -> None:
-    progress.succeeded_count += succeeded_delta
-    progress.failed_count += failed_delta
-    progress.partial_count += partial_delta
-    if progress.succeeded_count < 0 or progress.failed_count < 0 or progress.partial_count < 0:
-        raise RuntimeError("RunProgress counters entered invalid negative state.")
-
-
-def apply_outcome_to_progress(
-    *,
-    progress: RunProgress,
-    outcome: ScholarProcessingOutcome,
-) -> None:
-    scholar_profile_id = int_or_default(outcome.result_entry.get("scholar_profile_id"), 0)
-    if scholar_profile_id <= 0:
-        raise RuntimeError("Scholar outcome missing valid scholar_profile_id.")
-    prior_index = find_scholar_result_index(
-        scholar_results=progress.scholar_results,
-        scholar_profile_id=scholar_profile_id,
-    )
-    next_succeeded, next_failed, next_partial = result_counters(outcome.result_entry)
-    if prior_index is None:
-        progress.scholar_results.append(outcome.result_entry)
-        adjust_progress_counts(
-            progress=progress,
-            succeeded_delta=next_succeeded,
-            failed_delta=next_failed,
-            partial_delta=next_partial,
-        )
-        return
-    previous_entry = progress.scholar_results[prior_index]
-    prev_succeeded, prev_failed, prev_partial = result_counters(previous_entry)
-    progress.scholar_results[prior_index] = outcome.result_entry
-    adjust_progress_counts(
-        progress=progress,
-        succeeded_delta=next_succeeded - prev_succeeded,
-        failed_delta=next_failed - prev_failed,
-        partial_delta=next_partial - prev_partial,
-    )
-
-
-def run_execution_summary(
-    *,
-    run: CrawlRun,
-    scholars: list[ScholarProfile],
-    progress: RunProgress,
-) -> RunExecutionSummary:
-    return RunExecutionSummary(
-        crawl_run_id=run.id,
-        status=run.status,
-        scholar_count=len(scholars),
-        succeeded_count=progress.succeeded_count,
-        failed_count=progress.failed_count,
-        partial_count=progress.partial_count,
-        new_publication_count=run.new_pub_count,
-    )
-
-
-def log_run_completed(
-    *,
-    run: CrawlRun,
-    user_id: int,
-    scholars: list[ScholarProfile],
-    progress: RunProgress,
-    failure_summary: RunFailureSummary,
-    alert_summary: RunAlertSummary,
-) -> None:
-    structured_log(
-        logger,
-        "info",
-        "ingestion.run_completed",
-        user_id=user_id,
-        crawl_run_id=run.id,
-        status=run.status.value,
-        scholar_count=len(scholars),
-        succeeded_count=progress.succeeded_count,
-        failed_count=progress.failed_count,
-        partial_count=progress.partial_count,
-        new_publication_count=run.new_pub_count,
-        blocked_failure_count=alert_summary.blocked_failure_count,
-        network_failure_count=alert_summary.network_failure_count,
-        retries_scheduled_count=failure_summary.retries_scheduled_count,
-        alert_flags=alert_summary.alert_flags,
-    )

app/services/ingestion/run_enrichment.py

@@ -1,110 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-from typing import Any
-
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import CrawlRun, RunStatus
-from app.logging_utils import structured_log
-from app.services.ingestion.enrichment import EnrichmentRunner
-
-logger = logging.getLogger(__name__)
-
-_background_tasks: set[asyncio.Task[Any]] = set()
-
-
-async def background_enrich(
-    session_factory: Any,
-    enrichment_runner: EnrichmentRunner,
-    *,
-    run_id: int,
-    intended_final_status: RunStatus,
-    openalex_api_key: str | None = None,
-) -> None:
-    try:
-        async with session_factory() as session:
-            await enrichment_runner.enrich_pending_publications(
-                session,
-                run_id=run_id,
-                openalex_api_key=openalex_api_key,
-            )
-            run = await session.get(CrawlRun, run_id)
-            if run is not None and run.status == RunStatus.RESOLVING:
-                run.status = intended_final_status
-            await session.commit()
-            structured_log(
-                logger,
-                "info",
-                "ingestion.background_enrichment_complete",
-                run_id=run_id,
-                final_status=str(intended_final_status),
-            )
-    except Exception:
-        structured_log(
-            logger,
-            "exception",
-            "ingestion.background_enrichment_failed",
-            run_id=run_id,
-        )
-        try:
-            async with session_factory() as fallback_session:
-                run = await fallback_session.get(CrawlRun, run_id)
-                if run is not None and run.status == RunStatus.RESOLVING:
-                    run.status = intended_final_status
-                await fallback_session.commit()
-        except Exception:
-            structured_log(
-                logger,
-                "exception",
-                "ingestion.background_enrichment_fallback_failed",
-                run_id=run_id,
-            )
-
-
-async def inline_enrich_and_finalize(
-    db_session: AsyncSession,
-    enrichment_runner: EnrichmentRunner,
-    *,
-    run: CrawlRun,
-    user_settings: Any,
-    intended_final_status: RunStatus,
-) -> None:
-    try:
-        await enrichment_runner.enrich_pending_publications(
-            db_session,
-            run_id=run.id,
-            openalex_api_key=getattr(user_settings, "openalex_api_key", None),
-        )
-    except Exception:
-        structured_log(
-            logger,
-            "exception",
-            "ingestion.enrichment_failed",
-            run_id=run.id,
-        )
-    if run.status == RunStatus.RESOLVING:
-        run.status = intended_final_status
-    await db_session.commit()
-
-
-def spawn_background_enrichment_task(
-    session_factory: Any,
-    enrichment_runner: EnrichmentRunner,
-    *,
-    run_id: int,
-    intended_final_status: RunStatus,
-    openalex_api_key: str | None,
-) -> None:
-    task = asyncio.create_task(
-        background_enrich(
-            session_factory,
-            enrichment_runner,
-            run_id=run_id,
-            intended_final_status=intended_final_status,
-            openalex_api_key=openalex_api_key,
-        )
-    )
-    _background_tasks.add(task)
-    task.add_done_callback(_background_tasks.discard)

app/services/ingestion/run_guards.py

@@ -1,135 +0,0 @@
-from __future__ import annotations
-
-import logging
-from datetime import UTC, datetime
-from typing import Any
-
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import (
-    RunTriggerType,
-    ScholarProfile,
-)
-from app.logging_utils import structured_log
-from app.services.ingestion import safety as run_safety_service
-from app.services.ingestion.preflight import check_scholar_reachable
-from app.services.ingestion.types import RunBlockedBySafetyPolicyError
-from app.services.scholar.source import ScholarSource
-from app.services.settings import application as user_settings_service
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-
-async def load_user_settings_for_run(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    trigger_type: RunTriggerType,
-):
-    user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=user_id)
-    await enforce_safety_gate(db_session, user_settings=user_settings, user_id=user_id, trigger_type=trigger_type)
-    return user_settings
-
-
-async def enforce_safety_gate(
-    db_session: AsyncSession,
-    *,
-    user_settings,
-    user_id: int,
-    trigger_type: RunTriggerType,
-) -> None:
-    now_utc = datetime.now(UTC)
-    previous = run_safety_service.get_safety_event_context(user_settings, now_utc=now_utc)
-    if run_safety_service.clear_expired_cooldown(user_settings, now_utc=now_utc):
-        await db_session.commit()
-        await db_session.refresh(user_settings)
-        structured_log(
-            logger,
-            "info",
-            "ingestion.cooldown_cleared",
-            user_id=user_id,
-            reason=previous.get("cooldown_reason"),
-            cooldown_until=previous.get("cooldown_until"),
-        )
-        now_utc = datetime.now(UTC)
-    if run_safety_service.is_cooldown_active(user_settings, now_utc=now_utc):
-        await raise_safety_blocked_start(
-            db_session,
-            user_settings=user_settings,
-            user_id=user_id,
-            trigger_type=trigger_type,
-            now_utc=now_utc,
-        )
-
-
-async def raise_safety_blocked_start(
-    db_session: AsyncSession,
-    *,
-    user_settings,
-    user_id: int,
-    trigger_type: RunTriggerType,
-    now_utc: datetime,
-) -> None:
-    safety_state = run_safety_service.register_cooldown_blocked_start(user_settings, now_utc=now_utc)
-    await db_session.commit()
-    structured_log(
-        logger,
-        "warning",
-        "ingestion.safety_policy_blocked_run_start",
-        user_id=user_id,
-        trigger_type=trigger_type.value,
-        reason=safety_state.get("cooldown_reason"),
-        cooldown_until=safety_state.get("cooldown_until"),
-        cooldown_remaining_seconds=safety_state.get("cooldown_remaining_seconds"),
-        blocked_start_count=((safety_state.get("counters") or {}).get("blocked_start_count")),
-    )
-    raise RunBlockedBySafetyPolicyError(
-        code="scrape_cooldown_active",
-        message="Scrape safety cooldown is active; run start is temporarily blocked.",
-        safety_state=safety_state,
-    )
-
-
-async def run_preflight_guard(
-    db_session: AsyncSession,
-    source: ScholarSource,
-    *,
-    user_settings: Any,
-    user_id: int,
-    scholars: list[ScholarProfile],
-) -> None:
-    if not scholars:
-        return
-    result = await check_scholar_reachable(
-        source,
-        scholar_id=scholars[0].scholar_id,
-    )
-    if result.passed:
-        return
-    now_utc = datetime.now(UTC)
-    safety_state, _ = run_safety_service.apply_run_safety_outcome(
-        user_settings,
-        run_id=0,
-        blocked_failure_count=1,
-        network_failure_count=0,
-        blocked_failure_threshold=1,
-        network_failure_threshold=1,
-        blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds,
-        network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds,
-        now_utc=now_utc,
-    )
-    await db_session.commit()
-    structured_log(
-        logger,
-        "warning",
-        "ingestion.cooldown_entered_preflight",
-        user_id=user_id,
-        block_reason=result.block_reason,
-        cooldown_until=safety_state.get("cooldown_until"),
-    )
-    raise RunBlockedBySafetyPolicyError(
-        code="scrape_cooldown_active",
-        message=f"Preflight detected Scholar block ({result.block_reason}). Cooldown activated.",
-        safety_state=safety_state,
-    )

app/services/ingestion/scheduler.py

@@ -4,24 +4,26 @@ import asyncio
 import logging
 from dataclasses import dataclass
 from datetime import UTC, datetime, timedelta
-from typing import Any
 
 from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.db.background_session import background_session
 from app.db.models import (
     CrawlRun,
+    QueueItemStatus,
     RunTriggerType,
+    ScholarProfile,
     User,
     UserSetting,
 )
+from app.db.session import get_session_factory
 from app.logging_utils import structured_log
+from app.services.ingestion import queue as queue_service
 from app.services.ingestion.application import (
     RunAlreadyInProgressError,
     RunBlockedBySafetyPolicyError,
     ScholarIngestionService,
 )
-from app.services.ingestion.queue_runner import QueueJobRunner, effective_request_delay_seconds
 from app.services.scholar.source import LiveScholarSource
 from app.services.settings import application as user_settings_service
 from app.settings import settings
@@ -29,6 +31,19 @@ from app.settings import settings
 logger = logging.getLogger(__name__)
 
 
+def _request_delay_floor_seconds() -> int:
+    return user_settings_service.resolve_request_delay_minimum(settings.ingestion_min_request_delay_seconds)
+
+
+def _effective_request_delay_seconds(value: int | None) -> int:
+    floor = _request_delay_floor_seconds()
+    try:
+        parsed = int(value) if value is not None else floor
+    except (TypeError, ValueError):
+        parsed = floor
+    return max(floor, parsed)
+
+
 @dataclass(frozen=True)
 class _AutoRunCandidate:
     user_id: int
@@ -70,18 +85,6 @@ class SchedulerService:
         self._queue_batch_size = max(1, int(queue_batch_size))
         self._task: asyncio.Task[None] | None = None
         self._source = LiveScholarSource()
-        self._queue_runner = QueueJobRunner(
-            tick_seconds=self._tick_seconds,
-            network_error_retries=self._network_error_retries,
-            retry_backoff_seconds=self._retry_backoff_seconds,
-            max_pages_per_scholar=self._max_pages_per_scholar,
-            page_size=self._page_size,
-            continuation_queue_enabled=self._continuation_queue_enabled,
-            continuation_base_delay_seconds=self._continuation_base_delay_seconds,
-            continuation_max_delay_seconds=self._continuation_max_delay_seconds,
-            continuation_max_attempts=self._continuation_max_attempts,
-            queue_batch_size=self._queue_batch_size,
-        )
 
     async def start(self) -> None:
         if not self._enabled:
@@ -125,16 +128,15 @@ class SchedulerService:
             except asyncio.CancelledError:
                 raise
             except Exception:
-                structured_log(
+                logger.exception(
-                    logger,
-                    "exception",
                     "scheduler.tick_failed",
+                    extra={},
                 )
             await asyncio.sleep(float(self._tick_seconds))
 
     async def _tick_once(self) -> None:
         if self._continuation_queue_enabled:
-            await self._queue_runner.drain_continuation_queue()
+            await self._drain_continuation_queue()
 
         await self._drain_pdf_queue()
 
@@ -147,8 +149,9 @@ class SchedulerService:
                 continue
             await self._run_candidate(candidate)
 
-    async def _load_candidate_rows(self) -> list[Any]:
+    async def _load_candidate_rows(self) -> list[tuple]:
-        async with background_session() as session:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
             result = await session.execute(
                 select(
                     UserSetting.user_id,
@@ -161,10 +164,10 @@ class SchedulerService:
                 .where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True))
                 .order_by(UserSetting.user_id.asc())
             )
-            return list(result.all())
+            return result.all()
 
     @staticmethod
-    def _candidate_from_row(row: Any, *, now_utc: datetime) -> _AutoRunCandidate | None:
+    def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None:
         user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row
         if cooldown_until is not None and cooldown_until.tzinfo is None:
             cooldown_until = cooldown_until.replace(tzinfo=UTC)
@@ -182,10 +185,7 @@ class SchedulerService:
         return _AutoRunCandidate(
             user_id=int(user_id),
             run_interval_minutes=int(run_interval_minutes),
-            request_delay_seconds=effective_request_delay_seconds(
+            request_delay_seconds=_effective_request_delay_seconds(request_delay_seconds),
-                request_delay_seconds,
-                floor=user_settings_service.resolve_request_delay_minimum(settings.ingestion_min_request_delay_seconds),
-            ),
             cooldown_until=cooldown_until,
             cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
         )
@@ -203,7 +203,8 @@ class SchedulerService:
         return candidates
 
     async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
-        async with background_session() as session:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
             result = await session.execute(
                 select(CrawlRun.start_dt)
                 .where(
@@ -225,7 +226,8 @@ class SchedulerService:
         *,
         candidate: _AutoRunCandidate,
     ):
-        async with background_session() as session:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
             ingestion = ScholarIngestionService(source=self._source)
             try:
                 return await ingestion.run_for_user(
@@ -261,12 +263,7 @@ class SchedulerService:
                 return None
             except Exception:
                 await session.rollback()
-                structured_log(
+                logger.exception("scheduler.run_failed", extra={"user_id": candidate.user_id})
-                    logger,
-                    "exception",
-                    "scheduler.run_failed",
-                    user_id=candidate.user_id,
-                )
                 return None
 
     async def _run_candidate(self, candidate: _AutoRunCandidate) -> None:
@@ -284,14 +281,318 @@ class SchedulerService:
             new_publication_count=run_summary.new_publication_count,
         )
 
+    async def _drain_continuation_queue(self) -> None:
+        now = datetime.now(UTC)
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            jobs = await queue_service.list_due_jobs(
+                session,
+                now=now,
+                limit=self._queue_batch_size,
+            )
+        for job in jobs:
+            await self._run_queue_job(job)
+
+    async def _drop_queue_job_if_max_attempts(
+        self,
+        job: queue_service.ContinuationQueueJob,
+    ) -> bool:
+        if job.attempt_count < self._continuation_max_attempts:
+            return False
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            dropped = await queue_service.mark_dropped(
+                session,
+                job_id=job.id,
+                reason="max_attempts_reached",
+            )
+            await session.commit()
+        if dropped is not None:
+            structured_log(
+                logger,
+                "warning",
+                "scheduler.queue_item_dropped_max_attempts",
+                queue_item_id=job.id,
+                user_id=job.user_id,
+                scholar_profile_id=job.scholar_profile_id,
+                attempt_count=job.attempt_count,
+                max_attempts=self._continuation_max_attempts,
+            )
+        return True
+
+    async def _mark_queue_job_retrying(
+        self,
+        job: queue_service.ContinuationQueueJob,
+    ) -> bool:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            queue_item = await queue_service.mark_retrying(session, job_id=job.id)
+            await session.commit()
+        if queue_item is None:
+            return False
+        return queue_item.status != QueueItemStatus.DROPPED.value
+
+    async def _queue_job_has_available_scholar(
+        self,
+        job: queue_service.ContinuationQueueJob,
+    ) -> bool:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            scholar_result = await session.execute(
+                select(ScholarProfile.id).where(
+                    ScholarProfile.user_id == job.user_id,
+                    ScholarProfile.id == job.scholar_profile_id,
+                    ScholarProfile.is_enabled.is_(True),
+                )
+            )
+            scholar_id = scholar_result.scalar_one_or_none()
+            if scholar_id is not None:
+                return True
+            dropped = await queue_service.mark_dropped(
+                session,
+                job_id=job.id,
+                reason="scholar_unavailable",
+            )
+            await session.commit()
+        if dropped is not None:
+            structured_log(
+                logger,
+                "info",
+                "scheduler.queue_item_dropped_scholar_unavailable",
+                queue_item_id=job.id,
+                user_id=job.user_id,
+                scholar_profile_id=job.scholar_profile_id,
+            )
+        return False
+
+    async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
+        session_factory = get_session_factory()
+        async with session_factory() as recovery_session:
+            await queue_service.reschedule_job(
+                recovery_session,
+                job_id=job.id,
+                delay_seconds=max(self._tick_seconds, 15),
+                reason="user_run_lock_active",
+                error="run_already_in_progress",
+            )
+            await recovery_session.commit()
+        structured_log(
+            logger,
+            "info",
+            "scheduler.queue_item_deferred_lock",
+            queue_item_id=job.id,
+            user_id=job.user_id,
+        )
+
+    async def _reschedule_queue_job_safety_cooldown(
+        self,
+        job: queue_service.ContinuationQueueJob,
+        exc: RunBlockedBySafetyPolicyError,
+    ) -> None:
+        cooldown_remaining_seconds = max(
+            self._tick_seconds,
+            int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
+        )
+        session_factory = get_session_factory()
+        async with session_factory() as recovery_session:
+            await queue_service.reschedule_job(
+                recovery_session,
+                job_id=job.id,
+                delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds),
+                reason="scrape_safety_cooldown",
+                error=str(exc.message),
+            )
+            await recovery_session.commit()
+        structured_log(
+            logger,
+            "info",
+            "scheduler.queue_item_deferred_safety_cooldown",
+            queue_item_id=job.id,
+            user_id=job.user_id,
+            reason=exc.safety_state.get("cooldown_reason"),
+            cooldown_remaining_seconds=cooldown_remaining_seconds,
+        )
+
+    async def _reschedule_queue_job_after_exception(
+        self,
+        job: queue_service.ContinuationQueueJob,
+        *,
+        exc: Exception,
+    ) -> None:
+        session_factory = get_session_factory()
+        async with session_factory() as recovery_session:
+            queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
+            if queue_item is None:
+                await recovery_session.commit()
+                return
+            if int(queue_item.attempt_count) >= self._continuation_max_attempts:
+                await queue_service.mark_dropped(
+                    recovery_session,
+                    job_id=job.id,
+                    reason="scheduler_exception_max_attempts",
+                    error=str(exc),
+                )
+                await recovery_session.commit()
+                structured_log(
+                    logger,
+                    "warning",
+                    "scheduler.queue_item_dropped_after_exception",
+                    queue_item_id=job.id,
+                    user_id=job.user_id,
+                    attempt_count=queue_item.attempt_count,
+                )
+                return
+            delay_seconds = queue_service.compute_backoff_seconds(
+                base_seconds=self._continuation_base_delay_seconds,
+                attempt_count=int(queue_item.attempt_count),
+                max_seconds=self._continuation_max_delay_seconds,
+            )
+            await queue_service.reschedule_job(
+                recovery_session,
+                job_id=job.id,
+                delay_seconds=delay_seconds,
+                reason="scheduler_exception",
+                error=str(exc),
+            )
+            await recovery_session.commit()
+        logger.exception("scheduler.queue_item_run_failed", extra={"queue_item_id": job.id, "user_id": job.user_id})
+
+    async def _run_ingestion_for_queue_job(
+        self,
+        job: queue_service.ContinuationQueueJob,
+    ):
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id)
+            ingestion = ScholarIngestionService(source=self._source)
+            try:
+                return await ingestion.run_for_user(
+                    session,
+                    user_id=job.user_id,
+                    trigger_type=RunTriggerType.SCHEDULED,
+                    request_delay_seconds=request_delay_seconds,
+                    network_error_retries=self._network_error_retries,
+                    retry_backoff_seconds=self._retry_backoff_seconds,
+                    rate_limit_retries=settings.ingestion_rate_limit_retries,
+                    rate_limit_backoff_seconds=settings.ingestion_rate_limit_backoff_seconds,
+                    max_pages_per_scholar=self._max_pages_per_scholar,
+                    page_size=self._page_size,
+                    scholar_profile_ids={job.scholar_profile_id},
+                    start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart},
+                    auto_queue_continuations=self._continuation_queue_enabled,
+                    queue_delay_seconds=self._continuation_base_delay_seconds,
+                    alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
+                    alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
+                    alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
+                )
+            except RunAlreadyInProgressError:
+                await session.rollback()
+                await self._reschedule_queue_job_lock_active(job)
+            except RunBlockedBySafetyPolicyError as exc:
+                await session.rollback()
+                await self._reschedule_queue_job_safety_cooldown(job, exc)
+            except Exception as exc:
+                await session.rollback()
+                await self._reschedule_queue_job_after_exception(job, exc=exc)
+        return None
+
+    async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            if int(run_summary.failed_count) <= 0:
+                queue_item = await queue_service.reset_attempt_count(session, job_id=job.id)
+                await session.commit()
+                if queue_item is None:
+                    structured_log(
+                        logger,
+                        "info",
+                        "scheduler.queue_item_resolved",
+                        queue_item_id=job.id,
+                        user_id=job.user_id,
+                        run_id=run_summary.crawl_run_id,
+                        status=run_summary.status.value,
+                    )
+                    return
+                structured_log(
+                    logger,
+                    "info",
+                    "scheduler.queue_item_progressed",
+                    queue_item_id=job.id,
+                    user_id=job.user_id,
+                    run_id=run_summary.crawl_run_id,
+                    status=run_summary.status.value,
+                    attempt_count=int(queue_item.attempt_count),
+                )
+                return
+            queue_item = await queue_service.increment_attempt_count(session, job_id=job.id)
+            if queue_item is None:
+                await session.commit()
+                structured_log(
+                    logger,
+                    "info",
+                    "scheduler.queue_item_resolved",
+                    queue_item_id=job.id,
+                    user_id=job.user_id,
+                    run_id=run_summary.crawl_run_id,
+                    status=run_summary.status.value,
+                )
+                return
+            if int(queue_item.attempt_count) >= self._continuation_max_attempts:
+                await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
+                await session.commit()
+                structured_log(
+                    logger,
+                    "warning",
+                    "scheduler.queue_item_dropped_max_attempts_after_run",
+                    queue_item_id=job.id,
+                    user_id=job.user_id,
+                    attempt_count=queue_item.attempt_count,
+                    run_id=run_summary.crawl_run_id,
+                    status=run_summary.status.value,
+                )
+                return
+            delay_seconds = queue_service.compute_backoff_seconds(
+                base_seconds=self._continuation_base_delay_seconds,
+                attempt_count=int(queue_item.attempt_count),
+                max_seconds=self._continuation_max_delay_seconds,
+            )
+            await queue_service.reschedule_job(
+                session,
+                job_id=job.id,
+                delay_seconds=delay_seconds,
+                reason=queue_item.reason,
+                error=queue_item.last_error,
+            )
+            await session.commit()
+        structured_log(
+            logger,
+            "info",
+            "scheduler.queue_item_rescheduled_failed",
+            queue_item_id=job.id,
+            user_id=job.user_id,
+            run_id=run_summary.crawl_run_id,
+            status=run_summary.status.value,
+            attempt_count=int(queue_item.attempt_count),
+            delay_seconds=delay_seconds,
+        )
+
+    async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
+        if await self._drop_queue_job_if_max_attempts(job):
+            return
+        if not await self._mark_queue_job_retrying(job):
+            return
+        if not await self._queue_job_has_available_scholar(job):
+            return
+        run_summary = await self._run_ingestion_for_queue_job(job)
+        if run_summary is None:
+            return
+        await self._finalize_queue_job_after_run(job, run_summary)
+
     async def _drain_pdf_queue(self) -> None:
         from app.services.publications.pdf_queue import drain_ready_jobs
-        from app.services.publications.pdf_queue_resolution import is_budget_cooldown_active
 
-        if is_budget_cooldown_active():
+        session_factory = get_session_factory()
-            return
+        async with session_factory() as session:
-
-        async with background_session() as session:
             try:
                 processed = await drain_ready_jobs(
                     session,
@@ -306,8 +607,16 @@ class SchedulerService:
                         processed_count=processed,
                     )
             except Exception:
-                structured_log(
+                logger.exception("scheduler.pdf_queue_drain_failed", extra={})
-                    logger,
+
-                    "exception",
+    async def _load_request_delay_for_user(
-                    "scheduler.pdf_queue_drain_failed",
+        self,
-                )
+        db_session: AsyncSession,
+        *,
+        user_id: int,
+    ) -> int:
+        result = await db_session.execute(
+            select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
+        )
+        delay = result.scalar_one_or_none()
+        return _effective_request_delay_seconds(delay)

app/services/ingestion/scholar_outcomes.py

@@ -1,308 +0,0 @@
-from __future__ import annotations
-
-import logging
-from datetime import UTC, datetime
-from typing import Any
-
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import (
-    CrawlRun,
-    RunStatus,
-    ScholarProfile,
-)
-from app.logging_utils import structured_log
-from app.services.ingestion.run_completion import build_failure_debug_context
-from app.services.ingestion.types import (
-    PagedParseResult,
-    ScholarProcessingOutcome,
-)
-from app.services.scholar.parser import ParseState
-
-logger = logging.getLogger(__name__)
-
-
-def assert_valid_paged_parse_result(
-    *,
-    scholar_id: str,
-    paged_parse_result: PagedParseResult,
-) -> None:
-    parsed_page = paged_parse_result.parsed_page
-    if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} and any(
-        code.startswith("layout_") for code in parsed_page.warnings
-    ):
-        raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.")
-    for publication in paged_parse_result.publications:
-        if not publication.title.strip():
-            raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.")
-        if publication.citation_count is not None and int(publication.citation_count) < 0:
-            raise RuntimeError(f"Negative citation count for scholar_id={scholar_id}.")
-
-
-def apply_first_page_profile_metadata(
-    *,
-    scholar: ScholarProfile,
-    paged_parse_result: PagedParseResult,
-    run_dt: datetime,
-) -> None:
-    first_page = paged_parse_result.first_page_parsed_page
-    if first_page.profile_name and not (scholar.display_name or "").strip():
-        scholar.display_name = first_page.profile_name
-    if first_page.profile_image_url:
-        scholar.profile_image_url = first_page.profile_image_url
-    scholar.last_initial_page_checked_at = run_dt
-
-
-def build_result_entry(
-    *,
-    scholar: ScholarProfile,
-    start_cstart: int,
-    paged_parse_result: PagedParseResult,
-) -> dict[str, Any]:
-    parsed_page = paged_parse_result.parsed_page
-    return {
-        "scholar_profile_id": scholar.id,
-        "scholar_id": scholar.scholar_id,
-        "state": parsed_page.state.value,
-        "state_reason": parsed_page.state_reason,
-        "outcome": "failed",
-        "attempt_count": len(paged_parse_result.attempt_log),
-        "publication_count": len(paged_parse_result.publications),
-        "start_cstart": start_cstart,
-        "articles_range": parsed_page.articles_range,
-        "warnings": parsed_page.warnings,
-        "has_show_more_button": parsed_page.has_show_more_button,
-        "pages_fetched": paged_parse_result.pages_fetched,
-        "pages_attempted": paged_parse_result.pages_attempted,
-        "has_more_remaining": paged_parse_result.has_more_remaining,
-        "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason,
-        "continuation_cstart": paged_parse_result.continuation_cstart,
-        "skipped_no_change": paged_parse_result.skipped_no_change,
-        "initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256,
-    }
-
-
-def skipped_no_change_outcome(
-    *,
-    scholar: ScholarProfile,
-    run_dt: datetime,
-    paged_parse_result: PagedParseResult,
-    result_entry: dict[str, Any],
-) -> ScholarProcessingOutcome:
-    first_page = paged_parse_result.first_page_parsed_page
-    scholar.last_run_status = RunStatus.SUCCESS
-    scholar.last_run_dt = run_dt
-    result_entry["state"] = first_page.state.value
-    result_entry["state_reason"] = "no_change_initial_page_signature"
-    result_entry["outcome"] = "success"
-    result_entry["publication_count"] = 0
-    result_entry["warnings"] = first_page.warnings
-    result_entry["debug"] = {
-        "state_reason": "no_change_initial_page_signature",
-        "first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256,
-        "attempt_log": paged_parse_result.attempt_log,
-        "page_logs": paged_parse_result.page_logs,
-    }
-    return ScholarProcessingOutcome(
-        result_entry=result_entry,
-        succeeded_count_delta=1,
-        failed_count_delta=0,
-        partial_count_delta=0,
-        discovered_publication_count=0,
-    )
-
-
-async def upsert_publications_outcome(
-    db_session: AsyncSession,
-    *,
-    run: CrawlRun,
-    scholar: ScholarProfile,
-    run_dt: datetime,
-    paged_parse_result: PagedParseResult,
-    result_entry: dict[str, Any],
-) -> ScholarProcessingOutcome:
-    parsed_page = paged_parse_result.parsed_page
-    publications = paged_parse_result.publications
-    had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}
-    has_partial_set = len(publications) > 0 and had_page_failure
-    if (not had_page_failure) or has_partial_set:
-        return await _upsert_success_or_exception(
-            db_session,
-            run=run,
-            scholar=scholar,
-            run_dt=run_dt,
-            paged_parse_result=paged_parse_result,
-            result_entry=result_entry,
-            has_partial_publication_set=has_partial_set,
-        )
-    return _parse_failure_outcome(
-        scholar=scholar,
-        run_dt=run_dt,
-        paged_parse_result=paged_parse_result,
-        result_entry=result_entry,
-    )
-
-
-async def _upsert_success_or_exception(
-    db_session: AsyncSession,
-    *,
-    run: CrawlRun,
-    scholar: ScholarProfile,
-    run_dt: datetime,
-    paged_parse_result: PagedParseResult,
-    result_entry: dict[str, Any],
-    has_partial_publication_set: bool,
-) -> ScholarProcessingOutcome:
-    try:
-        return _upsert_success(
-            run=run,
-            scholar=scholar,
-            run_dt=run_dt,
-            paged_parse_result=paged_parse_result,
-            result_entry=result_entry,
-            has_partial_publication_set=has_partial_publication_set,
-        )
-    except Exception as exc:
-        return _upsert_exception_outcome(
-            run=run,
-            scholar=scholar,
-            run_dt=run_dt,
-            paged_parse_result=paged_parse_result,
-            result_entry=result_entry,
-            exc=exc,
-        )
-
-
-def _upsert_success(
-    *,
-    run: CrawlRun,
-    scholar: ScholarProfile,
-    run_dt: datetime,
-    paged_parse_result: PagedParseResult,
-    result_entry: dict[str, Any],
-    has_partial_publication_set: bool,
-) -> ScholarProcessingOutcome:
-    discovered_count = paged_parse_result.discovered_publication_count
-    is_partial = (
-        paged_parse_result.has_more_remaining
-        or paged_parse_result.pagination_truncated_reason is not None
-        or has_partial_publication_set
-    )
-    scholar.last_run_status = RunStatus.PARTIAL_FAILURE if is_partial else RunStatus.SUCCESS
-    scholar.last_run_dt = run_dt
-    if not is_partial and paged_parse_result.first_page_fingerprint_sha256:
-        scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256
-    result_entry["outcome"] = "partial" if is_partial else "success"
-    if is_partial:
-        result_entry["debug"] = build_failure_debug_context(
-            fetch_result=paged_parse_result.fetch_result,
-            parsed_page=paged_parse_result.parsed_page,
-            attempt_log=paged_parse_result.attempt_log,
-            page_logs=paged_parse_result.page_logs,
-        )
-    return ScholarProcessingOutcome(
-        result_entry=result_entry,
-        succeeded_count_delta=1,
-        failed_count_delta=0,
-        partial_count_delta=1 if is_partial else 0,
-        discovered_publication_count=discovered_count,
-    )
-
-
-def _upsert_exception_outcome(
-    *,
-    run: CrawlRun,
-    scholar: ScholarProfile,
-    run_dt: datetime,
-    paged_parse_result: PagedParseResult,
-    result_entry: dict[str, Any],
-    exc: Exception,
-) -> ScholarProcessingOutcome:
-    scholar.last_run_status = RunStatus.FAILED
-    scholar.last_run_dt = run_dt
-    result_entry["state"] = "ingestion_error"
-    result_entry["state_reason"] = "publication_upsert_exception"
-    result_entry["outcome"] = "failed"
-    result_entry["error"] = str(exc)
-    result_entry["debug"] = build_failure_debug_context(
-        fetch_result=paged_parse_result.fetch_result,
-        parsed_page=paged_parse_result.parsed_page,
-        attempt_log=paged_parse_result.attempt_log,
-        page_logs=paged_parse_result.page_logs,
-        exception=exc,
-    )
-    structured_log(
-        logger,
-        "exception",
-        "ingestion.scholar_failed",
-        crawl_run_id=run.id,
-        scholar_profile_id=scholar.id,
-        scholar_id=scholar.scholar_id,
-    )
-    return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0)
-
-
-def _parse_failure_outcome(
-    *,
-    scholar: ScholarProfile,
-    run_dt: datetime,
-    paged_parse_result: PagedParseResult,
-    result_entry: dict[str, Any],
-) -> ScholarProcessingOutcome:
-    scholar.last_run_status = RunStatus.FAILED
-    scholar.last_run_dt = run_dt
-    result_entry["debug"] = build_failure_debug_context(
-        fetch_result=paged_parse_result.fetch_result,
-        parsed_page=paged_parse_result.parsed_page,
-        attempt_log=paged_parse_result.attempt_log,
-        page_logs=paged_parse_result.page_logs,
-    )
-    structured_log(
-        logger,
-        "warning",
-        "ingestion.scholar_parse_failed",
-        scholar_profile_id=scholar.id,
-        scholar_id=scholar.scholar_id,
-        state=paged_parse_result.parsed_page.state.value,
-        state_reason=paged_parse_result.parsed_page.state_reason,
-        status_code=paged_parse_result.fetch_result.status_code,
-    )
-    return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0)
-
-
-def unexpected_scholar_exception_outcome(
-    *,
-    run: CrawlRun,
-    scholar: ScholarProfile,
-    start_cstart: int,
-    exc: Exception,
-) -> ScholarProcessingOutcome:
-    scholar.last_run_status = RunStatus.FAILED
-    scholar.last_run_dt = datetime.now(UTC)
-    structured_log(
-        logger,
-        "exception",
-        "ingestion.scholar_unexpected_failure",
-        crawl_run_id=run.id,
-        scholar_profile_id=scholar.id,
-        scholar_id=scholar.scholar_id,
-    )
-    return ScholarProcessingOutcome(
-        result_entry={
-            "scholar_profile_id": scholar.id,
-            "scholar_id": scholar.scholar_id,
-            "state": "ingestion_error",
-            "state_reason": "scholar_processing_exception",
-            "outcome": "failed",
-            "attempt_count": 0,
-            "publication_count": 0,
-            "start_cstart": start_cstart,
-            "warnings": [],
-            "error": str(exc),
-            "debug": {"exception_type": type(exc).__name__, "exception_message": str(exc)},
-        },
-        succeeded_count_delta=0,
-        failed_count_delta=1,
-        partial_count_delta=0,
-        discovered_publication_count=0,
-    )

app/services/ingestion/scholar_processing.py

@@ -1,458 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-import random
-from collections.abc import Callable, Coroutine
-from datetime import UTC, datetime
-from typing import Any
-
-from sqlalchemy.exc import InterfaceError, OperationalError
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import (
-    CrawlRun,
-    RunStatus,
-    ScholarProfile,
-)
-from app.logging_utils import structured_log
-from app.services.ingestion import queue as queue_service
-from app.services.ingestion.constants import (
-    RESUMABLE_PARTIAL_REASON_PREFIXES,
-    RESUMABLE_PARTIAL_REASONS,
-)
-from app.services.ingestion.pagination import PaginationEngine
-from app.services.ingestion.publication_upsert import upsert_profile_publications
-from app.services.ingestion.run_completion import apply_outcome_to_progress
-from app.services.ingestion.scholar_outcomes import (
-    apply_first_page_profile_metadata,
-    assert_valid_paged_parse_result,
-    build_result_entry,
-    skipped_no_change_outcome,
-    unexpected_scholar_exception_outcome,
-    upsert_publications_outcome,
-)
-from app.services.ingestion.types import (
-    PagedParseResult,
-    RunProgress,
-    ScholarProcessingOutcome,
-)
-from app.services.scholar.parser import ParseState
-from app.services.scholar.state_detection import is_hard_challenge_reason
-
-logger = logging.getLogger(__name__)
-
-
-async def sync_continuation_queue(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholar: ScholarProfile,
-    run: CrawlRun,
-    start_cstart: int,
-    result_entry: dict[str, Any],
-    paged_parse_result: PagedParseResult,
-    auto_queue_continuations: bool,
-    queue_delay_seconds: int,
-) -> None:
-    queue_reason, queue_cstart = resolve_continuation_queue_target(
-        outcome=str(result_entry.get("outcome", "")),
-        state=str(result_entry.get("state", "")),
-        pagination_truncated_reason=paged_parse_result.pagination_truncated_reason,
-        continuation_cstart=paged_parse_result.continuation_cstart,
-        fallback_cstart=start_cstart,
-    )
-    if auto_queue_continuations and queue_reason is not None:
-        await queue_service.upsert_job(
-            db_session,
-            user_id=user_id,
-            scholar_profile_id=scholar.id,
-            resume_cstart=queue_cstart,
-            reason=queue_reason,
-            run_id=run.id,
-            delay_seconds=queue_delay_seconds,
-        )
-        result_entry["continuation_enqueued"] = True
-        result_entry["continuation_reason"] = queue_reason
-        result_entry["continuation_cstart"] = queue_cstart
-        return
-    if await queue_service.clear_job_for_scholar(db_session, user_id=user_id, scholar_profile_id=scholar.id):
-        result_entry["continuation_cleared"] = True
-
-
-def resolve_continuation_queue_target(
-    *,
-    outcome: str,
-    state: str,
-    pagination_truncated_reason: str | None,
-    continuation_cstart: int | None,
-    fallback_cstart: int,
-) -> tuple[str | None, int]:
-    if outcome == "partial":
-        reason = (pagination_truncated_reason or "").strip()
-        if reason in RESUMABLE_PARTIAL_REASONS or reason.startswith(RESUMABLE_PARTIAL_REASON_PREFIXES):
-            return reason, queue_service.normalize_cstart(
-                continuation_cstart if continuation_cstart is not None else fallback_cstart
-            )
-        return None, queue_service.normalize_cstart(fallback_cstart)
-
-    if outcome == "failed" and state == ParseState.NETWORK_ERROR.value:
-        return "network_error_retry", queue_service.normalize_cstart(
-            continuation_cstart if continuation_cstart is not None else fallback_cstart
-        )
-
-    return None, queue_service.normalize_cstart(fallback_cstart)
-
-
-async def process_scholar(
-    db_session: AsyncSession,
-    *,
-    pagination: PaginationEngine,
-    run: CrawlRun,
-    scholar: ScholarProfile,
-    user_id: int,
-    request_delay_seconds: int,
-    network_error_retries: int,
-    retry_backoff_seconds: float,
-    rate_limit_retries: int,
-    rate_limit_backoff_seconds: float,
-    max_pages_per_scholar: int,
-    page_size: int,
-    start_cstart: int,
-    auto_queue_continuations: bool,
-    queue_delay_seconds: int,
-) -> ScholarProcessingOutcome:
-    try:
-        run_dt, paged_parse_result, result_entry = await _fetch_and_prepare_scholar_result(
-            db_session,
-            pagination=pagination,
-            run=run,
-            scholar=scholar,
-            user_id=user_id,
-            start_cstart=start_cstart,
-            request_delay_seconds=request_delay_seconds,
-            network_error_retries=network_error_retries,
-            retry_backoff_seconds=retry_backoff_seconds,
-            rate_limit_retries=rate_limit_retries,
-            rate_limit_backoff_seconds=rate_limit_backoff_seconds,
-            max_pages_per_scholar=max_pages_per_scholar,
-            page_size=page_size,
-        )
-        outcome = await _resolve_scholar_outcome(
-            db_session,
-            run=run,
-            scholar=scholar,
-            run_dt=run_dt,
-            paged_parse_result=paged_parse_result,
-            result_entry=result_entry,
-        )
-        await sync_continuation_queue(
-            db_session,
-            user_id=user_id,
-            scholar=scholar,
-            run=run,
-            start_cstart=start_cstart,
-            result_entry=outcome.result_entry,
-            paged_parse_result=paged_parse_result,
-            auto_queue_continuations=auto_queue_continuations,
-            queue_delay_seconds=queue_delay_seconds,
-        )
-        return outcome
-    except (OperationalError, InterfaceError):
-        raise
-    except Exception as exc:
-        return unexpected_scholar_exception_outcome(
-            run=run,
-            scholar=scholar,
-            start_cstart=start_cstart,
-            exc=exc,
-        )
-
-
-async def _fetch_and_prepare_scholar_result(
-    db_session: AsyncSession,
-    *,
-    pagination: PaginationEngine,
-    run: CrawlRun,
-    scholar: ScholarProfile,
-    user_id: int,
-    start_cstart: int,
-    request_delay_seconds: int,
-    network_error_retries: int,
-    retry_backoff_seconds: float,
-    rate_limit_retries: int,
-    rate_limit_backoff_seconds: float,
-    max_pages_per_scholar: int,
-    page_size: int,
-) -> tuple[datetime, PagedParseResult, dict[str, Any]]:
-    run_dt = datetime.now(UTC)
-    paged_parse_result = await pagination.fetch_and_parse_all_pages(
-        scholar=scholar,
-        run=run,
-        db_session=db_session,
-        start_cstart=start_cstart,
-        request_delay_seconds=request_delay_seconds,
-        network_error_retries=network_error_retries,
-        retry_backoff_seconds=retry_backoff_seconds,
-        rate_limit_retries=rate_limit_retries,
-        rate_limit_backoff_seconds=rate_limit_backoff_seconds,
-        max_pages=max_pages_per_scholar,
-        page_size=page_size,
-        previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256,
-        upsert_publications_fn=upsert_profile_publications,
-    )
-    assert_valid_paged_parse_result(scholar_id=scholar.scholar_id, paged_parse_result=paged_parse_result)
-    apply_first_page_profile_metadata(scholar=scholar, paged_parse_result=paged_parse_result, run_dt=run_dt)
-    parsed_page = paged_parse_result.parsed_page
-    structured_log(
-        logger,
-        "info",
-        "ingestion.scholar_parsed",
-        user_id=user_id,
-        crawl_run_id=run.id,
-        scholar_profile_id=scholar.id,
-        scholar_id=scholar.scholar_id,
-        state=parsed_page.state.value,
-        publication_count=len(paged_parse_result.publications),
-        has_show_more_button=parsed_page.has_show_more_button,
-        pages_fetched=paged_parse_result.pages_fetched,
-        pages_attempted=paged_parse_result.pages_attempted,
-        has_more_remaining=paged_parse_result.has_more_remaining,
-        pagination_truncated_reason=paged_parse_result.pagination_truncated_reason,
-        warning_count=len(parsed_page.warnings),
-        skipped_no_change=paged_parse_result.skipped_no_change,
-    )
-    result_entry = build_result_entry(
-        scholar=scholar,
-        start_cstart=start_cstart,
-        paged_parse_result=paged_parse_result,
-    )
-    return run_dt, paged_parse_result, result_entry
-
-
-async def _resolve_scholar_outcome(
-    db_session: AsyncSession,
-    *,
-    run: CrawlRun,
-    scholar: ScholarProfile,
-    run_dt: datetime,
-    paged_parse_result: PagedParseResult,
-    result_entry: dict[str, Any],
-) -> ScholarProcessingOutcome:
-    if paged_parse_result.skipped_no_change:
-        return skipped_no_change_outcome(
-            scholar=scholar,
-            run_dt=run_dt,
-            paged_parse_result=paged_parse_result,
-            result_entry=result_entry,
-        )
-    return await upsert_publications_outcome(
-        db_session,
-        run=run,
-        scholar=scholar,
-        run_dt=run_dt,
-        paged_parse_result=paged_parse_result,
-        result_entry=result_entry,
-    )
-
-
-def _is_hard_challenge_outcome(outcome: ScholarProcessingOutcome) -> bool:
-    entry = outcome.result_entry
-    return str(entry.get("state", "")) == ParseState.BLOCKED_OR_CAPTCHA.value and is_hard_challenge_reason(
-        str(entry.get("state_reason", ""))
-    )
-
-
-async def _run_first_pass(
-    db_session: AsyncSession,
-    *,
-    scholars: list[ScholarProfile],
-    pagination: PaginationEngine,
-    run: CrawlRun,
-    user_id: int,
-    start_cstart_map: dict[int, int],
-    scholar_kwargs: dict[str, Any],
-    request_delay_seconds: int,
-    queue_delay_seconds: int,
-    progress: RunProgress,
-    on_progress: Callable[[int, int], Coroutine[Any, Any, None]] | None = None,
-) -> dict[int, int]:
-    first_pass_cstarts: dict[int, int] = {}
-    for index, scholar in enumerate(scholars):
-        await db_session.refresh(run)
-        if run.status == RunStatus.CANCELED:
-            structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id)
-            return first_pass_cstarts
-        if index > 0 and request_delay_seconds > 0:
-            jitter = random.uniform(0.0, min(float(request_delay_seconds), 2.0))
-            await asyncio.sleep(float(request_delay_seconds) + jitter)
-        start_cstart = int(start_cstart_map.get(int(scholar.id), 0))
-        outcome = await process_scholar(
-            db_session,
-            pagination=pagination,
-            run=run,
-            scholar=scholar,
-            user_id=user_id,
-            start_cstart=start_cstart,
-            max_pages_per_scholar=1,
-            auto_queue_continuations=False,
-            queue_delay_seconds=queue_delay_seconds,
-            **scholar_kwargs,
-        )
-        apply_outcome_to_progress(progress=progress, outcome=outcome)
-        if _is_hard_challenge_outcome(outcome):
-            structured_log(
-                logger,
-                "warning",
-                "ingestion.run_aborted_hard_challenge",
-                run_id=run.id,
-                user_id=user_id,
-                scholar_id=scholar.scholar_id,
-                state_reason=outcome.result_entry.get("state_reason"),
-                scholars_remaining=len(scholars) - index - 1,
-            )
-            return first_pass_cstarts
-        if on_progress is not None:
-            await on_progress(1, 0)
-        resume_cstart = outcome.result_entry.get("continuation_cstart")
-        if resume_cstart is not None and int(resume_cstart) > start_cstart:
-            first_pass_cstarts[int(scholar.id)] = int(resume_cstart)
-    return first_pass_cstarts
-
-
-async def _run_depth_pass(
-    db_session: AsyncSession,
-    *,
-    scholars: list[ScholarProfile],
-    first_pass_cstarts: dict[int, int],
-    pagination: PaginationEngine,
-    run: CrawlRun,
-    user_id: int,
-    scholar_kwargs: dict[str, Any],
-    request_delay_seconds: int,
-    remaining_max: int,
-    auto_queue_continuations: bool,
-    queue_delay_seconds: int,
-    progress: RunProgress,
-    on_progress: Callable[[int, int], Coroutine[Any, Any, None]] | None = None,
-) -> None:
-    for index, scholar in enumerate(scholars):
-        resume_cstart = first_pass_cstarts.get(int(scholar.id))
-        if resume_cstart is None:
-            continue
-        await db_session.refresh(run)
-        if run.status == RunStatus.CANCELED:
-            structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id)
-            break
-        if index > 0 and request_delay_seconds > 0:
-            jitter = random.uniform(0.0, min(float(request_delay_seconds), 2.0))
-            await asyncio.sleep(float(request_delay_seconds) + jitter)
-        outcome = await process_scholar(
-            db_session,
-            pagination=pagination,
-            run=run,
-            scholar=scholar,
-            user_id=user_id,
-            start_cstart=resume_cstart,
-            max_pages_per_scholar=remaining_max,
-            auto_queue_continuations=auto_queue_continuations,
-            queue_delay_seconds=queue_delay_seconds,
-            **scholar_kwargs,
-        )
-        apply_outcome_to_progress(progress=progress, outcome=outcome)
-        if _is_hard_challenge_outcome(outcome):
-            structured_log(
-                logger,
-                "warning",
-                "ingestion.run_aborted_hard_challenge",
-                run_id=run.id,
-                user_id=user_id,
-                scholar_id=scholar.scholar_id,
-                state_reason=outcome.result_entry.get("state_reason"),
-                scholars_remaining=len(scholars) - index - 1,
-            )
-            break
-        if on_progress is not None:
-            await on_progress(0, 1)
-
-
-async def run_scholar_iteration(
-    db_session: AsyncSession,
-    *,
-    pagination: PaginationEngine,
-    run: CrawlRun,
-    scholars: list[ScholarProfile],
-    user_id: int,
-    start_cstart_map: dict[int, int],
-    request_delay_seconds: int,
-    network_error_retries: int,
-    retry_backoff_seconds: float,
-    rate_limit_retries: int,
-    rate_limit_backoff_seconds: float,
-    max_pages_per_scholar: int,
-    page_size: int,
-    auto_queue_continuations: bool,
-    queue_delay_seconds: int,
-) -> RunProgress:
-    from app.services.runs.events import run_events
-
-    progress = RunProgress()
-    scholar_kwargs: dict[str, Any] = {
-        "request_delay_seconds": request_delay_seconds,
-        "network_error_retries": network_error_retries,
-        "retry_backoff_seconds": retry_backoff_seconds,
-        "rate_limit_retries": rate_limit_retries,
-        "rate_limit_backoff_seconds": rate_limit_backoff_seconds,
-        "page_size": page_size,
-    }
-
-    visited = 0
-    finished = 0
-    total = len(scholars)
-
-    async def _emit(v: int = 0, f: int = 0) -> None:
-        nonlocal visited, finished
-        visited += v
-        finished += f
-        await run_events.publish(
-            run.id,
-            "scholar_progress",
-            {"visited": visited, "finished": finished, "total": total},
-        )
-
-    await _emit()
-    first_pass_cstarts = await _run_first_pass(
-        db_session,
-        scholars=scholars,
-        pagination=pagination,
-        run=run,
-        user_id=user_id,
-        start_cstart_map=start_cstart_map,
-        scholar_kwargs=scholar_kwargs,
-        request_delay_seconds=request_delay_seconds,
-        queue_delay_seconds=queue_delay_seconds,
-        progress=progress,
-        on_progress=_emit,
-    )
-    remaining_max = max(max_pages_per_scholar - 1, 0)
-    scholars_finished_in_first_pass = len(scholars) - len(first_pass_cstarts)
-    if scholars_finished_in_first_pass > 0:
-        await _emit(f=scholars_finished_in_first_pass)
-    if remaining_max <= 0:
-        return progress
-    await _run_depth_pass(
-        db_session,
-        scholars=scholars,
-        first_pass_cstarts=first_pass_cstarts,
-        pagination=pagination,
-        run=run,
-        user_id=user_id,
-        scholar_kwargs=scholar_kwargs,
-        request_delay_seconds=request_delay_seconds,
-        remaining_max=remaining_max,
-        auto_queue_continuations=auto_queue_continuations,
-        queue_delay_seconds=queue_delay_seconds,
-        progress=progress,
-        on_progress=_emit,
-    )
-    return progress

app/services/openalex/client.py

@@ -8,7 +8,6 @@ from tenacity import (
     wait_exponential,
 )
 
-from app.logging_utils import structured_log
 from app.services.openalex.types import OpenAlexWork
 
 logger = logging.getLogger(__name__)
@@ -83,13 +82,7 @@ class OpenAlexClient:
                 raise OpenAlexBudgetExhaustedError("Daily API budget exhausted; retrying won't help until midnight UTC")
             raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex work by DOI")
         if response.status_code >= 400:
-            structured_log(
+            logger.warning("OpenAlex API error: %s %s", response.status_code, response.text[:500])
-                logger,
-                "warning",
-                "openalex.api_error",
-                status_code=response.status_code,
-                response_preview=response.text[:500],
-            )
             raise OpenAlexClientError(f"API Error {response.status_code}")
 
         data = response.json()
@@ -137,14 +130,7 @@ class OpenAlexClient:
                 raise OpenAlexBudgetExhaustedError("Daily API budget exhausted; retrying won't help until midnight UTC")
             raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex works list")
         if response.status_code >= 400:
-            structured_log(
+            logger.warning("OpenAlex API error (filters=%s): %s %s", filters, response.status_code, response.text[:500])
-                logger,
-                "warning",
-                "openalex.api_error_with_filters",
-                filters=filters,
-                status_code=response.status_code,
-                response_preview=response.text[:500],
-            )
             raise OpenAlexClientError(f"API Error {response.status_code}")
 
         data = response.json()
@@ -154,13 +140,8 @@ class OpenAlexClient:
         for raw_work in results:
             try:
                 parsed_works.append(OpenAlexWork.from_api_dict(raw_work))
-            except Exception as exc:
+            except Exception as e:
-                structured_log(
+                logger.warning("Failed to parse OpenAlex raw dict: %s", e)
-                    logger,
-                    "warning",
-                    "openalex.parse_failed",
-                    error=str(exc),
-                )
                 continue
 
         return parsed_works

app/services/openalex/matching.py

@@ -93,12 +93,8 @@ def find_best_match(
 
     for original_score, cand in top_scored_candidates:
         tb_score = 0
-        if (
+        if target_year is not None and cand.publication_year is not None and abs(target_year - cand.publication_year) <= 1:
-            target_year is not None
+                tb_score += 1
-            and cand.publication_year is not None
-            and abs(target_year - cand.publication_year) <= 1
-        ):
-            tb_score += 1
 
         candidate_author_names = [a.display_name for a in cand.authors if a.display_name]
         if _author_overlap_score(target_authors, candidate_author_names):

app/services/portability/exporting.py

@@ -23,7 +23,7 @@ def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]:
     }
 
 
-def _serialize_export_publication(row: Any) -> dict[str, Any]:
+def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]:
     (
         scholar_id,
         cluster_id,
@@ -56,14 +56,11 @@ async def export_user_data(
     db_session: AsyncSession,
     *,
     user_id: int,
-    scholar_profile_ids: list[int] | None = None,
 ) -> dict[str, Any]:
-    scholar_query = select(ScholarProfile).where(ScholarProfile.user_id == user_id)
+    scholars_result = await db_session.execute(
-    if scholar_profile_ids:
+        select(ScholarProfile).where(ScholarProfile.user_id == user_id).order_by(ScholarProfile.id.asc())
-        scholar_query = scholar_query.where(ScholarProfile.id.in_(scholar_profile_ids))
+    )
-    scholars_result = await db_session.execute(scholar_query.order_by(ScholarProfile.id.asc()))
+    publication_result = await db_session.execute(
-
-    pub_query = (
         select(
             ScholarProfile.scholar_id,
             Publication.cluster_id,
@@ -80,13 +77,8 @@ async def export_user_data(
         .join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id)
         .join(Publication, Publication.id == ScholarPublication.publication_id)
         .where(ScholarProfile.user_id == user_id)
+        .order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
     )
-    if scholar_profile_ids:
-        pub_query = pub_query.where(ScholarProfile.id.in_(scholar_profile_ids))
-    publication_result = await db_session.execute(
-        pub_query.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
-    )
-
     scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()]
     publications = [_serialize_export_publication(row) for row in publication_result.all()]
     return {

app/services/portability/normalize.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import hashlib
 from typing import Any
 
-from app.services.ingestion.fingerprints import normalize_title
+from app.services.ingestion.application import normalize_title
 from app.services.portability.constants import (
     MAX_IMPORT_PUBLICATIONS,
     MAX_IMPORT_SCHOLARS,

app/services/portability/publication_import.py

@@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db.models import Publication, ScholarProfile, ScholarPublication
 from app.services.doi.normalize import normalize_doi
-from app.services.ingestion.fingerprints import build_publication_url, normalize_title
+from app.services.ingestion.application import build_publication_url, normalize_title
 from app.services.portability.normalize import (
     _normalize_citation_count,
     _normalize_optional_text,

app/services/publication_identifiers/application.py

@@ -21,7 +21,7 @@ from app.services.publication_identifiers.types import (
 )
 
 if TYPE_CHECKING:
-    from app.services.publications.pdf_queue_queries import PdfQueueListItem
+    from app.services.publications.pdf_queue import PdfQueueListItem
     from app.services.publications.types import PublicationListItem, UnreadPublicationItem
 
 CONFIDENCE_HIGH = 0.98

app/services/publications/application.py

@@ -24,11 +24,9 @@ from app.services.publications.modes import (
     resolve_publication_view_mode,
 )
 from app.services.publications.pdf_queue import (
+    count_pdf_queue_items,
     enqueue_all_missing_pdf_jobs,
     enqueue_retry_pdf_job_for_publication_id,
-)
-from app.services.publications.pdf_queue_queries import (
-    count_pdf_queue_items,
     list_pdf_queue_items,
     list_pdf_queue_page,
 )

app/services/publications/pdf_queue.py

@@ -1,38 +1,67 @@
 from __future__ import annotations
 
+import asyncio
 import logging
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import UTC, datetime, timedelta
 
-from sqlalchemy import select
+from sqlalchemy import Select, and_, func, literal, or_, select, union_all
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db.models import (
+    Publication,
     PublicationPdfJob,
+    PublicationPdfJobEvent,
+    ScholarProfile,
+    ScholarPublication,
     User,
 )
-from app.services.publications.pdf_queue_common import (
+from app.db.session import get_session_factory
-    PDF_STATUS_FAILED,
+from app.logging_utils import structured_log
-    PDF_STATUS_QUEUED,
+from app.services.publication_identifiers import application as identifier_service
-    PDF_STATUS_RESOLVED,
+from app.services.publication_identifiers.types import DisplayIdentifier
-    PDF_STATUS_RUNNING,
+from app.services.publications.pdf_resolution_pipeline import (
-    event_row,
+    resolve_publication_pdf_outcome_for_row,
-    queued_job,
-    utcnow,
 )
-from app.services.publications.pdf_queue_queries import (
-    missing_pdf_candidates,
-    retry_item_for_publication_id,
-)
-from app.services.publications.pdf_queue_resolution import schedule_rows
 from app.services.publications.types import PublicationListItem
+from app.services.unpaywall.application import (
+    FAILURE_RESOLUTION_EXCEPTION,
+    OaResolutionOutcome,
+)
 from app.settings import settings
 
 PDF_STATUS_UNTRACKED = "untracked"
+PDF_STATUS_QUEUED = "queued"
+PDF_STATUS_RUNNING = "running"
+PDF_STATUS_RESOLVED = "resolved"
+PDF_STATUS_FAILED = "failed"
 
 PDF_EVENT_QUEUED = "queued"
+PDF_EVENT_ATTEMPT_STARTED = "attempt_started"
+PDF_EVENT_RESOLVED = "resolved"
+PDF_EVENT_FAILED = "failed"
 
 logger = logging.getLogger(__name__)
+_scheduled_tasks: set[asyncio.Task[None]] = set()
+
+
+@dataclass(frozen=True)
+class PdfQueueListItem:
+    publication_id: int
+    title: str
+    pdf_url: str | None
+    status: str
+    attempt_count: int
+    last_failure_reason: str | None
+    last_failure_detail: str | None
+    last_source: str | None
+    requested_by_user_id: int | None
+    requested_by_email: str | None
+    queued_at: datetime | None
+    last_attempt_at: datetime | None
+    resolved_at: datetime | None
+    updated_at: datetime
+    display_identifier: DisplayIdentifier | None = None
 
 
 @dataclass(frozen=True)
@@ -47,6 +76,18 @@ class PdfBulkQueueResult:
     queued_count: int
 
 
+@dataclass(frozen=True)
+class PdfQueuePage:
+    items: list[PdfQueueListItem]
+    total_count: int
+    limit: int
+    offset: int
+
+
+def _utcnow() -> datetime:
+    return datetime.now(UTC)
+
+
 def _publication_ids(rows: list[PublicationListItem]) -> list[int]:
     return sorted({row.publication_id for row in rows})
 
@@ -97,6 +138,14 @@ def _queueable_rows(
     return candidates[:bounded]
 
 
+def _bounded_limit(limit: int, *, max_value: int = 500) -> int:
+    return max(1, min(int(limit), max_value))
+
+
+def _bounded_offset(offset: int) -> int:
+    return max(int(offset), 0)
+
+
 def _auto_retry_interval_seconds() -> int:
     return max(int(settings.pdf_auto_retry_interval_seconds), 1)
 
@@ -122,7 +171,7 @@ def _cooldown_active(
 ) -> bool:
     if last_attempt_at is None:
         return False
-    elapsed = (utcnow() - last_attempt_at).total_seconds()
+    elapsed = (_utcnow() - last_attempt_at).total_seconds()
     return elapsed < _retry_interval_seconds_for_attempt_count(int(attempt_count))
 
 
@@ -147,8 +196,43 @@ def _can_enqueue_job(
     )
 
 
+def _event_row(
+    *,
+    publication_id: int,
+    user_id: int | None,
+    event_type: str,
+    status: str | None,
+    source: str | None = None,
+    failure_reason: str | None = None,
+    message: str | None = None,
+) -> PublicationPdfJobEvent:
+    return PublicationPdfJobEvent(
+        publication_id=publication_id,
+        user_id=user_id,
+        event_type=event_type,
+        status=status,
+        source=source,
+        failure_reason=failure_reason,
+        message=message,
+    )
+
+
+def _queued_job(
+    *,
+    publication_id: int,
+    user_id: int,
+) -> PublicationPdfJob:
+    now = _utcnow()
+    return PublicationPdfJob(
+        publication_id=publication_id,
+        status=PDF_STATUS_QUEUED,
+        queued_at=now,
+        last_requested_by_user_id=user_id,
+    )
+
+
 def _mark_job_queued(job: PublicationPdfJob, *, user_id: int) -> None:
-    now = utcnow()
+    now = _utcnow()
     job.status = PDF_STATUS_QUEUED
     job.queued_at = now
     job.last_requested_by_user_id = user_id
@@ -207,13 +291,13 @@ async def _enqueue_rows(
         if not _can_enqueue_job(job, force_retry=force_retry):
             continue
         if job is None:
-            job = queued_job(publication_id=row.publication_id, user_id=user_id)
+            job = _queued_job(publication_id=row.publication_id, user_id=user_id)
             jobs[row.publication_id] = job
             db_session.add(job)
         else:
             _mark_job_queued(job, user_id=user_id)
         db_session.add(
-            event_row(
+            _event_row(
                 publication_id=row.publication_id,
                 user_id=user_id,
                 event_type=PDF_EVENT_QUEUED,
@@ -226,9 +310,247 @@ async def _enqueue_rows(
     return queued
 
 
-# ---------------------------------------------------------------------------
+def _register_task(task: asyncio.Task[None]) -> None:
-# Public API
+    _scheduled_tasks.add(task)
-# ---------------------------------------------------------------------------
+
+
+def _drop_finished_task(task: asyncio.Task[None]) -> None:
+    _scheduled_tasks.discard(task)
+    try:
+        task.result()
+    except Exception:
+        logger.exception("publications.pdf_queue.task_failed")
+
+
+async def _mark_attempt_started(
+    *,
+    publication_id: int,
+    user_id: int,
+) -> None:
+    session_factory = get_session_factory()
+    async with session_factory() as db_session:
+        job = await db_session.get(PublicationPdfJob, publication_id)
+        if job is None:
+            job = _queued_job(publication_id=publication_id, user_id=user_id)
+            db_session.add(job)
+        job.status = PDF_STATUS_RUNNING
+        job.last_attempt_at = _utcnow()
+        job.attempt_count = int(job.attempt_count) + 1
+        db_session.add(
+            _event_row(
+                publication_id=publication_id,
+                user_id=user_id,
+                event_type=PDF_EVENT_ATTEMPT_STARTED,
+                status=PDF_STATUS_RUNNING,
+            )
+        )
+        await db_session.commit()
+
+
+def _failed_outcome(
+    *,
+    row: PublicationListItem,
+) -> OaResolutionOutcome:
+    return OaResolutionOutcome(
+        publication_id=row.publication_id,
+        doi=None,
+        pdf_url=None,
+        failure_reason=FAILURE_RESOLUTION_EXCEPTION,
+        source=None,
+        used_crossref=False,
+    )
+
+
+async def _fetch_outcome_for_row(
+    *,
+    row: PublicationListItem,
+    request_email: str | None,
+    openalex_api_key: str | None = None,
+    allow_arxiv_lookup: bool = True,
+) -> tuple[OaResolutionOutcome, bool]:
+    pipeline_result = await resolve_publication_pdf_outcome_for_row(
+        row=row,
+        request_email=request_email,
+        openalex_api_key=openalex_api_key,
+        allow_arxiv_lookup=allow_arxiv_lookup,
+    )
+    outcome = pipeline_result.outcome
+    if outcome is not None:
+        return outcome, bool(pipeline_result.arxiv_rate_limited)
+    return _failed_outcome(row=row), bool(pipeline_result.arxiv_rate_limited)
+
+
+def _apply_publication_update(
+    publication: Publication,
+    *,
+    pdf_url: str | None,
+) -> None:
+    if pdf_url and publication.pdf_url != pdf_url:
+        publication.pdf_url = pdf_url
+
+
+def _apply_job_outcome(job: PublicationPdfJob, *, outcome: OaResolutionOutcome) -> None:
+    job.last_source = outcome.source
+    if outcome.pdf_url:
+        job.status = PDF_STATUS_RESOLVED
+        job.resolved_at = _utcnow()
+        job.last_failure_reason = None
+        job.last_failure_detail = None
+        return
+    job.status = PDF_STATUS_FAILED
+    job.last_failure_reason = outcome.failure_reason
+    job.last_failure_detail = outcome.failure_reason
+
+
+def _result_event(outcome: OaResolutionOutcome) -> tuple[str, str]:
+    if outcome.pdf_url:
+        return PDF_EVENT_RESOLVED, PDF_STATUS_RESOLVED
+    return PDF_EVENT_FAILED, PDF_STATUS_FAILED
+
+
+async def _persist_outcome(
+    *,
+    publication_id: int,
+    user_id: int,
+    outcome: OaResolutionOutcome,
+) -> None:
+    session_factory = get_session_factory()
+    async with session_factory() as db_session:
+        publication = await db_session.get(Publication, publication_id)
+        job = await db_session.get(PublicationPdfJob, publication_id)
+        if publication is None or job is None:
+            return
+        _apply_publication_update(publication, pdf_url=outcome.pdf_url)
+        await identifier_service.sync_identifiers_for_publication_resolution(
+            db_session,
+            publication=publication,
+            source=outcome.source,
+        )
+        _apply_job_outcome(job, outcome=outcome)
+        event_type, status = _result_event(outcome)
+        db_session.add(
+            _event_row(
+                publication_id=publication_id,
+                user_id=user_id,
+                event_type=event_type,
+                status=status,
+                source=outcome.source,
+                failure_reason=outcome.failure_reason,
+                message=outcome.failure_reason,
+            )
+        )
+        await db_session.commit()
+
+
+async def _resolve_publication_row(
+    *,
+    user_id: int,
+    request_email: str | None,
+    row: PublicationListItem,
+    openalex_api_key: str | None = None,
+    allow_arxiv_lookup: bool = True,
+) -> bool:
+    from app.services.openalex.client import OpenAlexBudgetExhaustedError
+
+    await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
+    try:
+        outcome, arxiv_rate_limited = await _fetch_outcome_for_row(
+            row=row,
+            request_email=request_email,
+            openalex_api_key=openalex_api_key,
+            allow_arxiv_lookup=allow_arxiv_lookup,
+        )
+    except OpenAlexBudgetExhaustedError:
+        # Persist a terminal outcome so jobs do not remain stuck in "running".
+        await _persist_outcome(
+            publication_id=row.publication_id,
+            user_id=user_id,
+            outcome=_failed_outcome(row=row),
+        )
+        # Propagate upward so the batch loop can stop immediately.
+        raise
+    except Exception as exc:  # pragma: no cover - defensive network boundary
+        structured_log(
+            logger,
+            "warning",
+            "publications.pdf_queue.resolve_failed",
+            publication_id=row.publication_id,
+            error=str(exc),
+        )
+        outcome = _failed_outcome(row=row)
+        arxiv_rate_limited = False
+    await _persist_outcome(
+        publication_id=row.publication_id,
+        user_id=user_id,
+        outcome=outcome,
+    )
+    return bool(arxiv_rate_limited)
+
+
+async def _run_resolution_task(
+    *,
+    user_id: int,
+    request_email: str | None,
+    rows: list[PublicationListItem],
+) -> None:
+    from app.services.openalex.client import OpenAlexBudgetExhaustedError
+    from app.services.settings import application as user_settings_service
+
+    # Resolve the best available API key: per-user setting → env var fallback.
+    openalex_api_key: str | None = None
+    try:
+        session_factory = get_session_factory()
+        async with session_factory() as key_session:
+            user_settings = await user_settings_service.get_or_create_settings(key_session, user_id=user_id)
+            openalex_api_key = getattr(user_settings, "openalex_api_key", None) or settings.openalex_api_key
+    except Exception:
+        openalex_api_key = settings.openalex_api_key
+
+    arxiv_lookup_allowed = True
+    for row in rows:
+        try:
+            arxiv_rate_limited = await _resolve_publication_row(
+                user_id=user_id,
+                request_email=request_email,
+                row=row,
+                openalex_api_key=openalex_api_key,
+                allow_arxiv_lookup=arxiv_lookup_allowed,
+            )
+            if arxiv_rate_limited and arxiv_lookup_allowed:
+                arxiv_lookup_allowed = False
+                structured_log(
+                    logger,
+                    "warning",
+                    "pdf_queue.arxiv_batch_disabled",
+                    detail="arXiv temporarily disabled for remaining batch after rate limit",
+                )
+        except OpenAlexBudgetExhaustedError:
+            structured_log(
+                logger,
+                "warning",
+                "pdf_queue.budget_exhausted",
+                detail="Stopping PDF resolution batch — OpenAlex daily budget exhausted",
+            )
+            break
+
+
+def _schedule_rows(
+    *,
+    user_id: int,
+    request_email: str | None,
+    rows: list[PublicationListItem],
+) -> None:
+    if not rows:
+        return
+    task = asyncio.create_task(
+        _run_resolution_task(
+            user_id=user_id,
+            request_email=request_email,
+            rows=rows,
+        )
+    )
+    _register_task(task)
+    task.add_done_callback(_drop_finished_task)
 
 
 async def enqueue_missing_pdf_jobs(
@@ -246,7 +568,7 @@ async def enqueue_missing_pdf_jobs(
         rows=queueable,
         force_retry=False,
     )
-    schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
+    _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
     return [row.publication_id for row in queued_rows]
 
 
@@ -263,10 +585,69 @@ async def enqueue_retry_pdf_job(
         rows=[row],
         force_retry=True,
     )
-    schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
+    _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
     return bool(queued_rows)
 
 
+def _retry_item_label(display_name: str | None, scholar_id: str | None) -> str:
+    return str(display_name or scholar_id or "unknown")
+
+
+def _retry_item_from_publication(
+    publication: Publication,
+    *,
+    link_row: tuple | None,
+) -> PublicationListItem:
+    if link_row is None:
+        scholar_profile_id = 0
+        scholar_label = "unknown"
+        is_read = True
+        first_seen_at = publication.created_at or _utcnow()
+    else:
+        scholar_profile_id = int(link_row[0])
+        scholar_label = _retry_item_label(link_row[1], link_row[2])
+        is_read = bool(link_row[3])
+        first_seen_at = link_row[4] or publication.created_at or _utcnow()
+    return PublicationListItem(
+        publication_id=int(publication.id),
+        scholar_profile_id=scholar_profile_id,
+        scholar_label=scholar_label,
+        title=publication.title_raw,
+        year=publication.year,
+        citation_count=int(publication.citation_count or 0),
+        venue_text=publication.venue_text,
+        pub_url=publication.pub_url,
+        pdf_url=publication.pdf_url,
+        is_read=is_read,
+        first_seen_at=first_seen_at,
+        is_new_in_latest_run=False,
+    )
+
+
+async def _retry_item_for_publication_id(
+    db_session: AsyncSession,
+    *,
+    publication_id: int,
+) -> PublicationListItem | None:
+    publication = await db_session.get(Publication, publication_id)
+    if publication is None:
+        return None
+    result = await db_session.execute(
+        select(
+            ScholarProfile.id,
+            ScholarProfile.display_name,
+            ScholarProfile.scholar_id,
+            ScholarPublication.is_read,
+            ScholarPublication.created_at,
+        )
+        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
+        .where(ScholarPublication.publication_id == publication_id)
+        .order_by(ScholarPublication.created_at.asc())
+        .limit(1)
+    )
+    return _retry_item_from_publication(publication, link_row=result.one_or_none())
+
+
 async def enqueue_retry_pdf_job_for_publication_id(
     db_session: AsyncSession,
     *,
@@ -274,7 +655,7 @@ async def enqueue_retry_pdf_job_for_publication_id(
     request_email: str | None,
     publication_id: int,
 ) -> PdfRequeueResult:
-    row = await retry_item_for_publication_id(
+    row = await _retry_item_for_publication_id(
         db_session,
         publication_id=publication_id,
     )
@@ -289,6 +670,54 @@ async def enqueue_retry_pdf_job_for_publication_id(
     return PdfRequeueResult(publication_exists=True, queued=queued)
 
 
+def _queue_candidate_from_publication(publication: Publication) -> PublicationListItem:
+    return PublicationListItem(
+        publication_id=int(publication.id),
+        scholar_profile_id=0,
+        scholar_label="",
+        title=publication.title_raw,
+        year=publication.year,
+        citation_count=int(publication.citation_count or 0),
+        venue_text=publication.venue_text,
+        pub_url=publication.pub_url,
+        pdf_url=publication.pdf_url,
+        is_read=True,
+        first_seen_at=publication.created_at or _utcnow(),
+        is_new_in_latest_run=False,
+    )
+
+
+async def _missing_pdf_candidates(
+    db_session: AsyncSession,
+    *,
+    limit: int,
+) -> list[PublicationListItem]:
+    bounded_limit = max(1, min(int(limit), 5000))
+    now = datetime.now(UTC)
+    cooldown_threshold = now - timedelta(days=7)
+
+    result = await db_session.execute(
+        select(Publication)
+        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
+        .where(Publication.pdf_url.is_(None))
+        .where(
+            or_(
+                PublicationPdfJob.publication_id.is_(None),
+                and_(
+                    PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
+                    or_(
+                        PublicationPdfJob.last_attempt_at.is_(None),
+                        PublicationPdfJob.last_attempt_at < cooldown_threshold,
+                    ),
+                ),
+            )
+        )
+        .order_by(Publication.updated_at.desc(), Publication.id.desc())
+        .limit(bounded_limit)
+    )
+    return [_queue_candidate_from_publication(publication) for publication in result.scalars()]
+
+
 async def enqueue_all_missing_pdf_jobs(
     db_session: AsyncSession,
     *,
@@ -296,20 +725,230 @@ async def enqueue_all_missing_pdf_jobs(
     request_email: str | None,
     limit: int = 1000,
 ) -> PdfBulkQueueResult:
-    candidates = await missing_pdf_candidates(db_session, limit=limit)
+    candidates = await _missing_pdf_candidates(db_session, limit=limit)
     queued_rows = await _enqueue_rows(
         db_session,
         user_id=user_id,
         rows=candidates,
         force_retry=True,
     )
-    schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
+    _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
     return PdfBulkQueueResult(
         requested_count=len(candidates),
         queued_count=len(queued_rows),
     )
 
 
+def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]:
+    stmt = (
+        select(
+            PublicationPdfJob.publication_id,
+            Publication.title_raw,
+            Publication.pdf_url,
+            PublicationPdfJob.status,
+            PublicationPdfJob.attempt_count,
+            PublicationPdfJob.last_failure_reason,
+            PublicationPdfJob.last_failure_detail,
+            PublicationPdfJob.last_source,
+            PublicationPdfJob.last_requested_by_user_id,
+            User.email,
+            PublicationPdfJob.queued_at,
+            PublicationPdfJob.last_attempt_at,
+            PublicationPdfJob.resolved_at,
+            PublicationPdfJob.updated_at,
+        )
+        .join(Publication, Publication.id == PublicationPdfJob.publication_id)
+        .outerjoin(User, User.id == PublicationPdfJob.last_requested_by_user_id)
+    )
+    if status:
+        stmt = stmt.where(PublicationPdfJob.status == status)
+    return stmt
+
+
+def _tracked_queue_select(*, limit: int, offset: int, status: str | None) -> Select[tuple]:
+    return (
+        _tracked_queue_select_base(status=status)
+        .order_by(PublicationPdfJob.updated_at.desc())
+        .limit(_bounded_limit(limit))
+        .offset(_bounded_offset(offset))
+    )
+
+
+def _untracked_queue_select_base() -> Select[tuple]:
+    return (
+        select(
+            Publication.id,
+            Publication.title_raw,
+            Publication.pdf_url,
+            literal(PDF_STATUS_UNTRACKED),
+            literal(0),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            literal(None),
+            Publication.updated_at,
+        )
+        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
+        .where(Publication.pdf_url.is_(None))
+        .where(PublicationPdfJob.publication_id.is_(None))
+    )
+
+
+def _untracked_queue_select(*, limit: int, offset: int) -> Select[tuple]:
+    return (
+        _untracked_queue_select_base()
+        .order_by(Publication.updated_at.desc(), Publication.id.desc())
+        .limit(_bounded_limit(limit))
+        .offset(_bounded_offset(offset))
+    )
+
+
+def _all_queue_select(*, limit: int, offset: int) -> Select[tuple]:
+    union_stmt = union_all(
+        _tracked_queue_select_base(status=None),
+        _untracked_queue_select_base(),
+    ).subquery()
+    return (
+        select(union_stmt)
+        .order_by(union_stmt.c.updated_at.desc())
+        .limit(_bounded_limit(limit))
+        .offset(_bounded_offset(offset))
+    )
+
+
+def _tracked_queue_count_select(*, status: str | None) -> Select[tuple]:
+    stmt = select(func.count()).select_from(PublicationPdfJob)
+    if status:
+        stmt = stmt.where(PublicationPdfJob.status == status)
+    return stmt
+
+
+def _untracked_queue_count_select() -> Select[tuple]:
+    return (
+        select(func.count())
+        .select_from(Publication)
+        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
+        .where(Publication.pdf_url.is_(None))
+        .where(PublicationPdfJob.publication_id.is_(None))
+    )
+
+
+def _queue_item_from_row(row: tuple) -> PdfQueueListItem:
+    return PdfQueueListItem(
+        publication_id=int(row[0]),
+        title=str(row[1] or ""),
+        pdf_url=row[2],
+        status=str(row[3] or PDF_STATUS_UNTRACKED),
+        attempt_count=int(row[4] or 0),
+        last_failure_reason=row[5],
+        last_failure_detail=row[6],
+        last_source=row[7],
+        requested_by_user_id=int(row[8]) if row[8] is not None else None,
+        requested_by_email=row[9],
+        queued_at=row[10],
+        last_attempt_at=row[11],
+        resolved_at=row[12],
+        updated_at=row[13],
+    )
+
+
+async def _hydrated_queue_items(
+    db_session: AsyncSession,
+    *,
+    rows: list[tuple],
+) -> list[PdfQueueListItem]:
+    items = [_queue_item_from_row(row) for row in rows]
+    return await identifier_service.overlay_pdf_queue_items_with_display_identifiers(
+        db_session,
+        items=items,
+    )
+
+
+async def list_pdf_queue_items(
+    db_session: AsyncSession,
+    *,
+    limit: int = 100,
+    offset: int = 0,
+    status: str | None = None,
+) -> list[PdfQueueListItem]:
+    bounded_limit = _bounded_limit(limit)
+    bounded_offset = _bounded_offset(offset)
+    normalized_status = (status or "").strip().lower() or None
+    if normalized_status == PDF_STATUS_UNTRACKED:
+        result = await db_session.execute(
+            _untracked_queue_select(
+                limit=bounded_limit,
+                offset=bounded_offset,
+            )
+        )
+        return await _hydrated_queue_items(db_session, rows=list(result.all()))
+    if normalized_status is None:
+        result = await db_session.execute(
+            _all_queue_select(
+                limit=bounded_limit,
+                offset=bounded_offset,
+            )
+        )
+        return await _hydrated_queue_items(db_session, rows=list(result.all()))
+    result = await db_session.execute(
+        _tracked_queue_select(
+            limit=bounded_limit,
+            offset=bounded_offset,
+            status=normalized_status,
+        )
+    )
+    return await _hydrated_queue_items(db_session, rows=list(result.all()))
+
+
+async def count_pdf_queue_items(
+    db_session: AsyncSession,
+    *,
+    status: str | None = None,
+) -> int:
+    normalized_status = (status or "").strip().lower() or None
+    if normalized_status == PDF_STATUS_UNTRACKED:
+        result = await db_session.execute(_untracked_queue_count_select())
+        return int(result.scalar_one() or 0)
+    tracked_result = await db_session.execute(_tracked_queue_count_select(status=normalized_status))
+    tracked_count = int(tracked_result.scalar_one() or 0)
+    if normalized_status is not None:
+        return tracked_count
+    untracked_result = await db_session.execute(_untracked_queue_count_select())
+    untracked_count = int(untracked_result.scalar_one() or 0)
+    return tracked_count + untracked_count
+
+
+async def list_pdf_queue_page(
+    db_session: AsyncSession,
+    *,
+    limit: int = 100,
+    offset: int = 0,
+    status: str | None = None,
+) -> PdfQueuePage:
+    bounded_limit = _bounded_limit(limit)
+    bounded_offset = _bounded_offset(offset)
+    items = await list_pdf_queue_items(
+        db_session,
+        limit=bounded_limit,
+        offset=bounded_offset,
+        status=status,
+    )
+    total_count = await count_pdf_queue_items(
+        db_session,
+        status=status,
+    )
+    return PdfQueuePage(
+        items=items,
+        total_count=total_count,
+        limit=bounded_limit,
+        offset=bounded_offset,
+    )
+
+
 async def drain_ready_jobs(
     db_session: AsyncSession,
     *,

app/services/publications/pdf_queue_common.py

@@ -1,51 +0,0 @@
-from __future__ import annotations
-
-from datetime import UTC, datetime
-
-from app.db.models import PublicationPdfJob, PublicationPdfJobEvent
-
-PDF_STATUS_UNTRACKED = "untracked"
-PDF_STATUS_QUEUED = "queued"
-PDF_STATUS_RUNNING = "running"
-PDF_STATUS_RESOLVED = "resolved"
-PDF_STATUS_FAILED = "failed"
-
-
-def utcnow() -> datetime:
-    return datetime.now(UTC)
-
-
-def event_row(
-    *,
-    publication_id: int,
-    user_id: int | None,
-    event_type: str,
-    status: str | None,
-    source: str | None = None,
-    failure_reason: str | None = None,
-    message: str | None = None,
-) -> PublicationPdfJobEvent:
-    return PublicationPdfJobEvent(
-        publication_id=publication_id,
-        user_id=user_id,
-        event_type=event_type,
-        status=status,
-        source=source,
-        failure_reason=failure_reason,
-        message=message,
-    )
-
-
-def queued_job(
-    *,
-    publication_id: int,
-    user_id: int,
-) -> PublicationPdfJob:
-    now = utcnow()
-    return PublicationPdfJob(
-        publication_id=publication_id,
-        status=PDF_STATUS_QUEUED,
-        queued_at=now,
-        last_requested_by_user_id=user_id,
-        attempt_count=0,
-    )

app/services/publications/pdf_queue_queries.py

@@ -1,395 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from datetime import UTC, datetime, timedelta
-from typing import Any
-
-from sqlalchemy import Select, and_, func, literal, or_, select, union_all
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import (
-    Publication,
-    PublicationPdfJob,
-    ScholarProfile,
-    ScholarPublication,
-    User,
-)
-from app.services.publication_identifiers import application as identifier_service
-from app.services.publication_identifiers.types import DisplayIdentifier
-from app.services.publications.pdf_queue_common import (
-    PDF_STATUS_QUEUED,
-    PDF_STATUS_RUNNING,
-    PDF_STATUS_UNTRACKED,
-)
-from app.services.publications.types import PublicationListItem
-
-
-def _bounded_limit(limit: int, *, max_value: int = 500) -> int:
-    return max(1, min(int(limit), max_value))
-
-
-def _bounded_offset(offset: int) -> int:
-    return max(int(offset), 0)
-
-
-def _utcnow() -> datetime:
-    return datetime.now(UTC)
-
-
-def _retry_item_label(display_name: str | None, scholar_id: str | None) -> str:
-    return str(display_name or scholar_id or "unknown")
-
-
-def _retry_item_from_publication(
-    publication: Publication,
-    *,
-    link_row: Any | None,
-) -> PublicationListItem:
-    if link_row is None:
-        scholar_profile_id = 0
-        scholar_label = "unknown"
-        is_read = True
-        first_seen_at = publication.created_at or _utcnow()
-    else:
-        scholar_profile_id = int(link_row[0])
-        scholar_label = _retry_item_label(link_row[1], link_row[2])
-        is_read = bool(link_row[3])
-        first_seen_at = link_row[4] or publication.created_at or _utcnow()
-    return PublicationListItem(
-        publication_id=int(publication.id),
-        scholar_profile_id=scholar_profile_id,
-        scholar_label=scholar_label,
-        title=publication.title_raw,
-        year=publication.year,
-        citation_count=int(publication.citation_count or 0),
-        venue_text=publication.venue_text,
-        pub_url=publication.pub_url,
-        pdf_url=publication.pdf_url,
-        is_read=is_read,
-        first_seen_at=first_seen_at,
-        is_new_in_latest_run=False,
-    )
-
-
-async def retry_item_for_publication_id(
-    db_session: AsyncSession,
-    *,
-    publication_id: int,
-) -> PublicationListItem | None:
-    publication = await db_session.get(Publication, publication_id)
-    if publication is None:
-        return None
-    result = await db_session.execute(
-        select(
-            ScholarProfile.id,
-            ScholarProfile.display_name,
-            ScholarProfile.scholar_id,
-            ScholarPublication.is_read,
-            ScholarPublication.created_at,
-        )
-        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
-        .where(ScholarPublication.publication_id == publication_id)
-        .order_by(ScholarPublication.created_at.asc())
-        .limit(1)
-    )
-    return _retry_item_from_publication(publication, link_row=result.one_or_none())
-
-
-def _queue_candidate_from_publication(publication: Publication) -> PublicationListItem:
-    return PublicationListItem(
-        publication_id=int(publication.id),
-        scholar_profile_id=0,
-        scholar_label="",
-        title=publication.title_raw,
-        year=publication.year,
-        citation_count=int(publication.citation_count or 0),
-        venue_text=publication.venue_text,
-        pub_url=publication.pub_url,
-        pdf_url=publication.pdf_url,
-        is_read=True,
-        first_seen_at=publication.created_at or _utcnow(),
-        is_new_in_latest_run=False,
-    )
-
-
-async def missing_pdf_candidates(
-    db_session: AsyncSession,
-    *,
-    limit: int,
-) -> list[PublicationListItem]:
-    bounded_limit = max(1, min(int(limit), 5000))
-    now = datetime.now(UTC)
-    cooldown_threshold = now - timedelta(days=7)
-
-    result = await db_session.execute(
-        select(Publication)
-        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
-        .where(Publication.pdf_url.is_(None))
-        .where(
-            or_(
-                PublicationPdfJob.publication_id.is_(None),
-                and_(
-                    PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
-                    or_(
-                        PublicationPdfJob.last_attempt_at.is_(None),
-                        PublicationPdfJob.last_attempt_at < cooldown_threshold,
-                    ),
-                ),
-            )
-        )
-        .order_by(Publication.updated_at.desc(), Publication.id.desc())
-        .limit(bounded_limit)
-    )
-    return [_queue_candidate_from_publication(publication) for publication in result.scalars()]
-
-
-# ---------------------------------------------------------------------------
-# Tracked / untracked queue SQL builders
-# ---------------------------------------------------------------------------
-
-
-def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]:
-    stmt = (
-        select(
-            PublicationPdfJob.publication_id,
-            Publication.title_raw,
-            Publication.pdf_url,
-            PublicationPdfJob.status,
-            PublicationPdfJob.attempt_count,
-            PublicationPdfJob.last_failure_reason,
-            PublicationPdfJob.last_failure_detail,
-            PublicationPdfJob.last_source,
-            PublicationPdfJob.last_requested_by_user_id,
-            User.email,
-            PublicationPdfJob.queued_at,
-            PublicationPdfJob.last_attempt_at,
-            PublicationPdfJob.resolved_at,
-            PublicationPdfJob.updated_at,
-        )
-        .join(Publication, Publication.id == PublicationPdfJob.publication_id)
-        .outerjoin(User, User.id == PublicationPdfJob.last_requested_by_user_id)
-    )
-    if status:
-        stmt = stmt.where(PublicationPdfJob.status == status)
-    return stmt
-
-
-def _tracked_queue_select(*, limit: int, offset: int, status: str | None) -> Select[tuple]:
-    return (
-        _tracked_queue_select_base(status=status)
-        .order_by(PublicationPdfJob.updated_at.desc())
-        .limit(_bounded_limit(limit))
-        .offset(_bounded_offset(offset))
-    )
-
-
-def _untracked_queue_select_base() -> Select[tuple]:
-    return (
-        select(
-            Publication.id,
-            Publication.title_raw,
-            Publication.pdf_url,
-            literal(PDF_STATUS_UNTRACKED),
-            literal(0),
-            literal(None),
-            literal(None),
-            literal(None),
-            literal(None),
-            literal(None),
-            literal(None),
-            literal(None),
-            literal(None),
-            Publication.updated_at,
-        )
-        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
-        .where(Publication.pdf_url.is_(None))
-        .where(PublicationPdfJob.publication_id.is_(None))
-    )
-
-
-def _untracked_queue_select(*, limit: int, offset: int) -> Select[tuple]:
-    return (
-        _untracked_queue_select_base()
-        .order_by(Publication.updated_at.desc(), Publication.id.desc())
-        .limit(_bounded_limit(limit))
-        .offset(_bounded_offset(offset))
-    )
-
-
-def _all_queue_select(*, limit: int, offset: int) -> Select[tuple]:
-    union_stmt = union_all(
-        _tracked_queue_select_base(status=None),
-        _untracked_queue_select_base(),
-    ).subquery()
-    return (
-        select(union_stmt)
-        .order_by(union_stmt.c.updated_at.desc())
-        .limit(_bounded_limit(limit))
-        .offset(_bounded_offset(offset))
-    )
-
-
-def _tracked_queue_count_select(*, status: str | None) -> Select[tuple]:
-    stmt = select(func.count()).select_from(PublicationPdfJob)
-    if status:
-        stmt = stmt.where(PublicationPdfJob.status == status)
-    return stmt
-
-
-def _untracked_queue_count_select() -> Select[tuple]:
-    return (
-        select(func.count())
-        .select_from(Publication)
-        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
-        .where(Publication.pdf_url.is_(None))
-        .where(PublicationPdfJob.publication_id.is_(None))
-    )
-
-
-# ---------------------------------------------------------------------------
-# Row hydration
-# ---------------------------------------------------------------------------
-
-
-@dataclass(frozen=True)
-class PdfQueueListItem:
-    publication_id: int
-    title: str
-    pdf_url: str | None
-    status: str
-    attempt_count: int
-    last_failure_reason: str | None
-    last_failure_detail: str | None
-    last_source: str | None
-    requested_by_user_id: int | None
-    requested_by_email: str | None
-    queued_at: datetime | None
-    last_attempt_at: datetime | None
-    resolved_at: datetime | None
-    updated_at: datetime
-    display_identifier: DisplayIdentifier | None = None
-
-
-def _queue_item_from_row(row: Any) -> PdfQueueListItem:
-    return PdfQueueListItem(
-        publication_id=int(row[0]),
-        title=str(row[1] or ""),
-        pdf_url=row[2],
-        status=str(row[3] or PDF_STATUS_UNTRACKED),
-        attempt_count=int(row[4] or 0),
-        last_failure_reason=row[5],
-        last_failure_detail=row[6],
-        last_source=row[7],
-        requested_by_user_id=int(row[8]) if row[8] is not None else None,
-        requested_by_email=row[9],
-        queued_at=row[10],
-        last_attempt_at=row[11],
-        resolved_at=row[12],
-        updated_at=row[13],
-    )
-
-
-async def _hydrated_queue_items(
-    db_session: AsyncSession,
-    *,
-    rows: list[Any],
-) -> list[PdfQueueListItem]:
-    items = [_queue_item_from_row(row) for row in rows]
-    return await identifier_service.overlay_pdf_queue_items_with_display_identifiers(
-        db_session,
-        items=items,
-    )
-
-
-# ---------------------------------------------------------------------------
-# Public listing / counting
-# ---------------------------------------------------------------------------
-
-
-@dataclass(frozen=True)
-class PdfQueuePage:
-    items: list[PdfQueueListItem]
-    total_count: int
-    limit: int
-    offset: int
-
-
-async def list_pdf_queue_items(
-    db_session: AsyncSession,
-    *,
-    limit: int = 100,
-    offset: int = 0,
-    status: str | None = None,
-) -> list[PdfQueueListItem]:
-    bounded_limit = _bounded_limit(limit)
-    bounded_offset = _bounded_offset(offset)
-    normalized_status = (status or "").strip().lower() or None
-    if normalized_status == PDF_STATUS_UNTRACKED:
-        result = await db_session.execute(
-            _untracked_queue_select(
-                limit=bounded_limit,
-                offset=bounded_offset,
-            )
-        )
-        return await _hydrated_queue_items(db_session, rows=list(result.all()))
-    if normalized_status is None:
-        result = await db_session.execute(
-            _all_queue_select(
-                limit=bounded_limit,
-                offset=bounded_offset,
-            )
-        )
-        return await _hydrated_queue_items(db_session, rows=list(result.all()))
-    result = await db_session.execute(
-        _tracked_queue_select(
-            limit=bounded_limit,
-            offset=bounded_offset,
-            status=normalized_status,
-        )
-    )
-    return await _hydrated_queue_items(db_session, rows=list(result.all()))
-
-
-async def count_pdf_queue_items(
-    db_session: AsyncSession,
-    *,
-    status: str | None = None,
-) -> int:
-    normalized_status = (status or "").strip().lower() or None
-    if normalized_status == PDF_STATUS_UNTRACKED:
-        result = await db_session.execute(_untracked_queue_count_select())
-        return int(result.scalar_one() or 0)
-    tracked_result = await db_session.execute(_tracked_queue_count_select(status=normalized_status))
-    tracked_count = int(tracked_result.scalar_one() or 0)
-    if normalized_status is not None:
-        return tracked_count
-    untracked_result = await db_session.execute(_untracked_queue_count_select())
-    untracked_count = int(untracked_result.scalar_one() or 0)
-    return tracked_count + untracked_count
-
-
-async def list_pdf_queue_page(
-    db_session: AsyncSession,
-    *,
-    limit: int = 100,
-    offset: int = 0,
-    status: str | None = None,
-) -> PdfQueuePage:
-    bounded_limit = _bounded_limit(limit)
-    bounded_offset = _bounded_offset(offset)
-    items = await list_pdf_queue_items(
-        db_session,
-        limit=bounded_limit,
-        offset=bounded_offset,
-        status=status,
-    )
-    total_count = await count_pdf_queue_items(
-        db_session,
-        status=status,
-    )
-    return PdfQueuePage(
-        items=items,
-        total_count=total_count,
-        limit=bounded_limit,
-        offset=bounded_offset,
-    )

app/services/publications/pdf_queue_resolution.py

@@ -1,305 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-from datetime import UTC, datetime, timedelta
-
-from app.db.background_session import background_session
-from app.db.models import Publication, PublicationPdfJob
-from app.logging_utils import structured_log
-from app.services.publication_identifiers import application as identifier_service
-from app.services.publications.pdf_queue_common import (
-    PDF_STATUS_FAILED,
-    PDF_STATUS_RESOLVED,
-    PDF_STATUS_RUNNING,
-    event_row,
-    queued_job,
-    utcnow,
-)
-from app.services.publications.pdf_resolution_pipeline import (
-    resolve_publication_pdf_outcome_for_row,
-)
-from app.services.publications.types import PublicationListItem
-from app.services.unpaywall.application import (
-    FAILURE_RESOLUTION_EXCEPTION,
-    OaResolutionOutcome,
-)
-from app.settings import settings
-
-PDF_EVENT_ATTEMPT_STARTED = "attempt_started"
-PDF_EVENT_RESOLVED = "resolved"
-PDF_EVENT_FAILED = "failed"
-
-_BUDGET_COOLDOWN_MINUTES = 15
-
-logger = logging.getLogger(__name__)
-_scheduled_tasks: set[asyncio.Task[None]] = set()
-_budget_cooldown_until: datetime | None = None
-
-
-def is_budget_cooldown_active() -> bool:
-    return _budget_cooldown_until is not None and datetime.now(UTC) < _budget_cooldown_until
-
-
-def _enter_budget_cooldown() -> None:
-    global _budget_cooldown_until
-    _budget_cooldown_until = datetime.now(UTC) + timedelta(minutes=_BUDGET_COOLDOWN_MINUTES)
-
-
-async def _mark_attempt_started(
-    *,
-    publication_id: int,
-    user_id: int,
-) -> None:
-    async with background_session() as db_session:
-        job = await db_session.get(PublicationPdfJob, publication_id)
-        if job is None:
-            job = queued_job(publication_id=publication_id, user_id=user_id)
-            db_session.add(job)
-        job.status = PDF_STATUS_RUNNING
-        job.last_attempt_at = utcnow()
-        job.attempt_count = int(job.attempt_count or 0) + 1
-        db_session.add(
-            event_row(
-                publication_id=publication_id,
-                user_id=user_id,
-                event_type=PDF_EVENT_ATTEMPT_STARTED,
-                status=PDF_STATUS_RUNNING,
-            )
-        )
-        await db_session.commit()
-
-
-def _failed_outcome(
-    *,
-    row: PublicationListItem,
-) -> OaResolutionOutcome:
-    return OaResolutionOutcome(
-        publication_id=row.publication_id,
-        doi=None,
-        pdf_url=None,
-        failure_reason=FAILURE_RESOLUTION_EXCEPTION,
-        source=None,
-        used_crossref=False,
-    )
-
-
-async def _fetch_outcome_for_row(
-    *,
-    row: PublicationListItem,
-    request_email: str | None,
-    openalex_api_key: str | None = None,
-    allow_arxiv_lookup: bool = True,
-) -> tuple[OaResolutionOutcome, bool]:
-    pipeline_result = await resolve_publication_pdf_outcome_for_row(
-        row=row,
-        request_email=request_email,
-        openalex_api_key=openalex_api_key,
-        allow_arxiv_lookup=allow_arxiv_lookup,
-    )
-    outcome = pipeline_result.outcome
-    if outcome is not None:
-        return outcome, bool(pipeline_result.arxiv_rate_limited)
-    return _failed_outcome(row=row), bool(pipeline_result.arxiv_rate_limited)
-
-
-def _apply_publication_update(
-    publication: Publication,
-    *,
-    pdf_url: str | None,
-) -> None:
-    if pdf_url and publication.pdf_url != pdf_url:
-        publication.pdf_url = pdf_url
-
-
-def _apply_job_outcome(job: PublicationPdfJob, *, outcome: OaResolutionOutcome) -> None:
-    job.last_source = outcome.source
-    if outcome.pdf_url:
-        job.status = PDF_STATUS_RESOLVED
-        job.resolved_at = utcnow()
-        job.last_failure_reason = None
-        job.last_failure_detail = None
-        return
-    job.status = PDF_STATUS_FAILED
-    job.last_failure_reason = outcome.failure_reason
-    job.last_failure_detail = outcome.failure_reason
-
-
-def _result_event(outcome: OaResolutionOutcome) -> tuple[str, str]:
-    if outcome.pdf_url:
-        return PDF_EVENT_RESOLVED, PDF_STATUS_RESOLVED
-    return PDF_EVENT_FAILED, PDF_STATUS_FAILED
-
-
-async def _persist_outcome(
-    *,
-    publication_id: int,
-    user_id: int,
-    outcome: OaResolutionOutcome,
-) -> None:
-    async with background_session() as db_session:
-        publication = await db_session.get(Publication, publication_id)
-        job = await db_session.get(PublicationPdfJob, publication_id)
-        if publication is None or job is None:
-            return
-        _apply_publication_update(publication, pdf_url=outcome.pdf_url)
-        await identifier_service.sync_identifiers_for_publication_resolution(
-            db_session,
-            publication=publication,
-            source=outcome.source,
-        )
-        _apply_job_outcome(job, outcome=outcome)
-        event_type, status = _result_event(outcome)
-        db_session.add(
-            event_row(
-                publication_id=publication_id,
-                user_id=user_id,
-                event_type=event_type,
-                status=status,
-                source=outcome.source,
-                failure_reason=outcome.failure_reason,
-                message=outcome.failure_reason,
-            )
-        )
-        await db_session.commit()
-
-
-async def _resolve_publication_row(
-    *,
-    user_id: int,
-    request_email: str | None,
-    row: PublicationListItem,
-    openalex_api_key: str | None = None,
-    allow_arxiv_lookup: bool = True,
-) -> bool:
-    from app.services.openalex.client import OpenAlexBudgetExhaustedError
-
-    await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
-    try:
-        outcome, arxiv_rate_limited = await _fetch_outcome_for_row(
-            row=row,
-            request_email=request_email,
-            openalex_api_key=openalex_api_key,
-            allow_arxiv_lookup=allow_arxiv_lookup,
-        )
-    except OpenAlexBudgetExhaustedError:
-        await _persist_outcome(
-            publication_id=row.publication_id,
-            user_id=user_id,
-            outcome=_failed_outcome(row=row),
-        )
-        raise
-    except Exception as exc:  # pragma: no cover - defensive network boundary
-        structured_log(
-            logger,
-            "warning",
-            "publications.pdf_queue.resolve_failed",
-            publication_id=row.publication_id,
-            error=str(exc),
-        )
-        outcome = _failed_outcome(row=row)
-        arxiv_rate_limited = False
-    await _persist_outcome(
-        publication_id=row.publication_id,
-        user_id=user_id,
-        outcome=outcome,
-    )
-    return bool(arxiv_rate_limited)
-
-
-async def _run_resolution_task(
-    *,
-    user_id: int,
-    request_email: str | None,
-    rows: list[PublicationListItem],
-) -> None:
-    from app.services.openalex.client import OpenAlexBudgetExhaustedError
-    from app.services.settings import application as user_settings_service
-
-    openalex_api_key: str | None = None
-    try:
-        async with background_session() as key_session:
-            user_settings = await user_settings_service.get_or_create_settings(key_session, user_id=user_id)
-            openalex_api_key = getattr(user_settings, "openalex_api_key", None) or settings.openalex_api_key
-    except Exception:
-        openalex_api_key = settings.openalex_api_key
-
-    arxiv_lookup_allowed = True
-    for row in rows:
-        try:
-            arxiv_rate_limited = await _resolve_publication_row(
-                user_id=user_id,
-                request_email=request_email,
-                row=row,
-                openalex_api_key=openalex_api_key,
-                allow_arxiv_lookup=arxiv_lookup_allowed,
-            )
-            if arxiv_rate_limited and arxiv_lookup_allowed:
-                arxiv_lookup_allowed = False
-                structured_log(
-                    logger,
-                    "warning",
-                    "pdf_queue.arxiv_batch_disabled",
-                    detail="arXiv temporarily disabled for remaining batch after rate limit",
-                )
-        except OpenAlexBudgetExhaustedError:
-            _enter_budget_cooldown()
-            structured_log(
-                logger,
-                "warning",
-                "pdf_queue.budget_exhausted",
-                detail="Stopping PDF resolution batch — OpenAlex daily budget exhausted",
-                cooldown_minutes=_BUDGET_COOLDOWN_MINUTES,
-            )
-            break
-        except Exception:
-            structured_log(
-                logger,
-                "exception",
-                "pdf_queue.row_failed",
-                publication_id=row.publication_id,
-            )
-            try:
-                await _persist_outcome(
-                    publication_id=row.publication_id,
-                    user_id=user_id,
-                    outcome=_failed_outcome(row=row),
-                )
-            except Exception:
-                structured_log(
-                    logger,
-                    "exception",
-                    "pdf_queue.row_fail_persist_error",
-                    publication_id=row.publication_id,
-                )
-
-
-def _register_task(task: asyncio.Task[None]) -> None:
-    _scheduled_tasks.add(task)
-
-
-def _drop_finished_task(task: asyncio.Task[None]) -> None:
-    _scheduled_tasks.discard(task)
-    try:
-        task.result()
-    except Exception:
-        structured_log(logger, "exception", "publications.pdf_queue.task_failed")
-
-
-def schedule_rows(
-    *,
-    user_id: int,
-    request_email: str | None,
-    rows: list[PublicationListItem],
-) -> None:
-    if not rows:
-        return
-    task = asyncio.create_task(
-        _run_resolution_task(
-            user_id=user_id,
-            request_email=request_email,
-            rows=rows,
-        )
-    )
-    _register_task(task)
-    task.add_done_callback(_drop_finished_task)

app/services/publications/queries.py

@@ -1,10 +1,8 @@
 from __future__ import annotations
 
 from datetime import datetime
-from typing import Any
 
 from sqlalchemy import Select, case, func, select
-from sqlalchemy import false as sa_false
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db.models import (
@@ -16,12 +14,6 @@ from app.db.models import (
     ScholarPublication,
 )
 from app.services.publications.modes import MODE_LATEST, MODE_UNREAD
-from app.services.publications.pdf_queue_common import (
-    PDF_STATUS_FAILED,
-    PDF_STATUS_QUEUED,
-    PDF_STATUS_RESOLVED,
-    PDF_STATUS_RUNNING,
-)
 from app.services.publications.types import PublicationListItem, UnreadPublicationItem
 
 
@@ -35,10 +27,10 @@ def _normalized_citation_count(value: object) -> int:
 def _pdf_status_sort_rank():
     return case(
         (Publication.pdf_url.is_not(None), 4),
-        (PublicationPdfJob.status == PDF_STATUS_RESOLVED, 4),
+        (PublicationPdfJob.status == "resolved", 4),
-        (PublicationPdfJob.status == PDF_STATUS_RUNNING, 3),
+        (PublicationPdfJob.status == "running", 3),
-        (PublicationPdfJob.status == PDF_STATUS_QUEUED, 2),
+        (PublicationPdfJob.status == "queued", 2),
-        (PublicationPdfJob.status == PDF_STATUS_FAILED, 0),
+        (PublicationPdfJob.status == "failed", 0),
         else_=1,
     )
 
@@ -132,7 +124,7 @@ def publications_query(
         stmt = stmt.where(ScholarPublication.is_read.is_(False))
     if mode == MODE_LATEST:
         if latest_run_id is None:
-            return stmt.where(sa_false())
+            return stmt.where(False)
         stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
     if snapshot_before is not None:
         stmt = stmt.where(ScholarPublication.created_at <= snapshot_before)
@@ -203,7 +195,7 @@ async def get_publication_item_for_user(
 
 
 def publication_list_item_from_row(
-    row: Any,
+    row: tuple,
     *,
     latest_run_id: int | None,
 ) -> PublicationListItem:
@@ -240,7 +232,7 @@ def publication_list_item_from_row(
     )
 
 
-def unread_item_from_row(row: Any) -> UnreadPublicationItem:
+def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
     (
         publication_id,
         scholar_profile_id,

app/services/publications/read_state.py

@@ -1,8 +1,6 @@
 from __future__ import annotations
 
-from typing import Any
+from sqlalchemy import select, tuple_, update
-
-from sqlalchemy import CursorResult, select, tuple_, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db.models import ScholarProfile, ScholarPublication
@@ -36,7 +34,7 @@ async def mark_all_unread_as_read_for_user(
         )
         .values(is_read=True)
     )
-    result: CursorResult[Any] = await db_session.execute(stmt)  # type: ignore[assignment]
+    result = await db_session.execute(stmt)
     await db_session.commit()
     return int(result.rowcount or 0)
 
@@ -64,7 +62,7 @@ async def mark_selected_as_read_for_user(
         )
         .values(is_read=True)
     )
-    result: CursorResult[Any] = await db_session.execute(stmt)  # type: ignore[assignment]
+    result = await db_session.execute(stmt)
     await db_session.commit()
     return int(result.rowcount or 0)
 
@@ -87,6 +85,6 @@ async def set_publication_favorite_for_user(
         )
         .values(is_favorite=bool(is_favorite))
     )
-    result: CursorResult[Any] = await db_session.execute(stmt)  # type: ignore[assignment]
+    result = await db_session.execute(stmt)
     await db_session.commit()
     return int(result.rowcount or 0)

app/services/runs/events.py

@@ -4,12 +4,8 @@ import logging
 from collections.abc import AsyncGenerator
 from typing import Any
 
-from app.logging_utils import structured_log
-
 logger = logging.getLogger(__name__)
 
-_SUBSCRIBER_QUEUE_MAXSIZE = 256
-
 
 class RunEventPublisher:
     def __init__(self) -> None:
@@ -19,15 +15,9 @@ class RunEventPublisher:
     def subscribe(self, run_id: int) -> asyncio.Queue:
         if run_id not in self._subscribers:
             self._subscribers[run_id] = set()
-        queue: asyncio.Queue[Any] = asyncio.Queue(maxsize=_SUBSCRIBER_QUEUE_MAXSIZE)
+        queue: asyncio.Queue[Any] = asyncio.Queue()
         self._subscribers[run_id].add(queue)
-        structured_log(
+        logger.debug(f"New subscriber for run {run_id}. Total: {len(self._subscribers[run_id])}")
-            logger,
-            "debug",
-            "runs.event_subscriber_added",
-            run_id=run_id,
-            subscriber_count=len(self._subscribers[run_id]),
-        )
         return queue
 
     def unsubscribe(self, run_id: int, queue: asyncio.Queue) -> None:
@@ -47,16 +37,7 @@ class RunEventPublisher:
             try:
                 queue.put_nowait(message)
             except asyncio.QueueFull:
-                structured_log(
+                logger.warning(f"Subscriber queue full for run {run_id}, dropping message")
-                    logger,
-                    "warning",
-                    "runs.event_subscriber_queue_full",
-                    run_id=run_id,
-                )
-                self._subscribers[run_id].discard(queue)
-
-    async def publish_run_complete(self, run_id: int) -> None:
-        await self.publish(run_id, "run_complete", {})
 
 
 run_events = RunEventPublisher()
@@ -68,20 +49,12 @@ async def event_generator(run_id: int) -> AsyncGenerator[str, None]:
         while True:
             # Wait for a new event
             message = await queue.get()
-            event_type = message["type"]
-            if event_type == "run_complete":
-                yield f"event: {event_type}\ndata: {{}}\n\n"
-                break
             # Server-Sent Events format: "event: <type>\ndata: <json>\n\n"
+            event_type = message["type"]
             data_str = json.dumps(message["data"])
             yield f"event: {event_type}\ndata: {data_str}\n\n"
     except asyncio.CancelledError:
-        structured_log(
+        logger.debug(f"Client disconnected from SSE stream for run {run_id}")
-            logger,
-            "debug",
-            "runs.event_stream_disconnected",
-            run_id=run_id,
-        )
         raise
     finally:
         run_events.unsubscribe(run_id, queue)

app/services/runs/queue_queries.py

@@ -1,7 +1,5 @@
 from __future__ import annotations
 
-from typing import Any
-
 from sqlalchemy import and_, select
 
 from app.db.models import IngestionQueueItem, ScholarProfile
@@ -40,7 +38,7 @@ def queue_item_select(*, user_id: int):
     )
 
 
-def queue_list_item_from_row(row: Any) -> QueueListItem:
+def queue_list_item_from_row(row: tuple) -> QueueListItem:
     (
         item_id,
         scholar_profile_id,

app/services/scholar/profile_rows.py

@@ -131,10 +131,7 @@ def parse_citation_count(parts: list[str]) -> int | None:
     text = normalize_space(" ".join(parts))
     if not text:
         return 0
-    match = re.search(r"[\d,]+", text)
+    digits = re.sub(r"\D+", "", text)
-    if not match:
-        return None
-    digits = match.group(0).replace(",", "")
     if not digits:
         return None
     return int(digits)

app/services/scholar/source.py

@@ -62,7 +62,6 @@ class LiveScholarSource:
         *,
         timeout_seconds: float = 25.0,
         min_interval_seconds: float | None = None,
-        rotate_user_agents: bool | None = None,
         user_agents: list[str] | None = None,
     ) -> None:
         self._timeout_seconds = timeout_seconds
@@ -73,36 +72,6 @@ class LiveScholarSource:
         )
         self._min_interval_seconds = max(configured_interval, 0.0)
         self._user_agents = user_agents or DEFAULT_USER_AGENTS
-        self._rotate_user_agents = (
-            bool(settings.scholar_http_rotate_user_agent) if rotate_user_agents is None else bool(rotate_user_agents)
-        )
-        configured_user_agent = settings.scholar_http_user_agent.strip()
-        self._configured_user_agent = configured_user_agent or None
-        self._accept_language = settings.scholar_http_accept_language.strip() or "en-US,en;q=0.9"
-        self._cookie_header = settings.scholar_http_cookie.strip() or None
-        self._stable_user_agent = self._resolve_initial_user_agent()
-
-    def _resolve_initial_user_agent(self) -> str:
-        if self._configured_user_agent is not None:
-            return self._configured_user_agent
-        return random.choice(self._user_agents)
-
-    def _resolve_user_agent_for_request(self) -> str:
-        if self._configured_user_agent is not None:
-            return self._configured_user_agent
-        if self._rotate_user_agents:
-            return random.choice(self._user_agents)
-        return self._stable_user_agent
-
-    def _request_headers(self) -> dict[str, str]:
-        headers = {
-            "User-Agent": self._resolve_user_agent_for_request(),
-            "Accept": "text/html,application/xhtml+xml",
-            "Accept-Language": self._accept_language,
-        }
-        if self._cookie_header is not None:
-            headers["Cookie"] = self._cookie_header
-        return headers
 
     async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
         return await self.fetch_profile_page_html(
@@ -147,21 +116,15 @@ class LiveScholarSource:
         return await asyncio.to_thread(self._fetch_sync, requested_url)
 
     def _build_request(self, requested_url: str) -> Request:
-        return Request(requested_url, headers=self._request_headers())
+        return Request(
-
+            requested_url,
-    @staticmethod
+            headers={
-    def _http_error_reason(*, status_code: int, final_url: str, body: str) -> str:
+                "User-Agent": random.choice(self._user_agents),
-        lowered_url = final_url.lower()
+                "Accept": "text/html,application/xhtml+xml",
-        lowered_body = body.lower()
+                "Accept-Language": "en-US,en;q=0.9",
-        if "sorry/index" in lowered_url or "sorry/index" in lowered_body:
+                "Connection": "close",
-            return "blocked_google_sorry_challenge"
+            },
-        if "our systems have detected" in lowered_body or "unusual traffic" in lowered_body:
+        )
-            return "blocked_unusual_traffic_detected"
-        if "automated queries" in lowered_body:
-            return "blocked_automated_queries_detected"
-        if status_code == 429:
-            return "blocked_http_429_rate_limited"
-        return f"http_error_status_{status_code}"
 
     @staticmethod
     def _http_error_body(exc: HTTPError) -> str:
@@ -188,27 +151,18 @@ class LiveScholarSource:
 
     @staticmethod
     def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult:
-        final_url = exc.geturl()
-        body = LiveScholarSource._http_error_body(exc)
-        block_reason = LiveScholarSource._http_error_reason(
-            status_code=exc.code,
-            final_url=final_url,
-            body=body,
-        )
         structured_log(
             logger,
             "warning",
             "scholar_source.fetch_http_error",
             requested_url=requested_url,
             status_code=exc.code,
-            final_url=final_url,
-            block_reason=block_reason,
         )
         return FetchResult(
             requested_url=requested_url,
             status_code=exc.code,
-            final_url=final_url,
+            final_url=exc.geturl(),
-            body=body,
+            body=LiveScholarSource._http_error_body(exc),
             error=str(exc),
         )
 

app/services/scholar/state_detection.py

@@ -30,11 +30,6 @@ def classify_network_error_reason(fetch_error: str | None) -> str:
     return "network_error_missing_status_code"
 
 
-def is_hard_challenge_reason(state_reason: str) -> bool:
-    """True if the block reason indicates an active Scholar challenge (not retryable in short loops)."""
-    return state_reason.startswith("blocked_") and state_reason != "blocked_http_429_rate_limited"
-
-
 def classify_block_or_captcha_reason(
     *,
     status_code: int,
@@ -43,18 +38,18 @@ def classify_block_or_captcha_reason(
 ) -> str | None:
     if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url):
         return "blocked_accounts_redirect"
+    if status_code == 429:
+        return "blocked_http_429_rate_limited"
+    if status_code == 403:
+        if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url:
+            return "blocked_http_403_captcha_challenge"
+        return "blocked_http_403_forbidden"
     if "sorry/index" in final_url or "sorry/index" in body_lowered:
         return "blocked_google_sorry_challenge"
     if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered:
         return "blocked_unusual_traffic_detected"
     if "automated queries" in body_lowered:
         return "blocked_automated_queries_detected"
-    if status_code == 429:
-        return "blocked_http_429_rate_limited"
-    if status_code == 403:
-        if "recaptcha" in body_lowered or "captcha" in body_lowered:
-            return "blocked_http_403_captcha_challenge"
-        return "blocked_http_403_forbidden"
     if "not a robot" in body_lowered:
         return "blocked_not_a_robot_challenge"
     if "recaptcha" in body_lowered:

app/services/scholars/application.py

@@ -1,22 +1,52 @@
 from __future__ import annotations
 
+import asyncio
+import logging
 import os
+import random
+from dataclasses import replace
+from datetime import UTC, datetime, timedelta
 from typing import Any
 from uuid import uuid4
 
+from sqlalchemy import delete, func, select, text
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.db.models import ScholarProfile
+from app.db.models import AuthorSearchCacheEntry, AuthorSearchRuntimeState, ScholarProfile
-from app.services.scholar.parser import ScholarParserError, parse_profile_page
+from app.logging_utils import structured_log
+from app.services.scholar.parser import (
+    ParsedAuthorSearchPage,
+    ParseState,
+    ScholarParserError,
+    parse_author_search_page,
+    parse_profile_page,
+)
 from app.services.scholar.source import ScholarSource
-from app.services.scholars.author_search import search_author_candidates
 from app.services.scholars.constants import (
     ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES,
+    AUTHOR_SEARCH_LOCK_KEY,
+    AUTHOR_SEARCH_LOCK_NAMESPACE,
+    AUTHOR_SEARCH_RUNTIME_STATE_KEY,
+    DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS,
+    DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES,
+    DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD,
+    DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD,
+    DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS,
+    DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS,
+    DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS,
+    DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD,
+    MAX_AUTHOR_SEARCH_LIMIT,
+    SEARCH_CACHED_BLOCK_REASON,
     SEARCH_COOLDOWN_REASON,
     SEARCH_DISABLED_REASON,
 )
 from app.services.scholars.exceptions import ScholarServiceError
+from app.services.scholars.search_hints import (
+    _merge_warnings,
+    _policy_blocked_author_search_result,
+    _trim_author_search_result,
+)
 from app.services.scholars.uploads import (
     _ensure_upload_root,
     _resolve_upload_path,
@@ -28,80 +58,280 @@ from app.services.scholars.validators import (
     validate_scholar_id,
 )
 
+logger = logging.getLogger(__name__)
 
-async def bulk_delete_scholars(
+
+async def _acquire_author_search_lock(db_session: AsyncSession) -> None:
+    await db_session.execute(
+        text("SELECT pg_advisory_xact_lock(:namespace, :lock_key)"),
+        {
+            "namespace": AUTHOR_SEARCH_LOCK_NAMESPACE,
+            "lock_key": AUTHOR_SEARCH_LOCK_KEY,
+        },
+    )
+
+
+async def _load_runtime_state_for_update(
     db_session: AsyncSession,
-    *,
+) -> AuthorSearchRuntimeState:
-    user_id: int,
-    scholar_profile_ids: list[int],
-    upload_dir: str | None = None,
-) -> int:
-    from sqlalchemy import select
-
     result = await db_session.execute(
-        select(ScholarProfile).where(
+        select(AuthorSearchRuntimeState)
-            ScholarProfile.id.in_(scholar_profile_ids),
+        .where(AuthorSearchRuntimeState.state_key == AUTHOR_SEARCH_RUNTIME_STATE_KEY)
-            ScholarProfile.user_id == user_id,
+        .with_for_update()
-        )
     )
-    profiles = list(result.scalars().all())
+    state = result.scalar_one_or_none()
-    if not profiles:
+    if state is not None:
-        return 0
+        return state
-    if upload_dir:
+
-        upload_root = _ensure_upload_root(upload_dir, create=True)
+    state = AuthorSearchRuntimeState(state_key=AUTHOR_SEARCH_RUNTIME_STATE_KEY)
-        for profile in profiles:
+    db_session.add(state)
-            _safe_remove_upload(upload_root, profile.profile_image_upload_path)
+    await db_session.flush()
-    for profile in profiles:
+    return state
-        await db_session.delete(profile)
+
+
+def _serialize_parsed_author_search_page(parsed: ParsedAuthorSearchPage) -> dict:
+    return {
+        "state": parsed.state.value,
+        "state_reason": parsed.state_reason,
+        "marker_counts": {str(key): int(value) for key, value in parsed.marker_counts.items()},
+        "warnings": [str(value) for value in parsed.warnings],
+        "candidates": [
+            {
+                "scholar_id": candidate.scholar_id,
+                "display_name": candidate.display_name,
+                "affiliation": candidate.affiliation,
+                "email_domain": candidate.email_domain,
+                "cited_by_count": candidate.cited_by_count,
+                "interests": [str(interest) for interest in candidate.interests],
+                "profile_url": candidate.profile_url,
+                "profile_image_url": candidate.profile_image_url,
+            }
+            for candidate in parsed.candidates
+        ],
+    }
+
+
+def _payload_state(payload: dict[str, object]) -> ParseState | None:
+    state_raw = str(payload.get("state", "")).strip()
     try:
-        await db_session.commit()
+        return ParseState(state_raw)
-    except IntegrityError as exc:
+    except ValueError:
-        await db_session.rollback()
+        return None
-        raise ScholarServiceError("Unable to bulk-delete scholars due to a database constraint.") from exc
-    return len(profiles)
 
 
-async def bulk_toggle_scholars(
+def _payload_marker_counts(payload: dict[str, object]) -> dict[str, int]:
+    marker_counts_payload = payload.get("marker_counts")
+    if not isinstance(marker_counts_payload, dict):
+        return {}
+    parsed: dict[str, int] = {}
+    for key, value in marker_counts_payload.items():
+        try:
+            parsed[str(key)] = int(value)
+        except (TypeError, ValueError):
+            continue
+    return parsed
+
+
+def _payload_warnings(payload: dict[str, object]) -> list[str]:
+    warnings_payload = payload.get("warnings")
+    if not isinstance(warnings_payload, list):
+        return []
+    return [str(value) for value in warnings_payload if isinstance(value, str)]
+
+
+def _parse_optional_string(value: object) -> str | None:
+    if value is None:
+        return None
+    normalized = str(value).strip()
+    return normalized or None
+
+
+def _parse_optional_int(value: object) -> int | None:
+    if isinstance(value, int):
+        return value
+    if isinstance(value, str) and value.strip():
+        try:
+            return int(value)
+        except ValueError:
+            return None
+    return None
+
+
+def _normalize_interests(value: object) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(item) for item in value if isinstance(item, str)]
+
+
+def _deserialize_candidate_payload(value: object) -> dict[str, object] | None:
+    if not isinstance(value, dict):
+        return None
+    scholar_id = str(value.get("scholar_id", "")).strip()
+    display_name = str(value.get("display_name", "")).strip()
+    profile_url = str(value.get("profile_url", "")).strip()
+    if not scholar_id or not display_name or not profile_url:
+        return None
+    return {
+        "scholar_id": scholar_id,
+        "display_name": display_name,
+        "affiliation": _parse_optional_string(value.get("affiliation")),
+        "email_domain": _parse_optional_string(value.get("email_domain")),
+        "cited_by_count": _parse_optional_int(value.get("cited_by_count")),
+        "interests": _normalize_interests(value.get("interests")),
+        "profile_url": profile_url,
+        "profile_image_url": _parse_optional_string(value.get("profile_image_url")),
+    }
+
+
+def _deserialize_candidates(payload: dict[str, object]) -> list[dict[str, Any]]:
+    candidates_payload = payload.get("candidates")
+    if not isinstance(candidates_payload, list):
+        return []
+    normalized: list[dict[str, Any]] = []
+    for value in candidates_payload:
+        candidate = _deserialize_candidate_payload(value)
+        if candidate is not None:
+            normalized.append(candidate)
+    return normalized
+
+
+def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearchPage | None:
+    if not isinstance(payload, dict):
+        return None
+
+    state = _payload_state(payload)
+    if state is None:
+        return None
+
+    marker_counts = _payload_marker_counts(payload)
+    warnings = _payload_warnings(payload)
+    from app.services.scholar.parser import ScholarSearchCandidate
+
+    normalized_candidates = _deserialize_candidates(payload)
+
+    return ParsedAuthorSearchPage(
+        state=state,
+        state_reason=str(payload.get("state_reason", "")).strip() or "unknown",
+        candidates=[
+            ScholarSearchCandidate(
+                scholar_id=item["scholar_id"],
+                display_name=item["display_name"],
+                affiliation=item["affiliation"],
+                email_domain=item["email_domain"],
+                cited_by_count=item["cited_by_count"],
+                interests=item["interests"],
+                profile_url=item["profile_url"],
+                profile_image_url=item["profile_image_url"],
+            )
+            for item in normalized_candidates
+        ],
+        marker_counts=marker_counts,
+        warnings=warnings,
+    )
+
+
+async def _cache_get_author_search_result(
     db_session: AsyncSession,
     *,
-    user_id: int,
+    query_key: str,
-    scholar_profile_ids: list[int],
+    now_utc: datetime,
-    is_enabled: bool,
+) -> ParsedAuthorSearchPage | None:
-) -> int:
+    result = await db_session.execute(
-    from sqlalchemy import CursorResult, update
+        select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key)
-
-    cursor: CursorResult[Any] = await db_session.execute(  # type: ignore[assignment]
-        update(ScholarProfile)
-        .where(
-            ScholarProfile.id.in_(scholar_profile_ids),
-            ScholarProfile.user_id == user_id,
-        )
-        .values(is_enabled=is_enabled)
     )
-    await db_session.commit()
+    entry = result.scalar_one_or_none()
-    return int(cursor.rowcount or 0)
+    if entry is None:
+        return None
+    expires_at = entry.expires_at
+    if expires_at.tzinfo is None:
+        expires_at = expires_at.replace(tzinfo=UTC)
+    if expires_at <= now_utc:
+        await db_session.delete(entry)
+        return None
+    parsed = _deserialize_parsed_author_search_page(entry.payload)
+    if parsed is None:
+        await db_session.delete(entry)
+        return None
+    return parsed
 
 
-__all__ = [
+async def _cache_set_author_search_result(
-    "SEARCH_COOLDOWN_REASON",
+    db_session: AsyncSession,
-    "SEARCH_DISABLED_REASON",
+    *,
-    "ScholarServiceError",
+    query_key: str,
-    "bulk_delete_scholars",
+    parsed: ParsedAuthorSearchPage,
-    "bulk_toggle_scholars",
+    ttl_seconds: float,
-    "clear_profile_image_customization",
+    max_entries: int,
-    "create_scholar_for_user",
+    now_utc: datetime,
-    "delete_scholar",
+) -> None:
-    "get_user_scholar_by_id",
+    ttl = max(float(ttl_seconds), 0.0)
-    "hydrate_profile_metadata",
+    existing_result = await db_session.execute(
-    "list_scholars_for_user",
+        select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key)
-    "normalize_display_name",
+    )
-    "normalize_profile_image_url",
+    existing = existing_result.scalar_one_or_none()
-    "search_author_candidates",
+
-    "set_profile_image_override_url",
+    if ttl <= 0.0:
-    "set_profile_image_upload",
+        if existing is not None:
-    "toggle_scholar_enabled",
+            await db_session.delete(existing)
-    "validate_scholar_id",
+        return
-]
+
+    expires_at = now_utc + timedelta(seconds=ttl)
+    payload = _serialize_parsed_author_search_page(parsed)
+    if existing is None:
+        db_session.add(
+            AuthorSearchCacheEntry(
+                query_key=query_key,
+                payload=payload,
+                expires_at=expires_at,
+                cached_at=now_utc,
+                updated_at=now_utc,
+            )
+        )
+    else:
+        existing.payload = payload
+        existing.expires_at = expires_at
+        existing.cached_at = now_utc
+        existing.updated_at = now_utc
+
+    await _prune_author_search_cache(db_session, now_utc=now_utc, max_entries=max_entries)
+
+
+async def _prune_author_search_cache(
+    db_session: AsyncSession,
+    *,
+    now_utc: datetime,
+    max_entries: int,
+) -> None:
+    await db_session.execute(delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.expires_at <= now_utc))
+    bounded_max_entries = max(1, int(max_entries))
+    count_result = await db_session.execute(select(func.count()).select_from(AuthorSearchCacheEntry))
+    entry_count = int(count_result.scalar_one() or 0)
+    overflow = max(0, entry_count - bounded_max_entries)
+    if overflow <= 0:
+        return
+    stale_keys_result = await db_session.execute(
+        select(AuthorSearchCacheEntry.query_key).order_by(AuthorSearchCacheEntry.cached_at.asc()).limit(overflow)
+    )
+    stale_keys = [str(row[0]) for row in stale_keys_result.all()]
+    if stale_keys:
+        await db_session.execute(delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key.in_(stale_keys)))
+
+
+def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool:
+    return parsed.state == ParseState.BLOCKED_OR_CAPTCHA
+
+
+def _author_search_cooldown_remaining_seconds(
+    runtime_state: AuthorSearchRuntimeState,
+    now_utc: datetime,
+) -> int:
+    cooldown_until = runtime_state.cooldown_until
+    if cooldown_until is None:
+        return 0
+    if cooldown_until.tzinfo is None:
+        cooldown_until = cooldown_until.replace(tzinfo=UTC)
+    remaining_seconds = int((cooldown_until - now_utc).total_seconds())
+    return max(0, remaining_seconds)
 
 
 async def list_scholars_for_user(
@@ -109,8 +339,6 @@ async def list_scholars_for_user(
     *,
     user_id: int,
 ) -> list[ScholarProfile]:
-    from sqlalchemy import select
-
     result = await db_session.execute(
         select(ScholarProfile)
         .where(ScholarProfile.user_id == user_id)
@@ -149,8 +377,6 @@ async def get_user_scholar_by_id(
     user_id: int,
     scholar_profile_id: int,
 ) -> ScholarProfile | None:
-    from sqlalchemy import select
-
     result = await db_session.execute(
         select(ScholarProfile).where(
             ScholarProfile.id == scholar_profile_id,
@@ -182,11 +408,493 @@ async def delete_scholar(
         _safe_remove_upload(upload_root, profile.profile_image_upload_path)
 
     await db_session.delete(profile)
-    try:
+    await db_session.commit()
+
+
+def _normalize_author_search_inputs(query: str, limit: int) -> tuple[str, int, str]:
+    normalized_query = query.strip()
+    if len(normalized_query) < 2:
+        raise ScholarServiceError("Search query must be at least 2 characters.")
+    bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))
+    return normalized_query, bounded_limit, normalized_query.casefold()
+
+
+def _disabled_search_result(*, normalized_query: str, bounded_limit: int) -> ParsedAuthorSearchPage:
+    structured_log(
+        logger,
+        "warning",
+        "scholar_search.disabled_by_configuration",
+        query=normalized_query,
+    )
+    return _policy_blocked_author_search_result(
+        reason=SEARCH_DISABLED_REASON,
+        warning_codes=["author_search_disabled_by_configuration"],
+        limit=bounded_limit,
+    )
+
+
+def _normalize_runtime_cooldown_state(
+    runtime_state: AuthorSearchRuntimeState,
+    *,
+    now_utc: datetime,
+) -> bool:
+    if runtime_state.cooldown_until is None:
+        return False
+    cooldown_until = runtime_state.cooldown_until
+    updated = False
+    if cooldown_until.tzinfo is None:
+        cooldown_until = cooldown_until.replace(tzinfo=UTC)
+        runtime_state.cooldown_until = cooldown_until
+        updated = True
+    if now_utc < cooldown_until:
+        return updated
+    structured_log(
+        logger,
+        "info",
+        "scholar_search.cooldown_expired",
+        cooldown_until_utc=cooldown_until.isoformat(),
+    )
+    runtime_state.cooldown_until = None
+    runtime_state.cooldown_rejection_count = 0
+    runtime_state.cooldown_alert_emitted = False
+    return True
+
+
+def _cooldown_warning_codes(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    cooldown_remaining_seconds: int,
+) -> list[str]:
+    warning_codes = [
+        "author_search_cooldown_active",
+        f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
+    ]
+    if bool(runtime_state.cooldown_alert_emitted):
+        warning_codes.append("author_search_cooldown_alert_threshold_exceeded")
+    return warning_codes
+
+
+def _emit_cooldown_threshold_alert(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    normalized_query: str,
+    cooldown_rejection_alert_threshold: int,
+) -> bool:
+    runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1
+    threshold = max(1, int(cooldown_rejection_alert_threshold))
+    if int(runtime_state.cooldown_rejection_count) < threshold:
+        return True
+    if bool(runtime_state.cooldown_alert_emitted):
+        return True
+    structured_log(
+        logger,
+        "error",
+        "scholar_search.cooldown_rejection_threshold_exceeded",
+        query=normalized_query,
+        cooldown_rejection_count=int(runtime_state.cooldown_rejection_count),
+        threshold=threshold,
+        cooldown_until_utc=runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
+    )
+    runtime_state.cooldown_alert_emitted = True
+    return True
+
+
+def _cooldown_block_result(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    normalized_query: str,
+    bounded_limit: int,
+    cooldown_rejection_alert_threshold: int,
+    cooldown_remaining_seconds: int,
+) -> ParsedAuthorSearchPage:
+    _emit_cooldown_threshold_alert(
+        runtime_state=runtime_state,
+        normalized_query=normalized_query,
+        cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
+    )
+    structured_log(
+        logger,
+        "warning",
+        "scholar_search.cooldown_active",
+        query=normalized_query,
+        cooldown_remaining_seconds=cooldown_remaining_seconds,
+        cooldown_until_utc=runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
+    )
+    return _policy_blocked_author_search_result(
+        reason=SEARCH_COOLDOWN_REASON,
+        warning_codes=_cooldown_warning_codes(
+            runtime_state=runtime_state,
+            cooldown_remaining_seconds=cooldown_remaining_seconds,
+        ),
+        limit=bounded_limit,
+    )
+
+
+async def _cache_hit_result(
+    db_session: AsyncSession,
+    *,
+    query_key: str,
+    now_utc: datetime,
+    normalized_query: str,
+    bounded_limit: int,
+) -> ParsedAuthorSearchPage | None:
+    cached = await _cache_get_author_search_result(
+        db_session,
+        query_key=query_key,
+        now_utc=now_utc,
+    )
+    if cached is None:
+        return None
+    structured_log(
+        logger,
+        "info",
+        "scholar_search.cache_hit",
+        query=normalized_query,
+        state=cached.state.value,
+        state_reason=cached.state_reason,
+    )
+    state_reason_override = SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None
+    return _trim_author_search_result(
+        cached,
+        limit=bounded_limit,
+        extra_warnings=["author_search_served_from_cache"],
+        state_reason_override=state_reason_override,
+    )
+
+
+def _throttle_sleep_seconds(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    now_utc: datetime,
+    min_interval_seconds: float,
+    interval_jitter_seconds: float,
+) -> tuple[float, bool]:
+    updated = False
+    if runtime_state.last_live_request_at is None:
+        enforced_wait_seconds = 0.0
+    else:
+        last_live_request_at = runtime_state.last_live_request_at
+        if last_live_request_at.tzinfo is None:
+            last_live_request_at = last_live_request_at.replace(tzinfo=UTC)
+            runtime_state.last_live_request_at = last_live_request_at
+            updated = True
+        enforced_wait_seconds = (
+            last_live_request_at + timedelta(seconds=max(float(min_interval_seconds), 0.0)) - now_utc
+        ).total_seconds()
+    jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0))
+    return max(0.0, float(enforced_wait_seconds)) + jitter_seconds, updated
+
+
+async def _wait_for_author_search_throttle(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    normalized_query: str,
+    now_utc: datetime,
+    min_interval_seconds: float,
+    interval_jitter_seconds: float,
+) -> bool:
+    sleep_seconds, updated = _throttle_sleep_seconds(
+        runtime_state=runtime_state,
+        now_utc=now_utc,
+        min_interval_seconds=min_interval_seconds,
+        interval_jitter_seconds=interval_jitter_seconds,
+    )
+    if sleep_seconds <= 0.0:
+        return updated
+    structured_log(
+        logger,
+        "info",
+        "scholar_search.throttle_wait",
+        query=normalized_query,
+        sleep_seconds=round(sleep_seconds, 3),
+    )
+    await asyncio.sleep(sleep_seconds)
+    return True
+
+
+async def _fetch_author_search_with_retries(
+    *,
+    source: ScholarSource,
+    normalized_query: str,
+    network_error_retries: int,
+    retry_backoff_seconds: float,
+) -> tuple[ParsedAuthorSearchPage, int, list[str]]:
+    max_attempts = max(1, int(network_error_retries) + 1)
+    parsed: ParsedAuthorSearchPage | None = None
+    retry_warnings: list[str] = []
+    retry_scheduled_count = 0
+    for attempt_index in range(max_attempts):
+        fetch_result = await source.fetch_author_search_html(normalized_query, start=0)
+        try:
+            parsed = parse_author_search_page(fetch_result)
+        except ScholarParserError as exc:
+            parsed = ParsedAuthorSearchPage(
+                state=ParseState.LAYOUT_CHANGED,
+                state_reason=exc.code,
+                candidates=[],
+                marker_counts={},
+                warnings=[exc.code],
+            )
+        if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1:
+            break
+        retry_warnings.append("network_retry_scheduled_for_author_search")
+        retry_scheduled_count += 1
+        retry_sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index)
+        if retry_sleep_seconds > 0:
+            await asyncio.sleep(retry_sleep_seconds)
+    if parsed is None:
+        raise ScholarServiceError("Unable to complete scholar author search.")
+    return parsed, retry_scheduled_count, retry_warnings
+
+
+def _with_retry_warnings(
+    parsed: ParsedAuthorSearchPage,
+    *,
+    retry_warnings: list[str],
+    retry_scheduled_count: int,
+    retry_alert_threshold: int,
+    normalized_query: str,
+) -> ParsedAuthorSearchPage:
+    merged = replace(parsed, warnings=_merge_warnings(parsed.warnings, retry_warnings))
+    threshold = max(1, int(retry_alert_threshold))
+    if retry_scheduled_count < threshold:
+        return merged
+    structured_log(
+        logger,
+        "warning",
+        "scholar_search.retry_threshold_exceeded",
+        query=normalized_query,
+        retry_scheduled_count=retry_scheduled_count,
+        threshold=threshold,
+        final_state=merged.state.value,
+        final_state_reason=merged.state_reason,
+    )
+    return replace(
+        merged,
+        warnings=_merge_warnings(
+            merged.warnings,
+            [f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"],
+        ),
+    )
+
+
+def _apply_block_circuit_breaker(
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    merged_parsed: ParsedAuthorSearchPage,
+    cooldown_block_threshold: int,
+    cooldown_seconds: int,
+    normalized_query: str,
+) -> ParsedAuthorSearchPage:
+    if not _is_author_search_block_state(merged_parsed):
+        runtime_state.consecutive_blocked_count = 0
+        return merged_parsed
+    runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1
+    structured_log(
+        logger,
+        "warning",
+        "scholar_search.block_detected",
+        query=normalized_query,
+        state_reason=merged_parsed.state_reason,
+        consecutive_blocked_count=int(runtime_state.consecutive_blocked_count),
+    )
+    if int(runtime_state.consecutive_blocked_count) < max(1, int(cooldown_block_threshold)):
+        return merged_parsed
+    runtime_state.cooldown_until = datetime.now(UTC) + timedelta(seconds=max(60, int(cooldown_seconds)))
+    runtime_state.consecutive_blocked_count = 0
+    runtime_state.cooldown_rejection_count = 0
+    runtime_state.cooldown_alert_emitted = False
+    structured_log(
+        logger,
+        "error",
+        "scholar_search.cooldown_activated",
+        query=normalized_query,
+        cooldown_until_utc=runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
+    )
+    return replace(
+        merged_parsed,
+        warnings=_merge_warnings(merged_parsed.warnings, ["author_search_circuit_breaker_armed"]),
+    )
+
+
+def _resolve_author_search_cache_ttl_seconds(
+    *,
+    merged_parsed: ParsedAuthorSearchPage,
+    blocked_cache_ttl_seconds: int,
+    cache_ttl_seconds: int,
+) -> int:
+    if _is_author_search_block_state(merged_parsed):
+        return min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds)))
+    return max(1, int(cache_ttl_seconds))
+
+
+async def _load_locked_runtime_state(
+    db_session: AsyncSession,
+) -> AuthorSearchRuntimeState:
+    await _acquire_author_search_lock(db_session)
+    return await _load_runtime_state_for_update(db_session)
+
+
+async def _cooldown_or_cache_result(
+    db_session: AsyncSession,
+    *,
+    runtime_state: AuthorSearchRuntimeState,
+    query_key: str,
+    normalized_query: str,
+    bounded_limit: int,
+    cooldown_rejection_alert_threshold: int,
+) -> tuple[ParsedAuthorSearchPage | None, bool]:
+    runtime_state_updated = _normalize_runtime_cooldown_state(
+        runtime_state,
+        now_utc=datetime.now(UTC),
+    )
+    cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(
+        runtime_state,
+        datetime.now(UTC),
+    )
+    if cooldown_remaining_seconds > 0:
+        return (
+            _cooldown_block_result(
+                runtime_state=runtime_state,
+                normalized_query=normalized_query,
+                bounded_limit=bounded_limit,
+                cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
+                cooldown_remaining_seconds=cooldown_remaining_seconds,
+            ),
+            True,
+        )
+    cached_result = await _cache_hit_result(
+        db_session,
+        query_key=query_key,
+        now_utc=datetime.now(UTC),
+        normalized_query=normalized_query,
+        bounded_limit=bounded_limit,
+    )
+    return cached_result, runtime_state_updated
+
+
+async def _perform_live_author_search(
+    db_session: AsyncSession,
+    *,
+    source: ScholarSource,
+    runtime_state: AuthorSearchRuntimeState,
+    normalized_query: str,
+    query_key: str,
+    network_error_retries: int,
+    retry_backoff_seconds: float,
+    min_interval_seconds: float,
+    interval_jitter_seconds: float,
+    retry_alert_threshold: int,
+    cooldown_block_threshold: int,
+    cooldown_seconds: int,
+    blocked_cache_ttl_seconds: int,
+    cache_ttl_seconds: int,
+    cache_max_entries: int,
+) -> tuple[ParsedAuthorSearchPage, bool]:
+    await _wait_for_author_search_throttle(
+        runtime_state=runtime_state,
+        normalized_query=normalized_query,
+        now_utc=datetime.now(UTC),
+        min_interval_seconds=min_interval_seconds,
+        interval_jitter_seconds=interval_jitter_seconds,
+    )
+    parsed, retry_count, retry_warnings = await _fetch_author_search_with_retries(
+        source=source,
+        normalized_query=normalized_query,
+        network_error_retries=network_error_retries,
+        retry_backoff_seconds=retry_backoff_seconds,
+    )
+    runtime_state.last_live_request_at = datetime.now(UTC)
+    merged = _with_retry_warnings(
+        parsed,
+        retry_warnings=retry_warnings,
+        retry_scheduled_count=retry_count,
+        retry_alert_threshold=retry_alert_threshold,
+        normalized_query=normalized_query,
+    )
+    merged = _apply_block_circuit_breaker(
+        runtime_state=runtime_state,
+        merged_parsed=merged,
+        cooldown_block_threshold=cooldown_block_threshold,
+        cooldown_seconds=cooldown_seconds,
+        normalized_query=normalized_query,
+    )
+    ttl_seconds = _resolve_author_search_cache_ttl_seconds(
+        merged_parsed=merged,
+        blocked_cache_ttl_seconds=blocked_cache_ttl_seconds,
+        cache_ttl_seconds=cache_ttl_seconds,
+    )
+    await _cache_set_author_search_result(
+        db_session,
+        query_key=query_key,
+        parsed=merged,
+        ttl_seconds=float(ttl_seconds),
+        max_entries=cache_max_entries,
+        now_utc=datetime.now(UTC),
+    )
+    return merged, True
+
+
+async def search_author_candidates(
+    *,
+    source: ScholarSource,
+    db_session: AsyncSession,
+    query: str,
+    limit: int,
+    network_error_retries: int = 1,
+    retry_backoff_seconds: float = 1.0,
+    search_enabled: bool = True,
+    cache_ttl_seconds: int = 21_600,
+    blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS,
+    cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES,
+    min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS,
+    interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS,
+    cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD,
+    cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS,
+    retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD,
+    cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD,
+) -> ParsedAuthorSearchPage:
+    normalized_query, bounded_limit, query_key = _normalize_author_search_inputs(query, limit)
+    if not search_enabled:
+        return _disabled_search_result(
+            normalized_query=normalized_query,
+            bounded_limit=bounded_limit,
+        )
+
+    runtime_state = await _load_locked_runtime_state(db_session)
+    early_result, runtime_state_updated = await _cooldown_or_cache_result(
+        db_session,
+        runtime_state=runtime_state,
+        query_key=query_key,
+        normalized_query=normalized_query,
+        bounded_limit=bounded_limit,
+        cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
+    )
+    if early_result is not None:
         await db_session.commit()
-    except IntegrityError as exc:
+        return early_result
-        await db_session.rollback()
+
-        raise ScholarServiceError("Unable to delete scholar due to a database constraint.") from exc
+    merged_parsed, live_runtime_updated = await _perform_live_author_search(
+        db_session,
+        source=source,
+        runtime_state=runtime_state,
+        normalized_query=normalized_query,
+        query_key=query_key,
+        network_error_retries=network_error_retries,
+        retry_backoff_seconds=retry_backoff_seconds,
+        min_interval_seconds=min_interval_seconds,
+        interval_jitter_seconds=interval_jitter_seconds,
+        retry_alert_threshold=retry_alert_threshold,
+        cooldown_block_threshold=cooldown_block_threshold,
+        cooldown_seconds=cooldown_seconds,
+        blocked_cache_ttl_seconds=blocked_cache_ttl_seconds,
+        cache_ttl_seconds=cache_ttl_seconds,
+        cache_max_entries=cache_max_entries,
+    )
+    runtime_state_updated = runtime_state_updated or live_runtime_updated
+    if runtime_state_updated:
+        await db_session.commit()
+    return _trim_author_search_result(merged_parsed, limit=bounded_limit)
 
 
 async def hydrate_profile_metadata(

app/services/scholars/author_search.py

@@ -1,580 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-import random
-from dataclasses import replace
-from datetime import UTC, datetime, timedelta
-
-from sqlalchemy import select, text
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import AuthorSearchRuntimeState
-from app.logging_utils import structured_log
-from app.services.scholar.parser import (
-    ParsedAuthorSearchPage,
-    ParseState,
-    ScholarParserError,
-    parse_author_search_page,
-)
-from app.services.scholar.source import ScholarSource
-from app.services.scholars.author_search_cache import (
-    cache_get_author_search_result,
-    cache_set_author_search_result,
-)
-from app.services.scholars.constants import (
-    AUTHOR_SEARCH_LOCK_KEY,
-    AUTHOR_SEARCH_LOCK_NAMESPACE,
-    AUTHOR_SEARCH_RUNTIME_STATE_KEY,
-    DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS,
-    DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES,
-    DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD,
-    DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD,
-    DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS,
-    DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS,
-    DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS,
-    DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD,
-    MAX_AUTHOR_SEARCH_LIMIT,
-    SEARCH_CACHED_BLOCK_REASON,
-    SEARCH_COOLDOWN_REASON,
-    SEARCH_DISABLED_REASON,
-)
-from app.services.scholars.exceptions import ScholarServiceError
-from app.services.scholars.search_hints import (
-    _merge_warnings,
-    _policy_blocked_author_search_result,
-    _trim_author_search_result,
-)
-
-logger = logging.getLogger(__name__)
-
-
-async def _acquire_author_search_lock(db_session: AsyncSession) -> None:
-    await db_session.execute(
-        text("SELECT pg_advisory_xact_lock(:namespace, :lock_key)"),
-        {
-            "namespace": AUTHOR_SEARCH_LOCK_NAMESPACE,
-            "lock_key": AUTHOR_SEARCH_LOCK_KEY,
-        },
-    )
-
-
-async def _load_runtime_state_for_update(
-    db_session: AsyncSession,
-) -> AuthorSearchRuntimeState:
-    result = await db_session.execute(
-        select(AuthorSearchRuntimeState)
-        .where(AuthorSearchRuntimeState.state_key == AUTHOR_SEARCH_RUNTIME_STATE_KEY)
-        .with_for_update()
-    )
-    state = result.scalar_one_or_none()
-    if state is not None:
-        return state
-
-    state = AuthorSearchRuntimeState(state_key=AUTHOR_SEARCH_RUNTIME_STATE_KEY)
-    db_session.add(state)
-    await db_session.flush()
-    return state
-
-
-def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool:
-    return parsed.state == ParseState.BLOCKED_OR_CAPTCHA
-
-
-def _author_search_cooldown_remaining_seconds(
-    runtime_state: AuthorSearchRuntimeState,
-    now_utc: datetime,
-) -> int:
-    cooldown_until = runtime_state.cooldown_until
-    if cooldown_until is None:
-        return 0
-    if cooldown_until.tzinfo is None:
-        cooldown_until = cooldown_until.replace(tzinfo=UTC)
-    remaining_seconds = int((cooldown_until - now_utc).total_seconds())
-    return max(0, remaining_seconds)
-
-
-def _normalize_author_search_inputs(query: str, limit: int) -> tuple[str, int, str]:
-    normalized_query = query.strip()
-    if len(normalized_query) < 2:
-        raise ScholarServiceError("Search query must be at least 2 characters.")
-    bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))
-    return normalized_query, bounded_limit, normalized_query.casefold()
-
-
-def _disabled_search_result(*, normalized_query: str, bounded_limit: int) -> ParsedAuthorSearchPage:
-    structured_log(
-        logger,
-        "warning",
-        "scholar_search.disabled_by_configuration",
-        query=normalized_query,
-    )
-    return _policy_blocked_author_search_result(
-        reason=SEARCH_DISABLED_REASON,
-        warning_codes=["author_search_disabled_by_configuration"],
-        limit=bounded_limit,
-    )
-
-
-def _normalize_runtime_cooldown_state(
-    runtime_state: AuthorSearchRuntimeState,
-    *,
-    now_utc: datetime,
-) -> bool:
-    if runtime_state.cooldown_until is None:
-        return False
-    cooldown_until = runtime_state.cooldown_until
-    updated = False
-    if cooldown_until.tzinfo is None:
-        cooldown_until = cooldown_until.replace(tzinfo=UTC)
-        runtime_state.cooldown_until = cooldown_until
-        updated = True
-    if now_utc < cooldown_until:
-        return updated
-    structured_log(
-        logger,
-        "info",
-        "scholar_search.cooldown_expired",
-        cooldown_until_utc=cooldown_until.isoformat(),
-    )
-    runtime_state.cooldown_until = None
-    runtime_state.cooldown_rejection_count = 0
-    runtime_state.cooldown_alert_emitted = False
-    return True
-
-
-def _cooldown_warning_codes(
-    *,
-    runtime_state: AuthorSearchRuntimeState,
-    cooldown_remaining_seconds: int,
-) -> list[str]:
-    warning_codes = [
-        "author_search_cooldown_active",
-        f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
-    ]
-    if bool(runtime_state.cooldown_alert_emitted):
-        warning_codes.append("author_search_cooldown_alert_threshold_exceeded")
-    return warning_codes
-
-
-def _emit_cooldown_threshold_alert(
-    *,
-    runtime_state: AuthorSearchRuntimeState,
-    normalized_query: str,
-    cooldown_rejection_alert_threshold: int,
-) -> bool:
-    runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1
-    threshold = max(1, int(cooldown_rejection_alert_threshold))
-    if int(runtime_state.cooldown_rejection_count) < threshold:
-        return True
-    if bool(runtime_state.cooldown_alert_emitted):
-        return True
-    structured_log(
-        logger,
-        "error",
-        "scholar_search.cooldown_rejection_threshold_exceeded",
-        query=normalized_query,
-        cooldown_rejection_count=int(runtime_state.cooldown_rejection_count),
-        threshold=threshold,
-        cooldown_until_utc=runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
-    )
-    runtime_state.cooldown_alert_emitted = True
-    return True
-
-
-def _cooldown_block_result(
-    *,
-    runtime_state: AuthorSearchRuntimeState,
-    normalized_query: str,
-    bounded_limit: int,
-    cooldown_rejection_alert_threshold: int,
-    cooldown_remaining_seconds: int,
-) -> ParsedAuthorSearchPage:
-    _emit_cooldown_threshold_alert(
-        runtime_state=runtime_state,
-        normalized_query=normalized_query,
-        cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
-    )
-    structured_log(
-        logger,
-        "warning",
-        "scholar_search.cooldown_active",
-        query=normalized_query,
-        cooldown_remaining_seconds=cooldown_remaining_seconds,
-        cooldown_until_utc=runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
-    )
-    return _policy_blocked_author_search_result(
-        reason=SEARCH_COOLDOWN_REASON,
-        warning_codes=_cooldown_warning_codes(
-            runtime_state=runtime_state,
-            cooldown_remaining_seconds=cooldown_remaining_seconds,
-        ),
-        limit=bounded_limit,
-    )
-
-
-async def _cache_hit_result(
-    db_session: AsyncSession,
-    *,
-    query_key: str,
-    now_utc: datetime,
-    normalized_query: str,
-    bounded_limit: int,
-) -> ParsedAuthorSearchPage | None:
-    cached = await cache_get_author_search_result(
-        db_session,
-        query_key=query_key,
-        now_utc=now_utc,
-    )
-    if cached is None:
-        return None
-    structured_log(
-        logger,
-        "info",
-        "scholar_search.cache_hit",
-        query=normalized_query,
-        state=cached.state.value,
-        state_reason=cached.state_reason,
-    )
-    state_reason_override = SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None
-    return _trim_author_search_result(
-        cached,
-        limit=bounded_limit,
-        extra_warnings=["author_search_served_from_cache"],
-        state_reason_override=state_reason_override,
-    )
-
-
-def _throttle_sleep_seconds(
-    *,
-    runtime_state: AuthorSearchRuntimeState,
-    now_utc: datetime,
-    min_interval_seconds: float,
-    interval_jitter_seconds: float,
-) -> tuple[float, bool]:
-    updated = False
-    if runtime_state.last_live_request_at is None:
-        enforced_wait_seconds = 0.0
-    else:
-        last_live_request_at = runtime_state.last_live_request_at
-        if last_live_request_at.tzinfo is None:
-            last_live_request_at = last_live_request_at.replace(tzinfo=UTC)
-            runtime_state.last_live_request_at = last_live_request_at
-            updated = True
-        enforced_wait_seconds = (
-            last_live_request_at + timedelta(seconds=max(float(min_interval_seconds), 0.0)) - now_utc
-        ).total_seconds()
-    jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0))
-    return max(0.0, float(enforced_wait_seconds)) + jitter_seconds, updated
-
-
-async def _wait_for_author_search_throttle(
-    *,
-    runtime_state: AuthorSearchRuntimeState,
-    normalized_query: str,
-    now_utc: datetime,
-    min_interval_seconds: float,
-    interval_jitter_seconds: float,
-) -> bool:
-    sleep_seconds, updated = _throttle_sleep_seconds(
-        runtime_state=runtime_state,
-        now_utc=now_utc,
-        min_interval_seconds=min_interval_seconds,
-        interval_jitter_seconds=interval_jitter_seconds,
-    )
-    if sleep_seconds <= 0.0:
-        return updated
-    structured_log(
-        logger,
-        "info",
-        "scholar_search.throttle_wait",
-        query=normalized_query,
-        sleep_seconds=round(sleep_seconds, 3),
-    )
-    await asyncio.sleep(sleep_seconds)
-    return True
-
-
-async def _fetch_author_search_with_retries(
-    *,
-    source: ScholarSource,
-    normalized_query: str,
-    network_error_retries: int,
-    retry_backoff_seconds: float,
-) -> tuple[ParsedAuthorSearchPage, int, list[str]]:
-    max_attempts = max(1, int(network_error_retries) + 1)
-    parsed: ParsedAuthorSearchPage | None = None
-    retry_warnings: list[str] = []
-    retry_scheduled_count = 0
-    for attempt_index in range(max_attempts):
-        fetch_result = await source.fetch_author_search_html(normalized_query, start=0)
-        try:
-            parsed = parse_author_search_page(fetch_result)
-        except ScholarParserError as exc:
-            parsed = ParsedAuthorSearchPage(
-                state=ParseState.LAYOUT_CHANGED,
-                state_reason=exc.code,
-                candidates=[],
-                marker_counts={},
-                warnings=[exc.code],
-            )
-        if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1:
-            break
-        retry_warnings.append("network_retry_scheduled_for_author_search")
-        retry_scheduled_count += 1
-        retry_sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index)
-        if retry_sleep_seconds > 0:
-            await asyncio.sleep(retry_sleep_seconds)
-    if parsed is None:
-        raise ScholarServiceError("Unable to complete scholar author search.")
-    return parsed, retry_scheduled_count, retry_warnings
-
-
-def _with_retry_warnings(
-    parsed: ParsedAuthorSearchPage,
-    *,
-    retry_warnings: list[str],
-    retry_scheduled_count: int,
-    retry_alert_threshold: int,
-    normalized_query: str,
-) -> ParsedAuthorSearchPage:
-    merged = replace(parsed, warnings=_merge_warnings(parsed.warnings, retry_warnings))
-    threshold = max(1, int(retry_alert_threshold))
-    if retry_scheduled_count < threshold:
-        return merged
-    structured_log(
-        logger,
-        "warning",
-        "scholar_search.retry_threshold_exceeded",
-        query=normalized_query,
-        retry_scheduled_count=retry_scheduled_count,
-        threshold=threshold,
-        final_state=merged.state.value,
-        final_state_reason=merged.state_reason,
-    )
-    return replace(
-        merged,
-        warnings=_merge_warnings(
-            merged.warnings,
-            [f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"],
-        ),
-    )
-
-
-def _apply_block_circuit_breaker(
-    *,
-    runtime_state: AuthorSearchRuntimeState,
-    merged_parsed: ParsedAuthorSearchPage,
-    cooldown_block_threshold: int,
-    cooldown_seconds: int,
-    normalized_query: str,
-) -> ParsedAuthorSearchPage:
-    if not _is_author_search_block_state(merged_parsed):
-        runtime_state.consecutive_blocked_count = 0
-        return merged_parsed
-    runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1
-    structured_log(
-        logger,
-        "warning",
-        "scholar_search.block_detected",
-        query=normalized_query,
-        state_reason=merged_parsed.state_reason,
-        consecutive_blocked_count=int(runtime_state.consecutive_blocked_count),
-    )
-    if int(runtime_state.consecutive_blocked_count) < max(1, int(cooldown_block_threshold)):
-        return merged_parsed
-    runtime_state.cooldown_until = datetime.now(UTC) + timedelta(seconds=max(60, int(cooldown_seconds)))
-    runtime_state.consecutive_blocked_count = 0
-    runtime_state.cooldown_rejection_count = 0
-    runtime_state.cooldown_alert_emitted = False
-    structured_log(
-        logger,
-        "error",
-        "scholar_search.cooldown_activated",
-        query=normalized_query,
-        cooldown_until_utc=runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
-    )
-    return replace(
-        merged_parsed,
-        warnings=_merge_warnings(merged_parsed.warnings, ["author_search_circuit_breaker_armed"]),
-    )
-
-
-def _resolve_author_search_cache_ttl_seconds(
-    *,
-    merged_parsed: ParsedAuthorSearchPage,
-    blocked_cache_ttl_seconds: int,
-    cache_ttl_seconds: int,
-) -> int:
-    if _is_author_search_block_state(merged_parsed):
-        return min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds)))
-    return max(1, int(cache_ttl_seconds))
-
-
-async def _load_locked_runtime_state(
-    db_session: AsyncSession,
-) -> AuthorSearchRuntimeState:
-    await _acquire_author_search_lock(db_session)
-    return await _load_runtime_state_for_update(db_session)
-
-
-async def _cooldown_or_cache_result(
-    db_session: AsyncSession,
-    *,
-    runtime_state: AuthorSearchRuntimeState,
-    query_key: str,
-    normalized_query: str,
-    bounded_limit: int,
-    cooldown_rejection_alert_threshold: int,
-) -> tuple[ParsedAuthorSearchPage | None, bool]:
-    runtime_state_updated = _normalize_runtime_cooldown_state(
-        runtime_state,
-        now_utc=datetime.now(UTC),
-    )
-    cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(
-        runtime_state,
-        datetime.now(UTC),
-    )
-    if cooldown_remaining_seconds > 0:
-        return (
-            _cooldown_block_result(
-                runtime_state=runtime_state,
-                normalized_query=normalized_query,
-                bounded_limit=bounded_limit,
-                cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
-                cooldown_remaining_seconds=cooldown_remaining_seconds,
-            ),
-            True,
-        )
-    cached_result = await _cache_hit_result(
-        db_session,
-        query_key=query_key,
-        now_utc=datetime.now(UTC),
-        normalized_query=normalized_query,
-        bounded_limit=bounded_limit,
-    )
-    return cached_result, runtime_state_updated
-
-
-async def _perform_live_author_search(
-    db_session: AsyncSession,
-    *,
-    source: ScholarSource,
-    runtime_state: AuthorSearchRuntimeState,
-    normalized_query: str,
-    query_key: str,
-    network_error_retries: int,
-    retry_backoff_seconds: float,
-    min_interval_seconds: float,
-    interval_jitter_seconds: float,
-    retry_alert_threshold: int,
-    cooldown_block_threshold: int,
-    cooldown_seconds: int,
-    blocked_cache_ttl_seconds: int,
-    cache_ttl_seconds: int,
-    cache_max_entries: int,
-) -> tuple[ParsedAuthorSearchPage, bool]:
-    await _wait_for_author_search_throttle(
-        runtime_state=runtime_state,
-        normalized_query=normalized_query,
-        now_utc=datetime.now(UTC),
-        min_interval_seconds=min_interval_seconds,
-        interval_jitter_seconds=interval_jitter_seconds,
-    )
-    parsed, retry_count, retry_warnings = await _fetch_author_search_with_retries(
-        source=source,
-        normalized_query=normalized_query,
-        network_error_retries=network_error_retries,
-        retry_backoff_seconds=retry_backoff_seconds,
-    )
-    runtime_state.last_live_request_at = datetime.now(UTC)
-    merged = _with_retry_warnings(
-        parsed,
-        retry_warnings=retry_warnings,
-        retry_scheduled_count=retry_count,
-        retry_alert_threshold=retry_alert_threshold,
-        normalized_query=normalized_query,
-    )
-    merged = _apply_block_circuit_breaker(
-        runtime_state=runtime_state,
-        merged_parsed=merged,
-        cooldown_block_threshold=cooldown_block_threshold,
-        cooldown_seconds=cooldown_seconds,
-        normalized_query=normalized_query,
-    )
-    ttl_seconds = _resolve_author_search_cache_ttl_seconds(
-        merged_parsed=merged,
-        blocked_cache_ttl_seconds=blocked_cache_ttl_seconds,
-        cache_ttl_seconds=cache_ttl_seconds,
-    )
-    await cache_set_author_search_result(
-        db_session,
-        query_key=query_key,
-        parsed=merged,
-        ttl_seconds=float(ttl_seconds),
-        max_entries=cache_max_entries,
-        now_utc=datetime.now(UTC),
-    )
-    return merged, True
-
-
-async def search_author_candidates(
-    *,
-    source: ScholarSource,
-    db_session: AsyncSession,
-    query: str,
-    limit: int,
-    network_error_retries: int = 1,
-    retry_backoff_seconds: float = 1.0,
-    search_enabled: bool = True,
-    cache_ttl_seconds: int = 21_600,
-    blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS,
-    cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES,
-    min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS,
-    interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS,
-    cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD,
-    cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS,
-    retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD,
-    cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD,
-) -> ParsedAuthorSearchPage:
-    normalized_query, bounded_limit, query_key = _normalize_author_search_inputs(query, limit)
-    if not search_enabled:
-        return _disabled_search_result(
-            normalized_query=normalized_query,
-            bounded_limit=bounded_limit,
-        )
-
-    runtime_state = await _load_locked_runtime_state(db_session)
-    early_result, runtime_state_updated = await _cooldown_or_cache_result(
-        db_session,
-        runtime_state=runtime_state,
-        query_key=query_key,
-        normalized_query=normalized_query,
-        bounded_limit=bounded_limit,
-        cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
-    )
-    if early_result is not None:
-        await db_session.commit()
-        return early_result
-
-    merged_parsed, live_runtime_updated = await _perform_live_author_search(
-        db_session,
-        source=source,
-        runtime_state=runtime_state,
-        normalized_query=normalized_query,
-        query_key=query_key,
-        network_error_retries=network_error_retries,
-        retry_backoff_seconds=retry_backoff_seconds,
-        min_interval_seconds=min_interval_seconds,
-        interval_jitter_seconds=interval_jitter_seconds,
-        retry_alert_threshold=retry_alert_threshold,
-        cooldown_block_threshold=cooldown_block_threshold,
-        cooldown_seconds=cooldown_seconds,
-        blocked_cache_ttl_seconds=blocked_cache_ttl_seconds,
-        cache_ttl_seconds=cache_ttl_seconds,
-        cache_max_entries=cache_max_entries,
-    )
-    runtime_state_updated = runtime_state_updated or live_runtime_updated
-    if runtime_state_updated:
-        await db_session.commit()
-    return _trim_author_search_result(merged_parsed, limit=bounded_limit)

app/services/scholars/author_search_cache.py

@@ -1,240 +0,0 @@
-from __future__ import annotations
-
-from datetime import UTC, datetime, timedelta
-from typing import cast
-
-from sqlalchemy import delete, func, select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import AuthorSearchCacheEntry
-from app.services.scholar.parser import (
-    ParsedAuthorSearchPage,
-    ParseState,
-    ScholarSearchCandidate,
-)
-
-
-def _serialize_parsed_author_search_page(parsed: ParsedAuthorSearchPage) -> dict:
-    return {
-        "state": parsed.state.value,
-        "state_reason": parsed.state_reason,
-        "marker_counts": {str(key): int(value) for key, value in parsed.marker_counts.items()},
-        "warnings": [str(value) for value in parsed.warnings],
-        "candidates": [
-            {
-                "scholar_id": candidate.scholar_id,
-                "display_name": candidate.display_name,
-                "affiliation": candidate.affiliation,
-                "email_domain": candidate.email_domain,
-                "cited_by_count": candidate.cited_by_count,
-                "interests": [str(interest) for interest in candidate.interests],
-                "profile_url": candidate.profile_url,
-                "profile_image_url": candidate.profile_image_url,
-            }
-            for candidate in parsed.candidates
-        ],
-    }
-
-
-def _payload_state(payload: dict[str, object]) -> ParseState | None:
-    state_raw = str(payload.get("state", "")).strip()
-    try:
-        return ParseState(state_raw)
-    except ValueError:
-        return None
-
-
-def _payload_marker_counts(payload: dict[str, object]) -> dict[str, int]:
-    marker_counts_payload = payload.get("marker_counts")
-    if not isinstance(marker_counts_payload, dict):
-        return {}
-    parsed: dict[str, int] = {}
-    for key, value in marker_counts_payload.items():
-        try:
-            parsed[str(key)] = int(value)
-        except (TypeError, ValueError):
-            continue
-    return parsed
-
-
-def _payload_warnings(payload: dict[str, object]) -> list[str]:
-    warnings_payload = payload.get("warnings")
-    if not isinstance(warnings_payload, list):
-        return []
-    return [str(value) for value in warnings_payload if isinstance(value, str)]
-
-
-def _parse_optional_string(value: object) -> str | None:
-    if value is None:
-        return None
-    normalized = str(value).strip()
-    return normalized or None
-
-
-def _parse_optional_int(value: object) -> int | None:
-    if isinstance(value, int):
-        return value
-    if isinstance(value, str) and value.strip():
-        try:
-            return int(value)
-        except ValueError:
-            return None
-    return None
-
-
-def _normalize_interests(value: object) -> list[str]:
-    if not isinstance(value, list):
-        return []
-    return [str(item) for item in value if isinstance(item, str)]
-
-
-def _deserialize_candidate_payload(value: object) -> dict[str, object] | None:
-    if not isinstance(value, dict):
-        return None
-    scholar_id = str(value.get("scholar_id", "")).strip()
-    display_name = str(value.get("display_name", "")).strip()
-    profile_url = str(value.get("profile_url", "")).strip()
-    if not scholar_id or not display_name or not profile_url:
-        return None
-    return {
-        "scholar_id": scholar_id,
-        "display_name": display_name,
-        "affiliation": _parse_optional_string(value.get("affiliation")),
-        "email_domain": _parse_optional_string(value.get("email_domain")),
-        "cited_by_count": _parse_optional_int(value.get("cited_by_count")),
-        "interests": _normalize_interests(value.get("interests")),
-        "profile_url": profile_url,
-        "profile_image_url": _parse_optional_string(value.get("profile_image_url")),
-    }
-
-
-def _deserialize_candidates(payload: dict[str, object]) -> list[dict[str, object]]:
-    candidates_payload = payload.get("candidates")
-    if not isinstance(candidates_payload, list):
-        return []
-    normalized: list[dict[str, object]] = []
-    for value in candidates_payload:
-        candidate = _deserialize_candidate_payload(value)
-        if candidate is not None:
-            normalized.append(candidate)
-    return normalized
-
-
-def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearchPage | None:
-    if not isinstance(payload, dict):
-        return None
-
-    state = _payload_state(payload)
-    if state is None:
-        return None
-
-    marker_counts = _payload_marker_counts(payload)
-    warnings = _payload_warnings(payload)
-    normalized_candidates = _deserialize_candidates(payload)
-
-    return ParsedAuthorSearchPage(
-        state=state,
-        state_reason=str(payload.get("state_reason", "")).strip() or "unknown",
-        candidates=[
-            ScholarSearchCandidate(
-                scholar_id=cast(str, item["scholar_id"]),
-                display_name=cast(str, item["display_name"]),
-                affiliation=cast("str | None", item["affiliation"]),
-                email_domain=cast("str | None", item["email_domain"]),
-                cited_by_count=cast("int | None", item["cited_by_count"]),
-                interests=cast("list[str]", item["interests"]),
-                profile_url=cast(str, item["profile_url"]),
-                profile_image_url=cast("str | None", item["profile_image_url"]),
-            )
-            for item in normalized_candidates
-        ],
-        marker_counts=marker_counts,
-        warnings=warnings,
-    )
-
-
-async def cache_get_author_search_result(
-    db_session: AsyncSession,
-    *,
-    query_key: str,
-    now_utc: datetime,
-) -> ParsedAuthorSearchPage | None:
-    result = await db_session.execute(
-        select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key)
-    )
-    entry = result.scalar_one_or_none()
-    if entry is None:
-        return None
-    expires_at = entry.expires_at
-    if expires_at.tzinfo is None:
-        expires_at = expires_at.replace(tzinfo=UTC)
-    if expires_at <= now_utc:
-        await db_session.delete(entry)
-        return None
-    parsed = _deserialize_parsed_author_search_page(entry.payload)
-    if parsed is None:
-        await db_session.delete(entry)
-        return None
-    return parsed
-
-
-async def cache_set_author_search_result(
-    db_session: AsyncSession,
-    *,
-    query_key: str,
-    parsed: ParsedAuthorSearchPage,
-    ttl_seconds: float,
-    max_entries: int,
-    now_utc: datetime,
-) -> None:
-    ttl = max(float(ttl_seconds), 0.0)
-    existing_result = await db_session.execute(
-        select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key)
-    )
-    existing = existing_result.scalar_one_or_none()
-
-    if ttl <= 0.0:
-        if existing is not None:
-            await db_session.delete(existing)
-        return
-
-    expires_at = now_utc + timedelta(seconds=ttl)
-    payload = _serialize_parsed_author_search_page(parsed)
-    if existing is None:
-        db_session.add(
-            AuthorSearchCacheEntry(
-                query_key=query_key,
-                payload=payload,
-                expires_at=expires_at,
-                cached_at=now_utc,
-                updated_at=now_utc,
-            )
-        )
-    else:
-        existing.payload = payload
-        existing.expires_at = expires_at
-        existing.cached_at = now_utc
-        existing.updated_at = now_utc
-
-    await prune_author_search_cache(db_session, now_utc=now_utc, max_entries=max_entries)
-
-
-async def prune_author_search_cache(
-    db_session: AsyncSession,
-    *,
-    now_utc: datetime,
-    max_entries: int,
-) -> None:
-    await db_session.execute(delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.expires_at <= now_utc))
-    bounded_max_entries = max(1, int(max_entries))
-    count_result = await db_session.execute(select(func.count()).select_from(AuthorSearchCacheEntry))
-    entry_count = int(count_result.scalar_one() or 0)
-    overflow = max(0, entry_count - bounded_max_entries)
-    if overflow <= 0:
-        return
-    stale_keys_result = await db_session.execute(
-        select(AuthorSearchCacheEntry.query_key).order_by(AuthorSearchCacheEntry.cached_at.asc()).limit(overflow)
-    )
-    stale_keys = [str(row[0]) for row in stale_keys_result.all()]
-    if stale_keys:
-        await db_session.execute(delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key.in_(stale_keys)))

app/services/unpaywall/application.py

@@ -393,7 +393,7 @@ async def resolve_publication_oa_outcomes(
         return {}
     email = _email_for_request(request_email)
     if email is None:
-        structured_log(logger, "debug", "unpaywall.resolve_skipped_missing_email")
+        logger.debug("unpaywall.resolve_skipped_missing_email")
         return {}
     import httpx
 

app/settings.py

@@ -235,13 +235,6 @@ class Settings:
         "SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD",
         3,
     )
-    scholar_http_user_agent: str = _env_str("SCHOLAR_HTTP_USER_AGENT", "")
-    scholar_http_rotate_user_agent: bool = _env_bool("SCHOLAR_HTTP_ROTATE_USER_AGENT", False)
-    scholar_http_accept_language: str = _env_str(
-        "SCHOLAR_HTTP_ACCEPT_LANGUAGE",
-        "en-US,en;q=0.9",
-    )
-    scholar_http_cookie: str = _env_str("SCHOLAR_HTTP_COOKIE", "")
     unpaywall_enabled: bool = _env_bool("UNPAYWALL_ENABLED", True)
     unpaywall_email: str = _env_str("UNPAYWALL_EMAIL", "")
     unpaywall_timeout_seconds: float = _env_float("UNPAYWALL_TIMEOUT_SECONDS", 4.0)
@@ -275,8 +268,6 @@ class Settings:
     crossref_max_lookups_per_request: int = _env_int("CROSSREF_MAX_LOOKUPS_PER_REQUEST", 8)
 
     openalex_api_key: str | None = os.getenv("OPENALEX_API_KEY")
-    database_reserved_api_connections: int = _env_int("DATABASE_RESERVED_API_CONNECTIONS", 3)
-
     crossref_api_token: str | None = os.getenv("CROSSREF_API_TOKEN")
     crossref_api_mailto: str | None = os.getenv("CROSSREF_API_MAILTO")
 

frontend/package-lock.json

@@ -15,9 +15,7 @@
       "devDependencies": {
         "@types/node": "^22.10.2",
         "@vitejs/plugin-vue": "^5.2.1",
-        "@vue/test-utils": "^2.4.6",
         "autoprefixer": "^10.4.21",
-        "happy-dom": "^20.7.0",
         "postcss": "^8.5.3",
         "tailwindcss": "^3.4.17",
         "typescript": "^5.7.2",
@@ -476,24 +474,6 @@
         "node": ">=12"
       }
     },
-    "node_modules/@isaacs/cliui": {
-      "version": "8.0.2",
-      "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
-      "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==",
-      "dev": true,
-      "license": "ISC",
-      "dependencies": {
-        "string-width": "^5.1.2",
-        "string-width-cjs": "npm:string-width@^4.2.0",
-        "strip-ansi": "^7.0.1",
-        "strip-ansi-cjs": "npm:strip-ansi@^6.0.1",
-        "wrap-ansi": "^8.1.0",
-        "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0"
-      },
-      "engines": {
-        "node": ">=12"
-      }
-    },
     "node_modules/@jridgewell/gen-mapping": {
       "version": "0.3.13",
       "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz",
@@ -570,24 +550,6 @@
         "node": ">= 8"
       }
     },
-    "node_modules/@one-ini/wasm": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/@one-ini/wasm/-/wasm-0.1.1.tgz",
-      "integrity": "sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==",
-      "dev": true,
-      "license": "MIT"
-    },
-    "node_modules/@pkgjs/parseargs": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
-      "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "engines": {
-        "node": ">=14"
-      }
-    },
     "node_modules/@rollup/rollup-android-arm-eabi": {
       "version": "4.57.1",
       "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.57.1.tgz",
@@ -955,23 +917,6 @@
         "undici-types": "~6.21.0"
       }
     },
-    "node_modules/@types/whatwg-mimetype": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@types/whatwg-mimetype/-/whatwg-mimetype-3.0.2.tgz",
-      "integrity": "sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA==",
-      "dev": true,
-      "license": "MIT"
-    },
-    "node_modules/@types/ws": {
-      "version": "8.18.1",
-      "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz",
-      "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@types/node": "*"
-      }
-    },
     "node_modules/@vitejs/plugin-vue": {
       "version": "5.2.4",
       "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-5.2.4.tgz",
@@ -1282,27 +1227,6 @@
       "integrity": "sha512-cfWa1fCGBxrvaHRhvV3Is0MgmrbSCxYTXCSCau2I0a1Xw1N1pHAvkWCiXPRAqjvToILvguNyEwjevUqAuBQWvQ==",
       "license": "MIT"
     },
-    "node_modules/@vue/test-utils": {
-      "version": "2.4.6",
-      "resolved": "https://registry.npmjs.org/@vue/test-utils/-/test-utils-2.4.6.tgz",
-      "integrity": "sha512-FMxEjOpYNYiFe0GkaHsnJPXFHxQ6m4t8vI/ElPGpMWxZKpmRvQ33OIrvRXemy6yha03RxhOlQuy+gZMC3CQSow==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "js-beautify": "^1.14.9",
-        "vue-component-type-helpers": "^2.0.0"
-      }
-    },
-    "node_modules/abbrev": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-2.0.0.tgz",
-      "integrity": "sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==",
-      "dev": true,
-      "license": "ISC",
-      "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
-      }
-    },
     "node_modules/alien-signals": {
       "version": "1.0.13",
       "resolved": "https://registry.npmjs.org/alien-signals/-/alien-signals-1.0.13.tgz",
@@ -1310,32 +1234,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/ansi-regex": {
-      "version": "6.2.2",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz",
-      "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/ansi-regex?sponsor=1"
-      }
-    },
-    "node_modules/ansi-styles": {
-      "version": "6.2.3",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz",
-      "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
-      }
-    },
     "node_modules/any-promise": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
@@ -1604,26 +1502,6 @@
         "node": ">= 6"
       }
     },
-    "node_modules/color-convert": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
-      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "color-name": "~1.1.4"
-      },
-      "engines": {
-        "node": ">=7.0.0"
-      }
-    },
-    "node_modules/color-name": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
-      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
-      "dev": true,
-      "license": "MIT"
-    },
     "node_modules/commander": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
@@ -1634,32 +1512,6 @@
         "node": ">= 6"
       }
     },
-    "node_modules/config-chain": {
-      "version": "1.1.13",
-      "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz",
-      "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "ini": "^1.3.4",
-        "proto-list": "~1.2.1"
-      }
-    },
-    "node_modules/cross-spawn": {
-      "version": "7.0.6",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
-      "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "path-key": "^3.1.0",
-        "shebang-command": "^2.0.0",
-        "which": "^2.0.1"
-      },
-      "engines": {
-        "node": ">= 8"
-      }
-    },
     "node_modules/cssesc": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz",
@@ -1728,42 +1580,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/eastasianwidth": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
-      "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==",
-      "dev": true,
-      "license": "MIT"
-    },
-    "node_modules/editorconfig": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-1.0.7.tgz",
-      "integrity": "sha512-e0GOtq/aTQhVdNyDU9e02+wz9oDDM+SIOQxWME2QRjzRX5yyLAuHDE+0aE8vHb9XRC8XD37eO2u57+F09JqFhw==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@one-ini/wasm": "0.1.1",
-        "commander": "^10.0.0",
-        "minimatch": "^9.0.1",
-        "semver": "^7.5.3"
-      },
-      "bin": {
-        "editorconfig": "bin/editorconfig"
-      },
-      "engines": {
-        "node": ">=14"
-      }
-    },
-    "node_modules/editorconfig/node_modules/commander": {
-      "version": "10.0.1",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz",
-      "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=14"
-      }
-    },
     "node_modules/electron-to-chromium": {
       "version": "1.5.286",
       "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.286.tgz",
@@ -1771,13 +1587,6 @@
       "dev": true,
       "license": "ISC"
     },
-    "node_modules/emoji-regex": {
-      "version": "9.2.2",
-      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
-      "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==",
-      "dev": true,
-      "license": "MIT"
-    },
     "node_modules/entities": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz",
@@ -1919,23 +1728,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/foreground-child": {
-      "version": "3.3.1",
-      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz",
-      "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==",
-      "dev": true,
-      "license": "ISC",
-      "dependencies": {
-        "cross-spawn": "^7.0.6",
-        "signal-exit": "^4.0.1"
-      },
-      "engines": {
-        "node": ">=14"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
     "node_modules/fraction.js": {
       "version": "5.3.4",
       "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.4.tgz",
@@ -1975,28 +1767,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/glob": {
-      "version": "10.5.0",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz",
-      "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==",
-      "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me",
-      "dev": true,
-      "license": "ISC",
-      "dependencies": {
-        "foreground-child": "^3.1.0",
-        "jackspeak": "^3.1.2",
-        "minimatch": "^9.0.4",
-        "minipass": "^7.1.2",
-        "package-json-from-dist": "^1.0.0",
-        "path-scurry": "^1.11.1"
-      },
-      "bin": {
-        "glob": "dist/esm/bin.mjs"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
     "node_modules/glob-parent": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
@@ -2010,24 +1780,6 @@
         "node": ">=10.13.0"
       }
     },
-    "node_modules/happy-dom": {
-      "version": "20.7.0",
-      "resolved": "https://registry.npmjs.org/happy-dom/-/happy-dom-20.7.0.tgz",
-      "integrity": "sha512-hR/uLYQdngTyEfxnOoa+e6KTcfBFyc1hgFj/Cc144A5JJUuHFYqIEBDcD4FeGqUeKLRZqJ9eN9u7/GDjYEgS1g==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@types/node": ">=20.0.0",
-        "@types/whatwg-mimetype": "^3.0.2",
-        "@types/ws": "^8.18.1",
-        "entities": "^7.0.1",
-        "whatwg-mimetype": "^3.0.0",
-        "ws": "^8.18.3"
-      },
-      "engines": {
-        "node": ">=20.0.0"
-      }
-    },
     "node_modules/hasown": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
@@ -2051,13 +1803,6 @@
         "he": "bin/he"
       }
     },
-    "node_modules/ini": {
-      "version": "1.3.8",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
-      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
-      "dev": true,
-      "license": "ISC"
-    },
     "node_modules/is-binary-path": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
@@ -2097,16 +1842,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/is-fullwidth-code-point": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
-      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/is-glob": {
       "version": "4.0.3",
       "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
@@ -2130,29 +1865,6 @@
         "node": ">=0.12.0"
       }
     },
-    "node_modules/isexe": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
-      "dev": true,
-      "license": "ISC"
-    },
-    "node_modules/jackspeak": {
-      "version": "3.4.3",
-      "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz",
-      "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==",
-      "dev": true,
-      "license": "BlueOak-1.0.0",
-      "dependencies": {
-        "@isaacs/cliui": "^8.0.2"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      },
-      "optionalDependencies": {
-        "@pkgjs/parseargs": "^0.11.0"
-      }
-    },
     "node_modules/jiti": {
       "version": "1.21.7",
       "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz",
@@ -2163,38 +1875,6 @@
         "jiti": "bin/jiti.js"
       }
     },
-    "node_modules/js-beautify": {
-      "version": "1.15.4",
-      "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.15.4.tgz",
-      "integrity": "sha512-9/KXeZUKKJwqCXUdBxFJ3vPh467OCckSBmYDwSK/EtV090K+iMJ7zx2S3HLVDIWFQdqMIsZWbnaGiba18aWhaA==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "config-chain": "^1.1.13",
-        "editorconfig": "^1.0.4",
-        "glob": "^10.4.2",
-        "js-cookie": "^3.0.5",
-        "nopt": "^7.2.1"
-      },
-      "bin": {
-        "css-beautify": "js/bin/css-beautify.js",
-        "html-beautify": "js/bin/html-beautify.js",
-        "js-beautify": "js/bin/js-beautify.js"
-      },
-      "engines": {
-        "node": ">=14"
-      }
-    },
-    "node_modules/js-cookie": {
-      "version": "3.0.5",
-      "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.5.tgz",
-      "integrity": "sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=14"
-      }
-    },
     "node_modules/lilconfig": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz",
@@ -2222,13 +1902,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/lru-cache": {
-      "version": "10.4.3",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz",
-      "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
-      "dev": true,
-      "license": "ISC"
-    },
     "node_modules/magic-string": {
       "version": "0.30.21",
       "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz",
@@ -2278,16 +1951,6 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/minipass": {
-      "version": "7.1.3",
-      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz",
-      "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==",
-      "dev": true,
-      "license": "BlueOak-1.0.0",
-      "engines": {
-        "node": ">=16 || 14 >=14.17"
-      }
-    },
     "node_modules/ms": {
       "version": "2.1.3",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
@@ -2339,22 +2002,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/nopt": {
-      "version": "7.2.1",
-      "resolved": "https://registry.npmjs.org/nopt/-/nopt-7.2.1.tgz",
-      "integrity": "sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w==",
-      "dev": true,
-      "license": "ISC",
-      "dependencies": {
-        "abbrev": "^2.0.0"
-      },
-      "bin": {
-        "nopt": "bin/nopt.js"
-      },
-      "engines": {
-        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
-      }
-    },
     "node_modules/normalize-path": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
@@ -2385,13 +2032,6 @@
         "node": ">= 6"
       }
     },
-    "node_modules/package-json-from-dist": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz",
-      "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==",
-      "dev": true,
-      "license": "BlueOak-1.0.0"
-    },
     "node_modules/path-browserify": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz",
@@ -2399,16 +2039,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/path-key": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
-      "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/path-parse": {
       "version": "1.0.7",
       "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz",
@@ -2416,23 +2046,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/path-scurry": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz",
-      "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==",
-      "dev": true,
-      "license": "BlueOak-1.0.0",
-      "dependencies": {
-        "lru-cache": "^10.2.0",
-        "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0"
-      },
-      "engines": {
-        "node": ">=16 || 14 >=14.18"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
     "node_modules/pathe": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz",
@@ -2673,13 +2286,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/proto-list": {
-      "version": "1.2.4",
-      "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz",
-      "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==",
-      "dev": true,
-      "license": "ISC"
-    },
     "node_modules/queue-microtask": {
       "version": "1.2.3",
       "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
@@ -2825,42 +2431,6 @@
         "queue-microtask": "^1.2.2"
       }
     },
-    "node_modules/semver": {
-      "version": "7.7.4",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz",
-      "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==",
-      "dev": true,
-      "license": "ISC",
-      "bin": {
-        "semver": "bin/semver.js"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/shebang-command": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
-      "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "shebang-regex": "^3.0.0"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/shebang-regex": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
-      "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/siginfo": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz",
@@ -2868,19 +2438,6 @@
       "dev": true,
       "license": "ISC"
     },
-    "node_modules/signal-exit": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
-      "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
-      "dev": true,
-      "license": "ISC",
-      "engines": {
-        "node": ">=14"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
     "node_modules/source-map-js": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz",
@@ -2904,110 +2461,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/string-width": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
-      "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "eastasianwidth": "^0.2.0",
-        "emoji-regex": "^9.2.2",
-        "strip-ansi": "^7.0.1"
-      },
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/string-width-cjs": {
-      "name": "string-width",
-      "version": "4.2.3",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
-      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "emoji-regex": "^8.0.0",
-        "is-fullwidth-code-point": "^3.0.0",
-        "strip-ansi": "^6.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/string-width-cjs/node_modules/ansi-regex": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
-      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/string-width-cjs/node_modules/emoji-regex": {
-      "version": "8.0.0",
-      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
-      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
-      "dev": true,
-      "license": "MIT"
-    },
-    "node_modules/string-width-cjs/node_modules/strip-ansi": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
-      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "ansi-regex": "^5.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/strip-ansi": {
-      "version": "7.2.0",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz",
-      "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "ansi-regex": "^6.2.2"
-      },
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/strip-ansi?sponsor=1"
-      }
-    },
-    "node_modules/strip-ansi-cjs": {
-      "name": "strip-ansi",
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
-      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "ansi-regex": "^5.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/strip-ansi-cjs/node_modules/ansi-regex": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
-      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/sucrase": {
       "version": "3.35.1",
       "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.1.tgz",
@@ -3453,13 +2906,6 @@
         }
       }
     },
-    "node_modules/vue-component-type-helpers": {
-      "version": "2.2.12",
-      "resolved": "https://registry.npmjs.org/vue-component-type-helpers/-/vue-component-type-helpers-2.2.12.tgz",
-      "integrity": "sha512-YbGqHZ5/eW4SnkPNR44mKVc6ZKQoRs/Rux1sxC6rdwXb4qpbOSYfDr9DsTHolOTGmIKgM9j141mZbBeg05R1pw==",
-      "dev": true,
-      "license": "MIT"
-    },
     "node_modules/vue-demi": {
       "version": "0.14.10",
       "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.10.tgz",
@@ -3518,32 +2964,6 @@
         "typescript": ">=5.0.0"
       }
     },
-    "node_modules/whatwg-mimetype": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz",
-      "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=12"
-      }
-    },
-    "node_modules/which": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
-      "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
-      "dev": true,
-      "license": "ISC",
-      "dependencies": {
-        "isexe": "^2.0.0"
-      },
-      "bin": {
-        "node-which": "bin/node-which"
-      },
-      "engines": {
-        "node": ">= 8"
-      }
-    },
     "node_modules/why-is-node-running": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz",
@@ -3560,126 +2980,6 @@
       "engines": {
         "node": ">=8"
       }
-    },
-    "node_modules/wrap-ansi": {
-      "version": "8.1.0",
-      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",
-      "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "ansi-styles": "^6.1.0",
-        "string-width": "^5.0.1",
-        "strip-ansi": "^7.0.1"
-      },
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
-      }
-    },
-    "node_modules/wrap-ansi-cjs": {
-      "name": "wrap-ansi",
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
-      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "ansi-styles": "^4.0.0",
-        "string-width": "^4.1.0",
-        "strip-ansi": "^6.0.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
-      }
-    },
-    "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
-      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
-      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "color-convert": "^2.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
-      }
-    },
-    "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": {
-      "version": "8.0.0",
-      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
-      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
-      "dev": true,
-      "license": "MIT"
-    },
-    "node_modules/wrap-ansi-cjs/node_modules/string-width": {
-      "version": "4.2.3",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
-      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "emoji-regex": "^8.0.0",
-        "is-fullwidth-code-point": "^3.0.0",
-        "strip-ansi": "^6.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
-      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "ansi-regex": "^5.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/ws": {
-      "version": "8.19.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz",
-      "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": ">=10.0.0"
-      },
-      "peerDependencies": {
-        "bufferutil": "^4.0.1",
-        "utf-8-validate": ">=5.0.2"
-      },
-      "peerDependenciesMeta": {
-        "bufferutil": {
-          "optional": true
-        },
-        "utf-8-validate": {
-          "optional": true
-        }
-      }
     }
   }
 }

frontend/package.json

@@ -21,9 +21,7 @@
   "devDependencies": {
     "@types/node": "^22.10.2",
     "@vitejs/plugin-vue": "^5.2.1",
-    "@vue/test-utils": "^2.4.6",
     "autoprefixer": "^10.4.21",
-    "happy-dom": "^20.7.0",
     "postcss": "^8.5.3",
     "tailwindcss": "^3.4.17",
     "typescript": "^5.7.2",

frontend/src/components/patterns/ScrapeSafetyBadge.test.ts

@@ -1,61 +0,0 @@
-// @vitest-environment happy-dom
-import { describe, expect, it } from "vitest";
-import { mount } from "@vue/test-utils";
-import ScrapeSafetyBadge from "./ScrapeSafetyBadge.vue";
-import { createDefaultSafetyState, type ScrapeSafetyState } from "@/features/safety";
-
-function buildState(overrides: Partial<ScrapeSafetyState> = {}): ScrapeSafetyState {
-  return { ...createDefaultSafetyState(), ...overrides };
-}
-
-describe("ScrapeSafetyBadge", () => {
-  it("shows ready tooltip when cooldown is inactive", () => {
-    const wrapper = mount(ScrapeSafetyBadge, {
-      props: { state: buildState({ cooldown_active: false }) },
-    });
-    expect(wrapper.text()).toContain("Safety ready");
-    const hint = wrapper.findComponent({ name: "AppHelpHint" });
-    expect(hint.exists()).toBe(true);
-    expect(hint.props("text")).toContain("No active cooldown");
-  });
-
-  it("shows cooldown tooltip with reason and action when active", () => {
-    const wrapper = mount(ScrapeSafetyBadge, {
-      props: {
-        state: buildState({
-          cooldown_active: true,
-          cooldown_reason: "blocked_failure_threshold_exceeded",
-          cooldown_reason_label: "Too many blocked requests",
-          recommended_action: "Wait for cooldown to expire",
-          cooldown_remaining_seconds: 120,
-        }),
-      },
-    });
-    expect(wrapper.text()).toContain("Safety cooldown");
-    const hint = wrapper.findComponent({ name: "AppHelpHint" });
-    expect(hint.exists()).toBe(true);
-    const text = hint.props("text") as string;
-    expect(text).toContain("Google Scholar rate-limits");
-    expect(text).toContain("Too many blocked requests");
-    expect(text).toContain("Wait for cooldown to expire");
-  });
-
-  it("shows cooldown tooltip without optional fields", () => {
-    const wrapper = mount(ScrapeSafetyBadge, {
-      props: {
-        state: buildState({
-          cooldown_active: true,
-          cooldown_reason: "some_reason",
-          cooldown_reason_label: null,
-          recommended_action: null,
-          cooldown_remaining_seconds: 60,
-        }),
-      },
-    });
-    const hint = wrapper.findComponent({ name: "AppHelpHint" });
-    const text = hint.props("text") as string;
-    expect(text).toContain("Google Scholar rate-limits");
-    expect(text).not.toContain("Why:");
-    expect(text).not.toContain("Action:");
-  });
-});

frontend/src/components/patterns/ScrapeSafetyBadge.vue

@@ -1,7 +1,6 @@
 <script setup lang="ts">
 import { computed } from "vue";
 
-import AppHelpHint from "@/components/ui/AppHelpHint.vue";
 import { type ScrapeSafetyState } from "@/features/safety";
 
 const props = defineProps<{
@@ -19,30 +18,10 @@ const toneClass = computed(() => {
   }
   return "border-state-warning-border bg-state-warning-bg text-state-warning-text";
 });
-
-const READY_TOOLTIP = "No active cooldown. Scraping can proceed normally.";
-const RATE_LIMIT_INTRO =
-  "Google Scholar rate-limits automated requests. The cooldown pauses scraping to avoid your IP being blocked.";
-
-const tooltipText = computed(() => {
-  if (!props.state.cooldown_active) return READY_TOOLTIP;
-
-  const parts = [RATE_LIMIT_INTRO];
-  if (props.state.cooldown_reason_label) {
-    parts.push(`Why: ${props.state.cooldown_reason_label}`);
-  }
-  if (props.state.recommended_action) {
-    parts.push(`Action: ${props.state.recommended_action}`);
-  }
-  return parts.join(" \u2014 ");
-});
 </script>
 
 <template>
-  <span class="inline-flex items-center gap-1">
+  <span class="inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-semibold" :class="toneClass">
-    <span class="inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-semibold" :class="toneClass">
+    {{ label }}
-      {{ label }}
-    </span>
-    <AppHelpHint :text="tooltipText" />
   </span>
 </template>

frontend/src/components/patterns/ScrapeSafetyBanner.vue

@@ -2,7 +2,6 @@
 import { computed, onBeforeUnmount, onMounted, ref } from "vue";
 
 import AppAlert from "@/components/ui/AppAlert.vue";
-import AppHelpHint from "@/components/ui/AppHelpHint.vue";
 import {
   formatCooldownCountdown,
   type ScrapeSafetyState,
@@ -80,10 +79,6 @@ const actionText = computed(() => {
   return props.safetyState.recommended_action;
 });
 
-const bannerHintText = computed(() => {
-  return "Google Scholar rate-limits automated requests. The cooldown pauses scraping to avoid your IP being blocked.";
-});
-
 onMounted(() => {
   timer = setInterval(() => {
     now.value = Date.now();
@@ -100,12 +95,7 @@ onBeforeUnmount(() => {
 
 <template>
   <AppAlert v-if="isVisible" :tone="tone">
-    <template #title>
+    <template #title>{{ title }}</template>
-      <span class="inline-flex items-center gap-1">
-        {{ title }}
-        <AppHelpHint :text="bannerHintText" />
-      </span>
-    </template>
     <p>{{ detailText }}</p>
     <p v-if="actionText" class="text-secondary">{{ actionText }}</p>
   </AppAlert>

frontend/src/components/ui/AppConfirmModal.vue

@@ -1,28 +0,0 @@
-<script setup lang="ts">
-import AppButton from "@/components/ui/AppButton.vue";
-import AppModal from "@/components/ui/AppModal.vue";
-
-withDefaults(
-  defineProps<{
-    open: boolean;
-    title: string;
-    message: string;
-    confirmLabel?: string;
-    cancelLabel?: string;
-    variant?: "danger" | "default";
-  }>(),
-  { confirmLabel: "Confirm", cancelLabel: "Cancel", variant: "default" },
-);
-
-const emit = defineEmits<{ confirm: []; cancel: [] }>();
-</script>
-
-<template>
-  <AppModal :open="open" :title="title" @close="emit('cancel')">
-    <p class="mb-6 text-sm text-secondary">{{ message }}</p>
-    <div class="flex justify-end gap-2">
-      <AppButton variant="secondary" @click="emit('cancel')">{{ cancelLabel }}</AppButton>
-      <AppButton :variant="variant === 'danger' ? 'danger' : 'primary'" @click="emit('confirm')">{{ confirmLabel }}</AppButton>
-    </div>
-  </AppModal>
-</template>

frontend/src/composables/useRequestState.test.ts

@@ -1,51 +0,0 @@
-import { describe, expect, it } from "vitest";
-import { ApiRequestError } from "@/lib/api/errors";
-import { useRequestState } from "@/composables/useRequestState";
-
-describe("useRequestState", () => {
-  it("initializes with all values null", () => {
-    const { errorMessage, errorRequestId, successMessage } = useRequestState();
-    expect(errorMessage.value).toBeNull();
-    expect(errorRequestId.value).toBeNull();
-    expect(successMessage.value).toBeNull();
-  });
-
-  it("assignError extracts message and requestId from ApiRequestError", () => {
-    const { errorMessage, errorRequestId, assignError } = useRequestState();
-    assignError(
-      new ApiRequestError({ status: 409, code: "conflict", message: "Conflict occurred", requestId: "req-42" }),
-      "fallback",
-    );
-    expect(errorMessage.value).toBe("Conflict occurred");
-    expect(errorRequestId.value).toBe("req-42");
-  });
-
-  it("assignError uses Error.message for generic errors", () => {
-    const { errorMessage, errorRequestId, assignError } = useRequestState();
-    assignError(new Error("something broke"), "fallback");
-    expect(errorMessage.value).toBe("something broke");
-    expect(errorRequestId.value).toBeNull();
-  });
-
-  it("assignError uses fallback for non-Error values", () => {
-    const { errorMessage, assignError } = useRequestState();
-    assignError("not an error object", "fallback text");
-    expect(errorMessage.value).toBe("fallback text");
-  });
-
-  it("setSuccess sets the success message", () => {
-    const { successMessage, setSuccess } = useRequestState();
-    setSuccess("Operation completed");
-    expect(successMessage.value).toBe("Operation completed");
-  });
-
-  it("clearAlerts resets all messages", () => {
-    const { errorMessage, errorRequestId, successMessage, assignError, setSuccess, clearAlerts } = useRequestState();
-    assignError(new ApiRequestError({ status: 500, code: "err", message: "fail", requestId: "r1" }), "fb");
-    setSuccess("ok");
-    clearAlerts();
-    expect(errorMessage.value).toBeNull();
-    expect(errorRequestId.value).toBeNull();
-    expect(successMessage.value).toBeNull();
-  });
-});

frontend/src/composables/useRequestState.ts

@@ -1,40 +0,0 @@
-import { ref } from "vue";
-import { ApiRequestError } from "@/lib/api/errors";
-
-export function useRequestState() {
-  const errorMessage = ref<string | null>(null);
-  const errorRequestId = ref<string | null>(null);
-  const successMessage = ref<string | null>(null);
-
-  function clearAlerts(): void {
-    errorMessage.value = null;
-    errorRequestId.value = null;
-    successMessage.value = null;
-  }
-
-  function assignError(error: unknown, fallback: string): void {
-    if (error instanceof ApiRequestError) {
-      errorMessage.value = error.message;
-      errorRequestId.value = error.requestId;
-      return;
-    }
-    if (error instanceof Error && error.message) {
-      errorMessage.value = error.message;
-      return;
-    }
-    errorMessage.value = fallback;
-  }
-
-  function setSuccess(message: string): void {
-    successMessage.value = message;
-  }
-
-  return {
-    errorMessage,
-    errorRequestId,
-    successMessage,
-    clearAlerts,
-    assignError,
-    setSuccess,
-  };
-}

frontend/src/features/publications/components/PublicationTableRow.test.ts

@@ -1,139 +0,0 @@
-// @vitest-environment happy-dom
-import { describe, expect, it } from "vitest";
-import { mount } from "@vue/test-utils";
-import PublicationTableRow from "./PublicationTableRow.vue";
-import type { PublicationItem } from "@/features/publications";
-
-function buildItem(overrides: Partial<PublicationItem> = {}): PublicationItem {
-  return {
-    publication_id: 1,
-    scholar_profile_id: 10,
-    title: "Test Publication",
-    year: 2025,
-    citation_count: 42,
-    pub_url: "https://example.com/pub",
-    pdf_url: null,
-    pdf_status: "untracked",
-    is_read: false,
-    is_favorite: false,
-    is_new_in_latest_run: true,
-    scholar_label: "Dr. Test",
-    first_seen_at: "2025-01-15T00:00:00Z",
-    display_identifier: null,
-    ...overrides,
-  } as PublicationItem;
-}
-
-const defaultProps = {
-  item: buildItem(),
-  itemKey: "pub-1",
-  selected: false,
-  favoriteUpdating: false,
-  retrying: false,
-  canRetry: false,
-};
-
-describe("PublicationTableRow", () => {
-  it("renders the publication title", () => {
-    const wrapper = mount(PublicationTableRow, { props: defaultProps });
-    expect(wrapper.text()).toContain("Test Publication");
-  });
-
-  it("renders scholar label", () => {
-    const wrapper = mount(PublicationTableRow, { props: defaultProps });
-    expect(wrapper.text()).toContain("Dr. Test");
-  });
-
-  it("renders year and citation count", () => {
-    const wrapper = mount(PublicationTableRow, { props: defaultProps });
-    expect(wrapper.text()).toContain("2025");
-    expect(wrapper.text()).toContain("42");
-  });
-
-  it("shows New badge when is_new_in_latest_run is true", () => {
-    const wrapper = mount(PublicationTableRow, { props: defaultProps });
-    expect(wrapper.text()).toContain("New");
-  });
-
-  it("shows Seen badge when is_new_in_latest_run is false", () => {
-    const wrapper = mount(PublicationTableRow, {
-      props: { ...defaultProps, item: buildItem({ is_new_in_latest_run: false }) },
-    });
-    expect(wrapper.text()).toContain("Seen");
-  });
-
-  it("shows Unread badge for unread items", () => {
-    const wrapper = mount(PublicationTableRow, { props: defaultProps });
-    expect(wrapper.text()).toContain("Unread");
-  });
-
-  it("shows Read badge for read items", () => {
-    const wrapper = mount(PublicationTableRow, {
-      props: { ...defaultProps, item: buildItem({ is_read: true }) },
-    });
-    expect(wrapper.text()).toContain("Read");
-  });
-
-  it("disables checkbox when item is read", () => {
-    const wrapper = mount(PublicationTableRow, {
-      props: { ...defaultProps, item: buildItem({ is_read: true }) },
-    });
-    const checkbox = wrapper.find('input[type="checkbox"]');
-    expect((checkbox.element as HTMLInputElement).disabled).toBe(true);
-  });
-
-  it("emits toggle-selection when checkbox is changed", async () => {
-    const wrapper = mount(PublicationTableRow, { props: defaultProps });
-    await wrapper.find('input[type="checkbox"]').trigger("change");
-    expect(wrapper.emitted("toggle-selection")).toHaveLength(1);
-  });
-
-  it("emits toggle-favorite when star button is clicked", async () => {
-    const wrapper = mount(PublicationTableRow, { props: defaultProps });
-    await wrapper.find(".favorite-star-button").trigger("click");
-    expect(wrapper.emitted("toggle-favorite")).toHaveLength(1);
-  });
-
-  it("shows filled star when item is favorite", () => {
-    const wrapper = mount(PublicationTableRow, {
-      props: { ...defaultProps, item: buildItem({ is_favorite: true }) },
-    });
-    expect(wrapper.find(".favorite-star-on").exists()).toBe(true);
-    expect(wrapper.text()).toContain("★");
-  });
-
-  it("shows empty star when item is not favorite", () => {
-    const wrapper = mount(PublicationTableRow, { props: defaultProps });
-    expect(wrapper.find(".favorite-star-off").exists()).toBe(true);
-    expect(wrapper.text()).toContain("☆");
-  });
-
-  it("shows Available link when pdf_url is set", () => {
-    const wrapper = mount(PublicationTableRow, {
-      props: { ...defaultProps, item: buildItem({ pdf_url: "https://example.com/paper.pdf" }) },
-    });
-    expect(wrapper.text()).toContain("Available");
-  });
-
-  it("shows retry button when canRetry is true and no pdf_url", () => {
-    const wrapper = mount(PublicationTableRow, {
-      props: { ...defaultProps, canRetry: true },
-    });
-    expect(wrapper.text()).toContain("Missing (Retry)");
-  });
-
-  it("emits retry-pdf when retry button is clicked", async () => {
-    const wrapper = mount(PublicationTableRow, {
-      props: { ...defaultProps, canRetry: true },
-    });
-    await wrapper.find(".pdf-retry-button").trigger("click");
-    expect(wrapper.emitted("retry-pdf")).toHaveLength(1);
-  });
-
-  it("shows Retrying... label when retrying", () => {
-    const wrapper = mount(PublicationTableRow, {
-      props: { ...defaultProps, canRetry: true, retrying: true },
-    });
-    expect(wrapper.text()).toContain("Retrying...");
-  });
-});

frontend/src/features/publications/components/PublicationTableRow.vue

@@ -1,178 +0,0 @@
-<script setup lang="ts">
-import AppBadge from "@/components/ui/AppBadge.vue";
-import type { PublicationItem } from "@/features/publications";
-
-const props = defineProps<{
-  item: PublicationItem;
-  itemKey: string;
-  selected: boolean;
-  favoriteUpdating: boolean;
-  retrying: boolean;
-  canRetry: boolean;
-}>();
-
-const emit = defineEmits<{
-  (e: "toggle-selection", event: Event): void;
-  (e: "toggle-favorite"): void;
-  (e: "retry-pdf"): void;
-}>();
-
-function primaryUrl(): string | null {
-  return props.item.pub_url || props.item.pdf_url;
-}
-
-function identifierUrl(): string | null {
-  return props.item.display_identifier?.url ?? null;
-}
-
-function identifierLabel(): string | null {
-  return props.item.display_identifier?.label ?? null;
-}
-
-function pdfPendingLabel(): string {
-  if (props.item.pdf_status === "queued") return "Queued";
-  if (props.item.pdf_status === "running") return "Resolving...";
-  if (props.item.pdf_status === "failed") return "Missing";
-  return "Untracked";
-}
-
-function formatDate(value: string): string {
-  const asDate = new Date(value);
-  if (Number.isNaN(asDate.getTime())) return value;
-  return asDate.toLocaleDateString();
-}
-</script>
-
-<template>
-  <tr>
-    <td>
-      <input
-        type="checkbox"
-        class="h-4 w-4 rounded border-stroke-interactive bg-surface-input text-brand-600 focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset"
-        :checked="selected"
-        :disabled="item.is_read"
-        :aria-label="`Select publication ${item.title}`"
-        @change="emit('toggle-selection', $event)"
-      />
-    </td>
-    <td>
-      <button
-        type="button"
-        class="favorite-star-button"
-        :class="item.is_favorite ? 'favorite-star-on' : 'favorite-star-off'"
-        :aria-label="item.is_favorite ? `Remove ${item.title} from favorites` : `Add ${item.title} to favorites`"
-        :aria-pressed="item.is_favorite"
-        :disabled="favoriteUpdating"
-        @click="emit('toggle-favorite')"
-      >
-        {{ item.is_favorite ? "★" : "☆" }}
-      </button>
-    </td>
-    <td class="max-w-0">
-      <div class="grid min-w-0 gap-1">
-        <a
-          v-if="primaryUrl()"
-          :href="primaryUrl() || ''"
-          target="_blank"
-          rel="noreferrer"
-          class="link-inline block truncate font-medium"
-          :title="item.title"
-        >
-          {{ item.title }}
-        </a>
-        <span v-else class="block truncate font-medium" :title="item.title">{{ item.title }}</span>
-        <a
-          v-if="identifierUrl()"
-          :href="identifierUrl() || ''"
-          target="_blank"
-          rel="noreferrer"
-          class="link-inline block truncate text-xs"
-          :title="identifierLabel() || ''"
-        >
-          {{ identifierLabel() }}
-        </a>
-      </div>
-    </td>
-    <td>
-      <span class="block truncate" :title="item.scholar_label">{{ item.scholar_label }}</span>
-    </td>
-    <td class="whitespace-nowrap">
-      <a
-        v-if="item.pdf_url"
-        :href="item.pdf_url"
-        target="_blank"
-        rel="noreferrer"
-        class="pdf-link-button"
-        title="Open PDF"
-      >
-        <svg class="mr-1 h-3 w-3" fill="none" viewBox="0 0 24 24" stroke="currentColor">
-          <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14" />
-        </svg>
-        Available
-      </a>
-      <button
-        v-else-if="canRetry"
-        type="button"
-        class="pdf-retry-button"
-        :disabled="retrying"
-        @click="emit('retry-pdf')"
-      >
-        <svg v-if="retrying" class="mr-1 h-3 w-3 animate-spin" viewBox="0 0 24 24" fill="none">
-          <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
-          <path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
-        </svg>
-        {{ retrying ? "Retrying..." : "Missing (Retry)" }}
-      </button>
-      <span v-else class="pdf-state-label" :class="{ 'bg-surface-accent-muted border-accent-300 text-accent-700': item.pdf_status === 'running' || item.pdf_status === 'queued' }">
-        <svg v-if="item.pdf_status === 'running'" class="mr-1 h-3 w-3 animate-spin text-accent-600" viewBox="0 0 24 24" fill="none">
-          <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
-          <path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
-        </svg>
-        {{ pdfPendingLabel() }}
-      </span>
-    </td>
-    <td class="whitespace-nowrap">{{ item.year ?? "n/a" }}</td>
-    <td class="whitespace-nowrap">{{ item.citation_count }}</td>
-    <td>
-      <div class="status-badges-row">
-        <AppBadge :tone="item.is_new_in_latest_run ? 'info' : 'neutral'">
-          {{ item.is_new_in_latest_run ? "New" : "Seen" }}
-        </AppBadge>
-        <AppBadge :tone="item.is_read ? 'success' : 'warning'">
-          {{ item.is_read ? "Read" : "Unread" }}
-        </AppBadge>
-      </div>
-    </td>
-    <td class="whitespace-nowrap">{{ formatDate(item.first_seen_at) }}</td>
-  </tr>
-</template>
-
-<style scoped>
-.favorite-star-button {
-  @apply inline-flex h-7 w-7 items-center justify-center rounded-full border text-sm leading-none transition focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset disabled:cursor-not-allowed disabled:opacity-50;
-}
-
-.favorite-star-on {
-  @apply border-warning-300 bg-warning-100 text-warning-700 hover:bg-warning-200;
-}
-
-.favorite-star-off {
-  @apply border-stroke-default bg-surface-card-muted text-ink-muted hover:border-stroke-interactive hover:text-ink-secondary;
-}
-
-.pdf-link-button {
-  @apply inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-medium border-state-success-border bg-state-success-bg text-state-success-text shadow-sm transition hover:brightness-95 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset;
-}
-
-.pdf-retry-button {
-  @apply inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-medium border-state-warning-border bg-state-warning-bg text-state-warning-text transition hover:brightness-95 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset disabled:cursor-not-allowed disabled:opacity-50;
-}
-
-.pdf-state-label {
-  @apply inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-medium border-stroke-default bg-surface-card-muted text-secondary;
-}
-
-.status-badges-row {
-  @apply inline-flex items-center gap-1 whitespace-nowrap;
-}
-</style>

frontend/src/features/publications/components/PublicationToolbar.test.ts

@@ -1,89 +0,0 @@
-// @vitest-environment happy-dom
-import { describe, expect, it } from "vitest";
-import { mount } from "@vue/test-utils";
-import PublicationToolbar from "./PublicationToolbar.vue";
-
-const defaultProps = {
-  mode: "all" as const,
-  selectedScholarFilter: "",
-  searchQuery: "",
-  favoriteOnly: false,
-  actionBusy: false,
-  loading: false,
-  scholars: [],
-  startRunDisabled: false,
-  startRunDisabledReason: null,
-  startRunButtonLabel: "Check now",
-};
-
-describe("PublicationToolbar", () => {
-  it("renders mode, scholar filter, and search inputs", () => {
-    const wrapper = mount(PublicationToolbar, { props: defaultProps });
-    expect(wrapper.text()).toContain("Status");
-    expect(wrapper.text()).toContain("Scholar");
-    expect(wrapper.text()).toContain("Search");
-  });
-
-  it("renders the start run button with provided label", () => {
-    const wrapper = mount(PublicationToolbar, { props: defaultProps });
-    expect(wrapper.text()).toContain("Check now");
-  });
-
-  it("disables start run button when startRunDisabled is true", () => {
-    const wrapper = mount(PublicationToolbar, {
-      props: { ...defaultProps, startRunDisabled: true, startRunDisabledReason: "Cooldown active" },
-    });
-    const buttons = wrapper.findAll("button");
-    const runButton = buttons.find((b) => b.text().includes("Check now"));
-    expect(runButton?.attributes("disabled")).toBeDefined();
-  });
-
-  it("emits start-run when start button is clicked", async () => {
-    const wrapper = mount(PublicationToolbar, { props: defaultProps });
-    const buttons = wrapper.findAll("button");
-    const runButton = buttons.find((b) => b.text().includes("Check now"));
-    await runButton?.trigger("click");
-    expect(wrapper.emitted("start-run")).toHaveLength(1);
-  });
-
-  it("emits favorite-only-changed when star filter is clicked", async () => {
-    const wrapper = mount(PublicationToolbar, { props: defaultProps });
-    await wrapper.find(".favorite-filter-button").trigger("click");
-    expect(wrapper.emitted("favorite-only-changed")).toHaveLength(1);
-  });
-
-  it("shows active star filter style when favoriteOnly is true", () => {
-    const wrapper = mount(PublicationToolbar, { props: { ...defaultProps, favoriteOnly: true } });
-    expect(wrapper.find(".favorite-filter-on").exists()).toBe(true);
-  });
-
-  it("shows inactive star filter style when favoriteOnly is false", () => {
-    const wrapper = mount(PublicationToolbar, { props: defaultProps });
-    expect(wrapper.find(".favorite-filter-off").exists()).toBe(true);
-  });
-
-  it("renders scholar options from props", () => {
-    const scholars = [
-      { id: 1, scholar_id: "abc123", display_name: "Alice", profile_image_url: null, profile_image_source: "none" as const, is_enabled: true, baseline_completed: true, last_run_dt: null, last_run_status: null },
-      { id: 2, scholar_id: "def456", display_name: "", profile_image_url: null, profile_image_source: "none" as const, is_enabled: true, baseline_completed: true, last_run_dt: null, last_run_status: null },
-    ];
-    const wrapper = mount(PublicationToolbar, { props: { ...defaultProps, scholars } });
-    expect(wrapper.text()).toContain("Alice");
-    expect(wrapper.text()).toContain("def456");
-  });
-
-  it("shows clear button only when search query is non-empty", () => {
-    const empty = mount(PublicationToolbar, { props: defaultProps });
-    expect(empty.text()).not.toContain("Clear");
-
-    const withQuery = mount(PublicationToolbar, { props: { ...defaultProps, searchQuery: "test" } });
-    expect(withQuery.text()).toContain("Clear");
-  });
-
-  it("emits reset-search when clear button is clicked", async () => {
-    const wrapper = mount(PublicationToolbar, { props: { ...defaultProps, searchQuery: "test" } });
-    const clearButton = wrapper.findAll("button").find((b) => b.text().includes("Clear"));
-    await clearButton?.trigger("click");
-    expect(wrapper.emitted("reset-search")).toHaveLength(1);
-  });
-});

frontend/src/features/publications/components/PublicationToolbar.vue

@@ -1,148 +0,0 @@
-<script setup lang="ts">
-import AppButton from "@/components/ui/AppButton.vue";
-import AppHelpHint from "@/components/ui/AppHelpHint.vue";
-import AppInput from "@/components/ui/AppInput.vue";
-import AppRefreshButton from "@/components/ui/AppRefreshButton.vue";
-import AppSelect from "@/components/ui/AppSelect.vue";
-import type { PublicationMode } from "@/features/publications";
-import type { ScholarProfile } from "@/features/scholars";
-
-defineProps<{
-  mode: PublicationMode;
-  selectedScholarFilter: string;
-  searchQuery: string;
-  favoriteOnly: boolean;
-  actionBusy: boolean;
-  loading: boolean;
-  scholars: ScholarProfile[];
-  startRunDisabled: boolean;
-  startRunDisabledReason: string | null;
-  startRunButtonLabel: string;
-}>();
-
-const emit = defineEmits<{
-  (e: "update:mode", value: PublicationMode): void;
-  (e: "update:selectedScholarFilter", value: string): void;
-  (e: "update:searchQuery", value: string): void;
-  (e: "mode-changed"): void;
-  (e: "scholar-filter-changed"): void;
-  (e: "favorite-only-changed"): void;
-  (e: "reset-search"): void;
-  (e: "start-run"): void;
-  (e: "refresh"): void;
-}>();
-
-function scholarLabel(item: ScholarProfile): string {
-  return item.display_name || item.scholar_id;
-}
-</script>
-
-<template>
-  <div class="grid gap-3 xl:grid-cols-[minmax(0,13rem)_minmax(0,18rem)_minmax(0,1fr)_auto] xl:items-end">
-    <div class="grid gap-1 text-xs text-secondary">
-      <div class="flex items-center gap-1">
-        <span>Status</span>
-        <AppHelpHint text="All shows full history. Unread and New narrow the dataset before search." />
-      </div>
-      <AppSelect
-        :model-value="mode"
-        :disabled="actionBusy"
-        @update:model-value="emit('update:mode', $event as PublicationMode)"
-        @change="emit('mode-changed')"
-      >
-        <option value="all">All records</option>
-        <option value="unread">Unread</option>
-        <option value="latest">New (latest run)</option>
-      </AppSelect>
-    </div>
-
-    <label class="grid gap-1 text-xs text-secondary" for="publications-scholar-filter">
-      <span class="inline-flex items-center gap-1">
-        Scholar
-        <AppHelpHint text="Filter to one tracked scholar profile. Filter is synced to URL query." />
-      </span>
-      <AppSelect
-        id="publications-scholar-filter"
-        :model-value="selectedScholarFilter"
-        :disabled="actionBusy"
-        @update:model-value="emit('update:selectedScholarFilter', $event as string)"
-        @change="emit('scholar-filter-changed')"
-      >
-        <option value="">All scholars</option>
-        <option v-for="scholar in scholars" :key="scholar.id" :value="String(scholar.id)">
-          {{ scholarLabel(scholar) }}
-        </option>
-      </AppSelect>
-    </label>
-
-    <label class="grid gap-1 text-xs text-secondary" for="publications-search-input">
-      <span class="inline-flex items-center gap-1">
-        Search
-        <AppHelpHint text="Searches title, scholar name, and venue." />
-      </span>
-      <div class="flex min-w-0 items-center gap-2">
-        <AppInput
-          id="publications-search-input"
-          :model-value="searchQuery"
-          placeholder="Search title, scholar, venue, year"
-          :disabled="loading"
-          @update:model-value="emit('update:searchQuery', $event as string)"
-        />
-        <AppButton
-          v-if="searchQuery.trim().length > 0"
-          variant="secondary"
-          class="shrink-0"
-          :disabled="loading"
-          @click="emit('reset-search')"
-        >
-          Clear
-        </AppButton>
-      </div>
-    </label>
-
-    <div class="flex flex-wrap items-end justify-end gap-2">
-      <button
-        type="button"
-        class="favorite-filter-button"
-        :class="favoriteOnly ? 'favorite-filter-on' : 'favorite-filter-off'"
-        :disabled="actionBusy"
-        :title="favoriteOnly ? 'Favorites-only filter is active' : 'Show only favorites'"
-        :aria-pressed="favoriteOnly"
-        :aria-label="favoriteOnly ? 'Disable favorites-only filter' : 'Enable favorites-only filter'"
-        @click="emit('favorite-only-changed')"
-      >
-        <span aria-hidden="true">{{ favoriteOnly ? "★" : "☆" }}</span>
-      </button>
-      <AppButton
-        variant="secondary"
-        :disabled="startRunDisabled"
-        :title="startRunDisabledReason || undefined"
-        @click="emit('start-run')"
-      >
-        {{ startRunButtonLabel }}
-      </AppButton>
-      <AppRefreshButton
-        variant="ghost"
-        :disabled="loading"
-        :loading="loading"
-        title="Refresh publications"
-        loading-title="Refreshing publications"
-        @click="emit('refresh')"
-      />
-    </div>
-  </div>
-</template>
-
-<style scoped>
-.favorite-filter-button {
-  @apply inline-flex min-h-10 h-10 w-10 items-center justify-center rounded-full border text-base leading-none transition focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset disabled:cursor-not-allowed disabled:opacity-50;
-}
-
-.favorite-filter-on {
-  @apply border-warning-300 bg-warning-100 text-warning-700 hover:bg-warning-200;
-}
-
-.favorite-filter-off {
-  @apply border-stroke-default bg-surface-card-muted text-ink-muted hover:border-stroke-interactive hover:text-ink-secondary;
-}
-</style>

frontend/src/features/publications/composables/usePublicationData.test.ts

@@ -1,118 +0,0 @@
-import { describe, expect, it, vi, beforeEach } from "vitest";
-import { createPinia, setActivePinia } from "pinia";
-
-vi.mock("vue-router", () => ({
-  useRoute: vi.fn(() => ({ query: {} })),
-  useRouter: vi.fn(() => ({ replace: vi.fn() })),
-}));
-
-vi.mock("@/features/publications", async (importOriginal) => {
-  const original = (await importOriginal()) as Record<string, unknown>;
-  return {
-    ...original,
-    listPublications: vi.fn(),
-  };
-});
-
-vi.mock("@/features/scholars", () => ({
-  listScholars: vi.fn(),
-}));
-
-import { listPublications } from "@/features/publications";
-import { listScholars } from "@/features/scholars";
-import { usePublicationData } from "./usePublicationData";
-
-const mockedListPublications = vi.mocked(listPublications);
-const mockedListScholars = vi.mocked(listScholars);
-
-describe("usePublicationData", () => {
-  beforeEach(() => {
-    setActivePinia(createPinia());
-    mockedListPublications.mockReset();
-    mockedListScholars.mockReset();
-  });
-
-  it("initializes with default filter values", () => {
-    const data = usePublicationData();
-    expect(data.mode.value).toBe("all");
-    expect(data.favoriteOnly.value).toBe(false);
-    expect(data.searchQuery.value).toBe("");
-    expect(data.selectedScholarFilter.value).toBe("");
-    expect(data.loading.value).toBe(true);
-  });
-
-  it("loadScholarFilters calls listScholars", async () => {
-    mockedListScholars.mockResolvedValueOnce([
-      { id: 1, scholar_id: "abc", display_name: "Test", profile_image_url: null, profile_image_source: "none" as const, is_enabled: true, baseline_completed: true, last_run_dt: null, last_run_status: null },
-    ]);
-    const data = usePublicationData();
-    await data.loadScholarFilters();
-    expect(mockedListScholars).toHaveBeenCalled();
-    expect(data.scholars.value).toHaveLength(1);
-  });
-
-  it("loadPublications calls listPublications with current filters", async () => {
-    mockedListPublications.mockResolvedValueOnce({
-      publications: [],
-      mode: "all" as const,
-      favorite_only: false,
-      selected_scholar_profile_id: null,
-      unread_count: 0,
-      favorites_count: 0,
-      latest_count: 0,
-      new_count: 0,
-      total_count: 0,
-      page: 1,
-      page_size: 25,
-      snapshot: "",
-      has_next: false,
-      has_prev: false,
-    });
-    const data = usePublicationData();
-    await data.loadPublications();
-    expect(mockedListPublications).toHaveBeenCalled();
-  });
-
-  it("resetPageAndSnapshot resets page to 1", () => {
-    const data = usePublicationData();
-    data.currentPage.value = 5;
-    data.resetPageAndSnapshot();
-    expect(data.currentPage.value).toBe(1);
-  });
-
-  it("toggleSort reverses direction for same key", async () => {
-    mockedListPublications.mockResolvedValue({
-      publications: [],
-      mode: "all" as const,
-      favorite_only: false,
-      selected_scholar_profile_id: null,
-      unread_count: 0,
-      favorites_count: 0,
-      latest_count: 0,
-      new_count: 0,
-      total_count: 0,
-      page: 1,
-      page_size: 25,
-      snapshot: "",
-      has_next: false,
-      has_prev: false,
-    });
-    const data = usePublicationData();
-    const initialDirection = data.sortDirection.value;
-    await data.toggleSort(data.sortKey.value);
-    expect(data.sortDirection.value).toBe(initialDirection === "asc" ? "desc" : "asc");
-  });
-
-  it("sortMarker returns arrow indicators", () => {
-    const data = usePublicationData();
-    expect(data.sortMarker(data.sortKey.value)).toMatch(/[↑↓]/);
-    expect(data.sortMarker("year" as never)).toBe("↕");
-  });
-
-  it("computed pagination properties work correctly", () => {
-    const data = usePublicationData();
-    expect(data.hasPrevPage.value).toBe(false);
-    expect(data.totalPages.value).toBe(1);
-    expect(data.totalCount.value).toBe(0);
-  });
-});

frontend/src/features/publications/composables/usePublicationData.ts

@@ -1,346 +0,0 @@
-import { computed, onScopeDispose, ref, watch } from "vue";
-import { useRoute, useRouter } from "vue-router";
-import {
-  listPublications,
-  type PublicationItem,
-  type PublicationMode,
-  type PublicationsResult,
-} from "@/features/publications";
-import { listScholars, type ScholarProfile } from "@/features/scholars";
-import { ApiRequestError } from "@/lib/api/errors";
-import { useRunStatusStore } from "@/stores/run_status";
-
-export type PublicationSortKey =
-  | "title"
-  | "scholar"
-  | "year"
-  | "citations"
-  | "first_seen"
-  | "pdf_status";
-
-export function publicationKey(item: PublicationItem): string {
-  return `${item.scholar_profile_id}:${item.publication_id}`;
-}
-
-export function usePublicationData() {
-  const loading = ref(true);
-  const mode = ref<PublicationMode>("all");
-  const favoriteOnly = ref(false);
-  const selectedScholarFilter = ref("");
-  const searchQuery = ref("");
-  const sortKey = ref<PublicationSortKey>("first_seen");
-  const sortDirection = ref<"asc" | "desc">("desc");
-  const currentPage = ref(1);
-  const pageSize = ref("50");
-  const publicationSnapshot = ref<string | null>(null);
-
-  const scholars = ref<ScholarProfile[]>([]);
-  const listState = ref<PublicationsResult | null>(null);
-
-  const errorMessage = ref<string | null>(null);
-  const errorRequestId = ref<string | null>(null);
-  const successMessage = ref<string | null>(null);
-  const route = useRoute();
-  const router = useRouter();
-  const textCollator = new Intl.Collator(undefined, { sensitivity: "base", numeric: true });
-  const runStatus = useRunStatusStore();
-
-  // --- Route sync ---
-
-  function normalizeScholarFilterQuery(value: unknown): string {
-    if (Array.isArray(value)) return normalizeScholarFilterQuery(value[0]);
-    if (typeof value !== "string") return "";
-    const parsed = Number(value);
-    return Number.isInteger(parsed) && parsed > 0 ? String(parsed) : "";
-  }
-
-  function normalizeFavoriteOnlyQuery(value: unknown): boolean {
-    if (Array.isArray(value)) return normalizeFavoriteOnlyQuery(value[0]);
-    if (typeof value !== "string") return false;
-    const normalized = value.trim().toLowerCase();
-    return normalized === "1" || normalized === "true" || normalized === "yes";
-  }
-
-  function normalizePageQuery(value: unknown): number {
-    if (Array.isArray(value)) return normalizePageQuery(value[0]);
-    if (typeof value !== "string") return 1;
-    const parsed = Number(value);
-    return Number.isInteger(parsed) && parsed > 0 ? parsed : 1;
-  }
-
-  function syncFiltersFromRoute(): boolean {
-    let changed = false;
-    const nextScholar = normalizeScholarFilterQuery(route.query.scholar);
-    const nextFavoriteOnly = normalizeFavoriteOnlyQuery(route.query.favorite);
-    const nextPage = normalizePageQuery(route.query.page);
-    if (selectedScholarFilter.value !== nextScholar) { selectedScholarFilter.value = nextScholar; changed = true; }
-    if (favoriteOnly.value !== nextFavoriteOnly) { favoriteOnly.value = nextFavoriteOnly; changed = true; }
-    if (currentPage.value !== nextPage) { currentPage.value = nextPage; changed = true; }
-    return changed;
-  }
-
-  async function syncFiltersToRoute(): Promise<void> {
-    const nextScholar = selectedScholarFilter.value.trim();
-    const currentScholar = normalizeScholarFilterQuery(route.query.scholar);
-    const currentFavoriteOnly = normalizeFavoriteOnlyQuery(route.query.favorite);
-    const currentPageQuery = normalizePageQuery(route.query.page);
-    if (
-      nextScholar === currentScholar
-      && favoriteOnly.value === currentFavoriteOnly
-      && currentPage.value === currentPageQuery
-    ) {
-      return;
-    }
-    await router.replace({
-      query: {
-        ...route.query,
-        scholar: nextScholar || undefined,
-        favorite: favoriteOnly.value ? "1" : undefined,
-        page: currentPage.value > 1 ? String(currentPage.value) : undefined,
-      },
-    });
-  }
-
-  // --- Sorting helpers ---
-
-  function publicationSortValue(item: PublicationItem, key: PublicationSortKey): number | string {
-    if (key === "title") return item.title;
-    if (key === "scholar") return item.scholar_label;
-    if (key === "year") return item.year ?? -1;
-    if (key === "citations") return item.citation_count;
-    if (key === "pdf_status") {
-      if (item.pdf_url || item.pdf_status === "resolved") return 4;
-      if (item.pdf_status === "running") return 3;
-      if (item.pdf_status === "queued") return 2;
-      if (item.pdf_status === "failed") return 0;
-      return 1;
-    }
-    const timestamp = Date.parse(item.first_seen_at);
-    return Number.isNaN(timestamp) ? 0 : timestamp;
-  }
-
-  // --- Computed ---
-
-  const selectedScholarName = computed(() => {
-    const selectedId = Number(selectedScholarFilter.value);
-    if (!Number.isInteger(selectedId) || selectedId <= 0) return "all scholars";
-    const profile = scholars.value.find((item) => item.id === selectedId);
-    return profile ? (profile.display_name || profile.scholar_id) : "the selected scholar";
-  });
-
-  const filteredPublications = computed(() => {
-    let stream = [...runStatus.livePublications];
-    if (favoriteOnly.value) stream = stream.filter((p) => p.is_favorite);
-    if (mode.value === "unread") stream = stream.filter((p) => !p.is_read);
-    const selectedScholarId = Number(selectedScholarFilter.value);
-    if (Number.isInteger(selectedScholarId) && selectedScholarId > 0) {
-      stream = stream.filter((p) => p.scholar_profile_id === selectedScholarId);
-    }
-    const base = listState.value?.publications ?? [];
-    const merged = [...stream, ...base];
-    const seenKeys = new Set<string>();
-    const deduped: typeof base = [];
-    for (const item of merged) {
-      const key = publicationKey(item);
-      if (!seenKeys.has(key)) { seenKeys.add(key); deduped.push(item); }
-    }
-    return deduped;
-  });
-
-  const sortedPublications = computed(() => {
-    const direction = sortDirection.value === "asc" ? 1 : -1;
-    return [...filteredPublications.value].sort((left, right) => {
-      const leftValue = publicationSortValue(left, sortKey.value);
-      const rightValue = publicationSortValue(right, sortKey.value);
-      let comparison = 0;
-      if (typeof leftValue === "string" && typeof rightValue === "string") {
-        comparison = textCollator.compare(leftValue, rightValue);
-      } else {
-        comparison = Number(leftValue) - Number(rightValue);
-      }
-      if (comparison !== 0) return comparison * direction;
-      return right.publication_id - left.publication_id;
-    });
-  });
-
-  const visibleUnreadKeys = computed(() => {
-    const keys = new Set<string>();
-    for (const item of sortedPublications.value) {
-      if (!item.is_read) keys.add(publicationKey(item));
-    }
-    return keys;
-  });
-
-  const pageSizeValue = computed(() => {
-    const parsed = Number(pageSize.value);
-    if (!Number.isInteger(parsed)) return 50;
-    return Math.max(10, Math.min(200, parsed));
-  });
-  const hasNextPage = computed(() => Boolean(listState.value?.has_next));
-  const hasPrevPage = computed(() => Boolean(listState.value?.has_prev));
-  const totalPages = computed(() => {
-    if (!listState.value) return 1;
-    return Math.max(1, Math.ceil(listState.value.total_count / Math.max(listState.value.page_size, 1)));
-  });
-  const totalCount = computed(() => listState.value?.total_count ?? 0);
-  const visibleCount = computed(() => sortedPublications.value.length);
-  const visibleUnreadCount = computed(() => visibleUnreadKeys.value.size);
-  const visibleFavoriteCount = computed(
-    () => sortedPublications.value.filter((item) => item.is_favorite).length,
-  );
-
-  // --- Data loading ---
-
-  async function loadScholarFilters(): Promise<void> {
-    try { scholars.value = await listScholars(); } catch { scholars.value = []; }
-  }
-
-  function selectedScholarId(): number | undefined {
-    const parsed = Number(selectedScholarFilter.value);
-    return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
-  }
-
-  async function loadPublications(): Promise<void> {
-    loading.value = true;
-    errorMessage.value = null;
-    errorRequestId.value = null;
-    try {
-      listState.value = await listPublications({
-        mode: mode.value,
-        favoriteOnly: favoriteOnly.value,
-        scholarProfileId: selectedScholarId(),
-        search: searchQuery.value.trim() || undefined,
-        sortBy: sortKey.value,
-        sortDir: sortDirection.value,
-        page: currentPage.value,
-        pageSize: pageSizeValue.value,
-        snapshot: publicationSnapshot.value ?? undefined,
-      });
-      publicationSnapshot.value = listState.value.snapshot;
-      currentPage.value = listState.value.page;
-      pageSize.value = String(listState.value.page_size);
-    } catch (error) {
-      listState.value = null;
-      if (error instanceof ApiRequestError) {
-        errorMessage.value = error.message;
-        errorRequestId.value = error.requestId;
-      } else {
-        errorMessage.value = "Unable to load publications.";
-      }
-    } finally {
-      loading.value = false;
-    }
-  }
-
-  function resetPageAndSnapshot(): void {
-    currentPage.value = 1;
-    publicationSnapshot.value = null;
-  }
-
-  async function toggleSort(nextKey: PublicationSortKey): Promise<void> {
-    if (sortKey.value === nextKey) {
-      sortDirection.value = sortDirection.value === "asc" ? "desc" : "asc";
-    } else {
-      sortKey.value = nextKey;
-      sortDirection.value = nextKey === "first_seen" || nextKey === "pdf_status" ? "desc" : "asc";
-    }
-    resetPageAndSnapshot();
-    await loadPublications();
-  }
-
-  function sortMarker(key: PublicationSortKey): string {
-    if (sortKey.value !== key) return "↕";
-    return sortDirection.value === "asc" ? "↑" : "↓";
-  }
-
-  function replacePublication(updated: PublicationItem): void {
-    if (!listState.value) return;
-    listState.value.publications = listState.value.publications.map((item) =>
-      publicationKey(item) !== publicationKey(updated) ? item : updated,
-    );
-  }
-
-  // --- Search debounce watcher setup ---
-
-  let searchDebounceTimer: ReturnType<typeof setTimeout> | null = null;
-  watch(searchQuery, () => {
-    if (searchDebounceTimer !== null) clearTimeout(searchDebounceTimer);
-    searchDebounceTimer = setTimeout(() => {
-      resetPageAndSnapshot();
-      void loadPublications();
-    }, 300);
-  });
-
-  onScopeDispose(() => {
-    if (searchDebounceTimer !== null) clearTimeout(searchDebounceTimer);
-  });
-
-  // --- Run-triggered refresh watcher ---
-
-  let previousRunStatusKey: string | null = null;
-
-  watch(
-    () => runStatus.latestRun,
-    async (nextRun) => {
-      const nextStatus = nextRun ? `${nextRun.id}:${nextRun.status}` : null;
-      if (nextStatus === previousRunStatusKey) return;
-      previousRunStatusKey = nextStatus;
-      const isActive = nextRun && (nextRun.status === "running" || nextRun.status === "resolving");
-      if (!isActive) return;
-      resetPageAndSnapshot();
-      await loadPublications();
-    },
-  );
-
-  // --- Route watcher ---
-
-  watch(
-    () => [route.query.scholar, route.query.favorite, route.query.page],
-    async () => {
-      const previousScholar = selectedScholarFilter.value;
-      const previousFavorite = favoriteOnly.value;
-      const changed = syncFiltersFromRoute();
-      if (!changed) return;
-      if (selectedScholarFilter.value !== previousScholar || favoriteOnly.value !== previousFavorite) {
-        publicationSnapshot.value = null;
-      }
-      await loadPublications();
-    },
-  );
-
-  return {
-    loading,
-    mode,
-    favoriteOnly,
-    selectedScholarFilter,
-    searchQuery,
-    sortKey,
-    sortDirection,
-    currentPage,
-    pageSize,
-    scholars,
-    listState,
-    errorMessage,
-    errorRequestId,
-    successMessage,
-    selectedScholarName,
-    sortedPublications,
-    visibleUnreadKeys,
-    pageSizeValue,
-    hasNextPage,
-    hasPrevPage,
-    totalPages,
-    totalCount,
-    visibleCount,
-    visibleUnreadCount,
-    visibleFavoriteCount,
-    loadScholarFilters,
-    loadPublications,
-    resetPageAndSnapshot,
-    toggleSort,
-    sortMarker,
-    replacePublication,
-    syncFiltersFromRoute,
-    syncFiltersToRoute,
-  };
-}

frontend/src/features/scholars/components/ScholarBatchAdd.test.ts

@@ -1,224 +0,0 @@
-// @vitest-environment happy-dom
-import { describe, expect, it } from "vitest";
-import { mount } from "@vue/test-utils";
-import ScholarBatchAdd from "./ScholarBatchAdd.vue";
-import {
-  parseScholarIds,
-  parseScholarTokens,
-  extractScholarIdFromUrl,
-  validateTokenAsId,
-} from "./scholar-batch-parsing";
-
-const defaultProps = { saving: false, loading: false };
-
-describe("parseScholarIds (unit)", () => {
-  it("parses a single bare ID", () => {
-    expect(parseScholarIds("A-UbBTPM15wL")).toEqual(["A-UbBTPM15wL"]);
-  });
-
-  it("parses multiple IDs separated by commas", () => {
-    expect(parseScholarIds("A-UbBTPM15wL, B-UbBTPM15wL")).toEqual(["A-UbBTPM15wL", "B-UbBTPM15wL"]);
-  });
-
-  it("deduplicates IDs", () => {
-    expect(parseScholarIds("A-UbBTPM15wL\nA-UbBTPM15wL")).toEqual(["A-UbBTPM15wL"]);
-  });
-
-  it("extracts ID from standard Google Scholar URL", () => {
-    expect(parseScholarIds("https://scholar.google.com/citations?hl=en&user=A-UbBTPM15wL")).toEqual(["A-UbBTPM15wL"]);
-  });
-
-  it("extracts ID from URL with trailing slash", () => {
-    expect(parseScholarIds("https://scholar.google.com/citations?user=A-UbBTPM15wL/")).toEqual(["A-UbBTPM15wL"]);
-  });
-
-  it("extracts ID from URL with fragment", () => {
-    expect(parseScholarIds("https://scholar.google.com/citations?user=A-UbBTPM15wL#section")).toEqual(["A-UbBTPM15wL"]);
-  });
-
-  it("extracts ID from URL with extra query params", () => {
-    expect(parseScholarIds("https://scholar.google.com/citations?hl=en&user=A-UbBTPM15wL&view_op=list_works&sortby=pubdate")).toEqual(["A-UbBTPM15wL"]);
-  });
-
-  it("handles URL-encoded characters in non-ID parts", () => {
-    expect(parseScholarIds("https://scholar.google.com/citations?hl=en&user=A-UbBTPM15wL&label=%E4%B8%AD%E6%96%87")).toEqual(["A-UbBTPM15wL"]);
-  });
-
-  it("rejects malformed IDs (too short)", () => {
-    expect(parseScholarIds("ABC123")).toEqual([]);
-  });
-
-  it("rejects malformed IDs (too long)", () => {
-    expect(parseScholarIds("ABCDEF1234567")).toEqual([]);
-  });
-
-  it("rejects IDs with special characters", () => {
-    expect(parseScholarIds("ABCDEF12345!")).toEqual([]);
-  });
-
-  it("rejects IDs with embedded whitespace", () => {
-    expect(parseScholarIds("ABC DEF12345")).toEqual([]);
-  });
-
-  it("handles mixed valid and invalid tokens", () => {
-    expect(parseScholarIds("A-UbBTPM15wL, invalid, B-UbBTPM15wL")).toEqual(["A-UbBTPM15wL", "B-UbBTPM15wL"]);
-  });
-
-  it("returns empty array for empty string", () => {
-    expect(parseScholarIds("")).toEqual([]);
-  });
-
-  it("returns empty array for whitespace only", () => {
-    expect(parseScholarIds("   ")).toEqual([]);
-  });
-});
-
-describe("extractScholarIdFromUrl", () => {
-  it("returns null for non-URL strings", () => {
-    expect(extractScholarIdFromUrl("not-a-url")).toBeNull();
-  });
-
-  it("returns null for URL without user param", () => {
-    expect(extractScholarIdFromUrl("https://scholar.google.com/citations?hl=en")).toBeNull();
-  });
-
-  it("extracts from URL with trailing slashes", () => {
-    expect(extractScholarIdFromUrl("https://scholar.google.com/citations?user=A-UbBTPM15wL///")).toBe("A-UbBTPM15wL");
-  });
-
-  it("strips fragment before parsing", () => {
-    expect(extractScholarIdFromUrl("https://scholar.google.com/citations?user=A-UbBTPM15wL#foo")).toBe("A-UbBTPM15wL");
-  });
-
-  it("returns null when user param is invalid length", () => {
-    expect(extractScholarIdFromUrl("https://scholar.google.com/citations?user=short")).toBeNull();
-  });
-});
-
-describe("validateTokenAsId", () => {
-  it("returns null for valid 12-char ID", () => {
-    expect(validateTokenAsId("ABCDEF123456")).toBeNull();
-  });
-
-  it("detects wrong length", () => {
-    expect(validateTokenAsId("ABC")).toContain("12 characters");
-  });
-
-  it("detects invalid characters", () => {
-    expect(validateTokenAsId("ABCDEF12345!")).toContain("invalid characters");
-  });
-
-  it("detects whitespace", () => {
-    expect(validateTokenAsId("ABC DEF12345")).toContain("whitespace");
-  });
-
-  it("detects empty input", () => {
-    expect(validateTokenAsId("")).toContain("empty");
-  });
-});
-
-describe("parseScholarTokens (per-token errors)", () => {
-  it("marks invalid tokens with error messages", () => {
-    const tokens = parseScholarTokens("A-UbBTPM15wL, short, B-UbBTPM15wL");
-    expect(tokens).toHaveLength(3);
-    expect(tokens[0].id).toBe("A-UbBTPM15wL");
-    expect(tokens[0].error).toBeNull();
-    expect(tokens[1].id).toBeNull();
-    expect(tokens[1].error).toContain("12 characters");
-    expect(tokens[2].id).toBe("B-UbBTPM15wL");
-  });
-
-  it("marks duplicate tokens", () => {
-    const tokens = parseScholarTokens("A-UbBTPM15wL, A-UbBTPM15wL");
-    expect(tokens[1].error).toBe("duplicate");
-  });
-
-  it("marks bad URL tokens", () => {
-    const tokens = parseScholarTokens("https://scholar.google.com/citations?hl=en");
-    expect(tokens[0].error).toContain("could not extract");
-  });
-});
-
-describe("ScholarBatchAdd (component)", () => {
-  it("renders the heading and input", () => {
-    const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
-    expect(wrapper.text()).toContain("Add Scholar Profiles");
-    expect(wrapper.find("textarea").exists()).toBe(true);
-  });
-
-  it("shows parsed ID count as 0 for empty input", () => {
-    const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
-    expect(wrapper.text()).toContain("Parsed IDs:");
-  });
-
-  it("parses bare scholar IDs from textarea input", async () => {
-    const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
-    await wrapper.find("textarea").setValue("A-UbBTPM15wL");
-    expect(wrapper.text()).toContain("1");
-  });
-
-  it("parses scholar IDs from Google Scholar URLs", async () => {
-    const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
-    await wrapper.find("textarea").setValue(
-      "https://scholar.google.com/citations?hl=en&user=A-UbBTPM15wL",
-    );
-    expect(wrapper.text()).toContain("1");
-  });
-
-  it("deduplicates IDs from mixed input", async () => {
-    const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
-    await wrapper.find("textarea").setValue(
-      "A-UbBTPM15wL\nhttps://scholar.google.com/citations?user=A-UbBTPM15wL",
-    );
-    expect(wrapper.text()).toContain("1");
-  });
-
-  it("parses multiple IDs separated by commas", async () => {
-    const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
-    await wrapper.find("textarea").setValue("A-UbBTPM15wL, B-UbBTPM15wL");
-    expect(wrapper.text()).toContain("2");
-  });
-
-  it("emits add-scholars with parsed IDs on submit", async () => {
-    const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
-    await wrapper.find("textarea").setValue("A-UbBTPM15wL");
-    await wrapper.find("form").trigger("submit");
-    const emitted = wrapper.emitted("add-scholars");
-    expect(emitted).toHaveLength(1);
-    expect(emitted![0][0]).toEqual(["A-UbBTPM15wL"]);
-  });
-
-  it("clears textarea after successful submit", async () => {
-    const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
-    const textarea = wrapper.find("textarea");
-    await textarea.setValue("A-UbBTPM15wL");
-    await wrapper.find("form").trigger("submit");
-    expect((textarea.element as HTMLTextAreaElement).value).toBe("");
-  });
-
-  it("does not emit when input contains no valid IDs", async () => {
-    const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
-    await wrapper.find("textarea").setValue("not-a-valid-id");
-    await wrapper.find("form").trigger("submit");
-    expect(wrapper.emitted("add-scholars")).toBeUndefined();
-  });
-
-  it("shows Adding... label when saving prop is true", () => {
-    const wrapper = mount(ScholarBatchAdd, { props: { ...defaultProps, saving: true } });
-    expect(wrapper.text()).toContain("Adding...");
-  });
-
-  it("shows validation errors for invalid tokens", async () => {
-    const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
-    await wrapper.find("textarea").setValue("A-UbBTPM15wL, bad!");
-    const errors = wrapper.find("[data-testid='validation-errors']");
-    expect(errors.exists()).toBe(true);
-  });
-
-  it("shows skipped count alongside valid count", async () => {
-    const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
-    await wrapper.find("textarea").setValue("A-UbBTPM15wL, tooshort");
-    expect(wrapper.text()).toContain("1 valid");
-    expect(wrapper.text()).toContain("1 skipped");
-  });
-});

frontend/src/features/scholars/components/ScholarBatchAdd.vue

@@ -1,87 +0,0 @@
-<script setup lang="ts">
-import { computed, ref } from "vue";
-import AppButton from "@/components/ui/AppButton.vue";
-import AppCard from "@/components/ui/AppCard.vue";
-import AppHelpHint from "@/components/ui/AppHelpHint.vue";
-import { parseScholarTokens } from "./scholar-batch-parsing";
-
-defineProps<{
-  saving: boolean;
-  loading: boolean;
-}>();
-
-const emit = defineEmits<{
-  (e: "add-scholars", ids: string[]): void;
-}>();
-
-const scholarBatchInput = ref("");
-
-const parsedTokens = computed(() => parseScholarTokens(scholarBatchInput.value));
-const validIds = computed(() => parsedTokens.value.filter((t) => t.id !== null));
-const invalidTokens = computed(() => parsedTokens.value.filter((t) => t.id === null));
-const parsedBatchCount = computed(() => validIds.value.length);
-
-function onSubmit(): void {
-  const ids = validIds.value.map((t) => t.id!);
-  if (ids.length > 0) {
-    emit("add-scholars", ids);
-    scholarBatchInput.value = "";
-  }
-}
-</script>
-
-<template>
-  <AppCard class="space-y-4 xl:flex xl:min-h-0 xl:flex-col">
-    <div class="space-y-1">
-      <div class="flex items-center gap-1">
-        <h2 class="text-lg font-semibold text-ink-primary">Add Scholar Profiles</h2>
-        <AppHelpHint text="A scholar profile is a Google Scholar author page that Scholarr will monitor for publication changes." />
-      </div>
-      <p class="text-sm text-secondary">Paste one or more Scholar IDs or profile URLs and add them in one action.</p>
-    </div>
-
-    <form class="grid gap-3" @submit.prevent="onSubmit">
-      <label class="grid gap-2 text-sm font-medium text-ink-secondary" for="scholar-batch-input">
-        <span>Scholar IDs or profile URLs</span>
-        <textarea
-          id="scholar-batch-input"
-          v-model="scholarBatchInput"
-          rows="5"
-          placeholder="A-UbBTPM15wL&#10;https://scholar.google.com/citations?hl=en&user=A-UbBTPM15wL"
-          class="w-full rounded-lg border border-stroke-interactive bg-surface-input px-3 py-2 text-sm text-ink-primary outline-none ring-focus-ring transition placeholder:text-ink-muted focus-visible:ring-2 focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset disabled:cursor-not-allowed disabled:opacity-60"
-        />
-      </label>
-
-      <div v-if="parsedTokens.length > 0" class="space-y-1">
-        <p class="text-xs text-secondary">
-          <strong class="text-ink-primary">{{ parsedBatchCount }}</strong> valid ID{{ parsedBatchCount === 1 ? "" : "s" }}
-          <template v-if="invalidTokens.length > 0">
-            · <span class="text-state-warning-text">{{ invalidTokens.length }} skipped</span>
-          </template>
-        </p>
-        <ul v-if="invalidTokens.length > 0" class="space-y-0.5" data-testid="validation-errors">
-          <li
-            v-for="item in invalidTokens.slice(0, 5)"
-            :key="item.index"
-            class="text-xs text-state-warning-text"
-          >
-            #{{ item.index }}: {{ item.error }}
-            <code class="ml-1 break-all text-ink-muted">{{ item.raw.length > 60 ? item.raw.slice(0, 57) + "..." : item.raw }}</code>
-          </li>
-          <li v-if="invalidTokens.length > 5" class="text-xs text-state-warning-text">
-            +{{ invalidTokens.length - 5 }} more skipped
-          </li>
-        </ul>
-      </div>
-      <div v-else class="text-xs text-secondary">
-        Parsed IDs: <strong class="text-ink-primary">0</strong>
-      </div>
-
-      <div class="flex flex-wrap items-center justify-end gap-2">
-        <AppButton type="submit" :disabled="saving || loading || parsedBatchCount === 0">
-          {{ saving ? "Adding..." : "Add scholars" }}
-        </AppButton>
-      </div>
-    </form>
-  </AppCard>
-</template>

frontend/src/features/scholars/components/ScholarNameSearch.test.ts

@@ -1,43 +0,0 @@
-// @vitest-environment happy-dom
-import { describe, expect, it } from "vitest";
-import { mount } from "@vue/test-utils";
-import ScholarNameSearch from "./ScholarNameSearch.vue";
-
-const defaultProps = {
-  trackedScholarIds: new Set<string>(),
-  addingCandidateScholarId: null,
-};
-
-describe("ScholarNameSearch", () => {
-  it("renders the search heading", () => {
-    const wrapper = mount(ScholarNameSearch, {
-      props: defaultProps,
-      global: { stubs: { ScholarAvatar: true } },
-    });
-    expect(wrapper.text()).toContain("Search by Name");
-  });
-
-  it("shows WIP badge indicating the feature is incomplete", () => {
-    const wrapper = mount(ScholarNameSearch, {
-      props: defaultProps,
-      global: { stubs: { ScholarAvatar: true } },
-    });
-    expect(wrapper.text()).toContain("WIP");
-  });
-
-  it("renders the search input", () => {
-    const wrapper = mount(ScholarNameSearch, {
-      props: defaultProps,
-      global: { stubs: { ScholarAvatar: true } },
-    });
-    expect(wrapper.find("form").exists()).toBe(true);
-  });
-
-  it("shows warning about Google Scholar login challenges", () => {
-    const wrapper = mount(ScholarNameSearch, {
-      props: defaultProps,
-      global: { stubs: { ScholarAvatar: true } },
-    });
-    expect(wrapper.text()).toContain("login challenge");
-  });
-});

frontend/src/features/scholars/components/ScholarNameSearch.vue

@@ -1,167 +0,0 @@
-<script setup lang="ts">
-import { computed, ref } from "vue";
-import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
-import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue";
-import AppAlert from "@/components/ui/AppAlert.vue";
-import AppBadge from "@/components/ui/AppBadge.vue";
-import AppButton from "@/components/ui/AppButton.vue";
-import AppCard from "@/components/ui/AppCard.vue";
-import AppHelpHint from "@/components/ui/AppHelpHint.vue";
-import AppInput from "@/components/ui/AppInput.vue";
-import ScholarAvatar from "@/features/scholars/components/ScholarAvatar.vue";
-import type { ScholarSearchCandidate, ScholarSearchResult } from "@/features/scholars";
-
-defineProps<{
-  trackedScholarIds: Set<string>;
-  addingCandidateScholarId: string | null;
-}>();
-
-const emit = defineEmits<{
-  (e: "add-candidate", candidate: ScholarSearchCandidate): void;
-}>();
-
-const nameSearchWip = true;
-const searchingByName = ref(false);
-const searchQuery = ref("");
-const searchResult = ref<ScholarSearchResult | null>(null);
-const searchErrorMessage = ref<string | null>(null);
-const searchErrorRequestId = ref<string | null>(null);
-
-const searchHasRun = computed(() => searchResult.value !== null);
-const searchIsDegraded = computed(() => {
-  if (!searchResult.value) return false;
-  return searchResult.value.state === "blocked_or_captcha" || searchResult.value.state === "network_error";
-});
-const searchStateTone = computed(() => {
-  const result = searchResult.value;
-  if (!result) return "neutral" as const;
-  if (result.state === "ok") return "success" as const;
-  if (result.state === "blocked_or_captcha" || result.state === "network_error") return "warning" as const;
-  return "neutral" as const;
-});
-const emptySearchCandidates = computed(() => {
-  const result = searchResult.value;
-  if (!result) return false;
-  return result.candidates.length === 0;
-});
-</script>
-
-<template>
-  <AppCard class="relative space-y-4 select-none xl:flex xl:min-h-0 xl:flex-col xl:overflow-hidden">
-    <div class="flex flex-wrap items-start justify-between gap-2">
-      <div class="space-y-1">
-        <div class="flex items-center gap-1">
-          <h2 class="text-lg font-semibold text-ink-primary">Search by Name</h2>
-          <AppHelpHint text="Google Scholar now challenges this endpoint with account login and anti-bot checks, so this workflow stays disabled." />
-        </div>
-        <p class="text-sm text-secondary">
-          This helper remains paused because Google Scholar currently requires account login for reliable name search access.
-        </p>
-      </div>
-      <AppHelpHint text="Name search is kept as WIP. Use direct Scholar ID or profile URL adds for production tracking.">
-        <template #trigger>
-          <AppBadge tone="warning">WIP</AppBadge>
-        </template>
-      </AppHelpHint>
-    </div>
-
-    <form class="pointer-events-none flex flex-wrap items-center gap-2 opacity-60">
-      <div class="min-w-0 flex-1">
-        <AppInput
-          id="scholar-search-name"
-          v-model="searchQuery"
-          placeholder="e.g. Geoffrey Hinton"
-          :disabled="nameSearchWip"
-        />
-      </div>
-      <AppButton type="button" :disabled="nameSearchWip">
-        Search
-      </AppButton>
-    </form>
-    <p class="text-xs text-secondary">
-      Direct Scholar ID/URL adds above are the supported path until name search can run without login challenges.
-    </p>
-
-    <div class="min-h-0 xl:flex-1 xl:overflow-y-auto xl:pr-1">
-      <RequestStateAlerts
-        :error-message="searchErrorMessage"
-        :error-request-id="searchErrorRequestId"
-        error-title="Search request failed"
-      />
-
-      <AsyncStateGate :loading="searchingByName" :loading-lines="4" :show-empty="false">
-        <template v-if="searchResult">
-          <div class="flex flex-wrap items-center justify-between gap-2">
-            <p class="text-sm text-secondary">
-              {{ searchResult.candidates.length }} candidate{{ searchResult.candidates.length === 1 ? "" : "s" }}
-              for <strong class="text-ink-primary">{{ searchResult.query }}</strong>
-            </p>
-            <AppBadge :tone="searchStateTone">{{ searchResult.state }}</AppBadge>
-          </div>
-
-          <p v-if="searchResult.state !== 'ok' || searchResult.warnings.length > 0" class="text-xs text-secondary">
-            <span>State reason: <code>{{ searchResult.state_reason }}</code></span>
-            <span v-if="searchResult.action_hint">. {{ searchResult.action_hint }}</span>
-            <span v-if="searchResult.warnings.length > 0">. Warnings: {{ searchResult.warnings.join(", ") }}</span>
-          </p>
-
-          <AppAlert v-if="searchIsDegraded" tone="warning">
-            <template #title>Name search is degraded</template>
-            <p>This endpoint throttles aggressively to avoid blocks. Use Scholar URL/ID adds for dependable tracking.</p>
-          </AppAlert>
-
-          <AsyncStateGate
-            :loading="false"
-            :show-empty="true"
-            :empty="emptySearchCandidates"
-            empty-title="No scholar matches returned"
-            empty-body="Try another query later or add directly by Scholar URL/ID."
-          >
-            <ul class="grid gap-3">
-              <li
-                v-for="candidate in searchResult.candidates"
-                :key="candidate.scholar_id"
-                class="rounded-xl border border-stroke-default bg-surface-card-muted/70 p-3"
-              >
-                <div class="flex items-start gap-3">
-                  <ScholarAvatar
-                    size="sm"
-                    :label="candidate.display_name"
-                    :scholar-id="candidate.scholar_id"
-                    :image-url="candidate.profile_image_url"
-                  />
-                  <div class="min-w-0 flex-1 space-y-1">
-                    <div class="flex flex-wrap items-center gap-2">
-                      <strong class="text-sm text-ink-primary">{{ candidate.display_name }}</strong>
-                      <code class="text-xs text-secondary">{{ candidate.scholar_id }}</code>
-                      <AppBadge v-if="trackedScholarIds.has(candidate.scholar_id)" tone="success">Tracked</AppBadge>
-                    </div>
-                    <p class="truncate text-xs text-secondary">{{ candidate.affiliation || "No affiliation provided" }}</p>
-                    <div class="flex flex-wrap items-center gap-2 text-xs text-secondary">
-                      <span v-if="candidate.email_domain">Email: {{ candidate.email_domain }}</span>
-                      <span v-if="candidate.cited_by_count !== null">Cited by: {{ candidate.cited_by_count }}</span>
-                    </div>
-                  </div>
-                  <div class="grid shrink-0 gap-2">
-                    <AppButton
-                      :disabled="trackedScholarIds.has(candidate.scholar_id) || addingCandidateScholarId === candidate.scholar_id"
-                      @click="emit('add-candidate', candidate)"
-                    >
-                      {{ addingCandidateScholarId === candidate.scholar_id ? "Adding..." : "Add" }}
-                    </AppButton>
-                    <a :href="candidate.profile_url" target="_blank" rel="noreferrer" class="link-inline text-xs text-center">
-                      Open profile
-                    </a>
-                  </div>
-                </div>
-              </li>
-            </ul>
-          </AsyncStateGate>
-        </template>
-        <p v-else-if="!searchErrorMessage && !searchHasRun" class="text-sm text-secondary">
-          Name search remains disabled while login-gated responses are unresolved.
-        </p>
-      </AsyncStateGate>
-    </div>
-  </AppCard>
-</template>

frontend/src/features/scholars/components/ScholarSettingsModal.test.ts

@@ -1,92 +0,0 @@
-// @vitest-environment happy-dom
-import { describe, expect, it } from "vitest";
-import { mount } from "@vue/test-utils";
-import ScholarSettingsModal from "./ScholarSettingsModal.vue";
-import type { ScholarProfile } from "@/features/scholars";
-
-function buildScholar(overrides: Partial<ScholarProfile> = {}): ScholarProfile {
-  return {
-    id: 1,
-    scholar_id: "abcDEF123456",
-    display_name: "Dr. Test Scholar",
-    profile_image_url: null,
-    profile_image_source: "none" as const,
-    is_enabled: true,
-    baseline_completed: true,
-    last_run_dt: null,
-    last_run_status: null,
-    ...overrides,
-  };
-}
-
-const defaultProps = {
-  scholar: buildScholar(),
-  imageUrlDraft: "",
-  imageBusy: false,
-  imageSaving: false,
-  imageUploading: false,
-  saving: false,
-};
-
-describe("ScholarSettingsModal", () => {
-  it("renders scholar name and ID when scholar is provided", () => {
-    const wrapper = mount(ScholarSettingsModal, {
-      props: defaultProps,
-      global: { stubs: { AppModal: false, RouterLink: true, ScholarAvatar: true } },
-    });
-    expect(wrapper.text()).toContain("Dr. Test Scholar");
-    expect(wrapper.text()).toContain("abcDEF123456");
-  });
-
-  it("renders nothing when scholar is null", () => {
-    const wrapper = mount(ScholarSettingsModal, {
-      props: { ...defaultProps, scholar: null },
-      global: { stubs: { AppModal: false, RouterLink: true, ScholarAvatar: true } },
-    });
-    expect(wrapper.text()).not.toContain("Dr. Test Scholar");
-  });
-
-  it("emits close when close event fires", async () => {
-    const wrapper = mount(ScholarSettingsModal, {
-      props: defaultProps,
-      global: { stubs: { RouterLink: true, ScholarAvatar: true } },
-    });
-    const modal = wrapper.findComponent({ name: "AppModal" });
-    if (modal.exists()) {
-      modal.vm.$emit("close");
-      expect(wrapper.emitted("close")).toHaveLength(1);
-    }
-  });
-
-  it("emits delete when delete button is clicked", async () => {
-    const wrapper = mount(ScholarSettingsModal, {
-      props: defaultProps,
-      global: { stubs: { RouterLink: true, ScholarAvatar: true } },
-    });
-    const deleteButton = wrapper.findAll("button").find((b) => b.text().includes("Delete"));
-    if (deleteButton) {
-      await deleteButton.trigger("click");
-      expect(wrapper.emitted("delete")).toHaveLength(1);
-    }
-  });
-
-  it("emits toggle when enable/disable button is clicked", async () => {
-    const wrapper = mount(ScholarSettingsModal, {
-      props: defaultProps,
-      global: { stubs: { RouterLink: true, ScholarAvatar: true } },
-    });
-    const toggleButton = wrapper.findAll("button").find((b) => b.text().includes("Disable"));
-    if (toggleButton) {
-      await toggleButton.trigger("click");
-      expect(wrapper.emitted("toggle")).toHaveLength(1);
-    }
-  });
-
-  it("shows Enable button for disabled scholars", () => {
-    const wrapper = mount(ScholarSettingsModal, {
-      props: { ...defaultProps, scholar: buildScholar({ is_enabled: false }) },
-      global: { stubs: { RouterLink: true, ScholarAvatar: true } },
-    });
-    expect(wrapper.text()).toContain("Enable");
-  });
-});

frontend/src/features/scholars/components/ScholarSettingsModal.vue

@@ -1,120 +0,0 @@
-<script setup lang="ts">
-import AppButton from "@/components/ui/AppButton.vue";
-import AppInput from "@/components/ui/AppInput.vue";
-import AppModal from "@/components/ui/AppModal.vue";
-import ScholarAvatar from "@/features/scholars/components/ScholarAvatar.vue";
-import type { ScholarProfile } from "@/features/scholars";
-
-const props = defineProps<{
-  scholar: ScholarProfile | null;
-  imageUrlDraft: string;
-  imageBusy: boolean;
-  imageSaving: boolean;
-  imageUploading: boolean;
-  saving: boolean;
-}>();
-
-const emit = defineEmits<{
-  (e: "close"): void;
-  (e: "update:image-url-draft", value: string): void;
-  (e: "save-image-url"): void;
-  (e: "upload-image", event: Event): void;
-  (e: "reset-image"): void;
-  (e: "toggle"): void;
-  (e: "delete"): void;
-}>();
-
-function scholarLabel(profile: ScholarProfile): string {
-  return profile.display_name || profile.scholar_id;
-}
-
-function scholarProfileUrl(scholarId: string): string {
-  return `https://scholar.google.com/citations?hl=en&user=${encodeURIComponent(scholarId)}`;
-}
-
-function scholarPublicationsRoute(profile: ScholarProfile): { name: string; query: { scholar: string } } {
-  return { name: "publications", query: { scholar: String(profile.id) } };
-}
-</script>
-
-<template>
-  <AppModal :open="scholar !== null" title="Scholar settings" @close="emit('close')">
-    <div v-if="scholar" class="grid gap-4">
-      <div class="flex items-start gap-3">
-        <ScholarAvatar
-          :label="scholar.display_name"
-          :scholar-id="scholar.scholar_id"
-          :image-url="scholar.profile_image_url"
-        />
-        <div class="min-w-0 space-y-1">
-          <p class="truncate text-sm font-semibold text-ink-primary">
-            {{ scholarLabel(scholar) }}
-            <span
-              v-if="!scholar.baseline_completed"
-              class="ml-1.5 inline-flex items-center rounded-full border border-state-warning-border bg-state-warning-bg px-1.5 py-0.5 text-[10px] font-medium text-state-warning-text"
-              title="This scholar hasn't been scraped yet. Publications will appear after the next run."
-            >Pending</span>
-          </p>
-          <p class="text-xs text-secondary">ID: <code>{{ scholar.scholar_id }}</code></p>
-          <div class="flex flex-wrap items-center gap-3">
-            <RouterLink :to="scholarPublicationsRoute(scholar)" class="link-inline text-xs">
-              View publications
-            </RouterLink>
-            <a :href="scholarProfileUrl(scholar.scholar_id)" target="_blank" rel="noreferrer" class="link-inline text-xs">
-              Open Google Scholar profile
-            </a>
-          </div>
-        </div>
-      </div>
-
-      <div class="grid gap-2">
-        <label class="text-sm font-medium text-ink-secondary" :for="`scholar-image-url-${scholar.id}`">
-          Profile image URL override
-        </label>
-        <div class="flex flex-wrap items-center gap-2">
-          <div class="min-w-0 flex-1">
-            <AppInput
-              :id="`scholar-image-url-${scholar.id}`"
-              :model-value="imageUrlDraft"
-              placeholder="https://example.com/avatar.jpg"
-              :disabled="imageBusy"
-              @update:model-value="emit('update:image-url-draft', $event as string)"
-            />
-          </div>
-          <AppButton variant="secondary" :disabled="imageBusy" @click="emit('save-image-url')">
-            {{ imageSaving ? "Saving..." : "Save URL" }}
-          </AppButton>
-        </div>
-        <div class="flex flex-wrap items-center gap-2">
-          <label
-            :for="`scholar-image-upload-${scholar.id}`"
-            class="inline-flex min-h-10 cursor-pointer items-center justify-center rounded-lg border border-stroke-strong bg-action-secondary-bg px-3 py-2 text-sm font-semibold text-action-secondary-text transition hover:bg-action-secondary-hover-bg focus-within:outline-none focus-within:ring-2 focus-within:ring-focus-ring focus-within:ring-offset-2 focus-within:ring-offset-focus-offset"
-            :class="{ 'pointer-events-none opacity-60': imageBusy }"
-          >
-            {{ imageUploading ? "Uploading..." : "Upload image" }}
-          </label>
-          <input
-            :id="`scholar-image-upload-${scholar.id}`"
-            type="file"
-            class="sr-only"
-            accept="image/jpeg,image/png,image/webp,image/gif"
-            :disabled="imageBusy"
-            @change="emit('upload-image', $event)"
-          />
-          <AppButton variant="ghost" :disabled="imageBusy" @click="emit('reset-image')">
-            Reset image
-          </AppButton>
-        </div>
-      </div>
-
-      <div class="flex flex-wrap items-center justify-between gap-2 border-t border-stroke-default pt-3">
-        <AppButton variant="secondary" :disabled="imageBusy || saving" @click="emit('toggle')">
-          {{ scholar.is_enabled ? "Disable scholar" : "Enable scholar" }}
-        </AppButton>
-        <AppButton variant="danger" :disabled="imageBusy || saving" @click="emit('delete')">
-          Delete scholar
-        </AppButton>
-      </div>
-    </div>
-  </AppModal>
-</template>

frontend/src/features/scholars/components/ScholarStatusBadges.vue

@@ -1,29 +0,0 @@
-<script setup lang="ts">
-defineProps<{
-  baselineCompleted: boolean;
-  lastRunStatus: string | null;
-}>();
-</script>
-
-<template>
-  <span
-    v-if="!baselineCompleted"
-    class="ml-1.5 inline-flex items-center rounded-full border border-state-warning-border bg-state-warning-bg px-1.5 py-0.5 text-[10px] font-medium text-state-warning-text"
-    title="This scholar hasn't been scraped yet. Publications will appear after the next run."
-  >Pending</span>
-  <span
-    v-if="lastRunStatus === 'failed'"
-    class="ml-1.5 inline-flex items-center rounded-full border border-state-danger-border bg-state-danger-bg px-1.5 py-0.5 text-[10px] font-medium text-state-danger-text"
-    title="The last scrape run for this scholar failed."
-  >Failed</span>
-  <span
-    v-else-if="lastRunStatus === 'partial'"
-    class="ml-1.5 inline-flex items-center rounded-full border border-state-warning-border bg-state-warning-bg px-1.5 py-0.5 text-[10px] font-medium text-state-warning-text"
-    title="The last scrape run for this scholar completed with partial results."
-  >Partial</span>
-  <span
-    v-else-if="lastRunStatus === 'success'"
-    class="ml-1.5 inline-flex items-center rounded-full border border-state-success-border bg-state-success-bg px-1.5 py-0.5 text-[10px] font-medium text-state-success-text"
-    title="The last scrape run for this scholar completed successfully."
-  >OK</span>
-</template>

frontend/src/features/scholars/components/scholar-batch-parsing.ts

@@ -1,76 +0,0 @@
-const SCHOLAR_ID_PATTERN = /^[a-zA-Z0-9_-]{12}$/;
-
-export interface ParsedToken {
-  index: number;
-  raw: string;
-  id: string | null;
-  error: string | null;
-}
-
-export function extractScholarIdFromUrl(token: string): string | null {
-  const cleaned = token.replace(/\/+$/, "").replace(/#.*$/, "");
-  try {
-    const parsed = new URL(cleaned);
-    const userParam = parsed.searchParams.get("user");
-    if (!userParam) return null;
-    const decoded = decodeURIComponent(userParam).trim();
-    if (SCHOLAR_ID_PATTERN.test(decoded)) return decoded;
-    return null;
-  } catch {
-    return null;
-  }
-}
-
-export function validateTokenAsId(token: string): string | null {
-  const trimmed = token.trim();
-  if (!trimmed) return "empty input";
-  if (/\s/.test(trimmed)) return "contains whitespace";
-  if (trimmed.length !== 12) return `must be 12 characters (got ${trimmed.length})`;
-  if (!/^[a-zA-Z0-9_-]+$/.test(trimmed)) {
-    return "contains invalid characters (only a-z, A-Z, 0-9, _ and - allowed)";
-  }
-  return null;
-}
-
-export function parseScholarTokens(raw: string): ParsedToken[] {
-  const tokens = raw.split(/[\s,;]+/).map((v) => v.trim()).filter((v) => v.length > 0);
-  const results: ParsedToken[] = [];
-  const seen = new Set<string>();
-
-  for (let i = 0; i < tokens.length; i++) {
-    const token = tokens[i];
-
-    if (SCHOLAR_ID_PATTERN.test(token)) {
-      if (seen.has(token)) {
-        results.push({ index: i + 1, raw: token, id: null, error: "duplicate" });
-        continue;
-      }
-      seen.add(token);
-      results.push({ index: i + 1, raw: token, id: token, error: null });
-      continue;
-    }
-
-    if (token.includes("scholar.google") || token.startsWith("http")) {
-      const extracted = extractScholarIdFromUrl(token);
-      if (extracted) {
-        if (seen.has(extracted)) {
-          results.push({ index: i + 1, raw: token, id: null, error: "duplicate" });
-          continue;
-        }
-        seen.add(extracted);
-        results.push({ index: i + 1, raw: token, id: extracted, error: null });
-        continue;
-      }
-      results.push({ index: i + 1, raw: token, id: null, error: "could not extract scholar ID from URL" });
-      continue;
-    }
-
-    const reason = validateTokenAsId(token);
-    results.push({ index: i + 1, raw: token, id: null, error: reason ?? "invalid scholar ID" });
-  }
-  return results;
-}
-
-export function parseScholarIds(raw: string): string[] {
-  return parseScholarTokens(raw).filter((t) => t.id !== null).map((t) => t.id!);
-}

frontend/src/features/scholars/composables/useScholarBulkActions.test.ts

@@ -1,148 +0,0 @@
-// @vitest-environment happy-dom
-import { describe, expect, it, vi } from "vitest";
-import { nextTick, ref } from "vue";
-import { useScholarBulkActions, type ScholarBulkAction } from "./useScholarBulkActions";
-import type { ScholarProfile } from "@/features/scholars";
-
-vi.mock("@/features/scholars", () => ({
-  bulkDeleteScholars: vi.fn(),
-  bulkToggleScholars: vi.fn(),
-  exportScholarData: vi.fn(),
-}));
-
-function makeProfile(id: number, name: string): ScholarProfile {
-  return {
-    id,
-    scholar_id: `scholar_${id}`,
-    display_name: name,
-    profile_image_url: null,
-    profile_image_source: "none",
-    is_enabled: true,
-    baseline_completed: false,
-    last_run_dt: null,
-    last_run_status: null,
-  };
-}
-
-function setup(profiles: ScholarProfile[] = []) {
-  const visibleScholars = ref(profiles);
-  const callbacks = {
-    clearMessages: vi.fn(),
-    assignError: vi.fn(),
-    setSuccess: vi.fn(),
-    reloadScholars: vi.fn(async () => {}),
-  };
-  const bulk = useScholarBulkActions(visibleScholars, callbacks);
-  return { visibleScholars, callbacks, bulk };
-}
-
-describe("useScholarBulkActions", () => {
-  it("starts with empty selection", () => {
-    const { bulk } = setup([makeProfile(1, "Alice")]);
-    expect(bulk.selectedIds.value.size).toBe(0);
-    expect(bulk.hasSelection.value).toBe(false);
-  });
-
-  it("toggles individual row selection", () => {
-    const { bulk } = setup([makeProfile(1, "Alice"), makeProfile(2, "Bob")]);
-    bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
-    expect(bulk.selectedIds.value.has(1)).toBe(true);
-    expect(bulk.selectedCount.value).toBe(1);
-
-    bulk.onToggleRow(1, { target: { checked: false } } as unknown as Event);
-    expect(bulk.selectedIds.value.has(1)).toBe(false);
-    expect(bulk.selectedCount.value).toBe(0);
-  });
-
-  it("toggles all visible scholars", () => {
-    const { bulk } = setup([makeProfile(1, "Alice"), makeProfile(2, "Bob")]);
-    bulk.onToggleAll({ target: { checked: true } } as unknown as Event);
-    expect(bulk.selectedIds.value.size).toBe(2);
-    expect(bulk.allVisibleSelected.value).toBe(true);
-
-    bulk.onToggleAll({ target: { checked: false } } as unknown as Event);
-    expect(bulk.selectedIds.value.size).toBe(0);
-  });
-
-  it("prunes stale selections when list changes", async () => {
-    const { visibleScholars, bulk } = setup([makeProfile(1, "Alice"), makeProfile(2, "Bob")]);
-    bulk.onToggleAll({ target: { checked: true } } as unknown as Event);
-    expect(bulk.selectedIds.value.size).toBe(2);
-
-    // Remove Bob from the visible list
-    visibleScholars.value = [makeProfile(1, "Alice")];
-    await nextTick();
-
-    expect(bulk.selectedIds.value.size).toBe(1);
-    expect(bulk.selectedIds.value.has(1)).toBe(true);
-    expect(bulk.selectedIds.value.has(2)).toBe(false);
-  });
-
-  it("shows correct bulk action options with and without selection", () => {
-    const { bulk } = setup([makeProfile(1, "Alice")]);
-    // Without selection
-    expect(bulk.bulkActionOptions.value.length).toBe(1);
-    expect(bulk.bulkActionOptions.value[0].value).toBe("select_all");
-
-    // With selection
-    bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
-    expect(bulk.bulkActionOptions.value.length).toBe(5);
-    const values = bulk.bulkActionOptions.value.map((o) => o.value);
-    expect(values).toContain("delete_selected");
-    expect(values).toContain("enable_selected");
-    expect(values).toContain("disable_selected");
-    expect(values).toContain("export_selected");
-    expect(values).toContain("clear_selection");
-  });
-
-  it("select all action selects all visible", async () => {
-    const { bulk } = setup([makeProfile(1, "Alice"), makeProfile(2, "Bob")]);
-    bulk.bulkAction.value = "select_all" as ScholarBulkAction;
-    await bulk.onApplyBulkAction();
-    expect(bulk.selectedIds.value.size).toBe(2);
-  });
-
-  it("clear selection action clears all", async () => {
-    const { bulk } = setup([makeProfile(1, "Alice")]);
-    bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
-    bulk.bulkAction.value = "clear_selection" as ScholarBulkAction;
-    await bulk.onApplyBulkAction();
-    expect(bulk.selectedIds.value.size).toBe(0);
-  });
-
-  it("bulk delete calls assignError on network failure", async () => {
-    const { bulkDeleteScholars } = await import("@/features/scholars");
-    vi.mocked(bulkDeleteScholars).mockRejectedValueOnce(new Error("Network error"));
-
-    const { bulk, callbacks } = setup([makeProfile(1, "Alice")]);
-    bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
-    bulk.bulkAction.value = "delete_selected" as ScholarBulkAction;
-    await bulk.onApplyBulkAction();
-
-    // Trigger the confirm callback
-    expect(bulk.confirmState.value.open).toBe(true);
-    await bulk.confirmState.value.onConfirm();
-
-    expect(callbacks.assignError).toHaveBeenCalledWith(
-      expect.any(Error),
-      "Unable to bulk delete scholars.",
-    );
-    expect(bulk.bulkBusy.value).toBe(false);
-  });
-
-  it("bulk toggle calls assignError on network failure", async () => {
-    const { bulkToggleScholars } = await import("@/features/scholars");
-    vi.mocked(bulkToggleScholars).mockRejectedValueOnce(new Error("Network error"));
-
-    const { bulk, callbacks } = setup([makeProfile(1, "Alice")]);
-    bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
-    bulk.bulkAction.value = "enable_selected" as ScholarBulkAction;
-    await bulk.onApplyBulkAction();
-
-    expect(callbacks.assignError).toHaveBeenCalledWith(
-      expect.any(Error),
-      "Unable to bulk enable scholars.",
-    );
-    expect(bulk.bulkBusy.value).toBe(false);
-  });
-});

frontend/src/features/scholars/composables/useScholarBulkActions.ts

@@ -1,221 +0,0 @@
-import { computed, ref, watch, type Ref } from "vue";
-
-import {
-  bulkDeleteScholars,
-  bulkToggleScholars,
-  exportScholarData,
-  type ScholarProfile,
-} from "@/features/scholars";
-
-export type ScholarBulkAction =
-  | "delete_selected"
-  | "enable_selected"
-  | "disable_selected"
-  | "export_selected"
-  | "clear_selection"
-  | "select_all";
-
-export interface ScholarBulkActionOption {
-  value: ScholarBulkAction;
-  label: string;
-}
-
-export interface ConfirmState {
-  open: boolean;
-  title: string;
-  message: string;
-  variant: "danger" | "default";
-  onConfirm: () => void;
-}
-
-export interface BulkActionCallbacks {
-  clearMessages: () => void;
-  assignError: (error: unknown, fallback: string) => void;
-  setSuccess: (msg: string) => void;
-  reloadScholars: () => Promise<void>;
-}
-
-export function useScholarBulkActions(
-  visibleScholars: Ref<ScholarProfile[]>,
-  callbacks: BulkActionCallbacks,
-) {
-  const selectedIds = ref<Set<number>>(new Set());
-  const bulkAction = ref<ScholarBulkAction>("select_all");
-  const bulkBusy = ref(false);
-  const confirmState = ref<ConfirmState>({ open: false, title: "", message: "", variant: "default", onConfirm: () => {} });
-
-  const selectedCount = computed(() => selectedIds.value.size);
-  const hasSelection = computed(() => selectedCount.value > 0);
-
-  const allVisibleSelected = computed(() => {
-    if (visibleScholars.value.length === 0) return false;
-    for (const item of visibleScholars.value) {
-      if (!selectedIds.value.has(item.id)) return false;
-    }
-    return true;
-  });
-
-  const bulkActionOptions = computed<ScholarBulkActionOption[]>(() => {
-    if (!hasSelection.value) return [{ value: "select_all", label: "Select all" }];
-    const n = selectedCount.value;
-    return [
-      { value: "delete_selected", label: `Delete selected (${n})` },
-      { value: "enable_selected", label: `Enable selected (${n})` },
-      { value: "disable_selected", label: `Disable selected (${n})` },
-      { value: "export_selected", label: `Export selected (${n})` },
-      { value: "clear_selection", label: "Clear selection" },
-    ];
-  });
-
-  const bulkApplyLabel = computed(() => {
-    if (bulkBusy.value) return "Applying...";
-    if (bulkAction.value === "select_all") return "Select";
-    if (bulkAction.value === "clear_selection") return "Clear";
-    return "Apply";
-  });
-
-  const bulkApplyDisabled = computed(() => {
-    if (bulkBusy.value) return true;
-    if (bulkAction.value === "select_all") return visibleScholars.value.length === 0;
-    return selectedCount.value === 0;
-  });
-
-  // Prune stale selections when visible list changes
-  watch(visibleScholars, (items) => {
-    const validIds = new Set(items.map((item) => item.id));
-    const next = new Set<number>();
-    for (const id of selectedIds.value) {
-      if (validIds.has(id)) next.add(id);
-    }
-    if (next.size !== selectedIds.value.size) selectedIds.value = next;
-  });
-
-  // Reset bulk action dropdown when selection state changes
-  watch(hasSelection, (has) => {
-    const validValues = new Set(bulkActionOptions.value.map((o) => o.value));
-    if (validValues.has(bulkAction.value)) return;
-    bulkAction.value = has ? "delete_selected" : "select_all";
-  });
-
-  function onToggleAll(event: Event): void {
-    const checked = (event.target as HTMLInputElement).checked;
-    const next = new Set(selectedIds.value);
-    for (const item of visibleScholars.value) {
-      if (checked) { next.add(item.id); } else { next.delete(item.id); }
-    }
-    selectedIds.value = next;
-  }
-
-  function onToggleRow(id: number, event: Event): void {
-    const checked = (event.target as HTMLInputElement).checked;
-    const next = new Set(selectedIds.value);
-    if (checked) { next.add(id); } else { next.delete(id); }
-    selectedIds.value = next;
-  }
-
-  function downloadJsonFile(filename: string, payload: unknown): void {
-    const blob = new Blob([JSON.stringify(payload, null, 2)], { type: "application/json" });
-    const url = URL.createObjectURL(blob);
-    const anchor = document.createElement("a");
-    anchor.href = url;
-    anchor.download = filename;
-    anchor.click();
-    URL.revokeObjectURL(url);
-  }
-
-  async function onApplyBulkAction(): Promise<void> {
-    if (bulkApplyDisabled.value) return;
-    if (bulkAction.value === "select_all") {
-      selectedIds.value = new Set(visibleScholars.value.map((item) => item.id));
-      return;
-    }
-    if (bulkAction.value === "clear_selection") { selectedIds.value = new Set(); return; }
-    if (bulkAction.value === "delete_selected") { await onBulkDelete(); return; }
-    if (bulkAction.value === "enable_selected") { await onBulkToggle(true); return; }
-    if (bulkAction.value === "disable_selected") { await onBulkToggle(false); return; }
-    if (bulkAction.value === "export_selected") { await onBulkExport(); return; }
-  }
-
-  function dismissConfirm(): void {
-    confirmState.value = { ...confirmState.value, open: false };
-  }
-
-  function requestConfirm(title: string, message: string, variant: "danger" | "default", onConfirm: () => void): void {
-    confirmState.value = { open: true, title, message, variant, onConfirm };
-  }
-
-  function onBulkDelete(): void {
-    const ids = [...selectedIds.value];
-    requestConfirm(
-      `Delete ${ids.length} scholar(s)?`,
-      "This removes all linked publications and queue data. This action cannot be undone.",
-      "danger",
-      async () => {
-        dismissConfirm();
-        bulkBusy.value = true;
-        callbacks.clearMessages();
-        try {
-          const result = await bulkDeleteScholars(ids);
-          callbacks.setSuccess(`${result.deleted_count} scholar(s) deleted.`);
-          selectedIds.value = new Set();
-          await callbacks.reloadScholars();
-        } catch (error) {
-          callbacks.assignError(error, "Unable to bulk delete scholars.");
-        } finally {
-          bulkBusy.value = false;
-        }
-      },
-    );
-  }
-
-  async function onBulkToggle(isEnabled: boolean): Promise<void> {
-    bulkBusy.value = true;
-    callbacks.clearMessages();
-    try {
-      const ids = [...selectedIds.value];
-      const result = await bulkToggleScholars(ids, isEnabled);
-      const verb = isEnabled ? "enabled" : "disabled";
-      callbacks.setSuccess(`${result.updated_count} scholar(s) ${verb}.`);
-      selectedIds.value = new Set();
-      await callbacks.reloadScholars();
-    } catch (error) {
-      callbacks.assignError(error, `Unable to bulk ${isEnabled ? "enable" : "disable"} scholars.`);
-    } finally {
-      bulkBusy.value = false;
-    }
-  }
-
-  async function onBulkExport(): Promise<void> {
-    bulkBusy.value = true;
-    callbacks.clearMessages();
-    try {
-      const ids = [...selectedIds.value];
-      const payload = await exportScholarData(ids);
-      const dateSlug = payload.exported_at.slice(0, 10) || "unknown-date";
-      downloadJsonFile(`scholarr-export-${dateSlug}.json`, payload);
-      callbacks.setSuccess("Export complete.");
-    } catch (error) {
-      callbacks.assignError(error, "Unable to export selected scholars.");
-    } finally {
-      bulkBusy.value = false;
-    }
-  }
-
-  return {
-    selectedIds,
-    selectedCount,
-    hasSelection,
-    allVisibleSelected,
-    bulkAction,
-    bulkBusy,
-    bulkActionOptions,
-    bulkApplyLabel,
-    bulkApplyDisabled,
-    confirmState,
-    dismissConfirm,
-    requestConfirm,
-    onToggleAll,
-    onToggleRow,
-    onApplyBulkAction,
-  };
-}

frontend/src/features/scholars/index.ts

@@ -160,30 +160,8 @@ export async function clearScholarImage(
   return response.data;
 }
 
-export interface BulkCountResult {
+export async function exportScholarData(): Promise<DataExportPayload> {
-  deleted_count: number;
+  const response = await apiRequest<DataExportPayload>("/scholars/export", {
-  updated_count: number;
-}
-
-export async function bulkDeleteScholars(scholarProfileIds: number[]): Promise<BulkCountResult> {
-  const response = await apiRequest<BulkCountResult>("/scholars/bulk-delete", {
-    method: "POST",
-    body: { scholar_profile_ids: scholarProfileIds },
-  });
-  return response.data;
-}
-
-export async function bulkToggleScholars(scholarProfileIds: number[], isEnabled: boolean): Promise<BulkCountResult> {
-  const response = await apiRequest<BulkCountResult>("/scholars/bulk-toggle", {
-    method: "POST",
-    body: { scholar_profile_ids: scholarProfileIds, is_enabled: isEnabled },
-  });
-  return response.data;
-}
-
-export async function exportScholarData(ids?: number[]): Promise<DataExportPayload> {
-  const path = ids && ids.length > 0 ? `/scholars/export?ids=${ids.join(",")}` : "/scholars/export";
-  const response = await apiRequest<DataExportPayload>(path, {
     method: "GET",
   });
   return response.data;

frontend/src/features/settings/SettingsAdminPanel.test.ts

@@ -1,94 +0,0 @@
-// @vitest-environment happy-dom
-import { describe, expect, it, vi, beforeEach } from "vitest";
-import { mount, flushPromises } from "@vue/test-utils";
-import { nextTick } from "vue";
-import SettingsAdminPanel from "./SettingsAdminPanel.vue";
-
-vi.mock("@/features/admin_users", () => ({
-  listAdminUsers: vi.fn().mockResolvedValue([]),
-  createAdminUser: vi.fn(),
-  setAdminUserActive: vi.fn(),
-  resetAdminUserPassword: vi.fn(),
-}));
-
-vi.mock("@/features/admin_dbops", () => ({
-  getAdminDbIntegrityReport: vi.fn().mockResolvedValue({
-    status: "ok",
-    warnings: [],
-    failures: [],
-    checked_at: null,
-    checks: [],
-  }),
-  listAdminPdfQueue: vi.fn().mockResolvedValue({
-    items: [],
-    total_count: 0,
-    has_next: false,
-    has_prev: false,
-    page: 1,
-    page_size: 50,
-  }),
-  requeueAdminPdfLookup: vi.fn(),
-  requeueAllAdminPdfLookups: vi.fn(),
-}));
-
-vi.mock("@/stores/auth", () => ({
-  useAuthStore: () => ({ isAdmin: true }),
-}));
-
-vi.mock("@/features/admin_repairs", () => ({
-  listAdminRepairTasks: vi.fn().mockResolvedValue([]),
-  runAdminRepairTask: vi.fn(),
-}));
-
-import { listAdminUsers } from "@/features/admin_users";
-import { getAdminDbIntegrityReport, listAdminPdfQueue } from "@/features/admin_dbops";
-
-const mockedListUsers = vi.mocked(listAdminUsers);
-const mockedGetReport = vi.mocked(getAdminDbIntegrityReport);
-const mockedListPdfQueue = vi.mocked(listAdminPdfQueue);
-
-describe("SettingsAdminPanel", () => {
-  beforeEach(() => {
-    mockedListUsers.mockClear();
-    mockedGetReport.mockClear();
-    mockedListPdfQueue.mockClear();
-  });
-
-  it("calls load on users section after nextTick when section=users", async () => {
-    mount(SettingsAdminPanel, { props: { section: "users" } });
-    await nextTick();
-    await nextTick();
-    await flushPromises();
-    expect(mockedListUsers).toHaveBeenCalled();
-  });
-
-  it("calls load on integrity section after nextTick when section=integrity", async () => {
-    mount(SettingsAdminPanel, { props: { section: "integrity" } });
-    await nextTick();
-    await nextTick();
-    await flushPromises();
-    expect(mockedGetReport).toHaveBeenCalled();
-  });
-
-  it("calls load on new section when section prop changes", async () => {
-    const wrapper = mount(SettingsAdminPanel, { props: { section: "users" } });
-    await nextTick();
-    await nextTick();
-    await flushPromises();
-    mockedListUsers.mockClear();
-    mockedGetReport.mockClear();
-
-    await wrapper.setProps({ section: "integrity" });
-    await nextTick();
-    await flushPromises();
-    expect(mockedGetReport).toHaveBeenCalled();
-  });
-
-  it("calls load on pdf section when section=pdf", async () => {
-    mount(SettingsAdminPanel, { props: { section: "pdf" } });
-    await nextTick();
-    await nextTick();
-    await flushPromises();
-    expect(mockedListPdfQueue).toHaveBeenCalled();
-  });
-});

frontend/src/features/settings/components/AdminIntegritySection.test.ts

@@ -1,72 +0,0 @@
-// @vitest-environment happy-dom
-import { describe, expect, it, vi, beforeEach } from "vitest";
-import { mount, flushPromises } from "@vue/test-utils";
-import AdminIntegritySection from "./AdminIntegritySection.vue";
-
-vi.mock("@/features/admin_dbops", () => ({
-  getAdminDbIntegrityReport: vi.fn(),
-}));
-
-import { getAdminDbIntegrityReport, type AdminDbIntegrityReport } from "@/features/admin_dbops";
-
-const mockedGetReport = vi.mocked(getAdminDbIntegrityReport);
-
-function buildReport(overrides: Partial<AdminDbIntegrityReport> = {}): AdminDbIntegrityReport {
-  return {
-    status: "ok",
-    warnings: [],
-    failures: [],
-    checked_at: "2025-01-15T12:00:00Z",
-    checks: [
-      { name: "orphaned_publications", count: 0, severity: "metric", message: "No orphaned publications found." },
-    ],
-    ...overrides,
-  };
-}
-
-describe("AdminIntegritySection", () => {
-  beforeEach(() => {
-    mockedGetReport.mockReset();
-  });
-
-  it("renders the section heading", () => {
-    const wrapper = mount(AdminIntegritySection);
-    expect(wrapper.text()).toContain("Integrity Report");
-  });
-
-  it("exposes load function that fetches integrity report", async () => {
-    mockedGetReport.mockResolvedValueOnce(buildReport());
-    const wrapper = mount(AdminIntegritySection);
-    const vm = wrapper.vm as unknown as { load: () => Promise<void> };
-    await vm.load();
-    await flushPromises();
-    expect(mockedGetReport).toHaveBeenCalled();
-  });
-
-  it("renders check results after loading", async () => {
-    mockedGetReport.mockResolvedValueOnce(buildReport());
-    const wrapper = mount(AdminIntegritySection);
-    const vm = wrapper.vm as unknown as { load: () => Promise<void> };
-    await vm.load();
-    await flushPromises();
-    expect(wrapper.text()).toContain("orphaned_publications");
-  });
-
-  it("displays warning count when present", async () => {
-    mockedGetReport.mockResolvedValueOnce(buildReport({ warnings: ["w1", "w2", "w3"] }));
-    const wrapper = mount(AdminIntegritySection);
-    const vm = wrapper.vm as unknown as { load: () => Promise<void> };
-    await vm.load();
-    await flushPromises();
-    expect(wrapper.text()).toContain("3");
-  });
-
-  it("renders status badge", async () => {
-    mockedGetReport.mockResolvedValueOnce(buildReport({ status: "failed" }));
-    const wrapper = mount(AdminIntegritySection);
-    const vm = wrapper.vm as unknown as { load: () => Promise<void> };
-    await vm.load();
-    await flushPromises();
-    expect(wrapper.text()).toContain("failed");
-  });
-});

frontend/src/features/settings/components/AdminIntegritySection.vue

@@ -1,88 +0,0 @@
-<script setup lang="ts">
-import { ref } from "vue";
-import AppBadge from "@/components/ui/AppBadge.vue";
-import AppCard from "@/components/ui/AppCard.vue";
-import AppHelpHint from "@/components/ui/AppHelpHint.vue";
-import AppRefreshButton from "@/components/ui/AppRefreshButton.vue";
-import AppTable from "@/components/ui/AppTable.vue";
-import {
-  getAdminDbIntegrityReport,
-  type AdminDbIntegrityCheck,
-  type AdminDbIntegrityReport,
-} from "@/features/admin_dbops";
-
-const refreshingIntegrity = ref(false);
-const integrityReport = ref<AdminDbIntegrityReport | null>(null);
-
-function formatTimestamp(value: string | null): string {
-  if (!value) return "n/a";
-  const parsed = new Date(value);
-  return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString();
-}
-
-function statusTone(status: string): "success" | "warning" | "danger" | "info" | "neutral" {
-  if (status === "ok" || status === "completed" || status === "resolved") return "success";
-  if (status === "warning" || status === "running" || status === "queued") return "warning";
-  if (status === "failed") return "danger";
-  return "info";
-}
-
-function checkTone(check: AdminDbIntegrityCheck): "warning" | "danger" | "neutral" | "info" {
-  if (check.severity === "metric") return "info";
-  if (check.count <= 0) return "neutral";
-  return check.severity === "failure" ? "danger" : "warning";
-}
-
-async function refreshIntegrity(): Promise<void> {
-  refreshingIntegrity.value = true;
-  try {
-    integrityReport.value = await getAdminDbIntegrityReport();
-  } finally {
-    refreshingIntegrity.value = false;
-  }
-}
-
-async function load(): Promise<void> {
-  await refreshIntegrity();
-}
-
-defineExpose({ load });
-</script>
-
-<template>
-  <AppCard class="space-y-3">
-    <div class="flex flex-wrap items-center justify-between gap-2">
-      <div class="flex items-center gap-1">
-        <h2 class="text-lg font-semibold text-ink-primary">Integrity Report</h2>
-        <AppHelpHint text="Read-only checks for known corruption patterns and data drift." />
-      </div>
-      <AppRefreshButton variant="secondary" :loading="refreshingIntegrity" title="Refresh integrity report" loading-title="Refreshing integrity report" @click="refreshIntegrity" />
-    </div>
-
-    <div v-if="integrityReport" class="flex flex-wrap items-center gap-2 text-xs">
-      <AppBadge :tone="statusTone(integrityReport.status)">Status: {{ integrityReport.status }}</AppBadge>
-      <AppBadge tone="warning">Warnings: {{ integrityReport.warnings.length }}</AppBadge>
-      <AppBadge tone="danger">Failures: {{ integrityReport.failures.length }}</AppBadge>
-      <span class="text-secondary">Checked: {{ formatTimestamp(integrityReport.checked_at) }}</span>
-    </div>
-
-    <AppTable v-if="integrityReport" label="Integrity checks">
-      <thead>
-        <tr>
-          <th scope="col">Check</th>
-          <th scope="col">Count</th>
-          <th scope="col">Severity</th>
-          <th scope="col">Message</th>
-        </tr>
-      </thead>
-      <tbody>
-        <tr v-for="check in integrityReport.checks" :key="check.name">
-          <td>{{ check.name }}</td>
-          <td>{{ check.count }}</td>
-          <td><AppBadge :tone="checkTone(check)">{{ check.severity }}</AppBadge></td>
-          <td>{{ check.message }}</td>
-        </tr>
-      </tbody>
-    </AppTable>
-  </AppCard>
-</template>