First product

.dockerignore

@@ -0,0 +1,14 @@
+.git
+__pycache__
+.pytest_cache
+.mypy_cache
+.ruff_cache
+.venv
+.uv-cache
+*.pyc
+*.pyo
+*.pyd
+*.egg-info
+*.log
+.env
+htmlcov

.env.example

@@ -0,0 +1,45 @@
+POSTGRES_DB=scholar
+POSTGRES_USER=scholar
+POSTGRES_PASSWORD=scholar
+POSTGRES_PORT=5432
+
+DATABASE_URL=postgresql+asyncpg://scholar:scholar@db:5432/scholar
+# Optional override. If empty, tests derive "<DATABASE_URL db name>_test".
+TEST_DATABASE_URL=
+
+APP_NAME=scholarr
+APP_HOST=0.0.0.0
+APP_PORT=8000
+APP_HOST_PORT=8000
+APP_RELOAD=1
+MIGRATE_ON_START=1
+
+SESSION_SECRET_KEY=replace-with-a-long-random-secret
+SESSION_COOKIE_SECURE=0
+LOGIN_RATE_LIMIT_ATTEMPTS=5
+LOGIN_RATE_LIMIT_WINDOW_SECONDS=60
+LOG_LEVEL=INFO
+LOG_FORMAT=console
+LOG_REQUESTS=1
+LOG_UVICORN_ACCESS=0
+LOG_REQUEST_SKIP_PATHS=/healthz,/static/
+LOG_REDACT_FIELDS=
+SCHEDULER_ENABLED=1
+SCHEDULER_TICK_SECONDS=60
+INGESTION_NETWORK_ERROR_RETRIES=1
+INGESTION_RETRY_BACKOFF_SECONDS=1.0
+INGESTION_MAX_PAGES_PER_SCHOLAR=30
+INGESTION_PAGE_SIZE=100
+INGESTION_CONTINUATION_QUEUE_ENABLED=1
+INGESTION_CONTINUATION_BASE_DELAY_SECONDS=120
+INGESTION_CONTINUATION_MAX_DELAY_SECONDS=3600
+INGESTION_CONTINUATION_MAX_ATTEMPTS=6
+SCHEDULER_QUEUE_BATCH_SIZE=10
+
+BOOTSTRAP_ADMIN_ON_START=0
+BOOTSTRAP_ADMIN_EMAIL=
+BOOTSTRAP_ADMIN_PASSWORD=
+BOOTSTRAP_ADMIN_FORCE_PASSWORD=0
+
+DB_WAIT_TIMEOUT_SECONDS=60
+DB_WAIT_INTERVAL_SECONDS=2

.github/workflows/ci.yml

@@ -0,0 +1,53 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_DB: scholar
+          POSTGRES_USER: scholar
+          POSTGRES_PASSWORD: scholar
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U scholar -d scholar"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+
+    env:
+      DATABASE_URL: postgresql+asyncpg://scholar:scholar@localhost:5432/scholar
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install dependencies
+        run: uv sync --extra dev
+
+      - name: Migration smoke
+        run: uv run alembic upgrade head
+
+      - name: Unit tests
+        run: uv run pytest tests/unit
+
+      - name: Integration tests
+        run: uv run pytest -m integration
+

.gitignore

@@ -0,0 +1,12 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+.venv/
+.uv-cache/
+.env
+*.log

.python-version

@@ -0,0 +1,2 @@
+3.12
+

Dockerfile

@@ -0,0 +1,54 @@
+FROM python:3.12-slim AS base
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/opt/venv \
+    PATH="/opt/venv/bin:$PATH"
+
+WORKDIR /app
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends build-essential curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=ghcr.io/astral-sh/uv:0.6.5 /uv /uvx /bin/
+
+COPY pyproject.toml uv.lock README.md ./
+COPY app ./app
+COPY alembic.ini ./alembic.ini
+COPY alembic ./alembic
+COPY scripts ./scripts
+
+RUN uv sync --frozen --extra dev
+
+FROM base AS dev
+ENTRYPOINT ["/bin/sh", "/app/scripts/entrypoint.sh"]
+
+FROM python:3.12-slim AS prod
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/opt/venv \
+    PATH="/opt/venv/bin:$PATH" \
+    APP_RELOAD=0 \
+    UVICORN_WORKERS=1
+
+WORKDIR /app
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=ghcr.io/astral-sh/uv:0.6.5 /uv /uvx /bin/
+
+COPY pyproject.toml uv.lock README.md ./
+COPY app ./app
+COPY alembic.ini ./alembic.ini
+COPY alembic ./alembic
+
+RUN uv sync --frozen
+
+COPY scripts ./scripts
+ENTRYPOINT ["/bin/sh", "/app/scripts/entrypoint.sh"]

README.md

@@ -0,0 +1,206 @@
+# scholarr
+
+Container-first, self-hosted scholar tracking in the spirit of the `*arr` ecosystem, with strict multi-user isolation and an intentionally small operational footprint.
+
+## Current Scope
+
+- Admin-managed users only (no public signup)
+- Session auth with CSRF protection and login rate limiting
+- Per-user scholar tracking list (add, enable/disable, delete)
+- Per-user automation settings
+- Manual per-user ingestion runs from dashboard
+- Per-user run history and "new since last run" publication stream
+- Publications library view with `new` and `all` modes plus scholar filtering
+- Dedicated run diagnostics pages (`/runs`, `/runs/{id}`) with failed-only filtering
+- Continuation queue visibility + controls on runs page (`retry`, `drop`, `clear`)
+- Baseline-aware discovery tracking (`new` = discovered in latest completed run)
+- Read state is user-controlled (`Mark All Read`), including baseline items
+- Multi-page profile ingestion (`cstart`) to capture full publication lists
+- Dashboard rendered via a presentation-layer view model and section template registry
+- Structured application logging with request IDs and field redaction
+- Ingestion overlap guard via Postgres advisory locks
+- Network-error retry support for ingestion attempts
+- Automatic continuation queue for resumable/limit-hit scholar runs
+- Background scheduler for per-user automatic runs
+- User self-service password changes
+- Admin user management (create users, activate/deactivate, reset passwords)
+- PostgreSQL + Alembic migrations + containerized test workflow
+- Versioned backend API (`/api/v1`) for frontend decoupling (same-origin cookie session)
+
+## Tech Stack
+
+- Python 3.12+
+- FastAPI
+- PostgreSQL 15+
+- SQLAlchemy 2 + Alembic
+- Jinja2 templates + modular theme tokens
+- `uv` for dependency and runtime workflow
+- Docker / Docker Compose for development and CI parity
+
+## Quick Start
+
+1. Copy environment defaults:
+
+```bash
+cp .env.example .env
+```
+
+2. Bootstrap the first admin on startup (recommended for first run):
+
+```bash
+export BOOTSTRAP_ADMIN_ON_START=1
+export BOOTSTRAP_ADMIN_EMAIL=admin@example.com
+export BOOTSTRAP_ADMIN_PASSWORD=change-me-now
+```
+
+3. Start the stack:
+
+```bash
+docker compose up --build
+```
+
+4. Open:
+
+- Login: `http://localhost:8000/login`
+- Dashboard: `http://localhost:8000/`
+- Healthcheck: `http://localhost:8000/healthz`
+
+5. After first successful admin login, disable auto-bootstrap:
+
+```bash
+export BOOTSTRAP_ADMIN_ON_START=0
+```
+
+## Admin Bootstrap (Manual)
+
+You can create/update an admin account without restarting the app:
+
+```bash
+docker compose run --rm app uv run python scripts/bootstrap_admin.py \
+  --email admin@example.com \
+  --password change-me-now \
+  --force-password
+```
+
+Notes:
+- If the user does not exist, this creates an active admin account.
+- If the user exists, it ensures admin + active flags.
+- `--force-password` resets the password for existing users.
+
+## Test Workflow
+
+- Integration/smoke tests run against `TEST_DATABASE_URL`.
+- If `TEST_DATABASE_URL` is empty, test runs derive an isolated database from `DATABASE_URL` by appending `_test`.
+- This keeps app data in `DATABASE_URL` untouched during test runs.
+- Parser regression tests are pinned to captured real-world fixture pages in `tests/fixtures/scholar/regression`.
+
+- Unit tests:
+
+```bash
+docker compose run --rm app uv run pytest tests/unit
+```
+
+- Integration tests:
+
+```bash
+docker compose run --rm app uv run pytest -m integration
+```
+
+- Smoke checks (container + DB + migration sanity):
+
+```bash
+./scripts/smoke_compose.sh
+```
+
+CI runs migration, unit, and integration checks on push/PR via `.github/workflows/ci.yml`.
+
+## Scholar Ingestion Notes
+
+- Current ingestion targets public profile pages: `/citations?user=<id>`.
+- Runs are intentionally conservative and prioritize reliability over aggressive scraping.
+- Ingestion follows pagination (`cstart`) to collect all reachable profile pages.
+- If pagination cannot be completed (max pages hit, cursor stall, or page-state failure), the scholar result is marked partial with explicit debug context.
+- Resumable partial/failure states are automatically queued and retried by the scheduler with bounded backoff.
+- Queue items keep explicit status (`queued`, `retrying`, `dropped`) and last-error context for UI diagnostics.
+- Failed scholar attempts persist structured debug context in `crawl_runs.error_log` (state reason, fetch metadata, marker evidence, and compact response excerpt).
+- Only resumable states are retried automatically (`network_error` and continuation-eligible pagination truncations); blocked/layout states are recorded immediately as failures.
+
+## API v1 (Frontend Prep)
+
+- Base path: `/api/v1`
+- Auth model: same-origin cookie session (no token auth added)
+- CSRF model: required for unsafe methods (`POST`, `PUT`, `PATCH`, `DELETE`) via `X-CSRF-Token` header.
+- Bootstrap CSRF via `GET /api/v1/auth/csrf` (works for anonymous and authenticated sessions).
+- You can also fetch `csrf_token` from `GET /api/v1/auth/me` after login.
+- Manual run idempotency is supported via `Idempotency-Key` header on `POST /api/v1/runs/manual`.
+- Response envelopes:
+  - Success: `{"data": ..., "meta": {"request_id": "..."}}`
+  - Error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}`
+- Initial API domains exposed:
+  - `auth`: `/auth/csrf`, `/auth/login`, `/auth/me`, `/auth/change-password`, `/auth/logout`
+  - `admin users`: `/admin/users` (list/create), `/admin/users/{id}/active`, `/admin/users/{id}/reset-password`
+  - `scholars`: list/create/toggle/delete
+  - `settings`: get/update
+  - `runs`: list/detail/manual trigger + queue list/retry/drop/clear
+  - `publications`: list + mark-all-read
+- Queue action semantics:
+  - `retry` is valid only for dropped items; retrying/queued states return `409`.
+  - `drop` returns the updated queue item (status `dropped`); dropping an already dropped item returns `409`.
+  - `clear` only works for dropped items and returns `{queue_item_id, previous_status, status="cleared"}`.
+
+## Logging
+
+- Default output is concise console logs to stdout.
+- Timestamp format is compact UTC: `YYYY-MM-DD HH:MM:SSZ`.
+- Every request gets an `X-Request-ID` (propagated if caller provides one).
+- Sensitive fields are redacted before log emission (`password`, `csrf_token`, `cookie`, etc.).
+- Configure via env:
+  - `LOG_LEVEL` (default `INFO`)
+  - `LOG_FORMAT` (`console` or `json`, default `console`)
+  - `LOG_REQUESTS` (`1`/`0`, default `1`)
+  - `LOG_UVICORN_ACCESS` (`1`/`0`, default `0`)
+  - `LOG_REQUEST_SKIP_PATHS` (comma-separated prefixes, default `/healthz,/static/`)
+  - `LOG_REDACT_FIELDS` (comma-separated additional keys)
+  - `SCHEDULER_ENABLED` (`1`/`0`, default `1`)
+  - `SCHEDULER_TICK_SECONDS` (default `60`)
+  - `INGESTION_NETWORK_ERROR_RETRIES` (default `1`)
+  - `INGESTION_RETRY_BACKOFF_SECONDS` (default `1.0`)
+  - `INGESTION_MAX_PAGES_PER_SCHOLAR` (default `30`)
+  - `INGESTION_PAGE_SIZE` (default `100`)
+  - `INGESTION_CONTINUATION_QUEUE_ENABLED` (`1`/`0`, default `1`)
+  - `INGESTION_CONTINUATION_BASE_DELAY_SECONDS` (default `120`)
+  - `INGESTION_CONTINUATION_MAX_DELAY_SECONDS` (default `3600`)
+  - `INGESTION_CONTINUATION_MAX_ATTEMPTS` (default `6`)
+  - `SCHEDULER_QUEUE_BATCH_SIZE` (default `10`)
+
+## Optional Local `uv` Workflow
+
+```bash
+uv sync --extra dev
+uv run pytest
+uv run uvicorn app.main:app --reload
+```
+
+Update the lockfile after dependency changes:
+
+```bash
+uv lock
+```
+
+## Theming
+
+- Theme registry: `app/theme.py`
+- Semantic tokens: `app/static/theme.css`
+- App styling: `app/static/app.css`
+- Theme persistence: `app/static/theme.js`
+- Dashboard presentation contract: `app/presentation/dashboard.py`
+
+## Project Layout
+
+```text
+app/         FastAPI app, auth/security modules, templates, services, DB wiring, presentation view-models
+app/web/     Router modules, shared web helpers, and request middleware
+alembic/     Migration environment and versions
+scripts/     Entrypoint, DB wait, bootstrap, smoke automation
+tests/       Unit, integration, and smoke suites
+```

alembic.ini

@@ -0,0 +1,37 @@
+[alembic]
+script_location = alembic
+prepend_sys_path = .
+path_separator = os
+sqlalchemy.url = postgresql+asyncpg://scholar:scholar@db:5432/scholar
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s

alembic/env.py

@@ -0,0 +1,66 @@
+from logging.config import fileConfig
+import os
+
+from alembic import context
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+
+from app.db.base import metadata
+from app.db import models  # noqa: F401
+
+config = context.config
+
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+database_url = os.getenv("DATABASE_URL")
+if database_url:
+    config.set_main_option("sqlalchemy.url", database_url)
+
+target_metadata = metadata
+
+
+def run_migrations_offline() -> None:
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+    context.configure(connection=connection, target_metadata=target_metadata)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+    connectable = async_engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+
+    await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+    import asyncio
+
+    asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,27 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: str | Sequence[str] | None = ${repr(down_revision)}
+branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
+depends_on: str | Sequence[str] | None = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}
+

alembic/versions/.gitkeep

@@ -0,0 +1 @@
+

alembic/versions/20260216_0001_multi_user_schema.py

@@ -0,0 +1,353 @@
+"""Create multi-user core schema
+
+Revision ID: 20260216_0001
+Revises:
+Create Date: 2026-02-16 17:10:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "20260216_0001"
+down_revision: str | Sequence[str] | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+run_status_enum = sa.Enum(
+    "running",
+    "success",
+    "partial_failure",
+    "failed",
+    name="run_status",
+)
+run_trigger_type_enum = sa.Enum(
+    "manual",
+    "scheduled",
+    name="run_trigger_type",
+)
+run_status_ref = postgresql.ENUM(
+    "running",
+    "success",
+    "partial_failure",
+    "failed",
+    name="run_status",
+    create_type=False,
+)
+run_trigger_type_ref = postgresql.ENUM(
+    "manual",
+    "scheduled",
+    name="run_trigger_type",
+    create_type=False,
+)
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    run_status_enum.create(bind, checkfirst=True)
+    run_trigger_type_enum.create(bind, checkfirst=True)
+
+    op.create_table(
+        "users",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("email", sa.String(length=255), nullable=False),
+        sa.Column("password_hash", sa.String(length=255), nullable=False),
+        sa.Column(
+            "is_active",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.text("true"),
+        ),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.PrimaryKeyConstraint("id", name="pk_users"),
+        sa.UniqueConstraint("email", name="uq_users_email"),
+    )
+
+    op.create_table(
+        "user_settings",
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "auto_run_enabled",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.text("false"),
+        ),
+        sa.Column(
+            "run_interval_minutes",
+            sa.Integer(),
+            nullable=False,
+            server_default=sa.text("1440"),
+        ),
+        sa.Column(
+            "request_delay_seconds",
+            sa.Integer(),
+            nullable=False,
+            server_default=sa.text("10"),
+        ),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.CheckConstraint(
+            "run_interval_minutes >= 15",
+            name="ck_user_settings_run_interval_minutes_min_15",
+        ),
+        sa.CheckConstraint(
+            "request_delay_seconds >= 1",
+            name="ck_user_settings_request_delay_seconds_min_1",
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["users.id"],
+            name="fk_user_settings_user_id_users",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("user_id", name="pk_user_settings"),
+    )
+
+    op.create_table(
+        "publications",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("cluster_id", sa.String(length=64), nullable=True),
+        sa.Column("fingerprint_sha256", sa.String(length=64), nullable=False),
+        sa.Column("title_raw", sa.Text(), nullable=False),
+        sa.Column("title_normalized", sa.Text(), nullable=False),
+        sa.Column("year", sa.Integer(), nullable=True),
+        sa.Column(
+            "citation_count",
+            sa.Integer(),
+            nullable=False,
+            server_default=sa.text("0"),
+        ),
+        sa.Column("author_text", sa.Text(), nullable=True),
+        sa.Column("venue_text", sa.Text(), nullable=True),
+        sa.Column("pub_url", sa.Text(), nullable=True),
+        sa.Column("pdf_url", sa.Text(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.PrimaryKeyConstraint("id", name="pk_publications"),
+        sa.UniqueConstraint(
+            "fingerprint_sha256",
+            name="uq_publications_fingerprint",
+        ),
+    )
+    op.create_index(
+        "uq_publications_cluster_id_not_null",
+        "publications",
+        ["cluster_id"],
+        unique=True,
+        postgresql_where=sa.text("cluster_id IS NOT NULL"),
+    )
+
+    op.create_table(
+        "scholar_profiles",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.Column("scholar_id", sa.String(length=64), nullable=False),
+        sa.Column("display_name", sa.String(length=255), nullable=True),
+        sa.Column(
+            "is_enabled",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.text("true"),
+        ),
+        sa.Column(
+            "baseline_completed",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.text("false"),
+        ),
+        sa.Column("last_run_dt", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_run_status", run_status_ref, nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["users.id"],
+            name="fk_scholar_profiles_user_id_users",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name="pk_scholar_profiles"),
+        sa.UniqueConstraint(
+            "user_id",
+            "scholar_id",
+            name="uq_scholar_profiles_user_scholar",
+        ),
+    )
+    op.create_index(
+        "ix_scholar_profiles_user_enabled",
+        "scholar_profiles",
+        ["user_id", "is_enabled"],
+        unique=False,
+    )
+
+    op.create_table(
+        "crawl_runs",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.Column("trigger_type", run_trigger_type_ref, nullable=False),
+        sa.Column("status", run_status_ref, nullable=False),
+        sa.Column(
+            "start_dt",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column("end_dt", sa.DateTime(timezone=True), nullable=True),
+        sa.Column(
+            "scholar_count",
+            sa.Integer(),
+            nullable=False,
+            server_default=sa.text("0"),
+        ),
+        sa.Column(
+            "new_pub_count",
+            sa.Integer(),
+            nullable=False,
+            server_default=sa.text("0"),
+        ),
+        sa.Column(
+            "error_log",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=False,
+            server_default=sa.text("'{}'::jsonb"),
+        ),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["users.id"],
+            name="fk_crawl_runs_user_id_users",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name="pk_crawl_runs"),
+    )
+    op.create_index(
+        "ix_crawl_runs_user_start",
+        "crawl_runs",
+        ["user_id", "start_dt"],
+        unique=False,
+    )
+
+    op.create_table(
+        "scholar_publications",
+        sa.Column("scholar_profile_id", sa.Integer(), nullable=False),
+        sa.Column("publication_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "is_read",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.text("false"),
+        ),
+        sa.Column("first_seen_run_id", sa.Integer(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.ForeignKeyConstraint(
+            ["first_seen_run_id"],
+            ["crawl_runs.id"],
+            name="fk_scholar_publications_first_seen_run_id_crawl_runs",
+            ondelete="SET NULL",
+        ),
+        sa.ForeignKeyConstraint(
+            ["publication_id"],
+            ["publications.id"],
+            name="fk_scholar_publications_publication_id_publications",
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["scholar_profile_id"],
+            ["scholar_profiles.id"],
+            name="fk_scholar_publications_scholar_profile_id_scholar_profiles",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint(
+            "scholar_profile_id",
+            "publication_id",
+            name="pk_scholar_publications",
+        ),
+    )
+    op.create_index(
+        "ix_scholar_publications_is_read",
+        "scholar_publications",
+        ["is_read"],
+        unique=False,
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_scholar_publications_is_read", table_name="scholar_publications")
+    op.drop_table("scholar_publications")
+
+    op.drop_index("ix_crawl_runs_user_start", table_name="crawl_runs")
+    op.drop_table("crawl_runs")
+
+    op.drop_index("ix_scholar_profiles_user_enabled", table_name="scholar_profiles")
+    op.drop_table("scholar_profiles")
+
+    op.drop_index("uq_publications_cluster_id_not_null", table_name="publications")
+    op.drop_table("publications")
+
+    op.drop_table("user_settings")
+    op.drop_table("users")
+
+    bind = op.get_bind()
+    run_trigger_type_enum.drop(bind, checkfirst=True)
+    run_status_enum.drop(bind, checkfirst=True)

alembic/versions/20260216_0002_add_user_admin_flag.py

@@ -0,0 +1,34 @@
+"""Add admin flag to users
+
+Revision ID: 20260216_0002
+Revises: 20260216_0001
+Create Date: 2026-02-16 17:40:00.000000
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision: str = "20260216_0002"
+down_revision: str | Sequence[str] | None = "20260216_0001"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "users",
+        sa.Column(
+            "is_admin",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.text("false"),
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("users", "is_admin")
+

alembic/versions/20260216_0003_add_ingestion_queue.py

@@ -0,0 +1,94 @@
+"""Add ingestion continuation queue table
+
+Revision ID: 20260216_0003
+Revises: 20260216_0002
+Create Date: 2026-02-16 23:45:00.000000
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision: str = "20260216_0003"
+down_revision: str | Sequence[str] | None = "20260216_0002"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "ingestion_queue_items",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.Column("scholar_profile_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "resume_cstart",
+            sa.Integer(),
+            nullable=False,
+            server_default=sa.text("0"),
+        ),
+        sa.Column("reason", sa.String(length=64), nullable=False),
+        sa.Column(
+            "attempt_count",
+            sa.Integer(),
+            nullable=False,
+            server_default=sa.text("0"),
+        ),
+        sa.Column(
+            "next_attempt_dt",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column("last_run_id", sa.Integer(), nullable=True),
+        sa.Column("last_error", sa.Text(), nullable=True),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.func.now(),
+        ),
+        sa.ForeignKeyConstraint(
+            ["last_run_id"],
+            ["crawl_runs.id"],
+            name="fk_ingestion_queue_items_last_run_id_crawl_runs",
+            ondelete="SET NULL",
+        ),
+        sa.ForeignKeyConstraint(
+            ["scholar_profile_id"],
+            ["scholar_profiles.id"],
+            name="fk_ingestion_queue_items_scholar_profile_id_scholar_profiles",
+            ondelete="CASCADE",
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["users.id"],
+            name="fk_ingestion_queue_items_user_id_users",
+            ondelete="CASCADE",
+        ),
+        sa.PrimaryKeyConstraint("id", name="pk_ingestion_queue_items"),
+        sa.UniqueConstraint(
+            "user_id",
+            "scholar_profile_id",
+            name="uq_ingestion_queue_user_scholar",
+        ),
+    )
+    op.create_index(
+        "ix_ingestion_queue_next_attempt",
+        "ingestion_queue_items",
+        ["next_attempt_dt"],
+        unique=False,
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_ingestion_queue_next_attempt", table_name="ingestion_queue_items")
+    op.drop_table("ingestion_queue_items")

alembic/versions/20260217_0004_queue_status_fields.py

@@ -0,0 +1,96 @@
+"""Add queue status and dropped diagnostics fields
+
+Revision ID: 20260217_0004
+Revises: 20260216_0003
+Create Date: 2026-02-17 10:15:00.000000
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision: str = "20260217_0004"
+down_revision: str | Sequence[str] | None = "20260216_0003"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    columns = {column["name"] for column in inspector.get_columns("ingestion_queue_items")}
+    indexes = {index["name"] for index in inspector.get_indexes("ingestion_queue_items")}
+    checks = {
+        constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items")
+    }
+
+    if "status" not in columns:
+        op.add_column(
+            "ingestion_queue_items",
+            sa.Column(
+                "status",
+                sa.String(length=16),
+                nullable=False,
+                server_default=sa.text("'queued'"),
+            ),
+        )
+        columns.add("status")
+
+    if "dropped_reason" not in columns:
+        op.add_column(
+            "ingestion_queue_items",
+            sa.Column("dropped_reason", sa.String(length=128), nullable=True),
+        )
+
+    if "dropped_at" not in columns:
+        op.add_column(
+            "ingestion_queue_items",
+            sa.Column("dropped_at", sa.DateTime(timezone=True), nullable=True),
+        )
+
+    if "status" in columns:
+        op.execute(
+            """
+            UPDATE ingestion_queue_items
+            SET status = 'queued'
+            WHERE status IS NULL OR status = ''
+            """
+        )
+
+    if "ingestion_queue_status_valid" not in checks:
+        op.create_check_constraint(
+            "ingestion_queue_status_valid",
+            "ingestion_queue_items",
+            "status IN ('queued', 'retrying', 'dropped')",
+        )
+
+    if "ix_ingestion_queue_status_next_attempt" not in indexes:
+        op.create_index(
+            "ix_ingestion_queue_status_next_attempt",
+            "ingestion_queue_items",
+            ["status", "next_attempt_dt"],
+            unique=False,
+        )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    columns = {column["name"] for column in inspector.get_columns("ingestion_queue_items")}
+    indexes = {index["name"] for index in inspector.get_indexes("ingestion_queue_items")}
+    checks = {
+        constraint["name"] for constraint in inspector.get_check_constraints("ingestion_queue_items")
+    }
+
+    if "ix_ingestion_queue_status_next_attempt" in indexes:
+        op.drop_index("ix_ingestion_queue_status_next_attempt", table_name="ingestion_queue_items")
+    if "ingestion_queue_status_valid" in checks:
+        op.drop_constraint("ingestion_queue_status_valid", "ingestion_queue_items", type_="check")
+    if "dropped_at" in columns:
+        op.drop_column("ingestion_queue_items", "dropped_at")
+    if "dropped_reason" in columns:
+        op.drop_column("ingestion_queue_items", "dropped_reason")
+    if "status" in columns:
+        op.drop_column("ingestion_queue_items", "status")

app/__init__.py

@@ -0,0 +1 @@
+

app/api/__init__.py

@@ -0,0 +1,2 @@
+from __future__ import annotations
+

app/api/deps.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from fastapi import Depends, Request
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import ApiException
+from app.db.models import User
+from app.db.session import get_db_session
+from app.web import common as web_common
+
+
+async def get_api_current_user(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> User:
+    current_user = await web_common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        raise ApiException(
+            status_code=401,
+            code="auth_required",
+            message="Authentication required.",
+        )
+    return current_user
+
+
+async def get_api_admin_user(
+    current_user: User = Depends(get_api_current_user),
+) -> User:
+    if not current_user.is_admin:
+        raise ApiException(
+            status_code=403,
+            code="forbidden",
+            message="Admin access required.",
+        )
+    return current_user
+

app/api/errors.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.exception_handlers import (
+    http_exception_handler as fastapi_http_exception_handler,
+)
+from fastapi.exception_handlers import (
+    request_validation_exception_handler as fastapi_validation_exception_handler,
+)
+from fastapi.exceptions import RequestValidationError
+
+from app.api.responses import error_response
+
+
+def _is_api_path(path: str) -> bool:
+    return path.startswith("/api/")
+
+
+def _status_code_to_error_code(status_code: int) -> str:
+    mapping = {
+        400: "bad_request",
+        401: "auth_required",
+        403: "forbidden",
+        404: "not_found",
+        409: "conflict",
+        422: "validation_error",
+        429: "rate_limited",
+        500: "internal_error",
+    }
+    return mapping.get(status_code, "error")
+
+
+class ApiException(Exception):
+    def __init__(
+        self,
+        *,
+        status_code: int,
+        code: str,
+        message: str,
+        details: Any | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+        self.code = code
+        self.message = message
+        self.details = details
+
+
+def register_api_exception_handlers(app: FastAPI) -> None:
+    @app.exception_handler(ApiException)
+    async def _handle_api_exception(request: Request, exc: ApiException):
+        return error_response(
+            request,
+            status_code=exc.status_code,
+            code=exc.code,
+            message=exc.message,
+            details=exc.details,
+        )
+
+    @app.exception_handler(HTTPException)
+    async def _handle_http_exception(request: Request, exc: HTTPException):
+        if not _is_api_path(request.url.path):
+            return await fastapi_http_exception_handler(request, exc)
+        return error_response(
+            request,
+            status_code=exc.status_code,
+            code=_status_code_to_error_code(exc.status_code),
+            message=str(exc.detail) if exc.detail is not None else "Request failed.",
+            details=exc.detail if isinstance(exc.detail, (dict, list)) else None,
+        )
+
+    @app.exception_handler(RequestValidationError)
+    async def _handle_validation_exception(request: Request, exc: RequestValidationError):
+        if not _is_api_path(request.url.path):
+            return await fastapi_validation_exception_handler(request, exc)
+        return error_response(
+            request,
+            status_code=422,
+            code="validation_error",
+            message="Request validation failed.",
+            details=exc.errors(),
+        )
+

app/api/responses.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi.encoders import jsonable_encoder
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+
+
+def _request_id(request: Request) -> str | None:
+    request_state = getattr(request, "state", None)
+    if request_state is None:
+        return None
+    return getattr(request_state, "request_id", None)
+
+
+def success_payload(
+    request: Request,
+    *,
+    data: Any,
+) -> dict[str, Any]:
+    return {
+        "data": data,
+        "meta": {
+            "request_id": _request_id(request),
+        },
+    }
+
+
+def success_response(
+    request: Request,
+    *,
+    data: Any,
+    status_code: int = 200,
+) -> JSONResponse:
+    payload = success_payload(request, data=data)
+    return JSONResponse(
+        status_code=status_code,
+        content=jsonable_encoder(payload),
+    )
+
+
+def error_response(
+    request: Request,
+    *,
+    status_code: int,
+    code: str,
+    message: str,
+    details: Any | None = None,
+) -> JSONResponse:
+    payload = {
+        "error": {
+            "code": code,
+            "message": message,
+            "details": details,
+        },
+        "meta": {
+            "request_id": _request_id(request),
+        },
+    }
+    return JSONResponse(
+        status_code=status_code,
+        content=jsonable_encoder(payload),
+    )

app/api/router.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+from app.api.routers import admin, auth, publications, runs, scholars, settings
+
+router = APIRouter(prefix="/api/v1")
+router.include_router(auth.router)
+router.include_router(admin.router)
+router.include_router(scholars.router)
+router.include_router(settings.router)
+router.include_router(runs.router)
+router.include_router(publications.router)

app/api/routers/__init__.py

@@ -0,0 +1,2 @@
+from __future__ import annotations
+

app/api/routers/admin.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import logging
+
+from fastapi import APIRouter, Depends, Request
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.deps import get_api_admin_user
+from app.api.errors import ApiException
+from app.api.responses import success_payload
+from app.api.schemas import (
+    AdminResetPasswordRequest,
+    AdminUserActiveUpdateRequest,
+    AdminUserCreateRequest,
+    AdminUserEnvelope,
+    AdminUsersListEnvelope,
+    MessageEnvelope,
+)
+from app.auth.deps import get_auth_service
+from app.auth.service import AuthService
+from app.db.models import User
+from app.db.session import get_db_session
+from app.services import users as user_service
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/admin/users", tags=["api-admin-users"])
+
+
+def _serialize_user(user: User) -> dict[str, object]:
+    return {
+        "id": int(user.id),
+        "email": user.email,
+        "is_active": bool(user.is_active),
+        "is_admin": bool(user.is_admin),
+        "created_at": user.created_at,
+        "updated_at": user.updated_at,
+    }
+
+
+@router.get(
+    "",
+    response_model=AdminUsersListEnvelope,
+)
+async def list_users(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    admin_user: User = Depends(get_api_admin_user),
+):
+    users = await user_service.list_users(db_session)
+    logger.info(
+        "api.admin.users_listed",
+        extra={
+            "event": "api.admin.users_listed",
+            "admin_user_id": int(admin_user.id),
+            "user_count": len(users),
+        },
+    )
+    return success_payload(
+        request,
+        data={
+            "users": [_serialize_user(user) for user in users],
+        },
+    )
+
+
+@router.post(
+    "",
+    response_model=AdminUserEnvelope,
+    status_code=201,
+)
+async def create_user(
+    payload: AdminUserCreateRequest,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    auth_service: AuthService = Depends(get_auth_service),
+    admin_user: User = Depends(get_api_admin_user),
+):
+    try:
+        validated_email = user_service.validate_email(payload.email)
+        validated_password = user_service.validate_password(payload.password)
+        created_user = await user_service.create_user(
+            db_session,
+            email=validated_email,
+            password_hash=auth_service.hash_password(validated_password),
+            is_admin=bool(payload.is_admin),
+        )
+    except user_service.UserServiceError as exc:
+        raise ApiException(
+            status_code=400,
+            code="invalid_user_input",
+            message=str(exc),
+        ) from exc
+
+    logger.info(
+        "api.admin.user_created",
+        extra={
+            "event": "api.admin.user_created",
+            "admin_user_id": int(admin_user.id),
+            "target_user_id": int(created_user.id),
+            "target_is_admin": bool(created_user.is_admin),
+        },
+    )
+    return success_payload(
+        request,
+        data=_serialize_user(created_user),
+    )
+
+
+@router.patch(
+    "/{user_id}/active",
+    response_model=AdminUserEnvelope,
+)
+async def set_user_active(
+    user_id: int,
+    payload: AdminUserActiveUpdateRequest,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    admin_user: User = Depends(get_api_admin_user),
+):
+    target_user = await user_service.get_user_by_id(db_session, user_id)
+    if target_user is None:
+        raise ApiException(
+            status_code=404,
+            code="user_not_found",
+            message="User not found.",
+        )
+    if (
+        int(target_user.id) == int(admin_user.id)
+        and bool(target_user.is_active)
+        and not bool(payload.is_active)
+    ):
+        raise ApiException(
+            status_code=400,
+            code="cannot_deactivate_self",
+            message="You cannot deactivate your own account.",
+        )
+    updated_user = await user_service.set_user_active(
+        db_session,
+        user=target_user,
+        is_active=bool(payload.is_active),
+    )
+    logger.info(
+        "api.admin.user_active_updated",
+        extra={
+            "event": "api.admin.user_active_updated",
+            "admin_user_id": int(admin_user.id),
+            "target_user_id": int(updated_user.id),
+            "is_active": bool(updated_user.is_active),
+        },
+    )
+    return success_payload(
+        request,
+        data=_serialize_user(updated_user),
+    )
+
+
+@router.post(
+    "/{user_id}/reset-password",
+    response_model=MessageEnvelope,
+)
+async def reset_user_password(
+    user_id: int,
+    payload: AdminResetPasswordRequest,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    auth_service: AuthService = Depends(get_auth_service),
+    admin_user: User = Depends(get_api_admin_user),
+):
+    target_user = await user_service.get_user_by_id(db_session, user_id)
+    if target_user is None:
+        raise ApiException(
+            status_code=404,
+            code="user_not_found",
+            message="User not found.",
+        )
+    try:
+        validated_password = user_service.validate_password(payload.new_password)
+    except user_service.UserServiceError as exc:
+        raise ApiException(
+            status_code=400,
+            code="invalid_password",
+            message=str(exc),
+        ) from exc
+
+    await user_service.set_user_password_hash(
+        db_session,
+        user=target_user,
+        password_hash=auth_service.hash_password(validated_password),
+    )
+    logger.info(
+        "api.admin.user_password_reset",
+        extra={
+            "event": "api.admin.user_password_reset",
+            "admin_user_id": int(admin_user.id),
+            "target_user_id": int(target_user.id),
+        },
+    )
+    return success_payload(
+        request,
+        data={"message": f"Password reset: {target_user.email}"},
+    )

app/api/routers/auth.py

@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+import logging
+
+from fastapi import APIRouter, Depends, Request
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.errors import ApiException
+from app.api.deps import get_api_current_user
+from app.api.responses import success_payload
+from app.api.schemas import (
+    AuthMeEnvelope,
+    ChangePasswordRequest,
+    CsrfBootstrapEnvelope,
+    LoginEnvelope,
+    LoginRequest,
+    MessageEnvelope,
+)
+from app.auth.deps import get_auth_service, get_login_rate_limiter
+from app.auth.rate_limit import SlidingWindowRateLimiter
+from app.auth.service import AuthService
+from app.auth.session import set_session_user
+from app.db.models import User
+from app.db.session import get_db_session
+from app.security.csrf import ensure_csrf_token
+from app.services import users as user_service
+from app.web import common as web_common
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/auth", tags=["api-auth"])
+
+
+@router.post(
+    "/login",
+    response_model=LoginEnvelope,
+)
+async def login(
+    payload: LoginRequest,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    auth_service: AuthService = Depends(get_auth_service),
+    rate_limiter: SlidingWindowRateLimiter = Depends(get_login_rate_limiter),
+):
+    limiter_key = web_common.login_rate_limit_key(request, payload.email)
+    decision = rate_limiter.check(limiter_key)
+    normalized_email = payload.email.strip().lower()
+    if not decision.allowed:
+        logger.warning(
+            "api.auth.login_rate_limited",
+            extra={
+                "event": "api.auth.login_rate_limited",
+                "email": normalized_email,
+                "retry_after_seconds": decision.retry_after_seconds,
+            },
+        )
+        raise ApiException(
+            status_code=429,
+            code="rate_limited",
+            message="Too many login attempts. Please try again later.",
+            details={"retry_after_seconds": decision.retry_after_seconds},
+        )
+
+    user = await auth_service.authenticate_user(
+        db_session,
+        email=payload.email,
+        password=payload.password,
+    )
+    if user is None:
+        rate_limiter.record_failure(limiter_key)
+        logger.info(
+            "api.auth.login_failed",
+            extra={
+                "event": "api.auth.login_failed",
+                "email": normalized_email,
+            },
+        )
+        raise ApiException(
+            status_code=401,
+            code="invalid_credentials",
+            message="Invalid email or password.",
+        )
+
+    rate_limiter.reset(limiter_key)
+    set_session_user(
+        request,
+        user_id=user.id,
+        email=user.email,
+        is_admin=user.is_admin,
+    )
+    logger.info(
+        "api.auth.login_succeeded",
+        extra={
+            "event": "api.auth.login_succeeded",
+            "user_id": user.id,
+            "is_admin": user.is_admin,
+        },
+    )
+    return success_payload(
+        request,
+        data={
+            "authenticated": True,
+            "csrf_token": ensure_csrf_token(request),
+            "user": {
+                "id": int(user.id),
+                "email": user.email,
+                "is_admin": bool(user.is_admin),
+                "is_active": bool(user.is_active),
+            },
+        },
+    )
+
+
+@router.get(
+    "/me",
+    response_model=AuthMeEnvelope,
+)
+async def get_current_session(
+    request: Request,
+    current_user: User = Depends(get_api_current_user),
+):
+    return success_payload(
+        request,
+        data={
+            "authenticated": True,
+            "csrf_token": ensure_csrf_token(request),
+            "user": {
+                "id": int(current_user.id),
+                "email": current_user.email,
+                "is_admin": bool(current_user.is_admin),
+                "is_active": bool(current_user.is_active),
+            },
+        },
+    )
+
+
+@router.get(
+    "/csrf",
+    response_model=CsrfBootstrapEnvelope,
+)
+async def get_csrf_bootstrap(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+):
+    current_user = await web_common.get_authenticated_user(request, db_session)
+    return success_payload(
+        request,
+        data={
+            "csrf_token": ensure_csrf_token(request),
+            "authenticated": current_user is not None,
+        },
+    )
+
+
+@router.post(
+    "/change-password",
+    response_model=MessageEnvelope,
+)
+async def change_password(
+    payload: ChangePasswordRequest,
+    request: Request,
+    current_user: User = Depends(get_api_current_user),
+    db_session: AsyncSession = Depends(get_db_session),
+    auth_service: AuthService = Depends(get_auth_service),
+):
+    if not auth_service.verify_password(
+        password_hash=current_user.password_hash,
+        password=payload.current_password,
+    ):
+        raise ApiException(
+            status_code=400,
+            code="invalid_current_password",
+            message="Current password is incorrect.",
+        )
+    if payload.new_password != payload.confirm_password:
+        raise ApiException(
+            status_code=400,
+            code="password_confirmation_mismatch",
+            message="New password and confirmation do not match.",
+        )
+    try:
+        validated_password = user_service.validate_password(payload.new_password)
+    except user_service.UserServiceError as exc:
+        raise ApiException(
+            status_code=400,
+            code="invalid_password",
+            message=str(exc),
+        ) from exc
+
+    await user_service.set_user_password_hash(
+        db_session,
+        user=current_user,
+        password_hash=auth_service.hash_password(validated_password),
+    )
+    logger.info(
+        "api.auth.password_changed",
+        extra={
+            "event": "api.auth.password_changed",
+            "user_id": int(current_user.id),
+        },
+    )
+    return success_payload(
+        request,
+        data={"message": "Password updated successfully."},
+    )
+
+
+@router.post(
+    "/logout",
+    response_model=MessageEnvelope,
+)
+async def logout(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+):
+    current_user = await web_common.get_authenticated_user(request, db_session)
+    web_common.invalidate_session(request)
+    logger.info(
+        "api.auth.logout",
+        extra={
+            "event": "api.auth.logout",
+            "user_id": int(current_user.id) if current_user is not None else None,
+        },
+    )
+    return success_payload(
+        request,
+        data={
+            "message": "Logged out.",
+        },
+    )

app/api/routers/publications.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import logging
+from typing import Literal
+
+from fastapi import APIRouter, Depends, Query, Request
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.deps import get_api_current_user
+from app.api.errors import ApiException
+from app.api.responses import success_payload
+from app.api.schemas import MarkAllReadEnvelope, PublicationsListEnvelope
+from app.db.models import User
+from app.db.session import get_db_session
+from app.services import publications as publication_service
+from app.services import scholars as scholar_service
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/publications", tags=["api-publications"])
+
+
+@router.get(
+    "",
+    response_model=PublicationsListEnvelope,
+)
+async def list_publications(
+    request: Request,
+    mode: Literal["all", "new"] | None = Query(default=None),
+    scholar_profile_id: int | None = Query(default=None, ge=1),
+    limit: int = Query(default=300, ge=1, le=1000),
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    resolved_mode = publication_service.resolve_mode(mode)
+    selected_scholar_id = scholar_profile_id
+    if selected_scholar_id is not None:
+        selected_profile = await scholar_service.get_user_scholar_by_id(
+            db_session,
+            user_id=current_user.id,
+            scholar_profile_id=selected_scholar_id,
+        )
+        if selected_profile is None:
+            raise ApiException(
+                status_code=404,
+                code="scholar_not_found",
+                message="Scholar filter not found.",
+            )
+
+    publications = await publication_service.list_for_user(
+        db_session,
+        user_id=current_user.id,
+        mode=resolved_mode,
+        scholar_profile_id=selected_scholar_id,
+        limit=limit,
+    )
+    new_count = await publication_service.count_for_user(
+        db_session,
+        user_id=current_user.id,
+        mode=publication_service.MODE_NEW,
+        scholar_profile_id=selected_scholar_id,
+    )
+    total_count = await publication_service.count_for_user(
+        db_session,
+        user_id=current_user.id,
+        mode=publication_service.MODE_ALL,
+        scholar_profile_id=selected_scholar_id,
+    )
+    return success_payload(
+        request,
+        data={
+            "mode": resolved_mode,
+            "selected_scholar_profile_id": selected_scholar_id,
+            "new_count": new_count,
+            "total_count": total_count,
+            "publications": [
+                {
+                    "publication_id": item.publication_id,
+                    "scholar_profile_id": item.scholar_profile_id,
+                    "scholar_label": item.scholar_label,
+                    "title": item.title,
+                    "year": item.year,
+                    "citation_count": item.citation_count,
+                    "venue_text": item.venue_text,
+                    "pub_url": item.pub_url,
+                    "is_read": item.is_read,
+                    "first_seen_at": item.first_seen_at,
+                    "is_new_in_latest_run": item.is_new_in_latest_run,
+                }
+                for item in publications
+            ],
+        },
+    )
+
+
+@router.post(
+    "/mark-all-read",
+    response_model=MarkAllReadEnvelope,
+)
+async def mark_all_publications_read(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    updated_count = await publication_service.mark_all_unread_as_read_for_user(
+        db_session,
+        user_id=current_user.id,
+    )
+    logger.info(
+        "api.publications.mark_all_read",
+        extra={
+            "event": "api.publications.mark_all_read",
+            "user_id": current_user.id,
+            "updated_count": updated_count,
+        },
+    )
+    return success_payload(
+        request,
+        data={
+            "message": "Marked all unread publications as read.",
+            "updated_count": updated_count,
+        },
+    )

app/api/routers/runs.py

@@ -0,0 +1,560 @@
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+
+from fastapi import APIRouter, Depends, Query, Request
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.deps import get_api_current_user
+from app.api.errors import ApiException
+from app.api.responses import success_payload
+from app.api.schemas import (
+    ManualRunEnvelope,
+    QueueClearEnvelope,
+    QueueItemEnvelope,
+    QueueListEnvelope,
+    RunDetailEnvelope,
+    RunsListEnvelope,
+)
+from app.db.models import RunStatus, RunTriggerType, User
+from app.db.session import get_db_session
+from app.services import ingestion as ingestion_service
+from app.services import runs as run_service
+from app.services import user_settings as user_settings_service
+from app.settings import settings
+from app.web.deps import get_ingestion_service
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/runs", tags=["api-runs"])
+
+IDEMPOTENCY_HEADER = "Idempotency-Key"
+IDEMPOTENCY_MAX_LENGTH = 128
+IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
+
+
+def _int_value(value: Any, default: int = 0) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _str_value(value: Any) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text if text else None
+
+
+def _bool_value(value: Any, default: bool = False) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        lowered = value.strip().lower()
+        if lowered in {"1", "true", "yes", "on"}:
+            return True
+        if lowered in {"0", "false", "no", "off"}:
+            return False
+    return default
+
+
+def _normalize_idempotency_key(raw_value: str | None) -> str | None:
+    if raw_value is None:
+        return None
+    candidate = raw_value.strip()
+    if not candidate:
+        return None
+    if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate):
+        raise ApiException(
+            status_code=400,
+            code="invalid_idempotency_key",
+            message=(
+                "Invalid Idempotency-Key. Use 8-128 characters from: "
+                "A-Z a-z 0-9 . _ : -"
+            ),
+        )
+    return candidate
+
+
+def _serialize_run(run) -> dict[str, Any]:
+    summary = run_service.extract_run_summary(run.error_log)
+    return {
+        "id": int(run.id),
+        "trigger_type": run.trigger_type.value,
+        "status": run.status.value,
+        "start_dt": run.start_dt,
+        "end_dt": run.end_dt,
+        "scholar_count": int(run.scholar_count or 0),
+        "new_publication_count": int(run.new_pub_count or 0),
+        "failed_count": int(summary["failed_count"]),
+        "partial_count": int(summary["partial_count"]),
+    }
+
+
+def _serialize_queue_item(item) -> dict[str, Any]:
+    return {
+        "id": int(item.id),
+        "scholar_profile_id": int(item.scholar_profile_id),
+        "scholar_label": item.scholar_label,
+        "status": item.status,
+        "reason": item.reason,
+        "dropped_reason": item.dropped_reason,
+        "attempt_count": int(item.attempt_count),
+        "resume_cstart": int(item.resume_cstart),
+        "next_attempt_dt": item.next_attempt_dt,
+        "updated_at": item.updated_at,
+        "last_error": item.last_error,
+        "last_run_id": item.last_run_id,
+    }
+
+
+def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]:
+    if not isinstance(value, list):
+        return []
+    normalized: list[dict[str, Any]] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        normalized.append(
+            {
+                "attempt": _int_value(item.get("attempt"), 0),
+                "cstart": _int_value(item.get("cstart"), 0),
+                "state": _str_value(item.get("state")),
+                "state_reason": _str_value(item.get("state_reason")),
+                "status_code": (
+                    _int_value(item.get("status_code"))
+                    if item.get("status_code") is not None
+                    else None
+                ),
+                "fetch_error": _str_value(item.get("fetch_error")),
+            }
+        )
+    return normalized
+
+
+def _normalize_page_logs(value: Any) -> list[dict[str, Any]]:
+    if not isinstance(value, list):
+        return []
+    normalized: list[dict[str, Any]] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        warning_codes = item.get("warning_codes")
+        normalized.append(
+            {
+                "page": _int_value(item.get("page"), 0),
+                "cstart": _int_value(item.get("cstart"), 0),
+                "state": _str_value(item.get("state")) or "unknown",
+                "state_reason": _str_value(item.get("state_reason")),
+                "status_code": (
+                    _int_value(item.get("status_code"))
+                    if item.get("status_code") is not None
+                    else None
+                ),
+                "publication_count": _int_value(item.get("publication_count"), 0),
+                "attempt_count": _int_value(item.get("attempt_count"), 0),
+                "has_show_more_button": _bool_value(item.get("has_show_more_button"), False),
+                "articles_range": _str_value(item.get("articles_range")),
+                "warning_codes": [
+                    str(code)
+                    for code in (warning_codes if isinstance(warning_codes, list) else [])
+                ],
+            }
+        )
+    return normalized
+
+
+def _normalize_debug(value: Any) -> dict[str, Any] | None:
+    if not isinstance(value, dict):
+        return None
+    marker_counts = value.get("marker_counts_nonzero")
+    warning_codes = value.get("warning_codes")
+    return {
+        "status_code": (
+            _int_value(value.get("status_code"))
+            if value.get("status_code") is not None
+            else None
+        ),
+        "final_url": _str_value(value.get("final_url")),
+        "fetch_error": _str_value(value.get("fetch_error")),
+        "body_sha256": _str_value(value.get("body_sha256")),
+        "body_length": (
+            _int_value(value.get("body_length"))
+            if value.get("body_length") is not None
+            else None
+        ),
+        "has_show_more_button": (
+            _bool_value(value.get("has_show_more_button"), False)
+            if value.get("has_show_more_button") is not None
+            else None
+        ),
+        "articles_range": _str_value(value.get("articles_range")),
+        "state_reason": _str_value(value.get("state_reason")),
+        "warning_codes": [
+            str(code)
+            for code in (warning_codes if isinstance(warning_codes, list) else [])
+        ],
+        "marker_counts_nonzero": {
+            str(key): _int_value(count, 0)
+            for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else [])
+        },
+        "page_logs": _normalize_page_logs(value.get("page_logs")),
+        "attempt_log": _normalize_attempt_log(value.get("attempt_log")),
+    }
+
+
+def _normalize_scholar_result(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {
+            "scholar_profile_id": 0,
+            "scholar_id": "unknown",
+            "state": "unknown",
+            "state_reason": None,
+            "outcome": "failed",
+            "attempt_count": 0,
+            "publication_count": 0,
+            "start_cstart": 0,
+            "continuation_cstart": None,
+            "continuation_enqueued": False,
+            "continuation_cleared": False,
+            "warnings": [],
+            "error": None,
+            "debug": None,
+        }
+    warnings = value.get("warnings")
+    return {
+        "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0),
+        "scholar_id": _str_value(value.get("scholar_id")) or "unknown",
+        "state": _str_value(value.get("state")) or "unknown",
+        "state_reason": _str_value(value.get("state_reason")),
+        "outcome": _str_value(value.get("outcome")) or "failed",
+        "attempt_count": _int_value(value.get("attempt_count"), 0),
+        "publication_count": _int_value(value.get("publication_count"), 0),
+        "start_cstart": _int_value(value.get("start_cstart"), 0),
+        "continuation_cstart": (
+            _int_value(value.get("continuation_cstart"))
+            if value.get("continuation_cstart") is not None
+            else None
+        ),
+        "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False),
+        "continuation_cleared": _bool_value(value.get("continuation_cleared"), False),
+        "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])],
+        "error": _str_value(value.get("error")),
+        "debug": _normalize_debug(value.get("debug")),
+    }
+
+
+def _manual_run_payload_from_run(
+    run,
+    *,
+    idempotency_key: str | None,
+    reused_existing_run: bool,
+) -> dict[str, Any]:
+    summary = run_service.extract_run_summary(run.error_log)
+    return {
+        "run_id": int(run.id),
+        "status": run.status.value,
+        "scholar_count": int(run.scholar_count or 0),
+        "succeeded_count": int(summary["succeeded_count"]),
+        "failed_count": int(summary["failed_count"]),
+        "partial_count": int(summary["partial_count"]),
+        "new_publication_count": int(run.new_pub_count or 0),
+        "reused_existing_run": reused_existing_run,
+        "idempotency_key": idempotency_key,
+    }
+
+
+@router.get(
+    "",
+    response_model=RunsListEnvelope,
+)
+async def list_runs(
+    request: Request,
+    failed_only: bool = Query(default=False),
+    limit: int = Query(default=100, ge=1, le=500),
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    runs = await run_service.list_runs_for_user(
+        db_session,
+        user_id=current_user.id,
+        limit=limit,
+        failed_only=failed_only,
+    )
+    return success_payload(
+        request,
+        data={
+            "runs": [_serialize_run(run) for run in runs],
+        },
+    )
+
+
+@router.get(
+    "/{run_id}",
+    response_model=RunDetailEnvelope,
+)
+async def get_run(
+    run_id: int,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    run = await run_service.get_run_for_user(
+        db_session,
+        user_id=current_user.id,
+        run_id=run_id,
+    )
+    if run is None:
+        raise ApiException(
+            status_code=404,
+            code="run_not_found",
+            message="Run not found.",
+        )
+    error_log = run.error_log if isinstance(run.error_log, dict) else {}
+    scholar_results = error_log.get("scholar_results")
+    if not isinstance(scholar_results, list):
+        scholar_results = []
+    return success_payload(
+        request,
+        data={
+            "run": _serialize_run(run),
+            "summary": run_service.extract_run_summary(error_log),
+            "scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
+        },
+    )
+
+
+@router.post(
+    "/manual",
+    response_model=ManualRunEnvelope,
+)
+async def run_manual(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+    ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
+):
+    idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
+    if idempotency_key is not None:
+        previous_run = await run_service.get_manual_run_by_idempotency_key(
+            db_session,
+            user_id=current_user.id,
+            idempotency_key=idempotency_key,
+        )
+        if previous_run is not None:
+            if previous_run.status == RunStatus.RUNNING:
+                raise ApiException(
+                    status_code=409,
+                    code="run_in_progress",
+                    message="A run with this idempotency key is still in progress.",
+                    details={
+                        "run_id": int(previous_run.id),
+                        "idempotency_key": idempotency_key,
+                    },
+                )
+            return success_payload(
+                request,
+                data=_manual_run_payload_from_run(
+                    previous_run,
+                    idempotency_key=idempotency_key,
+                    reused_existing_run=True,
+                ),
+            )
+
+    user_settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=current_user.id,
+    )
+    try:
+        run_summary = await ingest_service.run_for_user(
+            db_session,
+            user_id=current_user.id,
+            trigger_type=RunTriggerType.MANUAL,
+            request_delay_seconds=user_settings.request_delay_seconds,
+            network_error_retries=settings.ingestion_network_error_retries,
+            retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+            max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
+            page_size=settings.ingestion_page_size,
+            auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
+            queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
+        )
+    except ingestion_service.RunAlreadyInProgressError as exc:
+        await db_session.rollback()
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run is already in progress for this account.",
+        ) from exc
+    except Exception as exc:
+        await db_session.rollback()
+        logger.exception(
+            "api.runs.manual_failed",
+            extra={
+                "event": "api.runs.manual_failed",
+                "user_id": current_user.id,
+            },
+        )
+        raise ApiException(
+            status_code=500,
+            code="manual_run_failed",
+            message="Manual run failed.",
+        ) from exc
+
+    if idempotency_key is not None:
+        await run_service.set_manual_run_idempotency_key(
+            db_session,
+            user_id=current_user.id,
+            run_id=run_summary.crawl_run_id,
+            idempotency_key=idempotency_key,
+        )
+
+    return success_payload(
+        request,
+        data={
+            "run_id": run_summary.crawl_run_id,
+            "status": run_summary.status.value,
+            "scholar_count": run_summary.scholar_count,
+            "succeeded_count": run_summary.succeeded_count,
+            "failed_count": run_summary.failed_count,
+            "partial_count": run_summary.partial_count,
+            "new_publication_count": run_summary.new_publication_count,
+            "reused_existing_run": False,
+            "idempotency_key": idempotency_key,
+        },
+    )
+
+
+@router.get(
+    "/queue/items",
+    response_model=QueueListEnvelope,
+)
+async def list_queue_items(
+    request: Request,
+    limit: int = Query(default=200, ge=1, le=500),
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    items = await run_service.list_queue_items_for_user(
+        db_session,
+        user_id=current_user.id,
+        limit=limit,
+    )
+    return success_payload(
+        request,
+        data={
+            "queue_items": [_serialize_queue_item(item) for item in items],
+        },
+    )
+
+
+@router.post(
+    "/queue/{queue_item_id}/retry",
+    response_model=QueueItemEnvelope,
+)
+async def retry_queue_item(
+    queue_item_id: int,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    try:
+        queue_item = await run_service.retry_queue_item_for_user(
+            db_session,
+            user_id=current_user.id,
+            queue_item_id=queue_item_id,
+        )
+    except run_service.QueueTransitionError as exc:
+        raise ApiException(
+            status_code=409,
+            code=exc.code,
+            message=exc.message,
+            details={"current_status": exc.current_status},
+        ) from exc
+    if queue_item is None:
+        raise ApiException(
+            status_code=404,
+            code="queue_item_not_found",
+            message="Queue item not found.",
+        )
+    return success_payload(
+        request,
+        data=_serialize_queue_item(queue_item),
+    )
+
+
+@router.post(
+    "/queue/{queue_item_id}/drop",
+    response_model=QueueItemEnvelope,
+)
+async def drop_queue_item(
+    queue_item_id: int,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    try:
+        dropped = await run_service.drop_queue_item_for_user(
+            db_session,
+            user_id=current_user.id,
+            queue_item_id=queue_item_id,
+        )
+    except run_service.QueueTransitionError as exc:
+        raise ApiException(
+            status_code=409,
+            code=exc.code,
+            message=exc.message,
+            details={"current_status": exc.current_status},
+        ) from exc
+    if dropped is None:
+        raise ApiException(
+            status_code=404,
+            code="queue_item_not_found",
+            message="Queue item not found.",
+        )
+    return success_payload(
+        request,
+        data=_serialize_queue_item(dropped),
+    )
+
+
+@router.delete(
+    "/queue/{queue_item_id}",
+    response_model=QueueClearEnvelope,
+)
+async def clear_queue_item(
+    queue_item_id: int,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    try:
+        deleted = await run_service.clear_queue_item_for_user(
+            db_session,
+            user_id=current_user.id,
+            queue_item_id=queue_item_id,
+        )
+    except run_service.QueueTransitionError as exc:
+        raise ApiException(
+            status_code=409,
+            code=exc.code,
+            message=exc.message,
+            details={"current_status": exc.current_status},
+        ) from exc
+    if deleted is None:
+        raise ApiException(
+            status_code=404,
+            code="queue_item_not_found",
+            message="Queue item not found.",
+        )
+    return success_payload(
+        request,
+        data={
+            "queue_item_id": deleted.queue_item_id,
+            "previous_status": deleted.previous_status,
+            "status": "cleared",
+            "message": "Queue item cleared.",
+        },
+    )

app/api/routers/scholars.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+import logging
+
+from fastapi import APIRouter, Depends, Request
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.deps import get_api_current_user
+from app.api.errors import ApiException
+from app.api.responses import success_payload
+from app.api.schemas import (
+    MessageEnvelope,
+    ScholarCreateRequest,
+    ScholarEnvelope,
+    ScholarsListEnvelope,
+)
+from app.db.models import User
+from app.db.session import get_db_session
+from app.services import scholars as scholar_service
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/scholars", tags=["api-scholars"])
+
+
+def _serialize_scholar(profile) -> dict[str, object]:
+    return {
+        "id": int(profile.id),
+        "scholar_id": profile.scholar_id,
+        "display_name": profile.display_name,
+        "is_enabled": bool(profile.is_enabled),
+        "baseline_completed": bool(profile.baseline_completed),
+        "last_run_dt": profile.last_run_dt,
+        "last_run_status": (
+            profile.last_run_status.value if profile.last_run_status is not None else None
+        ),
+    }
+
+
+@router.get(
+    "",
+    response_model=ScholarsListEnvelope,
+)
+async def list_scholars(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    scholars = await scholar_service.list_scholars_for_user(
+        db_session,
+        user_id=current_user.id,
+    )
+    return success_payload(
+        request,
+        data={
+            "scholars": [_serialize_scholar(profile) for profile in scholars],
+        },
+    )
+
+
+@router.post(
+    "",
+    response_model=ScholarEnvelope,
+    status_code=201,
+)
+async def create_scholar(
+    payload: ScholarCreateRequest,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    try:
+        created = await scholar_service.create_scholar_for_user(
+            db_session,
+            user_id=current_user.id,
+            scholar_id=payload.scholar_id,
+            display_name=payload.display_name or "",
+        )
+    except scholar_service.ScholarServiceError as exc:
+        raise ApiException(
+            status_code=400,
+            code="invalid_scholar",
+            message=str(exc),
+        ) from exc
+    logger.info(
+        "api.scholars.created",
+        extra={
+            "event": "api.scholars.created",
+            "user_id": current_user.id,
+            "scholar_profile_id": created.id,
+        },
+    )
+    return success_payload(
+        request,
+        data=_serialize_scholar(created),
+    )
+
+
+@router.patch(
+    "/{scholar_profile_id}/toggle",
+    response_model=ScholarEnvelope,
+)
+async def toggle_scholar(
+    scholar_profile_id: int,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    profile = await scholar_service.get_user_scholar_by_id(
+        db_session,
+        user_id=current_user.id,
+        scholar_profile_id=scholar_profile_id,
+    )
+    if profile is None:
+        raise ApiException(
+            status_code=404,
+            code="scholar_not_found",
+            message="Scholar not found.",
+        )
+    updated = await scholar_service.toggle_scholar_enabled(db_session, profile=profile)
+    logger.info(
+        "api.scholars.toggled",
+        extra={
+            "event": "api.scholars.toggled",
+            "user_id": current_user.id,
+            "scholar_profile_id": updated.id,
+            "is_enabled": updated.is_enabled,
+        },
+    )
+    return success_payload(
+        request,
+        data=_serialize_scholar(updated),
+    )
+
+
+@router.delete(
+    "/{scholar_profile_id}",
+    response_model=MessageEnvelope,
+)
+async def delete_scholar(
+    scholar_profile_id: int,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    profile = await scholar_service.get_user_scholar_by_id(
+        db_session,
+        user_id=current_user.id,
+        scholar_profile_id=scholar_profile_id,
+    )
+    if profile is None:
+        raise ApiException(
+            status_code=404,
+            code="scholar_not_found",
+            message="Scholar not found.",
+        )
+    await scholar_service.delete_scholar(db_session, profile=profile)
+    logger.info(
+        "api.scholars.deleted",
+        extra={
+            "event": "api.scholars.deleted",
+            "user_id": current_user.id,
+            "scholar_profile_id": scholar_profile_id,
+        },
+    )
+    return success_payload(
+        request,
+        data={"message": "Scholar deleted."},
+    )

app/api/routers/settings.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import logging
+
+from fastapi import APIRouter, Depends, Request
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.deps import get_api_current_user
+from app.api.errors import ApiException
+from app.api.responses import success_payload
+from app.api.schemas import SettingsEnvelope, SettingsUpdateRequest
+from app.db.models import User
+from app.db.session import get_db_session
+from app.services import user_settings as user_settings_service
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/settings", tags=["api-settings"])
+
+
+def _serialize_settings(settings) -> dict[str, object]:
+    return {
+        "auto_run_enabled": bool(settings.auto_run_enabled),
+        "run_interval_minutes": int(settings.run_interval_minutes),
+        "request_delay_seconds": int(settings.request_delay_seconds),
+    }
+
+
+@router.get(
+    "",
+    response_model=SettingsEnvelope,
+)
+async def get_settings(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=current_user.id,
+    )
+    return success_payload(
+        request,
+        data=_serialize_settings(settings),
+    )
+
+
+@router.put(
+    "",
+    response_model=SettingsEnvelope,
+)
+async def update_settings(
+    payload: SettingsUpdateRequest,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    try:
+        parsed_interval = user_settings_service.parse_run_interval_minutes(
+            str(payload.run_interval_minutes)
+        )
+        parsed_delay = user_settings_service.parse_request_delay_seconds(
+            str(payload.request_delay_seconds)
+        )
+    except user_settings_service.UserSettingsServiceError as exc:
+        raise ApiException(
+            status_code=400,
+            code="invalid_settings",
+            message=str(exc),
+        ) from exc
+
+    settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=current_user.id,
+    )
+    updated = await user_settings_service.update_settings(
+        db_session,
+        settings=settings,
+        auto_run_enabled=bool(payload.auto_run_enabled),
+        run_interval_minutes=parsed_interval,
+        request_delay_seconds=parsed_delay,
+    )
+    logger.info(
+        "api.settings.updated",
+        extra={
+            "event": "api.settings.updated",
+            "user_id": current_user.id,
+            "auto_run_enabled": updated.auto_run_enabled,
+            "run_interval_minutes": updated.run_interval_minutes,
+            "request_delay_seconds": updated.request_delay_seconds,
+        },
+    )
+    return success_payload(
+        request,
+        data=_serialize_settings(updated),
+    )

app/api/schemas.py

@@ -0,0 +1,456 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ApiMeta(BaseModel):
+    request_id: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ApiErrorData(BaseModel):
+    code: str
+    message: str
+    details: Any | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ApiErrorEnvelope(BaseModel):
+    error: ApiErrorData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessageData(BaseModel):
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MessageEnvelope(BaseModel):
+    data: MessageData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SessionUserData(BaseModel):
+    id: int
+    email: str
+    is_admin: bool
+    is_active: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AuthMeData(BaseModel):
+    authenticated: bool
+    csrf_token: str
+    user: SessionUserData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AuthMeEnvelope(BaseModel):
+    data: AuthMeData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class CsrfBootstrapData(BaseModel):
+    csrf_token: str
+    authenticated: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class CsrfBootstrapEnvelope(BaseModel):
+    data: CsrfBootstrapData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginRequest(BaseModel):
+    email: str
+    password: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginData(BaseModel):
+    authenticated: bool
+    csrf_token: str
+    user: SessionUserData
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class LoginEnvelope(BaseModel):
+    data: LoginData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ChangePasswordRequest(BaseModel):
+    current_password: str
+    new_password: str
+    confirm_password: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarItemData(BaseModel):
+    id: int
+    scholar_id: str
+    display_name: str | None
+    is_enabled: bool
+    baseline_completed: bool
+    last_run_dt: datetime | None
+    last_run_status: str | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarsListData(BaseModel):
+    scholars: list[ScholarItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarsListEnvelope(BaseModel):
+    data: ScholarsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarCreateRequest(BaseModel):
+    scholar_id: str
+    display_name: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ScholarEnvelope(BaseModel):
+    data: ScholarItemData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunListItemData(BaseModel):
+    id: int
+    trigger_type: str
+    status: str
+    start_dt: datetime
+    end_dt: datetime | None
+    scholar_count: int
+    new_publication_count: int
+    failed_count: int
+    partial_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunsListData(BaseModel):
+    runs: list[RunListItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunsListEnvelope(BaseModel):
+    data: RunsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunSummaryData(BaseModel):
+    succeeded_count: int
+    failed_count: int
+    partial_count: int
+    failed_state_counts: dict[str, int] = Field(default_factory=dict)
+    failed_reason_counts: dict[str, int] = Field(default_factory=dict)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunAttemptLogData(BaseModel):
+    attempt: int
+    cstart: int
+    state: str | None = None
+    state_reason: str | None = None
+    status_code: int | None = None
+    fetch_error: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunPageLogData(BaseModel):
+    page: int
+    cstart: int
+    state: str
+    state_reason: str | None = None
+    status_code: int | None = None
+    publication_count: int = 0
+    attempt_count: int = 0
+    has_show_more_button: bool = False
+    articles_range: str | None = None
+    warning_codes: list[str] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDebugData(BaseModel):
+    status_code: int | None = None
+    final_url: str | None = None
+    fetch_error: str | None = None
+    body_sha256: str | None = None
+    body_length: int | None = None
+    has_show_more_button: bool | None = None
+    articles_range: str | None = None
+    state_reason: str | None = None
+    warning_codes: list[str] = Field(default_factory=list)
+    marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
+    page_logs: list[RunPageLogData] = Field(default_factory=list)
+    attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunScholarResultData(BaseModel):
+    scholar_profile_id: int
+    scholar_id: str
+    state: str
+    state_reason: str | None = None
+    outcome: str
+    attempt_count: int = 0
+    publication_count: int = 0
+    start_cstart: int = 0
+    continuation_cstart: int | None = None
+    continuation_enqueued: bool = False
+    continuation_cleared: bool = False
+    warnings: list[str] = Field(default_factory=list)
+    error: str | None = None
+    debug: RunDebugData | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDetailData(BaseModel):
+    run: RunListItemData
+    summary: RunSummaryData
+    scholar_results: list[RunScholarResultData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class RunDetailEnvelope(BaseModel):
+    data: RunDetailData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ManualRunData(BaseModel):
+    run_id: int
+    status: str
+    scholar_count: int
+    succeeded_count: int
+    failed_count: int
+    partial_count: int
+    new_publication_count: int
+    reused_existing_run: bool
+    idempotency_key: str | None = None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class ManualRunEnvelope(BaseModel):
+    data: ManualRunData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueItemData(BaseModel):
+    id: int
+    scholar_profile_id: int
+    scholar_label: str
+    status: str
+    reason: str
+    dropped_reason: str | None
+    attempt_count: int
+    resume_cstart: int
+    next_attempt_dt: datetime | None
+    updated_at: datetime
+    last_error: str | None
+    last_run_id: int | None
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueListData(BaseModel):
+    queue_items: list[QueueItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueListEnvelope(BaseModel):
+    data: QueueListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueItemEnvelope(BaseModel):
+    data: QueueItemData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueClearData(BaseModel):
+    queue_item_id: int
+    previous_status: str
+    status: str
+    message: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class QueueClearEnvelope(BaseModel):
+    data: QueueClearData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserData(BaseModel):
+    id: int
+    email: str
+    is_active: bool
+    is_admin: bool
+    created_at: datetime
+    updated_at: datetime
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUsersListData(BaseModel):
+    users: list[AdminUserData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUsersListEnvelope(BaseModel):
+    data: AdminUsersListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserCreateRequest(BaseModel):
+    email: str
+    password: str
+    is_admin: bool = False
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserActiveUpdateRequest(BaseModel):
+    is_active: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminUserEnvelope(BaseModel):
+    data: AdminUserData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AdminResetPasswordRequest(BaseModel):
+    new_password: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsData(BaseModel):
+    auto_run_enabled: bool
+    run_interval_minutes: int
+    request_delay_seconds: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsEnvelope(BaseModel):
+    data: SettingsData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SettingsUpdateRequest(BaseModel):
+    auto_run_enabled: bool
+    run_interval_minutes: int
+    request_delay_seconds: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationItemData(BaseModel):
+    publication_id: int
+    scholar_profile_id: int
+    scholar_label: str
+    title: str
+    year: int | None
+    citation_count: int
+    venue_text: str | None
+    pub_url: str | None
+    is_read: bool
+    first_seen_at: datetime
+    is_new_in_latest_run: bool
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationsListData(BaseModel):
+    mode: str
+    selected_scholar_profile_id: int | None
+    new_count: int
+    total_count: int
+    publications: list[PublicationItemData]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PublicationsListEnvelope(BaseModel):
+    data: PublicationsListData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkAllReadData(BaseModel):
+    message: str
+    updated_count: int
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class MarkAllReadEnvelope(BaseModel):
+    data: MarkAllReadData
+    meta: ApiMeta
+
+    model_config = ConfigDict(extra="forbid")

app/auth/__init__.py

@@ -0,0 +1,2 @@
+"""Authentication package for scholarr."""
+

app/auth/deps.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from functools import lru_cache
+
+from app.auth.rate_limit import SlidingWindowRateLimiter
+from app.auth.security import PasswordService
+from app.auth.service import AuthService
+from app.settings import settings
+
+
+@lru_cache
+def get_password_service() -> PasswordService:
+    return PasswordService()
+
+
+@lru_cache
+def get_auth_service() -> AuthService:
+    return AuthService(password_service=get_password_service())
+
+
+@lru_cache
+def get_login_rate_limiter() -> SlidingWindowRateLimiter:
+    return SlidingWindowRateLimiter(
+        max_attempts=settings.login_rate_limit_attempts,
+        window_seconds=settings.login_rate_limit_window_seconds,
+    )
+

app/auth/rate_limit.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from collections import defaultdict, deque
+from collections.abc import Callable
+from dataclasses import dataclass
+from math import ceil
+from threading import Lock
+from time import monotonic
+
+
+@dataclass(frozen=True)
+class RateLimitDecision:
+    allowed: bool
+    retry_after_seconds: int = 0
+
+
+class SlidingWindowRateLimiter:
+    def __init__(
+        self,
+        *,
+        max_attempts: int,
+        window_seconds: int,
+        now: Callable[[], float] = monotonic,
+    ) -> None:
+        self._max_attempts = max_attempts
+        self._window_seconds = window_seconds
+        self._now = now
+        self._attempts: dict[str, deque[float]] = defaultdict(deque)
+        self._lock = Lock()
+
+    def check(self, key: str) -> RateLimitDecision:
+        with self._lock:
+            now_value = self._now()
+            attempts = self._attempts[key]
+            self._trim_expired(attempts, now_value)
+            if len(attempts) >= self._max_attempts:
+                retry_after = self._window_seconds - (now_value - attempts[0])
+                return RateLimitDecision(
+                    allowed=False,
+                    retry_after_seconds=max(1, ceil(retry_after)),
+                )
+            return RateLimitDecision(allowed=True)
+
+    def record_failure(self, key: str) -> None:
+        with self._lock:
+            now_value = self._now()
+            attempts = self._attempts[key]
+            self._trim_expired(attempts, now_value)
+            attempts.append(now_value)
+
+    def reset(self, key: str) -> None:
+        with self._lock:
+            self._attempts.pop(key, None)
+
+    def clear_all(self) -> None:
+        with self._lock:
+            self._attempts.clear()
+
+    def _trim_expired(self, attempts: deque[float], now_value: float) -> None:
+        while attempts and now_value - attempts[0] >= self._window_seconds:
+            attempts.popleft()
+

app/auth/security.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from argon2 import PasswordHasher
+from argon2.exceptions import InvalidHashError, VerificationError
+
+
+class PasswordService:
+    def __init__(self, hasher: PasswordHasher | None = None) -> None:
+        self._hasher = hasher or PasswordHasher()
+
+    def hash_password(self, password: str) -> str:
+        return self._hasher.hash(password)
+
+    def verify_password(self, password_hash: str, password: str) -> bool:
+        try:
+            return bool(self._hasher.verify(password_hash, password))
+        except (InvalidHashError, VerificationError):
+            return False
+

app/auth/service.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.auth.security import PasswordService
+from app.db.models import User
+
+
+class AuthService:
+    def __init__(self, password_service: PasswordService) -> None:
+        self._password_service = password_service
+
+    async def authenticate_user(
+        self,
+        db_session: AsyncSession,
+        *,
+        email: str,
+        password: str,
+    ) -> User | None:
+        normalized_email = email.strip().lower()
+        if not normalized_email or not password:
+            return None
+        result = await db_session.execute(
+            select(User).where(User.email == normalized_email)
+        )
+        user = result.scalar_one_or_none()
+        if user is None or not user.is_active:
+            return None
+        if not self._password_service.verify_password(user.password_hash, password):
+            return None
+        return user
+
+    def hash_password(self, password: str) -> str:
+        return self._password_service.hash_password(password)
+
+    def verify_password(self, *, password_hash: str, password: str) -> bool:
+        return self._password_service.verify_password(password_hash, password)

app/auth/session.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from starlette.requests import Request
+
+
+SESSION_USER_ID_KEY = "auth_user_id"
+SESSION_USER_EMAIL_KEY = "auth_user_email"
+SESSION_USER_IS_ADMIN_KEY = "auth_user_is_admin"
+
+
+@dataclass(frozen=True)
+class SessionUser:
+    id: int
+    email: str
+    is_admin: bool
+
+
+def get_session_user(request: Request) -> SessionUser | None:
+    user_id = request.session.get(SESSION_USER_ID_KEY)
+    email = request.session.get(SESSION_USER_EMAIL_KEY)
+    is_admin = request.session.get(SESSION_USER_IS_ADMIN_KEY)
+    if user_id is None or email is None or is_admin is None:
+        return None
+    try:
+        parsed_user_id = int(user_id)
+    except (TypeError, ValueError):
+        return None
+    if not isinstance(email, str):
+        return None
+    return SessionUser(id=parsed_user_id, email=email, is_admin=bool(is_admin))
+
+
+def set_session_user(
+    request: Request,
+    *,
+    user_id: int,
+    email: str,
+    is_admin: bool,
+) -> None:
+    request.session[SESSION_USER_ID_KEY] = int(user_id)
+    request.session[SESSION_USER_EMAIL_KEY] = email
+    request.session[SESSION_USER_IS_ADMIN_KEY] = bool(is_admin)
+
+
+def clear_session_user(request: Request) -> None:
+    request.session.pop(SESSION_USER_ID_KEY, None)
+    request.session.pop(SESSION_USER_EMAIL_KEY, None)
+    request.session.pop(SESSION_USER_IS_ADMIN_KEY, None)

app/db/__init__.py

@@ -0,0 +1 @@
+

app/db/base.py

@@ -0,0 +1,18 @@
+from sqlalchemy import MetaData
+from sqlalchemy.orm import DeclarativeBase
+
+
+NAMING_CONVENTION = {
+    "ix": "ix_%(column_0_label)s",
+    "uq": "uq_%(table_name)s_%(column_0_name)s",
+    "ck": "ck_%(table_name)s_%(constraint_name)s",
+    "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+    "pk": "pk_%(table_name)s",
+}
+
+
+class Base(DeclarativeBase):
+    metadata = MetaData(naming_convention=NAMING_CONVENTION)
+
+
+metadata = Base.metadata

app/db/models.py

@@ -0,0 +1,292 @@
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+
+from sqlalchemy import (
+    Boolean,
+    CheckConstraint,
+    DateTime,
+    Enum,
+    ForeignKey,
+    Index,
+    Integer,
+    String,
+    Text,
+    UniqueConstraint,
+    func,
+    text,
+)
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.db.base import Base
+
+
+class RunTriggerType(StrEnum):
+    MANUAL = "manual"
+    SCHEDULED = "scheduled"
+
+
+class RunStatus(StrEnum):
+    RUNNING = "running"
+    SUCCESS = "success"
+    PARTIAL_FAILURE = "partial_failure"
+    FAILED = "failed"
+
+
+class QueueItemStatus(StrEnum):
+    QUEUED = "queued"
+    RETRYING = "retrying"
+    DROPPED = "dropped"
+
+
+RUN_STATUS_DB_ENUM = Enum(
+    RunStatus,
+    name="run_status",
+    values_callable=lambda members: [member.value for member in members],
+)
+RUN_TRIGGER_TYPE_DB_ENUM = Enum(
+    RunTriggerType,
+    name="run_trigger_type",
+    values_callable=lambda members: [member.value for member in members],
+)
+
+
+class User(Base):
+    __tablename__ = "users"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    email: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
+    password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
+    is_active: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, server_default=text("true")
+    )
+    is_admin: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, server_default=text("false")
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+
+
+class UserSetting(Base):
+    __tablename__ = "user_settings"
+    __table_args__ = (
+        CheckConstraint(
+            "run_interval_minutes >= 15",
+            name="run_interval_minutes_min_15",
+        ),
+        CheckConstraint(
+            "request_delay_seconds >= 1",
+            name="request_delay_seconds_min_1",
+        ),
+    )
+
+    user_id: Mapped[int] = mapped_column(
+        ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
+    )
+    auto_run_enabled: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, server_default=text("false")
+    )
+    run_interval_minutes: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default=text("1440")
+    )
+    request_delay_seconds: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default=text("10")
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+
+
+class ScholarProfile(Base):
+    __tablename__ = "scholar_profiles"
+    __table_args__ = (
+        UniqueConstraint("user_id", "scholar_id", name="uq_scholar_profiles_user_scholar"),
+        Index("ix_scholar_profiles_user_enabled", "user_id", "is_enabled"),
+    )
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    user_id: Mapped[int] = mapped_column(
+        ForeignKey("users.id", ondelete="CASCADE"), nullable=False
+    )
+    scholar_id: Mapped[str] = mapped_column(String(64), nullable=False)
+    display_name: Mapped[str | None] = mapped_column(String(255))
+    is_enabled: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, server_default=text("true")
+    )
+    baseline_completed: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, server_default=text("false")
+    )
+    last_run_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    last_run_status: Mapped[RunStatus | None] = mapped_column(
+        RUN_STATUS_DB_ENUM,
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+
+
+class CrawlRun(Base):
+    __tablename__ = "crawl_runs"
+    __table_args__ = (
+        Index("ix_crawl_runs_user_start", "user_id", "start_dt"),
+    )
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    user_id: Mapped[int] = mapped_column(
+        ForeignKey("users.id", ondelete="CASCADE"), nullable=False
+    )
+    trigger_type: Mapped[RunTriggerType] = mapped_column(
+        RUN_TRIGGER_TYPE_DB_ENUM, nullable=False
+    )
+    status: Mapped[RunStatus] = mapped_column(
+        RUN_STATUS_DB_ENUM, nullable=False
+    )
+    start_dt: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    end_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    scholar_count: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default=text("0")
+    )
+    new_pub_count: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default=text("0")
+    )
+    error_log: Mapped[dict] = mapped_column(
+        JSONB, nullable=False, server_default=text("'{}'::jsonb")
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+
+
+class Publication(Base):
+    __tablename__ = "publications"
+    __table_args__ = (
+        UniqueConstraint("fingerprint_sha256", name="uq_publications_fingerprint"),
+        Index(
+            "uq_publications_cluster_id_not_null",
+            "cluster_id",
+            unique=True,
+            postgresql_where=text("cluster_id IS NOT NULL"),
+        ),
+    )
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    cluster_id: Mapped[str | None] = mapped_column(String(64))
+    fingerprint_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
+    title_raw: Mapped[str] = mapped_column(Text, nullable=False)
+    title_normalized: Mapped[str] = mapped_column(Text, nullable=False)
+    year: Mapped[int | None] = mapped_column(Integer)
+    citation_count: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default=text("0")
+    )
+    author_text: Mapped[str | None] = mapped_column(Text)
+    venue_text: Mapped[str | None] = mapped_column(Text)
+    pub_url: Mapped[str | None] = mapped_column(Text)
+    pdf_url: Mapped[str | None] = mapped_column(Text)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+
+
+class ScholarPublication(Base):
+    __tablename__ = "scholar_publications"
+    __table_args__ = (
+        Index("ix_scholar_publications_is_read", "is_read"),
+    )
+
+    scholar_profile_id: Mapped[int] = mapped_column(
+        ForeignKey("scholar_profiles.id", ondelete="CASCADE"),
+        primary_key=True,
+    )
+    publication_id: Mapped[int] = mapped_column(
+        ForeignKey("publications.id", ondelete="CASCADE"),
+        primary_key=True,
+    )
+    is_read: Mapped[bool] = mapped_column(
+        Boolean, nullable=False, server_default=text("false")
+    )
+    first_seen_run_id: Mapped[int | None] = mapped_column(
+        ForeignKey("crawl_runs.id", ondelete="SET NULL")
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+
+
+class IngestionQueueItem(Base):
+    __tablename__ = "ingestion_queue_items"
+    __table_args__ = (
+        UniqueConstraint(
+            "user_id",
+            "scholar_profile_id",
+            name="uq_ingestion_queue_user_scholar",
+        ),
+        CheckConstraint(
+            "status IN ('queued', 'retrying', 'dropped')",
+            name="ingestion_queue_status_valid",
+        ),
+        Index("ix_ingestion_queue_next_attempt", "next_attempt_dt"),
+        Index("ix_ingestion_queue_status_next_attempt", "status", "next_attempt_dt"),
+    )
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    user_id: Mapped[int] = mapped_column(
+        ForeignKey("users.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    scholar_profile_id: Mapped[int] = mapped_column(
+        ForeignKey("scholar_profiles.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    resume_cstart: Mapped[int] = mapped_column(
+        Integer,
+        nullable=False,
+        server_default=text("0"),
+    )
+    reason: Mapped[str] = mapped_column(String(64), nullable=False)
+    status: Mapped[str] = mapped_column(
+        String(16),
+        nullable=False,
+        server_default=text("'queued'"),
+    )
+    attempt_count: Mapped[int] = mapped_column(
+        Integer,
+        nullable=False,
+        server_default=text("0"),
+    )
+    next_attempt_dt: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        nullable=False,
+        server_default=func.now(),
+    )
+    last_run_id: Mapped[int | None] = mapped_column(
+        ForeignKey("crawl_runs.id", ondelete="SET NULL"),
+    )
+    last_error: Mapped[str | None] = mapped_column(Text)
+    dropped_reason: Mapped[str | None] = mapped_column(String(128))
+    dropped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now()
+    )

app/db/session.py

@@ -0,0 +1,64 @@
+from collections.abc import AsyncIterator
+import logging
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import (
+    AsyncEngine,
+    AsyncSession,
+    async_sessionmaker,
+    create_async_engine,
+)
+from sqlalchemy.pool import NullPool
+
+from app.settings import settings
+
+logger = logging.getLogger(__name__)
+
+_engine: AsyncEngine | None = None
+_session_factory: async_sessionmaker[AsyncSession] | None = None
+
+
+def get_engine() -> AsyncEngine:
+    global _engine
+    if _engine is None:
+        # NullPool avoids cross-event-loop connection reuse in tests and dev tools.
+        _engine = create_async_engine(
+            settings.database_url,
+            pool_pre_ping=True,
+            poolclass=NullPool,
+        )
+        logger.info("db.engine_initialized", extra={"event": "db.engine_initialized"})
+    return _engine
+
+
+def get_session_factory() -> async_sessionmaker[AsyncSession]:
+    global _session_factory
+    if _session_factory is None:
+        _session_factory = async_sessionmaker(get_engine(), expire_on_commit=False)
+    return _session_factory
+
+
+async def get_db_session() -> AsyncIterator[AsyncSession]:
+    session_factory = get_session_factory()
+    async with session_factory() as session:
+        yield session
+
+
+async def check_database() -> bool:
+    engine = get_engine()
+    try:
+        async with engine.connect() as conn:
+            result = await conn.execute(text("SELECT 1"))
+            return result.scalar_one() == 1
+    except Exception:
+        logger.exception("db.healthcheck_failed", extra={"event": "db.healthcheck_failed"})
+        return False
+
+
+async def close_engine() -> None:
+    global _engine, _session_factory
+    if _engine is not None:
+        await _engine.dispose()
+        logger.info("db.engine_disposed", extra={"event": "db.engine_disposed"})
+        _engine = None
+        _session_factory = None

app/logging_config.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+import json
+import logging
+import sys
+from typing import Any
+
+from app.logging_context import get_request_id
+
+DEFAULT_REDACT_FIELDS = {
+    "authorization",
+    "cookie",
+    "csrf_token",
+    "new_password",
+    "password",
+    "password_hash",
+    "session",
+    "session_secret_key",
+}
+
+_BASE_RECORD = logging.makeLogRecord({})
+_STANDARD_RECORD_FIELDS = set(_BASE_RECORD.__dict__.keys()) | {"message", "asctime"}
+_NOISY_RECORD_FIELDS = {"color_message"}
+
+
+def parse_redact_fields(raw: str | None) -> set[str]:
+    fields = {field.strip().lower() for field in (raw or "").split(",") if field.strip()}
+    return DEFAULT_REDACT_FIELDS | fields
+
+
+def configure_logging(
+    *,
+    level: str,
+    log_format: str,
+    redact_fields: set[str],
+    include_uvicorn_access: bool,
+) -> None:
+    normalized_level = _normalize_level(level)
+    root_logger = logging.getLogger()
+    root_logger.handlers.clear()
+    root_logger.setLevel(normalized_level)
+
+    handler = logging.StreamHandler(stream=sys.stdout)
+    handler.setLevel(normalized_level)
+    handler.addFilter(RequestContextFilter())
+
+    normalized_format = log_format.strip().lower()
+    if normalized_format == "json":
+        handler.setFormatter(JsonLogFormatter(redact_fields=redact_fields))
+    else:
+        handler.setFormatter(ConsoleLogFormatter(redact_fields=redact_fields))
+
+    root_logger.addHandler(handler)
+
+    # Route server/framework logs through our single root handler.
+    for logger_name in ("uvicorn", "uvicorn.error"):
+        framework_logger = logging.getLogger(logger_name)
+        framework_logger.handlers.clear()
+        framework_logger.propagate = True
+        framework_logger.setLevel(normalized_level)
+
+    access_logger = logging.getLogger("uvicorn.access")
+    access_logger.handlers.clear()
+    access_logger.propagate = True
+    access_logger.setLevel(normalized_level if include_uvicorn_access else logging.WARNING)
+
+
+class RequestContextFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        if not getattr(record, "request_id", None):
+            request_id = get_request_id()
+            if request_id:
+                record.request_id = request_id
+        return True
+
+
+class JsonLogFormatter(logging.Formatter):
+    def __init__(self, *, redact_fields: set[str]) -> None:
+        super().__init__()
+        self._redact_fields = {field.lower() for field in redact_fields}
+
+    def format(self, record: logging.LogRecord) -> str:
+        payload: dict[str, Any] = {
+            "timestamp": _format_timestamp(record.created),
+            "level": record.levelname.lower(),
+            "logger": record.name,
+            "event": getattr(record, "event", record.getMessage()),
+        }
+        request_id = getattr(record, "request_id", None)
+        if request_id:
+            payload["request_id"] = request_id
+        payload.update(self._redact_mapping(_extra_fields(record)))
+        if record.exc_info:
+            payload["exception"] = self.formatException(record.exc_info)
+        return json.dumps(payload, ensure_ascii=True, default=str)
+
+    def _redact_mapping(self, value: dict[str, Any]) -> dict[str, Any]:
+        return {key: self._redact_value(key, item) for key, item in value.items()}
+
+    def _redact_value(self, key: str, value: Any) -> Any:
+        if key.lower() in self._redact_fields:
+            return "[REDACTED]"
+        if isinstance(value, dict):
+            return {nested_key: self._redact_value(nested_key, nested_value) for nested_key, nested_value in value.items()}
+        if isinstance(value, (list, tuple)):
+            return [self._redact_value(key, item) for item in value]
+        return value
+
+
+class ConsoleLogFormatter(logging.Formatter):
+    def __init__(self, *, redact_fields: set[str]) -> None:
+        super().__init__()
+        self._json_formatter = JsonLogFormatter(redact_fields=redact_fields)
+
+    def format(self, record: logging.LogRecord) -> str:
+        payload = json.loads(self._json_formatter.format(record))
+        timestamp = payload.get("timestamp", "")
+        level = _short_level(payload.get("level", "info"))
+        logger_name = str(payload.get("logger", "app"))
+        event = str(payload.get("event", ""))
+
+        parts = [timestamp, level, logger_name, event]
+
+        request_id = payload.pop("request_id", None)
+        method = payload.pop("method", None)
+        path = payload.pop("path", None)
+        status_code = payload.pop("status_code", None)
+        duration_ms = payload.pop("duration_ms", None)
+
+        if request_id:
+            parts.append(f"rid={request_id}")
+        if method and path:
+            parts.append(f"{method} {path}")
+        if status_code is not None:
+            parts.append(str(status_code))
+        if duration_ms is not None:
+            parts.append(f"{duration_ms}ms")
+
+        for key in sorted(payload.keys()):
+            if key in {"timestamp", "level", "logger", "event", "exception"}:
+                continue
+            parts.append(f"{key}={payload[key]}")
+
+        if "exception" in payload:
+            parts.append(f"exception={payload['exception']}")
+
+        return " | ".join(str(part) for part in parts if part)
+
+
+def _extra_fields(record: logging.LogRecord) -> dict[str, Any]:
+    extras: dict[str, Any] = {}
+    for key, value in record.__dict__.items():
+        if key in _STANDARD_RECORD_FIELDS or key.startswith("_"):
+            continue
+        if key in _NOISY_RECORD_FIELDS:
+            continue
+        extras[key] = value
+    return extras
+
+
+def _normalize_level(level: str) -> int:
+    normalized = level.strip().upper()
+    mapping = logging.getLevelNamesMapping()
+    if normalized not in mapping:
+        return logging.INFO
+    return mapping[normalized]
+
+
+def _format_timestamp(created_ts: float) -> str:
+    dt = datetime.fromtimestamp(created_ts, tz=timezone.utc)
+    return dt.strftime("%Y-%m-%d %H:%M:%SZ")
+
+
+def _short_level(level: str) -> str:
+    mapping = {
+        "debug": "DBG",
+        "info": "INF",
+        "warning": "WRN",
+        "error": "ERR",
+        "critical": "CRT",
+    }
+    return mapping.get(level.lower(), level[:3].upper())

app/logging_context.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from contextvars import ContextVar
+
+
+_request_id_ctx: ContextVar[str | None] = ContextVar("request_id", default=None)
+
+
+def get_request_id() -> str | None:
+    return _request_id_ctx.get()
+
+
+def set_request_id(value: str | None) -> None:
+    _request_id_ctx.set(value)
+

app/main.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+from starlette.middleware.sessions import SessionMiddleware
+
+from app.api.errors import register_api_exception_handlers
+from app.api.router import router as api_router
+from app.db.session import close_engine
+from app.logging_config import configure_logging, parse_redact_fields
+from app.security.csrf import CSRFMiddleware
+from app.services.scheduler import SchedulerService
+from app.settings import settings
+from app.web import common as web_common
+from app.web.deps import get_ingestion_service, get_scholar_source
+from app.web.middleware import RequestLoggingMiddleware, parse_skip_paths
+from app.web.routers import (
+    admin,
+    auth,
+    dashboard,
+    health,
+    publications,
+    runs,
+    scholars,
+    settings as settings_router,
+)
+
+configure_logging(
+    level=settings.log_level,
+    log_format=settings.log_format,
+    redact_fields=parse_redact_fields(settings.log_redact_fields),
+    include_uvicorn_access=settings.log_uvicorn_access,
+)
+
+scheduler_service = SchedulerService(
+    enabled=settings.scheduler_enabled,
+    tick_seconds=settings.scheduler_tick_seconds,
+    network_error_retries=settings.ingestion_network_error_retries,
+    retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+    max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
+    page_size=settings.ingestion_page_size,
+    continuation_queue_enabled=settings.ingestion_continuation_queue_enabled,
+    continuation_base_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
+    continuation_max_delay_seconds=settings.ingestion_continuation_max_delay_seconds,
+    continuation_max_attempts=settings.ingestion_continuation_max_attempts,
+    queue_batch_size=settings.scheduler_queue_batch_size,
+)
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI):
+    await scheduler_service.start()
+    yield
+    await scheduler_service.stop()
+    await close_engine()
+
+
+app = FastAPI(title=settings.app_name, lifespan=lifespan)
+register_api_exception_handlers(app)
+app.add_middleware(CSRFMiddleware)
+app.add_middleware(
+    SessionMiddleware,
+    secret_key=settings.session_secret_key,
+    same_site="lax",
+    https_only=settings.session_cookie_secure,
+)
+app.add_middleware(
+    RequestLoggingMiddleware,
+    log_requests=settings.log_requests,
+    skip_paths=parse_skip_paths(settings.log_request_skip_paths),
+)
+app.mount("/static", StaticFiles(directory="app/static"), name="static")
+
+app.include_router(api_router)
+app.include_router(auth.router)
+app.include_router(admin.router)
+app.include_router(scholars.router)
+app.include_router(settings_router.router)
+app.include_router(runs.router)
+app.include_router(publications.router)
+app.include_router(dashboard.router)
+app.include_router(health.router)
+
+# Backward-compatible export kept for tests and any existing local scripts.
+_get_authenticated_user = web_common.get_authenticated_user

app/presentation/__init__.py

@@ -0,0 +1,2 @@
+"""Presentation-layer view models used by template routes."""
+

app/presentation/dashboard.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Sequence
+
+from app.db.models import CrawlRun
+from app.services.publications import UnreadPublicationItem
+
+SECTION_TEMPLATES: tuple[str, ...] = (
+    "dashboard/_run_controls.html",
+    "dashboard/_unread_publications.html",
+    "dashboard/_run_history.html",
+)
+
+
+@dataclass(frozen=True)
+class RunControlsViewModel:
+    request_delay_seconds: int
+    queue_queued_count: int
+    queue_retrying_count: int
+    queue_dropped_count: int
+    run_manual_action: str = "/runs/manual"
+    mark_all_read_action: str = "/publications/mark-all-read"
+
+
+@dataclass(frozen=True)
+class UnreadPublicationViewModel:
+    title: str
+    scholar_label: str
+    year_display: str
+    citation_count: int
+    venue_text: str | None
+    pub_url: str | None
+
+
+@dataclass(frozen=True)
+class RunHistoryItemViewModel:
+    run_id: int | None
+    detail_url: str | None
+    started_at_display: str
+    status_label: str
+    status_badge: str
+    scholar_count: int
+    new_publication_count: int
+
+
+@dataclass(frozen=True)
+class DashboardViewModel:
+    section_templates: tuple[str, ...]
+    run_controls: RunControlsViewModel
+    unread_publications: list[UnreadPublicationViewModel]
+    run_history: list[RunHistoryItemViewModel]
+
+
+def build_dashboard_view_model(
+    *,
+    unread_publications: Sequence[UnreadPublicationItem],
+    recent_runs: Sequence[CrawlRun],
+    request_delay_seconds: int,
+    queue_counts: dict[str, int] | None = None,
+) -> DashboardViewModel:
+    queue_counts = queue_counts or {}
+    return DashboardViewModel(
+        section_templates=SECTION_TEMPLATES,
+        run_controls=RunControlsViewModel(
+            request_delay_seconds=request_delay_seconds,
+            queue_queued_count=int(queue_counts.get("queued", 0)),
+            queue_retrying_count=int(queue_counts.get("retrying", 0)),
+            queue_dropped_count=int(queue_counts.get("dropped", 0)),
+        ),
+        unread_publications=[
+            UnreadPublicationViewModel(
+                title=item.title,
+                scholar_label=item.scholar_label,
+                year_display=str(item.year) if item.year is not None else "-",
+                citation_count=item.citation_count,
+                venue_text=item.venue_text,
+                pub_url=item.pub_url,
+            )
+            for item in unread_publications
+        ],
+        run_history=[
+            RunHistoryItemViewModel(
+                run_id=run.id,
+                detail_url=f"/runs/{run.id}" if run.id is not None else None,
+                started_at_display=_format_started_at(run.start_dt),
+                status_label=run.status.value,
+                status_badge=_status_badge(run.status.value),
+                scholar_count=run.scholar_count,
+                new_publication_count=run.new_pub_count,
+            )
+            for run in recent_runs
+        ],
+    )
+
+
+def _status_badge(status: str) -> str:
+    if status == "success":
+        return "ok"
+    if status == "failed":
+        return "danger"
+    return "warn"
+
+
+def _format_started_at(dt: datetime) -> str:
+    local_aware = dt.astimezone(timezone.utc)
+    return local_aware.strftime("%Y-%m-%d %H:%M UTC")

app/security/__init__.py

@@ -0,0 +1,2 @@
+"""Security middleware and helpers for scholarr."""
+

app/security/csrf.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import logging
+from secrets import compare_digest, token_urlsafe
+from urllib.parse import parse_qs
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import JSONResponse, PlainTextResponse, Response
+from starlette.types import Message
+
+
+CSRF_SESSION_KEY = "csrf_token"
+CSRF_FORM_FIELD = "csrf_token"
+CSRF_HEADER_NAME = "x-csrf-token"
+SAFE_METHODS = {"GET", "HEAD", "OPTIONS", "TRACE"}
+logger = logging.getLogger(__name__)
+
+
+def ensure_csrf_token(request: Request) -> str:
+    token = request.session.get(CSRF_SESSION_KEY)
+    if token is None:
+        token = token_urlsafe(32)
+        request.session[CSRF_SESSION_KEY] = token
+    return str(token)
+
+
+class CSRFMiddleware(BaseHTTPMiddleware):
+    def __init__(self, app, *, exempt_paths: set[str] | None = None) -> None:
+        super().__init__(app)
+        self._exempt_paths = exempt_paths or set()
+
+    async def dispatch(self, request: Request, call_next) -> Response:
+        if self._should_skip(request):
+            return await call_next(request)
+
+        session_token = request.session.get(CSRF_SESSION_KEY)
+        if not session_token:
+            logger.warning(
+                "csrf.missing_session_token",
+                extra={
+                    "event": "csrf.missing_session_token",
+                    "method": request.method,
+                    "path": request.url.path,
+                },
+            )
+            return self._csrf_error_response(
+                request,
+                code="csrf_missing",
+                message="CSRF token missing.",
+            )
+
+        request_token = request.headers.get(CSRF_HEADER_NAME)
+        if request_token is None and self._is_form_payload(request):
+            body = await request.body()
+            request_token = self._token_from_form_body(request, body)
+            request._receive = self._build_receive(body)  # type: ignore[attr-defined]
+
+        if not request_token or not compare_digest(str(session_token), str(request_token)):
+            logger.warning(
+                "csrf.invalid_token",
+                extra={
+                    "event": "csrf.invalid_token",
+                    "method": request.method,
+                    "path": request.url.path,
+                },
+            )
+            return self._csrf_error_response(
+                request,
+                code="csrf_invalid",
+                message="CSRF token invalid.",
+            )
+
+        return await call_next(request)
+
+    def _should_skip(self, request: Request) -> bool:
+        if request.method in SAFE_METHODS:
+            return True
+        path = request.url.path
+        if path.startswith("/static/"):
+            return True
+        return path in self._exempt_paths
+
+    def _is_form_payload(self, request: Request) -> bool:
+        content_type = request.headers.get("content-type", "")
+        return content_type.startswith("application/x-www-form-urlencoded") or (
+            content_type.startswith("multipart/form-data")
+        )
+
+    def _token_from_form_body(self, request: Request, body: bytes) -> str | None:
+        content_type = request.headers.get("content-type", "")
+        if not content_type.startswith("application/x-www-form-urlencoded"):
+            return None
+        parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True)
+        values = parsed.get(CSRF_FORM_FIELD)
+        if not values:
+            return None
+        return values[0]
+
+    def _build_receive(self, body: bytes):
+        consumed = False
+
+        async def receive() -> Message:
+            nonlocal consumed
+            if consumed:
+                return {"type": "http.request", "body": b"", "more_body": False}
+            consumed = True
+            return {"type": "http.request", "body": body, "more_body": False}
+
+        return receive
+
+    def _csrf_error_response(
+        self,
+        request: Request,
+        *,
+        code: str,
+        message: str,
+    ) -> Response:
+        if request.url.path.startswith("/api/"):
+            request_state = getattr(request, "state", None)
+            request_id = getattr(request_state, "request_id", None) if request_state else None
+            return JSONResponse(
+                {
+                    "error": {
+                        "code": code,
+                        "message": message,
+                        "details": None,
+                    },
+                    "meta": {
+                        "request_id": request_id,
+                    },
+                },
+                status_code=403,
+            )
+        return PlainTextResponse(message, status_code=403)

app/services/__init__.py

@@ -0,0 +1,2 @@
+"""Service layer for scholarr application workflows."""
+

app/services/continuation_queue.py

@@ -0,0 +1,278 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import IngestionQueueItem, QueueItemStatus
+
+
+@dataclass(frozen=True)
+class ContinuationQueueJob:
+    id: int
+    user_id: int
+    scholar_profile_id: int
+    resume_cstart: int
+    reason: str
+    status: str
+    attempt_count: int
+    next_attempt_dt: datetime
+
+
+ACTIVE_QUEUE_STATUSES: tuple[str, ...] = (
+    QueueItemStatus.QUEUED.value,
+    QueueItemStatus.RETRYING.value,
+)
+
+
+def normalize_cstart(value: int | None) -> int:
+    if value is None:
+        return 0
+    return max(0, int(value))
+
+
+def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_seconds: int) -> int:
+    base = max(1, int(base_seconds))
+    attempts = max(1, int(attempt_count))
+    maximum = max(base, int(max_seconds))
+    seconds = base * (2 ** max(0, attempts - 1))
+    return min(seconds, maximum)
+
+
+async def upsert_job(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+    resume_cstart: int,
+    reason: str,
+    run_id: int | None,
+    delay_seconds: int,
+) -> IngestionQueueItem:
+    now = datetime.now(timezone.utc)
+    next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds)))
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(
+            IngestionQueueItem.user_id == user_id,
+            IngestionQueueItem.scholar_profile_id == scholar_profile_id,
+        )
+    )
+    item = result.scalar_one_or_none()
+    normalized_cstart = normalize_cstart(resume_cstart)
+    if item is None:
+        item = IngestionQueueItem(
+            user_id=user_id,
+            scholar_profile_id=scholar_profile_id,
+            resume_cstart=normalized_cstart,
+            reason=reason,
+            status=QueueItemStatus.QUEUED.value,
+            attempt_count=0,
+            next_attempt_dt=next_attempt_dt,
+            last_run_id=run_id,
+            last_error=None,
+            dropped_reason=None,
+            dropped_at=None,
+            created_at=now,
+            updated_at=now,
+        )
+        db_session.add(item)
+        return item
+
+    item.resume_cstart = normalized_cstart
+    item.reason = reason
+    if item.status == QueueItemStatus.DROPPED.value:
+        item.attempt_count = 0
+    item.status = QueueItemStatus.QUEUED.value
+    item.next_attempt_dt = next_attempt_dt
+    item.last_run_id = run_id
+    item.last_error = None
+    item.dropped_reason = None
+    item.dropped_at = None
+    item.updated_at = now
+    return item
+
+
+async def clear_job_for_scholar(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+) -> bool:
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(
+            IngestionQueueItem.user_id == user_id,
+            IngestionQueueItem.scholar_profile_id == scholar_profile_id,
+        )
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return False
+    await db_session.delete(item)
+    return True
+
+
+async def delete_job_by_id(
+    db_session: AsyncSession,
+    *,
+    job_id: int,
+) -> bool:
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return False
+    await db_session.delete(item)
+    return True
+
+
+async def list_due_jobs(
+    db_session: AsyncSession,
+    *,
+    now: datetime,
+    limit: int,
+) -> list[ContinuationQueueJob]:
+    result = await db_session.execute(
+        select(IngestionQueueItem)
+        .where(
+            IngestionQueueItem.next_attempt_dt <= now,
+            IngestionQueueItem.status.in_(ACTIVE_QUEUE_STATUSES),
+        )
+        .order_by(
+            IngestionQueueItem.next_attempt_dt.asc(),
+            IngestionQueueItem.id.asc(),
+        )
+        .limit(limit)
+    )
+    rows = list(result.scalars().all())
+    jobs: list[ContinuationQueueJob] = []
+    for row in rows:
+        jobs.append(
+            ContinuationQueueJob(
+                id=int(row.id),
+                user_id=int(row.user_id),
+                scholar_profile_id=int(row.scholar_profile_id),
+                resume_cstart=normalize_cstart(row.resume_cstart),
+                reason=row.reason,
+                status=row.status,
+                attempt_count=int(row.attempt_count),
+                next_attempt_dt=row.next_attempt_dt,
+            )
+        )
+    return jobs
+
+
+async def increment_attempt_count(
+    db_session: AsyncSession,
+    *,
+    job_id: int,
+) -> IngestionQueueItem | None:
+    now = datetime.now(timezone.utc)
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    item.attempt_count = int(item.attempt_count or 0) + 1
+    item.updated_at = now
+    return item
+
+
+async def reschedule_job(
+    db_session: AsyncSession,
+    *,
+    job_id: int,
+    delay_seconds: int,
+    reason: str,
+    error: str | None = None,
+) -> IngestionQueueItem | None:
+    now = datetime.now(timezone.utc)
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    item.next_attempt_dt = now + timedelta(seconds=max(1, int(delay_seconds)))
+    item.status = QueueItemStatus.QUEUED.value
+    item.reason = reason
+    item.last_error = error
+    item.dropped_reason = None
+    item.dropped_at = None
+    item.updated_at = now
+    return item
+
+
+async def mark_retrying(
+    db_session: AsyncSession,
+    *,
+    job_id: int,
+    reason: str | None = None,
+) -> IngestionQueueItem | None:
+    now = datetime.now(timezone.utc)
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    if item.status == QueueItemStatus.DROPPED.value:
+        return item
+    item.status = QueueItemStatus.RETRYING.value
+    if reason:
+        item.reason = reason
+    item.updated_at = now
+    return item
+
+
+async def mark_dropped(
+    db_session: AsyncSession,
+    *,
+    job_id: int,
+    reason: str,
+    error: str | None = None,
+) -> IngestionQueueItem | None:
+    now = datetime.now(timezone.utc)
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    item.status = QueueItemStatus.DROPPED.value
+    item.reason = "dropped"
+    item.dropped_reason = reason
+    item.dropped_at = now
+    if error is not None:
+        item.last_error = error
+    item.updated_at = now
+    return item
+
+
+async def mark_queued_now(
+    db_session: AsyncSession,
+    *,
+    job_id: int,
+    reason: str,
+    reset_attempt_count: bool = False,
+) -> IngestionQueueItem | None:
+    now = datetime.now(timezone.utc)
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    item.status = QueueItemStatus.QUEUED.value
+    item.reason = reason
+    item.next_attempt_dt = now
+    if reset_attempt_count:
+        item.attempt_count = 0
+    item.last_error = None
+    item.dropped_reason = None
+    item.dropped_at = None
+    item.updated_at = now
+    return item

app/services/ingestion.py

@@ -0,0 +1,967 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from datetime import datetime, timezone
+import hashlib
+import logging
+import re
+from typing import Any
+from urllib.parse import urljoin
+
+from sqlalchemy import select, text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import (
+    CrawlRun,
+    Publication,
+    RunStatus,
+    RunTriggerType,
+    ScholarProfile,
+    ScholarPublication,
+)
+from app.services import continuation_queue as queue_service
+from app.services.scholar_parser import (
+    ParseState,
+    ParsedProfilePage,
+    PublicationCandidate,
+    parse_profile_page,
+)
+from app.services.scholar_source import FetchResult, ScholarSource
+
+TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+")
+WORD_RE = re.compile(r"[a-z0-9]+")
+HTML_TAG_RE = re.compile(r"<[^>]+>", re.S)
+SPACE_RE = re.compile(r"\s+")
+FAILED_STATES = {
+    ParseState.BLOCKED_OR_CAPTCHA.value,
+    ParseState.LAYOUT_CHANGED.value,
+    ParseState.NETWORK_ERROR.value,
+    "ingestion_error",
+}
+RUN_LOCK_NAMESPACE = 8217
+RESUMABLE_PARTIAL_REASONS = {
+    "max_pages_reached",
+    "pagination_cursor_stalled",
+}
+RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",)
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class RunExecutionSummary:
+    crawl_run_id: int
+    status: RunStatus
+    scholar_count: int
+    succeeded_count: int
+    failed_count: int
+    partial_count: int
+    new_publication_count: int
+
+
+@dataclass(frozen=True)
+class PagedParseResult:
+    fetch_result: FetchResult
+    parsed_page: ParsedProfilePage
+    publications: list[PublicationCandidate]
+    attempt_log: list[dict[str, Any]]
+    page_logs: list[dict[str, Any]]
+    pages_fetched: int
+    pages_attempted: int
+    has_more_remaining: bool
+    pagination_truncated_reason: str | None
+    continuation_cstart: int | None
+
+
+class RunAlreadyInProgressError(RuntimeError):
+    """Raised when a run lock for a user is already held by another process."""
+
+
+class ScholarIngestionService:
+    def __init__(self, *, source: ScholarSource) -> None:
+        self._source = source
+
+    async def run_for_user(
+        self,
+        db_session: AsyncSession,
+        *,
+        user_id: int,
+        trigger_type: RunTriggerType,
+        request_delay_seconds: int,
+        network_error_retries: int = 1,
+        retry_backoff_seconds: float = 1.0,
+        max_pages_per_scholar: int = 30,
+        page_size: int = 100,
+        scholar_profile_ids: set[int] | None = None,
+        start_cstart_by_scholar_id: dict[int, int] | None = None,
+        auto_queue_continuations: bool = True,
+        queue_delay_seconds: int = 60,
+    ) -> RunExecutionSummary:
+        lock_acquired = await self._try_acquire_user_lock(
+            db_session,
+            user_id=user_id,
+        )
+        if not lock_acquired:
+            raise RunAlreadyInProgressError(f"Run already in progress for user_id={user_id}.")
+
+        filtered_scholar_ids = (
+            {int(value) for value in scholar_profile_ids}
+            if scholar_profile_ids is not None
+            else None
+        )
+        start_cstart_map = {
+            int(key): max(0, int(value))
+            for key, value in (start_cstart_by_scholar_id or {}).items()
+        }
+
+        scholars_stmt = (
+            select(ScholarProfile)
+            .where(
+                ScholarProfile.user_id == user_id,
+                ScholarProfile.is_enabled.is_(True),
+            )
+            .order_by(ScholarProfile.created_at.asc(), ScholarProfile.id.asc())
+        )
+        if filtered_scholar_ids is not None:
+            scholars_stmt = scholars_stmt.where(
+                ScholarProfile.id.in_(filtered_scholar_ids)
+            )
+
+        scholars_result = await db_session.execute(
+            scholars_stmt
+        )
+        scholars = list(scholars_result.scalars().all())
+        if filtered_scholar_ids is not None:
+            found_ids = {int(scholar.id) for scholar in scholars}
+            missing_ids = filtered_scholar_ids - found_ids
+            for scholar_profile_id in missing_ids:
+                await queue_service.clear_job_for_scholar(
+                    db_session,
+                    user_id=user_id,
+                    scholar_profile_id=scholar_profile_id,
+                )
+        logger.info(
+            "ingestion.run_started",
+            extra={
+                "event": "ingestion.run_started",
+                "user_id": user_id,
+                "trigger_type": trigger_type.value,
+                "scholar_count": len(scholars),
+                "is_filtered_run": filtered_scholar_ids is not None,
+                "request_delay_seconds": request_delay_seconds,
+                "network_error_retries": network_error_retries,
+                "retry_backoff_seconds": retry_backoff_seconds,
+                "max_pages_per_scholar": max_pages_per_scholar,
+                "page_size": page_size,
+            },
+        )
+
+        run = CrawlRun(
+            user_id=user_id,
+            trigger_type=trigger_type,
+            status=RunStatus.RUNNING,
+            scholar_count=len(scholars),
+            new_pub_count=0,
+            error_log={},
+        )
+        db_session.add(run)
+        await db_session.flush()
+
+        succeeded_count = 0
+        failed_count = 0
+        partial_count = 0
+        scholar_results: list[dict[str, Any]] = []
+
+        for index, scholar in enumerate(scholars):
+            if index > 0 and request_delay_seconds > 0:
+                await asyncio.sleep(float(request_delay_seconds))
+
+            run_dt = datetime.now(timezone.utc)
+            start_cstart = int(start_cstart_map.get(int(scholar.id), 0))
+
+            paged_parse_result = await self._fetch_and_parse_all_pages_with_retry(
+                scholar_id=scholar.scholar_id,
+                start_cstart=start_cstart,
+                request_delay_seconds=request_delay_seconds,
+                network_error_retries=network_error_retries,
+                retry_backoff_seconds=retry_backoff_seconds,
+                max_pages=max_pages_per_scholar,
+                page_size=page_size,
+            )
+            fetch_result = paged_parse_result.fetch_result
+            parsed_page = paged_parse_result.parsed_page
+            publications = paged_parse_result.publications
+            attempt_log = paged_parse_result.attempt_log
+            page_logs = paged_parse_result.page_logs
+
+            logger.info(
+                "ingestion.scholar_parsed",
+                extra={
+                    "event": "ingestion.scholar_parsed",
+                    "user_id": user_id,
+                    "crawl_run_id": run.id,
+                    "scholar_profile_id": scholar.id,
+                    "scholar_id": scholar.scholar_id,
+                    "state": parsed_page.state.value,
+                    "publication_count": len(publications),
+                    "has_show_more_button": parsed_page.has_show_more_button,
+                    "pages_fetched": paged_parse_result.pages_fetched,
+                    "pages_attempted": paged_parse_result.pages_attempted,
+                    "has_more_remaining": paged_parse_result.has_more_remaining,
+                    "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason,
+                    "warning_count": len(parsed_page.warnings),
+                },
+            )
+
+            result_entry = {
+                "scholar_profile_id": scholar.id,
+                "scholar_id": scholar.scholar_id,
+                "state": parsed_page.state.value,
+                "state_reason": parsed_page.state_reason,
+                "outcome": "failed",
+                "attempt_count": len(attempt_log),
+                "publication_count": len(publications),
+                "start_cstart": start_cstart,
+                "articles_range": parsed_page.articles_range,
+                "warnings": parsed_page.warnings,
+                "has_show_more_button": parsed_page.has_show_more_button,
+                "pages_fetched": paged_parse_result.pages_fetched,
+                "pages_attempted": paged_parse_result.pages_attempted,
+                "has_more_remaining": paged_parse_result.has_more_remaining,
+                "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason,
+                "continuation_cstart": paged_parse_result.continuation_cstart,
+            }
+            had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}
+            has_partial_publication_set = len(publications) > 0 and had_page_failure
+            is_partial_due_to_pagination = (
+                paged_parse_result.has_more_remaining
+                or paged_parse_result.pagination_truncated_reason is not None
+            )
+
+            if (not had_page_failure) or has_partial_publication_set:
+                try:
+                    discovered_publication_count = await self._upsert_profile_publications(
+                        db_session,
+                        run=run,
+                        scholar=scholar,
+                        publications=publications,
+                    )
+                    run.new_pub_count = int(run.new_pub_count or 0) + discovered_publication_count
+
+                    is_partial_scholar = is_partial_due_to_pagination or has_partial_publication_set
+                    scholar.last_run_status = (
+                        RunStatus.PARTIAL_FAILURE if is_partial_scholar else RunStatus.SUCCESS
+                    )
+                    scholar.last_run_dt = run_dt
+                    succeeded_count += 1
+                    result_entry["outcome"] = "partial" if is_partial_scholar else "success"
+
+                    if is_partial_scholar:
+                        partial_count += 1
+                        result_entry["debug"] = self._build_failure_debug_context(
+                            fetch_result=fetch_result,
+                            parsed_page=parsed_page,
+                            attempt_log=attempt_log,
+                            page_logs=page_logs,
+                        )
+                except Exception as exc:
+                    failed_count += 1
+                    scholar.last_run_status = RunStatus.FAILED
+                    scholar.last_run_dt = run_dt
+                    result_entry["state"] = "ingestion_error"
+                    result_entry["state_reason"] = "publication_upsert_exception"
+                    result_entry["outcome"] = "failed"
+                    result_entry["error"] = str(exc)
+                    result_entry["debug"] = self._build_failure_debug_context(
+                        fetch_result=fetch_result,
+                        parsed_page=parsed_page,
+                        attempt_log=attempt_log,
+                        page_logs=page_logs,
+                        exception=exc,
+                    )
+                    logger.exception(
+                        "ingestion.scholar_failed",
+                        extra={
+                            "event": "ingestion.scholar_failed",
+                            "user_id": user_id,
+                            "crawl_run_id": run.id,
+                            "scholar_profile_id": scholar.id,
+                            "scholar_id": scholar.scholar_id,
+                        },
+                    )
+            else:
+                failed_count += 1
+                scholar.last_run_status = RunStatus.FAILED
+                scholar.last_run_dt = run_dt
+                result_entry["debug"] = self._build_failure_debug_context(
+                    fetch_result=fetch_result,
+                    parsed_page=parsed_page,
+                    attempt_log=attempt_log,
+                    page_logs=page_logs,
+                )
+                logger.warning(
+                    "ingestion.scholar_parse_failed",
+                    extra={
+                        "event": "ingestion.scholar_parse_failed",
+                        "user_id": user_id,
+                        "crawl_run_id": run.id,
+                        "scholar_profile_id": scholar.id,
+                        "scholar_id": scholar.scholar_id,
+                        "state": parsed_page.state.value,
+                        "state_reason": parsed_page.state_reason,
+                        "status_code": fetch_result.status_code,
+                    },
+                )
+
+            queue_reason, queue_cstart = self._resolve_continuation_queue_target(
+                outcome=str(result_entry.get("outcome", "")),
+                state=str(result_entry.get("state", "")),
+                pagination_truncated_reason=paged_parse_result.pagination_truncated_reason,
+                continuation_cstart=paged_parse_result.continuation_cstart,
+                fallback_cstart=start_cstart,
+            )
+            if auto_queue_continuations and queue_reason is not None:
+                await queue_service.upsert_job(
+                    db_session,
+                    user_id=user_id,
+                    scholar_profile_id=scholar.id,
+                    resume_cstart=queue_cstart,
+                    reason=queue_reason,
+                    run_id=run.id,
+                    delay_seconds=queue_delay_seconds,
+                )
+                result_entry["continuation_enqueued"] = True
+                result_entry["continuation_reason"] = queue_reason
+                result_entry["continuation_cstart"] = queue_cstart
+            else:
+                cleared = await queue_service.clear_job_for_scholar(
+                    db_session,
+                    user_id=user_id,
+                    scholar_profile_id=scholar.id,
+                )
+                if cleared:
+                    result_entry["continuation_cleared"] = True
+
+            scholar_results.append(result_entry)
+
+        failed_state_counts: dict[str, int] = {}
+        failed_reason_counts: dict[str, int] = {}
+        for entry in scholar_results:
+            if str(entry.get("outcome", "")) != "failed":
+                continue
+            state = str(entry.get("state", ""))
+            if state not in FAILED_STATES:
+                continue
+            failed_state_counts[state] = failed_state_counts.get(state, 0) + 1
+            reason = str(entry.get("state_reason", "")).strip()
+            if reason:
+                failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1
+
+        run.end_dt = datetime.now(timezone.utc)
+        run.status = self._resolve_run_status(
+            scholar_count=len(scholars),
+            succeeded_count=succeeded_count,
+            failed_count=failed_count,
+            partial_count=partial_count,
+        )
+        run.error_log = {
+            "scholar_results": scholar_results,
+            "summary": {
+                "succeeded_count": succeeded_count,
+                "failed_count": failed_count,
+                "partial_count": partial_count,
+                "failed_state_counts": failed_state_counts,
+                "failed_reason_counts": failed_reason_counts,
+            },
+        }
+
+        await db_session.commit()
+        logger.info(
+            "ingestion.run_completed",
+            extra={
+                "event": "ingestion.run_completed",
+                "user_id": user_id,
+                "crawl_run_id": run.id,
+                "status": run.status.value,
+                "scholar_count": len(scholars),
+                "succeeded_count": succeeded_count,
+                "failed_count": failed_count,
+                "partial_count": partial_count,
+                "new_publication_count": run.new_pub_count,
+            },
+        )
+
+        return RunExecutionSummary(
+            crawl_run_id=run.id,
+            status=run.status,
+            scholar_count=len(scholars),
+            succeeded_count=succeeded_count,
+            failed_count=failed_count,
+            partial_count=partial_count,
+            new_publication_count=run.new_pub_count,
+        )
+
+    async def _fetch_profile_page(
+        self,
+        *,
+        scholar_id: str,
+        cstart: int,
+        page_size: int,
+    ) -> FetchResult:
+        try:
+            page_fetcher = getattr(self._source, "fetch_profile_page_html", None)
+            if callable(page_fetcher):
+                return await page_fetcher(
+                    scholar_id,
+                    cstart=cstart,
+                    pagesize=page_size,
+                )
+            if cstart <= 0:
+                return await self._source.fetch_profile_html(scholar_id)
+            return FetchResult(
+                requested_url=(
+                    "https://scholar.google.com/citations"
+                    f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}"
+                ),
+                status_code=None,
+                final_url=None,
+                body="",
+                error="source_does_not_support_pagination",
+            )
+        except Exception as exc:
+            logger.exception(
+                "ingestion.fetch_unexpected_error",
+                extra={
+                    "event": "ingestion.fetch_unexpected_error",
+                    "scholar_id": scholar_id,
+                    "cstart": cstart,
+                    "page_size": page_size,
+                },
+            )
+            return FetchResult(
+                requested_url=(
+                    "https://scholar.google.com/citations"
+                    f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}"
+                ),
+                status_code=None,
+                final_url=None,
+                body="",
+                error=str(exc),
+            )
+
+    async def _fetch_and_parse_page_with_retry(
+        self,
+        *,
+        scholar_id: str,
+        cstart: int,
+        page_size: int,
+        network_error_retries: int,
+        retry_backoff_seconds: float,
+    ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]:
+        max_attempts = max(1, int(network_error_retries) + 1)
+        backoff = max(float(retry_backoff_seconds), 0.0)
+        attempt_log: list[dict[str, Any]] = []
+        fetch_result: FetchResult | None = None
+        parsed_page: ParsedProfilePage | None = None
+
+        for attempt_index in range(max_attempts):
+            fetch_result = await self._fetch_profile_page(
+                scholar_id=scholar_id,
+                cstart=cstart,
+                page_size=page_size,
+            )
+            parsed_page = parse_profile_page(fetch_result)
+            attempt_log.append(
+                {
+                    "attempt": attempt_index + 1,
+                    "cstart": cstart,
+                    "state": parsed_page.state.value,
+                    "state_reason": parsed_page.state_reason,
+                    "status_code": fetch_result.status_code,
+                    "fetch_error": fetch_result.error,
+                }
+            )
+
+            should_retry = (
+                parsed_page.state == ParseState.NETWORK_ERROR
+                and attempt_index < max_attempts - 1
+            )
+            if not should_retry:
+                break
+
+            sleep_seconds = backoff * (2**attempt_index)
+            logger.warning(
+                "ingestion.scholar_retry_scheduled",
+                extra={
+                    "event": "ingestion.scholar_retry_scheduled",
+                    "scholar_id": scholar_id,
+                    "cstart": cstart,
+                    "attempt": attempt_index + 1,
+                    "next_attempt": attempt_index + 2,
+                    "sleep_seconds": sleep_seconds,
+                    "state_reason": parsed_page.state_reason,
+                },
+            )
+            if sleep_seconds > 0:
+                await asyncio.sleep(sleep_seconds)
+
+        if fetch_result is None or parsed_page is None:
+            raise RuntimeError("Fetch-and-parse retry loop produced no result.")
+        return fetch_result, parsed_page, attempt_log
+
+    async def _fetch_and_parse_all_pages_with_retry(
+        self,
+        *,
+        scholar_id: str,
+        start_cstart: int,
+        request_delay_seconds: int,
+        network_error_retries: int,
+        retry_backoff_seconds: float,
+        max_pages: int,
+        page_size: int,
+    ) -> PagedParseResult:
+        bounded_max_pages = max(1, int(max_pages))
+        bounded_page_size = max(1, int(page_size))
+
+        (
+            fetch_result,
+            parsed_page,
+            first_attempt_log,
+        ) = await self._fetch_and_parse_page_with_retry(
+            scholar_id=scholar_id,
+            cstart=start_cstart,
+            page_size=bounded_page_size,
+            network_error_retries=network_error_retries,
+            retry_backoff_seconds=retry_backoff_seconds,
+        )
+
+        attempt_log: list[dict[str, Any]] = list(first_attempt_log)
+        page_logs: list[dict[str, Any]] = []
+        page_logs.append(
+            {
+                "page": 1,
+                "cstart": start_cstart,
+                "state": parsed_page.state.value,
+                "state_reason": parsed_page.state_reason,
+                "status_code": fetch_result.status_code,
+                "publication_count": len(parsed_page.publications),
+                "articles_range": parsed_page.articles_range,
+                "has_show_more_button": parsed_page.has_show_more_button,
+                "warning_codes": parsed_page.warnings,
+                "attempt_count": len(first_attempt_log),
+            }
+        )
+        pages_attempted = 1
+
+        # Immediate hard failure: nothing to salvage.
+        if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
+            return PagedParseResult(
+                fetch_result=fetch_result,
+                parsed_page=parsed_page,
+                publications=[],
+                attempt_log=attempt_log,
+                page_logs=page_logs,
+                pages_fetched=0,
+                pages_attempted=pages_attempted,
+                has_more_remaining=False,
+                pagination_truncated_reason=None,
+                continuation_cstart=(
+                    start_cstart if parsed_page.state == ParseState.NETWORK_ERROR else None
+                ),
+            )
+
+        publications = list(parsed_page.publications)
+        pages_fetched = 1
+        has_more_remaining = False
+        pagination_truncated_reason: str | None = None
+        continuation_cstart: int | None = None
+        current_cstart = start_cstart
+        next_cstart = _next_cstart_value(
+            articles_range=parsed_page.articles_range,
+            fallback=current_cstart + len(parsed_page.publications),
+        )
+
+        while parsed_page.has_show_more_button:
+            if pages_fetched >= bounded_max_pages:
+                has_more_remaining = True
+                pagination_truncated_reason = "max_pages_reached"
+                continuation_cstart = next_cstart if next_cstart > current_cstart else current_cstart
+                break
+            if next_cstart <= current_cstart:
+                has_more_remaining = True
+                pagination_truncated_reason = "pagination_cursor_stalled"
+                continuation_cstart = current_cstart
+                break
+            if request_delay_seconds > 0:
+                await asyncio.sleep(float(request_delay_seconds))
+
+            current_cstart = next_cstart
+            (
+                page_fetch_result,
+                page_parsed,
+                page_attempt_log,
+            ) = await self._fetch_and_parse_page_with_retry(
+                scholar_id=scholar_id,
+                cstart=current_cstart,
+                page_size=bounded_page_size,
+                network_error_retries=network_error_retries,
+                retry_backoff_seconds=retry_backoff_seconds,
+            )
+
+            pages_attempted += 1
+            attempt_log.extend(page_attempt_log)
+            page_logs.append(
+                {
+                    "page": pages_attempted,
+                    "cstart": current_cstart,
+                    "state": page_parsed.state.value,
+                    "state_reason": page_parsed.state_reason,
+                    "status_code": page_fetch_result.status_code,
+                    "publication_count": len(page_parsed.publications),
+                    "articles_range": page_parsed.articles_range,
+                    "has_show_more_button": page_parsed.has_show_more_button,
+                    "warning_codes": page_parsed.warnings,
+                    "attempt_count": len(page_attempt_log),
+                }
+            )
+
+            fetch_result = page_fetch_result
+            parsed_page = page_parsed
+            if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
+                has_more_remaining = True
+                pagination_truncated_reason = f"page_state_{parsed_page.state.value}"
+                continuation_cstart = current_cstart
+                break
+
+            # Google may keep a stale/disabled "show more" marker while returning an empty tail page.
+            # Treat this as a terminal page to avoid false cursor-stalled partial runs.
+            if parsed_page.state == ParseState.NO_RESULTS and len(parsed_page.publications) == 0:
+                pages_fetched += 1
+                break
+
+            pages_fetched += 1
+            publications.extend(parsed_page.publications)
+            next_cstart = _next_cstart_value(
+                articles_range=parsed_page.articles_range,
+                fallback=current_cstart + len(parsed_page.publications),
+            )
+
+        return PagedParseResult(
+            fetch_result=fetch_result,
+            parsed_page=parsed_page,
+            publications=_dedupe_publication_candidates(publications),
+            attempt_log=attempt_log,
+            page_logs=page_logs,
+            pages_fetched=pages_fetched,
+            pages_attempted=pages_attempted,
+            has_more_remaining=has_more_remaining,
+            pagination_truncated_reason=pagination_truncated_reason,
+            continuation_cstart=continuation_cstart,
+        )
+
+    async def _upsert_profile_publications(
+        self,
+        db_session: AsyncSession,
+        *,
+        run: CrawlRun,
+        scholar: ScholarProfile,
+        publications: list[PublicationCandidate],
+    ) -> int:
+        seen_publication_ids: set[int] = set()
+        discovered_count = 0
+
+        for candidate in publications:
+            publication = await self._resolve_publication(db_session, candidate)
+            if publication.id in seen_publication_ids:
+                continue
+            seen_publication_ids.add(publication.id)
+
+            link_result = await db_session.execute(
+                select(ScholarPublication).where(
+                    ScholarPublication.scholar_profile_id == scholar.id,
+                    ScholarPublication.publication_id == publication.id,
+                )
+            )
+            link = link_result.scalar_one_or_none()
+            if link is not None:
+                continue
+
+            link = ScholarPublication(
+                scholar_profile_id=scholar.id,
+                publication_id=publication.id,
+                is_read=False,
+                first_seen_run_id=run.id,
+            )
+            db_session.add(link)
+            discovered_count += 1
+
+            logger.debug(
+                "ingestion.publication_discovered",
+                extra={
+                    "event": "ingestion.publication_discovered",
+                    "scholar_profile_id": scholar.id,
+                    "publication_id": publication.id,
+                    "crawl_run_id": run.id,
+                },
+            )
+
+        if not scholar.baseline_completed:
+            scholar.baseline_completed = True
+
+        return discovered_count
+
+    async def _resolve_publication(
+        self,
+        db_session: AsyncSession,
+        candidate: PublicationCandidate,
+    ) -> Publication:
+        fingerprint = build_publication_fingerprint(candidate)
+
+        publication: Publication | None = None
+        cluster_publication: Publication | None = None
+
+        if candidate.cluster_id:
+            cluster_result = await db_session.execute(
+                select(Publication).where(Publication.cluster_id == candidate.cluster_id)
+            )
+            cluster_publication = cluster_result.scalar_one_or_none()
+            publication = cluster_publication
+
+        if publication is None:
+            fingerprint_result = await db_session.execute(
+                select(Publication).where(Publication.fingerprint_sha256 == fingerprint)
+            )
+            publication = fingerprint_result.scalar_one_or_none()
+
+        if publication is not None and cluster_publication is not None and publication.id != cluster_publication.id:
+            publication = cluster_publication
+
+        if publication is None:
+            publication = Publication(
+                cluster_id=candidate.cluster_id,
+                fingerprint_sha256=fingerprint,
+                title_raw=candidate.title,
+                title_normalized=normalize_title(candidate.title),
+                year=candidate.year,
+                citation_count=int(candidate.citation_count or 0),
+                author_text=candidate.authors_text,
+                venue_text=candidate.venue_text,
+                pub_url=build_publication_url(candidate.title_url),
+                pdf_url=None,
+            )
+            db_session.add(publication)
+            await db_session.flush()
+            logger.debug(
+                "ingestion.publication_created",
+                extra={
+                    "event": "ingestion.publication_created",
+                    "publication_id": publication.id,
+                    "cluster_id": publication.cluster_id,
+                },
+            )
+            return publication
+
+        if candidate.cluster_id and publication.cluster_id is None:
+            publication.cluster_id = candidate.cluster_id
+        publication.title_raw = candidate.title
+        publication.title_normalized = normalize_title(candidate.title)
+        if candidate.year is not None:
+            publication.year = candidate.year
+        if candidate.citation_count is not None:
+            publication.citation_count = int(candidate.citation_count)
+        if candidate.authors_text:
+            publication.author_text = candidate.authors_text
+        if candidate.venue_text:
+            publication.venue_text = candidate.venue_text
+        if candidate.title_url:
+            publication.pub_url = build_publication_url(candidate.title_url)
+
+        return publication
+
+    def _resolve_run_status(
+        self,
+        *,
+        scholar_count: int,
+        succeeded_count: int,
+        failed_count: int,
+        partial_count: int,
+    ) -> RunStatus:
+        if scholar_count == 0:
+            return RunStatus.SUCCESS
+        if failed_count == scholar_count:
+            return RunStatus.FAILED
+        if failed_count > 0 or partial_count > 0:
+            return RunStatus.PARTIAL_FAILURE
+        if succeeded_count > 0:
+            return RunStatus.SUCCESS
+        return RunStatus.FAILED
+
+    def _resolve_continuation_queue_target(
+        self,
+        *,
+        outcome: str,
+        state: str,
+        pagination_truncated_reason: str | None,
+        continuation_cstart: int | None,
+        fallback_cstart: int,
+    ) -> tuple[str | None, int]:
+        if outcome == "partial":
+            reason = (pagination_truncated_reason or "").strip()
+            if reason in RESUMABLE_PARTIAL_REASONS or reason.startswith(
+                RESUMABLE_PARTIAL_REASON_PREFIXES
+            ):
+                return reason, queue_service.normalize_cstart(
+                    continuation_cstart if continuation_cstart is not None else fallback_cstart
+                )
+            return None, queue_service.normalize_cstart(fallback_cstart)
+
+        if outcome == "failed" and state == ParseState.NETWORK_ERROR.value:
+            return "network_error_retry", queue_service.normalize_cstart(
+                continuation_cstart if continuation_cstart is not None else fallback_cstart
+            )
+
+        return None, queue_service.normalize_cstart(fallback_cstart)
+
+    def _build_failure_debug_context(
+        self,
+        *,
+        fetch_result: FetchResult,
+        parsed_page: ParsedProfilePage,
+        attempt_log: list[dict[str, Any]],
+        page_logs: list[dict[str, Any]] | None = None,
+        exception: Exception | None = None,
+    ) -> dict[str, Any]:
+        context: dict[str, Any] = {
+            "requested_url": fetch_result.requested_url,
+            "final_url": fetch_result.final_url,
+            "status_code": fetch_result.status_code,
+            "fetch_error": fetch_result.error,
+            "state_reason": parsed_page.state_reason,
+            "profile_name": parsed_page.profile_name,
+            "articles_range": parsed_page.articles_range,
+            "has_show_more_button": parsed_page.has_show_more_button,
+            "has_operation_error_banner": parsed_page.has_operation_error_banner,
+            "warning_codes": parsed_page.warnings,
+            "marker_counts_nonzero": {
+                key: value for key, value in parsed_page.marker_counts.items() if value > 0
+            },
+            "body_length": len(fetch_result.body),
+            "body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest()
+            if fetch_result.body
+            else None,
+            "body_excerpt": _build_body_excerpt(fetch_result.body),
+            "attempt_log": attempt_log,
+        }
+        if page_logs:
+            context["page_logs"] = page_logs
+        if exception is not None:
+            context["exception_type"] = type(exception).__name__
+            context["exception_message"] = str(exception)
+        return context
+
+    async def _try_acquire_user_lock(
+        self,
+        db_session: AsyncSession,
+        *,
+        user_id: int,
+    ) -> bool:
+        result = await db_session.execute(
+            text(
+                "SELECT pg_try_advisory_xact_lock(:namespace, :user_key)"
+            ),
+            {
+                "namespace": RUN_LOCK_NAMESPACE,
+                "user_key": int(user_id),
+            },
+        )
+        return bool(result.scalar_one())
+
+
+def normalize_title(value: str) -> str:
+    lowered = value.lower()
+    cleaned = TITLE_ALNUM_RE.sub("", lowered)
+    return cleaned
+
+
+def _first_author_last_name(authors_text: str | None) -> str:
+    if not authors_text:
+        return ""
+    first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
+    words = WORD_RE.findall(first_author)
+    if not words:
+        return ""
+    return words[-1]
+
+
+def _first_venue_word(venue_text: str | None) -> str:
+    if not venue_text:
+        return ""
+    words = WORD_RE.findall(venue_text.lower())
+    if not words:
+        return ""
+    return words[0]
+
+
+def build_publication_fingerprint(candidate: PublicationCandidate) -> str:
+    canonical = "|".join(
+        [
+            normalize_title(candidate.title),
+            str(candidate.year) if candidate.year is not None else "",
+            _first_author_last_name(candidate.authors_text),
+            _first_venue_word(candidate.venue_text),
+        ]
+    )
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def build_publication_url(path_or_url: str | None) -> str | None:
+    if not path_or_url:
+        return None
+    return urljoin("https://scholar.google.com", path_or_url)
+
+
+def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int:
+    if articles_range:
+        numbers = re.findall(r"\d+", articles_range)
+        if len(numbers) >= 2:
+            try:
+                return int(numbers[1])
+            except ValueError:
+                pass
+    return int(fallback)
+
+
+def _dedupe_publication_candidates(
+    publications: list[PublicationCandidate],
+) -> list[PublicationCandidate]:
+    deduped: list[PublicationCandidate] = []
+    seen: set[str] = set()
+    for publication in publications:
+        if publication.cluster_id:
+            identity = f"cluster:{publication.cluster_id}"
+        else:
+            identity = "|".join(
+                [
+                    "fallback",
+                    normalize_title(publication.title),
+                    str(publication.year) if publication.year is not None else "",
+                    publication.authors_text or "",
+                    publication.venue_text or "",
+                ]
+            )
+        if identity in seen:
+            continue
+        seen.add(identity)
+        deduped.append(publication)
+    return deduped
+
+
+def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None:
+    if not body:
+        return None
+    flattened = SPACE_RE.sub(" ", HTML_TAG_RE.sub(" ", body)).strip()
+    if not flattened:
+        return None
+    if len(flattened) <= max_chars:
+        return flattened
+    return f"{flattened[:max_chars - 1]}..."

app/services/publications.py

@@ -0,0 +1,297 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+
+from sqlalchemy import Select, func, select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import (
+    CrawlRun,
+    Publication,
+    RunStatus,
+    ScholarProfile,
+    ScholarPublication,
+)
+
+MODE_NEW = "new"
+MODE_ALL = "all"
+
+
+@dataclass(frozen=True)
+class PublicationListItem:
+    publication_id: int
+    scholar_profile_id: int
+    scholar_label: str
+    title: str
+    year: int | None
+    citation_count: int
+    venue_text: str | None
+    pub_url: str | None
+    is_read: bool
+    first_seen_at: datetime
+    is_new_in_latest_run: bool
+
+
+@dataclass(frozen=True)
+class UnreadPublicationItem:
+    publication_id: int
+    scholar_profile_id: int
+    scholar_label: str
+    title: str
+    year: int | None
+    citation_count: int
+    venue_text: str | None
+    pub_url: str | None
+
+
+def resolve_mode(value: str | None) -> str:
+    if value == MODE_ALL:
+        return MODE_ALL
+    return MODE_NEW
+
+
+async def get_latest_completed_run_id_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> int | None:
+    result = await db_session.execute(
+        select(func.max(CrawlRun.id)).where(
+            CrawlRun.user_id == user_id,
+            CrawlRun.status != RunStatus.RUNNING,
+        )
+    )
+    latest_run_id = result.scalar_one_or_none()
+    return int(latest_run_id) if latest_run_id is not None else None
+
+
+def publications_query(
+    *,
+    user_id: int,
+    mode: str,
+    latest_run_id: int | None,
+    scholar_profile_id: int | None,
+    limit: int,
+) -> Select[tuple]:
+    scholar_label = ScholarProfile.display_name
+
+    stmt = (
+        select(
+            Publication.id,
+            ScholarProfile.id,
+            scholar_label,
+            ScholarProfile.scholar_id,
+            Publication.title_raw,
+            Publication.year,
+            Publication.citation_count,
+            Publication.venue_text,
+            Publication.pub_url,
+            ScholarPublication.is_read,
+            ScholarPublication.first_seen_run_id,
+            ScholarPublication.created_at,
+        )
+        .join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
+        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
+        .where(ScholarProfile.user_id == user_id)
+        .order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
+        .limit(limit)
+    )
+    if scholar_profile_id is not None:
+        stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
+    if mode == MODE_NEW:
+        # "New" means discovered in the latest completed run.
+        if latest_run_id is None:
+            stmt = stmt.where(False)
+        else:
+            stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
+    return stmt
+
+
+async def list_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    mode: str = MODE_ALL,
+    scholar_profile_id: int | None = None,
+    limit: int = 300,
+) -> list[PublicationListItem]:
+    resolved_mode = resolve_mode(mode)
+    latest_run_id = await get_latest_completed_run_id_for_user(
+        db_session,
+        user_id=user_id,
+    )
+    result = await db_session.execute(
+        publications_query(
+            user_id=user_id,
+            mode=resolved_mode,
+            latest_run_id=latest_run_id,
+            scholar_profile_id=scholar_profile_id,
+            limit=limit,
+        )
+    )
+
+    rows = result.all()
+    items: list[PublicationListItem] = []
+    for row in rows:
+        (
+            publication_id,
+            scholar_profile_id,
+            display_name,
+            scholar_id,
+            title_raw,
+            year,
+            citation_count,
+            venue_text,
+            pub_url,
+            is_read,
+            first_seen_run_id,
+            created_at,
+        ) = row
+        items.append(
+            PublicationListItem(
+                publication_id=int(publication_id),
+                scholar_profile_id=int(scholar_profile_id),
+                scholar_label=(display_name or scholar_id),
+                title=title_raw,
+                year=year,
+                citation_count=int(citation_count or 0),
+                venue_text=venue_text,
+                pub_url=pub_url,
+                is_read=bool(is_read),
+                first_seen_at=created_at,
+                is_new_in_latest_run=(
+                    latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
+                ),
+            )
+        )
+    return items
+
+
+async def list_unread_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    limit: int = 100,
+) -> list[UnreadPublicationItem]:
+    result = await db_session.execute(
+        publications_query(
+            user_id=user_id,
+            mode=MODE_ALL,
+            latest_run_id=None,
+            scholar_profile_id=None,
+            limit=limit,
+        ).where(ScholarPublication.is_read.is_(False))
+    )
+    rows = result.all()
+    items: list[UnreadPublicationItem] = []
+    for row in rows:
+        (
+            publication_id,
+            scholar_profile_id,
+            display_name,
+            scholar_id,
+            title_raw,
+            year,
+            citation_count,
+            venue_text,
+            pub_url,
+            _is_read,
+            _first_seen_run_id,
+            _created_at,
+        ) = row
+        items.append(
+            UnreadPublicationItem(
+                publication_id=int(publication_id),
+                scholar_profile_id=int(scholar_profile_id),
+                scholar_label=(display_name or scholar_id),
+                title=title_raw,
+                year=year,
+                citation_count=int(citation_count or 0),
+                venue_text=venue_text,
+                pub_url=pub_url,
+            )
+        )
+    return items
+
+
+async def list_new_for_latest_run_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    limit: int = 100,
+) -> list[UnreadPublicationItem]:
+    rows = await list_for_user(
+        db_session,
+        user_id=user_id,
+        mode=MODE_NEW,
+        scholar_profile_id=None,
+        limit=limit,
+    )
+    return [
+        UnreadPublicationItem(
+            publication_id=row.publication_id,
+            scholar_profile_id=row.scholar_profile_id,
+            scholar_label=row.scholar_label,
+            title=row.title,
+            year=row.year,
+            citation_count=row.citation_count,
+            venue_text=row.venue_text,
+            pub_url=row.pub_url,
+        )
+        for row in rows
+    ]
+
+
+async def count_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    mode: str = MODE_ALL,
+    scholar_profile_id: int | None = None,
+) -> int:
+    resolved_mode = resolve_mode(mode)
+    latest_run_id = await get_latest_completed_run_id_for_user(
+        db_session,
+        user_id=user_id,
+    )
+    stmt = (
+        select(func.count())
+        .select_from(ScholarPublication)
+        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
+        .where(ScholarProfile.user_id == user_id)
+    )
+    if scholar_profile_id is not None:
+        stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
+    if resolved_mode == MODE_NEW:
+        if latest_run_id is None:
+            return 0
+        stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
+    result = await db_session.execute(stmt)
+    return int(result.scalar_one() or 0)
+
+
+async def mark_all_unread_as_read_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> int:
+    scholar_ids = (
+        select(ScholarProfile.id)
+        .where(ScholarProfile.user_id == user_id)
+        .scalar_subquery()
+    )
+
+    stmt = (
+        update(ScholarPublication)
+        .where(
+            ScholarPublication.scholar_profile_id.in_(scholar_ids),
+            ScholarPublication.is_read.is_(False),
+        )
+        .values(is_read=True)
+    )
+    result = await db_session.execute(stmt)
+    await db_session.commit()
+
+    rowcount = result.rowcount
+    return int(rowcount or 0)

app/services/runs.py

@@ -0,0 +1,464 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+from sqlalchemy import and_, case, func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import (
+    CrawlRun,
+    IngestionQueueItem,
+    RunStatus,
+    RunTriggerType,
+    ScholarProfile,
+)
+from app.services import continuation_queue as queue_service
+
+QUEUE_STATUS_QUEUED = "queued"
+QUEUE_STATUS_RETRYING = "retrying"
+QUEUE_STATUS_DROPPED = "dropped"
+
+
+@dataclass(frozen=True)
+class QueueListItem:
+    id: int
+    scholar_profile_id: int
+    scholar_label: str
+    status: str
+    reason: str
+    dropped_reason: str | None
+    attempt_count: int
+    resume_cstart: int
+    next_attempt_dt: datetime | None
+    updated_at: datetime
+    last_error: str | None
+    last_run_id: int | None
+
+
+@dataclass(frozen=True)
+class QueueClearResult:
+    queue_item_id: int
+    previous_status: str
+
+
+class QueueTransitionError(RuntimeError):
+    def __init__(self, *, code: str, message: str, current_status: str) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+        self.current_status = current_status
+
+
+def _safe_int(value: object, default: int = 0) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _summary_dict(error_log: object) -> dict[str, Any]:
+    if not isinstance(error_log, dict):
+        return {}
+    summary = error_log.get("summary")
+    if not isinstance(summary, dict):
+        return {}
+    return summary
+
+
+def extract_run_summary(error_log: object) -> dict[str, Any]:
+    summary = _summary_dict(error_log)
+    return {
+        "succeeded_count": _safe_int(summary.get("succeeded_count", 0)),
+        "failed_count": _safe_int(summary.get("failed_count", 0)),
+        "partial_count": _safe_int(summary.get("partial_count", 0)),
+        "failed_state_counts": {
+            str(key): _safe_int(value, 0)
+            for key, value in summary.get("failed_state_counts", {}).items()
+            if isinstance(key, str)
+        }
+        if isinstance(summary.get("failed_state_counts"), dict)
+        else {},
+        "failed_reason_counts": {
+            str(key): _safe_int(value, 0)
+            for key, value in summary.get("failed_reason_counts", {}).items()
+            if isinstance(key, str)
+        }
+        if isinstance(summary.get("failed_reason_counts"), dict)
+        else {},
+    }
+
+
+async def list_recent_runs_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    limit: int = 20,
+) -> list[CrawlRun]:
+    result = await db_session.execute(
+        select(CrawlRun)
+        .where(CrawlRun.user_id == user_id)
+        .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
+        .limit(limit)
+    )
+    return list(result.scalars().all())
+
+
+async def list_runs_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    limit: int = 100,
+    failed_only: bool = False,
+) -> list[CrawlRun]:
+    stmt = (
+        select(CrawlRun)
+        .where(CrawlRun.user_id == user_id)
+        .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
+        .limit(limit)
+    )
+    if failed_only:
+        stmt = stmt.where(
+            CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE])
+        )
+    result = await db_session.execute(stmt)
+    return list(result.scalars().all())
+
+
+async def get_run_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    run_id: int,
+) -> CrawlRun | None:
+    result = await db_session.execute(
+        select(CrawlRun).where(
+            CrawlRun.user_id == user_id,
+            CrawlRun.id == run_id,
+        )
+    )
+    return result.scalar_one_or_none()
+
+
+async def get_manual_run_by_idempotency_key(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    idempotency_key: str,
+) -> CrawlRun | None:
+    result = await db_session.execute(
+        select(CrawlRun)
+        .where(
+            CrawlRun.user_id == user_id,
+            CrawlRun.trigger_type == RunTriggerType.MANUAL,
+            CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key,
+        )
+        .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
+        .limit(1)
+    )
+    return result.scalar_one_or_none()
+
+
+async def set_manual_run_idempotency_key(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    run_id: int,
+    idempotency_key: str,
+) -> bool:
+    run = await get_run_for_user(
+        db_session,
+        user_id=user_id,
+        run_id=run_id,
+    )
+    if run is None:
+        return False
+    if run.trigger_type != RunTriggerType.MANUAL:
+        return False
+
+    payload = dict(run.error_log) if isinstance(run.error_log, dict) else {}
+    meta = payload.get("meta")
+    meta_dict = dict(meta) if isinstance(meta, dict) else {}
+    meta_dict["idempotency_key"] = idempotency_key
+    payload["meta"] = meta_dict
+    run.error_log = payload
+    await db_session.commit()
+    return True
+
+
+async def list_queue_items_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    limit: int = 200,
+) -> list[QueueListItem]:
+    result = await db_session.execute(
+        select(
+            IngestionQueueItem.id,
+            IngestionQueueItem.scholar_profile_id,
+            ScholarProfile.display_name,
+            ScholarProfile.scholar_id,
+            IngestionQueueItem.status,
+            IngestionQueueItem.reason,
+            IngestionQueueItem.dropped_reason,
+            IngestionQueueItem.attempt_count,
+            IngestionQueueItem.resume_cstart,
+            IngestionQueueItem.next_attempt_dt,
+            IngestionQueueItem.updated_at,
+            IngestionQueueItem.last_error,
+            IngestionQueueItem.last_run_id,
+        )
+        .join(
+            ScholarProfile,
+            and_(
+                ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
+                ScholarProfile.user_id == IngestionQueueItem.user_id,
+            ),
+        )
+        .where(IngestionQueueItem.user_id == user_id)
+        .order_by(
+            case((IngestionQueueItem.status == "dropped", 1), else_=0).asc(),
+            IngestionQueueItem.next_attempt_dt.asc(),
+            IngestionQueueItem.id.asc(),
+        )
+        .limit(limit)
+    )
+
+    items: list[QueueListItem] = []
+    for row in result.all():
+        (
+            item_id,
+            scholar_profile_id,
+            display_name,
+            scholar_id,
+            status,
+            reason,
+            dropped_reason,
+            attempt_count,
+            resume_cstart,
+            next_attempt_dt,
+            updated_at,
+            last_error,
+            last_run_id,
+        ) = row
+        items.append(
+            QueueListItem(
+                id=int(item_id),
+                scholar_profile_id=int(scholar_profile_id),
+                scholar_label=(display_name or scholar_id),
+                status=str(status),
+                reason=str(reason),
+                dropped_reason=dropped_reason,
+                attempt_count=int(attempt_count or 0),
+                resume_cstart=int(resume_cstart or 0),
+                next_attempt_dt=next_attempt_dt,
+                updated_at=updated_at,
+                last_error=last_error,
+                last_run_id=int(last_run_id) if last_run_id is not None else None,
+            )
+        )
+    return items
+
+
+async def get_queue_item_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    queue_item_id: int,
+) -> QueueListItem | None:
+    result = await db_session.execute(
+        select(
+            IngestionQueueItem.id,
+            IngestionQueueItem.scholar_profile_id,
+            ScholarProfile.display_name,
+            ScholarProfile.scholar_id,
+            IngestionQueueItem.status,
+            IngestionQueueItem.reason,
+            IngestionQueueItem.dropped_reason,
+            IngestionQueueItem.attempt_count,
+            IngestionQueueItem.resume_cstart,
+            IngestionQueueItem.next_attempt_dt,
+            IngestionQueueItem.updated_at,
+            IngestionQueueItem.last_error,
+            IngestionQueueItem.last_run_id,
+        )
+        .join(
+            ScholarProfile,
+            and_(
+                ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
+                ScholarProfile.user_id == IngestionQueueItem.user_id,
+            ),
+        )
+        .where(
+            IngestionQueueItem.user_id == user_id,
+            IngestionQueueItem.id == queue_item_id,
+        )
+        .limit(1)
+    )
+    row = result.one_or_none()
+    if row is None:
+        return None
+    (
+        item_id,
+        scholar_profile_id,
+        display_name,
+        scholar_id,
+        status,
+        reason,
+        dropped_reason,
+        attempt_count,
+        resume_cstart,
+        next_attempt_dt,
+        updated_at,
+        last_error,
+        last_run_id,
+    ) = row
+    return QueueListItem(
+        id=int(item_id),
+        scholar_profile_id=int(scholar_profile_id),
+        scholar_label=(display_name or scholar_id),
+        status=str(status),
+        reason=str(reason),
+        dropped_reason=dropped_reason,
+        attempt_count=int(attempt_count or 0),
+        resume_cstart=int(resume_cstart or 0),
+        next_attempt_dt=next_attempt_dt,
+        updated_at=updated_at,
+        last_error=last_error,
+        last_run_id=int(last_run_id) if last_run_id is not None else None,
+    )
+
+
+async def retry_queue_item_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    queue_item_id: int,
+) -> QueueListItem | None:
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(
+            IngestionQueueItem.id == queue_item_id,
+            IngestionQueueItem.user_id == user_id,
+        )
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    if item.status == QUEUE_STATUS_QUEUED:
+        raise QueueTransitionError(
+            code="queue_item_already_queued",
+            message="Queue item is already queued.",
+            current_status=item.status,
+        )
+    if item.status == QUEUE_STATUS_RETRYING:
+        raise QueueTransitionError(
+            code="queue_item_retrying",
+            message="Queue item is currently retrying.",
+            current_status=item.status,
+        )
+
+    await queue_service.mark_queued_now(
+        db_session,
+        job_id=item.id,
+        reason="manual_retry",
+        reset_attempt_count=(item.status == QUEUE_STATUS_DROPPED),
+    )
+    await db_session.commit()
+
+    return await get_queue_item_for_user(
+        db_session,
+        user_id=user_id,
+        queue_item_id=queue_item_id,
+    )
+
+
+async def drop_queue_item_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    queue_item_id: int,
+) -> QueueListItem | None:
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(
+            IngestionQueueItem.id == queue_item_id,
+            IngestionQueueItem.user_id == user_id,
+        )
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    if item.status == QUEUE_STATUS_DROPPED:
+        raise QueueTransitionError(
+            code="queue_item_already_dropped",
+            message="Queue item is already dropped.",
+            current_status=item.status,
+        )
+
+    await queue_service.mark_dropped(
+        db_session,
+        job_id=int(item.id),
+        reason="manual_drop",
+    )
+    await db_session.commit()
+    return await get_queue_item_for_user(
+        db_session,
+        user_id=user_id,
+        queue_item_id=queue_item_id,
+    )
+
+
+async def clear_queue_item_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    queue_item_id: int,
+) -> QueueClearResult | None:
+    result = await db_session.execute(
+        select(IngestionQueueItem).where(
+            IngestionQueueItem.id == queue_item_id,
+            IngestionQueueItem.user_id == user_id,
+        )
+    )
+    item = result.scalar_one_or_none()
+    if item is None:
+        return None
+    if item.status != QUEUE_STATUS_DROPPED:
+        raise QueueTransitionError(
+            code="queue_item_not_dropped",
+            message="Queue item can only be cleared after it is dropped.",
+            current_status=item.status,
+        )
+
+    item_id = int(item.id)
+    previous_status = str(item.status)
+    deleted = await queue_service.delete_job_by_id(
+        db_session,
+        job_id=item_id,
+    )
+    await db_session.commit()
+    if not deleted:
+        return None
+    return QueueClearResult(
+        queue_item_id=item_id,
+        previous_status=previous_status,
+    )
+
+
+async def queue_status_counts_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> dict[str, int]:
+    result = await db_session.execute(
+        select(
+            IngestionQueueItem.status,
+            func.count(IngestionQueueItem.id),
+        )
+        .where(IngestionQueueItem.user_id == user_id)
+        .group_by(IngestionQueueItem.status)
+    )
+    counts: dict[str, int] = {"queued": 0, "retrying": 0, "dropped": 0}
+    for status, count in result.all():
+        counts[str(status)] = int(count or 0)
+    return counts

app/services/scheduler.py

@@ -0,0 +1,478 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+import logging
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import (
+    CrawlRun,
+    QueueItemStatus,
+    RunTriggerType,
+    ScholarProfile,
+    User,
+    UserSetting,
+)
+from app.db.session import get_session_factory
+from app.services import continuation_queue as queue_service
+from app.services.ingestion import RunAlreadyInProgressError, ScholarIngestionService
+from app.services.scholar_source import LiveScholarSource
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class _AutoRunCandidate:
+    user_id: int
+    run_interval_minutes: int
+    request_delay_seconds: int
+
+
+class SchedulerService:
+    def __init__(
+        self,
+        *,
+        enabled: bool,
+        tick_seconds: int,
+        network_error_retries: int,
+        retry_backoff_seconds: float,
+        max_pages_per_scholar: int,
+        page_size: int,
+        continuation_queue_enabled: bool,
+        continuation_base_delay_seconds: int,
+        continuation_max_delay_seconds: int,
+        continuation_max_attempts: int,
+        queue_batch_size: int,
+    ) -> None:
+        self._enabled = enabled
+        self._tick_seconds = max(5, int(tick_seconds))
+        self._network_error_retries = max(0, int(network_error_retries))
+        self._retry_backoff_seconds = max(0.0, float(retry_backoff_seconds))
+        self._max_pages_per_scholar = max(1, int(max_pages_per_scholar))
+        self._page_size = max(1, int(page_size))
+        self._continuation_queue_enabled = bool(continuation_queue_enabled)
+        self._continuation_base_delay_seconds = max(1, int(continuation_base_delay_seconds))
+        self._continuation_max_delay_seconds = max(
+            self._continuation_base_delay_seconds,
+            int(continuation_max_delay_seconds),
+        )
+        self._continuation_max_attempts = max(1, int(continuation_max_attempts))
+        self._queue_batch_size = max(1, int(queue_batch_size))
+        self._task: asyncio.Task[None] | None = None
+        self._source = LiveScholarSource()
+
+    async def start(self) -> None:
+        if not self._enabled:
+            logger.info(
+                "scheduler.disabled",
+                extra={
+                    "event": "scheduler.disabled",
+                },
+            )
+            return
+        if self._task is not None:
+            return
+        self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler")
+        logger.info(
+            "scheduler.started",
+            extra={
+                "event": "scheduler.started",
+                "tick_seconds": self._tick_seconds,
+                "network_error_retries": self._network_error_retries,
+                "retry_backoff_seconds": self._retry_backoff_seconds,
+                "max_pages_per_scholar": self._max_pages_per_scholar,
+                "page_size": self._page_size,
+                "continuation_queue_enabled": self._continuation_queue_enabled,
+                "continuation_base_delay_seconds": self._continuation_base_delay_seconds,
+                "continuation_max_delay_seconds": self._continuation_max_delay_seconds,
+                "continuation_max_attempts": self._continuation_max_attempts,
+                "queue_batch_size": self._queue_batch_size,
+            },
+        )
+
+    async def stop(self) -> None:
+        if self._task is None:
+            return
+        self._task.cancel()
+        try:
+            await self._task
+        except asyncio.CancelledError:
+            pass
+        finally:
+            self._task = None
+        logger.info("scheduler.stopped", extra={"event": "scheduler.stopped"})
+
+    async def _run_loop(self) -> None:
+        while True:
+            try:
+                await self._tick_once()
+            except asyncio.CancelledError:
+                raise
+            except Exception:
+                logger.exception(
+                    "scheduler.tick_failed",
+                    extra={
+                        "event": "scheduler.tick_failed",
+                    },
+                )
+            await asyncio.sleep(float(self._tick_seconds))
+
+    async def _tick_once(self) -> None:
+        if self._continuation_queue_enabled:
+            await self._drain_continuation_queue()
+        candidates = await self._load_candidates()
+        if not candidates:
+            return
+        now = datetime.now(timezone.utc)
+        for candidate in candidates:
+            if not await self._is_due(candidate, now=now):
+                continue
+            await self._run_candidate(candidate)
+
+    async def _load_candidates(self) -> list[_AutoRunCandidate]:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            result = await session.execute(
+                select(
+                    UserSetting.user_id,
+                    UserSetting.run_interval_minutes,
+                    UserSetting.request_delay_seconds,
+                )
+                .join(User, User.id == UserSetting.user_id)
+                .where(
+                    User.is_active.is_(True),
+                    UserSetting.auto_run_enabled.is_(True),
+                )
+                .order_by(UserSetting.user_id.asc())
+            )
+            rows = result.all()
+        return [
+            _AutoRunCandidate(
+                user_id=int(user_id),
+                run_interval_minutes=int(run_interval_minutes),
+                request_delay_seconds=int(request_delay_seconds),
+            )
+            for user_id, run_interval_minutes, request_delay_seconds in rows
+        ]
+
+    async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            result = await session.execute(
+                select(CrawlRun.start_dt)
+                .where(
+                    CrawlRun.user_id == candidate.user_id,
+                )
+                .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
+                .limit(1)
+            )
+            last_run = result.scalar_one_or_none()
+
+        if last_run is None:
+            return True
+
+        next_due_dt = last_run + timedelta(
+            minutes=candidate.run_interval_minutes
+        )
+        return now >= next_due_dt
+
+    async def _run_candidate(self, candidate: _AutoRunCandidate) -> None:
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            ingestion = ScholarIngestionService(source=self._source)
+            try:
+                run_summary = await ingestion.run_for_user(
+                    session,
+                    user_id=candidate.user_id,
+                    trigger_type=RunTriggerType.SCHEDULED,
+                    request_delay_seconds=candidate.request_delay_seconds,
+                    network_error_retries=self._network_error_retries,
+                    retry_backoff_seconds=self._retry_backoff_seconds,
+                    max_pages_per_scholar=self._max_pages_per_scholar,
+                    page_size=self._page_size,
+                    auto_queue_continuations=self._continuation_queue_enabled,
+                    queue_delay_seconds=self._continuation_base_delay_seconds,
+                )
+            except RunAlreadyInProgressError:
+                await session.rollback()
+                logger.info(
+                    "scheduler.run_skipped_locked",
+                    extra={
+                        "event": "scheduler.run_skipped_locked",
+                        "user_id": candidate.user_id,
+                    },
+                )
+                return
+            except Exception:
+                await session.rollback()
+                logger.exception(
+                    "scheduler.run_failed",
+                    extra={
+                        "event": "scheduler.run_failed",
+                        "user_id": candidate.user_id,
+                    },
+                )
+                return
+
+        logger.info(
+            "scheduler.run_completed",
+            extra={
+                "event": "scheduler.run_completed",
+                "user_id": candidate.user_id,
+                "run_id": run_summary.crawl_run_id,
+                "status": run_summary.status.value,
+                "scholar_count": run_summary.scholar_count,
+                "new_publication_count": run_summary.new_publication_count,
+            },
+        )
+
+    async def _drain_continuation_queue(self) -> None:
+        now = datetime.now(timezone.utc)
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            jobs = await queue_service.list_due_jobs(
+                session,
+                now=now,
+                limit=self._queue_batch_size,
+            )
+        for job in jobs:
+            await self._run_queue_job(job)
+
+    async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
+        if job.attempt_count >= self._continuation_max_attempts:
+            session_factory = get_session_factory()
+            async with session_factory() as session:
+                dropped = await queue_service.mark_dropped(
+                    session,
+                    job_id=job.id,
+                    reason="max_attempts_reached",
+                )
+                await session.commit()
+            if dropped is not None:
+                logger.warning(
+                    "scheduler.queue_item_dropped_max_attempts",
+                    extra={
+                        "event": "scheduler.queue_item_dropped_max_attempts",
+                        "queue_item_id": job.id,
+                        "user_id": job.user_id,
+                        "scholar_profile_id": job.scholar_profile_id,
+                        "attempt_count": job.attempt_count,
+                        "max_attempts": self._continuation_max_attempts,
+                    },
+                )
+            return
+
+        session_factory = get_session_factory()
+        async with session_factory() as session:
+            queue_item = await queue_service.mark_retrying(session, job_id=job.id)
+            if queue_item is None:
+                await session.commit()
+                return
+            if queue_item.status == QueueItemStatus.DROPPED.value:
+                await session.commit()
+                return
+            await session.commit()
+
+        async with session_factory() as session:
+            scholar_result = await session.execute(
+                select(ScholarProfile.id).where(
+                    ScholarProfile.user_id == job.user_id,
+                    ScholarProfile.id == job.scholar_profile_id,
+                    ScholarProfile.is_enabled.is_(True),
+                )
+            )
+            scholar_id = scholar_result.scalar_one_or_none()
+            if scholar_id is None:
+                dropped = await queue_service.mark_dropped(
+                    session,
+                    job_id=job.id,
+                    reason="scholar_unavailable",
+                )
+                await session.commit()
+                if dropped is not None:
+                    logger.info(
+                        "scheduler.queue_item_dropped_scholar_unavailable",
+                        extra={
+                            "event": "scheduler.queue_item_dropped_scholar_unavailable",
+                            "queue_item_id": job.id,
+                            "user_id": job.user_id,
+                            "scholar_profile_id": job.scholar_profile_id,
+                        },
+                    )
+                return
+
+        async with session_factory() as session:
+            request_delay_seconds = await self._load_request_delay_for_user(
+                session,
+                user_id=job.user_id,
+            )
+            ingestion = ScholarIngestionService(source=self._source)
+            try:
+                run_summary = await ingestion.run_for_user(
+                    session,
+                    user_id=job.user_id,
+                    trigger_type=RunTriggerType.SCHEDULED,
+                    request_delay_seconds=request_delay_seconds,
+                    network_error_retries=self._network_error_retries,
+                    retry_backoff_seconds=self._retry_backoff_seconds,
+                    max_pages_per_scholar=self._max_pages_per_scholar,
+                    page_size=self._page_size,
+                    scholar_profile_ids={job.scholar_profile_id},
+                    start_cstart_by_scholar_id={
+                        job.scholar_profile_id: job.resume_cstart,
+                    },
+                    auto_queue_continuations=self._continuation_queue_enabled,
+                    queue_delay_seconds=self._continuation_base_delay_seconds,
+                )
+            except RunAlreadyInProgressError:
+                await session.rollback()
+                async with session_factory() as recovery_session:
+                    await queue_service.reschedule_job(
+                        recovery_session,
+                        job_id=job.id,
+                        delay_seconds=max(self._tick_seconds, 15),
+                        reason="user_run_lock_active",
+                        error="run_already_in_progress",
+                    )
+                    await recovery_session.commit()
+                logger.info(
+                    "scheduler.queue_item_deferred_lock",
+                    extra={
+                        "event": "scheduler.queue_item_deferred_lock",
+                        "queue_item_id": job.id,
+                        "user_id": job.user_id,
+                    },
+                )
+                return
+            except Exception as exc:
+                await session.rollback()
+                async with session_factory() as recovery_session:
+                    queue_item = await queue_service.increment_attempt_count(
+                        recovery_session,
+                        job_id=job.id,
+                    )
+                    if queue_item is None:
+                        await recovery_session.commit()
+                        return
+                    if int(queue_item.attempt_count) >= self._continuation_max_attempts:
+                        await queue_service.mark_dropped(
+                            recovery_session,
+                            job_id=job.id,
+                            reason="scheduler_exception_max_attempts",
+                            error=str(exc),
+                        )
+                        await recovery_session.commit()
+                        logger.warning(
+                            "scheduler.queue_item_dropped_after_exception",
+                            extra={
+                                "event": "scheduler.queue_item_dropped_after_exception",
+                                "queue_item_id": job.id,
+                                "user_id": job.user_id,
+                                "attempt_count": queue_item.attempt_count,
+                            },
+                        )
+                        return
+                    delay_seconds = queue_service.compute_backoff_seconds(
+                        base_seconds=self._continuation_base_delay_seconds,
+                        attempt_count=int(queue_item.attempt_count),
+                        max_seconds=self._continuation_max_delay_seconds,
+                    )
+                    await queue_service.reschedule_job(
+                        recovery_session,
+                        job_id=job.id,
+                        delay_seconds=delay_seconds,
+                        reason="scheduler_exception",
+                        error=str(exc),
+                    )
+                    await recovery_session.commit()
+                logger.exception(
+                    "scheduler.queue_item_run_failed",
+                    extra={
+                        "event": "scheduler.queue_item_run_failed",
+                        "queue_item_id": job.id,
+                        "user_id": job.user_id,
+                    },
+                )
+                return
+
+        async with session_factory() as session:
+            queue_item = await queue_service.increment_attempt_count(
+                session,
+                job_id=job.id,
+            )
+            if queue_item is None:
+                await session.commit()
+                logger.info(
+                    "scheduler.queue_item_resolved",
+                    extra={
+                        "event": "scheduler.queue_item_resolved",
+                        "queue_item_id": job.id,
+                        "user_id": job.user_id,
+                        "run_id": run_summary.crawl_run_id,
+                        "status": run_summary.status.value,
+                    },
+                )
+                return
+
+            if int(queue_item.attempt_count) >= self._continuation_max_attempts:
+                await queue_service.mark_dropped(
+                    session,
+                    job_id=job.id,
+                    reason="max_attempts_after_run",
+                )
+                await session.commit()
+                logger.warning(
+                    "scheduler.queue_item_dropped_max_attempts_after_run",
+                    extra={
+                        "event": "scheduler.queue_item_dropped_max_attempts_after_run",
+                        "queue_item_id": job.id,
+                        "user_id": job.user_id,
+                        "attempt_count": queue_item.attempt_count,
+                        "run_id": run_summary.crawl_run_id,
+                        "status": run_summary.status.value,
+                    },
+                )
+                return
+
+            delay_seconds = queue_service.compute_backoff_seconds(
+                base_seconds=self._continuation_base_delay_seconds,
+                attempt_count=int(queue_item.attempt_count),
+                max_seconds=self._continuation_max_delay_seconds,
+            )
+            await queue_service.reschedule_job(
+                session,
+                job_id=job.id,
+                delay_seconds=delay_seconds,
+                reason=queue_item.reason,
+                error=queue_item.last_error,
+            )
+            await session.commit()
+            logger.info(
+                "scheduler.queue_item_rescheduled",
+                extra={
+                    "event": "scheduler.queue_item_rescheduled",
+                    "queue_item_id": job.id,
+                    "user_id": job.user_id,
+                    "attempt_count": queue_item.attempt_count,
+                    "delay_seconds": delay_seconds,
+                    "run_id": run_summary.crawl_run_id,
+                    "status": run_summary.status.value,
+                },
+            )
+
+    async def _load_request_delay_for_user(
+        self,
+        db_session: AsyncSession,
+        *,
+        user_id: int,
+    ) -> int:
+        result = await db_session.execute(
+            select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
+        )
+        delay = result.scalar_one_or_none()
+        if delay is None:
+            return 10
+        return max(1, int(delay))

app/services/scholar_parser.py

@@ -0,0 +1,370 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from enum import StrEnum
+from html import unescape
+from html.parser import HTMLParser
+from typing import Any
+from urllib.parse import parse_qs, urlparse
+
+from app.services.scholar_source import FetchResult
+
+BLOCKED_KEYWORDS = [
+    "unusual traffic",
+    "sorry/index",
+    "not a robot",
+    "our systems have detected",
+    "automated queries",
+]
+
+NO_RESULTS_KEYWORDS = [
+    "didn't match any articles",
+    "did not match any articles",
+    "no articles",
+    "no documents",
+]
+
+MARKER_KEYS = [
+    "gsc_a_tr",
+    "gsc_a_at",
+    "gsc_a_ac",
+    "gsc_a_h",
+    "gsc_a_y",
+    "gs_gray",
+    "gsc_prf_in",
+    "gsc_rsb_st",
+]
+
+TAG_RE = re.compile(r"<[^>]+>", re.S)
+SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.I | re.S)
+SHOW_MORE_BUTTON_RE = re.compile(
+    r"<button\b[^>]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>",
+    re.I | re.S,
+)
+
+
+class ParseState(StrEnum):
+    OK = "ok"
+    NO_RESULTS = "no_results"
+    BLOCKED_OR_CAPTCHA = "blocked_or_captcha"
+    LAYOUT_CHANGED = "layout_changed"
+    NETWORK_ERROR = "network_error"
+
+
+@dataclass(frozen=True)
+class PublicationCandidate:
+    title: str
+    title_url: str | None
+    cluster_id: str | None
+    year: int | None
+    citation_count: int | None
+    authors_text: str | None
+    venue_text: str | None
+
+
+@dataclass(frozen=True)
+class ParsedProfilePage:
+    state: ParseState
+    state_reason: str
+    profile_name: str | None
+    publications: list[PublicationCandidate]
+    marker_counts: dict[str, int]
+    warnings: list[str]
+    has_show_more_button: bool
+    has_operation_error_banner: bool
+    articles_range: str | None
+
+
+def normalize_space(value: str) -> str:
+    return " ".join(unescape(value).split())
+
+
+def strip_tags(value: str) -> str:
+    return normalize_space(TAG_RE.sub(" ", value))
+
+
+def attr_class(attrs: list[tuple[str, str | None]]) -> str:
+    for name, raw_value in attrs:
+        if name.lower() == "class":
+            return raw_value or ""
+    return ""
+
+
+def attr_href(attrs: list[tuple[str, str | None]]) -> str | None:
+    for name, raw_value in attrs:
+        if name.lower() == "href":
+            return raw_value
+    return None
+
+
+class ScholarRowParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.title_href: str | None = None
+        self.title_parts: list[str] = []
+        self.citation_parts: list[str] = []
+        self.year_parts: list[str] = []
+        self.gray_texts: list[str] = []
+
+        self._title_depth = 0
+        self._citation_depth = 0
+        self._year_depth = 0
+        self._gray_stack: list[dict[str, Any]] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if self._title_depth > 0:
+            self._title_depth += 1
+        if self._citation_depth > 0:
+            self._citation_depth += 1
+        if self._year_depth > 0:
+            self._year_depth += 1
+        if self._gray_stack:
+            self._gray_stack[-1]["depth"] += 1
+
+        classes = attr_class(attrs)
+
+        if tag == "a" and "gsc_a_at" in classes:
+            self._title_depth = 1
+            self.title_href = attr_href(attrs)
+            return
+
+        if tag == "a" and "gsc_a_ac" in classes:
+            self._citation_depth = 1
+            return
+
+        if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes):
+            self._year_depth = 1
+            return
+
+        if tag == "div" and "gs_gray" in classes:
+            self._gray_stack.append({"depth": 1, "parts": []})
+            return
+
+    def handle_data(self, data: str) -> None:
+        if self._title_depth > 0:
+            self.title_parts.append(data)
+        if self._citation_depth > 0:
+            self.citation_parts.append(data)
+        if self._year_depth > 0:
+            self.year_parts.append(data)
+        if self._gray_stack:
+            self._gray_stack[-1]["parts"].append(data)
+
+    def handle_endtag(self, _tag: str) -> None:
+        if self._title_depth > 0:
+            self._title_depth -= 1
+        if self._citation_depth > 0:
+            self._citation_depth -= 1
+        if self._year_depth > 0:
+            self._year_depth -= 1
+        if self._gray_stack:
+            self._gray_stack[-1]["depth"] -= 1
+            if self._gray_stack[-1]["depth"] == 0:
+                text = normalize_space("".join(self._gray_stack[-1]["parts"]))
+                if text:
+                    self.gray_texts.append(text)
+                self._gray_stack.pop()
+
+
+def extract_rows(html: str) -> list[str]:
+    pattern = re.compile(
+        r"<tr\b(?=[^>]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)</tr>",
+        re.I | re.S,
+    )
+    return [match.group(1) for match in pattern.finditer(html)]
+
+
+def parse_cluster_id_from_href(href: str | None) -> str | None:
+    if not href:
+        return None
+    parsed = urlparse(href)
+    query = parse_qs(parsed.query)
+
+    citation_for_view = query.get("citation_for_view")
+    if citation_for_view:
+        token = citation_for_view[0].strip()
+        if token:
+            if ":" in token:
+                return token.rsplit(":", 1)[-1] or None
+            return token
+
+    cluster = query.get("cluster")
+    if cluster:
+        token = cluster[0].strip()
+        if token:
+            return token
+    return None
+
+
+def parse_year(parts: list[str]) -> int | None:
+    text = normalize_space(" ".join(parts))
+    match = re.search(r"\b(19|20)\d{2}\b", text)
+    if not match:
+        return None
+    try:
+        return int(match.group(0))
+    except ValueError:
+        return None
+
+
+def parse_citation_count(parts: list[str]) -> int | None:
+    text = normalize_space(" ".join(parts))
+    if not text:
+        return 0
+    match = re.search(r"\d+", text)
+    if not match:
+        return None
+    return int(match.group(0))
+
+
+def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]:
+    rows = extract_rows(html)
+    warnings: list[str] = []
+    publications: list[PublicationCandidate] = []
+
+    for row_html in rows:
+        parser = ScholarRowParser()
+        parser.feed(row_html)
+
+        title = normalize_space("".join(parser.title_parts))
+        if not title:
+            warnings.append("row_missing_title")
+            continue
+
+        authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
+        venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
+
+        publications.append(
+            PublicationCandidate(
+                title=title,
+                title_url=parser.title_href,
+                cluster_id=parse_cluster_id_from_href(parser.title_href),
+                year=parse_year(parser.year_parts),
+                citation_count=parse_citation_count(parser.citation_parts),
+                authors_text=authors_text,
+                venue_text=venue_text,
+            )
+        )
+
+    if not rows:
+        warnings.append("no_rows_detected")
+
+    return publications, sorted(set(warnings))
+
+
+def extract_profile_name(html: str) -> str | None:
+    pattern = re.compile(
+        r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)</[^>]+>",
+        re.I | re.S,
+    )
+    match = pattern.search(html)
+    if not match:
+        return None
+    value = strip_tags(match.group(1))
+    return value or None
+
+
+def extract_articles_range(html: str) -> str | None:
+    pattern = re.compile(
+        r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)</[^>]+>",
+        re.I | re.S,
+    )
+    match = pattern.search(html)
+    if not match:
+        return None
+    value = strip_tags(match.group(1))
+    return value or None
+
+
+def has_show_more_button(html: str) -> bool:
+    match = SHOW_MORE_BUTTON_RE.search(html)
+    if match is None:
+        return False
+
+    button_tag = match.group(0).lower()
+    if "disabled" in button_tag:
+        return False
+    if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag:
+        return False
+    if "gs_dis" in button_tag:
+        return False
+    return True
+
+
+def has_operation_error_banner(html: str) -> bool:
+    lowered = html.lower()
+    if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered:
+        return False
+    return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered
+
+
+def count_markers(html: str) -> dict[str, int]:
+    lowered = html.lower()
+    return {key: lowered.count(key.lower()) for key in MARKER_KEYS}
+
+
+def detect_state(
+    fetch_result: FetchResult,
+    publications: list[PublicationCandidate],
+    marker_counts: dict[str, int],
+    *,
+    visible_text: str,
+) -> tuple[ParseState, str]:
+    if fetch_result.status_code is None:
+        return ParseState.NETWORK_ERROR, "network_error_missing_status_code"
+
+    lowered = fetch_result.body.lower()
+    final = (fetch_result.final_url or "").lower()
+
+    if "accounts.google.com" in final and ("signin" in final or "servicelogin" in final):
+        return ParseState.BLOCKED_OR_CAPTCHA, "blocked_accounts_redirect"
+    if any(keyword in lowered for keyword in BLOCKED_KEYWORDS) or "sorry/index" in final:
+        return ParseState.BLOCKED_OR_CAPTCHA, "blocked_keyword_detected"
+
+    if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS):
+        return ParseState.NO_RESULTS, "no_results_keyword_detected"
+
+    if not publications:
+        has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
+        has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
+        if not has_profile_markers and not has_table_markers:
+            return ParseState.LAYOUT_CHANGED, "layout_markers_missing"
+        return ParseState.OK, "no_rows_with_known_markers"
+
+    return ParseState.OK, "publications_extracted"
+
+
+def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
+    publications, warnings = parse_publications(fetch_result.body)
+    marker_counts = count_markers(fetch_result.body)
+    visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
+
+    show_more = has_show_more_button(fetch_result.body)
+    operation_error_banner = has_operation_error_banner(fetch_result.body)
+
+    if show_more:
+        warnings.append("possible_partial_page_show_more_present")
+    if operation_error_banner:
+        warnings.append("operation_error_banner_present")
+
+    warnings = sorted(set(warnings))
+
+    state, state_reason = detect_state(
+        fetch_result,
+        publications,
+        marker_counts,
+        visible_text=visible_text,
+    )
+
+    return ParsedProfilePage(
+        state=state,
+        state_reason=state_reason,
+        profile_name=extract_profile_name(fetch_result.body),
+        publications=publications,
+        marker_counts=marker_counts,
+        warnings=warnings,
+        has_show_more_button=show_more,
+        has_operation_error_banner=operation_error_banner,
+        articles_range=extract_articles_range(fetch_result.body),
+    )

app/services/scholar_source.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import random
+from dataclasses import dataclass
+from typing import Protocol
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations"
+DEFAULT_PAGE_SIZE = 100
+
+DEFAULT_USER_AGENTS = [
+    (
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+    ),
+    (
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 "
+        "(KHTML, like Gecko) Version/18.1 Safari/605.1.15"
+    ),
+    (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) "
+        "Gecko/20100101 Firefox/131.0"
+    ),
+]
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class FetchResult:
+    requested_url: str
+    status_code: int | None
+    final_url: str | None
+    body: str
+    error: str | None
+
+
+class ScholarSource(Protocol):
+    async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
+        ...
+
+    async def fetch_profile_page_html(
+        self,
+        scholar_id: str,
+        *,
+        cstart: int,
+        pagesize: int,
+    ) -> FetchResult:
+        ...
+
+
+class LiveScholarSource:
+    def __init__(
+        self,
+        *,
+        timeout_seconds: float = 25.0,
+        user_agents: list[str] | None = None,
+    ) -> None:
+        self._timeout_seconds = timeout_seconds
+        self._user_agents = user_agents or DEFAULT_USER_AGENTS
+
+    async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
+        return await self.fetch_profile_page_html(
+            scholar_id,
+            cstart=0,
+            pagesize=DEFAULT_PAGE_SIZE,
+        )
+
+    async def fetch_profile_page_html(
+        self,
+        scholar_id: str,
+        *,
+        cstart: int,
+        pagesize: int = DEFAULT_PAGE_SIZE,
+    ) -> FetchResult:
+        requested_url = _build_profile_url(
+            scholar_id=scholar_id,
+            cstart=cstart,
+            pagesize=pagesize,
+        )
+        logger.debug(
+            "scholar_source.fetch_started",
+            extra={
+                "event": "scholar_source.fetch_started",
+                "scholar_id": scholar_id,
+                "requested_url": requested_url,
+                "cstart": cstart,
+                "pagesize": pagesize,
+            },
+        )
+        return await asyncio.to_thread(self._fetch_sync, requested_url)
+
+    def _fetch_sync(self, requested_url: str) -> FetchResult:
+        request = Request(
+            requested_url,
+            headers={
+                "User-Agent": random.choice(self._user_agents),
+                "Accept": "text/html,application/xhtml+xml",
+                "Accept-Language": "en-US,en;q=0.9",
+                "Connection": "close",
+            },
+        )
+
+        try:
+            with urlopen(request, timeout=self._timeout_seconds) as response:
+                body = response.read().decode("utf-8", errors="replace")
+                status_code = getattr(response, "status", 200)
+                logger.debug(
+                    "scholar_source.fetch_succeeded",
+                    extra={
+                        "event": "scholar_source.fetch_succeeded",
+                        "requested_url": requested_url,
+                        "status_code": status_code,
+                    },
+                )
+                return FetchResult(
+                    requested_url=requested_url,
+                    status_code=status_code,
+                    final_url=response.geturl(),
+                    body=body,
+                    error=None,
+                )
+        except HTTPError as exc:
+            body = ""
+            try:
+                body = exc.read().decode("utf-8", errors="replace")
+            except Exception:
+                body = ""
+            logger.warning(
+                "scholar_source.fetch_http_error",
+                extra={
+                    "event": "scholar_source.fetch_http_error",
+                    "requested_url": requested_url,
+                    "status_code": exc.code,
+                },
+            )
+            return FetchResult(
+                requested_url=requested_url,
+                status_code=exc.code,
+                final_url=exc.geturl(),
+                body=body,
+                error=str(exc),
+            )
+        except URLError as exc:
+            logger.warning(
+                "scholar_source.fetch_network_error",
+                extra={
+                    "event": "scholar_source.fetch_network_error",
+                    "requested_url": requested_url,
+                },
+            )
+            return FetchResult(
+                requested_url=requested_url,
+                status_code=None,
+                final_url=None,
+                body="",
+                error=str(exc),
+            )
+
+
+def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:
+    query: dict[str, int | str] = {"hl": "en", "user": scholar_id}
+    if cstart > 0:
+        query["cstart"] = int(cstart)
+    if pagesize > 0 and cstart > 0:
+        query["pagesize"] = int(pagesize)
+    return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}"

app/services/scholars.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import re
+
+from sqlalchemy import select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import ScholarProfile
+
+SCHOLAR_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]{12}$")
+
+
+class ScholarServiceError(ValueError):
+    """Raised for expected scholar-management validation failures."""
+
+
+def validate_scholar_id(value: str) -> str:
+    scholar_id = value.strip()
+    if not SCHOLAR_ID_PATTERN.fullmatch(scholar_id):
+        raise ScholarServiceError("Scholar ID must match [a-zA-Z0-9_-]{12}.")
+    return scholar_id
+
+
+def normalize_display_name(value: str) -> str | None:
+    normalized = value.strip()
+    return normalized if normalized else None
+
+
+async def list_scholars_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> list[ScholarProfile]:
+    result = await db_session.execute(
+        select(ScholarProfile)
+        .where(ScholarProfile.user_id == user_id)
+        .order_by(ScholarProfile.created_at.desc(), ScholarProfile.id.desc())
+    )
+    return list(result.scalars().all())
+
+
+async def create_scholar_for_user(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_id: str,
+    display_name: str,
+) -> ScholarProfile:
+    profile = ScholarProfile(
+        user_id=user_id,
+        scholar_id=validate_scholar_id(scholar_id),
+        display_name=normalize_display_name(display_name),
+    )
+    db_session.add(profile)
+    try:
+        await db_session.commit()
+    except IntegrityError as exc:
+        await db_session.rollback()
+        raise ScholarServiceError("That scholar is already tracked for this account.") from exc
+    await db_session.refresh(profile)
+    return profile
+
+
+async def get_user_scholar_by_id(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+    scholar_profile_id: int,
+) -> ScholarProfile | None:
+    result = await db_session.execute(
+        select(ScholarProfile).where(
+            ScholarProfile.id == scholar_profile_id,
+            ScholarProfile.user_id == user_id,
+        )
+    )
+    return result.scalar_one_or_none()
+
+
+async def toggle_scholar_enabled(
+    db_session: AsyncSession,
+    *,
+    profile: ScholarProfile,
+) -> ScholarProfile:
+    profile.is_enabled = not profile.is_enabled
+    await db_session.commit()
+    await db_session.refresh(profile)
+    return profile
+
+
+async def delete_scholar(
+    db_session: AsyncSession,
+    *,
+    profile: ScholarProfile,
+) -> None:
+    await db_session.delete(profile)
+    await db_session.commit()
+

app/services/user_settings.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import UserSetting
+
+
+class UserSettingsServiceError(ValueError):
+    """Raised for expected settings-validation failures."""
+
+
+def parse_run_interval_minutes(value: str) -> int:
+    try:
+        parsed = int(value)
+    except ValueError as exc:
+        raise UserSettingsServiceError("Run interval must be a whole number.") from exc
+    if parsed < 15:
+        raise UserSettingsServiceError("Run interval must be at least 15 minutes.")
+    return parsed
+
+
+def parse_request_delay_seconds(value: str) -> int:
+    try:
+        parsed = int(value)
+    except ValueError as exc:
+        raise UserSettingsServiceError("Request delay must be a whole number.") from exc
+    if parsed < 1:
+        raise UserSettingsServiceError("Request delay must be at least 1 second.")
+    return parsed
+
+
+async def get_or_create_settings(
+    db_session: AsyncSession,
+    *,
+    user_id: int,
+) -> UserSetting:
+    result = await db_session.execute(
+        select(UserSetting).where(UserSetting.user_id == user_id)
+    )
+    settings = result.scalar_one_or_none()
+    if settings is not None:
+        return settings
+
+    settings = UserSetting(user_id=user_id)
+    db_session.add(settings)
+    await db_session.commit()
+    await db_session.refresh(settings)
+    return settings
+
+
+async def update_settings(
+    db_session: AsyncSession,
+    *,
+    settings: UserSetting,
+    auto_run_enabled: bool,
+    run_interval_minutes: int,
+    request_delay_seconds: int,
+) -> UserSetting:
+    settings.auto_run_enabled = auto_run_enabled
+    settings.run_interval_minutes = run_interval_minutes
+    settings.request_delay_seconds = request_delay_seconds
+    await db_session.commit()
+    await db_session.refresh(settings)
+    return settings
+

app/services/users.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import re
+
+from sqlalchemy import select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import User
+
+EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
+
+
+class UserServiceError(ValueError):
+    """Raised for expected user-management validation failures."""
+
+
+def normalize_email(value: str) -> str:
+    return value.strip().lower()
+
+
+def validate_email(value: str) -> str:
+    email = normalize_email(value)
+    if not EMAIL_PATTERN.fullmatch(email):
+        raise UserServiceError("Enter a valid email address.")
+    return email
+
+
+def validate_password(value: str) -> str:
+    password = value.strip()
+    if len(password) < 8:
+        raise UserServiceError("Password must be at least 8 characters.")
+    return password
+
+
+async def get_user_by_id(db_session: AsyncSession, user_id: int) -> User | None:
+    result = await db_session.execute(select(User).where(User.id == user_id))
+    return result.scalar_one_or_none()
+
+
+async def get_user_by_email(db_session: AsyncSession, email: str) -> User | None:
+    result = await db_session.execute(
+        select(User).where(User.email == normalize_email(email))
+    )
+    return result.scalar_one_or_none()
+
+
+async def list_users(db_session: AsyncSession) -> list[User]:
+    result = await db_session.execute(select(User).order_by(User.email.asc()))
+    return list(result.scalars().all())
+
+
+async def create_user(
+    db_session: AsyncSession,
+    *,
+    email: str,
+    password_hash: str,
+    is_admin: bool,
+) -> User:
+    user = User(
+        email=validate_email(email),
+        password_hash=password_hash,
+        is_admin=is_admin,
+        is_active=True,
+    )
+    db_session.add(user)
+    try:
+        await db_session.commit()
+    except IntegrityError as exc:
+        await db_session.rollback()
+        raise UserServiceError("A user with that email already exists.") from exc
+    await db_session.refresh(user)
+    return user
+
+
+async def set_user_active(
+    db_session: AsyncSession,
+    *,
+    user: User,
+    is_active: bool,
+) -> User:
+    user.is_active = is_active
+    await db_session.commit()
+    await db_session.refresh(user)
+    return user
+
+
+async def set_user_password_hash(
+    db_session: AsyncSession,
+    *,
+    user: User,
+    password_hash: str,
+) -> User:
+    user.password_hash = password_hash
+    await db_session.commit()
+    await db_session.refresh(user)
+    return user
+

app/settings.py

@@ -0,0 +1,84 @@
+from dataclasses import dataclass
+import os
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    value = os.getenv(name)
+    if value is None:
+        return default
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _env_int(name: str, default: int) -> int:
+    value = os.getenv(name)
+    if value is None:
+        return default
+    return int(value)
+
+
+def _env_float(name: str, default: float) -> float:
+    value = os.getenv(name)
+    if value is None:
+        return default
+    return float(value)
+
+
+def _env_str(name: str, default: str) -> str:
+    value = os.getenv(name)
+    if value is None:
+        return default
+    return value.strip() or default
+
+
+@dataclass(frozen=True)
+class Settings:
+    app_name: str = os.getenv("APP_NAME", "scholarr")
+    database_url: str = os.getenv(
+        "DATABASE_URL",
+        "postgresql+asyncpg://scholar:scholar@db:5432/scholar",
+    )
+    session_secret_key: str = os.getenv("SESSION_SECRET_KEY", "dev-insecure-session-key")
+    session_cookie_secure: bool = _env_bool("SESSION_COOKIE_SECURE", False)
+    login_rate_limit_attempts: int = _env_int("LOGIN_RATE_LIMIT_ATTEMPTS", 5)
+    login_rate_limit_window_seconds: int = _env_int(
+        "LOGIN_RATE_LIMIT_WINDOW_SECONDS",
+        60,
+    )
+    log_level: str = _env_str("LOG_LEVEL", "INFO")
+    log_format: str = _env_str("LOG_FORMAT", "console")
+    log_requests: bool = _env_bool("LOG_REQUESTS", True)
+    log_uvicorn_access: bool = _env_bool("LOG_UVICORN_ACCESS", False)
+    log_request_skip_paths: str = _env_str("LOG_REQUEST_SKIP_PATHS", "/healthz,/static/")
+    log_redact_fields: str = os.getenv("LOG_REDACT_FIELDS", "")
+    scheduler_enabled: bool = _env_bool("SCHEDULER_ENABLED", True)
+    scheduler_tick_seconds: int = _env_int("SCHEDULER_TICK_SECONDS", 60)
+    ingestion_network_error_retries: int = _env_int("INGESTION_NETWORK_ERROR_RETRIES", 1)
+    ingestion_retry_backoff_seconds: float = _env_float(
+        "INGESTION_RETRY_BACKOFF_SECONDS",
+        1.0,
+    )
+    ingestion_max_pages_per_scholar: int = _env_int(
+        "INGESTION_MAX_PAGES_PER_SCHOLAR",
+        30,
+    )
+    ingestion_page_size: int = _env_int("INGESTION_PAGE_SIZE", 100)
+    ingestion_continuation_queue_enabled: bool = _env_bool(
+        "INGESTION_CONTINUATION_QUEUE_ENABLED",
+        True,
+    )
+    ingestion_continuation_base_delay_seconds: int = _env_int(
+        "INGESTION_CONTINUATION_BASE_DELAY_SECONDS",
+        120,
+    )
+    ingestion_continuation_max_delay_seconds: int = _env_int(
+        "INGESTION_CONTINUATION_MAX_DELAY_SECONDS",
+        3600,
+    )
+    ingestion_continuation_max_attempts: int = _env_int(
+        "INGESTION_CONTINUATION_MAX_ATTEMPTS",
+        6,
+    )
+    scheduler_queue_batch_size: int = _env_int("SCHEDULER_QUEUE_BATCH_SIZE", 10)
+
+
+settings = Settings()

app/static/app.css

@@ -0,0 +1,501 @@
+* {
+  box-sizing: border-box;
+}
+
+body {
+  margin: 0;
+  min-height: 100vh;
+  color: var(--md-sys-color-on-surface);
+  font-family: var(--font-body);
+  background:
+    radial-gradient(circle at 10% 5%, var(--md-sys-color-primary-container) 0%, transparent 30%),
+    radial-gradient(circle at 90% 0%, var(--md-sys-color-secondary-container) 0%, transparent 28%),
+    var(--md-sys-color-surface);
+}
+
+.loading-indicator {
+  position: fixed;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 3px;
+  z-index: 40;
+  transform: scaleX(0);
+  transform-origin: left;
+  background: linear-gradient(90deg, var(--md-sys-color-primary), var(--md-sys-color-tertiary));
+  opacity: 0;
+}
+
+body.is-page-loading .loading-indicator {
+  opacity: 1;
+  animation: loading-bar 1.2s ease-in-out infinite;
+}
+
+.app-backdrop {
+  position: fixed;
+  inset: 0;
+  pointer-events: none;
+  background:
+    linear-gradient(130deg, var(--md-sys-color-primary-tint), transparent 40%),
+    linear-gradient(230deg, var(--md-sys-color-surface-tint), transparent 45%);
+  opacity: 0.28;
+}
+
+.top-app-bar {
+  position: sticky;
+  top: 0;
+  z-index: 10;
+  display: flex;
+  align-items: center;
+  gap: 0.8rem;
+  flex-wrap: wrap;
+  padding: 0.8rem 1.1rem;
+  border-bottom: 1px solid var(--md-sys-color-outline-variant);
+  backdrop-filter: blur(9px);
+  background: color-mix(in srgb, var(--md-sys-color-surface) 88%, white 12%);
+}
+
+.brand-wrap {
+  display: flex;
+  align-items: center;
+  gap: 0.65rem;
+}
+
+.brand {
+  color: var(--md-sys-color-primary);
+  text-decoration: none;
+  font-family: var(--font-heading);
+  font-weight: 700;
+  font-size: 1.2rem;
+  letter-spacing: 0.02em;
+}
+
+.session-user {
+  margin: 0;
+  padding: 0.24rem 0.58rem;
+  border-radius: 999px;
+  border: 1px solid var(--md-sys-color-outline-variant);
+  background: var(--md-sys-color-surface-container-high);
+  color: var(--md-sys-color-on-surface-variant);
+  font-size: 0.78rem;
+}
+
+.site-nav {
+  display: flex;
+  align-items: center;
+  gap: 0.35rem;
+  flex-wrap: wrap;
+}
+
+.nav-link {
+  color: var(--md-sys-color-on-surface-variant);
+  text-decoration: none;
+  padding: 0.42rem 0.78rem;
+  border-radius: 999px;
+  border: 1px solid transparent;
+  font-size: 0.92rem;
+}
+
+.nav-link:hover {
+  background: var(--md-sys-color-surface-container);
+  border-color: var(--md-sys-color-outline-variant);
+}
+
+.nav-link.is-active {
+  color: var(--md-sys-color-on-secondary-container);
+  background: var(--md-sys-color-secondary-container);
+  border-color: transparent;
+}
+
+.header-actions {
+  margin-left: auto;
+  display: flex;
+  align-items: center;
+  gap: 0.7rem;
+}
+
+.theme-control-wrap {
+  display: flex;
+  align-items: center;
+  gap: 0.45rem;
+}
+
+.theme-control-wrap label {
+  color: var(--md-sys-color-on-surface-variant);
+  font-size: 0.82rem;
+}
+
+.theme-control {
+  border: 1px solid var(--md-sys-color-outline);
+  border-radius: 999px;
+  background: var(--md-sys-color-surface-container-low);
+  color: var(--md-sys-color-on-surface);
+  padding: 0.32rem 0.64rem;
+  font: inherit;
+}
+
+.logout-form {
+  margin: 0;
+}
+
+.page-shell {
+  position: relative;
+  display: grid;
+  gap: 0.9rem;
+  width: min(1100px, calc(100vw - 1.4rem));
+  margin: 0 auto;
+  padding: 1.1rem 0 1.6rem;
+}
+
+.flash {
+  margin: 0;
+  padding: 0.72rem 0.82rem;
+  border-radius: 14px;
+  border: 1px solid var(--md-sys-color-outline-variant);
+  background: var(--md-sys-color-surface-container);
+}
+
+.flash-notice {
+  color: var(--md-sys-color-on-tertiary-container);
+  border-color: var(--md-sys-color-tertiary-container);
+  background: color-mix(in srgb, var(--md-sys-color-tertiary-container) 30%, white 70%);
+}
+
+.flash-error {
+  color: var(--md-sys-color-on-error-container);
+  border-color: var(--md-sys-color-error-container);
+  background: color-mix(in srgb, var(--md-sys-color-error-container) 25%, white 75%);
+}
+
+.hero,
+.sandbox {
+  border-radius: 24px;
+  border: 1px solid var(--md-sys-color-outline-variant);
+  background: var(--md-sys-color-surface-container-low);
+  box-shadow: var(--elevation-card);
+}
+
+.hero {
+  padding: 1.3rem;
+}
+
+.sandbox {
+  padding: 1.05rem;
+}
+
+.eyebrow {
+  margin: 0;
+  font-size: 0.78rem;
+  color: var(--md-sys-color-primary);
+  text-transform: uppercase;
+  letter-spacing: 0.11em;
+  font-weight: 600;
+}
+
+h1 {
+  margin: 0.28rem 0 0.5rem;
+  color: var(--md-sys-color-on-surface);
+  font-family: var(--font-heading);
+  font-size: clamp(1.45rem, 3vw, 2.05rem);
+  line-height: 1.15;
+}
+
+h2 {
+  margin: 0;
+  color: var(--md-sys-color-on-surface);
+  font-family: var(--font-heading);
+  font-size: 1.08rem;
+}
+
+.lede {
+  margin: 0;
+  color: var(--md-sys-color-on-surface-variant);
+  max-width: 68ch;
+}
+
+.section-heading {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 0.55rem;
+  margin-bottom: 0.7rem;
+  flex-wrap: wrap;
+}
+
+.stat-strip {
+  margin-top: 0.95rem;
+  display: grid;
+  gap: 0.75rem;
+  grid-template-columns: repeat(auto-fit, minmax(170px, 1fr));
+}
+
+.stat-item {
+  margin: 0;
+  padding: 0.72rem 0.78rem;
+  border-radius: 16px;
+  border: 1px solid var(--md-sys-color-outline-variant);
+  background: var(--md-sys-color-surface-container);
+}
+
+.stat-label {
+  margin: 0;
+  color: var(--md-sys-color-on-surface-variant);
+  font-size: 0.8rem;
+}
+
+.stat-value {
+  margin: 0.2rem 0 0;
+  color: var(--md-sys-color-on-surface);
+  font-size: 1.36rem;
+  font-weight: 700;
+}
+
+form {
+  display: grid;
+  gap: 0.45rem;
+}
+
+.stack-form {
+  max-width: 560px;
+}
+
+label {
+  font-size: 0.9rem;
+  color: var(--md-sys-color-on-surface-variant);
+}
+
+input,
+select {
+  width: 100%;
+  border: 1px solid var(--md-sys-color-outline);
+  border-radius: 12px;
+  padding: 0.56rem 0.64rem;
+  font: inherit;
+  color: var(--md-sys-color-on-surface);
+  background: var(--md-sys-color-surface);
+}
+
+input:focus,
+select:focus {
+  outline: 2px solid color-mix(in srgb, var(--md-sys-color-primary) 35%, transparent);
+  border-color: var(--md-sys-color-primary);
+}
+
+.checkbox-row {
+  display: flex;
+  align-items: center;
+  gap: 0.54rem;
+}
+
+.checkbox-row input {
+  width: auto;
+}
+
+button,
+.button {
+  border: 0;
+  border-radius: 999px;
+  padding: 0.54rem 0.86rem;
+  font: inherit;
+  font-weight: 600;
+  color: var(--md-sys-color-on-primary);
+  background: var(--md-sys-color-primary);
+  cursor: pointer;
+}
+
+button:hover,
+.button:hover {
+  filter: brightness(0.97);
+}
+
+button:disabled,
+.button:disabled {
+  cursor: not-allowed;
+  opacity: 0.5;
+}
+
+.button-text {
+  color: var(--md-sys-color-primary);
+  background: transparent;
+  border: 1px solid var(--md-sys-color-outline-variant);
+}
+
+.danger-button {
+  color: var(--md-sys-color-on-error);
+  background: var(--md-sys-color-error);
+}
+
+.inline-actions {
+  display: flex;
+  align-items: center;
+  flex-wrap: wrap;
+  gap: 0.45rem;
+}
+
+.inline-actions form {
+  display: inline-flex;
+  gap: 0.42rem;
+  align-items: center;
+}
+
+.inline-reset-form input {
+  min-width: 160px;
+}
+
+form.is-loading button[type="submit"],
+form.is-loading input[type="submit"] {
+  opacity: 0.75;
+}
+
+table {
+  width: 100%;
+  border-collapse: collapse;
+  border-radius: 12px;
+  overflow: hidden;
+  background: var(--md-sys-color-surface);
+}
+
+th,
+td {
+  padding: 0.54rem;
+  text-align: left;
+  border-bottom: 1px solid var(--md-sys-color-outline-variant);
+  vertical-align: top;
+}
+
+thead th {
+  color: var(--md-sys-color-on-surface-variant);
+  font-size: 0.84rem;
+  font-weight: 600;
+  background: var(--md-sys-color-surface-container-high);
+}
+
+tbody tr:last-child td {
+  border-bottom: 0;
+}
+
+a {
+  color: var(--md-sys-color-primary);
+}
+
+.link-button {
+  display: inline-flex;
+  align-items: center;
+  padding: 0.45rem 0.78rem;
+  border: 1px solid var(--md-sys-color-outline-variant);
+  border-radius: 999px;
+  color: var(--md-sys-color-primary);
+  text-decoration: none;
+  background: var(--md-sys-color-surface-container-low);
+}
+
+.link-button.is-active {
+  color: var(--md-sys-color-on-secondary-container);
+  background: var(--md-sys-color-secondary-container);
+  border-color: transparent;
+}
+
+.badge {
+  display: inline-block;
+  border-radius: 999px;
+  padding: 0.16rem 0.48rem;
+  font-size: 0.76rem;
+  text-transform: lowercase;
+  border: 1px solid var(--md-sys-color-outline-variant);
+}
+
+.badge.ok {
+  color: var(--md-sys-color-on-tertiary-container);
+  background: var(--md-sys-color-tertiary-container);
+  border-color: transparent;
+}
+
+.badge.warn,
+.badge.danger {
+  color: var(--md-sys-color-on-error-container);
+  background: var(--md-sys-color-error-container);
+  border-color: transparent;
+}
+
+.helper-text {
+  margin: 0.24rem 0 0;
+  color: var(--md-sys-color-on-surface-variant);
+  font-size: 0.82rem;
+}
+
+.auth-card {
+  max-width: 520px;
+}
+
+.auth-form {
+  max-width: 360px;
+}
+
+.form-error {
+  margin: 0.5rem 0 0;
+  color: var(--md-sys-color-error);
+  font-weight: 600;
+}
+
+.form-note {
+  margin: 0.35rem 0 0;
+  color: var(--md-sys-color-on-surface-variant);
+}
+
+code {
+  font-family: var(--font-code);
+  background: var(--md-sys-color-surface-container-high);
+  border-radius: 6px;
+  padding: 0.1rem 0.25rem;
+}
+
+pre {
+  margin: 0.6rem 0 0;
+  border-radius: 12px;
+  border: 1px solid var(--md-sys-color-outline-variant);
+  background: var(--md-sys-color-surface-container-high);
+  color: var(--md-sys-color-on-surface);
+  padding: 0.75rem;
+  white-space: pre-wrap;
+  word-break: break-word;
+}
+
+@media (max-width: 880px) {
+  .top-app-bar {
+    padding: 0.7rem 0.8rem;
+  }
+
+  .header-actions {
+    width: 100%;
+    justify-content: space-between;
+    margin-left: 0;
+  }
+
+  .page-shell {
+    width: min(100vw, calc(100vw - 0.9rem));
+    padding-top: 0.8rem;
+  }
+
+  .hero,
+  .sandbox {
+    border-radius: 18px;
+    padding: 0.9rem;
+  }
+
+  .inline-actions form {
+    width: 100%;
+    justify-content: flex-start;
+  }
+}
+
+@keyframes loading-bar {
+  0% {
+    transform: scaleX(0.1);
+  }
+  50% {
+    transform: scaleX(0.7);
+  }
+  100% {
+    transform: scaleX(1);
+  }
+}

app/static/app.js

@@ -0,0 +1,76 @@
+(() => {
+  const body = document.body;
+  if (!body) {
+    return;
+  }
+
+  const defaultLoadingText = "Working...";
+
+  const shouldHandleAnchor = (anchor) => {
+    if (!anchor || !anchor.getAttribute) {
+      return false;
+    }
+    const href = anchor.getAttribute("href");
+    if (!href) {
+      return false;
+    }
+    if (href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("javascript:")) {
+      return false;
+    }
+    if (anchor.hasAttribute("download")) {
+      return false;
+    }
+    if ((anchor.getAttribute("target") || "").toLowerCase() === "_blank") {
+      return false;
+    }
+    return true;
+  };
+
+  document.addEventListener("click", (event) => {
+    const target = event.target;
+    if (!(target instanceof Element)) {
+      return;
+    }
+    const anchor = target.closest("a");
+    if (!shouldHandleAnchor(anchor)) {
+      return;
+    }
+    body.classList.add("is-page-loading");
+  });
+
+  document.addEventListener("submit", (event) => {
+    const form = event.target;
+    if (!(form instanceof HTMLFormElement)) {
+      return;
+    }
+    if (form.dataset.noLoading === "1") {
+      return;
+    }
+
+    body.classList.add("is-page-loading");
+    form.classList.add("is-loading");
+
+    const submitElements = form.querySelectorAll("button[type='submit'], input[type='submit']");
+    submitElements.forEach((element) => {
+      if (element.hasAttribute("disabled")) {
+        return;
+      }
+      element.setAttribute("disabled", "disabled");
+      if (element instanceof HTMLInputElement) {
+        const loadingText = element.dataset.loadingText || defaultLoadingText;
+        element.dataset.originalValue = element.value;
+        element.value = loadingText;
+        return;
+      }
+      if (element instanceof HTMLButtonElement) {
+        const loadingText = element.dataset.loadingText || defaultLoadingText;
+        element.dataset.originalText = element.textContent || "";
+        element.textContent = loadingText;
+      }
+    });
+  });
+
+  window.addEventListener("pageshow", () => {
+    body.classList.remove("is-page-loading");
+  });
+})();

app/static/theme.css

@@ -0,0 +1,98 @@
+:root {
+  --font-body: "Plus Jakarta Sans", "Segoe UI", sans-serif;
+  --font-heading: "DM Serif Text", "Georgia", serif;
+  --font-code: "JetBrains Mono", "Fira Code", monospace;
+
+  --md-sys-color-primary: #73594d;
+  --md-sys-color-on-primary: #ffffff;
+  --md-sys-color-primary-container: #fbded0;
+  --md-sys-color-on-primary-container: #2b160f;
+  --md-sys-color-primary-tint: rgba(115, 89, 77, 0.18);
+
+  --md-sys-color-secondary: #6f5b52;
+  --md-sys-color-on-secondary: #ffffff;
+  --md-sys-color-secondary-container: #f8ddd2;
+  --md-sys-color-on-secondary-container: #281913;
+
+  --md-sys-color-tertiary: #5b6550;
+  --md-sys-color-on-tertiary: #ffffff;
+  --md-sys-color-tertiary-container: #ddebcf;
+  --md-sys-color-on-tertiary-container: #17200f;
+
+  --md-sys-color-error: #b3261e;
+  --md-sys-color-on-error: #ffffff;
+  --md-sys-color-error-container: #f9dedc;
+  --md-sys-color-on-error-container: #410e0b;
+
+  --md-sys-color-surface: #fff8f5;
+  --md-sys-color-surface-tint: rgba(111, 87, 74, 0.12);
+  --md-sys-color-surface-container-low: #fffaf8;
+  --md-sys-color-surface-container: #fdf2ec;
+  --md-sys-color-surface-container-high: #f8e9e1;
+  --md-sys-color-on-surface: #221a17;
+  --md-sys-color-on-surface-variant: #55423a;
+  --md-sys-color-outline: #85736b;
+  --md-sys-color-outline-variant: #d8c2b8;
+
+  --elevation-card: 0 10px 30px rgba(55, 39, 28, 0.08);
+}
+
+:root[data-theme="fjord"] {
+  --md-sys-color-primary: #2f6678;
+  --md-sys-color-on-primary: #ffffff;
+  --md-sys-color-primary-container: #cde9f4;
+  --md-sys-color-on-primary-container: #051f28;
+  --md-sys-color-primary-tint: rgba(47, 102, 120, 0.2);
+
+  --md-sys-color-secondary: #4d626b;
+  --md-sys-color-on-secondary: #ffffff;
+  --md-sys-color-secondary-container: #d0e6f2;
+  --md-sys-color-on-secondary-container: #091f27;
+
+  --md-sys-color-tertiary: #456179;
+  --md-sys-color-on-tertiary: #ffffff;
+  --md-sys-color-tertiary-container: #d2e5ff;
+  --md-sys-color-on-tertiary-container: #031d33;
+
+  --md-sys-color-surface: #f5fbff;
+  --md-sys-color-surface-tint: rgba(47, 102, 120, 0.12);
+  --md-sys-color-surface-container-low: #fcfeff;
+  --md-sys-color-surface-container: #ecf5fa;
+  --md-sys-color-surface-container-high: #dfeef6;
+  --md-sys-color-on-surface: #172126;
+  --md-sys-color-on-surface-variant: #3f4f57;
+  --md-sys-color-outline: #6f7f88;
+  --md-sys-color-outline-variant: #becfd8;
+
+  --elevation-card: 0 10px 30px rgba(22, 43, 53, 0.09);
+}
+
+:root[data-theme="spruce"] {
+  --md-sys-color-primary: #39684c;
+  --md-sys-color-on-primary: #ffffff;
+  --md-sys-color-primary-container: #bcefc9;
+  --md-sys-color-on-primary-container: #062111;
+  --md-sys-color-primary-tint: rgba(57, 104, 76, 0.18);
+
+  --md-sys-color-secondary: #4f6354;
+  --md-sys-color-on-secondary: #ffffff;
+  --md-sys-color-secondary-container: #d2e8d4;
+  --md-sys-color-on-secondary-container: #0d1f12;
+
+  --md-sys-color-tertiary: #3f655f;
+  --md-sys-color-on-tertiary: #ffffff;
+  --md-sys-color-tertiary-container: #c2ece4;
+  --md-sys-color-on-tertiary-container: #021f1a;
+
+  --md-sys-color-surface: #f5fbf5;
+  --md-sys-color-surface-tint: rgba(57, 104, 76, 0.1);
+  --md-sys-color-surface-container-low: #fcfffb;
+  --md-sys-color-surface-container: #ebf4ea;
+  --md-sys-color-surface-container-high: #deecdd;
+  --md-sys-color-on-surface: #172119;
+  --md-sys-color-on-surface-variant: #415246;
+  --md-sys-color-outline: #708174;
+  --md-sys-color-outline-variant: #c1d3c2;
+
+  --elevation-card: 0 10px 30px rgba(24, 48, 32, 0.08);
+}

app/static/theme.js

@@ -0,0 +1,56 @@
+(() => {
+  const STORAGE_KEY = "scholarr_theme";
+  const LEGACY_STORAGE_KEY = "scholar_tracker_theme";
+  const root = document.documentElement;
+  const themeControl = document.querySelector("[data-theme-control]");
+
+  if (!root) {
+    return;
+  }
+
+  const defaultTheme = root.dataset.defaultTheme || "terracotta";
+  const supportedThemes = themeControl
+    ? Array.from(themeControl.options).map((option) => option.value)
+    : [];
+
+  const isSupported = (theme) =>
+    supportedThemes.length === 0 || supportedThemes.includes(theme);
+
+  const applyTheme = (theme) => {
+    if (!isSupported(theme)) {
+      return;
+    }
+    root.dataset.theme = theme;
+  };
+
+  let activeTheme = defaultTheme;
+  try {
+    const savedTheme = window.localStorage.getItem(STORAGE_KEY);
+    const legacyTheme = window.localStorage.getItem(LEGACY_STORAGE_KEY);
+    if (savedTheme && isSupported(savedTheme)) {
+      activeTheme = savedTheme;
+    } else if (legacyTheme && isSupported(legacyTheme)) {
+      activeTheme = legacyTheme;
+      window.localStorage.setItem(STORAGE_KEY, legacyTheme);
+    }
+  } catch {
+    activeTheme = defaultTheme;
+  }
+
+  applyTheme(activeTheme);
+
+  if (!themeControl) {
+    return;
+  }
+
+  themeControl.value = activeTheme;
+  themeControl.addEventListener("change", (event) => {
+    const selectedTheme = event.target.value;
+    applyTheme(selectedTheme);
+    try {
+      window.localStorage.setItem(STORAGE_KEY, selectedTheme);
+    } catch {
+      // Ignore storage errors in locked-down browsers.
+    }
+  });
+})();

app/templates/account_password.html

@@ -0,0 +1,23 @@
+{% extends "base.html" %}
+
+{% block content %}
+<section class="hero" data-test="account-password-page">
+  <p class="eyebrow">Account</p>
+  <h1>Change Password</h1>
+  <p class="lede">Use a strong password. This updates only your own account.</p>
+</section>
+
+<section class="sandbox" data-test="account-password-form-card">
+  <h2>Password Update</h2>
+  <form method="post" action="/account/password" class="stack-form">
+    <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+    <label for="current_password">Current Password</label>
+    <input id="current_password" name="current_password" type="password" autocomplete="current-password" required>
+    <label for="new_password">New Password</label>
+    <input id="new_password" name="new_password" type="password" autocomplete="new-password" minlength="8" required>
+    <label for="confirm_password">Confirm New Password</label>
+    <input id="confirm_password" name="confirm_password" type="password" autocomplete="new-password" minlength="8" required>
+    <button type="submit" data-loading-text="Updating...">Update Password</button>
+  </form>
+</section>
+{% endblock %}

app/templates/base.html

@@ -0,0 +1,66 @@
+<!doctype html>
+<html lang="en" data-theme="{{ theme_name | default('terracotta') }}" data-default-theme="{{ theme_name | default('terracotta') }}">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>{{ page_title }} | scholarr</title>
+    <link rel="stylesheet" href="{{ url_for('static', path='theme.css') }}">
+    <link rel="stylesheet" href="{{ url_for('static', path='app.css') }}">
+  </head>
+  <body>
+    <div class="loading-indicator" data-loading-indicator aria-hidden="true"></div>
+    <div class="app-backdrop" aria-hidden="true"></div>
+    <header class="top-app-bar">
+      <div class="brand-wrap">
+        <a class="brand" href="/">scholarr</a>
+        {% if session_user %}
+        <p class="session-user" data-test="session-user">{{ session_user.email }}</p>
+        {% endif %}
+      </div>
+
+      <nav class="site-nav" aria-label="Primary">
+        {% if session_user %}
+        <a class="nav-link {% if active_nav == 'home' %}is-active{% endif %}" href="/">Dashboard</a>
+        <a class="nav-link {% if active_nav == 'scholars' %}is-active{% endif %}" href="/scholars">Scholars</a>
+        <a class="nav-link {% if active_nav == 'publications' %}is-active{% endif %}" href="/publications?mode=new">Publications</a>
+        <a class="nav-link {% if active_nav == 'runs' %}is-active{% endif %}" href="/runs">Runs</a>
+        <a class="nav-link {% if active_nav == 'settings' %}is-active{% endif %}" href="/settings">Settings</a>
+        <a class="nav-link {% if active_nav == 'account_password' %}is-active{% endif %}" href="/account/password">Password</a>
+        {% if session_user.is_admin %}
+        <a class="nav-link {% if active_nav == 'users' %}is-active{% endif %}" href="/users">Users</a>
+        {% endif %}
+        {% else %}
+        <a class="nav-link {% if active_nav == 'login' %}is-active{% endif %}" href="/login">Login</a>
+        {% endif %}
+      </nav>
+
+      <div class="header-actions">
+        <div class="theme-control-wrap">
+          <label for="theme-control">Theme</label>
+          <select id="theme-control" class="theme-control" data-theme-control aria-label="Color theme">
+            {% for theme in themes %}
+            <option value="{{ theme.slug }}" {% if theme.slug == theme_name %}selected{% endif %}>{{ theme.label }}</option>
+            {% endfor %}
+          </select>
+        </div>
+        {% if session_user %}
+        <form method="post" action="/logout" class="logout-form">
+          <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+          <button type="submit" class="button button-text" data-loading-text="Logging out...">Log out</button>
+        </form>
+        {% endif %}
+      </div>
+    </header>
+    <main class="page-shell">
+      {% if notice %}
+      <p class="flash flash-notice" data-test="flash-notice">{{ notice }}</p>
+      {% endif %}
+      {% if page_error %}
+      <p class="flash flash-error" data-test="flash-error">{{ page_error }}</p>
+      {% endif %}
+      {% block content %}{% endblock %}
+    </main>
+    <script defer src="{{ url_for('static', path='app.js') }}"></script>
+    <script defer src="{{ url_for('static', path='theme.js') }}"></script>
+  </body>
+</html>

app/templates/dashboard/_run_controls.html

@@ -0,0 +1,23 @@
+<section class="sandbox" data-test="dashboard-run-controls">
+  <div class="section-heading">
+    <h2>Run Tracking</h2>
+    <div class="inline-actions">
+      <a class="link-button" href="/publications?mode=new">New Publications</a>
+      <a class="link-button" href="/publications?mode=all">All Publications</a>
+      <a class="link-button" href="/runs?failed_only=1">Run Diagnostics</a>
+      <a class="link-button" href="/settings">Automation Settings</a>
+      <a class="link-button" href="/scholars">Manage Scholars</a>
+    </div>
+  </div>
+  <p class="lede">Manual runs use your request delay of {{ dashboard.run_controls.request_delay_seconds }} second(s).</p>
+  <p class="helper-text">
+    Queue state:
+    {{ dashboard.run_controls.queue_queued_count }} queued,
+    {{ dashboard.run_controls.queue_retrying_count }} retrying,
+    {{ dashboard.run_controls.queue_dropped_count }} dropped.
+  </p>
+  <form method="post" action="{{ dashboard.run_controls.run_manual_action }}" class="inline-actions">
+    <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+    <button type="submit" data-loading-text="Running...">Run Now</button>
+  </form>
+</section>

app/templates/dashboard/_run_history.html

@@ -0,0 +1,38 @@
+<section class="sandbox" data-test="dashboard-run-history">
+  <div class="section-heading">
+    <h2>Run History</h2>
+    <div class="inline-actions">
+      <a class="link-button" href="/runs">View All Runs</a>
+    </div>
+  </div>
+  {% if dashboard.run_history %}
+  <table>
+    <thead>
+      <tr>
+        <th>Started</th>
+        <th>Status</th>
+        <th>Scholars</th>
+        <th>New Publications</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for run in dashboard.run_history %}
+      <tr>
+        <td>
+          {% if run.detail_url %}
+          <a href="{{ run.detail_url }}">{{ run.started_at_display }}</a>
+          {% else %}
+          {{ run.started_at_display }}
+          {% endif %}
+        </td>
+        <td><span class="badge {{ run.status_badge }}">{{ run.status_label }}</span></td>
+        <td>{{ run.scholar_count }}</td>
+        <td>{{ run.new_publication_count }}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  {% else %}
+  <p>No runs yet.</p>
+  {% endif %}
+</section>

app/templates/dashboard/_unread_publications.html

@@ -0,0 +1,47 @@
+<section class="sandbox" data-test="dashboard-unread-publications">
+  <div class="section-heading">
+    <h2>New Since Last Run</h2>
+    <div class="inline-actions">
+      <a class="link-button" href="/publications?mode=all">View All</a>
+      <form method="post" action="{{ dashboard.run_controls.mark_all_read_action }}">
+        <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+        <input type="hidden" name="return_to" value="/">
+        <button type="submit" data-loading-text="Marking...">Mark All Read</button>
+      </form>
+    </div>
+  </div>
+
+  {% if dashboard.unread_publications %}
+  <table>
+    <thead>
+      <tr>
+        <th>Title</th>
+        <th>Scholar</th>
+        <th>Year</th>
+        <th>Citations</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for item in dashboard.unread_publications %}
+      <tr>
+        <td>
+          {% if item.pub_url %}
+          <a href="{{ item.pub_url }}" target="_blank" rel="noreferrer">{{ item.title }}</a>
+          {% else %}
+          {{ item.title }}
+          {% endif %}
+          {% if item.venue_text %}
+          <p class="helper-text">{{ item.venue_text }}</p>
+          {% endif %}
+        </td>
+        <td>{{ item.scholar_label }}</td>
+        <td>{{ item.year_display }}</td>
+        <td>{{ item.citation_count }}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  {% else %}
+  <p>No new publications in the latest run.</p>
+  {% endif %}
+</section>

app/templates/index.html

@@ -0,0 +1,27 @@
+{% extends "base.html" %}
+
+{% block content %}
+<section class="hero" data-test="home-hero">
+  <p class="eyebrow">Dashboard</p>
+  <h1>Keep up with your tracked scholars</h1>
+  <p class="lede">Run ingestion, review fresh publications, and inspect recent outcomes from one screen.</p>
+  <div class="stat-strip">
+    <article class="stat-item">
+      <p class="stat-label">New Since Last Run</p>
+      <p class="stat-value">{{ dashboard.unread_publications | length }}</p>
+    </article>
+    <article class="stat-item">
+      <p class="stat-label">Recent Runs</p>
+      <p class="stat-value">{{ dashboard.run_history | length }}</p>
+    </article>
+    <article class="stat-item">
+      <p class="stat-label">Automation Delay</p>
+      <p class="stat-value">{{ dashboard.run_controls.request_delay_seconds }}s</p>
+    </article>
+  </div>
+</section>
+
+{% for section_template in dashboard.section_templates %}
+{% include section_template %}
+{% endfor %}
+{% endblock %}

app/templates/login.html

@@ -0,0 +1,23 @@
+{% extends "base.html" %}
+
+{% block content %}
+<section class="hero auth-card" data-test="login-card">
+  <p class="eyebrow">Secure Access</p>
+  <h1>Sign in</h1>
+  <p class="lede">Use an admin-created account to access your tracked scholars and settings.</p>
+  {% if error_message %}
+  <p class="form-error" role="alert">{{ error_message }}</p>
+  {% endif %}
+  {% if retry_after_seconds %}
+  <p class="form-note">Retry after {{ retry_after_seconds }} seconds.</p>
+  {% endif %}
+  <form method="post" action="/login" class="auth-form" data-test="login-form">
+    <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+    <label for="email">Email</label>
+    <input id="email" name="email" type="email" autocomplete="username" required autofocus>
+    <label for="password">Password</label>
+    <input id="password" name="password" type="password" autocomplete="current-password" required>
+    <button type="submit" data-loading-text="Signing in...">Sign in</button>
+  </form>
+</section>
+{% endblock %}

app/templates/publications.html

@@ -0,0 +1,104 @@
+{% extends "base.html" %}
+
+{% block content %}
+<section class="hero" data-test="publications-page">
+  <p class="eyebrow">Publications</p>
+  <h1>Publications</h1>
+  <p class="lede">
+    Browse publications discovered in the latest run or your complete tracked library.
+    {% if selected_scholar %}
+    Current scholar filter: <strong>{{ selected_scholar.display_name or selected_scholar.scholar_id }}</strong>.
+    {% endif %}
+  </p>
+  <div class="inline-actions">
+    <a class="link-button {% if mode == 'new' %}is-active{% endif %}" href="{{ mode_new_url }}">New Since Last Run ({{ new_count }})</a>
+    <a class="link-button {% if mode == 'all' %}is-active{% endif %}" href="{{ mode_all_url }}">All ({{ total_count }})</a>
+    <a class="link-button" href="/scholars">Manage Scholars</a>
+    <a class="link-button" href="/runs?failed_only=1">Run Diagnostics</a>
+    {% if mode == 'new' and new_count > 0 %}
+    <form method="post" action="/publications/mark-all-read">
+      <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+      <input type="hidden" name="return_to" value="{{ current_publications_url }}">
+      <button type="submit" data-loading-text="Marking...">Mark All Read</button>
+    </form>
+    {% endif %}
+  </div>
+</section>
+
+<section class="sandbox" data-test="publications-table">
+  <div class="section-heading">
+    <h2>Filter</h2>
+  </div>
+  <form method="get" action="/publications" class="inline-actions">
+    <label for="mode">Mode</label>
+    <select id="mode" name="mode">
+      <option value="new" {% if mode == 'new' %}selected{% endif %}>New Since Last Run</option>
+      <option value="all" {% if mode == 'all' %}selected{% endif %}>All Publications</option>
+    </select>
+
+    <label for="scholar_profile_id">Scholar</label>
+    <select id="scholar_profile_id" name="scholar_profile_id">
+      <option value="">All Scholars</option>
+      {% for scholar in scholars %}
+      <option value="{{ scholar.id }}" {% if selected_scholar_id == scholar.id %}selected{% endif %}>
+        {{ scholar.display_name or scholar.scholar_id }}
+      </option>
+      {% endfor %}
+    </select>
+    <button type="submit" data-loading-text="Filtering...">Apply</button>
+  </form>
+
+  <div class="section-heading">
+    <h2>Results</h2>
+  </div>
+  {% if publications %}
+  <table>
+    <thead>
+      <tr>
+        <th>Title</th>
+        <th>Scholar</th>
+        <th>Year</th>
+        <th>Citations</th>
+        <th>New This Run</th>
+        <th>Status</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for item in publications %}
+      <tr>
+        <td>
+          {% if item.pub_url %}
+          <a href="{{ item.pub_url }}" target="_blank" rel="noreferrer">{{ item.title }}</a>
+          {% else %}
+          {{ item.title }}
+          {% endif %}
+          {% if item.venue_text %}
+          <p class="helper-text">{{ item.venue_text }}</p>
+          {% endif %}
+        </td>
+        <td>{{ item.scholar_label }}</td>
+        <td>{{ item.year if item.year is not none else "-" }}</td>
+        <td>{{ item.citation_count }}</td>
+        <td>
+          {% if item.is_new_in_latest_run %}
+          <span class="badge warn">new</span>
+          {% else %}
+          <span class="badge">older</span>
+          {% endif %}
+        </td>
+        <td>
+          {% if item.is_read %}
+          <span class="badge ok">read</span>
+          {% else %}
+          <span class="badge warn">new</span>
+          {% endif %}
+        </td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  {% else %}
+  <p>No publications match this mode yet.</p>
+  {% endif %}
+</section>
+{% endblock %}

app/templates/run_detail.html

@@ -0,0 +1,98 @@
+{% extends "base.html" %}
+
+{% block content %}
+<section class="hero" data-test="run-detail-page">
+  <p class="eyebrow">Diagnostics</p>
+  <h1>Run #{{ run.id }}</h1>
+  <p class="lede">Detailed state and failure context for this execution.</p>
+</section>
+
+<section class="sandbox" data-test="run-detail-summary">
+  <h2>Summary</h2>
+  <table>
+    <tbody>
+      <tr><th>Started</th><td>{{ run.started_at }}</td></tr>
+      <tr><th>Finished</th><td>{{ run.finished_at }}</td></tr>
+      <tr><th>Status</th><td><span class="badge {{ run.status_badge }}">{{ run.status }}</span></td></tr>
+      <tr><th>Trigger</th><td>{{ run.trigger_type }}</td></tr>
+      <tr><th>Scholars</th><td>{{ run.scholar_count }}</td></tr>
+      <tr><th>New Publications</th><td>{{ run.new_publication_count }}</td></tr>
+      <tr><th>Succeeded</th><td>{{ run_summary.succeeded_count or 0 }}</td></tr>
+      <tr><th>Failed</th><td>{{ run_summary.failed_count or 0 }}</td></tr>
+      <tr><th>Partial</th><td>{{ run_summary.partial_count or 0 }}</td></tr>
+    </tbody>
+  </table>
+  {% if run_summary.failed_state_counts or run_summary.failed_reason_counts %}
+  <details>
+    <summary>Failure Breakdown</summary>
+    <pre>{{ {"failed_state_counts": run_summary.failed_state_counts, "failed_reason_counts": run_summary.failed_reason_counts} | tojson(indent=2) }}</pre>
+  </details>
+  {% endif %}
+</section>
+
+<section class="sandbox" data-test="run-detail-results">
+  <h2>Scholar Results</h2>
+  {% if scholar_results %}
+  <table>
+    <thead>
+      <tr>
+        <th>Scholar</th>
+        <th>Outcome</th>
+        <th>State</th>
+        <th>Reason</th>
+        <th>Publications</th>
+        <th>Pages</th>
+        <th>Attempts</th>
+        <th>Continuation</th>
+        <th>Warnings</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for item in scholar_results %}
+      <tr>
+        <td><code>{{ item.scholar_id }}</code></td>
+        <td>{{ item.outcome or "-" }}</td>
+        <td>{{ item.state }}</td>
+        <td>{{ item.state_reason or "-" }}</td>
+        <td>{{ item.publication_count }}</td>
+        <td>{{ item.pages_fetched or 0 }} / {{ item.pages_attempted or 0 }}</td>
+        <td>{{ item.attempt_count or 1 }}</td>
+        <td>
+          {% if item.continuation_enqueued %}
+          <span class="badge warn">queued</span>
+          <p class="helper-text">
+            reason: {{ item.continuation_reason or "-" }},
+            cstart: {{ item.continuation_cstart or "-" }}
+          </p>
+          {% elif item.continuation_cleared %}
+          <span class="badge ok">cleared</span>
+          {% else %}
+          -
+          {% endif %}
+        </td>
+        <td>
+          {% if item.warnings %}
+          {{ item.warnings | join(", ") }}
+          {% else %}
+          -
+          {% endif %}
+        </td>
+      </tr>
+      {% if item.debug %}
+      <tr>
+        <td colspan="9">
+          <details>
+            <summary>Debug Context</summary>
+            <pre>{{ item.debug | tojson(indent=2) }}</pre>
+          </details>
+        </td>
+      </tr>
+      {% endif %}
+      {% endfor %}
+    </tbody>
+  </table>
+  {% else %}
+  <p>No scholar result details were captured.</p>
+  {% endif %}
+</section>
+{% endblock %}

app/templates/runs.html

@@ -0,0 +1,129 @@
+{% extends "base.html" %}
+
+{% block content %}
+<section class="hero" data-test="runs-page">
+  <p class="eyebrow">Runs</p>
+  <h1>Run History</h1>
+  <p class="lede">Review execution outcomes and drill into run diagnostics when needed.</p>
+</section>
+
+<section class="sandbox" data-test="runs-filter-card">
+  <h2>Filters</h2>
+  <form method="get" action="/runs" class="inline-actions">
+    <label class="checkbox-row" for="failed_only">
+      <input id="failed_only" type="checkbox" name="failed_only" value="1" {% if failed_only %}checked{% endif %}>
+      <span>Failed / partial only</span>
+    </label>
+    <button type="submit" data-loading-text="Filtering...">Apply</button>
+    {% if failed_only %}
+    <a class="link-button" href="/runs">Clear</a>
+    {% endif %}
+  </form>
+</section>
+
+<section class="sandbox" data-test="runs-table-card">
+  <h2>Recent Runs</h2>
+  {% if runs %}
+  <table>
+    <thead>
+      <tr>
+        <th>ID</th>
+        <th>Started</th>
+        <th>Status</th>
+        <th>Trigger</th>
+        <th>Scholars</th>
+        <th>New Publications</th>
+        <th>Failures</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for run in runs %}
+      <tr>
+        <td><a href="/runs/{{ run.id }}">#{{ run.id }}</a></td>
+        <td>{{ run.started_at }}</td>
+        <td><span class="badge {{ run.status_badge }}">{{ run.status }}</span></td>
+        <td>{{ run.trigger_type }}</td>
+        <td>{{ run.scholar_count }}</td>
+        <td>{{ run.new_publication_count }}</td>
+        <td>{{ run.failed_count }} failed / {{ run.partial_count }} partial</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  {% else %}
+  <p>No runs match the current filter.</p>
+  {% endif %}
+</section>
+
+<section class="sandbox" data-test="runs-queue-card">
+  <div class="section-heading">
+    <h2>Continuation Queue</h2>
+    <a class="link-button" href="/runs?failed_only=1">Focus Failed Runs</a>
+  </div>
+  {% if queue_items %}
+  <table>
+    <thead>
+      <tr>
+        <th>Scholar</th>
+        <th>Status</th>
+        <th>Reason</th>
+        <th>Attempts</th>
+        <th>Next Attempt</th>
+        <th>Last Error</th>
+        <th>Actions</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for item in queue_items %}
+      <tr>
+        <td>
+          <a href="/publications?mode=new&scholar_profile_id={{ item.scholar_profile_id }}">{{ item.scholar_label }}</a>
+          <p class="helper-text">resume cstart: {{ item.resume_cstart }}</p>
+        </td>
+        <td><span class="badge {{ item.status_badge }}">{{ item.status }}</span></td>
+        <td>
+          {% if item.status == "dropped" and item.dropped_reason %}
+          <code>{{ item.dropped_reason }}</code>
+          {% else %}
+          <code>{{ item.reason }}</code>
+          {% endif %}
+        </td>
+        <td>{{ item.attempt_count }}</td>
+        <td>{{ item.next_attempt_at }}</td>
+        <td>
+          {% if item.last_error %}
+          <details>
+            <summary>show</summary>
+            <pre>{{ item.last_error }}</pre>
+          </details>
+          {% else %}
+          -
+          {% endif %}
+        </td>
+        <td>
+          <div class="inline-actions">
+            <form method="post" action="/runs/queue/{{ item.id }}/retry">
+              <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+              <button type="submit" data-loading-text="Retrying...">Retry Now</button>
+            </form>
+            {% if item.status != "dropped" %}
+            <form method="post" action="/runs/queue/{{ item.id }}/drop">
+              <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+              <button type="submit" class="danger-button" data-loading-text="Dropping...">Drop</button>
+            </form>
+            {% endif %}
+            <form method="post" action="/runs/queue/{{ item.id }}/clear">
+              <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+              <button type="submit" class="button-text" data-loading-text="Clearing...">Clear</button>
+            </form>
+          </div>
+        </td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  {% else %}
+  <p>No queued continuation items.</p>
+  {% endif %}
+</section>
+{% endblock %}

app/templates/scholars.html

@@ -0,0 +1,99 @@
+{% extends "base.html" %}
+
+{% block content %}
+<section class="hero" data-test="scholars-page">
+  <p class="eyebrow">Scholars</p>
+  <h1>Your Scholars</h1>
+  <p class="lede">Add, disable, or remove profiles scoped to your account.</p>
+</section>
+
+<section class="sandbox" data-test="scholars-create-card">
+  <h2>Add Scholar</h2>
+  <form method="post" action="/scholars" class="stack-form">
+    <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+    <label for="scholar_id">Scholar ID</label>
+    <input
+      id="scholar_id"
+      name="scholar_id"
+      type="text"
+      value="{{ form_scholar_id | default('') }}"
+      placeholder="abcDEF123456"
+      pattern="[a-zA-Z0-9_-]{12}"
+      required
+    >
+    <label for="display_name">Display Name (Optional)</label>
+    <input
+      id="display_name"
+      name="display_name"
+      type="text"
+      value="{{ form_display_name | default('') }}"
+      placeholder="Ada Lovelace"
+    >
+    <button type="submit" data-loading-text="Adding...">Add Scholar</button>
+  </form>
+</section>
+
+<section class="sandbox" data-test="scholars-list-card">
+  <h2>Tracked Scholars</h2>
+  {% if scholars %}
+  <table>
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Scholar ID</th>
+        <th>Status</th>
+        <th>Last Run</th>
+        <th>Publications</th>
+        <th>Actions</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for scholar in scholars %}
+      <tr>
+        <td>{{ scholar.display_name }}</td>
+        <td><code>{{ scholar.scholar_id }}</code></td>
+        <td>
+          {% if scholar.is_enabled %}
+          <span class="badge ok">enabled</span>
+          {% else %}
+          <span class="badge warn">disabled</span>
+          {% endif %}
+          {% if scholar.baseline_completed %}
+          <p class="helper-text">baseline complete</p>
+          {% else %}
+          <p class="helper-text">awaiting first run</p>
+          {% endif %}
+        </td>
+        <td>
+          <span class="badge {% if scholar.last_run_status == 'success' %}ok{% elif scholar.last_run_status in ['failed', 'partial_failure'] %}danger{% else %}warn{% endif %}">
+            {{ scholar.last_run_status }}
+          </span>
+          <p class="helper-text">{{ scholar.last_run_dt }}</p>
+        </td>
+        <td>
+          <div class="inline-actions">
+            <a class="link-button" href="/publications?mode=new&scholar_profile_id={{ scholar.id }}">New</a>
+            <a class="link-button" href="/publications?mode=all&scholar_profile_id={{ scholar.id }}">All</a>
+          </div>
+        </td>
+        <td>
+          <div class="inline-actions">
+            <form method="post" action="/scholars/{{ scholar.id }}/toggle">
+              <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+              <button type="submit" data-loading-text="Saving...">{% if scholar.is_enabled %}Disable{% else %}Enable{% endif %}</button>
+            </form>
+            <form method="post" action="/scholars/{{ scholar.id }}/delete">
+              <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+              <button type="submit" class="danger-button" data-loading-text="Deleting...">Delete</button>
+            </form>
+          </div>
+        </td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  {% else %}
+  <p>No scholars tracked yet.</p>
+  {% endif %}
+</section>
+{% endblock %}

app/templates/settings.html

@@ -0,0 +1,44 @@
+{% extends "base.html" %}
+
+{% block content %}
+<section class="hero" data-test="settings-page">
+  <p class="eyebrow">Settings</p>
+  <h1>Your Settings</h1>
+  <p class="lede">Control automation cadence and request pacing for your own account.</p>
+</section>
+
+<section class="sandbox" data-test="settings-form-card">
+  <h2>Automation Settings</h2>
+  <form method="post" action="/settings" class="stack-form">
+    <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+    <label class="checkbox-row" for="auto_run_enabled">
+      <input id="auto_run_enabled" name="auto_run_enabled" type="checkbox" {% if user_settings.auto_run_enabled %}checked{% endif %}>
+      <span>Enable automatic runs</span>
+    </label>
+
+    <label for="run_interval_minutes">Run Interval (minutes)</label>
+    <input
+      id="run_interval_minutes"
+      name="run_interval_minutes"
+      type="number"
+      min="15"
+      step="1"
+      value="{{ user_settings.run_interval_minutes }}"
+      required
+    >
+
+    <label for="request_delay_seconds">Request Delay (seconds)</label>
+    <input
+      id="request_delay_seconds"
+      name="request_delay_seconds"
+      type="number"
+      min="1"
+      step="1"
+      value="{{ user_settings.request_delay_seconds }}"
+      required
+    >
+
+    <button type="submit" data-loading-text="Saving...">Save Settings</button>
+  </form>
+</section>
+{% endblock %}

app/templates/users.html

@@ -0,0 +1,75 @@
+{% extends "base.html" %}
+
+{% block content %}
+<section class="hero" data-test="users-page">
+  <p class="eyebrow">Admin</p>
+  <h1>User Management</h1>
+  <p class="lede">Admin-only controls for account creation, activation, and password resets.</p>
+</section>
+
+<section class="sandbox" data-test="users-create-card">
+  <h2>Create User</h2>
+  <form method="post" action="/users" class="stack-form">
+    <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+    <label for="email">Email</label>
+    <input id="email" name="email" type="email" value="{{ form_email | default('') }}" required>
+    <label for="password">Temporary Password</label>
+    <input id="password" name="password" type="password" required>
+    <label class="checkbox-row" for="is_admin">
+      <input id="is_admin" name="is_admin" type="checkbox" {% if form_is_admin %}checked{% endif %}>
+      <span>Admin account</span>
+    </label>
+    <button type="submit" data-loading-text="Creating...">Create User</button>
+  </form>
+</section>
+
+<section class="sandbox" data-test="users-list-card">
+  <h2>Users</h2>
+  <table>
+    <thead>
+      <tr>
+        <th>Email</th>
+        <th>Role</th>
+        <th>Status</th>
+        <th>Actions</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for user in users %}
+      <tr>
+        <td>{{ user.email }}</td>
+        <td>
+          {% if user.is_admin %}
+          <span class="badge ok">admin</span>
+          {% else %}
+          <span class="badge">user</span>
+          {% endif %}
+        </td>
+        <td>
+          {% if user.is_active %}
+          <span class="badge ok">active</span>
+          {% else %}
+          <span class="badge warn">inactive</span>
+          {% endif %}
+        </td>
+        <td>
+          <div class="inline-actions">
+            <form method="post" action="/users/{{ user.id }}/toggle-active">
+              <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+              <button type="submit" data-loading-text="Saving..." {% if user.id == current_user_id and user.is_active %}disabled{% endif %}>
+                {% if user.is_active %}Deactivate{% else %}Activate{% endif %}
+              </button>
+            </form>
+            <form method="post" action="/users/{{ user.id }}/reset-password" class="inline-reset-form">
+              <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
+              <input name="new_password" type="password" placeholder="New password" minlength="8" required>
+              <button type="submit" data-loading-text="Resetting...">Reset Password</button>
+            </form>
+          </div>
+        </td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</section>
+{% endblock %}

app/theme.py

@@ -0,0 +1,24 @@
+from dataclasses import dataclass
+from typing import Final
+
+
+@dataclass(frozen=True)
+class ThemeDefinition:
+    slug: str
+    label: str
+
+
+THEMES: Final[tuple[ThemeDefinition, ...]] = (
+    ThemeDefinition(slug="terracotta", label="Terracotta"),
+    ThemeDefinition(slug="fjord", label="Fjord"),
+    ThemeDefinition(slug="spruce", label="Spruce"),
+)
+_THEME_SLUGS: Final[frozenset[str]] = frozenset(theme.slug for theme in THEMES)
+DEFAULT_THEME: Final[str] = THEMES[0].slug
+
+
+def resolve_theme(candidate: str | None) -> str:
+    if candidate and candidate in _THEME_SLUGS:
+        return candidate
+    return DEFAULT_THEME
+

app/web/__init__.py

@@ -0,0 +1,2 @@
+"""Web-layer routers, middleware, and shared helpers."""
+

app/web/common.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import logging
+from urllib.parse import urlencode
+
+from fastapi import Request, status
+from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.templating import Jinja2Templates
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.auth.session import (
+    SessionUser,
+    clear_session_user,
+    get_session_user,
+    set_session_user,
+)
+from app.db.models import User
+from app.security.csrf import CSRF_SESSION_KEY, ensure_csrf_token
+from app.services import users as user_service
+from app.theme import THEMES
+
+logger = logging.getLogger(__name__)
+
+templates = Jinja2Templates(directory="app/templates")
+
+
+def to_session_user(user: User | None) -> SessionUser | None:
+    if user is None:
+        return None
+    return SessionUser(id=user.id, email=user.email, is_admin=user.is_admin)
+
+
+def build_template_context(
+    request: Request,
+    *,
+    page_title: str,
+    active_nav: str,
+    theme_name: str,
+    session_user: SessionUser | None,
+    notice: str | None = None,
+    page_error: str | None = None,
+) -> dict[str, object]:
+    return {
+        "active_nav": active_nav,
+        "page_title": page_title,
+        "theme_name": theme_name,
+        "themes": THEMES,
+        "session_user": session_user,
+        "csrf_token": ensure_csrf_token(request),
+        "notice": notice,
+        "page_error": page_error,
+    }
+
+
+def redirect_with_message(
+    path: str,
+    *,
+    notice: str | None = None,
+    error: str | None = None,
+) -> RedirectResponse:
+    params: dict[str, str] = {}
+    if notice:
+        params["notice"] = notice
+    if error:
+        params["error"] = error
+    if params:
+        path = f"{path}?{urlencode(params)}"
+    return RedirectResponse(path, status_code=status.HTTP_303_SEE_OTHER)
+
+
+def redirect_to_login() -> RedirectResponse:
+    return RedirectResponse("/login", status_code=status.HTTP_303_SEE_OTHER)
+
+
+def invalidate_session(request: Request) -> None:
+    clear_session_user(request)
+    request.session.pop(CSRF_SESSION_KEY, None)
+
+
+async def get_authenticated_user(
+    request: Request,
+    db_session: AsyncSession,
+) -> User | None:
+    session_user = get_session_user(request)
+    if session_user is None:
+        return None
+
+    user = await user_service.get_user_by_id(db_session, session_user.id)
+    if user is None or not user.is_active:
+        logger.info(
+            "auth.session_invalidated",
+            extra={
+                "event": "auth.session_invalidated",
+                "session_user_id": session_user.id,
+            },
+        )
+        invalidate_session(request)
+        return None
+
+    if user.email != session_user.email or user.is_admin != session_user.is_admin:
+        set_session_user(
+            request,
+            user_id=user.id,
+            email=user.email,
+            is_admin=user.is_admin,
+        )
+
+    return user
+
+
+def login_rate_limit_key(request: Request, email: str) -> str:
+    client_host = request.client.host if request.client is not None else "unknown"
+    normalized_email = email.strip().lower()
+    return f"{client_host}:{normalized_email or '<empty>'}"
+
+
+def render_login_page(
+    request: Request,
+    *,
+    theme_name: str,
+    error_message: str | None = None,
+    status_code: int = status.HTTP_200_OK,
+    retry_after_seconds: int | None = None,
+) -> HTMLResponse:
+    context = build_template_context(
+        request,
+        page_title="Login",
+        active_nav="login",
+        theme_name=theme_name,
+        session_user=None,
+    )
+    context["error_message"] = error_message
+    context["retry_after_seconds"] = retry_after_seconds
+    return templates.TemplateResponse(
+        request=request,
+        name="login.html",
+        context=context,
+        status_code=status_code,
+    )
+

app/web/deps.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from fastapi import Depends
+
+from app.services import ingestion as ingestion_service
+from app.services import scholar_source as scholar_source_service
+
+
+def get_scholar_source() -> scholar_source_service.ScholarSource:
+    return scholar_source_service.LiveScholarSource()
+
+
+def get_ingestion_service(
+    source: scholar_source_service.ScholarSource = Depends(get_scholar_source),
+) -> ingestion_service.ScholarIngestionService:
+    return ingestion_service.ScholarIngestionService(source=source)
+

app/web/middleware.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from secrets import token_urlsafe
+import time
+import logging
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import Response
+
+from app.logging_context import set_request_id
+
+REQUEST_ID_HEADER = "X-Request-ID"
+
+logger = logging.getLogger(__name__)
+
+
+class RequestLoggingMiddleware(BaseHTTPMiddleware):
+    def __init__(
+        self,
+        app,
+        *,
+        log_requests: bool = True,
+        skip_paths: tuple[str, ...] = (),
+    ) -> None:
+        super().__init__(app)
+        self._log_requests = log_requests
+        self._skip_paths = tuple(path for path in skip_paths if path)
+
+    async def dispatch(self, request: Request, call_next) -> Response:
+        request_id = request.headers.get(REQUEST_ID_HEADER) or token_urlsafe(12)
+        request.state.request_id = request_id
+        set_request_id(request_id)
+
+        start = time.perf_counter()
+        should_log = self._log_requests and not self._is_skipped_path(request.url.path)
+        if should_log:
+            logger.info(
+                "request.started",
+                extra={
+                    "event": "request.started",
+                    "method": request.method,
+                    "path": request.url.path,
+                },
+            )
+
+        try:
+            response = await call_next(request)
+        except Exception:
+            duration_ms = int((time.perf_counter() - start) * 1000)
+            logger.exception(
+                "request.failed",
+                extra={
+                    "event": "request.failed",
+                    "method": request.method,
+                    "path": request.url.path,
+                    "duration_ms": duration_ms,
+                },
+            )
+            raise
+        else:
+            duration_ms = int((time.perf_counter() - start) * 1000)
+            response.headers[REQUEST_ID_HEADER] = request_id
+            if should_log:
+                logger.info(
+                    "request.completed",
+                    extra={
+                        "event": "request.completed",
+                        "method": request.method,
+                        "path": request.url.path,
+                        "status_code": response.status_code,
+                        "duration_ms": duration_ms,
+                    },
+                )
+            return response
+        finally:
+            set_request_id(None)
+
+    def _is_skipped_path(self, path: str) -> bool:
+        return any(path.startswith(prefix) for prefix in self._skip_paths)
+
+
+def parse_skip_paths(raw_value: str) -> tuple[str, ...]:
+    parts = [part.strip() for part in raw_value.split(",")]
+    return tuple(part for part in parts if part)

app/web/routers/__init__.py

@@ -0,0 +1,2 @@
+"""Application routers grouped by concern."""
+

app/web/routers/admin.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+import logging
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
+from fastapi.responses import HTMLResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.auth.deps import get_auth_service
+from app.auth.service import AuthService
+from app.db.session import get_db_session
+from app.services import users as user_service
+from app.theme import resolve_theme
+from app.web import common
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+async def _render_users_page(
+    request: Request,
+    *,
+    db_session: AsyncSession,
+    current_user,
+    theme_name: str,
+    notice: str | None = None,
+    page_error: str | None = None,
+    status_code: int = status.HTTP_200_OK,
+    form_email: str = "",
+    form_is_admin: bool = False,
+) -> HTMLResponse:
+    users = await user_service.list_users(db_session)
+    context = common.build_template_context(
+        request,
+        page_title="Users",
+        active_nav="users",
+        theme_name=theme_name,
+        session_user=common.to_session_user(current_user),
+        notice=notice,
+        page_error=page_error,
+    )
+    context["users"] = users
+    context["current_user_id"] = current_user.id
+    context["form_email"] = form_email
+    context["form_is_admin"] = form_is_admin
+    return common.templates.TemplateResponse(
+        request=request,
+        name="users.html",
+        context=context,
+        status_code=status_code,
+    )
+
+
+@router.get("/users", response_class=HTMLResponse)
+async def users_page(
+    request: Request,
+    theme: str | None = None,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> HTMLResponse:
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+    if not current_user.is_admin:
+        raise HTTPException(status_code=403, detail="Admin access required.")
+    return await _render_users_page(
+        request,
+        db_session=db_session,
+        current_user=current_user,
+        theme_name=resolve_theme(theme),
+        notice=request.query_params.get("notice"),
+        page_error=request.query_params.get("error"),
+    )
+
+
+@router.post("/users")
+async def create_user(
+    request: Request,
+    email: Annotated[str, Form()],
+    password: Annotated[str, Form()],
+    is_admin: Annotated[str | None, Form()] = None,
+    db_session: AsyncSession = Depends(get_db_session),
+    auth_service: AuthService = Depends(get_auth_service),
+) -> HTMLResponse:
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+    if not current_user.is_admin:
+        raise HTTPException(status_code=403, detail="Admin access required.")
+
+    active_theme = resolve_theme(None)
+    try:
+        validated_email = user_service.validate_email(email)
+        validated_password = user_service.validate_password(password)
+        created_user = await user_service.create_user(
+            db_session,
+            email=validated_email,
+            password_hash=auth_service.hash_password(validated_password),
+            is_admin=is_admin == "on",
+        )
+    except user_service.UserServiceError as exc:
+        logger.info(
+            "admin.user_create_failed",
+            extra={
+                "event": "admin.user_create_failed",
+                "admin_user_id": current_user.id,
+                "reason": "validation_or_conflict",
+            },
+        )
+        return await _render_users_page(
+            request,
+            db_session=db_session,
+            current_user=current_user,
+            theme_name=active_theme,
+            page_error=str(exc),
+            status_code=status.HTTP_400_BAD_REQUEST,
+            form_email=email,
+            form_is_admin=is_admin == "on",
+        )
+
+    logger.info(
+        "admin.user_created",
+        extra={
+            "event": "admin.user_created",
+            "admin_user_id": current_user.id,
+            "target_user_id": created_user.id,
+            "target_is_admin": created_user.is_admin,
+        },
+    )
+    return common.redirect_with_message(
+        "/users",
+        notice=f"User created: {created_user.email}",
+    )
+
+
+@router.post("/users/{user_id}/toggle-active")
+async def toggle_user_active(
+    request: Request,
+    user_id: int,
+    db_session: AsyncSession = Depends(get_db_session),
+):
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+    if not current_user.is_admin:
+        raise HTTPException(status_code=403, detail="Admin access required.")
+
+    target_user = await user_service.get_user_by_id(db_session, user_id)
+    if target_user is None:
+        return common.redirect_with_message("/users", error="User not found.")
+    if target_user.id == current_user.id and target_user.is_active:
+        return common.redirect_with_message("/users", error="You cannot deactivate your own account.")
+
+    updated_user = await user_service.set_user_active(
+        db_session,
+        user=target_user,
+        is_active=not target_user.is_active,
+    )
+    status_label = "activated" if updated_user.is_active else "deactivated"
+    logger.info(
+        "admin.user_active_toggled",
+        extra={
+            "event": "admin.user_active_toggled",
+            "admin_user_id": current_user.id,
+            "target_user_id": updated_user.id,
+            "is_active": updated_user.is_active,
+        },
+    )
+    return common.redirect_with_message("/users", notice=f"User {status_label}: {updated_user.email}")
+
+
+@router.post("/users/{user_id}/reset-password")
+async def reset_user_password(
+    request: Request,
+    user_id: int,
+    new_password: Annotated[str, Form()],
+    db_session: AsyncSession = Depends(get_db_session),
+    auth_service: AuthService = Depends(get_auth_service),
+):
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+    if not current_user.is_admin:
+        raise HTTPException(status_code=403, detail="Admin access required.")
+
+    target_user = await user_service.get_user_by_id(db_session, user_id)
+    if target_user is None:
+        return common.redirect_with_message("/users", error="User not found.")
+    try:
+        validated_password = user_service.validate_password(new_password)
+    except user_service.UserServiceError as exc:
+        return common.redirect_with_message("/users", error=str(exc))
+
+    await user_service.set_user_password_hash(
+        db_session,
+        user=target_user,
+        password_hash=auth_service.hash_password(validated_password),
+    )
+    logger.info(
+        "admin.user_password_reset",
+        extra={
+            "event": "admin.user_password_reset",
+            "admin_user_id": current_user.id,
+            "target_user_id": target_user.id,
+        },
+    )
+    return common.redirect_with_message("/users", notice=f"Password reset: {target_user.email}")
+

app/web/routers/auth.py

@@ -0,0 +1,214 @@
+from __future__ import annotations
+
+import logging
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Form, Request, status
+from fastapi.responses import HTMLResponse, RedirectResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.auth.deps import get_auth_service, get_login_rate_limiter
+from app.auth.rate_limit import SlidingWindowRateLimiter
+from app.auth.service import AuthService
+from app.auth.session import get_session_user, set_session_user
+from app.db.session import get_db_session
+from app.security.csrf import ensure_csrf_token
+from app.services import users as user_service
+from app.theme import resolve_theme
+from app.web import common
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/login", response_class=HTMLResponse)
+async def login_page(request: Request, theme: str | None = None) -> HTMLResponse:
+    active_theme = resolve_theme(theme)
+    ensure_csrf_token(request)
+    if get_session_user(request) is not None:
+        return RedirectResponse("/", status_code=status.HTTP_303_SEE_OTHER)
+    return common.render_login_page(request, theme_name=active_theme)
+
+
+@router.post("/login", response_class=HTMLResponse)
+async def login(
+    request: Request,
+    email: Annotated[str, Form()],
+    password: Annotated[str, Form()],
+    db_session: AsyncSession = Depends(get_db_session),
+    auth_service: AuthService = Depends(get_auth_service),
+    rate_limiter: SlidingWindowRateLimiter = Depends(get_login_rate_limiter),
+) -> HTMLResponse:
+    active_theme = resolve_theme(None)
+    limiter_key = common.login_rate_limit_key(request, email)
+    decision = rate_limiter.check(limiter_key)
+    normalized_email = email.strip().lower()
+    if not decision.allowed:
+        logger.warning(
+            "auth.login_rate_limited",
+            extra={
+                "event": "auth.login_rate_limited",
+                "email": normalized_email,
+                "retry_after_seconds": decision.retry_after_seconds,
+            },
+        )
+        response = common.render_login_page(
+            request,
+            theme_name=active_theme,
+            error_message="Too many login attempts. Please try again later.",
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            retry_after_seconds=decision.retry_after_seconds,
+        )
+        response.headers["Retry-After"] = str(decision.retry_after_seconds)
+        return response
+
+    user = await auth_service.authenticate_user(
+        db_session,
+        email=email,
+        password=password,
+    )
+    if user is None:
+        rate_limiter.record_failure(limiter_key)
+        logger.info(
+            "auth.login_failed",
+            extra={
+                "event": "auth.login_failed",
+                "email": normalized_email,
+            },
+        )
+        return common.render_login_page(
+            request,
+            theme_name=active_theme,
+            error_message="Invalid email or password.",
+            status_code=status.HTTP_401_UNAUTHORIZED,
+        )
+
+    rate_limiter.reset(limiter_key)
+    set_session_user(
+        request,
+        user_id=user.id,
+        email=user.email,
+        is_admin=user.is_admin,
+    )
+    ensure_csrf_token(request)
+    logger.info(
+        "auth.login_succeeded",
+        extra={
+            "event": "auth.login_succeeded",
+            "user_id": user.id,
+            "is_admin": user.is_admin,
+        },
+    )
+    return RedirectResponse("/", status_code=status.HTTP_303_SEE_OTHER)
+
+
+@router.post("/logout")
+async def logout(request: Request) -> RedirectResponse:
+    session_user = get_session_user(request)
+    common.invalidate_session(request)
+    logger.info(
+        "auth.logout",
+        extra={
+            "event": "auth.logout",
+            "user_id": session_user.id if session_user else None,
+        },
+    )
+    return RedirectResponse("/login", status_code=status.HTTP_303_SEE_OTHER)
+
+
+@router.get("/account/password", response_class=HTMLResponse)
+async def account_password_page(
+    request: Request,
+    theme: str | None = None,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> HTMLResponse:
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+    return common.templates.TemplateResponse(
+        request=request,
+        name="account_password.html",
+        context=common.build_template_context(
+            request,
+            page_title="Change Password",
+            active_nav="account_password",
+            theme_name=resolve_theme(theme),
+            session_user=common.to_session_user(current_user),
+            notice=request.query_params.get("notice"),
+            page_error=request.query_params.get("error"),
+        ),
+    )
+
+
+@router.post("/account/password")
+async def update_account_password(
+    request: Request,
+    current_password: Annotated[str, Form()],
+    new_password: Annotated[str, Form()],
+    confirm_password: Annotated[str, Form()],
+    db_session: AsyncSession = Depends(get_db_session),
+    auth_service: AuthService = Depends(get_auth_service),
+) -> RedirectResponse:
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+    if not auth_service.verify_password(
+        password_hash=current_user.password_hash,
+        password=current_password,
+    ):
+        logger.info(
+            "account.password_change_failed",
+            extra={
+                "event": "account.password_change_failed",
+                "user_id": current_user.id,
+                "reason": "invalid_current_password",
+            },
+        )
+        return common.redirect_with_message(
+            "/account/password",
+            error="Current password is incorrect.",
+        )
+    if new_password != confirm_password:
+        logger.info(
+            "account.password_change_failed",
+            extra={
+                "event": "account.password_change_failed",
+                "user_id": current_user.id,
+                "reason": "confirmation_mismatch",
+            },
+        )
+        return common.redirect_with_message(
+            "/account/password",
+            error="New password and confirmation do not match.",
+        )
+    try:
+        validated_password = user_service.validate_password(new_password)
+    except user_service.UserServiceError as exc:
+        logger.info(
+            "account.password_change_failed",
+            extra={
+                "event": "account.password_change_failed",
+                "user_id": current_user.id,
+                "reason": "validation_error",
+            },
+        )
+        return common.redirect_with_message("/account/password", error=str(exc))
+
+    await user_service.set_user_password_hash(
+        db_session,
+        user=current_user,
+        password_hash=auth_service.hash_password(validated_password),
+    )
+    logger.info(
+        "account.password_changed",
+        extra={
+            "event": "account.password_changed",
+            "user_id": current_user.id,
+        },
+    )
+    return common.redirect_with_message(
+        "/account/password",
+        notice="Password updated successfully.",
+    )
+

app/web/routers/dashboard.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+import logging
+
+from fastapi import APIRouter, Depends, Request
+from fastapi.responses import HTMLResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models import RunTriggerType
+from app.db.session import get_db_session
+from app.presentation import dashboard as dashboard_presenter
+from app.services import ingestion as ingestion_service
+from app.services import publications as publication_service
+from app.services import runs as run_service
+from app.services import user_settings as user_settings_service
+from app.settings import settings
+from app.theme import resolve_theme
+from app.web import common
+from app.web.deps import get_ingestion_service
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+async def _render_dashboard_page(
+    request: Request,
+    *,
+    db_session: AsyncSession,
+    current_user,
+    theme_name: str,
+    notice: str | None = None,
+    page_error: str | None = None,
+) -> HTMLResponse:
+    unread_publications = await publication_service.list_new_for_latest_run_for_user(
+        db_session,
+        user_id=current_user.id,
+        limit=50,
+    )
+    recent_runs = await run_service.list_recent_runs_for_user(
+        db_session,
+        user_id=current_user.id,
+        limit=20,
+    )
+    user_settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=current_user.id,
+    )
+    queue_counts = await run_service.queue_status_counts_for_user(
+        db_session,
+        user_id=current_user.id,
+    )
+    context = common.build_template_context(
+        request,
+        page_title="Dashboard",
+        active_nav="home",
+        theme_name=theme_name,
+        session_user=common.to_session_user(current_user),
+        notice=notice,
+        page_error=page_error,
+    )
+    context["dashboard"] = dashboard_presenter.build_dashboard_view_model(
+        unread_publications=unread_publications,
+        recent_runs=recent_runs,
+        request_delay_seconds=user_settings.request_delay_seconds,
+        queue_counts=queue_counts,
+    )
+    return common.templates.TemplateResponse(
+        request=request,
+        name="index.html",
+        context=context,
+    )
+
+
+@router.get("/", response_class=HTMLResponse)
+async def home(
+    request: Request,
+    theme: str | None = None,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> HTMLResponse:
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+    return await _render_dashboard_page(
+        request,
+        db_session=db_session,
+        current_user=current_user,
+        theme_name=resolve_theme(theme),
+        notice=request.query_params.get("notice"),
+        page_error=request.query_params.get("error"),
+    )
+
+
+@router.post("/runs/manual")
+async def run_manual_ingestion(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
+):
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+
+    user_settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=current_user.id,
+    )
+    logger.info(
+        "runs.manual_started",
+        extra={
+            "event": "runs.manual_started",
+            "user_id": current_user.id,
+            "request_delay_seconds": user_settings.request_delay_seconds,
+            "network_error_retries": settings.ingestion_network_error_retries,
+            "max_pages_per_scholar": settings.ingestion_max_pages_per_scholar,
+            "page_size": settings.ingestion_page_size,
+        },
+    )
+    try:
+        run_summary = await ingest_service.run_for_user(
+            db_session,
+            user_id=current_user.id,
+            trigger_type=RunTriggerType.MANUAL,
+            request_delay_seconds=user_settings.request_delay_seconds,
+            network_error_retries=settings.ingestion_network_error_retries,
+            retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+            max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
+            page_size=settings.ingestion_page_size,
+            auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
+            queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
+        )
+    except ingestion_service.RunAlreadyInProgressError:
+        await db_session.rollback()
+        return common.redirect_with_message(
+            "/",
+            error="A run is already in progress for this account.",
+        )
+    except Exception:
+        await db_session.rollback()
+        logger.exception(
+            "runs.manual_failed",
+            extra={
+                "event": "runs.manual_failed",
+                "user_id": current_user.id,
+            },
+        )
+        return common.redirect_with_message("/", error="Manual run failed. Check logs for details.")
+
+    logger.info(
+        "runs.manual_completed",
+        extra={
+            "event": "runs.manual_completed",
+            "user_id": current_user.id,
+            "run_id": run_summary.crawl_run_id,
+            "status": run_summary.status.value,
+            "scholar_count": run_summary.scholar_count,
+            "new_publication_count": run_summary.new_publication_count,
+        },
+    )
+    return common.redirect_with_message(
+        "/",
+        notice=(
+            f"Run #{run_summary.crawl_run_id} complete ({run_summary.status.value}). "
+            f"Scholars: {run_summary.scholar_count}, "
+            f"new publications: {run_summary.new_publication_count}."
+        ),
+    )

app/web/routers/health.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException
+
+from app.db.session import check_database
+
+router = APIRouter()
+
+
+@router.get("/healthz")
+async def healthz() -> dict[str, str]:
+    if await check_database():
+        return {"status": "ok"}
+    raise HTTPException(status_code=500, detail="database unavailable")
+

app/web/routers/publications.py

@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+import logging
+from urllib.parse import urlencode, urlparse
+
+from fastapi import APIRouter, Depends, Form, Request
+from fastapi.responses import HTMLResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.session import get_db_session
+from app.services import publications as publication_service
+from app.services import scholars as scholar_service
+from app.theme import resolve_theme
+from app.web import common
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+MODE_NEW = "new"
+MODE_ALL = "all"
+
+
+def _resolve_mode(raw_mode: str | None) -> str:
+    return publication_service.resolve_mode(raw_mode)
+
+
+def _parse_scholar_profile_id(value: str | None) -> int | None:
+    if not value:
+        return None
+    try:
+        parsed = int(value)
+    except ValueError:
+        return None
+    return parsed if parsed > 0 else None
+
+
+def _build_publications_url(*, mode: str, scholar_profile_id: int | None) -> str:
+    params: dict[str, str] = {"mode": mode}
+    if scholar_profile_id is not None:
+        params["scholar_profile_id"] = str(scholar_profile_id)
+    return f"/publications?{urlencode(params)}"
+
+
+def _is_safe_return_to(value: str | None) -> bool:
+    if not value:
+        return False
+    parsed = urlparse(value)
+    if parsed.scheme or parsed.netloc:
+        return False
+    if not value.startswith("/"):
+        return False
+    return value == "/" or value.startswith("/publications")
+
+
+@router.get("/publications", response_class=HTMLResponse)
+async def publications_page(
+    request: Request,
+    mode: str | None = None,
+    scholar_profile_id: str | None = None,
+    theme: str | None = None,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> HTMLResponse:
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+
+    resolved_mode = _resolve_mode(mode)
+    scholar_id_filter = _parse_scholar_profile_id(scholar_profile_id)
+    scholars = await scholar_service.list_scholars_for_user(
+        db_session,
+        user_id=current_user.id,
+    )
+    scholar_lookup = {scholar.id: scholar for scholar in scholars}
+    if scholar_id_filter not in scholar_lookup:
+        scholar_id_filter = None
+
+    publications = await publication_service.list_for_user(
+        db_session,
+        user_id=current_user.id,
+        mode=resolved_mode,
+        scholar_profile_id=scholar_id_filter,
+        limit=500,
+    )
+    new_count = await publication_service.count_for_user(
+        db_session,
+        user_id=current_user.id,
+        mode=MODE_NEW,
+        scholar_profile_id=scholar_id_filter,
+    )
+    total_count = await publication_service.count_for_user(
+        db_session,
+        user_id=current_user.id,
+        mode=MODE_ALL,
+        scholar_profile_id=scholar_id_filter,
+    )
+    mode_new_url = _build_publications_url(
+        mode=MODE_NEW,
+        scholar_profile_id=scholar_id_filter,
+    )
+    mode_all_url = _build_publications_url(
+        mode=MODE_ALL,
+        scholar_profile_id=scholar_id_filter,
+    )
+    selected_scholar = scholar_lookup.get(scholar_id_filter) if scholar_id_filter else None
+    current_publications_url = _build_publications_url(
+        mode=resolved_mode,
+        scholar_profile_id=scholar_id_filter,
+    )
+
+    context = common.build_template_context(
+        request,
+        page_title="Publications",
+        active_nav="publications",
+        theme_name=resolve_theme(theme),
+        session_user=common.to_session_user(current_user),
+        notice=request.query_params.get("notice"),
+        page_error=request.query_params.get("error"),
+    )
+    context["mode"] = resolved_mode
+    context["publications"] = publications
+    context["new_count"] = new_count
+    context["total_count"] = total_count
+    context["mode_new_url"] = mode_new_url
+    context["mode_all_url"] = mode_all_url
+    context["scholars"] = scholars
+    context["selected_scholar_id"] = scholar_id_filter
+    context["selected_scholar"] = selected_scholar
+    context["current_publications_url"] = current_publications_url
+    return common.templates.TemplateResponse(
+        request=request,
+        name="publications.html",
+        context=context,
+    )
+
+
+@router.post("/publications/mark-all-read")
+async def mark_all_publications_read(
+    request: Request,
+    return_to: str | None = Form(default=None),
+    db_session: AsyncSession = Depends(get_db_session),
+):
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+
+    updated_count = await publication_service.mark_all_unread_as_read_for_user(
+        db_session,
+        user_id=current_user.id,
+    )
+    logger.info(
+        "publications.mark_all_read",
+        extra={
+            "event": "publications.mark_all_read",
+            "user_id": current_user.id,
+            "updated_count": updated_count,
+        },
+    )
+    redirect_target = "/publications?mode=new"
+    if _is_safe_return_to(return_to):
+        redirect_target = return_to
+    return common.redirect_with_message(
+        redirect_target,
+        notice=f"Marked {updated_count} publication(s) as read.",
+    )

app/web/routers/runs.py

@@ -0,0 +1,274 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi.responses import HTMLResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.session import get_db_session
+from app.services import runs as run_service
+from app.theme import resolve_theme
+from app.web import common
+
+router = APIRouter()
+
+
+def _as_bool(value: str | None) -> bool:
+    if value is None:
+        return False
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _status_badge(status_value: str) -> str:
+    if status_value == "success":
+        return "ok"
+    if status_value == "failed":
+        return "danger"
+    return "warn"
+
+
+def _queue_status_badge(status_value: str) -> str:
+    if status_value == "dropped":
+        return "danger"
+    if status_value == "retrying":
+        return "ok"
+    return "warn"
+
+
+def _format_dt(value: datetime | None) -> str:
+    if value is None:
+        return "-"
+    return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+
+
+def _summary_dict(error_log: object) -> dict[str, object]:
+    if not isinstance(error_log, dict):
+        return {}
+    summary = error_log.get("summary")
+    if not isinstance(summary, dict):
+        return {}
+    return summary
+
+
+@router.get("/runs", response_class=HTMLResponse)
+async def runs_page(
+    request: Request,
+    theme: str | None = None,
+    failed_only: str | None = None,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> HTMLResponse:
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+
+    failed_only_enabled = _as_bool(failed_only)
+    runs = await run_service.list_runs_for_user(
+        db_session,
+        user_id=current_user.id,
+        limit=200,
+        failed_only=failed_only_enabled,
+    )
+
+    run_items = []
+    for run in runs:
+        summary = _summary_dict(run.error_log)
+        failed_count = summary.get("failed_count", 0)
+        partial_count = summary.get("partial_count", 0)
+        run_items.append(
+            {
+                "id": run.id,
+                "started_at": _format_dt(run.start_dt),
+                "finished_at": _format_dt(run.end_dt),
+                "status": run.status.value,
+                "status_badge": _status_badge(run.status.value),
+                "trigger_type": run.trigger_type.value,
+                "scholar_count": run.scholar_count,
+                "new_publication_count": run.new_pub_count,
+                "failed_count": failed_count,
+                "partial_count": partial_count,
+            }
+        )
+
+    queue_entries = await run_service.list_queue_items_for_user(
+        db_session,
+        user_id=current_user.id,
+        limit=200,
+    )
+    queue_items = []
+    for item in queue_entries:
+        queue_items.append(
+            {
+                "id": item.id,
+                "scholar_profile_id": item.scholar_profile_id,
+                "scholar_label": item.scholar_label,
+                "status": item.status,
+                "status_badge": _queue_status_badge(item.status),
+                "reason": item.reason,
+                "dropped_reason": item.dropped_reason,
+                "attempt_count": item.attempt_count,
+                "resume_cstart": item.resume_cstart,
+                "next_attempt_at": _format_dt(item.next_attempt_dt),
+                "updated_at": _format_dt(item.updated_at),
+                "last_error": item.last_error,
+                "last_run_id": item.last_run_id,
+            }
+        )
+
+    context = common.build_template_context(
+        request,
+        page_title="Runs",
+        active_nav="runs",
+        theme_name=resolve_theme(theme),
+        session_user=common.to_session_user(current_user),
+        notice=request.query_params.get("notice"),
+        page_error=request.query_params.get("error"),
+    )
+    context["runs"] = run_items
+    context["failed_only"] = failed_only_enabled
+    context["queue_items"] = queue_items
+    return common.templates.TemplateResponse(
+        request=request,
+        name="runs.html",
+        context=context,
+    )
+
+
+@router.get("/runs/{run_id}", response_class=HTMLResponse)
+async def run_detail_page(
+    request: Request,
+    run_id: int,
+    theme: str | None = None,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> HTMLResponse:
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+
+    run = await run_service.get_run_for_user(
+        db_session,
+        user_id=current_user.id,
+        run_id=run_id,
+    )
+    if run is None:
+        raise HTTPException(status_code=404, detail="Run not found.")
+
+    error_log = run.error_log if isinstance(run.error_log, dict) else {}
+    scholar_results = error_log.get("scholar_results", [])
+    if not isinstance(scholar_results, list):
+        scholar_results = []
+
+    context = common.build_template_context(
+        request,
+        page_title=f"Run #{run.id}",
+        active_nav="runs",
+        theme_name=resolve_theme(theme),
+        session_user=common.to_session_user(current_user),
+    )
+    context["run"] = {
+        "id": run.id,
+        "started_at": _format_dt(run.start_dt),
+        "finished_at": _format_dt(run.end_dt),
+        "status": run.status.value,
+        "status_badge": _status_badge(run.status.value),
+        "trigger_type": run.trigger_type.value,
+        "scholar_count": run.scholar_count,
+        "new_publication_count": run.new_pub_count,
+    }
+    context["run_summary"] = _summary_dict(error_log)
+    context["scholar_results"] = scholar_results
+    return common.templates.TemplateResponse(
+        request=request,
+        name="run_detail.html",
+        context=context,
+    )
+
+
+@router.post("/runs/queue/{queue_item_id}/drop")
+async def drop_queue_item(
+    request: Request,
+    queue_item_id: int,
+    db_session: AsyncSession = Depends(get_db_session),
+):
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+
+    try:
+        dropped = await run_service.drop_queue_item_for_user(
+            db_session,
+            user_id=current_user.id,
+            queue_item_id=queue_item_id,
+        )
+    except run_service.QueueTransitionError as exc:
+        return common.redirect_with_message(
+            "/runs",
+            error=f"Queue item #{queue_item_id}: {exc.message}",
+        )
+    if dropped is None:
+        raise HTTPException(status_code=404, detail="Queue item not found.")
+    return common.redirect_with_message(
+        "/runs",
+        notice=f"Queue item #{queue_item_id} marked as dropped.",
+    )
+
+
+@router.post("/runs/queue/{queue_item_id}/clear")
+async def clear_queue_item(
+    request: Request,
+    queue_item_id: int,
+    db_session: AsyncSession = Depends(get_db_session),
+):
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+
+    try:
+        cleared = await run_service.clear_queue_item_for_user(
+            db_session,
+            user_id=current_user.id,
+            queue_item_id=queue_item_id,
+        )
+    except run_service.QueueTransitionError as exc:
+        return common.redirect_with_message(
+            "/runs",
+            error=f"Queue item #{queue_item_id}: {exc.message}",
+        )
+    if cleared is None:
+        raise HTTPException(status_code=404, detail="Queue item not found.")
+    return common.redirect_with_message(
+        "/runs",
+        notice=f"Queue item #{queue_item_id} cleared.",
+    )
+
+
+@router.post("/runs/queue/{queue_item_id}/retry")
+async def retry_queue_item(
+    request: Request,
+    queue_item_id: int,
+    db_session: AsyncSession = Depends(get_db_session),
+):
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+
+    try:
+        queue_item = await run_service.retry_queue_item_for_user(
+            db_session,
+            user_id=current_user.id,
+            queue_item_id=queue_item_id,
+        )
+    except run_service.QueueTransitionError as exc:
+        return common.redirect_with_message(
+            "/runs",
+            error=f"Queue item #{queue_item_id}: {exc.message}",
+        )
+    if queue_item is None:
+        raise HTTPException(status_code=404, detail="Queue item not found.")
+    return common.redirect_with_message(
+        "/runs",
+        notice=(
+            f"Queue item #{queue_item_id} queued for retry "
+            f"(scholar: {queue_item.scholar_label})."
+        ),
+    )

app/web/routers/scholars.py

@@ -0,0 +1,212 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+import logging
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
+from fastapi.responses import HTMLResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.session import get_db_session
+from app.services import scholars as scholar_service
+from app.theme import resolve_theme
+from app.web import common
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+def _format_dt(value: datetime | None) -> str:
+    if value is None:
+        return "-"
+    return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+
+
+async def _render_scholars_page(
+    request: Request,
+    *,
+    db_session: AsyncSession,
+    current_user,
+    theme_name: str,
+    notice: str | None = None,
+    page_error: str | None = None,
+    status_code: int = status.HTTP_200_OK,
+    form_scholar_id: str = "",
+    form_display_name: str = "",
+) -> HTMLResponse:
+    scholars = await scholar_service.list_scholars_for_user(
+        db_session,
+        user_id=current_user.id,
+    )
+    scholar_items = []
+    for scholar in scholars:
+        scholar_items.append(
+            {
+                "id": scholar.id,
+                "scholar_id": scholar.scholar_id,
+                "display_name": scholar.display_name or "Unnamed",
+                "is_enabled": scholar.is_enabled,
+                "baseline_completed": scholar.baseline_completed,
+                "last_run_status": (
+                    scholar.last_run_status.value
+                    if scholar.last_run_status is not None
+                    else "never"
+                ),
+                "last_run_dt": _format_dt(scholar.last_run_dt),
+            }
+        )
+    context = common.build_template_context(
+        request,
+        page_title="Scholars",
+        active_nav="scholars",
+        theme_name=theme_name,
+        session_user=common.to_session_user(current_user),
+        notice=notice,
+        page_error=page_error,
+    )
+    context["scholars"] = scholar_items
+    context["form_scholar_id"] = form_scholar_id
+    context["form_display_name"] = form_display_name
+    return common.templates.TemplateResponse(
+        request=request,
+        name="scholars.html",
+        context=context,
+        status_code=status_code,
+    )
+
+
+@router.get("/scholars", response_class=HTMLResponse)
+async def scholars_page(
+    request: Request,
+    theme: str | None = None,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> HTMLResponse:
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+    return await _render_scholars_page(
+        request,
+        db_session=db_session,
+        current_user=current_user,
+        theme_name=resolve_theme(theme),
+        notice=request.query_params.get("notice"),
+        page_error=request.query_params.get("error"),
+    )
+
+
+@router.post("/scholars")
+async def create_scholar(
+    request: Request,
+    scholar_id: Annotated[str, Form()],
+    display_name: Annotated[str, Form()] = "",
+    db_session: AsyncSession = Depends(get_db_session),
+) -> HTMLResponse:
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+    active_theme = resolve_theme(None)
+    try:
+        created_profile = await scholar_service.create_scholar_for_user(
+            db_session,
+            user_id=current_user.id,
+            scholar_id=scholar_id,
+            display_name=display_name,
+        )
+    except scholar_service.ScholarServiceError as exc:
+        logger.info(
+            "scholars.create_failed",
+            extra={
+                "event": "scholars.create_failed",
+                "user_id": current_user.id,
+                "scholar_id": scholar_id.strip(),
+            },
+        )
+        return await _render_scholars_page(
+            request,
+            db_session=db_session,
+            current_user=current_user,
+            theme_name=active_theme,
+            page_error=str(exc),
+            status_code=status.HTTP_400_BAD_REQUEST,
+            form_scholar_id=scholar_id,
+            form_display_name=display_name,
+        )
+    label = created_profile.display_name or created_profile.scholar_id
+    logger.info(
+        "scholars.created",
+        extra={
+            "event": "scholars.created",
+            "user_id": current_user.id,
+            "scholar_profile_id": created_profile.id,
+        },
+    )
+    return common.redirect_with_message("/scholars", notice=f"Scholar added: {label}")
+
+
+@router.post("/scholars/{scholar_profile_id}/toggle")
+async def toggle_scholar(
+    request: Request,
+    scholar_profile_id: int,
+    db_session: AsyncSession = Depends(get_db_session),
+):
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+
+    profile = await scholar_service.get_user_scholar_by_id(
+        db_session,
+        user_id=current_user.id,
+        scholar_profile_id=scholar_profile_id,
+    )
+    if profile is None:
+        raise HTTPException(status_code=404, detail="Scholar not found.")
+    updated_profile = await scholar_service.toggle_scholar_enabled(
+        db_session,
+        profile=profile,
+    )
+    status_label = "enabled" if updated_profile.is_enabled else "disabled"
+    logger.info(
+        "scholars.toggled",
+        extra={
+            "event": "scholars.toggled",
+            "user_id": current_user.id,
+            "scholar_profile_id": updated_profile.id,
+            "is_enabled": updated_profile.is_enabled,
+        },
+    )
+    return common.redirect_with_message(
+        "/scholars",
+        notice=f"Scholar {status_label}: {updated_profile.scholar_id}",
+    )
+
+
+@router.post("/scholars/{scholar_profile_id}/delete")
+async def delete_scholar(
+    request: Request,
+    scholar_profile_id: int,
+    db_session: AsyncSession = Depends(get_db_session),
+):
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+
+    profile = await scholar_service.get_user_scholar_by_id(
+        db_session,
+        user_id=current_user.id,
+        scholar_profile_id=scholar_profile_id,
+    )
+    if profile is None:
+        raise HTTPException(status_code=404, detail="Scholar not found.")
+    deleted_label = profile.display_name or profile.scholar_id
+    await scholar_service.delete_scholar(db_session, profile=profile)
+    logger.info(
+        "scholars.deleted",
+        extra={
+            "event": "scholars.deleted",
+            "user_id": current_user.id,
+            "scholar_profile_id": scholar_profile_id,
+        },
+    )
+    return common.redirect_with_message("/scholars", notice=f"Scholar removed: {deleted_label}")

app/web/routers/settings.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import logging
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Form, Request
+from fastapi.responses import HTMLResponse
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.session import get_db_session
+from app.services import user_settings as user_settings_service
+from app.theme import resolve_theme
+from app.web import common
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/settings", response_class=HTMLResponse)
+async def settings_page(
+    request: Request,
+    theme: str | None = None,
+    db_session: AsyncSession = Depends(get_db_session),
+) -> HTMLResponse:
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+
+    user_settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=current_user.id,
+    )
+    context = common.build_template_context(
+        request,
+        page_title="Settings",
+        active_nav="settings",
+        theme_name=resolve_theme(theme),
+        session_user=common.to_session_user(current_user),
+        notice=request.query_params.get("notice"),
+        page_error=request.query_params.get("error"),
+    )
+    context["user_settings"] = user_settings
+    return common.templates.TemplateResponse(
+        request=request,
+        name="settings.html",
+        context=context,
+    )
+
+
+@router.post("/settings")
+async def update_settings(
+    request: Request,
+    run_interval_minutes: Annotated[str, Form()],
+    request_delay_seconds: Annotated[str, Form()],
+    auto_run_enabled: Annotated[str | None, Form()] = None,
+    db_session: AsyncSession = Depends(get_db_session),
+):
+    current_user = await common.get_authenticated_user(request, db_session)
+    if current_user is None:
+        return common.redirect_to_login()
+    try:
+        parsed_interval = user_settings_service.parse_run_interval_minutes(
+            run_interval_minutes
+        )
+        parsed_delay = user_settings_service.parse_request_delay_seconds(
+            request_delay_seconds
+        )
+    except user_settings_service.UserSettingsServiceError as exc:
+        return common.redirect_with_message("/settings", error=str(exc))
+
+    user_settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=current_user.id,
+    )
+    await user_settings_service.update_settings(
+        db_session,
+        settings=user_settings,
+        auto_run_enabled=auto_run_enabled == "on",
+        run_interval_minutes=parsed_interval,
+        request_delay_seconds=parsed_delay,
+    )
+    logger.info(
+        "settings.updated",
+        extra={
+            "event": "settings.updated",
+            "user_id": current_user.id,
+            "auto_run_enabled": auto_run_enabled == "on",
+            "run_interval_minutes": parsed_interval,
+            "request_delay_seconds": parsed_delay,
+        },
+    )
+    return common.redirect_with_message("/settings", notice="Settings updated.")
+

docker-compose.yml

@@ -0,0 +1,68 @@
+name: scholarr
+
+services:
+  db:
+    image: postgres:15-alpine
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-scholar}
+      POSTGRES_USER: ${POSTGRES_USER:-scholar}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-scholar}
+    ports:
+      - "${POSTGRES_PORT:-5432}:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-scholar} -d ${POSTGRES_DB:-scholar}"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+
+  app:
+    build:
+      context: .
+      target: dev
+    environment:
+      DATABASE_URL: ${DATABASE_URL:-postgresql+asyncpg://scholar:scholar@db:5432/scholar}
+      TEST_DATABASE_URL: ${TEST_DATABASE_URL:-}
+      MIGRATE_ON_START: ${MIGRATE_ON_START:-1}
+      APP_NAME: ${APP_NAME:-scholarr}
+      APP_HOST: ${APP_HOST:-0.0.0.0}
+      APP_PORT: ${APP_PORT:-8000}
+      APP_RELOAD: ${APP_RELOAD:-1}
+      SESSION_SECRET_KEY: ${SESSION_SECRET_KEY:-dev-insecure-session-key}
+      SESSION_COOKIE_SECURE: ${SESSION_COOKIE_SECURE:-0}
+      LOGIN_RATE_LIMIT_ATTEMPTS: ${LOGIN_RATE_LIMIT_ATTEMPTS:-5}
+      LOGIN_RATE_LIMIT_WINDOW_SECONDS: ${LOGIN_RATE_LIMIT_WINDOW_SECONDS:-60}
+      LOG_LEVEL: ${LOG_LEVEL:-INFO}
+      LOG_FORMAT: ${LOG_FORMAT:-console}
+      LOG_REQUESTS: ${LOG_REQUESTS:-1}
+      LOG_UVICORN_ACCESS: ${LOG_UVICORN_ACCESS:-0}
+      LOG_REQUEST_SKIP_PATHS: ${LOG_REQUEST_SKIP_PATHS:-/healthz,/static/}
+      LOG_REDACT_FIELDS: ${LOG_REDACT_FIELDS:-}
+      SCHEDULER_ENABLED: ${SCHEDULER_ENABLED:-1}
+      SCHEDULER_TICK_SECONDS: ${SCHEDULER_TICK_SECONDS:-60}
+      INGESTION_NETWORK_ERROR_RETRIES: ${INGESTION_NETWORK_ERROR_RETRIES:-1}
+      INGESTION_RETRY_BACKOFF_SECONDS: ${INGESTION_RETRY_BACKOFF_SECONDS:-1.0}
+      INGESTION_MAX_PAGES_PER_SCHOLAR: ${INGESTION_MAX_PAGES_PER_SCHOLAR:-30}
+      INGESTION_PAGE_SIZE: ${INGESTION_PAGE_SIZE:-100}
+      BOOTSTRAP_ADMIN_ON_START: ${BOOTSTRAP_ADMIN_ON_START:-0}
+      BOOTSTRAP_ADMIN_EMAIL: ${BOOTSTRAP_ADMIN_EMAIL:-}
+      BOOTSTRAP_ADMIN_PASSWORD: ${BOOTSTRAP_ADMIN_PASSWORD:-}
+      BOOTSTRAP_ADMIN_FORCE_PASSWORD: ${BOOTSTRAP_ADMIN_FORCE_PASSWORD:-0}
+      DB_WAIT_TIMEOUT_SECONDS: ${DB_WAIT_TIMEOUT_SECONDS:-60}
+      DB_WAIT_INTERVAL_SECONDS: ${DB_WAIT_INTERVAL_SECONDS:-2}
+    ports:
+      - "${APP_HOST_PORT:-8000}:8000"
+    volumes:
+      - ./:/app
+    depends_on:
+      db:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://localhost:8000/healthz >/dev/null || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 12
+
+volumes:
+  postgres_data:

planning/mvp_implementation_reminders.md

@@ -0,0 +1,92 @@
+# MVP Implementation Reminders (From Scholar Probe)
+
+Last validated probe run: `planning/scholar_probe_tmp/notes/probe_report_run_20260216T182334Z.md`.
+
+## Decision: Are we ready?
+
+Yes, for the scoped MVP.
+
+- We have stable selectors and field coverage for first-page profile parsing.
+- We have explicit failure modes for blocked/inaccessible pages.
+- We have enough fixtures to start parser + ingestion tests immediately.
+
+Not guaranteed (and intentionally out of MVP):
+
+- Full historical completeness from profile pagination.
+- Robots currently disallows `citations?*cstart=`; treat deep pagination as out of scope unless policy changes.
+
+## Non-negotiables
+
+- Keep app container-first (`docker compose`) and test while developing.
+- Keep feature scope small; prefer deterministic behavior over clever scraping.
+- Never crash a whole run because one scholar page fails.
+- Preserve strict tenant isolation for all run/output/read-state data.
+
+## Scraping Contract
+
+Target URL:
+
+- `https://scholar.google.com/citations?hl=en&user=<scholar_id>`
+
+Primary parse markers:
+
+- Row: `tr.gsc_a_tr`
+- Title/link: `a.gsc_a_at`
+- Citation count: `a.gsc_a_ac`
+- Year: `span.gsc_a_h` (fallback year regex)
+- Metadata lines: first/second `div.gs_gray` => authors/venue
+- Cluster id from `citation_for_view=<user>:<cluster_id>`
+
+## Required Parse States
+
+Use and persist one of:
+
+- `ok`
+- `no_results`
+- `blocked_or_captcha`
+- `layout_changed`
+- `network_error`
+
+Plus these page-level flags:
+
+- `has_show_more_button`
+- `articles_range`
+
+## Data Quality Expectations
+
+Observed from probe sample:
+
+- `title`: 100%
+- `cluster_id`: 100%
+- `citation_count`: 100%
+- `authors_text`: 100%
+- `year`: 98.2%
+- `venue_text`: 94.7%
+
+Implications:
+
+- `year` and `venue_text` must remain nullable.
+- Dedupe must not depend solely on `year`/`venue_text` presence.
+
+## Run Semantics
+
+- First successful run per scholar = baseline.
+- If `has_show_more_button=true`, label result as partial in run status/UI.
+- Invalid/inaccessible scholar IDs are expected and should become structured run errors, not exceptions.
+
+## Testing Priorities (Do Before Feature Expansion)
+
+- Fixture parser unit tests using probe HTML snapshots.
+- Parse-state classification tests (ok/blocked/layout/no_results).
+- Dedupe integration tests (`cluster_id` first, fingerprint fallback).
+- Tenant isolation tests for run records and read-state.
+- Smoke test for manual run path through Docker.
+
+## Stop Conditions
+
+Pause implementation and re-probe if:
+
+- Selector markers drop out unexpectedly.
+- Login/redirect pages become frequent for valid IDs.
+- Robots policy for profile endpoints changes.
+

planning/scholar_probe_tmp/README.md

@@ -0,0 +1,13 @@
+# Scholar Scrape Probe (Temporary)
+
+Purpose: short-lived workspace for information gathering and planning.
+
+Rules:
+- No production code changes here.
+- Use this for HTML samples, parsing experiments, and extraction notes.
+- Delete this directory after we lock the scrape contract and parser plan.
+
+Structure:
+- `fixtures/` saved HTML samples for parser validation
+- `notes/` extraction decisions, edge cases, and risk log
+- `scripts/` one-off probe scripts used only during planning

planning/scholar_probe_tmp/fixtures/run_20260216T182029Z/robots.txt

@@ -0,0 +1,24 @@
+User-agent: *
+Disallow: /search
+Disallow: /index.html
+Disallow: /scholar
+Disallow: /citations?
+Allow: /citations?user=
+Disallow: /citations?*cstart=
+Disallow: /citations?user=*%40
+Disallow: /citations?user=*@
+Allow: /citations?view_op=list_classic_articles
+Allow: /citations?view_op=mandates_leaderboard
+Allow: /citations?view_op=metrics_intro
+Allow: /citations?view_op=new_profile
+Allow: /citations?view_op=sitemap
+Allow: /citations?view_op=top_venues
+
+User-agent: Twitterbot
+Disallow:
+
+User-agent: facebookexternalhit
+Disallow:
+
+User-agent: PetalBot
+Disallow: /