justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

Feature/improved UI #1

Merged
JustinZeus merged 3 commits from feature/improved-UI into main 2026-02-17 20:24:52 +01:00
Conversation 0 Commits 3 Files changed 182 +11056 -13262
182 changed files with 11056 additions and 13262 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.dockerignore
View file

@ -12,3 +12,6 @@ __pycache__
*.log
.env
htmlcov
frontend/node_modules
frontend/dist
planning

27
.env.example
View file

@ -1,28 +1,33 @@
POSTGRES_DB=scholar
POSTGRES_USER=scholar
POSTGRES_PASSWORD=scholar
POSTGRES_PORT=5432
POSTGRES_PASSWORD=change-me
DATABASE_URL=postgresql+asyncpg://scholar:scholar@db:5432/scholar
# Optional override. If empty, tests derive "<DATABASE_URL db name>_test".
TEST_DATABASE_URL=
SCHOLARR_IMAGE=justinzeus/scholarr:latest
APP_NAME=scholarr
APP_HOST=0.0.0.0
APP_PORT=8000
APP_HOST_PORT=8000
APP_RELOAD=1
APP_RELOAD=0
FRONTEND_ENABLED=1
FRONTEND_DIST_DIR=/app/frontend/dist
FRONTEND_HOST_PORT=5173
CHOKIDAR_USEPOLLING=1
VITE_DEV_API_PROXY_TARGET=http://app:8000
MIGRATE_ON_START=1
SESSION_SECRET_KEY=replace-with-a-long-random-secret
SESSION_COOKIE_SECURE=0
SESSION_SECRET_KEY=replace-with-a-long-random-secret-at-least-32-characters
SESSION_COOKIE_SECURE=1
LOGIN_RATE_LIMIT_ATTEMPTS=5
LOGIN_RATE_LIMIT_WINDOW_SECONDS=60
LOG_LEVEL=INFO
LOG_FORMAT=console
LOG_REQUESTS=1
LOG_UVICORN_ACCESS=0
LOG_REQUEST_SKIP_PATHS=/healthz,/static/
LOG_REQUEST_SKIP_PATHS=/healthz
LOG_REDACT_FIELDS=
SCHEDULER_ENABLED=1
SCHEDULER_TICK_SECONDS=60
@ -35,6 +40,16 @@ INGESTION_CONTINUATION_BASE_DELAY_SECONDS=120
INGESTION_CONTINUATION_MAX_DELAY_SECONDS=3600
INGESTION_CONTINUATION_MAX_ATTEMPTS=6
SCHEDULER_QUEUE_BATCH_SIZE=10
SCHOLAR_IMAGE_UPLOAD_DIR=/var/lib/scholarr/uploads
SCHOLAR_IMAGE_UPLOAD_MAX_BYTES=2000000
SCHOLAR_NAME_SEARCH_ENABLED=1
SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS=21600
SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS=300
SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES=512
SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS=8.0
SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS=2.0
SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD=1
SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS=1800
BOOTSTRAP_ADMIN_ON_START=0
BOOTSTRAP_ADMIN_EMAIL=

82
.github/workflows/ci.yml vendored
View file

@ -51,3 +51,85 @@ jobs:
- name: Integration tests
run: uv run pytest -m integration
frontend-quality:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: API contract drift check
run: python3 scripts/check_frontend_api_contract.py
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Install frontend dependencies
working-directory: frontend
run: npm install
- name: Frontend typecheck
working-directory: frontend
run: npm run typecheck
- name: Frontend unit tests
working-directory: frontend
run: npm run test:run
- name: Frontend build
working-directory: frontend
run: npm run build
docker-publish:
runs-on: ubuntu-latest
needs:
- test
- frontend-quality
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Derive image metadata
id: meta
uses: docker/metadata-action@v5
with:
images: justinzeus/scholarr
tags: |
type=raw,value=latest
type=sha,format=short,prefix=sha-
- name: Build and push multi-arch image
uses: docker/build-push-action@v6
with:
context: .
file: ./Dockerfile
target: prod
push: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max

3
.gitignore vendored
View file

@ -10,3 +10,6 @@ htmlcov/
.uv-cache/
.env
*.log
planning/
frontend/node_modules/
frontend/dist/

314
AGENTS.MD Normal file
View file

@ -0,0 +1,314 @@
# scholarr Agent Handbook
This file is the consolidated source of truth for project scope, constraints, scrape contract, and UI implementation flow.
## 1. Product Objective
- Build a self-hosted scholar tracking system ("scholarr") with reliable, low-cost scraping and clear, actionable UI.
- Keep the MVP scope intentionally small.
- Prioritize reliability and ease of use over advanced/fancy processing.
## 2. Locked Constraints and Preferences
- Multi-user system with strict tenant isolation.
- Users are admin-created only.
- Users can change their own password.
- Admin features are shown only to admin users.
- Same-origin cookie session model with CSRF protection.
- Container-first development workflow (`docker compose`, `uv`).
- Test while developing; avoid shipping untested flows.
- Keep backend modular and DRY.
- UI is being rebuilt from scratch against API contracts.
## 3. Current Backend Architecture
- Runtime: Python + FastAPI.
- DB: PostgreSQL + SQLAlchemy + Alembic migrations.
- Scheduler: background scheduler + continuation queue processing.
- Auth:
- Argon2 password hashing.
- Session cookie (`HttpOnly`, `SameSite=Lax`, secure flag configurable).
- CSRF required for unsafe methods via `X-CSRF-Token`.
- Logging:
- structured request logging with request IDs.
- redaction controls for sensitive fields.
## 4. Scraping Contract (Probe-Based)
Status: sufficient for MVP implementation with graceful degradation.
Target endpoint:
- `https://scholar.google.com/citations?hl=en&user=<scholar_id>`
Primary selectors:
- Row: `tr.gsc_a_tr`
- Title/link: `a.gsc_a_at`
- Citation count: `a.gsc_a_ac`
- Year: `span.gsc_a_h` (fallback regex on year cell text)
- Metadata lines: first/second `div.gs_gray` -> authors/venue
- Cluster ID: from `citation_for_view=<user>:<cluster_id>` in title URL
Required parser states:
- `ok`
- `no_results`
- `blocked_or_captcha`
- `layout_changed`
- `network_error`
Required page flags:
- `has_show_more_button`
- `articles_range`
Quality assumptions from probe:
- title/cluster_id/citation/authors coverage observed near 100%
- year and venue may be missing and must remain nullable
Guardrails:
- Never crash the full run when one scholar fails.
- Persist structured failure/debug information for diagnosis.
- Handle inaccessible/redirected IDs as states, not exceptions.
## 5. Current API Scope
Base path: `/api/v1`
Envelope model:
- success: `{"data": ..., "meta": {"request_id": "..."}}`
- error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}`
Auth/session:
- `GET /api/v1/auth/csrf`
- `POST /api/v1/auth/login`
- `GET /api/v1/auth/me`
- `POST /api/v1/auth/change-password`
- `POST /api/v1/auth/logout`
Admin users:
- `GET /api/v1/admin/users`
- `POST /api/v1/admin/users`
- `PATCH /api/v1/admin/users/{id}/active`
- `POST /api/v1/admin/users/{id}/reset-password`
Scholars:
- `GET /api/v1/scholars`
- `POST /api/v1/scholars`
- `GET /api/v1/scholars/search`
- `PATCH /api/v1/scholars/{id}/toggle`
- `PUT /api/v1/scholars/{id}/image/url`
- `POST /api/v1/scholars/{id}/image/upload`
- `GET /api/v1/scholars/{id}/image/upload`
- `DELETE /api/v1/scholars/{id}/image`
- `DELETE /api/v1/scholars/{id}`
Settings:
- `GET /api/v1/settings`
- `PUT /api/v1/settings`
Runs/diagnostics/queue:
- `GET /api/v1/runs`
- `GET /api/v1/runs/{id}`
- `POST /api/v1/runs/manual` (`Idempotency-Key` supported)
- `GET /api/v1/runs/queue/items`
- `POST /api/v1/runs/queue/{id}/retry`
- `POST /api/v1/runs/queue/{id}/drop`
- `DELETE /api/v1/runs/queue/{id}`
Publications:
- `GET /api/v1/publications`
- `POST /api/v1/publications/mark-read`
- `POST /api/v1/publications/mark-all-read`
Ops:
- `GET /healthz`
## 5.1 Runtime Semantics (Locked)
Manual run idempotency:
- `Idempotency-Key` is optional on `POST /api/v1/runs/manual`.
- Idempotency scope is `(user_id, idempotency_key)`.
- Replaying the same key for the same user returns the same run payload after completion.
- If the keyed run is still in progress, respond with `409 run_in_progress` and include `run_id`.
- Enforcement is database-backed, not best-effort in-memory logic.
- Frontend triggers generate keys internally; key input is not user-facing.
Continuation queue attempt policy:
- `attempt_count` is a failed-attempt budget, not a total execution counter.
- Increment attempts only when a continuation execution fails to make progress.
- Reset attempts to `0` after successful progress (success or resumable partial progress).
- Drop queue items only after failed-attempt threshold is reached.
Scholar no-change skip policy:
- Persist first-page fingerprint snapshots per scholar.
- If the first-page fingerprint is unchanged for a normal run (`cstart=0`), skip deep pagination/upsert work.
- Record run scholar result `state_reason` as `no_change_initial_page_signature`.
Scholar create naming policy:
- Do not accept custom display names during scholar create.
- Hydrate names from scraped profile metadata.
Scholar name-search safety policy:
- Treat name search as best-effort only.
- Apply single-flight execution, pacing/jitter, cache, and cooldown circuit breaker after repeated blocked responses.
- Prefer Scholar URL/ID adds for reliable ingestion onboarding.
## 6. Required UI Behavior (MVP)
Core UX entities:
- scholars
- publications (`all` and `new`)
- runs + run diagnostics
- queue visibility/actions
- settings/account
- admin users (role-gated)
Critical semantics:
- Keep `is_new_in_latest_run` separate from `is_read`.
- First-time ingestion should not imply user read-state.
- Surface partial runs and warnings clearly.
- Show loading, empty, and error states explicitly.
- Support both light and dark modes with consistent status semantics.
## 7. UI Information Architecture
Public:
- Login
Authenticated:
- Dashboard
- Scholars
- Publications
- Settings
- Admin Users (admin only)
## 8. End-to-End UI Flow Plan
## 8.1 Session bootstrap
1. On load call `GET /api/v1/auth/csrf`.
2. Call `GET /api/v1/auth/me`.
3. Route to login or dashboard based on auth state.
## 8.2 Login/logout
1. Login form submits `POST /api/v1/auth/login` with `X-CSRF-Token`.
2. Store CSRF token from response for future unsafe requests.
3. Logout calls `POST /api/v1/auth/logout`.
## 8.3 Dashboard
1. Fetch latest publications summary from `GET /api/v1/publications?mode=new&limit=20`.
2. Fetch recent runs from `GET /api/v1/runs?limit=5`.
3. Fetch queue status from `GET /api/v1/runs/queue/items?limit=200` (aggregate in UI).
4. Show:
- new publications count (`new_count`)
- total tracked publications (`total_count`)
- latest run status/started time
- queue badges (`queued`, `retrying`, `dropped`)
5. Admin users additionally see a diagnostics shortcut block to full runs/queue page.
6. Every dashboard error panel must display request-id for support.
## 8.4 Scholars
1. List via `GET /api/v1/scholars`.
2. Add/toggle/delete actions mapped to scholar endpoints.
3. Search by name via `GET /api/v1/scholars/search` with candidate add actions.
4. Profile image controls:
- scraped image fallback from Scholar metadata
- custom URL override
- image upload + reset to scraped/default
5. Per-row quick links:
- all publications filtered by scholar
- new publications filtered by scholar
## 8.5 Publications
1. Default view mode is `new` (`GET /api/v1/publications?mode=new`).
2. Support mode toggle:
- `new`: scoped to latest completed run
- `all`: full history
3. Support scholar filter via `scholar_profile_id` query param.
4. Render both badges independently:
- `is_new_in_latest_run`
- `is_read`
5. "Mark all read" action calls `POST /api/v1/publications/mark-all-read` with CSRF.
6. Show explicit empty states:
- no publications tracked
- no new publications in latest run
- filtered scholar has no items
7. Surface run recency context when mode is `new` (latest completed run semantics).
## 8.6 Runs and diagnostics (admin)
1. List runs from `GET /api/v1/runs`.
2. Run details from `GET /api/v1/runs/{id}`:
- scholar result state/reason
- warnings
- page logs + attempt logs
- debug metadata
3. Queue operations from queue endpoints (`retry`, `drop`, `clear`).
## 8.7 Settings + account
1. User ingest settings from `/api/v1/settings`.
2. Password change via `/api/v1/auth/change-password`.
## 8.8 Admin user management
1. Show section only when `current_user.is_admin=true`.
2. Use admin user endpoints for create/activate/deactivate/reset password.
## 9. Development Method
- API-contract-first frontend.
- Vertical slices (complete flow before moving on).
- Shared API client module:
- always send credentials
- inject CSRF header for unsafe methods
- normalize envelope parsing and error handling
- Keep frontend modules separate (`auth`, `scholars`, `publications`, `runs`, `settings`, `admin`).
- Write tests while implementing each slice.
## 10. Definition of Done (UI Readiness)
- All main flows above implemented and mapped to existing API endpoints.
- Role-gated admin experience enforced in UI.
- Distinct new-vs-read publication semantics visible.
- Run/queue diagnostics visible and actionable.
- Reliable loading/empty/error states with request-id surfaced for support.
- No dependence on removed server-rendered UI layer.
## 11. Open Items (Known Future Work)
- API contract freeze/changelog discipline for external UI clients.
- Additional API-first smoke coverage for end-to-end UI critical flows.
- Optional background image enrichment/refresh jobs for already-tracked scholars.
## 12. UI Rebuild Program Artifacts
- Phase 0 through Phase 3 decisions are consolidated into this handbook and `README.md`.
- Local scratch notes, probes, and temporary planning artifacts are intentionally not tracked in git.
## 13. Frontend Scaffold Status (Implemented)
- Frontend scaffold path: `frontend/`
- Implemented core:
- App shell + routing + auth/role route guards
- Shared API client with envelope parsing, credentials, and CSRF header injection
- Pinia stores for auth, theme, and global UI errors
- Tailwind-first design foundation with light/dark themes and pre-paint theme initialization
- Page shells for login, dashboard, scholars, publications, runs, run detail, settings, and admin users
- Initial feature modules under `frontend/src/features/*` mapped to current API endpoints
- Deployment boundary:
- No in-repo reverse-proxy implementation
- Frontend edge/proxy routing is managed externally (for example, Caddy)
## 14. Frontend Quality Gates (Phase 3 Implemented)
- CI now includes frontend quality gates in `.github/workflows/ci.yml`:
- API contract drift check (`scripts/check_frontend_api_contract.py`)
- frontend typecheck (`npm run typecheck`)
- frontend unit tests (`npm run test:run`)
- frontend build (`npm run build`)
- Frontend test harness:
- `vitest` configured in `frontend/vitest.config.ts`
- baseline tests in `frontend/src/lib/api/envelope.test.ts` and `frontend/src/lib/api/errors.test.ts`

15
Dockerfile
View file

@ -1,3 +1,13 @@
FROM node:20-alpine AS frontend-builder
WORKDIR /frontend
COPY frontend/package.json frontend/package-lock.json ./
RUN npm ci
COPY frontend/ ./
RUN npm run build
FROM python:3.12-slim AS base
ENV PYTHONDONTWRITEBYTECODE=1 \
@ -33,7 +43,9 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
UV_PROJECT_ENVIRONMENT=/opt/venv \
PATH="/opt/venv/bin:$PATH" \
APP_RELOAD=0 \
UVICORN_WORKERS=1
UVICORN_WORKERS=1 \
FRONTEND_ENABLED=1 \
FRONTEND_DIST_DIR=/app/frontend/dist
WORKDIR /app
@ -51,4 +63,5 @@ COPY alembic ./alembic
RUN uv sync --frozen
COPY scripts ./scripts
COPY --from=frontend-builder /frontend/dist /app/frontend/dist
ENTRYPOINT ["/bin/sh", "/app/scripts/entrypoint.sh"]

273
README.md
View file

@ -1,206 +1,147 @@
# scholarr
Container-first, self-hosted scholar tracking in the spirit of the `*arr` ecosystem, with strict multi-user isolation and an intentionally small operational footprint.
<div align="center">
## Current Scope
API-first, self-hosted scholar tracking in the spirit of the `*arr` ecosystem.
- Admin-managed users only (no public signup)
- Session auth with CSRF protection and login rate limiting
- Per-user scholar tracking list (add, enable/disable, delete)
- Per-user automation settings
- Manual per-user ingestion runs from dashboard
- Per-user run history and "new since last run" publication stream
- Publications library view with `new` and `all` modes plus scholar filtering
- Dedicated run diagnostics pages (`/runs`, `/runs/{id}`) with failed-only filtering
- Continuation queue visibility + controls on runs page (`retry`, `drop`, `clear`)
- Baseline-aware discovery tracking (`new` = discovered in latest completed run)
- Read state is user-controlled (`Mark All Read`), including baseline items
- Multi-page profile ingestion (`cstart`) to capture full publication lists
- Dashboard rendered via a presentation-layer view model and section template registry
- Structured application logging with request IDs and field redaction
- Ingestion overlap guard via Postgres advisory locks
- Network-error retry support for ingestion attempts
- Automatic continuation queue for resumable/limit-hit scholar runs
- Background scheduler for per-user automatic runs
- User self-service password changes
- Admin user management (create users, activate/deactivate, reset passwords)
- PostgreSQL + Alembic migrations + containerized test workflow
- Versioned backend API (`/api/v1`) for frontend decoupling (same-origin cookie session)
[![CI](https://img.shields.io/github/actions/workflow/status/justinzeus/scholarr/ci.yml?style=for-the-badge)](https://github.com/justinzeus/scholarr/actions/workflows/ci.yml)
[![Docker Pulls](https://img.shields.io/docker/pulls/justinzeus/scholarr?style=for-the-badge&logo=docker)](https://hub.docker.com/r/justinzeus/scholarr)
[![Docker Image](https://img.shields.io/badge/docker-justinzeus%2Fscholarr-2496ED?style=for-the-badge&logo=docker&logoColor=white)](https://hub.docker.com/r/justinzeus/scholarr)
## Tech Stack
</div>
- Python 3.12+
- FastAPI
- PostgreSQL 15+
- SQLAlchemy 2 + Alembic
- Jinja2 templates + modular theme tokens
- `uv` for dependency and runtime workflow
- Docker / Docker Compose for development and CI parity
## What This Includes
## Quick Start
- Multi-user accounts with admin-managed lifecycle
- Cookie session auth with CSRF enforcement
- Scholar CRUD + name-search discovery + profile image management
- Per-user ingestion settings and scheduler
- Manual runs with idempotency behavior
- Run history and continuation queue diagnostics/actions
- Publications workflow (`new` / `all`, mark-selected-read, mark-all-read)
- Vue 3 + Vite frontend served from the same container image as the API
1. Copy environment defaults:
## Docker Image
- Image: `justinzeus/scholarr`
- Published by GitHub Actions on every push to `main`
- Architectures: `linux/amd64`, `linux/arm64`
- Tags:
- `latest`
- `sha-<short_commit>`
## Quick Deploy (Copy Compose + Fill Env)
1. Copy `docker-compose.yml` and `.env.example` into your deployment directory.
2. Create `.env` from `.env.example`.
3. Set at minimum:
- `SESSION_SECRET_KEY`
- `POSTGRES_PASSWORD`
4. Pull and start:
```bash
cp .env.example .env
docker compose pull
docker compose up -d
```
2. Bootstrap the first admin on startup (recommended for first run):
5. Open:
- App + API: `http://localhost:8000`
- Health check: `http://localhost:8000/healthz`
The SPA and API are same-origin by default in this deployment model.
## Required Environment Variables
| Variable | Required | Purpose |
| --- | --- | --- |
| `SESSION_SECRET_KEY` | Yes | Session signing key. Use a long random value. |
| `POSTGRES_PASSWORD` | Yes | Database password for the bundled Postgres service. |
Optional startup bootstrap:
| Variable | Default | Purpose |
| --- | --- | --- |
| `BOOTSTRAP_ADMIN_ON_START` | `0` | Auto-create admin on app start. |
| `BOOTSTRAP_ADMIN_EMAIL` | empty | Admin email for bootstrap. |
| `BOOTSTRAP_ADMIN_PASSWORD` | empty | Admin password for bootstrap. |
| `BOOTSTRAP_ADMIN_FORCE_PASSWORD` | `0` | Force-reset bootstrap admin password if exists. |
## Ports
| Port | Service | Description |
| --- | --- | --- |
| `8000` | `app` | scholarr API + frontend |
## Volumes
| Volume | Container Path | Purpose |
| --- | --- | --- |
| `postgres_data` | `/var/lib/postgresql/data` | Postgres persistence |
| `scholar_uploads` | `/var/lib/scholarr/uploads` | Scholar image upload persistence |
## Upgrade
```bash
export BOOTSTRAP_ADMIN_ON_START=1
export BOOTSTRAP_ADMIN_EMAIL=admin@example.com
export BOOTSTRAP_ADMIN_PASSWORD=change-me-now
docker compose pull
docker compose up -d
```
3. Start the stack:
## Development Workflow
Default `docker-compose.yml` is deployment-oriented (prebuilt image).
For local development with source mounts + Vite dev server:
```bash
docker compose up --build
docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
```
4. Open:
Then open:
- Login: `http://localhost:8000/login`
- Dashboard: `http://localhost:8000/`
- Healthcheck: `http://localhost:8000/healthz`
- API: `http://localhost:8000`
- Frontend dev server: `http://localhost:5173`
5. After first successful admin login, disable auto-bootstrap:
## Test and Quality Commands
Backend:
```bash
export BOOTSTRAP_ADMIN_ON_START=0
docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest tests/unit
docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest -m integration
```
## Admin Bootstrap (Manual)
You can create/update an admin account without restarting the app:
Frontend:
```bash
docker compose run --rm app uv run python scripts/bootstrap_admin.py \
--email admin@example.com \
--password change-me-now \
--force-password
cd frontend
npm install
npm run typecheck
npm run test:run
npm run build
```
Notes:
- If the user does not exist, this creates an active admin account.
- If the user exists, it ensures admin + active flags.
- `--force-password` resets the password for existing users.
## Test Workflow
- Integration/smoke tests run against `TEST_DATABASE_URL`.
- If `TEST_DATABASE_URL` is empty, test runs derive an isolated database from `DATABASE_URL` by appending `_test`.
- This keeps app data in `DATABASE_URL` untouched during test runs.
- Parser regression tests are pinned to captured real-world fixture pages in `tests/fixtures/scholar/regression`.
- Unit tests:
Contract drift check:
```bash
docker compose run --rm app uv run pytest tests/unit
python3 scripts/check_frontend_api_contract.py
```
- Integration tests:
```bash
docker compose run --rm app uv run pytest -m integration
```
- Smoke checks (container + DB + migration sanity):
```bash
./scripts/smoke_compose.sh
```
CI runs migration, unit, and integration checks on push/PR via `.github/workflows/ci.yml`.
## Scholar Ingestion Notes
- Current ingestion targets public profile pages: `/citations?user=<id>`.
- Runs are intentionally conservative and prioritize reliability over aggressive scraping.
- Ingestion follows pagination (`cstart`) to collect all reachable profile pages.
- If pagination cannot be completed (max pages hit, cursor stall, or page-state failure), the scholar result is marked partial with explicit debug context.
- Resumable partial/failure states are automatically queued and retried by the scheduler with bounded backoff.
- Queue items keep explicit status (`queued`, `retrying`, `dropped`) and last-error context for UI diagnostics.
- Failed scholar attempts persist structured debug context in `crawl_runs.error_log` (state reason, fetch metadata, marker evidence, and compact response excerpt).
- Only resumable states are retried automatically (`network_error` and continuation-eligible pagination truncations); blocked/layout states are recorded immediately as failures.
## API v1 (Frontend Prep)
## API Base
- Base path: `/api/v1`
- Auth model: same-origin cookie session (no token auth added)
- CSRF model: required for unsafe methods (`POST`, `PUT`, `PATCH`, `DELETE`) via `X-CSRF-Token` header.
- Bootstrap CSRF via `GET /api/v1/auth/csrf` (works for anonymous and authenticated sessions).
- You can also fetch `csrf_token` from `GET /api/v1/auth/me` after login.
- Manual run idempotency is supported via `Idempotency-Key` header on `POST /api/v1/runs/manual`.
- Response envelopes:
- Success: `{"data": ..., "meta": {"request_id": "..."}}`
- Error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}`
- Initial API domains exposed:
- `auth`: `/auth/csrf`, `/auth/login`, `/auth/me`, `/auth/change-password`, `/auth/logout`
- `admin users`: `/admin/users` (list/create), `/admin/users/{id}/active`, `/admin/users/{id}/reset-password`
- `scholars`: list/create/toggle/delete
- `settings`: get/update
- `runs`: list/detail/manual trigger + queue list/retry/drop/clear
- `publications`: list + mark-all-read
- Queue action semantics:
- `retry` is valid only for dropped items; retrying/queued states return `409`.
- `drop` returns the updated queue item (status `dropped`); dropping an already dropped item returns `409`.
- `clear` only works for dropped items and returns `{queue_item_id, previous_status, status="cleared"}`.
- Success envelope:
- `{"data": ..., "meta": {"request_id": "..."}}`
- Error envelope:
- `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}`
## Logging
- Default output is concise console logs to stdout.
- Timestamp format is compact UTC: `YYYY-MM-DD HH:MM:SSZ`.
- Every request gets an `X-Request-ID` (propagated if caller provides one).
- Sensitive fields are redacted before log emission (`password`, `csrf_token`, `cookie`, etc.).
- Configure via env:
- `LOG_LEVEL` (default `INFO`)
- `LOG_FORMAT` (`console` or `json`, default `console`)
- `LOG_REQUESTS` (`1`/`0`, default `1`)
- `LOG_UVICORN_ACCESS` (`1`/`0`, default `0`)
- `LOG_REQUEST_SKIP_PATHS` (comma-separated prefixes, default `/healthz,/static/`)
- `LOG_REDACT_FIELDS` (comma-separated additional keys)
- `SCHEDULER_ENABLED` (`1`/`0`, default `1`)
- `SCHEDULER_TICK_SECONDS` (default `60`)
- `INGESTION_NETWORK_ERROR_RETRIES` (default `1`)
- `INGESTION_RETRY_BACKOFF_SECONDS` (default `1.0`)
- `INGESTION_MAX_PAGES_PER_SCHOLAR` (default `30`)
- `INGESTION_PAGE_SIZE` (default `100`)
- `INGESTION_CONTINUATION_QUEUE_ENABLED` (`1`/`0`, default `1`)
- `INGESTION_CONTINUATION_BASE_DELAY_SECONDS` (default `120`)
- `INGESTION_CONTINUATION_MAX_DELAY_SECONDS` (default `3600`)
- `INGESTION_CONTINUATION_MAX_ATTEMPTS` (default `6`)
- `SCHEDULER_QUEUE_BATCH_SIZE` (default `10`)
Important envs:
## Optional Local `uv` Workflow
```bash
uv sync --extra dev
uv run pytest
uv run uvicorn app.main:app --reload
```
Update the lockfile after dependency changes:
```bash
uv lock
```
## Theming
- Theme registry: `app/theme.py`
- Semantic tokens: `app/static/theme.css`
- App styling: `app/static/app.css`
- Theme persistence: `app/static/theme.js`
- Dashboard presentation contract: `app/presentation/dashboard.py`
## Project Layout
```text
app/ FastAPI app, auth/security modules, templates, services, DB wiring, presentation view-models
app/web/ Router modules, shared web helpers, and request middleware
alembic/ Migration environment and versions
scripts/ Entrypoint, DB wait, bootstrap, smoke automation
tests/ Unit, integration, and smoke suites
```
- `LOG_LEVEL`
- `LOG_FORMAT` (`console` or `json`)
- `LOG_REQUESTS`
- `LOG_UVICORN_ACCESS`
- `LOG_REQUEST_SKIP_PATHS`
- `LOG_REDACT_FIELDS`

88
alembic/versions/20260217_0005_manual_run_idempotency.py Normal file
View file

@ -0,0 +1,88 @@
"""Add database-backed manual run idempotency key
Revision ID: 20260217_0005
Revises: 20260217_0004
Create Date: 2026-02-17 16:20:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "20260217_0005"
down_revision: str | Sequence[str] | None = "20260217_0004"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
INDEX_NAME = "uq_crawl_runs_user_manual_idempotency_key"
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns("crawl_runs")}
indexes = {index["name"] for index in inspector.get_indexes("crawl_runs")}
if "idempotency_key" not in columns:
op.add_column(
"crawl_runs",
sa.Column("idempotency_key", sa.String(length=128), nullable=True),
)
columns.add("idempotency_key")
if "idempotency_key" in columns:
op.execute(
"""
UPDATE crawl_runs
SET idempotency_key = NULLIF(BTRIM(error_log #>> '{meta,idempotency_key}'), '')
WHERE trigger_type = 'manual'
AND idempotency_key IS NULL
"""
)
# Preserve one winning row per (user_id, idempotency_key) before creating uniqueness.
op.execute(
"""
WITH ranked AS (
SELECT
id,
ROW_NUMBER() OVER (
PARTITION BY user_id, idempotency_key
ORDER BY start_dt DESC, id DESC
) AS rn
FROM crawl_runs
WHERE trigger_type = 'manual'
AND idempotency_key IS NOT NULL
)
UPDATE crawl_runs AS runs
SET idempotency_key = NULL
FROM ranked
WHERE runs.id = ranked.id
AND ranked.rn > 1
"""
)
if INDEX_NAME not in indexes:
op.create_index(
INDEX_NAME,
"crawl_runs",
["user_id", "idempotency_key"],
unique=True,
postgresql_where=sa.text(
"idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"
),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns("crawl_runs")}
indexes = {index["name"] for index in inspector.get_indexes("crawl_runs")}
if INDEX_NAME in indexes:
op.drop_index(INDEX_NAME, table_name="crawl_runs")
if "idempotency_key" in columns:
op.drop_column("crawl_runs", "idempotency_key")

59
alembic/versions/20260217_0006_scholar_profile_images.py Normal file
View file

@ -0,0 +1,59 @@
"""Add scholar profile image metadata and overrides.
Revision ID: 20260217_0006
Revises: 20260217_0005
Create Date: 2026-02-17 18:30:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "20260217_0006"
down_revision: str | Sequence[str] | None = "20260217_0005"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
TABLE_NAME = "scholar_profiles"
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns(TABLE_NAME)}
if "profile_image_url" not in columns:
op.add_column(
TABLE_NAME,
sa.Column("profile_image_url", sa.Text(), nullable=True),
)
if "profile_image_override_url" not in columns:
op.add_column(
TABLE_NAME,
sa.Column("profile_image_override_url", sa.Text(), nullable=True),
)
if "profile_image_upload_path" not in columns:
op.add_column(
TABLE_NAME,
sa.Column("profile_image_upload_path", sa.Text(), nullable=True),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns(TABLE_NAME)}
if "profile_image_upload_path" in columns:
op.drop_column(TABLE_NAME, "profile_image_upload_path")
if "profile_image_override_url" in columns:
op.drop_column(TABLE_NAME, "profile_image_override_url")
if "profile_image_url" in columns:
op.drop_column(TABLE_NAME, "profile_image_url")

50
alembic/versions/20260217_0007_scholar_initial_page_fingerprint.py Normal file
View file

@ -0,0 +1,50 @@
"""Add per-scholar initial page fingerprint snapshot columns.
Revision ID: 20260217_0007
Revises: 20260217_0006
Create Date: 2026-02-17 18:55:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "20260217_0007"
down_revision: str | Sequence[str] | None = "20260217_0006"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
TABLE_NAME = "scholar_profiles"
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns(TABLE_NAME)}
if "last_initial_page_fingerprint_sha256" not in columns:
op.add_column(
TABLE_NAME,
sa.Column("last_initial_page_fingerprint_sha256", sa.String(length=64), nullable=True),
)
if "last_initial_page_checked_at" not in columns:
op.add_column(
TABLE_NAME,
sa.Column("last_initial_page_checked_at", sa.DateTime(timezone=True), nullable=True),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns(TABLE_NAME)}
if "last_initial_page_checked_at" in columns:
op.drop_column(TABLE_NAME, "last_initial_page_checked_at")
if "last_initial_page_fingerprint_sha256" in columns:
op.drop_column(TABLE_NAME, "last_initial_page_fingerprint_sha256")

5
app/api/deps.py
View file

@ -4,16 +4,16 @@ from fastapi import Depends, Request
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.errors import ApiException
from app.auth import runtime as auth_runtime
from app.db.models import User
from app.db.session import get_db_session
from app.web import common as web_common
async def get_api_current_user(
request: Request,
db_session: AsyncSession = Depends(get_db_session),
) -> User:
current_user = await web_common.get_authenticated_user(request, db_session)
current_user = await auth_runtime.get_authenticated_user(request, db_session)
if current_user is None:
raise ApiException(
status_code=401,
@ -33,4 +33,3 @@ async def get_api_admin_user(
message="Admin access required.",
)
return current_user

10
app/api/routers/auth.py
View file

@ -18,13 +18,13 @@ from app.api.schemas import (
)
from app.auth.deps import get_auth_service, get_login_rate_limiter
from app.auth.rate_limit import SlidingWindowRateLimiter
from app.auth import runtime as auth_runtime
from app.auth.service import AuthService
from app.auth.session import set_session_user
from app.db.models import User
from app.db.session import get_db_session
from app.security.csrf import ensure_csrf_token
from app.services import users as user_service
from app.web import common as web_common
logger = logging.getLogger(__name__)
@ -42,7 +42,7 @@ async def login(
auth_service: AuthService = Depends(get_auth_service),
rate_limiter: SlidingWindowRateLimiter = Depends(get_login_rate_limiter),
):
limiter_key = web_common.login_rate_limit_key(request, payload.email)
limiter_key = auth_runtime.login_rate_limit_key(request, payload.email)
decision = rate_limiter.check(limiter_key)
normalized_email = payload.email.strip().lower()
if not decision.allowed:
@ -142,7 +142,7 @@ async def get_csrf_bootstrap(
request: Request,
db_session: AsyncSession = Depends(get_db_session),
):
current_user = await web_common.get_authenticated_user(request, db_session)
current_user = await auth_runtime.get_authenticated_user(request, db_session)
return success_payload(
request,
data={
@ -213,8 +213,8 @@ async def logout(
request: Request,
db_session: AsyncSession = Depends(get_db_session),
):
current_user = await web_common.get_authenticated_user(request, db_session)
web_common.invalidate_session(request)
current_user = await auth_runtime.get_authenticated_user(request, db_session)
auth_runtime.invalidate_session(request)
logger.info(
"api.auth.logout",
extra={

47
app/api/routers/publications.py
View file

@ -9,7 +9,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_current_user
from app.api.errors import ApiException
from app.api.responses import success_payload
from app.api.schemas import MarkAllReadEnvelope, PublicationsListEnvelope
from app.api.schemas import (
MarkAllReadEnvelope,
MarkSelectedReadEnvelope,
MarkSelectedReadRequest,
PublicationsListEnvelope,
)
from app.db.models import User
from app.db.session import get_db_session
from app.services import publications as publication_service
@ -121,3 +126,43 @@ async def mark_all_publications_read(
"updated_count": updated_count,
},
)
@router.post(
"/mark-read",
response_model=MarkSelectedReadEnvelope,
)
async def mark_selected_publications_read(
payload: MarkSelectedReadRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
selection_pairs = sorted(
{
(int(item.scholar_profile_id), int(item.publication_id))
for item in payload.selections
}
)
updated_count = await publication_service.mark_selected_as_read_for_user(
db_session,
user_id=current_user.id,
selections=selection_pairs,
)
logger.info(
"api.publications.mark_selected_read",
extra={
"event": "api.publications.mark_selected_read",
"user_id": current_user.id,
"requested_count": len(selection_pairs),
"updated_count": updated_count,
},
)
return success_payload(
request,
data={
"message": "Marked selected publications as read.",
"requested_count": len(selection_pairs),
"updated_count": updated_count,
},
)

51
app/api/routers/runs.py
View file

@ -5,6 +5,7 @@ import re
from typing import Any
from fastapi import APIRouter, Depends, Query, Request
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_current_user
@ -24,7 +25,7 @@ from app.services import ingestion as ingestion_service
from app.services import runs as run_service
from app.services import user_settings as user_settings_service
from app.settings import settings
from app.web.deps import get_ingestion_service
from app.api.runtime_deps import get_ingestion_service
logger = logging.getLogger(__name__)
@ -380,6 +381,7 @@ async def run_manual(
page_size=settings.ingestion_page_size,
auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
idempotency_key=idempotency_key,
)
except ingestion_service.RunAlreadyInProgressError as exc:
await db_session.rollback()
@ -388,6 +390,45 @@ async def run_manual(
code="run_in_progress",
message="A run is already in progress for this account.",
) from exc
except IntegrityError as exc:
await db_session.rollback()
if idempotency_key is not None:
existing_run = await run_service.get_manual_run_by_idempotency_key(
db_session,
user_id=current_user.id,
idempotency_key=idempotency_key,
)
if existing_run is not None:
if existing_run.status == RunStatus.RUNNING:
raise ApiException(
status_code=409,
code="run_in_progress",
message="A run with this idempotency key is still in progress.",
details={
"run_id": int(existing_run.id),
"idempotency_key": idempotency_key,
},
) from exc
return success_payload(
request,
data=_manual_run_payload_from_run(
existing_run,
idempotency_key=idempotency_key,
reused_existing_run=True,
),
)
logger.exception(
"api.runs.manual_integrity_error",
extra={
"event": "api.runs.manual_integrity_error",
"user_id": current_user.id,
},
)
raise ApiException(
status_code=500,
code="manual_run_failed",
message="Manual run failed.",
) from exc
except Exception as exc:
await db_session.rollback()
logger.exception(
@ -403,14 +444,6 @@ async def run_manual(
message="Manual run failed.",
) from exc
if idempotency_key is not None:
await run_service.set_manual_run_idempotency_key(
db_session,
user_id=current_user.id,
run_id=run_summary.crawl_run_id,
idempotency_key=idempotency_key,
)
return success_payload(
request,
data={

326
app/api/routers/scholars.py
View file

@ -1,33 +1,56 @@
from __future__ import annotations
import asyncio
import logging
import mimetypes
from fastapi import APIRouter, Depends, Request
from fastapi import APIRouter, Depends, File, Query, Request, UploadFile
from fastapi.responses import FileResponse
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_current_user
from app.api.errors import ApiException
from app.api.responses import success_payload
from app.api.runtime_deps import get_scholar_source
from app.api.schemas import (
MessageEnvelope,
ScholarCreateRequest,
ScholarEnvelope,
ScholarImageUrlUpdateRequest,
ScholarSearchEnvelope,
ScholarsListEnvelope,
)
from app.db.models import User
from app.db.session import get_db_session
from app.services import scholars as scholar_service
from app.services.scholar_source import ScholarSource
from app.settings import settings
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/scholars", tags=["api-scholars"])
def _uploaded_image_api_path(scholar_profile_id: int) -> str:
return f"/api/v1/scholars/{scholar_profile_id}/image/upload"
def _serialize_scholar(profile) -> dict[str, object]:
uploaded_image_url = None
if profile.profile_image_upload_path:
uploaded_image_url = _uploaded_image_api_path(int(profile.id))
profile_image_url, profile_image_source = scholar_service.resolve_profile_image(
profile,
uploaded_image_url=uploaded_image_url,
)
return {
"id": int(profile.id),
"scholar_id": profile.scholar_id,
"display_name": profile.display_name,
"profile_image_url": profile_image_url,
"profile_image_source": profile_image_source,
"is_enabled": bool(profile.is_enabled),
"baseline_completed": bool(profile.baseline_completed),
"last_run_dt": profile.last_run_dt,
@ -67,6 +90,7 @@ async def create_scholar(
payload: ScholarCreateRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
source: ScholarSource = Depends(get_scholar_source),
current_user: User = Depends(get_api_current_user),
):
try:
@ -74,7 +98,8 @@ async def create_scholar(
db_session,
user_id=current_user.id,
scholar_id=payload.scholar_id,
display_name=payload.display_name or "",
display_name="",
profile_image_url=payload.profile_image_url,
)
except scholar_service.ScholarServiceError as exc:
raise ApiException(
@ -90,12 +115,104 @@ async def create_scholar(
"scholar_profile_id": created.id,
},
)
try:
if not created.profile_image_url or not (created.display_name or "").strip():
created = await asyncio.wait_for(
scholar_service.hydrate_profile_metadata(
db_session,
profile=created,
source=source,
),
timeout=5.0,
)
except Exception:
logger.warning(
"api.scholars.create_metadata_hydration_failed",
extra={
"event": "api.scholars.create_metadata_hydration_failed",
"user_id": current_user.id,
"scholar_profile_id": created.id,
},
)
return success_payload(
request,
data=_serialize_scholar(created),
)
@router.get(
"/search",
response_model=ScholarSearchEnvelope,
)
async def search_scholars(
request: Request,
query: str = Query(..., min_length=2, max_length=120),
limit: int = Query(10, ge=1, le=25),
source: ScholarSource = Depends(get_scholar_source),
current_user: User = Depends(get_api_current_user),
):
try:
parsed = await scholar_service.search_author_candidates(
source=source,
query=query,
limit=limit,
network_error_retries=settings.ingestion_network_error_retries,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
search_enabled=settings.scholar_name_search_enabled,
cache_ttl_seconds=settings.scholar_name_search_cache_ttl_seconds,
blocked_cache_ttl_seconds=settings.scholar_name_search_blocked_cache_ttl_seconds,
cache_max_entries=settings.scholar_name_search_cache_max_entries,
min_interval_seconds=settings.scholar_name_search_min_interval_seconds,
interval_jitter_seconds=settings.scholar_name_search_interval_jitter_seconds,
cooldown_block_threshold=settings.scholar_name_search_cooldown_block_threshold,
cooldown_seconds=settings.scholar_name_search_cooldown_seconds,
)
except scholar_service.ScholarServiceError as exc:
raise ApiException(
status_code=400,
code="invalid_scholar_search",
message=str(exc),
) from exc
logger.info(
"api.scholars.search.completed",
extra={
"event": "api.scholars.search.completed",
"user_id": current_user.id,
"query": query.strip(),
"candidate_count": len(parsed.candidates),
"state": parsed.state.value,
},
)
return success_payload(
request,
data={
"query": query.strip(),
"state": parsed.state.value,
"state_reason": parsed.state_reason,
"action_hint": scholar_service.scrape_state_hint(
state=parsed.state,
state_reason=parsed.state_reason,
),
"candidates": [
{
"scholar_id": item.scholar_id,
"display_name": item.display_name,
"affiliation": item.affiliation,
"email_domain": item.email_domain,
"cited_by_count": item.cited_by_count,
"interests": item.interests,
"profile_url": item.profile_url,
"profile_image_url": item.profile_image_url,
}
for item in parsed.candidates
],
"warnings": parsed.warnings,
},
)
@router.patch(
"/{scholar_profile_id}/toggle",
response_model=ScholarEnvelope,
@ -154,7 +271,11 @@ async def delete_scholar(
code="scholar_not_found",
message="Scholar not found.",
)
await scholar_service.delete_scholar(db_session, profile=profile)
await scholar_service.delete_scholar(
db_session,
profile=profile,
upload_dir=settings.scholar_image_upload_dir,
)
logger.info(
"api.scholars.deleted",
extra={
@ -167,3 +288,202 @@ async def delete_scholar(
request,
data={"message": "Scholar deleted."},
)
@router.put(
"/{scholar_profile_id}/image/url",
response_model=ScholarEnvelope,
)
async def update_scholar_image_url(
scholar_profile_id: int,
payload: ScholarImageUrlUpdateRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
profile = await scholar_service.get_user_scholar_by_id(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
)
if profile is None:
raise ApiException(
status_code=404,
code="scholar_not_found",
message="Scholar not found.",
)
try:
updated = await scholar_service.set_profile_image_override_url(
db_session,
profile=profile,
image_url=payload.image_url,
upload_dir=settings.scholar_image_upload_dir,
)
except scholar_service.ScholarServiceError as exc:
raise ApiException(
status_code=400,
code="invalid_scholar_image",
message=str(exc),
) from exc
logger.info(
"api.scholars.image_url_updated",
extra={
"event": "api.scholars.image_url_updated",
"user_id": current_user.id,
"scholar_profile_id": updated.id,
},
)
return success_payload(
request,
data=_serialize_scholar(updated),
)
@router.post(
"/{scholar_profile_id}/image/upload",
response_model=ScholarEnvelope,
)
async def upload_scholar_image(
scholar_profile_id: int,
request: Request,
image: UploadFile = File(...),
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
profile = await scholar_service.get_user_scholar_by_id(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
)
if profile is None:
raise ApiException(
status_code=404,
code="scholar_not_found",
message="Scholar not found.",
)
try:
image_bytes = await image.read()
updated = await scholar_service.set_profile_image_upload(
db_session,
profile=profile,
content_type=image.content_type,
image_bytes=image_bytes,
upload_dir=settings.scholar_image_upload_dir,
max_upload_bytes=settings.scholar_image_upload_max_bytes,
)
except scholar_service.ScholarServiceError as exc:
raise ApiException(
status_code=400,
code="invalid_scholar_image",
message=str(exc),
) from exc
finally:
await image.close()
logger.info(
"api.scholars.image_uploaded",
extra={
"event": "api.scholars.image_uploaded",
"user_id": current_user.id,
"scholar_profile_id": updated.id,
"content_type": image.content_type,
"size_bytes": len(image_bytes),
},
)
return success_payload(
request,
data=_serialize_scholar(updated),
)
@router.delete(
"/{scholar_profile_id}/image",
response_model=ScholarEnvelope,
)
async def clear_scholar_image_customization(
scholar_profile_id: int,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
profile = await scholar_service.get_user_scholar_by_id(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
)
if profile is None:
raise ApiException(
status_code=404,
code="scholar_not_found",
message="Scholar not found.",
)
updated = await scholar_service.clear_profile_image_customization(
db_session,
profile=profile,
upload_dir=settings.scholar_image_upload_dir,
)
logger.info(
"api.scholars.image_customization_cleared",
extra={
"event": "api.scholars.image_customization_cleared",
"user_id": current_user.id,
"scholar_profile_id": updated.id,
},
)
return success_payload(
request,
data=_serialize_scholar(updated),
)
@router.get(
"/{scholar_profile_id}/image/upload",
)
async def get_uploaded_scholar_image(
scholar_profile_id: int,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
profile = await scholar_service.get_user_scholar_by_id(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
)
if profile is None:
raise ApiException(
status_code=404,
code="scholar_not_found",
message="Scholar not found.",
)
if not profile.profile_image_upload_path:
raise ApiException(
status_code=404,
code="scholar_image_not_found",
message="Scholar image not found.",
)
try:
image_path = scholar_service.resolve_upload_file_path(
upload_dir=settings.scholar_image_upload_dir,
relative_path=profile.profile_image_upload_path,
)
except scholar_service.ScholarServiceError as exc:
raise ApiException(
status_code=404,
code="scholar_image_not_found",
message="Scholar image not found.",
) from exc
if not image_path.exists() or not image_path.is_file():
raise ApiException(
status_code=404,
code="scholar_image_not_found",
message="Scholar image not found.",
)
media_type = mimetypes.guess_type(str(image_path))[0] or "application/octet-stream"
return FileResponse(path=image_path, media_type=media_type)

1
app/web/deps.py → app/api/runtime_deps.py
View file

@ -14,4 +14,3 @@ def get_ingestion_service(
source: scholar_source_service.ScholarSource = Depends(get_scholar_source),
) -> ingestion_service.ScholarIngestionService:
return ingestion_service.ScholarIngestionService(source=source)

69
app/api/schemas.py
View file

@ -112,6 +112,8 @@ class ScholarItemData(BaseModel):
id: int
scholar_id: str
display_name: str | None
profile_image_url: str | None
profile_image_source: str
is_enabled: bool
baseline_completed: bool
last_run_dt: datetime | None
@ -135,7 +137,7 @@ class ScholarsListEnvelope(BaseModel):
class ScholarCreateRequest(BaseModel):
scholar_id: str
display_name: str | None = None
profile_image_url: str | None = None
model_config = ConfigDict(extra="forbid")
@ -147,6 +149,43 @@ class ScholarEnvelope(BaseModel):
model_config = ConfigDict(extra="forbid")
class ScholarSearchCandidateData(BaseModel):
scholar_id: str
display_name: str
affiliation: str | None
email_domain: str | None
cited_by_count: int | None
interests: list[str] = Field(default_factory=list)
profile_url: str
profile_image_url: str | None
model_config = ConfigDict(extra="forbid")
class ScholarSearchData(BaseModel):
query: str
state: str
state_reason: str
action_hint: str | None = None
candidates: list[ScholarSearchCandidateData]
warnings: list[str] = Field(default_factory=list)
model_config = ConfigDict(extra="forbid")
class ScholarSearchEnvelope(BaseModel):
data: ScholarSearchData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class ScholarImageUrlUpdateRequest(BaseModel):
image_url: str
model_config = ConfigDict(extra="forbid")
class RunListItemData(BaseModel):
id: int
trigger_type: str
@ -454,3 +493,31 @@ class MarkAllReadEnvelope(BaseModel):
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class PublicationSelectionItem(BaseModel):
scholar_profile_id: int
publication_id: int
model_config = ConfigDict(extra="forbid")
class MarkSelectedReadRequest(BaseModel):
selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
model_config = ConfigDict(extra="forbid")
class MarkSelectedReadData(BaseModel):
message: str
requested_count: int
updated_count: int
model_config = ConfigDict(extra="forbid")
class MarkSelectedReadEnvelope(BaseModel):
data: MarkSelectedReadData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")

55
app/auth/runtime.py Normal file
View file

@ -0,0 +1,55 @@
from __future__ import annotations
import logging
from fastapi import Request
from sqlalchemy.ext.asyncio import AsyncSession
from app.auth.session import clear_session_user, get_session_user, set_session_user
from app.db.models import User
from app.security.csrf import CSRF_SESSION_KEY
from app.services import users as user_service
logger = logging.getLogger(__name__)
def invalidate_session(request: Request) -> None:
clear_session_user(request)
request.session.pop(CSRF_SESSION_KEY, None)
async def get_authenticated_user(
request: Request,
db_session: AsyncSession,
) -> User | None:
session_user = get_session_user(request)
if session_user is None:
return None
user = await user_service.get_user_by_id(db_session, session_user.id)
if user is None or not user.is_active:
logger.info(
"auth.session_invalidated",
extra={
"event": "auth.session_invalidated",
"session_user_id": session_user.id,
},
)
invalidate_session(request)
return None
if user.email != session_user.email or user.is_admin != session_user.is_admin:
set_session_user(
request,
user_id=user.id,
email=user.email,
is_admin=user.is_admin,
)
return user
def login_rate_limit_key(request: Request, email: str) -> str:
client_host = request.client.host if request.client is not None else "unknown"
normalized_email = email.strip().lower()
return f"{client_host}:{normalized_email or '<empty>'}"

11
app/db/base.py
View file

@ -1,4 +1,6 @@
from sqlalchemy import MetaData
from datetime import datetime, timezone
from sqlalchemy import MetaData, event
from sqlalchemy.orm import DeclarativeBase
@ -15,4 +17,11 @@ class Base(DeclarativeBase):
metadata = MetaData(naming_convention=NAMING_CONVENTION)
@event.listens_for(Base, "before_update", propagate=True)
def _set_updated_at_before_update(_mapper, _connection, target) -> None:
# Keep audit timestamps current for ORM-managed updates.
if hasattr(target, "updated_at"):
target.updated_at = datetime.now(timezone.utc)
metadata = Base.metadata

15
app/db/models.py
View file

@ -119,6 +119,11 @@ class ScholarProfile(Base):
)
scholar_id: Mapped[str] = mapped_column(String(64), nullable=False)
display_name: Mapped[str | None] = mapped_column(String(255))
profile_image_url: Mapped[str | None] = mapped_column(Text)
profile_image_override_url: Mapped[str | None] = mapped_column(Text)
profile_image_upload_path: Mapped[str | None] = mapped_column(Text)
last_initial_page_fingerprint_sha256: Mapped[str | None] = mapped_column(String(64))
last_initial_page_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
is_enabled: Mapped[bool] = mapped_column(
Boolean, nullable=False, server_default=text("true")
)
@ -141,6 +146,15 @@ class CrawlRun(Base):
__tablename__ = "crawl_runs"
__table_args__ = (
Index("ix_crawl_runs_user_start", "user_id", "start_dt"),
Index(
"uq_crawl_runs_user_manual_idempotency_key",
"user_id",
"idempotency_key",
unique=True,
postgresql_where=text(
"idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"
),
),
)
id: Mapped[int] = mapped_column(primary_key=True)
@ -163,6 +177,7 @@ class CrawlRun(Base):
new_pub_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default=text("0")
)
idempotency_key: Mapped[str | None] = mapped_column(String(128))
error_log: Mapped[dict] = mapped_column(
JSONB, nullable=False, server_default=text("'{}'::jsonb")
)

1
app/http/__init__.py Normal file
View file

@ -0,0 +1 @@
from __future__ import annotations

2
app/web/middleware.py → app/http/middleware.py
View file

@ -1,8 +1,8 @@
from __future__ import annotations
from secrets import token_urlsafe
import time
import logging
import time
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request

77
app/main.py
View file

@ -1,31 +1,23 @@
from __future__ import annotations
from contextlib import asynccontextmanager
from pathlib import Path
from fastapi import FastAPI
from fastapi import FastAPI, HTTPException
from fastapi.responses import FileResponse
from fastapi.staticfiles import StaticFiles
from starlette.middleware.sessions import SessionMiddleware
from app.api.errors import register_api_exception_handlers
from app.api.router import router as api_router
from app.api.runtime_deps import get_ingestion_service, get_scholar_source
from app.db.session import check_database
from app.db.session import close_engine
from app.http.middleware import RequestLoggingMiddleware, parse_skip_paths
from app.logging_config import configure_logging, parse_redact_fields
from app.security.csrf import CSRFMiddleware
from app.services.scheduler import SchedulerService
from app.settings import settings
from app.web import common as web_common
from app.web.deps import get_ingestion_service, get_scholar_source
from app.web.middleware import RequestLoggingMiddleware, parse_skip_paths
from app.web.routers import (
admin,
auth,
dashboard,
health,
publications,
runs,
scholars,
settings as settings_router,
)
configure_logging(
level=settings.log_level,
@ -71,17 +63,50 @@ app.add_middleware(
log_requests=settings.log_requests,
skip_paths=parse_skip_paths(settings.log_request_skip_paths),
)
app.mount("/static", StaticFiles(directory="app/static"), name="static")
app.include_router(api_router)
app.include_router(auth.router)
app.include_router(admin.router)
app.include_router(scholars.router)
app.include_router(settings_router.router)
app.include_router(runs.router)
app.include_router(publications.router)
app.include_router(dashboard.router)
app.include_router(health.router)
# Backward-compatible export kept for tests and any existing local scripts.
_get_authenticated_user = web_common.get_authenticated_user
@app.get("/healthz")
async def healthz() -> dict[str, str]:
if await check_database():
return {"status": "ok"}
raise HTTPException(status_code=500, detail="database unavailable")
def _configure_frontend_routes(application: FastAPI) -> None:
if not settings.frontend_enabled:
return
dist_dir = Path(settings.frontend_dist_dir)
index_file = dist_dir / "index.html"
if not index_file.is_file():
return
assets_dir = dist_dir / "assets"
if assets_dir.is_dir():
application.mount("/assets", StaticFiles(directory=assets_dir), name="frontend-assets")
@application.get("/", include_in_schema=False)
async def frontend_index() -> FileResponse:
return FileResponse(index_file)
@application.get("/{full_path:path}", include_in_schema=False)
async def frontend_entry(full_path: str) -> FileResponse:
normalized = full_path.lstrip("/")
if normalized in {"healthz", "api"} or normalized.startswith("api/"):
raise HTTPException(status_code=404, detail="Not Found")
candidate = (dist_dir / normalized).resolve()
try:
candidate.relative_to(dist_dir.resolve())
except ValueError as error:
raise HTTPException(status_code=404, detail="Not Found") from error
if candidate.is_file():
return FileResponse(candidate)
return FileResponse(index_file)
_configure_frontend_routes(app)

2
app/presentation/__init__.py
View file

@ -1,2 +0,0 @@
"""Presentation-layer view models used by template routes."""

108
app/presentation/dashboard.py
View file

@ -1,108 +0,0 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Sequence
from app.db.models import CrawlRun
from app.services.publications import UnreadPublicationItem
SECTION_TEMPLATES: tuple[str, ...] = (
"dashboard/_run_controls.html",
"dashboard/_unread_publications.html",
"dashboard/_run_history.html",
)
@dataclass(frozen=True)
class RunControlsViewModel:
request_delay_seconds: int
queue_queued_count: int
queue_retrying_count: int
queue_dropped_count: int
run_manual_action: str = "/runs/manual"
mark_all_read_action: str = "/publications/mark-all-read"
@dataclass(frozen=True)
class UnreadPublicationViewModel:
title: str
scholar_label: str
year_display: str
citation_count: int
venue_text: str | None
pub_url: str | None
@dataclass(frozen=True)
class RunHistoryItemViewModel:
run_id: int | None
detail_url: str | None
started_at_display: str
status_label: str
status_badge: str
scholar_count: int
new_publication_count: int
@dataclass(frozen=True)
class DashboardViewModel:
section_templates: tuple[str, ...]
run_controls: RunControlsViewModel
unread_publications: list[UnreadPublicationViewModel]
run_history: list[RunHistoryItemViewModel]
def build_dashboard_view_model(
*,
unread_publications: Sequence[UnreadPublicationItem],
recent_runs: Sequence[CrawlRun],
request_delay_seconds: int,
queue_counts: dict[str, int] | None = None,
) -> DashboardViewModel:
queue_counts = queue_counts or {}
return DashboardViewModel(
section_templates=SECTION_TEMPLATES,
run_controls=RunControlsViewModel(
request_delay_seconds=request_delay_seconds,
queue_queued_count=int(queue_counts.get("queued", 0)),
queue_retrying_count=int(queue_counts.get("retrying", 0)),
queue_dropped_count=int(queue_counts.get("dropped", 0)),
),
unread_publications=[
UnreadPublicationViewModel(
title=item.title,
scholar_label=item.scholar_label,
year_display=str(item.year) if item.year is not None else "-",
citation_count=item.citation_count,
venue_text=item.venue_text,
pub_url=item.pub_url,
)
for item in unread_publications
],
run_history=[
RunHistoryItemViewModel(
run_id=run.id,
detail_url=f"/runs/{run.id}" if run.id is not None else None,
started_at_display=_format_started_at(run.start_dt),
status_label=run.status.value,
status_badge=_status_badge(run.status.value),
scholar_count=run.scholar_count,
new_publication_count=run.new_pub_count,
)
for run in recent_runs
],
)
def _status_badge(status: str) -> str:
if status == "success":
return "ok"
if status == "failed":
return "danger"
return "warn"
def _format_started_at(dt: datetime) -> str:
local_aware = dt.astimezone(timezone.utc)
return local_aware.strftime("%Y-%m-%d %H:%M UTC")

2
app/security/csrf.py
View file

@ -77,8 +77,6 @@ class CSRFMiddleware(BaseHTTPMiddleware):
if request.method in SAFE_METHODS:
return True
path = request.url.path
if path.startswith("/static/"):
return True
return path in self._exempt_paths
def _is_form_payload(self, request: Request) -> bool:

17
app/services/continuation_queue.py
View file

@ -181,6 +181,23 @@ async def increment_attempt_count(
return item
async def reset_attempt_count(
db_session: AsyncSession,
*,
job_id: int,
) -> IngestionQueueItem | None:
now = datetime.now(timezone.utc)
result = await db_session.execute(
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
)
item = result.scalar_one_or_none()
if item is None:
return None
item.attempt_count = 0
item.updated_at = now
return item
async def reschedule_job(
db_session: AsyncSession,
*,

144
app/services/ingestion.py
View file

@ -4,6 +4,7 @@ import asyncio
from dataclasses import dataclass
from datetime import datetime, timezone
import hashlib
import json
import logging
import re
from typing import Any
@ -45,6 +46,7 @@ RESUMABLE_PARTIAL_REASONS = {
"pagination_cursor_stalled",
}
RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",)
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS = 30
logger = logging.getLogger(__name__)
@ -63,6 +65,9 @@ class RunExecutionSummary:
class PagedParseResult:
fetch_result: FetchResult
parsed_page: ParsedProfilePage
first_page_fetch_result: FetchResult
first_page_parsed_page: ParsedProfilePage
first_page_fingerprint_sha256: str | None
publications: list[PublicationCandidate]
attempt_log: list[dict[str, Any]]
page_logs: list[dict[str, Any]]
@ -71,6 +76,7 @@ class PagedParseResult:
has_more_remaining: bool
pagination_truncated_reason: str | None
continuation_cstart: int | None
skipped_no_change: bool
class RunAlreadyInProgressError(RuntimeError):
@ -96,6 +102,7 @@ class ScholarIngestionService:
start_cstart_by_scholar_id: dict[int, int] | None = None,
auto_queue_continuations: bool = True,
queue_delay_seconds: int = 60,
idempotency_key: str | None = None,
) -> RunExecutionSummary:
lock_acquired = await self._try_acquire_user_lock(
db_session,
@ -153,6 +160,7 @@ class ScholarIngestionService:
"retry_backoff_seconds": retry_backoff_seconds,
"max_pages_per_scholar": max_pages_per_scholar,
"page_size": page_size,
"idempotency_key": idempotency_key,
},
)
@ -162,6 +170,7 @@ class ScholarIngestionService:
status=RunStatus.RUNNING,
scholar_count=len(scholars),
new_pub_count=0,
idempotency_key=idempotency_key,
error_log={},
)
db_session.add(run)
@ -187,6 +196,7 @@ class ScholarIngestionService:
retry_backoff_seconds=retry_backoff_seconds,
max_pages=max_pages_per_scholar,
page_size=page_size,
previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256,
)
fetch_result = paged_parse_result.fetch_result
parsed_page = paged_parse_result.parsed_page
@ -194,6 +204,17 @@ class ScholarIngestionService:
attempt_log = paged_parse_result.attempt_log
page_logs = paged_parse_result.page_logs
first_page = paged_parse_result.first_page_parsed_page
if first_page.profile_name and not (scholar.display_name or "").strip():
scholar.display_name = first_page.profile_name
if first_page.profile_image_url:
scholar.profile_image_url = first_page.profile_image_url
if paged_parse_result.first_page_fingerprint_sha256:
scholar.last_initial_page_fingerprint_sha256 = (
paged_parse_result.first_page_fingerprint_sha256
)
scholar.last_initial_page_checked_at = run_dt
logger.info(
"ingestion.scholar_parsed",
extra={
@ -210,6 +231,7 @@ class ScholarIngestionService:
"has_more_remaining": paged_parse_result.has_more_remaining,
"pagination_truncated_reason": paged_parse_result.pagination_truncated_reason,
"warning_count": len(parsed_page.warnings),
"skipped_no_change": paged_parse_result.skipped_no_change,
},
)
@ -230,7 +252,55 @@ class ScholarIngestionService:
"has_more_remaining": paged_parse_result.has_more_remaining,
"pagination_truncated_reason": paged_parse_result.pagination_truncated_reason,
"continuation_cstart": paged_parse_result.continuation_cstart,
"skipped_no_change": paged_parse_result.skipped_no_change,
"initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256,
}
if paged_parse_result.skipped_no_change:
scholar.last_run_status = RunStatus.SUCCESS
scholar.last_run_dt = run_dt
succeeded_count += 1
result_entry["state"] = first_page.state.value
result_entry["state_reason"] = "no_change_initial_page_signature"
result_entry["outcome"] = "success"
result_entry["publication_count"] = 0
result_entry["warnings"] = first_page.warnings
result_entry["debug"] = {
"state_reason": "no_change_initial_page_signature",
"first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256,
"attempt_log": attempt_log,
"page_logs": page_logs,
}
scholar_results.append(result_entry)
queue_reason, queue_cstart = self._resolve_continuation_queue_target(
outcome=str(result_entry.get("outcome", "")),
state=str(result_entry.get("state", "")),
pagination_truncated_reason=paged_parse_result.pagination_truncated_reason,
continuation_cstart=paged_parse_result.continuation_cstart,
fallback_cstart=start_cstart,
)
if auto_queue_continuations and queue_reason is not None:
await queue_service.upsert_job(
db_session,
user_id=user_id,
scholar_profile_id=scholar.id,
resume_cstart=queue_cstart,
reason=queue_reason,
run_id=run.id,
delay_seconds=queue_delay_seconds,
)
result_entry["continuation_enqueued"] = True
result_entry["continuation_reason"] = queue_reason
result_entry["continuation_cstart"] = queue_cstart
else:
cleared = await queue_service.clear_job_for_scholar(
db_session,
user_id=user_id,
scholar_profile_id=scholar.id,
)
if cleared:
result_entry["continuation_cleared"] = True
continue
had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}
has_partial_publication_set = len(publications) > 0 and had_page_failure
is_partial_due_to_pagination = (
@ -373,6 +443,11 @@ class ScholarIngestionService:
"failed_state_counts": failed_state_counts,
"failed_reason_counts": failed_reason_counts,
},
"meta": {
"idempotency_key": idempotency_key,
}
if idempotency_key
else {},
}
await db_session.commit()
@ -519,6 +594,7 @@ class ScholarIngestionService:
retry_backoff_seconds: float,
max_pages: int,
page_size: int,
previous_initial_page_fingerprint_sha256: str | None = None,
) -> PagedParseResult:
bounded_max_pages = max(1, int(max_pages))
bounded_page_size = max(1, int(page_size))
@ -534,6 +610,9 @@ class ScholarIngestionService:
network_error_retries=network_error_retries,
retry_backoff_seconds=retry_backoff_seconds,
)
first_page_fetch_result = fetch_result
first_page_parsed_page = parsed_page
first_page_fingerprint_sha256 = build_initial_page_fingerprint(parsed_page)
attempt_log: list[dict[str, Any]] = list(first_attempt_log)
page_logs: list[dict[str, Any]] = []
@ -553,11 +632,39 @@ class ScholarIngestionService:
)
pages_attempted = 1
should_skip_no_change = (
start_cstart <= 0
and first_page_fingerprint_sha256 is not None
and previous_initial_page_fingerprint_sha256 is not None
and first_page_fingerprint_sha256 == previous_initial_page_fingerprint_sha256
and parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS}
)
if should_skip_no_change:
return PagedParseResult(
fetch_result=fetch_result,
parsed_page=parsed_page,
first_page_fetch_result=first_page_fetch_result,
first_page_parsed_page=first_page_parsed_page,
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
publications=[],
attempt_log=attempt_log,
page_logs=page_logs,
pages_fetched=1,
pages_attempted=pages_attempted,
has_more_remaining=False,
pagination_truncated_reason=None,
continuation_cstart=None,
skipped_no_change=True,
)
# Immediate hard failure: nothing to salvage.
if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
return PagedParseResult(
fetch_result=fetch_result,
parsed_page=parsed_page,
first_page_fetch_result=first_page_fetch_result,
first_page_parsed_page=first_page_parsed_page,
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
publications=[],
attempt_log=attempt_log,
page_logs=page_logs,
@ -568,6 +675,7 @@ class ScholarIngestionService:
continuation_cstart=(
start_cstart if parsed_page.state == ParseState.NETWORK_ERROR else None
),
skipped_no_change=False,
)
publications = list(parsed_page.publications)
@ -649,6 +757,9 @@ class ScholarIngestionService:
return PagedParseResult(
fetch_result=fetch_result,
parsed_page=parsed_page,
first_page_fetch_result=first_page_fetch_result,
first_page_parsed_page=first_page_parsed_page,
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
publications=_dedupe_publication_candidates(publications),
attempt_log=attempt_log,
page_logs=page_logs,
@ -657,6 +768,7 @@ class ScholarIngestionService:
has_more_remaining=has_more_remaining,
pagination_truncated_reason=pagination_truncated_reason,
continuation_cstart=continuation_cstart,
skipped_no_change=False,
)
async def _upsert_profile_publications(
@ -838,6 +950,7 @@ class ScholarIngestionService:
"fetch_error": fetch_result.error,
"state_reason": parsed_page.state_reason,
"profile_name": parsed_page.profile_name,
"profile_image_url": parsed_page.profile_image_url,
"articles_range": parsed_page.articles_range,
"has_show_more_button": parsed_page.has_show_more_button,
"has_operation_error_banner": parsed_page.has_operation_error_banner,
@ -914,6 +1027,37 @@ def build_publication_fingerprint(candidate: PublicationCandidate) -> str:
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def build_initial_page_fingerprint(parsed_page: ParsedProfilePage) -> str | None:
if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
return None
normalized_rows: list[dict[str, Any]] = []
for publication in parsed_page.publications[:INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS]:
normalized_rows.append(
{
"cluster_id": publication.cluster_id or "",
"title_normalized": normalize_title(publication.title),
"year": publication.year,
"citation_count": publication.citation_count,
}
)
payload = {
"state": parsed_page.state.value,
"articles_range": parsed_page.articles_range or "",
"has_show_more_button": parsed_page.has_show_more_button,
"profile_name": parsed_page.profile_name or "",
"publications": normalized_rows,
}
canonical = json.dumps(
payload,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def build_publication_url(path_or_url: str | None) -> str | None:
if not path_or_url:
return None

38
app/services/publications.py
View file

@ -3,7 +3,7 @@ from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from sqlalchemy import Select, func, select, update
from sqlalchemy import Select, func, select, tuple_, update
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import (
@ -295,3 +295,39 @@ async def mark_all_unread_as_read_for_user(
rowcount = result.rowcount
return int(rowcount or 0)
async def mark_selected_as_read_for_user(
db_session: AsyncSession,
*,
user_id: int,
selections: list[tuple[int, int]],
) -> int:
normalized_pairs = {
(int(scholar_profile_id), int(publication_id))
for scholar_profile_id, publication_id in selections
if int(scholar_profile_id) > 0 and int(publication_id) > 0
}
if not normalized_pairs:
return 0
scholar_ids = (
select(ScholarProfile.id)
.where(ScholarProfile.user_id == user_id)
.scalar_subquery()
)
stmt = (
update(ScholarPublication)
.where(
ScholarPublication.scholar_profile_id.in_(scholar_ids),
tuple_(
ScholarPublication.scholar_profile_id,
ScholarPublication.publication_id,
).in_(list(normalized_pairs)),
ScholarPublication.is_read.is_(False),
)
.values(is_read=True)
)
result = await db_session.execute(stmt)
await db_session.commit()
return int(result.rowcount or 0)

34
app/services/runs.py
View file

@ -4,7 +4,7 @@ from dataclasses import dataclass
from datetime import datetime
from typing import Any
from sqlalchemy import and_, case, func, select
from sqlalchemy import and_, case, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import (
@ -152,7 +152,10 @@ async def get_manual_run_by_idempotency_key(
.where(
CrawlRun.user_id == user_id,
CrawlRun.trigger_type == RunTriggerType.MANUAL,
CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key,
or_(
CrawlRun.idempotency_key == idempotency_key,
CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key,
),
)
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
.limit(1)
@ -160,33 +163,6 @@ async def get_manual_run_by_idempotency_key(
return result.scalar_one_or_none()
async def set_manual_run_idempotency_key(
db_session: AsyncSession,
*,
user_id: int,
run_id: int,
idempotency_key: str,
) -> bool:
run = await get_run_for_user(
db_session,
user_id=user_id,
run_id=run_id,
)
if run is None:
return False
if run.trigger_type != RunTriggerType.MANUAL:
return False
payload = dict(run.error_log) if isinstance(run.error_log, dict) else {}
meta = payload.get("meta")
meta_dict = dict(meta) if isinstance(meta, dict) else {}
meta_dict["idempotency_key"] = idempotency_key
payload["meta"] = meta_dict
run.error_log = payload
await db_session.commit()
return True
async def list_queue_items_for_user(
db_session: AsyncSession,
*,

37
app/services/scheduler.py
View file

@ -399,6 +399,39 @@ class SchedulerService:
return
async with session_factory() as session:
# Failed-attempt budget should advance only when continuation execution fails.
if int(run_summary.failed_count) <= 0:
queue_item = await queue_service.reset_attempt_count(
session,
job_id=job.id,
)
if queue_item is None:
await session.commit()
logger.info(
"scheduler.queue_item_resolved",
extra={
"event": "scheduler.queue_item_resolved",
"queue_item_id": job.id,
"user_id": job.user_id,
"run_id": run_summary.crawl_run_id,
"status": run_summary.status.value,
},
)
return
await session.commit()
logger.info(
"scheduler.queue_item_progressed",
extra={
"event": "scheduler.queue_item_progressed",
"queue_item_id": job.id,
"user_id": job.user_id,
"attempt_count": int(queue_item.attempt_count),
"run_id": run_summary.crawl_run_id,
"status": run_summary.status.value,
},
)
return
queue_item = await queue_service.increment_attempt_count(
session,
job_id=job.id,
@ -451,9 +484,9 @@ class SchedulerService:
)
await session.commit()
logger.info(
"scheduler.queue_item_rescheduled",
"scheduler.queue_item_rescheduled_failed",
extra={
"event": "scheduler.queue_item_rescheduled",
"event": "scheduler.queue_item_rescheduled_failed",
"queue_item_id": job.id,
"user_id": job.user_id,
"attempt_count": queue_item.attempt_count,

408
app/services/scholar_parser.py
View file

@ -6,7 +6,7 @@ from enum import StrEnum
from html import unescape
from html.parser import HTMLParser
from typing import Any
from urllib.parse import parse_qs, urlparse
from urllib.parse import parse_qs, urljoin, urlparse
from app.services.scholar_source import FetchResult
@ -16,6 +16,8 @@ BLOCKED_KEYWORDS = [
"not a robot",
"our systems have detected",
"automated queries",
"recaptcha",
"captcha",
]
NO_RESULTS_KEYWORDS = [
@ -25,6 +27,14 @@ NO_RESULTS_KEYWORDS = [
"no documents",
]
NO_AUTHOR_RESULTS_KEYWORDS = [
"didn't match any user profiles",
"did not match any user profiles",
"didn't match any scholars",
"did not match any scholars",
"no user profiles",
]
MARKER_KEYS = [
"gsc_a_tr",
"gsc_a_at",
@ -36,6 +46,33 @@ MARKER_KEYS = [
"gsc_rsb_st",
]
AUTHOR_SEARCH_MARKER_KEYS = [
"gsc_1usr",
"gs_ai_name",
"gs_ai_aff",
"gs_ai_eml",
"gs_ai_cby",
"gs_ai_one_int",
]
NETWORK_DNS_ERROR_KEYWORDS = [
"temporary failure in name resolution",
"name or service not known",
"nodename nor servname provided",
"getaddrinfo failed",
]
NETWORK_TIMEOUT_KEYWORDS = [
"timed out",
"timeout",
]
NETWORK_TLS_ERROR_KEYWORDS = [
"ssl",
"tls",
"certificate verify failed",
]
TAG_RE = re.compile(r"<[^>]+>", re.S)
SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.I | re.S)
SHOW_MORE_BUTTON_RE = re.compile(
@ -63,11 +100,24 @@ class PublicationCandidate:
venue_text: str | None
@dataclass(frozen=True)
class ScholarSearchCandidate:
scholar_id: str
display_name: str
affiliation: str | None
email_domain: str | None
cited_by_count: int | None
interests: list[str]
profile_url: str
profile_image_url: str | None
@dataclass(frozen=True)
class ParsedProfilePage:
state: ParseState
state_reason: str
profile_name: str | None
profile_image_url: str | None
publications: list[PublicationCandidate]
marker_counts: dict[str, int]
warnings: list[str]
@ -76,6 +126,15 @@ class ParsedProfilePage:
articles_range: str | None
@dataclass(frozen=True)
class ParsedAuthorSearchPage:
state: ParseState
state_reason: str
candidates: list[ScholarSearchCandidate]
marker_counts: dict[str, int]
warnings: list[str]
def normalize_space(value: str) -> str:
return " ".join(unescape(value).split())
@ -98,6 +157,19 @@ def attr_href(attrs: list[tuple[str, str | None]]) -> str | None:
return None
def attr_src(attrs: list[tuple[str, str | None]]) -> str | None:
for name, raw_value in attrs:
if name.lower() == "src":
return raw_value
return None
def build_absolute_scholar_url(path_or_url: str | None) -> str | None:
if not path_or_url:
return None
return urljoin("https://scholar.google.com", path_or_url)
class ScholarRowParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
@ -197,6 +269,18 @@ def parse_cluster_id_from_href(href: str | None) -> str | None:
return None
def parse_scholar_id_from_href(href: str | None) -> str | None:
if not href:
return None
parsed = urlparse(href)
query = parse_qs(parsed.query)
user_values = query.get("user")
if not user_values:
return None
candidate = user_values[0].strip()
return candidate or None
def parse_year(parts: list[str]) -> int | None:
text = normalize_space(" ".join(parts))
match = re.search(r"\b(19|20)\d{2}\b", text)
@ -265,6 +349,30 @@ def extract_profile_name(html: str) -> str | None:
return value or None
def extract_profile_image_url(html: str) -> str | None:
og_image_pattern = re.compile(
r"<meta[^>]+property=['\"]og:image['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>",
re.I | re.S,
)
og_match = og_image_pattern.search(html)
if og_match:
value = normalize_space(og_match.group(1))
absolute = build_absolute_scholar_url(value)
if absolute:
return absolute
image_pattern = re.compile(
r"<img[^>]*\bid=['\"]gsc_prf_pup-img['\"][^>]*\bsrc=['\"]([^'\"]+)['\"][^>]*>",
re.I | re.S,
)
image_match = image_pattern.search(html)
if not image_match:
return None
value = normalize_space(image_match.group(1))
return build_absolute_scholar_url(value)
def extract_articles_range(html: str) -> str | None:
pattern = re.compile(
r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)</[^>]+>",
@ -304,6 +412,229 @@ def count_markers(html: str) -> dict[str, int]:
return {key: lowered.count(key.lower()) for key in MARKER_KEYS}
def count_author_search_markers(html: str) -> dict[str, int]:
lowered = html.lower()
return {key: lowered.count(key.lower()) for key in AUTHOR_SEARCH_MARKER_KEYS}
def _extract_verified_email_domain(value: str | None) -> str | None:
if not value:
return None
match = re.search(r"verified email at\s+(.+)$", value.strip(), re.I)
if not match:
return None
domain = normalize_space(match.group(1))
return domain or None
class ScholarAuthorSearchParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.candidates: list[ScholarSearchCandidate] = []
self._candidate: dict[str, Any] | None = None
def _begin_candidate(self) -> None:
self._candidate = {
"depth": 1,
"name_href": None,
"name_parts": [],
"aff_depth": 0,
"aff_parts": [],
"name_depth": 0,
"eml_depth": 0,
"eml_parts": [],
"cby_depth": 0,
"cby_parts": [],
"interest_depth": 0,
"interest_parts": [],
"interests": [],
"image_src": None,
}
def _increment_capture_depths(self) -> None:
if self._candidate is None:
return
for key in ("name_depth", "aff_depth", "eml_depth", "cby_depth", "interest_depth"):
if self._candidate[key] > 0:
self._candidate[key] += 1
def _finalize_candidate(self) -> None:
if self._candidate is None:
return
name = normalize_space("".join(self._candidate["name_parts"]))
scholar_id = parse_scholar_id_from_href(self._candidate["name_href"])
if not name or not scholar_id:
return
affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None
email_domain = _extract_verified_email_domain(
normalize_space("".join(self._candidate["eml_parts"])) or None
)
cited_by_text = normalize_space("".join(self._candidate["cby_parts"]))
cited_by_match = re.search(r"\d+", cited_by_text)
cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None
seen_interests: set[str] = set()
interests: list[str] = []
for interest in self._candidate["interests"]:
normalized = normalize_space(interest)
if not normalized or normalized in seen_interests:
continue
seen_interests.add(normalized)
interests.append(normalized)
profile_url = build_absolute_scholar_url(self._candidate["name_href"])
if not profile_url:
profile_url = (
"https://scholar.google.com/citations"
f"?hl=en&user={scholar_id}"
)
self.candidates.append(
ScholarSearchCandidate(
scholar_id=scholar_id,
display_name=name,
affiliation=affiliation,
email_domain=email_domain,
cited_by_count=cited_by_count,
interests=interests,
profile_url=profile_url,
profile_image_url=build_absolute_scholar_url(self._candidate["image_src"]),
)
)
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
classes = attr_class(attrs)
if self._candidate is None:
if tag == "div" and "gsc_1usr" in classes:
self._begin_candidate()
return
self._candidate["depth"] += 1
self._increment_capture_depths()
if tag == "a" and "gs_ai_name" in classes:
self._candidate["name_depth"] = 1
self._candidate["name_href"] = attr_href(attrs)
return
if tag == "div" and "gs_ai_aff" in classes:
self._candidate["aff_depth"] = 1
return
if tag == "div" and "gs_ai_eml" in classes:
self._candidate["eml_depth"] = 1
return
if tag == "div" and "gs_ai_cby" in classes:
self._candidate["cby_depth"] = 1
return
if tag == "a" and "gs_ai_one_int" in classes:
self._candidate["interest_depth"] = 1
self._candidate["interest_parts"] = []
return
if tag == "img" and self._candidate["image_src"] is None:
self._candidate["image_src"] = attr_src(attrs)
def handle_data(self, data: str) -> None:
if self._candidate is None:
return
if self._candidate["name_depth"] > 0:
self._candidate["name_parts"].append(data)
if self._candidate["aff_depth"] > 0:
self._candidate["aff_parts"].append(data)
if self._candidate["eml_depth"] > 0:
self._candidate["eml_parts"].append(data)
if self._candidate["cby_depth"] > 0:
self._candidate["cby_parts"].append(data)
if self._candidate["interest_depth"] > 0:
self._candidate["interest_parts"].append(data)
def _decrement_capture_depth(self, key: str) -> bool:
if self._candidate is None:
return False
if self._candidate[key] <= 0:
return False
self._candidate[key] -= 1
return self._candidate[key] == 0
def handle_endtag(self, _tag: str) -> None:
if self._candidate is None:
return
interest_closed = self._decrement_capture_depth("interest_depth")
self._decrement_capture_depth("name_depth")
self._decrement_capture_depth("aff_depth")
self._decrement_capture_depth("eml_depth")
self._decrement_capture_depth("cby_depth")
if interest_closed:
interest_text = normalize_space("".join(self._candidate["interest_parts"]))
if interest_text:
self._candidate["interests"].append(interest_text)
self._candidate["interest_parts"] = []
self._candidate["depth"] -= 1
if self._candidate["depth"] > 0:
return
self._finalize_candidate()
self._candidate = None
def classify_network_error_reason(fetch_error: str | None) -> str:
lowered = (fetch_error or "").lower()
if lowered:
if any(keyword in lowered for keyword in NETWORK_DNS_ERROR_KEYWORDS):
return "network_dns_resolution_failed"
if any(keyword in lowered for keyword in NETWORK_TIMEOUT_KEYWORDS):
return "network_timeout"
if any(keyword in lowered for keyword in NETWORK_TLS_ERROR_KEYWORDS):
return "network_tls_error"
if "connection reset" in lowered:
return "network_connection_reset"
if "connection refused" in lowered:
return "network_connection_refused"
if "network is unreachable" in lowered:
return "network_unreachable"
return "network_error_missing_status_code"
def classify_block_or_captcha_reason(
*,
status_code: int,
final_url: str,
body_lowered: str,
) -> str | None:
if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url):
return "blocked_accounts_redirect"
if status_code == 429:
return "blocked_http_429_rate_limited"
if status_code == 403:
if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url:
return "blocked_http_403_captcha_challenge"
return "blocked_http_403_forbidden"
if "sorry/index" in final_url or "sorry/index" in body_lowered:
return "blocked_google_sorry_challenge"
if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered:
return "blocked_unusual_traffic_detected"
if "automated queries" in body_lowered:
return "blocked_automated_queries_detected"
if "not a robot" in body_lowered:
return "blocked_not_a_robot_challenge"
if "recaptcha" in body_lowered:
return "blocked_recaptcha_challenge"
if "captcha" in body_lowered:
return "blocked_captcha_challenge"
if any(keyword in body_lowered for keyword in BLOCKED_KEYWORDS):
return "blocked_keyword_detected"
return None
def detect_state(
fetch_result: FetchResult,
publications: list[PublicationCandidate],
@ -312,15 +643,19 @@ def detect_state(
visible_text: str,
) -> tuple[ParseState, str]:
if fetch_result.status_code is None:
return ParseState.NETWORK_ERROR, "network_error_missing_status_code"
return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
lowered = fetch_result.body.lower()
final = (fetch_result.final_url or "").lower()
status_code = int(fetch_result.status_code)
if "accounts.google.com" in final and ("signin" in final or "servicelogin" in final):
return ParseState.BLOCKED_OR_CAPTCHA, "blocked_accounts_redirect"
if any(keyword in lowered for keyword in BLOCKED_KEYWORDS) or "sorry/index" in final:
return ParseState.BLOCKED_OR_CAPTCHA, "blocked_keyword_detected"
block_reason = classify_block_or_captcha_reason(
status_code=status_code,
final_url=final,
body_lowered=lowered,
)
if block_reason is not None:
return ParseState.BLOCKED_OR_CAPTCHA, block_reason
if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS):
return ParseState.NO_RESULTS, "no_results_keyword_detected"
@ -335,6 +670,40 @@ def detect_state(
return ParseState.OK, "publications_extracted"
def detect_author_search_state(
fetch_result: FetchResult,
candidates: list[ScholarSearchCandidate],
marker_counts: dict[str, int],
*,
visible_text: str,
) -> tuple[ParseState, str]:
if fetch_result.status_code is None:
return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
lowered = fetch_result.body.lower()
final = (fetch_result.final_url or "").lower()
status_code = int(fetch_result.status_code)
block_reason = classify_block_or_captcha_reason(
status_code=status_code,
final_url=final,
body_lowered=lowered,
)
if block_reason is not None:
return ParseState.BLOCKED_OR_CAPTCHA, block_reason
if not candidates and any(keyword in visible_text for keyword in NO_AUTHOR_RESULTS_KEYWORDS):
return ParseState.NO_RESULTS, "no_results_keyword_detected"
if not candidates:
has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0
if not has_search_markers:
return ParseState.NO_RESULTS, "no_search_candidates_detected"
return ParseState.OK, "search_markers_present_with_empty_results"
return ParseState.OK, "author_candidates_extracted"
def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
publications, warnings = parse_publications(fetch_result.body)
marker_counts = count_markers(fetch_result.body)
@ -361,6 +730,7 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
state=state,
state_reason=state_reason,
profile_name=extract_profile_name(fetch_result.body),
profile_image_url=extract_profile_image_url(fetch_result.body),
publications=publications,
marker_counts=marker_counts,
warnings=warnings,
@ -368,3 +738,29 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
has_operation_error_banner=operation_error_banner,
articles_range=extract_articles_range(fetch_result.body),
)
def parse_author_search_page(fetch_result: FetchResult) -> ParsedAuthorSearchPage:
parser = ScholarAuthorSearchParser()
parser.feed(fetch_result.body)
marker_counts = count_author_search_markers(fetch_result.body)
visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
warnings: list[str] = []
if not parser.candidates:
warnings.append("no_author_candidates_detected")
state, state_reason = detect_author_search_state(
fetch_result,
parser.candidates,
marker_counts,
visible_text=visible_text,
)
return ParsedAuthorSearchPage(
state=state,
state_reason=state_reason,
candidates=parser.candidates,
marker_counts=marker_counts,
warnings=warnings,
)

40
app/services/scholar_source.py
View file

@ -52,6 +52,14 @@ class ScholarSource(Protocol):
) -> FetchResult:
...
async def fetch_author_search_html(
self,
query: str,
*,
start: int,
) -> FetchResult:
...
class LiveScholarSource:
def __init__(
@ -94,6 +102,27 @@ class LiveScholarSource:
)
return await asyncio.to_thread(self._fetch_sync, requested_url)
async def fetch_author_search_html(
self,
query: str,
*,
start: int = 0,
) -> FetchResult:
requested_url = _build_author_search_url(
query=query,
start=start,
)
logger.debug(
"scholar_source.search_fetch_started",
extra={
"event": "scholar_source.search_fetch_started",
"query": query,
"requested_url": requested_url,
"start": start,
},
)
return await asyncio.to_thread(self._fetch_sync, requested_url)
def _fetch_sync(self, requested_url: str) -> FetchResult:
request = Request(
requested_url,
@ -169,3 +198,14 @@ def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:
if pagesize > 0 and cstart > 0:
query["pagesize"] = int(pagesize)
return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}"
def _build_author_search_url(*, query: str, start: int) -> str:
params: dict[str, int | str] = {
"hl": "en",
"view_op": "search_authors",
"mauthors": query,
}
if start > 0:
params["astart"] = int(start)
return f"{SCHOLAR_PROFILE_URL}?{urlencode(params)}"

548
app/services/scholars.py
View file

@ -1,14 +1,106 @@
from __future__ import annotations
import asyncio
from collections import OrderedDict
from dataclasses import dataclass
from dataclasses import replace
from datetime import datetime
from datetime import timedelta
from datetime import timezone
import logging
import os
import random
import re
import time
from pathlib import Path
from urllib.parse import urlparse
from uuid import uuid4
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import ScholarProfile
from app.services.scholar_parser import (
ParseState,
ParsedAuthorSearchPage,
parse_author_search_page,
parse_profile_page,
)
from app.services.scholar_source import ScholarSource
SCHOLAR_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]{12}$")
MAX_IMAGE_URL_LENGTH = 2048
MAX_AUTHOR_SEARCH_LIMIT = 25
DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES = 512
DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS = 300
DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD = 1
DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS = 1800
DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS = 3.0
DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS = 1.0
ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES = {
"image/jpeg": ".jpg",
"image/png": ".png",
"image/webp": ".webp",
"image/gif": ".gif",
}
SEARCH_DISABLED_REASON = "search_disabled_by_configuration"
SEARCH_COOLDOWN_REASON = "search_temporarily_disabled_due_to_repeated_blocks"
SEARCH_CACHED_BLOCK_REASON = "search_temporarily_disabled_from_cached_blocked_response"
STATE_REASON_HINTS: dict[str, str] = {
SEARCH_DISABLED_REASON: (
"Scholar name search is currently disabled by service policy. "
"Add scholars by profile URL or Scholar ID."
),
SEARCH_COOLDOWN_REASON: (
"Scholar name search is temporarily paused after repeated block responses. "
"Use Scholar URL/ID adds until cooldown expires."
),
SEARCH_CACHED_BLOCK_REASON: (
"A recent blocked response was cached to reduce traffic. "
"Retry later or add by Scholar URL/ID."
),
"network_dns_resolution_failed": (
"DNS resolution failed while reaching scholar.google.com. "
"Verify container DNS/network and retry."
),
"network_timeout": (
"Request timed out before Google Scholar responded. "
"Increase delay/backoff and retry."
),
"network_tls_error": (
"TLS handshake/certificate validation failed. "
"Verify outbound TLS/network configuration."
),
"blocked_http_429_rate_limited": (
"Google Scholar rate-limited the request. "
"Slow request cadence and retry later."
),
"blocked_unusual_traffic_detected": (
"Google Scholar flagged traffic as unusual. "
"Increase delay/jitter and reduce concurrent scraping."
),
"blocked_accounts_redirect": (
"Request was redirected to Google Account sign-in. "
"Treat as access block and retry later."
),
}
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class _AuthorSearchCacheEntry:
parsed: ParsedAuthorSearchPage
expires_at_monotonic: float
cached_at_utc: datetime
_AUTHOR_SEARCH_EXECUTION_LOCK = asyncio.Lock()
_AUTHOR_SEARCH_CACHE: OrderedDict[str, _AuthorSearchCacheEntry] = OrderedDict()
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = 0.0
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC: datetime | None = None
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
class ScholarServiceError(ValueError):
@ -27,6 +119,174 @@ def normalize_display_name(value: str) -> str | None:
return normalized if normalized else None
def normalize_profile_image_url(value: str | None) -> str | None:
if value is None:
return None
candidate = value.strip()
if not candidate:
return None
if len(candidate) > MAX_IMAGE_URL_LENGTH:
raise ScholarServiceError(
f"Image URL must be {MAX_IMAGE_URL_LENGTH} characters or fewer."
)
parsed = urlparse(candidate)
if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc:
raise ScholarServiceError("Image URL must be an absolute http(s) URL.")
return candidate
def _ensure_upload_root(upload_dir: str, *, create: bool) -> Path:
root = Path(upload_dir).expanduser().resolve()
if create:
root.mkdir(parents=True, exist_ok=True)
return root
def _resolve_upload_path(upload_root: Path, relative_path: str) -> Path:
candidate = (upload_root / relative_path).resolve()
if upload_root != candidate and upload_root not in candidate.parents:
raise ScholarServiceError("Invalid scholar image path.")
return candidate
def _safe_remove_upload(upload_root: Path, relative_path: str | None) -> None:
if not relative_path:
return
try:
file_path = _resolve_upload_path(upload_root, relative_path)
except ScholarServiceError:
return
try:
if file_path.exists() and file_path.is_file():
file_path.unlink()
except OSError:
return
def resolve_profile_image(
profile: ScholarProfile,
*,
uploaded_image_url: str | None,
) -> tuple[str | None, str]:
if profile.profile_image_upload_path and uploaded_image_url:
return uploaded_image_url, "upload"
if profile.profile_image_override_url:
return profile.profile_image_override_url, "override"
if profile.profile_image_url:
return profile.profile_image_url, "scraped"
return None, "none"
def resolve_upload_file_path(*, upload_dir: str, relative_path: str) -> Path:
root = _ensure_upload_root(upload_dir, create=False)
return _resolve_upload_path(root, relative_path)
def scrape_state_hint(*, state: ParseState, state_reason: str) -> str | None:
if state not in {ParseState.NETWORK_ERROR, ParseState.BLOCKED_OR_CAPTCHA}:
return None
return STATE_REASON_HINTS.get(state_reason)
def _merge_warnings(base: list[str], extra: list[str]) -> list[str]:
if not extra:
return sorted(set(base))
return sorted(set(base + extra))
def _trim_author_search_result(
parsed: ParsedAuthorSearchPage,
*,
limit: int,
extra_warnings: list[str] | None = None,
state_reason_override: str | None = None,
) -> ParsedAuthorSearchPage:
return ParsedAuthorSearchPage(
state=parsed.state,
state_reason=state_reason_override or parsed.state_reason,
candidates=parsed.candidates[: max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))],
marker_counts=parsed.marker_counts,
warnings=_merge_warnings(parsed.warnings, extra_warnings or []),
)
def _policy_blocked_author_search_result(
*,
reason: str,
warning_codes: list[str],
limit: int,
) -> ParsedAuthorSearchPage:
_ = limit
return ParsedAuthorSearchPage(
state=ParseState.BLOCKED_OR_CAPTCHA,
state_reason=reason,
candidates=[],
marker_counts={},
warnings=_merge_warnings([], warning_codes),
)
def _cache_get_author_search_result(query_key: str, now_monotonic: float) -> _AuthorSearchCacheEntry | None:
entry = _AUTHOR_SEARCH_CACHE.get(query_key)
if entry is None:
return None
if entry.expires_at_monotonic <= now_monotonic:
_AUTHOR_SEARCH_CACHE.pop(query_key, None)
return None
_AUTHOR_SEARCH_CACHE.move_to_end(query_key)
return entry
def _cache_set_author_search_result(
*,
query_key: str,
parsed: ParsedAuthorSearchPage,
ttl_seconds: float,
max_entries: int,
) -> None:
ttl = max(float(ttl_seconds), 0.0)
if ttl <= 0.0:
_AUTHOR_SEARCH_CACHE.pop(query_key, None)
return
_AUTHOR_SEARCH_CACHE[query_key] = _AuthorSearchCacheEntry(
parsed=parsed,
expires_at_monotonic=time.monotonic() + ttl,
cached_at_utc=datetime.now(timezone.utc),
)
_AUTHOR_SEARCH_CACHE.move_to_end(query_key)
bounded_max_entries = max(1, int(max_entries))
while len(_AUTHOR_SEARCH_CACHE) > bounded_max_entries:
_AUTHOR_SEARCH_CACHE.popitem(last=False)
def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool:
return parsed.state == ParseState.BLOCKED_OR_CAPTCHA
def _author_search_cooldown_remaining_seconds(now_utc: datetime) -> int:
if _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC is None:
return 0
remaining_seconds = int((_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC - now_utc).total_seconds())
return max(0, remaining_seconds)
def _reset_author_search_runtime_state_for_tests() -> None:
global _AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC
global _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
global _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT
_AUTHOR_SEARCH_CACHE.clear()
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = 0.0
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC = None
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
async def list_scholars_for_user(
db_session: AsyncSession,
*,
@ -46,11 +306,13 @@ async def create_scholar_for_user(
user_id: int,
scholar_id: str,
display_name: str,
profile_image_url: str | None = None,
) -> ScholarProfile:
profile = ScholarProfile(
user_id=user_id,
scholar_id=validate_scholar_id(scholar_id),
display_name=normalize_display_name(display_name),
profile_image_url=normalize_profile_image_url(profile_image_url),
)
db_session.add(profile)
try:
@ -92,7 +354,293 @@ async def delete_scholar(
db_session: AsyncSession,
*,
profile: ScholarProfile,
upload_dir: str | None = None,
) -> None:
if upload_dir:
upload_root = _ensure_upload_root(upload_dir, create=True)
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
await db_session.delete(profile)
await db_session.commit()
async def search_author_candidates(
*,
source: ScholarSource,
query: str,
limit: int,
network_error_retries: int = 1,
retry_backoff_seconds: float = 1.0,
search_enabled: bool = True,
cache_ttl_seconds: int = 21_600,
blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS,
cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES,
min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS,
interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS,
cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD,
cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS,
) -> ParsedAuthorSearchPage:
global _AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC
global _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
global _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT
normalized_query = query.strip()
if len(normalized_query) < 2:
raise ScholarServiceError("Search query must be at least 2 characters.")
bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))
query_key = normalized_query.casefold()
if not search_enabled:
logger.warning(
"scholar_search.disabled_by_configuration",
extra={
"event": "scholar_search.disabled_by_configuration",
"query": normalized_query,
},
)
return _policy_blocked_author_search_result(
reason=SEARCH_DISABLED_REASON,
warning_codes=["author_search_disabled_by_configuration"],
limit=bounded_limit,
)
async with _AUTHOR_SEARCH_EXECUTION_LOCK:
now_utc = datetime.now(timezone.utc)
now_monotonic = time.monotonic()
cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(now_utc)
if cooldown_remaining_seconds > 0:
logger.warning(
"scholar_search.cooldown_active",
extra={
"event": "scholar_search.cooldown_active",
"query": normalized_query,
"cooldown_remaining_seconds": cooldown_remaining_seconds,
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat()
if _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
else None,
},
)
return _policy_blocked_author_search_result(
reason=SEARCH_COOLDOWN_REASON,
warning_codes=[
"author_search_cooldown_active",
f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
],
limit=bounded_limit,
)
cached_entry = _cache_get_author_search_result(query_key, now_monotonic)
if cached_entry is not None:
cached = cached_entry.parsed
state_reason_override = (
SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None
)
logger.info(
"scholar_search.cache_hit",
extra={
"event": "scholar_search.cache_hit",
"query": normalized_query,
"state": cached.state.value,
"state_reason": cached.state_reason,
},
)
return _trim_author_search_result(
cached,
limit=bounded_limit,
extra_warnings=["author_search_served_from_cache"],
state_reason_override=state_reason_override,
)
enforced_wait_seconds = (
(_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC + max(float(min_interval_seconds), 0.0))
- now_monotonic
)
jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0))
sleep_seconds = max(0.0, enforced_wait_seconds) + jitter_seconds
if sleep_seconds > 0.0:
logger.info(
"scholar_search.throttle_wait",
extra={
"event": "scholar_search.throttle_wait",
"query": normalized_query,
"sleep_seconds": round(sleep_seconds, 3),
},
)
await asyncio.sleep(sleep_seconds)
max_attempts = max(1, int(network_error_retries) + 1)
parsed: ParsedAuthorSearchPage | None = None
retry_warnings: list[str] = []
for attempt_index in range(max_attempts):
fetch_result = await source.fetch_author_search_html(normalized_query, start=0)
parsed = parse_author_search_page(fetch_result)
if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1:
break
retry_warnings.append("network_retry_scheduled_for_author_search")
sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index)
if sleep_seconds > 0:
await asyncio.sleep(sleep_seconds)
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = time.monotonic()
if parsed is None:
raise ScholarServiceError("Unable to complete scholar author search.")
merged_parsed = replace(
parsed,
warnings=_merge_warnings(parsed.warnings, retry_warnings),
)
if _is_author_search_block_state(merged_parsed):
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT += 1
logger.warning(
"scholar_search.block_detected",
extra={
"event": "scholar_search.block_detected",
"query": normalized_query,
"state_reason": merged_parsed.state_reason,
"consecutive_blocked_count": _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT,
},
)
if _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT >= max(1, int(cooldown_block_threshold)):
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC = datetime.now(timezone.utc) + timedelta(
seconds=max(60, int(cooldown_seconds))
)
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
merged_parsed = replace(
merged_parsed,
warnings=_merge_warnings(
merged_parsed.warnings,
["author_search_circuit_breaker_armed"],
),
)
logger.error(
"scholar_search.cooldown_activated",
extra={
"event": "scholar_search.cooldown_activated",
"query": normalized_query,
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat(),
},
)
else:
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
ttl_seconds = (
min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds)))
if _is_author_search_block_state(merged_parsed)
else max(1, int(cache_ttl_seconds))
)
_cache_set_author_search_result(
query_key=query_key,
parsed=merged_parsed,
ttl_seconds=float(ttl_seconds),
max_entries=cache_max_entries,
)
return _trim_author_search_result(
merged_parsed,
limit=bounded_limit,
)
async def hydrate_profile_metadata(
db_session: AsyncSession,
*,
profile: ScholarProfile,
source: ScholarSource,
) -> ScholarProfile:
fetch_result = await source.fetch_profile_html(profile.scholar_id)
parsed_page = parse_profile_page(fetch_result)
if parsed_page.profile_name and not (profile.display_name or "").strip():
profile.display_name = parsed_page.profile_name
if parsed_page.profile_image_url and not profile.profile_image_url:
profile.profile_image_url = parsed_page.profile_image_url
await db_session.commit()
await db_session.refresh(profile)
return profile
async def set_profile_image_override_url(
db_session: AsyncSession,
*,
profile: ScholarProfile,
image_url: str | None,
upload_dir: str,
) -> ScholarProfile:
upload_root = _ensure_upload_root(upload_dir, create=True)
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
profile.profile_image_upload_path = None
profile.profile_image_override_url = normalize_profile_image_url(image_url)
await db_session.commit()
await db_session.refresh(profile)
return profile
async def clear_profile_image_customization(
db_session: AsyncSession,
*,
profile: ScholarProfile,
upload_dir: str,
) -> ScholarProfile:
upload_root = _ensure_upload_root(upload_dir, create=True)
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
profile.profile_image_upload_path = None
profile.profile_image_override_url = None
await db_session.commit()
await db_session.refresh(profile)
return profile
async def set_profile_image_upload(
db_session: AsyncSession,
*,
profile: ScholarProfile,
content_type: str | None,
image_bytes: bytes,
upload_dir: str,
max_upload_bytes: int,
) -> ScholarProfile:
normalized_content_type = (content_type or "").strip().lower()
extension = ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES.get(normalized_content_type)
if extension is None:
raise ScholarServiceError(
"Unsupported image type. Use JPEG, PNG, WEBP, or GIF."
)
if not image_bytes:
raise ScholarServiceError("Uploaded image file is empty.")
if len(image_bytes) > max_upload_bytes:
raise ScholarServiceError(
f"Uploaded image exceeds {max_upload_bytes} bytes."
)
upload_root = _ensure_upload_root(upload_dir, create=True)
user_dir = upload_root / str(profile.user_id)
user_dir.mkdir(parents=True, exist_ok=True)
filename = f"{profile.id}_{uuid4().hex}{extension}"
relative_path = os.path.join(str(profile.user_id), filename)
absolute_path = _resolve_upload_path(upload_root, relative_path)
absolute_path.write_bytes(image_bytes)
old_path = profile.profile_image_upload_path
profile.profile_image_upload_path = relative_path
profile.profile_image_override_url = None
await db_session.commit()
await db_session.refresh(profile)
if old_path and old_path != relative_path:
_safe_remove_upload(upload_root, old_path)
return profile

41
app/settings.py
View file

@ -48,7 +48,7 @@ class Settings:
log_format: str = _env_str("LOG_FORMAT", "console")
log_requests: bool = _env_bool("LOG_REQUESTS", True)
log_uvicorn_access: bool = _env_bool("LOG_UVICORN_ACCESS", False)
log_request_skip_paths: str = _env_str("LOG_REQUEST_SKIP_PATHS", "/healthz,/static/")
log_request_skip_paths: str = _env_str("LOG_REQUEST_SKIP_PATHS", "/healthz")
log_redact_fields: str = os.getenv("LOG_REDACT_FIELDS", "")
scheduler_enabled: bool = _env_bool("SCHEDULER_ENABLED", True)
scheduler_tick_seconds: int = _env_int("SCHEDULER_TICK_SECONDS", 60)
@ -79,6 +79,45 @@ class Settings:
6,
)
scheduler_queue_batch_size: int = _env_int("SCHEDULER_QUEUE_BATCH_SIZE", 10)
frontend_enabled: bool = _env_bool("FRONTEND_ENABLED", True)
frontend_dist_dir: str = _env_str("FRONTEND_DIST_DIR", "/app/frontend/dist")
scholar_image_upload_dir: str = _env_str(
"SCHOLAR_IMAGE_UPLOAD_DIR",
"/var/lib/scholarr/uploads",
)
scholar_image_upload_max_bytes: int = _env_int(
"SCHOLAR_IMAGE_UPLOAD_MAX_BYTES",
2_000_000,
)
scholar_name_search_enabled: bool = _env_bool("SCHOLAR_NAME_SEARCH_ENABLED", True)
scholar_name_search_cache_ttl_seconds: int = _env_int(
"SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS",
21_600,
)
scholar_name_search_blocked_cache_ttl_seconds: int = _env_int(
"SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS",
300,
)
scholar_name_search_cache_max_entries: int = _env_int(
"SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES",
512,
)
scholar_name_search_min_interval_seconds: float = _env_float(
"SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS",
3.0,
)
scholar_name_search_interval_jitter_seconds: float = _env_float(
"SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS",
1.0,
)
scholar_name_search_cooldown_block_threshold: int = _env_int(
"SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD",
1,
)
scholar_name_search_cooldown_seconds: int = _env_int(
"SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS",
1800,
)
settings = Settings()

501
app/static/app.css
View file

@ -1,501 +0,0 @@
* {
box-sizing: border-box;
}
body {
margin: 0;
min-height: 100vh;
color: var(--md-sys-color-on-surface);
font-family: var(--font-body);
background:
radial-gradient(circle at 10% 5%, var(--md-sys-color-primary-container) 0%, transparent 30%),
radial-gradient(circle at 90% 0%, var(--md-sys-color-secondary-container) 0%, transparent 28%),
var(--md-sys-color-surface);
}
.loading-indicator {
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 3px;
z-index: 40;
transform: scaleX(0);
transform-origin: left;
background: linear-gradient(90deg, var(--md-sys-color-primary), var(--md-sys-color-tertiary));
opacity: 0;
}
body.is-page-loading .loading-indicator {
opacity: 1;
animation: loading-bar 1.2s ease-in-out infinite;
}
.app-backdrop {
position: fixed;
inset: 0;
pointer-events: none;
background:
linear-gradient(130deg, var(--md-sys-color-primary-tint), transparent 40%),
linear-gradient(230deg, var(--md-sys-color-surface-tint), transparent 45%);
opacity: 0.28;
}
.top-app-bar {
position: sticky;
top: 0;
z-index: 10;
display: flex;
align-items: center;
gap: 0.8rem;
flex-wrap: wrap;
padding: 0.8rem 1.1rem;
border-bottom: 1px solid var(--md-sys-color-outline-variant);
backdrop-filter: blur(9px);
background: color-mix(in srgb, var(--md-sys-color-surface) 88%, white 12%);
}
.brand-wrap {
display: flex;
align-items: center;
gap: 0.65rem;
}
.brand {
color: var(--md-sys-color-primary);
text-decoration: none;
font-family: var(--font-heading);
font-weight: 700;
font-size: 1.2rem;
letter-spacing: 0.02em;
}
.session-user {
margin: 0;
padding: 0.24rem 0.58rem;
border-radius: 999px;
border: 1px solid var(--md-sys-color-outline-variant);
background: var(--md-sys-color-surface-container-high);
color: var(--md-sys-color-on-surface-variant);
font-size: 0.78rem;
}
.site-nav {
display: flex;
align-items: center;
gap: 0.35rem;
flex-wrap: wrap;
}
.nav-link {
color: var(--md-sys-color-on-surface-variant);
text-decoration: none;
padding: 0.42rem 0.78rem;
border-radius: 999px;
border: 1px solid transparent;
font-size: 0.92rem;
}
.nav-link:hover {
background: var(--md-sys-color-surface-container);
border-color: var(--md-sys-color-outline-variant);
}
.nav-link.is-active {
color: var(--md-sys-color-on-secondary-container);
background: var(--md-sys-color-secondary-container);
border-color: transparent;
}
.header-actions {
margin-left: auto;
display: flex;
align-items: center;
gap: 0.7rem;
}
.theme-control-wrap {
display: flex;
align-items: center;
gap: 0.45rem;
}
.theme-control-wrap label {
color: var(--md-sys-color-on-surface-variant);
font-size: 0.82rem;
}
.theme-control {
border: 1px solid var(--md-sys-color-outline);
border-radius: 999px;
background: var(--md-sys-color-surface-container-low);
color: var(--md-sys-color-on-surface);
padding: 0.32rem 0.64rem;
font: inherit;
}
.logout-form {
margin: 0;
}
.page-shell {
position: relative;
display: grid;
gap: 0.9rem;
width: min(1100px, calc(100vw - 1.4rem));
margin: 0 auto;
padding: 1.1rem 0 1.6rem;
}
.flash {
margin: 0;
padding: 0.72rem 0.82rem;
border-radius: 14px;
border: 1px solid var(--md-sys-color-outline-variant);
background: var(--md-sys-color-surface-container);
}
.flash-notice {
color: var(--md-sys-color-on-tertiary-container);
border-color: var(--md-sys-color-tertiary-container);
background: color-mix(in srgb, var(--md-sys-color-tertiary-container) 30%, white 70%);
}
.flash-error {
color: var(--md-sys-color-on-error-container);
border-color: var(--md-sys-color-error-container);
background: color-mix(in srgb, var(--md-sys-color-error-container) 25%, white 75%);
}
.hero,
.sandbox {
border-radius: 24px;
border: 1px solid var(--md-sys-color-outline-variant);
background: var(--md-sys-color-surface-container-low);
box-shadow: var(--elevation-card);
}
.hero {
padding: 1.3rem;
}
.sandbox {
padding: 1.05rem;
}
.eyebrow {
margin: 0;
font-size: 0.78rem;
color: var(--md-sys-color-primary);
text-transform: uppercase;
letter-spacing: 0.11em;
font-weight: 600;
}
h1 {
margin: 0.28rem 0 0.5rem;
color: var(--md-sys-color-on-surface);
font-family: var(--font-heading);
font-size: clamp(1.45rem, 3vw, 2.05rem);
line-height: 1.15;
}
h2 {
margin: 0;
color: var(--md-sys-color-on-surface);
font-family: var(--font-heading);
font-size: 1.08rem;
}
.lede {
margin: 0;
color: var(--md-sys-color-on-surface-variant);
max-width: 68ch;
}
.section-heading {
display: flex;
align-items: center;
justify-content: space-between;
gap: 0.55rem;
margin-bottom: 0.7rem;
flex-wrap: wrap;
}
.stat-strip {
margin-top: 0.95rem;
display: grid;
gap: 0.75rem;
grid-template-columns: repeat(auto-fit, minmax(170px, 1fr));
}
.stat-item {
margin: 0;
padding: 0.72rem 0.78rem;
border-radius: 16px;
border: 1px solid var(--md-sys-color-outline-variant);
background: var(--md-sys-color-surface-container);
}
.stat-label {
margin: 0;
color: var(--md-sys-color-on-surface-variant);
font-size: 0.8rem;
}
.stat-value {
margin: 0.2rem 0 0;
color: var(--md-sys-color-on-surface);
font-size: 1.36rem;
font-weight: 700;
}
form {
display: grid;
gap: 0.45rem;
}
.stack-form {
max-width: 560px;
}
label {
font-size: 0.9rem;
color: var(--md-sys-color-on-surface-variant);
}
input,
select {
width: 100%;
border: 1px solid var(--md-sys-color-outline);
border-radius: 12px;
padding: 0.56rem 0.64rem;
font: inherit;
color: var(--md-sys-color-on-surface);
background: var(--md-sys-color-surface);
}
input:focus,
select:focus {
outline: 2px solid color-mix(in srgb, var(--md-sys-color-primary) 35%, transparent);
border-color: var(--md-sys-color-primary);
}
.checkbox-row {
display: flex;
align-items: center;
gap: 0.54rem;
}
.checkbox-row input {
width: auto;
}
button,
.button {
border: 0;
border-radius: 999px;
padding: 0.54rem 0.86rem;
font: inherit;
font-weight: 600;
color: var(--md-sys-color-on-primary);
background: var(--md-sys-color-primary);
cursor: pointer;
}
button:hover,
.button:hover {
filter: brightness(0.97);
}
button:disabled,
.button:disabled {
cursor: not-allowed;
opacity: 0.5;
}
.button-text {
color: var(--md-sys-color-primary);
background: transparent;
border: 1px solid var(--md-sys-color-outline-variant);
}
.danger-button {
color: var(--md-sys-color-on-error);
background: var(--md-sys-color-error);
}
.inline-actions {
display: flex;
align-items: center;
flex-wrap: wrap;
gap: 0.45rem;
}
.inline-actions form {
display: inline-flex;
gap: 0.42rem;
align-items: center;
}
.inline-reset-form input {
min-width: 160px;
}
form.is-loading button[type="submit"],
form.is-loading input[type="submit"] {
opacity: 0.75;
}
table {
width: 100%;
border-collapse: collapse;
border-radius: 12px;
overflow: hidden;
background: var(--md-sys-color-surface);
}
th,
td {
padding: 0.54rem;
text-align: left;
border-bottom: 1px solid var(--md-sys-color-outline-variant);
vertical-align: top;
}
thead th {
color: var(--md-sys-color-on-surface-variant);
font-size: 0.84rem;
font-weight: 600;
background: var(--md-sys-color-surface-container-high);
}
tbody tr:last-child td {
border-bottom: 0;
}
a {
color: var(--md-sys-color-primary);
}
.link-button {
display: inline-flex;
align-items: center;
padding: 0.45rem 0.78rem;
border: 1px solid var(--md-sys-color-outline-variant);
border-radius: 999px;
color: var(--md-sys-color-primary);
text-decoration: none;
background: var(--md-sys-color-surface-container-low);
}
.link-button.is-active {
color: var(--md-sys-color-on-secondary-container);
background: var(--md-sys-color-secondary-container);
border-color: transparent;
}
.badge {
display: inline-block;
border-radius: 999px;
padding: 0.16rem 0.48rem;
font-size: 0.76rem;
text-transform: lowercase;
border: 1px solid var(--md-sys-color-outline-variant);
}
.badge.ok {
color: var(--md-sys-color-on-tertiary-container);
background: var(--md-sys-color-tertiary-container);
border-color: transparent;
}
.badge.warn,
.badge.danger {
color: var(--md-sys-color-on-error-container);
background: var(--md-sys-color-error-container);
border-color: transparent;
}
.helper-text {
margin: 0.24rem 0 0;
color: var(--md-sys-color-on-surface-variant);
font-size: 0.82rem;
}
.auth-card {
max-width: 520px;
}
.auth-form {
max-width: 360px;
}
.form-error {
margin: 0.5rem 0 0;
color: var(--md-sys-color-error);
font-weight: 600;
}
.form-note {
margin: 0.35rem 0 0;
color: var(--md-sys-color-on-surface-variant);
}
code {
font-family: var(--font-code);
background: var(--md-sys-color-surface-container-high);
border-radius: 6px;
padding: 0.1rem 0.25rem;
}
pre {
margin: 0.6rem 0 0;
border-radius: 12px;
border: 1px solid var(--md-sys-color-outline-variant);
background: var(--md-sys-color-surface-container-high);
color: var(--md-sys-color-on-surface);
padding: 0.75rem;
white-space: pre-wrap;
word-break: break-word;
}
@media (max-width: 880px) {
.top-app-bar {
padding: 0.7rem 0.8rem;
}
.header-actions {
width: 100%;
justify-content: space-between;
margin-left: 0;
}
.page-shell {
width: min(100vw, calc(100vw - 0.9rem));
padding-top: 0.8rem;
}
.hero,
.sandbox {
border-radius: 18px;
padding: 0.9rem;
}
.inline-actions form {
width: 100%;
justify-content: flex-start;
}
}
@keyframes loading-bar {
0% {
transform: scaleX(0.1);
}
50% {
transform: scaleX(0.7);
}
100% {
transform: scaleX(1);
}
}

76
app/static/app.js
View file

@ -1,76 +0,0 @@
(() => {
const body = document.body;
if (!body) {
return;
}
const defaultLoadingText = "Working...";
const shouldHandleAnchor = (anchor) => {
if (!anchor || !anchor.getAttribute) {
return false;
}
const href = anchor.getAttribute("href");
if (!href) {
return false;
}
if (href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("javascript:")) {
return false;
}
if (anchor.hasAttribute("download")) {
return false;
}
if ((anchor.getAttribute("target") || "").toLowerCase() === "_blank") {
return false;
}
return true;
};
document.addEventListener("click", (event) => {
const target = event.target;
if (!(target instanceof Element)) {
return;
}
const anchor = target.closest("a");
if (!shouldHandleAnchor(anchor)) {
return;
}
body.classList.add("is-page-loading");
});
document.addEventListener("submit", (event) => {
const form = event.target;
if (!(form instanceof HTMLFormElement)) {
return;
}
if (form.dataset.noLoading === "1") {
return;
}
body.classList.add("is-page-loading");
form.classList.add("is-loading");
const submitElements = form.querySelectorAll("button[type='submit'], input[type='submit']");
submitElements.forEach((element) => {
if (element.hasAttribute("disabled")) {
return;
}
element.setAttribute("disabled", "disabled");
if (element instanceof HTMLInputElement) {
const loadingText = element.dataset.loadingText || defaultLoadingText;
element.dataset.originalValue = element.value;
element.value = loadingText;
return;
}
if (element instanceof HTMLButtonElement) {
const loadingText = element.dataset.loadingText || defaultLoadingText;
element.dataset.originalText = element.textContent || "";
element.textContent = loadingText;
}
});
});
window.addEventListener("pageshow", () => {
body.classList.remove("is-page-loading");
});
})();

98
app/static/theme.css
View file

@ -1,98 +0,0 @@
:root {
--font-body: "Plus Jakarta Sans", "Segoe UI", sans-serif;
--font-heading: "DM Serif Text", "Georgia", serif;
--font-code: "JetBrains Mono", "Fira Code", monospace;
--md-sys-color-primary: #73594d;
--md-sys-color-on-primary: #ffffff;
--md-sys-color-primary-container: #fbded0;
--md-sys-color-on-primary-container: #2b160f;
--md-sys-color-primary-tint: rgba(115, 89, 77, 0.18);
--md-sys-color-secondary: #6f5b52;
--md-sys-color-on-secondary: #ffffff;
--md-sys-color-secondary-container: #f8ddd2;
--md-sys-color-on-secondary-container: #281913;
--md-sys-color-tertiary: #5b6550;
--md-sys-color-on-tertiary: #ffffff;
--md-sys-color-tertiary-container: #ddebcf;
--md-sys-color-on-tertiary-container: #17200f;
--md-sys-color-error: #b3261e;
--md-sys-color-on-error: #ffffff;
--md-sys-color-error-container: #f9dedc;
--md-sys-color-on-error-container: #410e0b;
--md-sys-color-surface: #fff8f5;
--md-sys-color-surface-tint: rgba(111, 87, 74, 0.12);
--md-sys-color-surface-container-low: #fffaf8;
--md-sys-color-surface-container: #fdf2ec;
--md-sys-color-surface-container-high: #f8e9e1;
--md-sys-color-on-surface: #221a17;
--md-sys-color-on-surface-variant: #55423a;
--md-sys-color-outline: #85736b;
--md-sys-color-outline-variant: #d8c2b8;
--elevation-card: 0 10px 30px rgba(55, 39, 28, 0.08);
}
:root[data-theme="fjord"] {
--md-sys-color-primary: #2f6678;
--md-sys-color-on-primary: #ffffff;
--md-sys-color-primary-container: #cde9f4;
--md-sys-color-on-primary-container: #051f28;
--md-sys-color-primary-tint: rgba(47, 102, 120, 0.2);
--md-sys-color-secondary: #4d626b;
--md-sys-color-on-secondary: #ffffff;
--md-sys-color-secondary-container: #d0e6f2;
--md-sys-color-on-secondary-container: #091f27;
--md-sys-color-tertiary: #456179;
--md-sys-color-on-tertiary: #ffffff;
--md-sys-color-tertiary-container: #d2e5ff;
--md-sys-color-on-tertiary-container: #031d33;
--md-sys-color-surface: #f5fbff;
--md-sys-color-surface-tint: rgba(47, 102, 120, 0.12);
--md-sys-color-surface-container-low: #fcfeff;
--md-sys-color-surface-container: #ecf5fa;
--md-sys-color-surface-container-high: #dfeef6;
--md-sys-color-on-surface: #172126;
--md-sys-color-on-surface-variant: #3f4f57;
--md-sys-color-outline: #6f7f88;
--md-sys-color-outline-variant: #becfd8;
--elevation-card: 0 10px 30px rgba(22, 43, 53, 0.09);
}
:root[data-theme="spruce"] {
--md-sys-color-primary: #39684c;
--md-sys-color-on-primary: #ffffff;
--md-sys-color-primary-container: #bcefc9;
--md-sys-color-on-primary-container: #062111;
--md-sys-color-primary-tint: rgba(57, 104, 76, 0.18);
--md-sys-color-secondary: #4f6354;
--md-sys-color-on-secondary: #ffffff;
--md-sys-color-secondary-container: #d2e8d4;
--md-sys-color-on-secondary-container: #0d1f12;
--md-sys-color-tertiary: #3f655f;
--md-sys-color-on-tertiary: #ffffff;
--md-sys-color-tertiary-container: #c2ece4;
--md-sys-color-on-tertiary-container: #021f1a;
--md-sys-color-surface: #f5fbf5;
--md-sys-color-surface-tint: rgba(57, 104, 76, 0.1);
--md-sys-color-surface-container-low: #fcfffb;
--md-sys-color-surface-container: #ebf4ea;
--md-sys-color-surface-container-high: #deecdd;
--md-sys-color-on-surface: #172119;
--md-sys-color-on-surface-variant: #415246;
--md-sys-color-outline: #708174;
--md-sys-color-outline-variant: #c1d3c2;
--elevation-card: 0 10px 30px rgba(24, 48, 32, 0.08);
}

56
app/static/theme.js
View file

@ -1,56 +0,0 @@
(() => {
const STORAGE_KEY = "scholarr_theme";
const LEGACY_STORAGE_KEY = "scholar_tracker_theme";
const root = document.documentElement;
const themeControl = document.querySelector("[data-theme-control]");
if (!root) {
return;
}
const defaultTheme = root.dataset.defaultTheme || "terracotta";
const supportedThemes = themeControl
? Array.from(themeControl.options).map((option) => option.value)
: [];
const isSupported = (theme) =>
supportedThemes.length === 0 || supportedThemes.includes(theme);
const applyTheme = (theme) => {
if (!isSupported(theme)) {
return;
}
root.dataset.theme = theme;
};
let activeTheme = defaultTheme;
try {
const savedTheme = window.localStorage.getItem(STORAGE_KEY);
const legacyTheme = window.localStorage.getItem(LEGACY_STORAGE_KEY);
if (savedTheme && isSupported(savedTheme)) {
activeTheme = savedTheme;
} else if (legacyTheme && isSupported(legacyTheme)) {
activeTheme = legacyTheme;
window.localStorage.setItem(STORAGE_KEY, legacyTheme);
}
} catch {
activeTheme = defaultTheme;
}
applyTheme(activeTheme);
if (!themeControl) {
return;
}
themeControl.value = activeTheme;
themeControl.addEventListener("change", (event) => {
const selectedTheme = event.target.value;
applyTheme(selectedTheme);
try {
window.localStorage.setItem(STORAGE_KEY, selectedTheme);
} catch {
// Ignore storage errors in locked-down browsers.
}
});
})();

23
app/templates/account_password.html
View file

@ -1,23 +0,0 @@
{% extends "base.html" %}
{% block content %}
<section class="hero" data-test="account-password-page">
<p class="eyebrow">Account</p>
<h1>Change Password</h1>
<p class="lede">Use a strong password. This updates only your own account.</p>
</section>
<section class="sandbox" data-test="account-password-form-card">
<h2>Password Update</h2>
<form method="post" action="/account/password" class="stack-form">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<label for="current_password">Current Password</label>
<input id="current_password" name="current_password" type="password" autocomplete="current-password" required>
<label for="new_password">New Password</label>
<input id="new_password" name="new_password" type="password" autocomplete="new-password" minlength="8" required>
<label for="confirm_password">Confirm New Password</label>
<input id="confirm_password" name="confirm_password" type="password" autocomplete="new-password" minlength="8" required>
<button type="submit" data-loading-text="Updating...">Update Password</button>
</form>
</section>
{% endblock %}

66
app/templates/base.html
View file

@ -1,66 +0,0 @@
<!doctype html>
<html lang="en" data-theme="{{ theme_name | default('terracotta') }}" data-default-theme="{{ theme_name | default('terracotta') }}">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{{ page_title }} | scholarr</title>
<link rel="stylesheet" href="{{ url_for('static', path='theme.css') }}">
<link rel="stylesheet" href="{{ url_for('static', path='app.css') }}">
</head>
<body>
<div class="loading-indicator" data-loading-indicator aria-hidden="true"></div>
<div class="app-backdrop" aria-hidden="true"></div>
<header class="top-app-bar">
<div class="brand-wrap">
<a class="brand" href="/">scholarr</a>
{% if session_user %}
<p class="session-user" data-test="session-user">{{ session_user.email }}</p>
{% endif %}
</div>
<nav class="site-nav" aria-label="Primary">
{% if session_user %}
<a class="nav-link {% if active_nav == 'home' %}is-active{% endif %}" href="/">Dashboard</a>
<a class="nav-link {% if active_nav == 'scholars' %}is-active{% endif %}" href="/scholars">Scholars</a>
<a class="nav-link {% if active_nav == 'publications' %}is-active{% endif %}" href="/publications?mode=new">Publications</a>
<a class="nav-link {% if active_nav == 'runs' %}is-active{% endif %}" href="/runs">Runs</a>
<a class="nav-link {% if active_nav == 'settings' %}is-active{% endif %}" href="/settings">Settings</a>
<a class="nav-link {% if active_nav == 'account_password' %}is-active{% endif %}" href="/account/password">Password</a>
{% if session_user.is_admin %}
<a class="nav-link {% if active_nav == 'users' %}is-active{% endif %}" href="/users">Users</a>
{% endif %}
{% else %}
<a class="nav-link {% if active_nav == 'login' %}is-active{% endif %}" href="/login">Login</a>
{% endif %}
</nav>
<div class="header-actions">
<div class="theme-control-wrap">
<label for="theme-control">Theme</label>
<select id="theme-control" class="theme-control" data-theme-control aria-label="Color theme">
{% for theme in themes %}
<option value="{{ theme.slug }}" {% if theme.slug == theme_name %}selected{% endif %}>{{ theme.label }}</option>
{% endfor %}
</select>
</div>
{% if session_user %}
<form method="post" action="/logout" class="logout-form">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<button type="submit" class="button button-text" data-loading-text="Logging out...">Log out</button>
</form>
{% endif %}
</div>
</header>
<main class="page-shell">
{% if notice %}
<p class="flash flash-notice" data-test="flash-notice">{{ notice }}</p>
{% endif %}
{% if page_error %}
<p class="flash flash-error" data-test="flash-error">{{ page_error }}</p>
{% endif %}
{% block content %}{% endblock %}
</main>
<script defer src="{{ url_for('static', path='app.js') }}"></script>
<script defer src="{{ url_for('static', path='theme.js') }}"></script>
</body>
</html>

23
app/templates/dashboard/_run_controls.html
View file

@ -1,23 +0,0 @@
<section class="sandbox" data-test="dashboard-run-controls">
<div class="section-heading">
<h2>Run Tracking</h2>
<div class="inline-actions">
<a class="link-button" href="/publications?mode=new">New Publications</a>
<a class="link-button" href="/publications?mode=all">All Publications</a>
<a class="link-button" href="/runs?failed_only=1">Run Diagnostics</a>
<a class="link-button" href="/settings">Automation Settings</a>
<a class="link-button" href="/scholars">Manage Scholars</a>
</div>
</div>
<p class="lede">Manual runs use your request delay of {{ dashboard.run_controls.request_delay_seconds }} second(s).</p>
<p class="helper-text">
Queue state:
{{ dashboard.run_controls.queue_queued_count }} queued,
{{ dashboard.run_controls.queue_retrying_count }} retrying,
{{ dashboard.run_controls.queue_dropped_count }} dropped.
</p>
<form method="post" action="{{ dashboard.run_controls.run_manual_action }}" class="inline-actions">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<button type="submit" data-loading-text="Running...">Run Now</button>
</form>
</section>

38
app/templates/dashboard/_run_history.html
View file

@ -1,38 +0,0 @@
<section class="sandbox" data-test="dashboard-run-history">
<div class="section-heading">
<h2>Run History</h2>
<div class="inline-actions">
<a class="link-button" href="/runs">View All Runs</a>
</div>
</div>
{% if dashboard.run_history %}
<table>
<thead>
<tr>
<th>Started</th>
<th>Status</th>
<th>Scholars</th>
<th>New Publications</th>
</tr>
</thead>
<tbody>
{% for run in dashboard.run_history %}
<tr>
<td>
{% if run.detail_url %}
<a href="{{ run.detail_url }}">{{ run.started_at_display }}</a>
{% else %}
{{ run.started_at_display }}
{% endif %}
</td>
<td><span class="badge {{ run.status_badge }}">{{ run.status_label }}</span></td>
<td>{{ run.scholar_count }}</td>
<td>{{ run.new_publication_count }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p>No runs yet.</p>
{% endif %}
</section>

47
app/templates/dashboard/_unread_publications.html
View file

@ -1,47 +0,0 @@
<section class="sandbox" data-test="dashboard-unread-publications">
<div class="section-heading">
<h2>New Since Last Run</h2>
<div class="inline-actions">
<a class="link-button" href="/publications?mode=all">View All</a>
<form method="post" action="{{ dashboard.run_controls.mark_all_read_action }}">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<input type="hidden" name="return_to" value="/">
<button type="submit" data-loading-text="Marking...">Mark All Read</button>
</form>
</div>
</div>
{% if dashboard.unread_publications %}
<table>
<thead>
<tr>
<th>Title</th>
<th>Scholar</th>
<th>Year</th>
<th>Citations</th>
</tr>
</thead>
<tbody>
{% for item in dashboard.unread_publications %}
<tr>
<td>
{% if item.pub_url %}
<a href="{{ item.pub_url }}" target="_blank" rel="noreferrer">{{ item.title }}</a>
{% else %}
{{ item.title }}
{% endif %}
{% if item.venue_text %}
<p class="helper-text">{{ item.venue_text }}</p>
{% endif %}
</td>
<td>{{ item.scholar_label }}</td>
<td>{{ item.year_display }}</td>
<td>{{ item.citation_count }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p>No new publications in the latest run.</p>
{% endif %}
</section>

27
app/templates/index.html
View file

@ -1,27 +0,0 @@
{% extends "base.html" %}
{% block content %}
<section class="hero" data-test="home-hero">
<p class="eyebrow">Dashboard</p>
<h1>Keep up with your tracked scholars</h1>
<p class="lede">Run ingestion, review fresh publications, and inspect recent outcomes from one screen.</p>
<div class="stat-strip">
<article class="stat-item">
<p class="stat-label">New Since Last Run</p>
<p class="stat-value">{{ dashboard.unread_publications | length }}</p>
</article>
<article class="stat-item">
<p class="stat-label">Recent Runs</p>
<p class="stat-value">{{ dashboard.run_history | length }}</p>
</article>
<article class="stat-item">
<p class="stat-label">Automation Delay</p>
<p class="stat-value">{{ dashboard.run_controls.request_delay_seconds }}s</p>
</article>
</div>
</section>
{% for section_template in dashboard.section_templates %}
{% include section_template %}
{% endfor %}
{% endblock %}

23
app/templates/login.html
View file

@ -1,23 +0,0 @@
{% extends "base.html" %}
{% block content %}
<section class="hero auth-card" data-test="login-card">
<p class="eyebrow">Secure Access</p>
<h1>Sign in</h1>
<p class="lede">Use an admin-created account to access your tracked scholars and settings.</p>
{% if error_message %}
<p class="form-error" role="alert">{{ error_message }}</p>
{% endif %}
{% if retry_after_seconds %}
<p class="form-note">Retry after {{ retry_after_seconds }} seconds.</p>
{% endif %}
<form method="post" action="/login" class="auth-form" data-test="login-form">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<label for="email">Email</label>
<input id="email" name="email" type="email" autocomplete="username" required autofocus>
<label for="password">Password</label>
<input id="password" name="password" type="password" autocomplete="current-password" required>
<button type="submit" data-loading-text="Signing in...">Sign in</button>
</form>
</section>
{% endblock %}

104
app/templates/publications.html
View file

@ -1,104 +0,0 @@
{% extends "base.html" %}
{% block content %}
<section class="hero" data-test="publications-page">
<p class="eyebrow">Publications</p>
<h1>Publications</h1>
<p class="lede">
Browse publications discovered in the latest run or your complete tracked library.
{% if selected_scholar %}
Current scholar filter: <strong>{{ selected_scholar.display_name or selected_scholar.scholar_id }}</strong>.
{% endif %}
</p>
<div class="inline-actions">
<a class="link-button {% if mode == 'new' %}is-active{% endif %}" href="{{ mode_new_url }}">New Since Last Run ({{ new_count }})</a>
<a class="link-button {% if mode == 'all' %}is-active{% endif %}" href="{{ mode_all_url }}">All ({{ total_count }})</a>
<a class="link-button" href="/scholars">Manage Scholars</a>
<a class="link-button" href="/runs?failed_only=1">Run Diagnostics</a>
{% if mode == 'new' and new_count > 0 %}
<form method="post" action="/publications/mark-all-read">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<input type="hidden" name="return_to" value="{{ current_publications_url }}">
<button type="submit" data-loading-text="Marking...">Mark All Read</button>
</form>
{% endif %}
</div>
</section>
<section class="sandbox" data-test="publications-table">
<div class="section-heading">
<h2>Filter</h2>
</div>
<form method="get" action="/publications" class="inline-actions">
<label for="mode">Mode</label>
<select id="mode" name="mode">
<option value="new" {% if mode == 'new' %}selected{% endif %}>New Since Last Run</option>
<option value="all" {% if mode == 'all' %}selected{% endif %}>All Publications</option>
</select>
<label for="scholar_profile_id">Scholar</label>
<select id="scholar_profile_id" name="scholar_profile_id">
<option value="">All Scholars</option>
{% for scholar in scholars %}
<option value="{{ scholar.id }}" {% if selected_scholar_id == scholar.id %}selected{% endif %}>
{{ scholar.display_name or scholar.scholar_id }}
</option>
{% endfor %}
</select>
<button type="submit" data-loading-text="Filtering...">Apply</button>
</form>
<div class="section-heading">
<h2>Results</h2>
</div>
{% if publications %}
<table>
<thead>
<tr>
<th>Title</th>
<th>Scholar</th>
<th>Year</th>
<th>Citations</th>
<th>New This Run</th>
<th>Status</th>
</tr>
</thead>
<tbody>
{% for item in publications %}
<tr>
<td>
{% if item.pub_url %}
<a href="{{ item.pub_url }}" target="_blank" rel="noreferrer">{{ item.title }}</a>
{% else %}
{{ item.title }}
{% endif %}
{% if item.venue_text %}
<p class="helper-text">{{ item.venue_text }}</p>
{% endif %}
</td>
<td>{{ item.scholar_label }}</td>
<td>{{ item.year if item.year is not none else "-" }}</td>
<td>{{ item.citation_count }}</td>
<td>
{% if item.is_new_in_latest_run %}
<span class="badge warn">new</span>
{% else %}
<span class="badge">older</span>
{% endif %}
</td>
<td>
{% if item.is_read %}
<span class="badge ok">read</span>
{% else %}
<span class="badge warn">new</span>
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p>No publications match this mode yet.</p>
{% endif %}
</section>
{% endblock %}

98
app/templates/run_detail.html
View file

@ -1,98 +0,0 @@
{% extends "base.html" %}
{% block content %}
<section class="hero" data-test="run-detail-page">
<p class="eyebrow">Diagnostics</p>
<h1>Run #{{ run.id }}</h1>
<p class="lede">Detailed state and failure context for this execution.</p>
</section>
<section class="sandbox" data-test="run-detail-summary">
<h2>Summary</h2>
<table>
<tbody>
<tr><th>Started</th><td>{{ run.started_at }}</td></tr>
<tr><th>Finished</th><td>{{ run.finished_at }}</td></tr>
<tr><th>Status</th><td><span class="badge {{ run.status_badge }}">{{ run.status }}</span></td></tr>
<tr><th>Trigger</th><td>{{ run.trigger_type }}</td></tr>
<tr><th>Scholars</th><td>{{ run.scholar_count }}</td></tr>
<tr><th>New Publications</th><td>{{ run.new_publication_count }}</td></tr>
<tr><th>Succeeded</th><td>{{ run_summary.succeeded_count or 0 }}</td></tr>
<tr><th>Failed</th><td>{{ run_summary.failed_count or 0 }}</td></tr>
<tr><th>Partial</th><td>{{ run_summary.partial_count or 0 }}</td></tr>
</tbody>
</table>
{% if run_summary.failed_state_counts or run_summary.failed_reason_counts %}
<details>
<summary>Failure Breakdown</summary>
<pre>{{ {"failed_state_counts": run_summary.failed_state_counts, "failed_reason_counts": run_summary.failed_reason_counts} | tojson(indent=2) }}</pre>
</details>
{% endif %}
</section>
<section class="sandbox" data-test="run-detail-results">
<h2>Scholar Results</h2>
{% if scholar_results %}
<table>
<thead>
<tr>
<th>Scholar</th>
<th>Outcome</th>
<th>State</th>
<th>Reason</th>
<th>Publications</th>
<th>Pages</th>
<th>Attempts</th>
<th>Continuation</th>
<th>Warnings</th>
</tr>
</thead>
<tbody>
{% for item in scholar_results %}
<tr>
<td><code>{{ item.scholar_id }}</code></td>
<td>{{ item.outcome or "-" }}</td>
<td>{{ item.state }}</td>
<td>{{ item.state_reason or "-" }}</td>
<td>{{ item.publication_count }}</td>
<td>{{ item.pages_fetched or 0 }} / {{ item.pages_attempted or 0 }}</td>
<td>{{ item.attempt_count or 1 }}</td>
<td>
{% if item.continuation_enqueued %}
<span class="badge warn">queued</span>
<p class="helper-text">
reason: {{ item.continuation_reason or "-" }},
cstart: {{ item.continuation_cstart or "-" }}
</p>
{% elif item.continuation_cleared %}
<span class="badge ok">cleared</span>
{% else %}
-
{% endif %}
</td>
<td>
{% if item.warnings %}
{{ item.warnings | join(", ") }}
{% else %}
-
{% endif %}
</td>
</tr>
{% if item.debug %}
<tr>
<td colspan="9">
<details>
<summary>Debug Context</summary>
<pre>{{ item.debug | tojson(indent=2) }}</pre>
</details>
</td>
</tr>
{% endif %}
{% endfor %}
</tbody>
</table>
{% else %}
<p>No scholar result details were captured.</p>
{% endif %}
</section>
{% endblock %}

129
app/templates/runs.html
View file

@ -1,129 +0,0 @@
{% extends "base.html" %}
{% block content %}
<section class="hero" data-test="runs-page">
<p class="eyebrow">Runs</p>
<h1>Run History</h1>
<p class="lede">Review execution outcomes and drill into run diagnostics when needed.</p>
</section>
<section class="sandbox" data-test="runs-filter-card">
<h2>Filters</h2>
<form method="get" action="/runs" class="inline-actions">
<label class="checkbox-row" for="failed_only">
<input id="failed_only" type="checkbox" name="failed_only" value="1" {% if failed_only %}checked{% endif %}>
<span>Failed / partial only</span>
</label>
<button type="submit" data-loading-text="Filtering...">Apply</button>
{% if failed_only %}
<a class="link-button" href="/runs">Clear</a>
{% endif %}
</form>
</section>
<section class="sandbox" data-test="runs-table-card">
<h2>Recent Runs</h2>
{% if runs %}
<table>
<thead>
<tr>
<th>ID</th>
<th>Started</th>
<th>Status</th>
<th>Trigger</th>
<th>Scholars</th>
<th>New Publications</th>
<th>Failures</th>
</tr>
</thead>
<tbody>
{% for run in runs %}
<tr>
<td><a href="/runs/{{ run.id }}">#{{ run.id }}</a></td>
<td>{{ run.started_at }}</td>
<td><span class="badge {{ run.status_badge }}">{{ run.status }}</span></td>
<td>{{ run.trigger_type }}</td>
<td>{{ run.scholar_count }}</td>
<td>{{ run.new_publication_count }}</td>
<td>{{ run.failed_count }} failed / {{ run.partial_count }} partial</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p>No runs match the current filter.</p>
{% endif %}
</section>
<section class="sandbox" data-test="runs-queue-card">
<div class="section-heading">
<h2>Continuation Queue</h2>
<a class="link-button" href="/runs?failed_only=1">Focus Failed Runs</a>
</div>
{% if queue_items %}
<table>
<thead>
<tr>
<th>Scholar</th>
<th>Status</th>
<th>Reason</th>
<th>Attempts</th>
<th>Next Attempt</th>
<th>Last Error</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
{% for item in queue_items %}
<tr>
<td>
<a href="/publications?mode=new&scholar_profile_id={{ item.scholar_profile_id }}">{{ item.scholar_label }}</a>
<p class="helper-text">resume cstart: {{ item.resume_cstart }}</p>
</td>
<td><span class="badge {{ item.status_badge }}">{{ item.status }}</span></td>
<td>
{% if item.status == "dropped" and item.dropped_reason %}
<code>{{ item.dropped_reason }}</code>
{% else %}
<code>{{ item.reason }}</code>
{% endif %}
</td>
<td>{{ item.attempt_count }}</td>
<td>{{ item.next_attempt_at }}</td>
<td>
{% if item.last_error %}
<details>
<summary>show</summary>
<pre>{{ item.last_error }}</pre>
</details>
{% else %}
-
{% endif %}
</td>
<td>
<div class="inline-actions">
<form method="post" action="/runs/queue/{{ item.id }}/retry">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<button type="submit" data-loading-text="Retrying...">Retry Now</button>
</form>
{% if item.status != "dropped" %}
<form method="post" action="/runs/queue/{{ item.id }}/drop">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<button type="submit" class="danger-button" data-loading-text="Dropping...">Drop</button>
</form>
{% endif %}
<form method="post" action="/runs/queue/{{ item.id }}/clear">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<button type="submit" class="button-text" data-loading-text="Clearing...">Clear</button>
</form>
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p>No queued continuation items.</p>
{% endif %}
</section>
{% endblock %}

99
app/templates/scholars.html
View file

@ -1,99 +0,0 @@
{% extends "base.html" %}
{% block content %}
<section class="hero" data-test="scholars-page">
<p class="eyebrow">Scholars</p>
<h1>Your Scholars</h1>
<p class="lede">Add, disable, or remove profiles scoped to your account.</p>
</section>
<section class="sandbox" data-test="scholars-create-card">
<h2>Add Scholar</h2>
<form method="post" action="/scholars" class="stack-form">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<label for="scholar_id">Scholar ID</label>
<input
id="scholar_id"
name="scholar_id"
type="text"
value="{{ form_scholar_id | default('') }}"
placeholder="abcDEF123456"
pattern="[a-zA-Z0-9_-]{12}"
required
>
<label for="display_name">Display Name (Optional)</label>
<input
id="display_name"
name="display_name"
type="text"
value="{{ form_display_name | default('') }}"
placeholder="Ada Lovelace"
>
<button type="submit" data-loading-text="Adding...">Add Scholar</button>
</form>
</section>
<section class="sandbox" data-test="scholars-list-card">
<h2>Tracked Scholars</h2>
{% if scholars %}
<table>
<thead>
<tr>
<th>Name</th>
<th>Scholar ID</th>
<th>Status</th>
<th>Last Run</th>
<th>Publications</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
{% for scholar in scholars %}
<tr>
<td>{{ scholar.display_name }}</td>
<td><code>{{ scholar.scholar_id }}</code></td>
<td>
{% if scholar.is_enabled %}
<span class="badge ok">enabled</span>
{% else %}
<span class="badge warn">disabled</span>
{% endif %}
{% if scholar.baseline_completed %}
<p class="helper-text">baseline complete</p>
{% else %}
<p class="helper-text">awaiting first run</p>
{% endif %}
</td>
<td>
<span class="badge {% if scholar.last_run_status == 'success' %}ok{% elif scholar.last_run_status in ['failed', 'partial_failure'] %}danger{% else %}warn{% endif %}">
{{ scholar.last_run_status }}
</span>
<p class="helper-text">{{ scholar.last_run_dt }}</p>
</td>
<td>
<div class="inline-actions">
<a class="link-button" href="/publications?mode=new&scholar_profile_id={{ scholar.id }}">New</a>
<a class="link-button" href="/publications?mode=all&scholar_profile_id={{ scholar.id }}">All</a>
</div>
</td>
<td>
<div class="inline-actions">
<form method="post" action="/scholars/{{ scholar.id }}/toggle">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<button type="submit" data-loading-text="Saving...">{% if scholar.is_enabled %}Disable{% else %}Enable{% endif %}</button>
</form>
<form method="post" action="/scholars/{{ scholar.id }}/delete">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<button type="submit" class="danger-button" data-loading-text="Deleting...">Delete</button>
</form>
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% else %}
<p>No scholars tracked yet.</p>
{% endif %}
</section>
{% endblock %}

44
app/templates/settings.html
View file

@ -1,44 +0,0 @@
{% extends "base.html" %}
{% block content %}
<section class="hero" data-test="settings-page">
<p class="eyebrow">Settings</p>
<h1>Your Settings</h1>
<p class="lede">Control automation cadence and request pacing for your own account.</p>
</section>
<section class="sandbox" data-test="settings-form-card">
<h2>Automation Settings</h2>
<form method="post" action="/settings" class="stack-form">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<label class="checkbox-row" for="auto_run_enabled">
<input id="auto_run_enabled" name="auto_run_enabled" type="checkbox" {% if user_settings.auto_run_enabled %}checked{% endif %}>
<span>Enable automatic runs</span>
</label>
<label for="run_interval_minutes">Run Interval (minutes)</label>
<input
id="run_interval_minutes"
name="run_interval_minutes"
type="number"
min="15"
step="1"
value="{{ user_settings.run_interval_minutes }}"
required
>
<label for="request_delay_seconds">Request Delay (seconds)</label>
<input
id="request_delay_seconds"
name="request_delay_seconds"
type="number"
min="1"
step="1"
value="{{ user_settings.request_delay_seconds }}"
required
>
<button type="submit" data-loading-text="Saving...">Save Settings</button>
</form>
</section>
{% endblock %}

75
app/templates/users.html
View file

@ -1,75 +0,0 @@
{% extends "base.html" %}
{% block content %}
<section class="hero" data-test="users-page">
<p class="eyebrow">Admin</p>
<h1>User Management</h1>
<p class="lede">Admin-only controls for account creation, activation, and password resets.</p>
</section>
<section class="sandbox" data-test="users-create-card">
<h2>Create User</h2>
<form method="post" action="/users" class="stack-form">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<label for="email">Email</label>
<input id="email" name="email" type="email" value="{{ form_email | default('') }}" required>
<label for="password">Temporary Password</label>
<input id="password" name="password" type="password" required>
<label class="checkbox-row" for="is_admin">
<input id="is_admin" name="is_admin" type="checkbox" {% if form_is_admin %}checked{% endif %}>
<span>Admin account</span>
</label>
<button type="submit" data-loading-text="Creating...">Create User</button>
</form>
</section>
<section class="sandbox" data-test="users-list-card">
<h2>Users</h2>
<table>
<thead>
<tr>
<th>Email</th>
<th>Role</th>
<th>Status</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
{% for user in users %}
<tr>
<td>{{ user.email }}</td>
<td>
{% if user.is_admin %}
<span class="badge ok">admin</span>
{% else %}
<span class="badge">user</span>
{% endif %}
</td>
<td>
{% if user.is_active %}
<span class="badge ok">active</span>
{% else %}
<span class="badge warn">inactive</span>
{% endif %}
</td>
<td>
<div class="inline-actions">
<form method="post" action="/users/{{ user.id }}/toggle-active">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<button type="submit" data-loading-text="Saving..." {% if user.id == current_user_id and user.is_active %}disabled{% endif %}>
{% if user.is_active %}Deactivate{% else %}Activate{% endif %}
</button>
</form>
<form method="post" action="/users/{{ user.id }}/reset-password" class="inline-reset-form">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
<input name="new_password" type="password" placeholder="New password" minlength="8" required>
<button type="submit" data-loading-text="Resetting...">Reset Password</button>
</form>
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</section>
{% endblock %}

24
app/theme.py
View file

@ -1,24 +0,0 @@
from dataclasses import dataclass
from typing import Final
@dataclass(frozen=True)
class ThemeDefinition:
slug: str
label: str
THEMES: Final[tuple[ThemeDefinition, ...]] = (
ThemeDefinition(slug="terracotta", label="Terracotta"),
ThemeDefinition(slug="fjord", label="Fjord"),
ThemeDefinition(slug="spruce", label="Spruce"),
)
_THEME_SLUGS: Final[frozenset[str]] = frozenset(theme.slug for theme in THEMES)
DEFAULT_THEME: Final[str] = THEMES[0].slug
def resolve_theme(candidate: str | None) -> str:
if candidate and candidate in _THEME_SLUGS:
return candidate
return DEFAULT_THEME

2
app/web/__init__.py
View file

@ -1,2 +0,0 @@
"""Web-layer routers, middleware, and shared helpers."""

140
app/web/common.py
View file

@ -1,140 +0,0 @@
from __future__ import annotations
import logging
from urllib.parse import urlencode
from fastapi import Request, status
from fastapi.responses import HTMLResponse, RedirectResponse
from fastapi.templating import Jinja2Templates
from sqlalchemy.ext.asyncio import AsyncSession
from app.auth.session import (
SessionUser,
clear_session_user,
get_session_user,
set_session_user,
)
from app.db.models import User
from app.security.csrf import CSRF_SESSION_KEY, ensure_csrf_token
from app.services import users as user_service
from app.theme import THEMES
logger = logging.getLogger(__name__)
templates = Jinja2Templates(directory="app/templates")
def to_session_user(user: User | None) -> SessionUser | None:
if user is None:
return None
return SessionUser(id=user.id, email=user.email, is_admin=user.is_admin)
def build_template_context(
request: Request,
*,
page_title: str,
active_nav: str,
theme_name: str,
session_user: SessionUser | None,
notice: str | None = None,
page_error: str | None = None,
) -> dict[str, object]:
return {
"active_nav": active_nav,
"page_title": page_title,
"theme_name": theme_name,
"themes": THEMES,
"session_user": session_user,
"csrf_token": ensure_csrf_token(request),
"notice": notice,
"page_error": page_error,
}
def redirect_with_message(
path: str,
*,
notice: str | None = None,
error: str | None = None,
) -> RedirectResponse:
params: dict[str, str] = {}
if notice:
params["notice"] = notice
if error:
params["error"] = error
if params:
path = f"{path}?{urlencode(params)}"
return RedirectResponse(path, status_code=status.HTTP_303_SEE_OTHER)
def redirect_to_login() -> RedirectResponse:
return RedirectResponse("/login", status_code=status.HTTP_303_SEE_OTHER)
def invalidate_session(request: Request) -> None:
clear_session_user(request)
request.session.pop(CSRF_SESSION_KEY, None)
async def get_authenticated_user(
request: Request,
db_session: AsyncSession,
) -> User | None:
session_user = get_session_user(request)
if session_user is None:
return None
user = await user_service.get_user_by_id(db_session, session_user.id)
if user is None or not user.is_active:
logger.info(
"auth.session_invalidated",
extra={
"event": "auth.session_invalidated",
"session_user_id": session_user.id,
},
)
invalidate_session(request)
return None
if user.email != session_user.email or user.is_admin != session_user.is_admin:
set_session_user(
request,
user_id=user.id,
email=user.email,
is_admin=user.is_admin,
)
return user
def login_rate_limit_key(request: Request, email: str) -> str:
client_host = request.client.host if request.client is not None else "unknown"
normalized_email = email.strip().lower()
return f"{client_host}:{normalized_email or '<empty>'}"
def render_login_page(
request: Request,
*,
theme_name: str,
error_message: str | None = None,
status_code: int = status.HTTP_200_OK,
retry_after_seconds: int | None = None,
) -> HTMLResponse:
context = build_template_context(
request,
page_title="Login",
active_nav="login",
theme_name=theme_name,
session_user=None,
)
context["error_message"] = error_message
context["retry_after_seconds"] = retry_after_seconds
return templates.TemplateResponse(
request=request,
name="login.html",
context=context,
status_code=status_code,
)

2
app/web/routers/__init__.py
View file

@ -1,2 +0,0 @@
"""Application routers grouped by concern."""

209
app/web/routers/admin.py
View file

@ -1,209 +0,0 @@
from __future__ import annotations
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
from fastapi.responses import HTMLResponse
from sqlalchemy.ext.asyncio import AsyncSession
from app.auth.deps import get_auth_service
from app.auth.service import AuthService
from app.db.session import get_db_session
from app.services import users as user_service
from app.theme import resolve_theme
from app.web import common
logger = logging.getLogger(__name__)
router = APIRouter()
async def _render_users_page(
request: Request,
*,
db_session: AsyncSession,
current_user,
theme_name: str,
notice: str | None = None,
page_error: str | None = None,
status_code: int = status.HTTP_200_OK,
form_email: str = "",
form_is_admin: bool = False,
) -> HTMLResponse:
users = await user_service.list_users(db_session)
context = common.build_template_context(
request,
page_title="Users",
active_nav="users",
theme_name=theme_name,
session_user=common.to_session_user(current_user),
notice=notice,
page_error=page_error,
)
context["users"] = users
context["current_user_id"] = current_user.id
context["form_email"] = form_email
context["form_is_admin"] = form_is_admin
return common.templates.TemplateResponse(
request=request,
name="users.html",
context=context,
status_code=status_code,
)
@router.get("/users", response_class=HTMLResponse)
async def users_page(
request: Request,
theme: str | None = None,
db_session: AsyncSession = Depends(get_db_session),
) -> HTMLResponse:
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
if not current_user.is_admin:
raise HTTPException(status_code=403, detail="Admin access required.")
return await _render_users_page(
request,
db_session=db_session,
current_user=current_user,
theme_name=resolve_theme(theme),
notice=request.query_params.get("notice"),
page_error=request.query_params.get("error"),
)
@router.post("/users")
async def create_user(
request: Request,
email: Annotated[str, Form()],
password: Annotated[str, Form()],
is_admin: Annotated[str | None, Form()] = None,
db_session: AsyncSession = Depends(get_db_session),
auth_service: AuthService = Depends(get_auth_service),
) -> HTMLResponse:
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
if not current_user.is_admin:
raise HTTPException(status_code=403, detail="Admin access required.")
active_theme = resolve_theme(None)
try:
validated_email = user_service.validate_email(email)
validated_password = user_service.validate_password(password)
created_user = await user_service.create_user(
db_session,
email=validated_email,
password_hash=auth_service.hash_password(validated_password),
is_admin=is_admin == "on",
)
except user_service.UserServiceError as exc:
logger.info(
"admin.user_create_failed",
extra={
"event": "admin.user_create_failed",
"admin_user_id": current_user.id,
"reason": "validation_or_conflict",
},
)
return await _render_users_page(
request,
db_session=db_session,
current_user=current_user,
theme_name=active_theme,
page_error=str(exc),
status_code=status.HTTP_400_BAD_REQUEST,
form_email=email,
form_is_admin=is_admin == "on",
)
logger.info(
"admin.user_created",
extra={
"event": "admin.user_created",
"admin_user_id": current_user.id,
"target_user_id": created_user.id,
"target_is_admin": created_user.is_admin,
},
)
return common.redirect_with_message(
"/users",
notice=f"User created: {created_user.email}",
)
@router.post("/users/{user_id}/toggle-active")
async def toggle_user_active(
request: Request,
user_id: int,
db_session: AsyncSession = Depends(get_db_session),
):
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
if not current_user.is_admin:
raise HTTPException(status_code=403, detail="Admin access required.")
target_user = await user_service.get_user_by_id(db_session, user_id)
if target_user is None:
return common.redirect_with_message("/users", error="User not found.")
if target_user.id == current_user.id and target_user.is_active:
return common.redirect_with_message("/users", error="You cannot deactivate your own account.")
updated_user = await user_service.set_user_active(
db_session,
user=target_user,
is_active=not target_user.is_active,
)
status_label = "activated" if updated_user.is_active else "deactivated"
logger.info(
"admin.user_active_toggled",
extra={
"event": "admin.user_active_toggled",
"admin_user_id": current_user.id,
"target_user_id": updated_user.id,
"is_active": updated_user.is_active,
},
)
return common.redirect_with_message("/users", notice=f"User {status_label}: {updated_user.email}")
@router.post("/users/{user_id}/reset-password")
async def reset_user_password(
request: Request,
user_id: int,
new_password: Annotated[str, Form()],
db_session: AsyncSession = Depends(get_db_session),
auth_service: AuthService = Depends(get_auth_service),
):
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
if not current_user.is_admin:
raise HTTPException(status_code=403, detail="Admin access required.")
target_user = await user_service.get_user_by_id(db_session, user_id)
if target_user is None:
return common.redirect_with_message("/users", error="User not found.")
try:
validated_password = user_service.validate_password(new_password)
except user_service.UserServiceError as exc:
return common.redirect_with_message("/users", error=str(exc))
await user_service.set_user_password_hash(
db_session,
user=target_user,
password_hash=auth_service.hash_password(validated_password),
)
logger.info(
"admin.user_password_reset",
extra={
"event": "admin.user_password_reset",
"admin_user_id": current_user.id,
"target_user_id": target_user.id,
},
)
return common.redirect_with_message("/users", notice=f"Password reset: {target_user.email}")

214
app/web/routers/auth.py
View file

@ -1,214 +0,0 @@
from __future__ import annotations
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, Form, Request, status
from fastapi.responses import HTMLResponse, RedirectResponse
from sqlalchemy.ext.asyncio import AsyncSession
from app.auth.deps import get_auth_service, get_login_rate_limiter
from app.auth.rate_limit import SlidingWindowRateLimiter
from app.auth.service import AuthService
from app.auth.session import get_session_user, set_session_user
from app.db.session import get_db_session
from app.security.csrf import ensure_csrf_token
from app.services import users as user_service
from app.theme import resolve_theme
from app.web import common
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/login", response_class=HTMLResponse)
async def login_page(request: Request, theme: str | None = None) -> HTMLResponse:
active_theme = resolve_theme(theme)
ensure_csrf_token(request)
if get_session_user(request) is not None:
return RedirectResponse("/", status_code=status.HTTP_303_SEE_OTHER)
return common.render_login_page(request, theme_name=active_theme)
@router.post("/login", response_class=HTMLResponse)
async def login(
request: Request,
email: Annotated[str, Form()],
password: Annotated[str, Form()],
db_session: AsyncSession = Depends(get_db_session),
auth_service: AuthService = Depends(get_auth_service),
rate_limiter: SlidingWindowRateLimiter = Depends(get_login_rate_limiter),
) -> HTMLResponse:
active_theme = resolve_theme(None)
limiter_key = common.login_rate_limit_key(request, email)
decision = rate_limiter.check(limiter_key)
normalized_email = email.strip().lower()
if not decision.allowed:
logger.warning(
"auth.login_rate_limited",
extra={
"event": "auth.login_rate_limited",
"email": normalized_email,
"retry_after_seconds": decision.retry_after_seconds,
},
)
response = common.render_login_page(
request,
theme_name=active_theme,
error_message="Too many login attempts. Please try again later.",
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
retry_after_seconds=decision.retry_after_seconds,
)
response.headers["Retry-After"] = str(decision.retry_after_seconds)
return response
user = await auth_service.authenticate_user(
db_session,
email=email,
password=password,
)
if user is None:
rate_limiter.record_failure(limiter_key)
logger.info(
"auth.login_failed",
extra={
"event": "auth.login_failed",
"email": normalized_email,
},
)
return common.render_login_page(
request,
theme_name=active_theme,
error_message="Invalid email or password.",
status_code=status.HTTP_401_UNAUTHORIZED,
)
rate_limiter.reset(limiter_key)
set_session_user(
request,
user_id=user.id,
email=user.email,
is_admin=user.is_admin,
)
ensure_csrf_token(request)
logger.info(
"auth.login_succeeded",
extra={
"event": "auth.login_succeeded",
"user_id": user.id,
"is_admin": user.is_admin,
},
)
return RedirectResponse("/", status_code=status.HTTP_303_SEE_OTHER)
@router.post("/logout")
async def logout(request: Request) -> RedirectResponse:
session_user = get_session_user(request)
common.invalidate_session(request)
logger.info(
"auth.logout",
extra={
"event": "auth.logout",
"user_id": session_user.id if session_user else None,
},
)
return RedirectResponse("/login", status_code=status.HTTP_303_SEE_OTHER)
@router.get("/account/password", response_class=HTMLResponse)
async def account_password_page(
request: Request,
theme: str | None = None,
db_session: AsyncSession = Depends(get_db_session),
) -> HTMLResponse:
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
return common.templates.TemplateResponse(
request=request,
name="account_password.html",
context=common.build_template_context(
request,
page_title="Change Password",
active_nav="account_password",
theme_name=resolve_theme(theme),
session_user=common.to_session_user(current_user),
notice=request.query_params.get("notice"),
page_error=request.query_params.get("error"),
),
)
@router.post("/account/password")
async def update_account_password(
request: Request,
current_password: Annotated[str, Form()],
new_password: Annotated[str, Form()],
confirm_password: Annotated[str, Form()],
db_session: AsyncSession = Depends(get_db_session),
auth_service: AuthService = Depends(get_auth_service),
) -> RedirectResponse:
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
if not auth_service.verify_password(
password_hash=current_user.password_hash,
password=current_password,
):
logger.info(
"account.password_change_failed",
extra={
"event": "account.password_change_failed",
"user_id": current_user.id,
"reason": "invalid_current_password",
},
)
return common.redirect_with_message(
"/account/password",
error="Current password is incorrect.",
)
if new_password != confirm_password:
logger.info(
"account.password_change_failed",
extra={
"event": "account.password_change_failed",
"user_id": current_user.id,
"reason": "confirmation_mismatch",
},
)
return common.redirect_with_message(
"/account/password",
error="New password and confirmation do not match.",
)
try:
validated_password = user_service.validate_password(new_password)
except user_service.UserServiceError as exc:
logger.info(
"account.password_change_failed",
extra={
"event": "account.password_change_failed",
"user_id": current_user.id,
"reason": "validation_error",
},
)
return common.redirect_with_message("/account/password", error=str(exc))
await user_service.set_user_password_hash(
db_session,
user=current_user,
password_hash=auth_service.hash_password(validated_password),
)
logger.info(
"account.password_changed",
extra={
"event": "account.password_changed",
"user_id": current_user.id,
},
)
return common.redirect_with_message(
"/account/password",
notice="Password updated successfully.",
)

167
app/web/routers/dashboard.py
View file

@ -1,167 +0,0 @@
from __future__ import annotations
import logging
from fastapi import APIRouter, Depends, Request
from fastapi.responses import HTMLResponse
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import RunTriggerType
from app.db.session import get_db_session
from app.presentation import dashboard as dashboard_presenter
from app.services import ingestion as ingestion_service
from app.services import publications as publication_service
from app.services import runs as run_service
from app.services import user_settings as user_settings_service
from app.settings import settings
from app.theme import resolve_theme
from app.web import common
from app.web.deps import get_ingestion_service
logger = logging.getLogger(__name__)
router = APIRouter()
async def _render_dashboard_page(
request: Request,
*,
db_session: AsyncSession,
current_user,
theme_name: str,
notice: str | None = None,
page_error: str | None = None,
) -> HTMLResponse:
unread_publications = await publication_service.list_new_for_latest_run_for_user(
db_session,
user_id=current_user.id,
limit=50,
)
recent_runs = await run_service.list_recent_runs_for_user(
db_session,
user_id=current_user.id,
limit=20,
)
user_settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=current_user.id,
)
queue_counts = await run_service.queue_status_counts_for_user(
db_session,
user_id=current_user.id,
)
context = common.build_template_context(
request,
page_title="Dashboard",
active_nav="home",
theme_name=theme_name,
session_user=common.to_session_user(current_user),
notice=notice,
page_error=page_error,
)
context["dashboard"] = dashboard_presenter.build_dashboard_view_model(
unread_publications=unread_publications,
recent_runs=recent_runs,
request_delay_seconds=user_settings.request_delay_seconds,
queue_counts=queue_counts,
)
return common.templates.TemplateResponse(
request=request,
name="index.html",
context=context,
)
@router.get("/", response_class=HTMLResponse)
async def home(
request: Request,
theme: str | None = None,
db_session: AsyncSession = Depends(get_db_session),
) -> HTMLResponse:
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
return await _render_dashboard_page(
request,
db_session=db_session,
current_user=current_user,
theme_name=resolve_theme(theme),
notice=request.query_params.get("notice"),
page_error=request.query_params.get("error"),
)
@router.post("/runs/manual")
async def run_manual_ingestion(
request: Request,
db_session: AsyncSession = Depends(get_db_session),
ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
):
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
user_settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=current_user.id,
)
logger.info(
"runs.manual_started",
extra={
"event": "runs.manual_started",
"user_id": current_user.id,
"request_delay_seconds": user_settings.request_delay_seconds,
"network_error_retries": settings.ingestion_network_error_retries,
"max_pages_per_scholar": settings.ingestion_max_pages_per_scholar,
"page_size": settings.ingestion_page_size,
},
)
try:
run_summary = await ingest_service.run_for_user(
db_session,
user_id=current_user.id,
trigger_type=RunTriggerType.MANUAL,
request_delay_seconds=user_settings.request_delay_seconds,
network_error_retries=settings.ingestion_network_error_retries,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
page_size=settings.ingestion_page_size,
auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
)
except ingestion_service.RunAlreadyInProgressError:
await db_session.rollback()
return common.redirect_with_message(
"/",
error="A run is already in progress for this account.",
)
except Exception:
await db_session.rollback()
logger.exception(
"runs.manual_failed",
extra={
"event": "runs.manual_failed",
"user_id": current_user.id,
},
)
return common.redirect_with_message("/", error="Manual run failed. Check logs for details.")
logger.info(
"runs.manual_completed",
extra={
"event": "runs.manual_completed",
"user_id": current_user.id,
"run_id": run_summary.crawl_run_id,
"status": run_summary.status.value,
"scholar_count": run_summary.scholar_count,
"new_publication_count": run_summary.new_publication_count,
},
)
return common.redirect_with_message(
"/",
notice=(
f"Run #{run_summary.crawl_run_id} complete ({run_summary.status.value}). "
f"Scholars: {run_summary.scholar_count}, "
f"new publications: {run_summary.new_publication_count}."
),
)

15
app/web/routers/health.py
View file

@ -1,15 +0,0 @@
from __future__ import annotations
from fastapi import APIRouter, HTTPException
from app.db.session import check_database
router = APIRouter()
@router.get("/healthz")
async def healthz() -> dict[str, str]:
if await check_database():
return {"status": "ok"}
raise HTTPException(status_code=500, detail="database unavailable")

165
app/web/routers/publications.py
View file

@ -1,165 +0,0 @@
from __future__ import annotations
import logging
from urllib.parse import urlencode, urlparse
from fastapi import APIRouter, Depends, Form, Request
from fastapi.responses import HTMLResponse
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.session import get_db_session
from app.services import publications as publication_service
from app.services import scholars as scholar_service
from app.theme import resolve_theme
from app.web import common
logger = logging.getLogger(__name__)
router = APIRouter()
MODE_NEW = "new"
MODE_ALL = "all"
def _resolve_mode(raw_mode: str | None) -> str:
return publication_service.resolve_mode(raw_mode)
def _parse_scholar_profile_id(value: str | None) -> int | None:
if not value:
return None
try:
parsed = int(value)
except ValueError:
return None
return parsed if parsed > 0 else None
def _build_publications_url(*, mode: str, scholar_profile_id: int | None) -> str:
params: dict[str, str] = {"mode": mode}
if scholar_profile_id is not None:
params["scholar_profile_id"] = str(scholar_profile_id)
return f"/publications?{urlencode(params)}"
def _is_safe_return_to(value: str | None) -> bool:
if not value:
return False
parsed = urlparse(value)
if parsed.scheme or parsed.netloc:
return False
if not value.startswith("/"):
return False
return value == "/" or value.startswith("/publications")
@router.get("/publications", response_class=HTMLResponse)
async def publications_page(
request: Request,
mode: str | None = None,
scholar_profile_id: str | None = None,
theme: str | None = None,
db_session: AsyncSession = Depends(get_db_session),
) -> HTMLResponse:
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
resolved_mode = _resolve_mode(mode)
scholar_id_filter = _parse_scholar_profile_id(scholar_profile_id)
scholars = await scholar_service.list_scholars_for_user(
db_session,
user_id=current_user.id,
)
scholar_lookup = {scholar.id: scholar for scholar in scholars}
if scholar_id_filter not in scholar_lookup:
scholar_id_filter = None
publications = await publication_service.list_for_user(
db_session,
user_id=current_user.id,
mode=resolved_mode,
scholar_profile_id=scholar_id_filter,
limit=500,
)
new_count = await publication_service.count_for_user(
db_session,
user_id=current_user.id,
mode=MODE_NEW,
scholar_profile_id=scholar_id_filter,
)
total_count = await publication_service.count_for_user(
db_session,
user_id=current_user.id,
mode=MODE_ALL,
scholar_profile_id=scholar_id_filter,
)
mode_new_url = _build_publications_url(
mode=MODE_NEW,
scholar_profile_id=scholar_id_filter,
)
mode_all_url = _build_publications_url(
mode=MODE_ALL,
scholar_profile_id=scholar_id_filter,
)
selected_scholar = scholar_lookup.get(scholar_id_filter) if scholar_id_filter else None
current_publications_url = _build_publications_url(
mode=resolved_mode,
scholar_profile_id=scholar_id_filter,
)
context = common.build_template_context(
request,
page_title="Publications",
active_nav="publications",
theme_name=resolve_theme(theme),
session_user=common.to_session_user(current_user),
notice=request.query_params.get("notice"),
page_error=request.query_params.get("error"),
)
context["mode"] = resolved_mode
context["publications"] = publications
context["new_count"] = new_count
context["total_count"] = total_count
context["mode_new_url"] = mode_new_url
context["mode_all_url"] = mode_all_url
context["scholars"] = scholars
context["selected_scholar_id"] = scholar_id_filter
context["selected_scholar"] = selected_scholar
context["current_publications_url"] = current_publications_url
return common.templates.TemplateResponse(
request=request,
name="publications.html",
context=context,
)
@router.post("/publications/mark-all-read")
async def mark_all_publications_read(
request: Request,
return_to: str | None = Form(default=None),
db_session: AsyncSession = Depends(get_db_session),
):
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
updated_count = await publication_service.mark_all_unread_as_read_for_user(
db_session,
user_id=current_user.id,
)
logger.info(
"publications.mark_all_read",
extra={
"event": "publications.mark_all_read",
"user_id": current_user.id,
"updated_count": updated_count,
},
)
redirect_target = "/publications?mode=new"
if _is_safe_return_to(return_to):
redirect_target = return_to
return common.redirect_with_message(
redirect_target,
notice=f"Marked {updated_count} publication(s) as read.",
)

274
app/web/routers/runs.py
View file

@ -1,274 +0,0 @@
from __future__ import annotations
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi.responses import HTMLResponse
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.session import get_db_session
from app.services import runs as run_service
from app.theme import resolve_theme
from app.web import common
router = APIRouter()
def _as_bool(value: str | None) -> bool:
if value is None:
return False
return value.strip().lower() in {"1", "true", "yes", "on"}
def _status_badge(status_value: str) -> str:
if status_value == "success":
return "ok"
if status_value == "failed":
return "danger"
return "warn"
def _queue_status_badge(status_value: str) -> str:
if status_value == "dropped":
return "danger"
if status_value == "retrying":
return "ok"
return "warn"
def _format_dt(value: datetime | None) -> str:
if value is None:
return "-"
return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
def _summary_dict(error_log: object) -> dict[str, object]:
if not isinstance(error_log, dict):
return {}
summary = error_log.get("summary")
if not isinstance(summary, dict):
return {}
return summary
@router.get("/runs", response_class=HTMLResponse)
async def runs_page(
request: Request,
theme: str | None = None,
failed_only: str | None = None,
db_session: AsyncSession = Depends(get_db_session),
) -> HTMLResponse:
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
failed_only_enabled = _as_bool(failed_only)
runs = await run_service.list_runs_for_user(
db_session,
user_id=current_user.id,
limit=200,
failed_only=failed_only_enabled,
)
run_items = []
for run in runs:
summary = _summary_dict(run.error_log)
failed_count = summary.get("failed_count", 0)
partial_count = summary.get("partial_count", 0)
run_items.append(
{
"id": run.id,
"started_at": _format_dt(run.start_dt),
"finished_at": _format_dt(run.end_dt),
"status": run.status.value,
"status_badge": _status_badge(run.status.value),
"trigger_type": run.trigger_type.value,
"scholar_count": run.scholar_count,
"new_publication_count": run.new_pub_count,
"failed_count": failed_count,
"partial_count": partial_count,
}
)
queue_entries = await run_service.list_queue_items_for_user(
db_session,
user_id=current_user.id,
limit=200,
)
queue_items = []
for item in queue_entries:
queue_items.append(
{
"id": item.id,
"scholar_profile_id": item.scholar_profile_id,
"scholar_label": item.scholar_label,
"status": item.status,
"status_badge": _queue_status_badge(item.status),
"reason": item.reason,
"dropped_reason": item.dropped_reason,
"attempt_count": item.attempt_count,
"resume_cstart": item.resume_cstart,
"next_attempt_at": _format_dt(item.next_attempt_dt),
"updated_at": _format_dt(item.updated_at),
"last_error": item.last_error,
"last_run_id": item.last_run_id,
}
)
context = common.build_template_context(
request,
page_title="Runs",
active_nav="runs",
theme_name=resolve_theme(theme),
session_user=common.to_session_user(current_user),
notice=request.query_params.get("notice"),
page_error=request.query_params.get("error"),
)
context["runs"] = run_items
context["failed_only"] = failed_only_enabled
context["queue_items"] = queue_items
return common.templates.TemplateResponse(
request=request,
name="runs.html",
context=context,
)
@router.get("/runs/{run_id}", response_class=HTMLResponse)
async def run_detail_page(
request: Request,
run_id: int,
theme: str | None = None,
db_session: AsyncSession = Depends(get_db_session),
) -> HTMLResponse:
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
run = await run_service.get_run_for_user(
db_session,
user_id=current_user.id,
run_id=run_id,
)
if run is None:
raise HTTPException(status_code=404, detail="Run not found.")
error_log = run.error_log if isinstance(run.error_log, dict) else {}
scholar_results = error_log.get("scholar_results", [])
if not isinstance(scholar_results, list):
scholar_results = []
context = common.build_template_context(
request,
page_title=f"Run #{run.id}",
active_nav="runs",
theme_name=resolve_theme(theme),
session_user=common.to_session_user(current_user),
)
context["run"] = {
"id": run.id,
"started_at": _format_dt(run.start_dt),
"finished_at": _format_dt(run.end_dt),
"status": run.status.value,
"status_badge": _status_badge(run.status.value),
"trigger_type": run.trigger_type.value,
"scholar_count": run.scholar_count,
"new_publication_count": run.new_pub_count,
}
context["run_summary"] = _summary_dict(error_log)
context["scholar_results"] = scholar_results
return common.templates.TemplateResponse(
request=request,
name="run_detail.html",
context=context,
)
@router.post("/runs/queue/{queue_item_id}/drop")
async def drop_queue_item(
request: Request,
queue_item_id: int,
db_session: AsyncSession = Depends(get_db_session),
):
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
try:
dropped = await run_service.drop_queue_item_for_user(
db_session,
user_id=current_user.id,
queue_item_id=queue_item_id,
)
except run_service.QueueTransitionError as exc:
return common.redirect_with_message(
"/runs",
error=f"Queue item #{queue_item_id}: {exc.message}",
)
if dropped is None:
raise HTTPException(status_code=404, detail="Queue item not found.")
return common.redirect_with_message(
"/runs",
notice=f"Queue item #{queue_item_id} marked as dropped.",
)
@router.post("/runs/queue/{queue_item_id}/clear")
async def clear_queue_item(
request: Request,
queue_item_id: int,
db_session: AsyncSession = Depends(get_db_session),
):
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
try:
cleared = await run_service.clear_queue_item_for_user(
db_session,
user_id=current_user.id,
queue_item_id=queue_item_id,
)
except run_service.QueueTransitionError as exc:
return common.redirect_with_message(
"/runs",
error=f"Queue item #{queue_item_id}: {exc.message}",
)
if cleared is None:
raise HTTPException(status_code=404, detail="Queue item not found.")
return common.redirect_with_message(
"/runs",
notice=f"Queue item #{queue_item_id} cleared.",
)
@router.post("/runs/queue/{queue_item_id}/retry")
async def retry_queue_item(
request: Request,
queue_item_id: int,
db_session: AsyncSession = Depends(get_db_session),
):
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
try:
queue_item = await run_service.retry_queue_item_for_user(
db_session,
user_id=current_user.id,
queue_item_id=queue_item_id,
)
except run_service.QueueTransitionError as exc:
return common.redirect_with_message(
"/runs",
error=f"Queue item #{queue_item_id}: {exc.message}",
)
if queue_item is None:
raise HTTPException(status_code=404, detail="Queue item not found.")
return common.redirect_with_message(
"/runs",
notice=(
f"Queue item #{queue_item_id} queued for retry "
f"(scholar: {queue_item.scholar_label})."
),
)

212
app/web/routers/scholars.py
View file

@ -1,212 +0,0 @@
from __future__ import annotations
from datetime import datetime, timezone
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, Form, HTTPException, Request, status
from fastapi.responses import HTMLResponse
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.session import get_db_session
from app.services import scholars as scholar_service
from app.theme import resolve_theme
from app.web import common
logger = logging.getLogger(__name__)
router = APIRouter()
def _format_dt(value: datetime | None) -> str:
if value is None:
return "-"
return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
async def _render_scholars_page(
request: Request,
*,
db_session: AsyncSession,
current_user,
theme_name: str,
notice: str | None = None,
page_error: str | None = None,
status_code: int = status.HTTP_200_OK,
form_scholar_id: str = "",
form_display_name: str = "",
) -> HTMLResponse:
scholars = await scholar_service.list_scholars_for_user(
db_session,
user_id=current_user.id,
)
scholar_items = []
for scholar in scholars:
scholar_items.append(
{
"id": scholar.id,
"scholar_id": scholar.scholar_id,
"display_name": scholar.display_name or "Unnamed",
"is_enabled": scholar.is_enabled,
"baseline_completed": scholar.baseline_completed,
"last_run_status": (
scholar.last_run_status.value
if scholar.last_run_status is not None
else "never"
),
"last_run_dt": _format_dt(scholar.last_run_dt),
}
)
context = common.build_template_context(
request,
page_title="Scholars",
active_nav="scholars",
theme_name=theme_name,
session_user=common.to_session_user(current_user),
notice=notice,
page_error=page_error,
)
context["scholars"] = scholar_items
context["form_scholar_id"] = form_scholar_id
context["form_display_name"] = form_display_name
return common.templates.TemplateResponse(
request=request,
name="scholars.html",
context=context,
status_code=status_code,
)
@router.get("/scholars", response_class=HTMLResponse)
async def scholars_page(
request: Request,
theme: str | None = None,
db_session: AsyncSession = Depends(get_db_session),
) -> HTMLResponse:
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
return await _render_scholars_page(
request,
db_session=db_session,
current_user=current_user,
theme_name=resolve_theme(theme),
notice=request.query_params.get("notice"),
page_error=request.query_params.get("error"),
)
@router.post("/scholars")
async def create_scholar(
request: Request,
scholar_id: Annotated[str, Form()],
display_name: Annotated[str, Form()] = "",
db_session: AsyncSession = Depends(get_db_session),
) -> HTMLResponse:
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
active_theme = resolve_theme(None)
try:
created_profile = await scholar_service.create_scholar_for_user(
db_session,
user_id=current_user.id,
scholar_id=scholar_id,
display_name=display_name,
)
except scholar_service.ScholarServiceError as exc:
logger.info(
"scholars.create_failed",
extra={
"event": "scholars.create_failed",
"user_id": current_user.id,
"scholar_id": scholar_id.strip(),
},
)
return await _render_scholars_page(
request,
db_session=db_session,
current_user=current_user,
theme_name=active_theme,
page_error=str(exc),
status_code=status.HTTP_400_BAD_REQUEST,
form_scholar_id=scholar_id,
form_display_name=display_name,
)
label = created_profile.display_name or created_profile.scholar_id
logger.info(
"scholars.created",
extra={
"event": "scholars.created",
"user_id": current_user.id,
"scholar_profile_id": created_profile.id,
},
)
return common.redirect_with_message("/scholars", notice=f"Scholar added: {label}")
@router.post("/scholars/{scholar_profile_id}/toggle")
async def toggle_scholar(
request: Request,
scholar_profile_id: int,
db_session: AsyncSession = Depends(get_db_session),
):
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
profile = await scholar_service.get_user_scholar_by_id(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
)
if profile is None:
raise HTTPException(status_code=404, detail="Scholar not found.")
updated_profile = await scholar_service.toggle_scholar_enabled(
db_session,
profile=profile,
)
status_label = "enabled" if updated_profile.is_enabled else "disabled"
logger.info(
"scholars.toggled",
extra={
"event": "scholars.toggled",
"user_id": current_user.id,
"scholar_profile_id": updated_profile.id,
"is_enabled": updated_profile.is_enabled,
},
)
return common.redirect_with_message(
"/scholars",
notice=f"Scholar {status_label}: {updated_profile.scholar_id}",
)
@router.post("/scholars/{scholar_profile_id}/delete")
async def delete_scholar(
request: Request,
scholar_profile_id: int,
db_session: AsyncSession = Depends(get_db_session),
):
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
profile = await scholar_service.get_user_scholar_by_id(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
)
if profile is None:
raise HTTPException(status_code=404, detail="Scholar not found.")
deleted_label = profile.display_name or profile.scholar_id
await scholar_service.delete_scholar(db_session, profile=profile)
logger.info(
"scholars.deleted",
extra={
"event": "scholars.deleted",
"user_id": current_user.id,
"scholar_profile_id": scholar_profile_id,
},
)
return common.redirect_with_message("/scholars", notice=f"Scholar removed: {deleted_label}")

94
app/web/routers/settings.py
View file

@ -1,94 +0,0 @@
from __future__ import annotations
import logging
from typing import Annotated
from fastapi import APIRouter, Depends, Form, Request
from fastapi.responses import HTMLResponse
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.session import get_db_session
from app.services import user_settings as user_settings_service
from app.theme import resolve_theme
from app.web import common
logger = logging.getLogger(__name__)
router = APIRouter()
@router.get("/settings", response_class=HTMLResponse)
async def settings_page(
request: Request,
theme: str | None = None,
db_session: AsyncSession = Depends(get_db_session),
) -> HTMLResponse:
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
user_settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=current_user.id,
)
context = common.build_template_context(
request,
page_title="Settings",
active_nav="settings",
theme_name=resolve_theme(theme),
session_user=common.to_session_user(current_user),
notice=request.query_params.get("notice"),
page_error=request.query_params.get("error"),
)
context["user_settings"] = user_settings
return common.templates.TemplateResponse(
request=request,
name="settings.html",
context=context,
)
@router.post("/settings")
async def update_settings(
request: Request,
run_interval_minutes: Annotated[str, Form()],
request_delay_seconds: Annotated[str, Form()],
auto_run_enabled: Annotated[str | None, Form()] = None,
db_session: AsyncSession = Depends(get_db_session),
):
current_user = await common.get_authenticated_user(request, db_session)
if current_user is None:
return common.redirect_to_login()
try:
parsed_interval = user_settings_service.parse_run_interval_minutes(
run_interval_minutes
)
parsed_delay = user_settings_service.parse_request_delay_seconds(
request_delay_seconds
)
except user_settings_service.UserSettingsServiceError as exc:
return common.redirect_with_message("/settings", error=str(exc))
user_settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=current_user.id,
)
await user_settings_service.update_settings(
db_session,
settings=user_settings,
auto_run_enabled=auto_run_enabled == "on",
run_interval_minutes=parsed_interval,
request_delay_seconds=parsed_delay,
)
logger.info(
"settings.updated",
extra={
"event": "settings.updated",
"user_id": current_user.id,
"auto_run_enabled": auto_run_enabled == "on",
"run_interval_minutes": parsed_interval,
"request_delay_seconds": parsed_delay,
},
)
return common.redirect_with_message("/settings", notice="Settings updated.")

31
docker-compose.dev.yml Normal file
View file

@ -0,0 +1,31 @@
services:
app:
image: scholarr-dev:local
build:
context: .
target: dev
environment:
APP_RELOAD: ${APP_RELOAD:-1}
SESSION_COOKIE_SECURE: ${SESSION_COOKIE_SECURE:-0}
FRONTEND_ENABLED: ${FRONTEND_ENABLED:-0}
volumes:
- ./:/app
frontend:
image: node:20-alpine
working_dir: /frontend
command: ["sh", "-lc", "npm install && npm run dev"]
environment:
CHOKIDAR_USEPOLLING: ${CHOKIDAR_USEPOLLING:-1}
VITE_DEV_API_PROXY_TARGET: ${VITE_DEV_API_PROXY_TARGET:-http://app:8000}
ports:
- "${FRONTEND_HOST_PORT:-5173}:5173"
volumes:
- ./frontend:/frontend
- frontend_node_modules:/frontend/node_modules
depends_on:
app:
condition: service_healthy
volumes:
frontend_node_modules:

41
docker-compose.yml
View file

@ -3,12 +3,11 @@ name: scholarr
services:
db:
image: postgres:15-alpine
restart: unless-stopped
environment:
POSTGRES_DB: ${POSTGRES_DB:-scholar}
POSTGRES_USER: ${POSTGRES_USER:-scholar}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-scholar}
ports:
- "${POSTGRES_PORT:-5432}:5432"
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
@ -18,26 +17,26 @@ services:
retries: 20
app:
build:
context: .
target: dev
image: ${SCHOLARR_IMAGE:-justinzeus/scholarr:latest}
restart: unless-stopped
environment:
DATABASE_URL: ${DATABASE_URL:-postgresql+asyncpg://scholar:scholar@db:5432/scholar}
TEST_DATABASE_URL: ${TEST_DATABASE_URL:-}
DATABASE_URL: ${DATABASE_URL:-postgresql+asyncpg://${POSTGRES_USER:-scholar}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-scholar}}
MIGRATE_ON_START: ${MIGRATE_ON_START:-1}
APP_NAME: ${APP_NAME:-scholarr}
APP_HOST: ${APP_HOST:-0.0.0.0}
APP_PORT: ${APP_PORT:-8000}
APP_RELOAD: ${APP_RELOAD:-1}
SESSION_SECRET_KEY: ${SESSION_SECRET_KEY:-dev-insecure-session-key}
SESSION_COOKIE_SECURE: ${SESSION_COOKIE_SECURE:-0}
APP_RELOAD: ${APP_RELOAD:-0}
SESSION_SECRET_KEY: ${SESSION_SECRET_KEY:?set SESSION_SECRET_KEY}
SESSION_COOKIE_SECURE: ${SESSION_COOKIE_SECURE:-1}
FRONTEND_ENABLED: ${FRONTEND_ENABLED:-1}
FRONTEND_DIST_DIR: ${FRONTEND_DIST_DIR:-/app/frontend/dist}
LOGIN_RATE_LIMIT_ATTEMPTS: ${LOGIN_RATE_LIMIT_ATTEMPTS:-5}
LOGIN_RATE_LIMIT_WINDOW_SECONDS: ${LOGIN_RATE_LIMIT_WINDOW_SECONDS:-60}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
LOG_FORMAT: ${LOG_FORMAT:-console}
LOG_REQUESTS: ${LOG_REQUESTS:-1}
LOG_UVICORN_ACCESS: ${LOG_UVICORN_ACCESS:-0}
LOG_REQUEST_SKIP_PATHS: ${LOG_REQUEST_SKIP_PATHS:-/healthz,/static/}
LOG_REQUEST_SKIP_PATHS: ${LOG_REQUEST_SKIP_PATHS:-/healthz}
LOG_REDACT_FIELDS: ${LOG_REDACT_FIELDS:-}
SCHEDULER_ENABLED: ${SCHEDULER_ENABLED:-1}
SCHEDULER_TICK_SECONDS: ${SCHEDULER_TICK_SECONDS:-60}
@ -45,6 +44,21 @@ services:
INGESTION_RETRY_BACKOFF_SECONDS: ${INGESTION_RETRY_BACKOFF_SECONDS:-1.0}
INGESTION_MAX_PAGES_PER_SCHOLAR: ${INGESTION_MAX_PAGES_PER_SCHOLAR:-30}
INGESTION_PAGE_SIZE: ${INGESTION_PAGE_SIZE:-100}
INGESTION_CONTINUATION_QUEUE_ENABLED: ${INGESTION_CONTINUATION_QUEUE_ENABLED:-1}
INGESTION_CONTINUATION_BASE_DELAY_SECONDS: ${INGESTION_CONTINUATION_BASE_DELAY_SECONDS:-120}
INGESTION_CONTINUATION_MAX_DELAY_SECONDS: ${INGESTION_CONTINUATION_MAX_DELAY_SECONDS:-3600}
INGESTION_CONTINUATION_MAX_ATTEMPTS: ${INGESTION_CONTINUATION_MAX_ATTEMPTS:-6}
SCHEDULER_QUEUE_BATCH_SIZE: ${SCHEDULER_QUEUE_BATCH_SIZE:-10}
SCHOLAR_IMAGE_UPLOAD_DIR: ${SCHOLAR_IMAGE_UPLOAD_DIR:-/var/lib/scholarr/uploads}
SCHOLAR_IMAGE_UPLOAD_MAX_BYTES: ${SCHOLAR_IMAGE_UPLOAD_MAX_BYTES:-2000000}
SCHOLAR_NAME_SEARCH_ENABLED: ${SCHOLAR_NAME_SEARCH_ENABLED:-1}
SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS: ${SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS:-21600}
SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS: ${SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS:-300}
SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES: ${SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES:-512}
SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS: ${SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS:-8.0}
SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS: ${SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS:-2.0}
SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD: ${SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD:-1}
SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS: ${SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS:-1800}
BOOTSTRAP_ADMIN_ON_START: ${BOOTSTRAP_ADMIN_ON_START:-0}
BOOTSTRAP_ADMIN_EMAIL: ${BOOTSTRAP_ADMIN_EMAIL:-}
BOOTSTRAP_ADMIN_PASSWORD: ${BOOTSTRAP_ADMIN_PASSWORD:-}
@ -54,7 +68,7 @@ services:
ports:
- "${APP_HOST_PORT:-8000}:8000"
volumes:
- ./:/app
- scholar_uploads:/var/lib/scholarr/uploads
depends_on:
db:
condition: service_healthy
@ -66,3 +80,4 @@ services:
volumes:
postgres_data:
scholar_uploads:

3
frontend/.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
node_modules/
dist/
.vite/

31
frontend/index.html Normal file
View file

@ -0,0 +1,31 @@
<!doctype html>
<html lang="en" class="h-full">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>scholarr</title>
<script>
(function () {
var storageKey = "scholarr-theme-preference";
var root = document.documentElement;
var stored = null;
try {
stored = localStorage.getItem(storageKey);
} catch (_err) {
stored = null;
}
var preference = stored === "light" || stored === "dark" || stored === "system" ? stored : "system";
var systemDark = window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches;
var effective = preference === "system" ? (systemDark ? "dark" : "light") : preference;
root.classList.toggle("dark", effective === "dark");
root.setAttribute("data-theme-preference", preference);
})();
</script>
</head>
<body class="h-full">
<div id="app"></div>
<script type="module" src="/src/main.ts"></script>
</body>
</html>

2985
frontend/package-lock.json generated Normal file
View file

File diff suppressed because it is too large Load diff

30
frontend/package.json Normal file
View file

@ -0,0 +1,30 @@
{
"name": "scholarr-frontend",
"private": true,
"version": "0.1.0",
"type": "module",
"scripts": {
"dev": "vite --host 0.0.0.0 --port 5173",
"build": "vue-tsc --noEmit && vite build",
"preview": "vite preview --host 0.0.0.0 --port 4173",
"typecheck": "vue-tsc --noEmit",
"test": "vitest",
"test:run": "vitest run"
},
"dependencies": {
"pinia": "^2.1.7",
"vue": "^3.5.13",
"vue-router": "^4.5.0"
},
"devDependencies": {
"@types/node": "^22.10.2",
"@vitejs/plugin-vue": "^5.2.1",
"autoprefixer": "^10.4.21",
"postcss": "^8.5.3",
"tailwindcss": "^3.4.17",
"typescript": "^5.7.2",
"vite": "^5.4.11",
"vitest": "^2.1.8",
"vue-tsc": "^2.1.10"
}
}

6
frontend/postcss.config.cjs Normal file
View file

@ -0,0 +1,6 @@
module.exports = {
plugins: {
tailwindcss: {},
autoprefixer: {},
},
};

35
frontend/src/app/AppShell.vue Normal file
View file

@ -0,0 +1,35 @@
<script setup lang="ts">
import { computed } from "vue";
import { RouterView } from "vue-router";
import AppHeader from "@/components/layout/AppHeader.vue";
import AppNav from "@/components/layout/AppNav.vue";
import RequestErrorPanel from "@/components/patterns/RequestErrorPanel.vue";
import { useAuthStore } from "@/stores/auth";
const auth = useAuthStore();
const showChrome = computed(() => auth.isAuthenticated);
</script>
<template>
<div class="min-h-screen overflow-x-clip">
<a href="#app-main" class="skip-link">Skip to main content</a>
<RequestErrorPanel />
<template v-if="showChrome">
<AppHeader />
<div
class="grid min-h-[calc(100dvh-4.5rem)] grid-cols-1 lg:h-[calc(100dvh-4.5rem)] lg:min-h-[calc(100dvh-4.5rem)] lg:grid-cols-[17rem_minmax(0,1fr)]"
>
<AppNav class="lg:min-h-0 lg:overflow-y-auto" />
<main id="app-main" class="min-w-0 px-4 py-6 sm:px-6 lg:min-h-0 lg:overflow-y-auto lg:px-8">
<RouterView />
</main>
</div>
</template>
<main id="app-main" v-else class="min-h-screen overflow-x-clip">
<RouterView />
</main>
</div>
</template>

23
frontend/src/app/guards.ts Normal file
View file

@ -0,0 +1,23 @@
import type { Router } from "vue-router";
import { useAuthStore } from "@/stores/auth";
export function applyRouteGuards(router: Router): void {
router.beforeEach((to) => {
const auth = useAuthStore();
if (to.meta.requiresAuth && !auth.isAuthenticated) {
return { name: "login" };
}
if (to.meta.requiresAdmin && !auth.isAdmin) {
return { name: "dashboard" };
}
if (to.meta.guestOnly && auth.isAuthenticated) {
return { name: "dashboard" };
}
return true;
});
}

12
frontend/src/app/providers.ts Normal file
View file

@ -0,0 +1,12 @@
import { setCsrfTokenProvider } from "@/lib/api/client";
import { useAuthStore } from "@/stores/auth";
import { useThemeStore } from "@/stores/theme";
export async function bootstrapAppProviders(): Promise<void> {
const theme = useThemeStore();
theme.initialize();
const auth = useAuthStore();
setCsrfTokenProvider(() => auth.csrfToken);
await auth.bootstrapSession();
}

84
frontend/src/app/router.ts Normal file
View file

@ -0,0 +1,84 @@
import { createRouter, createWebHistory } from "vue-router";
import { applyRouteGuards } from "@/app/guards";
import LoginPage from "@/pages/LoginPage.vue";
import DashboardPage from "@/pages/DashboardPage.vue";
import ScholarsPage from "@/pages/ScholarsPage.vue";
import PublicationsPage from "@/pages/PublicationsPage.vue";
import RunsPage from "@/pages/RunsPage.vue";
import RunDetailPage from "@/pages/RunDetailPage.vue";
import SettingsPage from "@/pages/SettingsPage.vue";
import AdminUsersPage from "@/pages/AdminUsersPage.vue";
import StyleGuidePage from "@/pages/StyleGuidePage.vue";
export const router = createRouter({
history: createWebHistory(),
routes: [
{
path: "/login",
name: "login",
component: LoginPage,
meta: { guestOnly: true },
},
{
path: "/",
redirect: "/dashboard",
},
{
path: "/dashboard",
name: "dashboard",
component: DashboardPage,
meta: { requiresAuth: true },
},
{
path: "/scholars",
name: "scholars",
component: ScholarsPage,
meta: { requiresAuth: true },
},
{
path: "/publications",
name: "publications",
component: PublicationsPage,
meta: { requiresAuth: true },
},
{
path: "/settings",
name: "settings",
component: SettingsPage,
meta: { requiresAuth: true },
},
{
path: "/admin/style-guide",
name: "style-guide",
component: StyleGuidePage,
meta: { requiresAuth: true, requiresAdmin: true },
},
{
path: "/admin/runs",
name: "runs",
component: RunsPage,
meta: { requiresAuth: true, requiresAdmin: true },
},
{
path: "/admin/runs/:id",
name: "run-detail",
component: RunDetailPage,
meta: { requiresAuth: true, requiresAdmin: true },
},
{
path: "/admin/users",
name: "admin-users",
component: AdminUsersPage,
meta: { requiresAuth: true, requiresAdmin: true },
},
{
path: "/:pathMatch(.*)*",
redirect: "/dashboard",
},
],
});
applyRouteGuards(router);
export default router;

71
frontend/src/components/layout/AppHeader.vue Normal file
View file

@ -0,0 +1,71 @@
<script setup lang="ts">
import { computed } from "vue";
import AppButton from "@/components/ui/AppButton.vue";
import { useThemeStore } from "@/stores/theme";
const theme = useThemeStore();
const isDarkTheme = computed(() => theme.active === "dark");
const toggleThemeLabel = computed(() =>
isDarkTheme.value ? "Switch to light theme" : "Switch to dark theme",
);
function onToggleTheme(): void {
theme.setPreference(isDarkTheme.value ? "light" : "dark");
}
</script>
<template>
<header class="sticky top-0 z-30 border-b border-zinc-200/70 bg-white/90 backdrop-blur dark:border-zinc-800 dark:bg-zinc-900/90">
<div class="flex min-h-[4.5rem] w-full items-center justify-between gap-4 px-4 sm:px-6 lg:px-8">
<div class="flex min-w-0 items-center gap-2">
<RouterLink
to="/dashboard"
class="rounded-sm font-display text-lg tracking-tight text-zinc-900 transition hover:text-zinc-700 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-brand-500 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 dark:text-zinc-100 dark:hover:text-zinc-300 dark:focus-visible:ring-brand-400 dark:focus-visible:ring-offset-zinc-950"
>
scholarr
</RouterLink>
</div>
<div class="flex items-center justify-end">
<AppButton
variant="ghost"
class="h-10 w-10 rounded-full p-0"
:aria-label="toggleThemeLabel"
:title="toggleThemeLabel"
@click="onToggleTheme"
>
<span class="sr-only">{{ toggleThemeLabel }}</span>
<svg
v-if="isDarkTheme"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="1.8"
class="h-5 w-5"
aria-hidden="true"
>
<circle cx="12" cy="12" r="4" />
<path
d="M12 2v2.5M12 19.5V22M4.9 4.9l1.8 1.8M17.3 17.3l1.8 1.8M2 12h2.5M19.5 12H22M4.9 19.1l1.8-1.8M17.3 6.7l1.8-1.8"
/>
</svg>
<svg
v-else
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="1.8"
class="h-5 w-5"
aria-hidden="true"
>
<path d="M21 14.5A8.5 8.5 0 1 1 9.5 3 7 7 0 0 0 21 14.5z" />
</svg>
</AppButton>
</div>
</div>
</header>
</template>

56
frontend/src/components/layout/AppNav.vue Normal file
View file

@ -0,0 +1,56 @@
<script setup lang="ts">
import { computed } from "vue";
import { useRouter } from "vue-router";
import AppButton from "@/components/ui/AppButton.vue";
import { useAuthStore } from "@/stores/auth";
const auth = useAuthStore();
const router = useRouter();
const links = computed(() => {
const base = [
{ to: "/dashboard", label: "Dashboard" },
{ to: "/scholars", label: "Scholars" },
{ to: "/publications", label: "Publications" },
{ to: "/settings", label: "Settings" },
];
if (auth.isAdmin) {
base.push({ to: "/admin/runs", label: "Runs" });
base.push({ to: "/admin/users", label: "Users" });
}
return base;
});
async function onLogout(): Promise<void> {
await auth.logout();
await router.replace({ name: "login" });
}
</script>
<template>
<aside
class="min-w-0 border-b border-zinc-200 bg-white/70 px-4 py-4 dark:border-zinc-800 dark:bg-zinc-900/70 lg:h-full lg:min-h-0 lg:overflow-y-auto lg:border-b-0 lg:border-r lg:px-5 lg:py-6"
>
<div class="flex h-full min-h-0 flex-col gap-4">
<nav class="grid grid-cols-2 content-start gap-2 sm:grid-cols-3 lg:grid-cols-1" aria-label="Primary">
<RouterLink
v-for="link in links"
:key="link.to"
:to="link.to"
class="inline-flex min-h-10 min-w-0 items-center rounded-lg border border-transparent px-3 py-2 text-sm font-medium text-zinc-600 transition hover:border-zinc-300 hover:bg-zinc-50 hover:text-zinc-900 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-brand-500 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 dark:text-zinc-300 dark:hover:border-zinc-700 dark:hover:bg-zinc-800 dark:hover:text-zinc-100 dark:focus-visible:ring-brand-400 dark:focus-visible:ring-offset-zinc-950"
active-class="border-zinc-900 bg-zinc-900 text-white hover:border-zinc-900 hover:bg-zinc-900 hover:text-white dark:border-brand-400 dark:bg-brand-400 dark:text-zinc-950 dark:hover:border-brand-300 dark:hover:bg-brand-300"
>
{{ link.label }}
</RouterLink>
</nav>
<div class="mt-auto grid gap-2 border-t border-zinc-200 pt-3 dark:border-zinc-800">
<p class="truncate text-xs text-zinc-500 dark:text-zinc-400">{{ auth.user?.email }}</p>
<AppButton variant="ghost" class="w-full justify-start" @click="onLogout">Logout</AppButton>
</div>
</div>
</aside>
</template>

16
frontend/src/components/layout/AppPage.vue Normal file
View file

@ -0,0 +1,16 @@
<script setup lang="ts">
defineProps<{
title: string;
subtitle?: string;
}>();
</script>
<template>
<section class="page-stack min-w-0">
<header class="space-y-2">
<h1 class="page-title">{{ title }}</h1>
<p v-if="subtitle" class="page-subtitle">{{ subtitle }}</p>
</header>
<slot />
</section>
</template>

22
frontend/src/components/patterns/QueueHealthBadge.vue Normal file
View file

@ -0,0 +1,22 @@
<script setup lang="ts">
const props = defineProps<{
queued: number;
retrying: number;
dropped: number;
}>();
</script>
<template>
<span
class="inline-flex flex-wrap items-center gap-2 rounded-full border border-zinc-300 bg-zinc-100 px-3 py-1 text-xs text-zinc-700 dark:border-zinc-700 dark:bg-zinc-800 dark:text-zinc-300"
>
<span class="font-medium">Queue</span>
<span class="rounded-full bg-zinc-200 px-2 py-0.5 dark:bg-zinc-700">{{ props.queued }} queued</span>
<span class="rounded-full bg-amber-100 px-2 py-0.5 text-amber-800 dark:bg-amber-950/50 dark:text-amber-300">
{{ props.retrying }} retrying
</span>
<span class="rounded-full bg-rose-100 px-2 py-0.5 text-rose-700 dark:bg-rose-950/50 dark:text-rose-300">
{{ props.dropped }} dropped
</span>
</span>
</template>

18
frontend/src/components/patterns/RequestErrorPanel.vue Normal file
View file

@ -0,0 +1,18 @@
<script setup lang="ts">
import AppAlert from "@/components/ui/AppAlert.vue";
import { useUiStore } from "@/stores/ui";
const ui = useUiStore();
</script>
<template>
<div v-if="ui.globalError" class="sticky top-[4.5rem] z-20 px-4 pb-2 sm:px-6 lg:px-8">
<div class="mx-auto w-full max-w-7xl">
<AppAlert tone="danger" :dismissible="true" @dismiss="ui.clearGlobalError()">
<template #title>Request failed</template>
<p>{{ ui.globalError.message }}</p>
<p class="text-secondary">Request ID: {{ ui.globalError.requestId || "n/a" }}</p>
</AppAlert>
</div>
</div>
</template>

34
frontend/src/components/patterns/RunStatusBadge.vue Normal file
View file

@ -0,0 +1,34 @@
<script setup lang="ts">
import { computed } from "vue";
const props = defineProps<{
status: string;
}>();
const label = computed(() => props.status.replaceAll("_", " "));
const toneClass = computed(() => {
if (props.status === "success") {
return "border-emerald-300 bg-emerald-50 text-emerald-700 dark:border-emerald-800 dark:bg-emerald-950/50 dark:text-emerald-300";
}
if (props.status === "partial_failure") {
return "border-amber-300 bg-amber-50 text-amber-800 dark:border-amber-800 dark:bg-amber-950/50 dark:text-amber-300";
}
if (props.status === "failed") {
return "border-rose-300 bg-rose-50 text-rose-700 dark:border-rose-800 dark:bg-rose-950/50 dark:text-rose-300";
}
if (props.status === "running") {
return "border-brand-300 bg-brand-50 text-brand-700 dark:border-brand-800 dark:bg-brand-950/50 dark:text-brand-300";
}
return "border-zinc-300 bg-zinc-100 text-zinc-700 dark:border-zinc-700 dark:bg-zinc-800 dark:text-zinc-300";
});
</script>
<template>
<span
class="inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-semibold capitalize"
:class="toneClass"
>
{{ label }}
</span>
</template>

49
frontend/src/components/ui/AppAlert.vue Normal file
View file

@ -0,0 +1,49 @@
<script setup lang="ts">
import { computed } from "vue";
import AppButton from "@/components/ui/AppButton.vue";
const props = withDefaults(
defineProps<{
tone?: "danger" | "warning" | "info" | "success";
dismissible?: boolean;
}>(),
{
tone: "info",
dismissible: false,
},
);
const emit = defineEmits<{ dismiss: [] }>();
const toneClass = computed(() => {
if (props.tone === "danger") {
return "border-rose-300 bg-rose-50 text-rose-900 dark:border-rose-800 dark:bg-rose-950/45 dark:text-rose-200";
}
if (props.tone === "warning") {
return "border-amber-300 bg-amber-50 text-amber-900 dark:border-amber-800 dark:bg-amber-950/45 dark:text-amber-200";
}
if (props.tone === "success") {
return "border-emerald-300 bg-emerald-50 text-emerald-900 dark:border-emerald-800 dark:bg-emerald-950/45 dark:text-emerald-200";
}
return "border-brand-300 bg-brand-50 text-brand-900 dark:border-brand-800 dark:bg-brand-950/45 dark:text-brand-200";
});
const alertRole = computed(() => (props.tone === "danger" || props.tone === "warning" ? "alert" : "status"));
const alertLive = computed(() => (props.tone === "danger" || props.tone === "warning" ? "assertive" : "polite"));
</script>
<template>
<div
class="flex items-start justify-between gap-3 rounded-xl border px-4 py-3 text-sm"
:class="toneClass"
:role="alertRole"
:aria-live="alertLive"
>
<div class="space-y-1">
<strong v-if="$slots.title" class="block font-semibold"><slot name="title" /></strong>
<slot />
</div>
<AppButton v-if="props.dismissible" variant="ghost" @click="emit('dismiss')">Dismiss</AppButton>
</div>
</template>

33
frontend/src/components/ui/AppBadge.vue Normal file
View file

@ -0,0 +1,33 @@
<script setup lang="ts">
import { computed } from "vue";
const props = defineProps<{
tone?: "neutral" | "success" | "warning" | "danger" | "info";
}>();
const toneClass = computed(() => {
if (props.tone === "success") {
return "border-emerald-300 bg-emerald-50 text-emerald-700 dark:border-emerald-800 dark:bg-emerald-950/50 dark:text-emerald-300";
}
if (props.tone === "warning") {
return "border-amber-300 bg-amber-50 text-amber-800 dark:border-amber-800 dark:bg-amber-950/50 dark:text-amber-300";
}
if (props.tone === "danger") {
return "border-rose-300 bg-rose-50 text-rose-700 dark:border-rose-800 dark:bg-rose-950/50 dark:text-rose-300";
}
if (props.tone === "info") {
return "border-brand-300 bg-brand-50 text-brand-700 dark:border-brand-800 dark:bg-brand-950/50 dark:text-brand-300";
}
return "border-zinc-300 bg-zinc-100 text-zinc-700 dark:border-zinc-700 dark:bg-zinc-800 dark:text-zinc-300";
});
</script>
<template>
<span class="inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-medium" :class="toneClass">
<slot />
</span>
</template>

43
frontend/src/components/ui/AppButton.vue Normal file
View file

@ -0,0 +1,43 @@
<script setup lang="ts">
import { computed } from "vue";
const props = withDefaults(
defineProps<{
variant?: "primary" | "secondary" | "ghost" | "danger";
type?: "button" | "submit" | "reset";
disabled?: boolean;
}>(),
{
variant: "primary",
type: "button",
disabled: false,
},
);
const variantClass = computed(() => {
if (props.variant === "secondary") {
return "border-zinc-300 bg-zinc-100 text-zinc-900 hover:bg-zinc-200 dark:border-zinc-700 dark:bg-zinc-800 dark:text-zinc-100 dark:hover:bg-zinc-700";
}
if (props.variant === "ghost") {
return "border-zinc-300 bg-transparent text-zinc-700 hover:bg-zinc-100 dark:border-zinc-700 dark:text-zinc-200 dark:hover:bg-zinc-800";
}
if (props.variant === "danger") {
return "border-rose-300 bg-rose-100 text-rose-800 hover:bg-rose-200 dark:border-rose-800 dark:bg-rose-950/60 dark:text-rose-200 dark:hover:bg-rose-900/70";
}
return "border-brand-700 bg-brand-700 text-white hover:bg-brand-600 dark:border-brand-400 dark:bg-brand-400 dark:text-zinc-950 dark:hover:bg-brand-300";
});
</script>
<template>
<button
class="inline-flex min-h-10 items-center justify-center rounded-lg border px-3 py-2 text-sm font-semibold transition focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-brand-500 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 disabled:cursor-not-allowed disabled:opacity-60 dark:focus-visible:ring-brand-400 dark:focus-visible:ring-offset-zinc-950"
:class="variantClass"
:type="props.type"
:disabled="props.disabled"
>
<slot />
</button>
</template>

5
frontend/src/components/ui/AppCard.vue Normal file
View file

@ -0,0 +1,5 @@
<template>
<article class="min-w-0 rounded-2xl border border-zinc-200 bg-white p-5 shadow-panel dark:border-zinc-800 dark:bg-zinc-900">
<slot />
</article>
</template>

22
frontend/src/components/ui/AppCheckbox.vue Normal file
View file

@ -0,0 +1,22 @@
<script setup lang="ts">
const model = defineModel<boolean>({ default: false });
defineProps<{
id?: string;
disabled?: boolean;
label?: string;
}>();
</script>
<template>
<label class="inline-flex min-h-10 items-center gap-2 text-sm text-zinc-800 dark:text-zinc-200" :for="id">
<input
:id="id"
v-model="model"
class="h-4 w-4 rounded border-zinc-400 text-brand-600 focus-visible:ring-2 focus-visible:ring-brand-500 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 dark:border-zinc-600 dark:bg-zinc-900 dark:focus-visible:ring-brand-400 dark:focus-visible:ring-offset-zinc-950"
type="checkbox"
:disabled="disabled"
/>
<span><slot>{{ label }}</slot></span>
</label>
</template>

16
frontend/src/components/ui/AppEmptyState.vue Normal file
View file

@ -0,0 +1,16 @@
<script setup lang="ts">
defineProps<{
title: string;
body: string;
}>();
</script>
<template>
<section class="rounded-xl border border-dashed border-zinc-300 bg-zinc-50 p-5 dark:border-zinc-700 dark:bg-zinc-900/60">
<h3 class="font-display text-lg font-semibold text-zinc-900 dark:text-zinc-100">{{ title }}</h3>
<p class="mt-1 text-sm text-zinc-600 dark:text-zinc-400">{{ body }}</p>
<div v-if="$slots.default" class="mt-4">
<slot />
</div>
</section>
</template>

21
frontend/src/components/ui/AppInput.vue Normal file
View file

@ -0,0 +1,21 @@
<script setup lang="ts">
const model = defineModel<string>({ default: "" });
defineProps<{
id?: string;
type?: string;
placeholder?: string;
disabled?: boolean;
}>();
</script>
<template>
<input
v-model="model"
class="w-full rounded-lg border border-zinc-300 bg-white px-3 py-2 text-sm text-zinc-900 outline-none ring-brand-300 transition placeholder:text-zinc-400 focus-visible:ring-2 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 disabled:cursor-not-allowed disabled:opacity-60 dark:border-zinc-700 dark:bg-zinc-900 dark:text-zinc-100 dark:ring-brand-400 dark:focus-visible:ring-offset-zinc-950 dark:placeholder:text-zinc-500"
:id="id"
:type="type || 'text'"
:placeholder="placeholder"
:disabled="disabled"
/>
</template>

17
frontend/src/components/ui/AppModal.vue Normal file
View file

@ -0,0 +1,17 @@
<script setup lang="ts">
const props = defineProps<{
open: boolean;
title: string;
}>();
const emit = defineEmits<{ close: [] }>();
</script>
<template>
<div v-if="props.open" class="fixed inset-0 z-50 grid place-items-center bg-zinc-950/60 p-4" @click.self="emit('close')">
<div class="w-full max-w-lg rounded-2xl border border-zinc-200 bg-white p-6 shadow-panel dark:border-zinc-800 dark:bg-zinc-900">
<h3 class="mb-4 font-display text-xl font-semibold text-zinc-900 dark:text-zinc-100">{{ props.title }}</h3>
<slot />
</div>
</div>
</template>

19
frontend/src/components/ui/AppSelect.vue Normal file
View file

@ -0,0 +1,19 @@
<script setup lang="ts">
const model = defineModel<string>({ default: "" });
defineProps<{
id?: string;
disabled?: boolean;
}>();
</script>
<template>
<select
v-model="model"
class="w-full rounded-lg border border-zinc-300 bg-white px-3 py-2 text-sm text-zinc-900 outline-none ring-brand-300 transition focus-visible:ring-2 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 disabled:cursor-not-allowed disabled:opacity-60 dark:border-zinc-700 dark:bg-zinc-900 dark:text-zinc-100 dark:ring-brand-400 dark:focus-visible:ring-offset-zinc-950"
:id="id"
:disabled="disabled"
>
<slot />
</select>
</template>

15
frontend/src/components/ui/AppSkeleton.vue Normal file
View file

@ -0,0 +1,15 @@
<script setup lang="ts">
defineProps<{
lines?: number;
}>();
</script>
<template>
<div class="space-y-2" role="status" aria-live="polite">
<span
v-for="line in lines || 3"
:key="line"
class="block h-3 animate-pulse rounded-full bg-zinc-200 dark:bg-zinc-800"
/>
</div>
</template>

31
frontend/src/components/ui/AppTable.vue Normal file
View file

@ -0,0 +1,31 @@
<script setup lang="ts">
defineProps<{
label?: string;
}>();
</script>
<template>
<div
class="max-w-full overflow-x-auto rounded-xl border border-zinc-200 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-brand-500 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 dark:border-zinc-800 dark:focus-visible:ring-brand-400 dark:focus-visible:ring-offset-zinc-950"
tabindex="0"
>
<table class="min-w-full border-collapse bg-white text-sm dark:bg-zinc-900" :aria-label="label || 'Data table'">
<slot />
</table>
</div>
</template>
<style scoped>
:deep(th),
:deep(td) {
@apply border-b border-zinc-200 px-3 py-2 text-left align-top dark:border-zinc-800;
}
:deep(th) {
@apply sticky top-0 z-10 bg-zinc-100 font-semibold text-zinc-800 dark:bg-zinc-900 dark:text-zinc-200;
}
:deep(td) {
@apply text-zinc-700 dark:text-zinc-300;
}
</style>

1
frontend/src/env.d.ts vendored Normal file
View file

@ -0,0 +1 @@
/// <reference types="vite/client" />

49
frontend/src/features/admin_users/index.ts Normal file
View file

@ -0,0 +1,49 @@
import { apiRequest } from "@/lib/api/client";
export interface AdminUser {
id: number;
email: string;
is_active: boolean;
is_admin: boolean;
created_at: string;
updated_at: string;
}
interface UsersListData {
users: AdminUser[];
}
export interface CreateAdminUserPayload {
email: string;
password: string;
is_admin: boolean;
}
export async function listAdminUsers(): Promise<AdminUser[]> {
const response = await apiRequest<UsersListData>("/admin/users", { method: "GET" });
return response.data.users;
}
export async function createAdminUser(payload: CreateAdminUserPayload): Promise<AdminUser> {
const response = await apiRequest<AdminUser>("/admin/users", {
method: "POST",
body: payload,
});
return response.data;
}
export async function setAdminUserActive(userId: number, isActive: boolean): Promise<AdminUser> {
const response = await apiRequest<AdminUser>(`/admin/users/${userId}/active`, {
method: "PATCH",
body: { is_active: isActive },
});
return response.data;
}
export async function resetAdminUserPassword(userId: number, newPassword: string): Promise<{ message: string }> {
const response = await apiRequest<{ message: string }>(`/admin/users/${userId}/reset-password`, {
method: "POST",
body: { new_password: newPassword },
});
return response.data;
}

9
frontend/src/features/auth/index.ts Normal file
View file

@ -0,0 +1,9 @@
export { fetchCsrfBootstrap, type CsrfBootstrapData } from "@/lib/api/csrf";
export {
fetchMe,
loginSession,
logoutSession,
type AuthSessionData,
type MessageData,
type SessionUser,
} from "@/lib/auth/session";

58
frontend/src/features/dashboard/index.ts Normal file
View file

@ -0,0 +1,58 @@
import {
listPublications,
type PublicationItem,
type PublicationMode,
} from "@/features/publications";
import { listQueueItems, listRuns, type RunListItem } from "@/features/runs";
export interface QueueHealth {
queued: number;
retrying: number;
dropped: number;
}
export interface DashboardSnapshot {
newCount: number;
totalCount: number;
mode: PublicationMode;
latestRun: RunListItem | null;
recentRuns: RunListItem[];
recentPublications: PublicationItem[];
queue: QueueHealth;
}
function countQueueStatuses(statuses: string[]): QueueHealth {
return statuses.reduce<QueueHealth>(
(acc, status) => {
if (status === "queued") {
acc.queued += 1;
} else if (status === "retrying") {
acc.retrying += 1;
} else if (status === "dropped") {
acc.dropped += 1;
}
return acc;
},
{ queued: 0, retrying: 0, dropped: 0 },
);
}
export async function fetchDashboardSnapshot(): Promise<DashboardSnapshot> {
const [publications, runs, queueItems] = await Promise.all([
listPublications({ mode: "new", limit: 20 }),
listRuns({ limit: 5 }),
listQueueItems(200),
]);
const queueHealth = countQueueStatuses(queueItems.map((item) => item.status));
return {
newCount: publications.new_count,
totalCount: publications.total_count,
mode: publications.mode,
latestRun: runs[0] ?? null,
recentRuns: runs,
recentPublications: publications.publications,
queue: queueHealth,
};
}

81
frontend/src/features/publications/index.ts Normal file
View file

@ -0,0 +1,81 @@
import { apiRequest } from "@/lib/api/client";
export type PublicationMode = "all" | "new";
export interface PublicationItem {
publication_id: number;
scholar_profile_id: number;
scholar_label: string;
title: string;
year: number | null;
citation_count: number;
venue_text: string | null;
pub_url: string | null;
is_read: boolean;
first_seen_at: string;
is_new_in_latest_run: boolean;
}
export interface PublicationsResult {
mode: PublicationMode;
selected_scholar_profile_id: number | null;
new_count: number;
total_count: number;
publications: PublicationItem[];
}
export interface PublicationsQuery {
mode?: PublicationMode;
scholarProfileId?: number;
limit?: number;
}
export interface PublicationSelection {
scholar_profile_id: number;
publication_id: number;
}
export async function listPublications(query: PublicationsQuery = {}): Promise<PublicationsResult> {
const params = new URLSearchParams();
if (query.mode) {
params.set("mode", query.mode);
}
if (query.scholarProfileId) {
params.set("scholar_profile_id", String(query.scholarProfileId));
}
if (query.limit) {
params.set("limit", String(query.limit));
}
const suffix = params.toString();
const response = await apiRequest<PublicationsResult>(
`/publications${suffix ? `?${suffix}` : ""}`,
{ method: "GET" },
);
return response.data;
}
export async function markAllRead(): Promise<{ message: string; updated_count: number }> {
const response = await apiRequest<{ message: string; updated_count: number }>(
"/publications/mark-all-read",
{ method: "POST" },
);
return response.data;
}
export async function markSelectedRead(selections: PublicationSelection[]): Promise<{
message: string;
requested_count: number;
updated_count: number;
}> {
const response = await apiRequest<{
message: string;
requested_count: number;
updated_count: number;
}>("/publications/mark-read", {
method: "POST",
body: { selections },
});
return response.data;
}

171
frontend/src/features/runs/index.ts Normal file
View file

@ -0,0 +1,171 @@
import { apiRequest } from "@/lib/api/client";
export interface RunListItem {
id: number;
trigger_type: string;
status: string;
start_dt: string;
end_dt: string | null;
scholar_count: number;
new_publication_count: number;
failed_count: number;
partial_count: number;
}
export interface RunSummary {
succeeded_count: number;
failed_count: number;
partial_count: number;
failed_state_counts: Record<string, number>;
failed_reason_counts: Record<string, number>;
}
export interface RunScholarResult {
scholar_profile_id: number;
scholar_id: string;
state: string;
state_reason: string | null;
outcome: string;
attempt_count: number;
publication_count: number;
start_cstart: number;
continuation_cstart: number | null;
continuation_enqueued: boolean;
continuation_cleared: boolean;
warnings: string[];
error: string | null;
debug: Record<string, unknown> | null;
}
export interface RunDetail {
run: RunListItem;
summary: RunSummary;
scholar_results: RunScholarResult[];
}
export interface QueueItem {
id: number;
scholar_profile_id: number;
scholar_label: string;
status: string;
reason: string;
dropped_reason: string | null;
attempt_count: number;
resume_cstart: number;
next_attempt_dt: string | null;
updated_at: string;
last_error: string | null;
last_run_id: number | null;
}
interface RunsListData {
runs: RunListItem[];
}
interface QueueListData {
queue_items: QueueItem[];
}
export interface RunsListQuery {
failedOnly?: boolean;
limit?: number;
}
export async function listRuns(query: RunsListQuery = {}): Promise<RunListItem[]> {
const params = new URLSearchParams();
if (query.failedOnly) {
params.set("failed_only", "true");
}
if (query.limit) {
params.set("limit", String(query.limit));
}
const suffix = params.toString();
const response = await apiRequest<RunsListData>(`/runs${suffix ? `?${suffix}` : ""}`, {
method: "GET",
});
return response.data.runs;
}
export async function getRunDetail(runId: number): Promise<RunDetail> {
const response = await apiRequest<RunDetail>(`/runs/${runId}`, { method: "GET" });
return response.data;
}
function generateIdempotencyKey(): string {
const randomUuid = globalThis.crypto?.randomUUID?.();
if (randomUuid) {
return randomUuid;
}
return `${Date.now()}-${Math.random().toString(16).slice(2)}`;
}
export async function triggerManualRun(): Promise<{
run_id: number;
status: string;
scholar_count: number;
succeeded_count: number;
failed_count: number;
partial_count: number;
new_publication_count: number;
reused_existing_run: boolean;
idempotency_key: string | null;
}> {
const headers: Record<string, string> = {
"Idempotency-Key": generateIdempotencyKey(),
};
const response = await apiRequest<{
run_id: number;
status: string;
scholar_count: number;
succeeded_count: number;
failed_count: number;
partial_count: number;
new_publication_count: number;
reused_existing_run: boolean;
idempotency_key: string | null;
}>("/runs/manual", {
method: "POST",
headers,
});
return response.data;
}
export async function listQueueItems(limit = 200): Promise<QueueItem[]> {
const response = await apiRequest<QueueListData>(`/runs/queue/items?limit=${limit}`, {
method: "GET",
});
return response.data.queue_items;
}
export async function retryQueueItem(queueItemId: number): Promise<QueueItem> {
const response = await apiRequest<QueueItem>(`/runs/queue/${queueItemId}/retry`, {
method: "POST",
});
return response.data;
}
export async function dropQueueItem(queueItemId: number): Promise<QueueItem> {
const response = await apiRequest<QueueItem>(`/runs/queue/${queueItemId}/drop`, {
method: "POST",
});
return response.data;
}
export async function clearQueueItem(queueItemId: number): Promise<{
queue_item_id: number;
previous_status: string;
status: string;
message: string;
}> {
const response = await apiRequest<{
queue_item_id: number;
previous_status: string;
status: string;
message: string;
}>(`/runs/queue/${queueItemId}`, {
method: "DELETE",
});
return response.data;
}

117
frontend/src/features/scholars/index.ts Normal file
View file

@ -0,0 +1,117 @@
import { apiRequest } from "@/lib/api/client";
export interface ScholarProfile {
id: number;
scholar_id: string;
display_name: string | null;
profile_image_url: string | null;
profile_image_source: "upload" | "override" | "scraped" | "none";
is_enabled: boolean;
baseline_completed: boolean;
last_run_dt: string | null;
last_run_status: string | null;
}
export interface ScholarCreatePayload {
scholar_id: string;
profile_image_url?: string;
}
export interface ScholarSearchCandidate {
scholar_id: string;
display_name: string;
affiliation: string | null;
email_domain: string | null;
cited_by_count: number | null;
interests: string[];
profile_url: string;
profile_image_url: string | null;
}
export interface ScholarSearchResult {
query: string;
state: "ok" | "no_results" | "blocked_or_captcha" | "layout_changed" | "network_error";
state_reason: string;
action_hint: string | null;
candidates: ScholarSearchCandidate[];
warnings: string[];
}
interface ScholarsListData {
scholars: ScholarProfile[];
}
interface ScholarSearchData extends ScholarSearchResult {}
export async function listScholars(): Promise<ScholarProfile[]> {
const response = await apiRequest<ScholarsListData>("/scholars", { method: "GET" });
return response.data.scholars;
}
export async function createScholar(payload: ScholarCreatePayload): Promise<ScholarProfile> {
const response = await apiRequest<ScholarProfile>("/scholars", {
method: "POST",
body: payload,
});
return response.data;
}
export async function toggleScholar(scholarProfileId: number): Promise<ScholarProfile> {
const response = await apiRequest<ScholarProfile>(`/scholars/${scholarProfileId}/toggle`, {
method: "PATCH",
});
return response.data;
}
export async function deleteScholar(scholarProfileId: number): Promise<void> {
await apiRequest<{ message: string }>(`/scholars/${scholarProfileId}`, {
method: "DELETE",
});
}
export async function searchScholarsByName(
query: string,
limit = 10,
): Promise<ScholarSearchResult> {
const searchParams = new URLSearchParams({
query,
limit: String(limit),
});
const response = await apiRequest<ScholarSearchData>(`/scholars/search?${searchParams.toString()}`, {
method: "GET",
});
return response.data;
}
export async function setScholarImageUrl(
scholarProfileId: number,
imageUrl: string,
): Promise<ScholarProfile> {
const response = await apiRequest<ScholarProfile>(`/scholars/${scholarProfileId}/image/url`, {
method: "PUT",
body: { image_url: imageUrl },
});
return response.data;
}
export async function uploadScholarImage(
scholarProfileId: number,
file: File,
): Promise<ScholarProfile> {
const form = new FormData();
form.append("image", file);
const response = await apiRequest<ScholarProfile>(`/scholars/${scholarProfileId}/image/upload`, {
method: "POST",
body: form,
});
return response.data;
}
export async function clearScholarImage(
scholarProfileId: number,
): Promise<ScholarProfile> {
const response = await apiRequest<ScholarProfile>(`/scholars/${scholarProfileId}/image`, {
method: "DELETE",
});
return response.data;
}

34
frontend/src/features/settings/index.ts Normal file
View file

@ -0,0 +1,34 @@
import { apiRequest } from "@/lib/api/client";
export interface UserSettings {
auto_run_enabled: boolean;
run_interval_minutes: number;
request_delay_seconds: number;
}
export interface ChangePasswordPayload {
current_password: string;
new_password: string;
confirm_password: string;
}
export async function fetchSettings(): Promise<UserSettings> {
const response = await apiRequest<UserSettings>("/settings", { method: "GET" });
return response.data;
}
export async function updateSettings(payload: UserSettings): Promise<UserSettings> {
const response = await apiRequest<UserSettings>("/settings", {
method: "PUT",
body: payload,
});
return response.data;
}
export async function changePassword(payload: ChangePasswordPayload): Promise<{ message: string }> {
const response = await apiRequest<{ message: string }>("/auth/change-password", {
method: "POST",
body: payload,
});
return response.data;
}

Some files were not shown because too many files have changed in this diff Show more