From f71841e92275c92196d8dd61c01726e62c41a7c0 Mon Sep 17 00:00:00 2001 From: Justin Visser Date: Tue, 17 Feb 2026 15:21:08 +0100 Subject: [PATCH] removed old UI --- .env.example | 2 +- AGENTS.MD | 220 ++++ README.md | 223 ++-- app/api/deps.py | 5 +- app/api/routers/auth.py | 10 +- app/api/routers/runs.py | 2 +- app/{web/deps.py => api/runtime_deps.py} | 1 - app/auth/runtime.py | 55 + app/http/__init__.py | 1 + app/{web => http}/middleware.py | 2 +- app/main.py | 37 +- app/presentation/__init__.py | 2 - app/presentation/dashboard.py | 108 -- app/security/csrf.py | 2 - app/settings.py | 2 +- app/static/app.css | 501 ------- app/static/app.js | 76 -- app/static/theme.css | 98 -- app/static/theme.js | 56 - app/templates/account_password.html | 23 - app/templates/base.html | 66 - app/templates/dashboard/_run_controls.html | 23 - app/templates/dashboard/_run_history.html | 38 - .../dashboard/_unread_publications.html | 47 - app/templates/index.html | 27 - app/templates/login.html | 23 - app/templates/publications.html | 104 -- app/templates/run_detail.html | 98 -- app/templates/runs.html | 129 -- app/templates/scholars.html | 99 -- app/templates/settings.html | 44 - app/templates/users.html | 75 -- app/theme.py | 24 - app/web/__init__.py | 2 - app/web/common.py | 140 -- app/web/routers/__init__.py | 2 - app/web/routers/admin.py | 209 --- app/web/routers/auth.py | 214 --- app/web/routers/dashboard.py | 167 --- app/web/routers/health.py | 15 - app/web/routers/publications.py | 165 --- app/web/routers/runs.py | 274 ---- app/web/routers/scholars.py | 212 --- app/web/routers/settings.py | 94 -- docker-compose.yml | 7 +- planning/mvp_implementation_reminders.md | 92 -- planning/scholar_probe_tmp/README.md | 13 - .../notes/findings_phase1_input.md | 71 - pyproject.toml | 2 - tests/integration/helpers.py | 30 +- .../integration/test_admin_user_management.py | 200 --- tests/integration/test_auth_flow.py | 109 -- .../integration/test_manual_ingestion_flow.py | 1151 ----------------- tests/integration/test_queue_ui.py | 227 ---- tests/integration/test_tenant_routes.py | 197 --- tests/unit/test_auth.py | 204 --- tests/unit/test_dashboard_view_model.py | 88 -- tests/unit/test_healthz.py | 4 +- tests/unit/test_logging.py | 15 +- tests/unit/test_pages.py | 54 - tests/unit/test_phase2_pages.py | 20 - uv.lock | 25 - 62 files changed, 411 insertions(+), 5815 deletions(-) create mode 100644 AGENTS.MD rename app/{web/deps.py => api/runtime_deps.py} (99%) create mode 100644 app/auth/runtime.py create mode 100644 app/http/__init__.py rename app/{web => http}/middleware.py (100%) delete mode 100644 app/presentation/__init__.py delete mode 100644 app/presentation/dashboard.py delete mode 100644 app/static/app.css delete mode 100644 app/static/app.js delete mode 100644 app/static/theme.css delete mode 100644 app/static/theme.js delete mode 100644 app/templates/account_password.html delete mode 100644 app/templates/base.html delete mode 100644 app/templates/dashboard/_run_controls.html delete mode 100644 app/templates/dashboard/_run_history.html delete mode 100644 app/templates/dashboard/_unread_publications.html delete mode 100644 app/templates/index.html delete mode 100644 app/templates/login.html delete mode 100644 app/templates/publications.html delete mode 100644 app/templates/run_detail.html delete mode 100644 app/templates/runs.html delete mode 100644 app/templates/scholars.html delete mode 100644 app/templates/settings.html delete mode 100644 app/templates/users.html delete mode 100644 app/theme.py delete mode 100644 app/web/__init__.py delete mode 100644 app/web/common.py delete mode 100644 app/web/routers/__init__.py delete mode 100644 app/web/routers/admin.py delete mode 100644 app/web/routers/auth.py delete mode 100644 app/web/routers/dashboard.py delete mode 100644 app/web/routers/health.py delete mode 100644 app/web/routers/publications.py delete mode 100644 app/web/routers/runs.py delete mode 100644 app/web/routers/scholars.py delete mode 100644 app/web/routers/settings.py delete mode 100644 planning/mvp_implementation_reminders.md delete mode 100644 planning/scholar_probe_tmp/README.md delete mode 100644 planning/scholar_probe_tmp/notes/findings_phase1_input.md delete mode 100644 tests/integration/test_admin_user_management.py delete mode 100644 tests/integration/test_auth_flow.py delete mode 100644 tests/integration/test_manual_ingestion_flow.py delete mode 100644 tests/integration/test_queue_ui.py delete mode 100644 tests/integration/test_tenant_routes.py delete mode 100644 tests/unit/test_auth.py delete mode 100644 tests/unit/test_dashboard_view_model.py delete mode 100644 tests/unit/test_pages.py delete mode 100644 tests/unit/test_phase2_pages.py diff --git a/.env.example b/.env.example index 82480f1..dc1a02a 100644 --- a/.env.example +++ b/.env.example @@ -22,7 +22,7 @@ LOG_LEVEL=INFO LOG_FORMAT=console LOG_REQUESTS=1 LOG_UVICORN_ACCESS=0 -LOG_REQUEST_SKIP_PATHS=/healthz,/static/ +LOG_REQUEST_SKIP_PATHS=/healthz LOG_REDACT_FIELDS= SCHEDULER_ENABLED=1 SCHEDULER_TICK_SECONDS=60 diff --git a/AGENTS.MD b/AGENTS.MD new file mode 100644 index 0000000..b2aa84e --- /dev/null +++ b/AGENTS.MD @@ -0,0 +1,220 @@ +# scholarr Agent Handbook + +This file is the consolidated source of truth for project scope, constraints, scrape contract, and UI implementation flow. + +Raw probe artifacts in `planning/scholar_probe_tmp/notes/*.json` and `planning/scholar_probe_tmp/notes/*.md` remain historical evidence and fixture references. + +## 1. Product Objective + +- Build a self-hosted scholar tracking system ("scholarr") with reliable, low-cost scraping and clear, actionable UI. +- Keep the MVP scope intentionally small. +- Prioritize reliability and ease of use over advanced/fancy processing. + +## 2. Locked Constraints and Preferences + +- Multi-user system with strict tenant isolation. +- Users are admin-created only. +- Users can change their own password. +- Admin features are shown only to admin users. +- Same-origin cookie session model with CSRF protection. +- Container-first development workflow (`docker compose`, `uv`). +- Test while developing; avoid shipping untested flows. +- Keep backend modular and DRY. +- UI is being rebuilt from scratch against API contracts. + +## 3. Current Backend Architecture + +- Runtime: Python + FastAPI. +- DB: PostgreSQL + SQLAlchemy + Alembic migrations. +- Scheduler: background scheduler + continuation queue processing. +- Auth: + - Argon2 password hashing. + - Session cookie (`HttpOnly`, `SameSite=Lax`, secure flag configurable). + - CSRF required for unsafe methods via `X-CSRF-Token`. +- Logging: + - structured request logging with request IDs. + - redaction controls for sensitive fields. + +## 4. Scraping Contract (Probe-Based) + +Status: sufficient for MVP implementation with graceful degradation. + +Target endpoint: +- `https://scholar.google.com/citations?hl=en&user=` + +Primary selectors: +- Row: `tr.gsc_a_tr` +- Title/link: `a.gsc_a_at` +- Citation count: `a.gsc_a_ac` +- Year: `span.gsc_a_h` (fallback regex on year cell text) +- Metadata lines: first/second `div.gs_gray` -> authors/venue +- Cluster ID: from `citation_for_view=:` in title URL + +Required parser states: +- `ok` +- `no_results` +- `blocked_or_captcha` +- `layout_changed` +- `network_error` + +Required page flags: +- `has_show_more_button` +- `articles_range` + +Quality assumptions from probe: +- title/cluster_id/citation/authors coverage observed near 100% +- year and venue may be missing and must remain nullable + +Guardrails: +- Never crash the full run when one scholar fails. +- Persist structured failure/debug information for diagnosis. +- Handle inaccessible/redirected IDs as states, not exceptions. + +## 5. Current API Scope + +Base path: `/api/v1` + +Envelope model: +- success: `{"data": ..., "meta": {"request_id": "..."}}` +- error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}` + +Auth/session: +- `GET /api/v1/auth/csrf` +- `POST /api/v1/auth/login` +- `GET /api/v1/auth/me` +- `POST /api/v1/auth/change-password` +- `POST /api/v1/auth/logout` + +Admin users: +- `GET /api/v1/admin/users` +- `POST /api/v1/admin/users` +- `PATCH /api/v1/admin/users/{id}/active` +- `POST /api/v1/admin/users/{id}/reset-password` + +Scholars: +- `GET /api/v1/scholars` +- `POST /api/v1/scholars` +- `PATCH /api/v1/scholars/{id}/toggle` +- `DELETE /api/v1/scholars/{id}` + +Settings: +- `GET /api/v1/settings` +- `PUT /api/v1/settings` + +Runs/diagnostics/queue: +- `GET /api/v1/runs` +- `GET /api/v1/runs/{id}` +- `POST /api/v1/runs/manual` (`Idempotency-Key` supported) +- `GET /api/v1/runs/queue/items` +- `POST /api/v1/runs/queue/{id}/retry` +- `POST /api/v1/runs/queue/{id}/drop` +- `DELETE /api/v1/runs/queue/{id}` + +Publications: +- `GET /api/v1/publications` +- `POST /api/v1/publications/mark-all-read` + +Ops: +- `GET /healthz` + +## 6. Required UI Behavior (MVP) + +Core UX entities: +- scholars +- publications (`all` and `new`) +- runs + run diagnostics +- queue visibility/actions +- settings/account +- admin users (role-gated) + +Critical semantics: +- Keep `is_new_in_latest_run` separate from `is_read`. +- First-time ingestion should not imply user read-state. +- Surface partial runs and warnings clearly. +- Show loading, empty, and error states explicitly. + +## 7. UI Information Architecture + +Public: +- Login + +Authenticated: +- Dashboard +- Scholars +- Publications +- Settings +- Admin Users (admin only) + +## 8. End-to-End UI Flow Plan + +## 8.1 Session bootstrap + +1. On load call `GET /api/v1/auth/csrf`. +2. Call `GET /api/v1/auth/me`. +3. Route to login or dashboard based on auth state. + +## 8.2 Login/logout + +1. Login form submits `POST /api/v1/auth/login` with `X-CSRF-Token`. +2. Store CSRF token from response for future unsafe requests. +3. Logout calls `POST /api/v1/auth/logout`. + +## 8.3 Dashboard + + + +## 8.4 Scholars + +1. List via `GET /api/v1/scholars`. +2. Add/toggle/delete actions mapped to scholar endpoints. +3. Per-row quick links: + - all publications filtered by scholar + - new publications filtered by scholar + +## 8.5 Publications + +## 8.6 Runs and diagnostics (admin) + +1. List runs from `GET /api/v1/runs`. +2. Run details from `GET /api/v1/runs/{id}`: + - scholar result state/reason + - warnings + - page logs + attempt logs + - debug metadata +3. Queue operations from queue endpoints (`retry`, `drop`, `clear`). + +## 8.7 Settings + account + +1. User ingest settings from `/api/v1/settings`. +2. Password change via `/api/v1/auth/change-password`. + +## 8.8 Admin user management + +1. Show section only when `current_user.is_admin=true`. +2. Use admin user endpoints for create/activate/deactivate/reset password. + +## 9. Development Method + +- API-contract-first frontend. +- Vertical slices (complete flow before moving on). +- Shared API client module: + - always send credentials + - inject CSRF header for unsafe methods + - normalize envelope parsing and error handling +- Keep frontend modules separate (`auth`, `scholars`, `publications`, `runs`, `settings`, `admin`). +- Write tests while implementing each slice. + +## 10. Definition of Done (UI Readiness) + +- All main flows above implemented and mapped to existing API endpoints. +- Role-gated admin experience enforced in UI. +- Distinct new-vs-read publication semantics visible. +- Run/queue diagnostics visible and actionable. +- Reliable loading/empty/error states with request-id surfaced for support. +- No dependence on removed server-rendered UI layer. + +## 11. Open Items (Known Future Work) + +- Scholar discovery by real name (search Google Scholar candidates, select one, then add scholar ID). +- API contract freeze/changelog discipline for external UI clients. +- Additional API-first smoke coverage for end-to-end UI critical flows. diff --git a/README.md b/README.md index 21c0da2..934599a 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,80 @@ # scholarr -Container-first, self-hosted scholar tracking in the spirit of the `*arr` ecosystem, with strict multi-user isolation and an intentionally small operational footprint. +API-first, self-hosted scholar tracking backend in the spirit of the `*arr` ecosystem. + +The legacy server-rendered UI has been removed. This repository now focuses on core ingestion, scheduling, and multi-user API functionality. ## Current Scope -- Admin-managed users only (no public signup) -- Session auth with CSRF protection and login rate limiting -- Per-user scholar tracking list (add, enable/disable, delete) -- Per-user automation settings -- Manual per-user ingestion runs from dashboard -- Per-user run history and "new since last run" publication stream -- Publications library view with `new` and `all` modes plus scholar filtering -- Dedicated run diagnostics pages (`/runs`, `/runs/{id}`) with failed-only filtering -- Continuation queue visibility + controls on runs page (`retry`, `drop`, `clear`) -- Baseline-aware discovery tracking (`new` = discovered in latest completed run) -- Read state is user-controlled (`Mark All Read`), including baseline items -- Multi-page profile ingestion (`cstart`) to capture full publication lists -- Dashboard rendered via a presentation-layer view model and section template registry -- Structured application logging with request IDs and field redaction -- Ingestion overlap guard via Postgres advisory locks -- Network-error retry support for ingestion attempts -- Automatic continuation queue for resumable/limit-hit scholar runs -- Background scheduler for per-user automatic runs -- User self-service password changes -- Admin user management (create users, activate/deactivate, reset passwords) -- PostgreSQL + Alembic migrations + containerized test workflow -- Versioned backend API (`/api/v1`) for frontend decoupling (same-origin cookie session) +- Multi-user accounts with admin-managed user lifecycle +- Same-origin cookie sessions with CSRF enforcement +- API auth flows (login, logout, me, password change) +- Admin user management API +- Scholar CRUD per user +- Per-user ingestion settings +- Manual runs with idempotency support +- Run history, run detail diagnostics, and continuation queue actions +- Publication listing (`new` / `all`) and mark-all-read +- Ingestion scheduler + continuation queue retries +- Structured logging with request IDs + redaction +- PostgreSQL + Alembic migrations +- Container-first development workflow with `uv` -## Tech Stack +## Functionality Tracking -- Python 3.12+ -- FastAPI -- PostgreSQL 15+ -- SQLAlchemy 2 + Alembic -- Jinja2 templates + modular theme tokens -- `uv` for dependency and runtime workflow -- Docker / Docker Compose for development and CI parity +Planned and supported backend functionality is tracked in: + +- `AGENTS.MD` + +## API Base + +- Base path: `/api/v1` +- Success envelope: + - `{"data": ..., "meta": {"request_id": "..."}}` +- Error envelope: + - `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}` + +## Auth & Session Model + +- Session transport: same-origin cookie session (`HttpOnly`, `SameSite=Lax`) +- CSRF: + - Required for unsafe methods (`POST`, `PUT`, `PATCH`, `DELETE`) via `X-CSRF-Token` + - Bootstrap token via `GET /api/v1/auth/csrf` + +## API Surface + +- Auth: + - `GET /api/v1/auth/csrf` + - `POST /api/v1/auth/login` + - `GET /api/v1/auth/me` + - `POST /api/v1/auth/change-password` + - `POST /api/v1/auth/logout` +- Admin users: + - `GET /api/v1/admin/users` + - `POST /api/v1/admin/users` + - `PATCH /api/v1/admin/users/{id}/active` + - `POST /api/v1/admin/users/{id}/reset-password` +- Scholars: + - `GET /api/v1/scholars` + - `POST /api/v1/scholars` + - `PATCH /api/v1/scholars/{id}/toggle` + - `DELETE /api/v1/scholars/{id}` +- Settings: + - `GET /api/v1/settings` + - `PUT /api/v1/settings` +- Runs: + - `GET /api/v1/runs` + - `GET /api/v1/runs/{id}` + - `POST /api/v1/runs/manual` (`Idempotency-Key` supported) + - `GET /api/v1/runs/queue/items` + - `POST /api/v1/runs/queue/{id}/retry` + - `POST /api/v1/runs/queue/{id}/drop` + - `DELETE /api/v1/runs/queue/{id}` +- Publications: + - `GET /api/v1/publications` + - `POST /api/v1/publications/mark-all-read` +- Ops: + - `GET /healthz` ## Quick Start @@ -45,7 +84,7 @@ Container-first, self-hosted scholar tracking in the spirit of the `*arr` ecosys cp .env.example .env ``` -2. Bootstrap the first admin on startup (recommended for first run): +2. Optional bootstrap admin on startup: ```bash export BOOTSTRAP_ADMIN_ON_START=1 @@ -53,28 +92,20 @@ export BOOTSTRAP_ADMIN_EMAIL=admin@example.com export BOOTSTRAP_ADMIN_PASSWORD=change-me-now ``` -3. Start the stack: +3. Start stack: ```bash docker compose up --build ``` -4. Open: +4. Health check: -- Login: `http://localhost:8000/login` -- Dashboard: `http://localhost:8000/` -- Healthcheck: `http://localhost:8000/healthz` - -5. After first successful admin login, disable auto-bootstrap: - -```bash -export BOOTSTRAP_ADMIN_ON_START=0 +```text +http://localhost:8000/healthz ``` ## Admin Bootstrap (Manual) -You can create/update an admin account without restarting the app: - ```bash docker compose run --rm app uv run python scripts/bootstrap_admin.py \ --email admin@example.com \ @@ -82,18 +113,8 @@ docker compose run --rm app uv run python scripts/bootstrap_admin.py \ --force-password ``` -Notes: -- If the user does not exist, this creates an active admin account. -- If the user exists, it ensures admin + active flags. -- `--force-password` resets the password for existing users. - ## Test Workflow -- Integration/smoke tests run against `TEST_DATABASE_URL`. -- If `TEST_DATABASE_URL` is empty, test runs derive an isolated database from `DATABASE_URL` by appending `_test`. -- This keeps app data in `DATABASE_URL` untouched during test runs. -- Parser regression tests are pinned to captured real-world fixture pages in `tests/fixtures/scholar/regression`. - - Unit tests: ```bash @@ -106,101 +127,29 @@ docker compose run --rm app uv run pytest tests/unit docker compose run --rm app uv run pytest -m integration ``` -- Smoke checks (container + DB + migration sanity): +- Smoke checks: ```bash ./scripts/smoke_compose.sh ``` -CI runs migration, unit, and integration checks on push/PR via `.github/workflows/ci.yml`. - -## Scholar Ingestion Notes - -- Current ingestion targets public profile pages: `/citations?user=`. -- Runs are intentionally conservative and prioritize reliability over aggressive scraping. -- Ingestion follows pagination (`cstart`) to collect all reachable profile pages. -- If pagination cannot be completed (max pages hit, cursor stall, or page-state failure), the scholar result is marked partial with explicit debug context. -- Resumable partial/failure states are automatically queued and retried by the scheduler with bounded backoff. -- Queue items keep explicit status (`queued`, `retrying`, `dropped`) and last-error context for UI diagnostics. -- Failed scholar attempts persist structured debug context in `crawl_runs.error_log` (state reason, fetch metadata, marker evidence, and compact response excerpt). -- Only resumable states are retried automatically (`network_error` and continuation-eligible pagination truncations); blocked/layout states are recorded immediately as failures. - -## API v1 (Frontend Prep) - -- Base path: `/api/v1` -- Auth model: same-origin cookie session (no token auth added) -- CSRF model: required for unsafe methods (`POST`, `PUT`, `PATCH`, `DELETE`) via `X-CSRF-Token` header. -- Bootstrap CSRF via `GET /api/v1/auth/csrf` (works for anonymous and authenticated sessions). -- You can also fetch `csrf_token` from `GET /api/v1/auth/me` after login. -- Manual run idempotency is supported via `Idempotency-Key` header on `POST /api/v1/runs/manual`. -- Response envelopes: - - Success: `{"data": ..., "meta": {"request_id": "..."}}` - - Error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}` -- Initial API domains exposed: - - `auth`: `/auth/csrf`, `/auth/login`, `/auth/me`, `/auth/change-password`, `/auth/logout` - - `admin users`: `/admin/users` (list/create), `/admin/users/{id}/active`, `/admin/users/{id}/reset-password` - - `scholars`: list/create/toggle/delete - - `settings`: get/update - - `runs`: list/detail/manual trigger + queue list/retry/drop/clear - - `publications`: list + mark-all-read -- Queue action semantics: - - `retry` is valid only for dropped items; retrying/queued states return `409`. - - `drop` returns the updated queue item (status `dropped`); dropping an already dropped item returns `409`. - - `clear` only works for dropped items and returns `{queue_item_id, previous_status, status="cleared"}`. - ## Logging -- Default output is concise console logs to stdout. -- Timestamp format is compact UTC: `YYYY-MM-DD HH:MM:SSZ`. -- Every request gets an `X-Request-ID` (propagated if caller provides one). -- Sensitive fields are redacted before log emission (`password`, `csrf_token`, `cookie`, etc.). -- Configure via env: - - `LOG_LEVEL` (default `INFO`) - - `LOG_FORMAT` (`console` or `json`, default `console`) - - `LOG_REQUESTS` (`1`/`0`, default `1`) - - `LOG_UVICORN_ACCESS` (`1`/`0`, default `0`) - - `LOG_REQUEST_SKIP_PATHS` (comma-separated prefixes, default `/healthz,/static/`) - - `LOG_REDACT_FIELDS` (comma-separated additional keys) - - `SCHEDULER_ENABLED` (`1`/`0`, default `1`) - - `SCHEDULER_TICK_SECONDS` (default `60`) - - `INGESTION_NETWORK_ERROR_RETRIES` (default `1`) - - `INGESTION_RETRY_BACKOFF_SECONDS` (default `1.0`) - - `INGESTION_MAX_PAGES_PER_SCHOLAR` (default `30`) - - `INGESTION_PAGE_SIZE` (default `100`) - - `INGESTION_CONTINUATION_QUEUE_ENABLED` (`1`/`0`, default `1`) - - `INGESTION_CONTINUATION_BASE_DELAY_SECONDS` (default `120`) - - `INGESTION_CONTINUATION_MAX_DELAY_SECONDS` (default `3600`) - - `INGESTION_CONTINUATION_MAX_ATTEMPTS` (default `6`) - - `SCHEDULER_QUEUE_BATCH_SIZE` (default `10`) +Configurable env vars: -## Optional Local `uv` Workflow - -```bash -uv sync --extra dev -uv run pytest -uv run uvicorn app.main:app --reload -``` - -Update the lockfile after dependency changes: - -```bash -uv lock -``` - -## Theming - -- Theme registry: `app/theme.py` -- Semantic tokens: `app/static/theme.css` -- App styling: `app/static/app.css` -- Theme persistence: `app/static/theme.js` -- Dashboard presentation contract: `app/presentation/dashboard.py` +- `LOG_LEVEL` (default `INFO`) +- `LOG_FORMAT` (`console` or `json`, default `console`) +- `LOG_REQUESTS` (`1`/`0`, default `1`) +- `LOG_UVICORN_ACCESS` (`1`/`0`, default `0`) +- `LOG_REQUEST_SKIP_PATHS` (default `/healthz`) +- `LOG_REDACT_FIELDS` (additional redact keys) ## Project Layout ```text -app/ FastAPI app, auth/security modules, templates, services, DB wiring, presentation view-models -app/web/ Router modules, shared web helpers, and request middleware +app/ FastAPI app (API routers, auth, services, db, middleware) alembic/ Migration environment and versions -scripts/ Entrypoint, DB wait, bootstrap, smoke automation +scripts/ Entrypoint, db wait/bootstrap, smoke automation tests/ Unit, integration, and smoke suites +planning/ Scope and implementation planning notes ``` diff --git a/app/api/deps.py b/app/api/deps.py index 9dd3a68..f9c5377 100644 --- a/app/api/deps.py +++ b/app/api/deps.py @@ -4,16 +4,16 @@ from fastapi import Depends, Request from sqlalchemy.ext.asyncio import AsyncSession from app.api.errors import ApiException +from app.auth import runtime as auth_runtime from app.db.models import User from app.db.session import get_db_session -from app.web import common as web_common async def get_api_current_user( request: Request, db_session: AsyncSession = Depends(get_db_session), ) -> User: - current_user = await web_common.get_authenticated_user(request, db_session) + current_user = await auth_runtime.get_authenticated_user(request, db_session) if current_user is None: raise ApiException( status_code=401, @@ -33,4 +33,3 @@ async def get_api_admin_user( message="Admin access required.", ) return current_user - diff --git a/app/api/routers/auth.py b/app/api/routers/auth.py index 3b53dc8..1a76ced 100644 --- a/app/api/routers/auth.py +++ b/app/api/routers/auth.py @@ -18,13 +18,13 @@ from app.api.schemas import ( ) from app.auth.deps import get_auth_service, get_login_rate_limiter from app.auth.rate_limit import SlidingWindowRateLimiter +from app.auth import runtime as auth_runtime from app.auth.service import AuthService from app.auth.session import set_session_user from app.db.models import User from app.db.session import get_db_session from app.security.csrf import ensure_csrf_token from app.services import users as user_service -from app.web import common as web_common logger = logging.getLogger(__name__) @@ -42,7 +42,7 @@ async def login( auth_service: AuthService = Depends(get_auth_service), rate_limiter: SlidingWindowRateLimiter = Depends(get_login_rate_limiter), ): - limiter_key = web_common.login_rate_limit_key(request, payload.email) + limiter_key = auth_runtime.login_rate_limit_key(request, payload.email) decision = rate_limiter.check(limiter_key) normalized_email = payload.email.strip().lower() if not decision.allowed: @@ -142,7 +142,7 @@ async def get_csrf_bootstrap( request: Request, db_session: AsyncSession = Depends(get_db_session), ): - current_user = await web_common.get_authenticated_user(request, db_session) + current_user = await auth_runtime.get_authenticated_user(request, db_session) return success_payload( request, data={ @@ -213,8 +213,8 @@ async def logout( request: Request, db_session: AsyncSession = Depends(get_db_session), ): - current_user = await web_common.get_authenticated_user(request, db_session) - web_common.invalidate_session(request) + current_user = await auth_runtime.get_authenticated_user(request, db_session) + auth_runtime.invalidate_session(request) logger.info( "api.auth.logout", extra={ diff --git a/app/api/routers/runs.py b/app/api/routers/runs.py index 00f18d1..55bffa3 100644 --- a/app/api/routers/runs.py +++ b/app/api/routers/runs.py @@ -24,7 +24,7 @@ from app.services import ingestion as ingestion_service from app.services import runs as run_service from app.services import user_settings as user_settings_service from app.settings import settings -from app.web.deps import get_ingestion_service +from app.api.runtime_deps import get_ingestion_service logger = logging.getLogger(__name__) diff --git a/app/web/deps.py b/app/api/runtime_deps.py similarity index 99% rename from app/web/deps.py rename to app/api/runtime_deps.py index 001e2b8..167e2ef 100644 --- a/app/web/deps.py +++ b/app/api/runtime_deps.py @@ -14,4 +14,3 @@ def get_ingestion_service( source: scholar_source_service.ScholarSource = Depends(get_scholar_source), ) -> ingestion_service.ScholarIngestionService: return ingestion_service.ScholarIngestionService(source=source) - diff --git a/app/auth/runtime.py b/app/auth/runtime.py new file mode 100644 index 0000000..5bb1a74 --- /dev/null +++ b/app/auth/runtime.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import logging + +from fastapi import Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.auth.session import clear_session_user, get_session_user, set_session_user +from app.db.models import User +from app.security.csrf import CSRF_SESSION_KEY +from app.services import users as user_service + +logger = logging.getLogger(__name__) + + +def invalidate_session(request: Request) -> None: + clear_session_user(request) + request.session.pop(CSRF_SESSION_KEY, None) + + +async def get_authenticated_user( + request: Request, + db_session: AsyncSession, +) -> User | None: + session_user = get_session_user(request) + if session_user is None: + return None + + user = await user_service.get_user_by_id(db_session, session_user.id) + if user is None or not user.is_active: + logger.info( + "auth.session_invalidated", + extra={ + "event": "auth.session_invalidated", + "session_user_id": session_user.id, + }, + ) + invalidate_session(request) + return None + + if user.email != session_user.email or user.is_admin != session_user.is_admin: + set_session_user( + request, + user_id=user.id, + email=user.email, + is_admin=user.is_admin, + ) + + return user + + +def login_rate_limit_key(request: Request, email: str) -> str: + client_host = request.client.host if request.client is not None else "unknown" + normalized_email = email.strip().lower() + return f"{client_host}:{normalized_email or ''}" diff --git a/app/http/__init__.py b/app/http/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/app/http/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/app/web/middleware.py b/app/http/middleware.py similarity index 100% rename from app/web/middleware.py rename to app/http/middleware.py index 9de69a6..9d85458 100644 --- a/app/web/middleware.py +++ b/app/http/middleware.py @@ -1,8 +1,8 @@ from __future__ import annotations from secrets import token_urlsafe -import time import logging +import time from starlette.middleware.base import BaseHTTPMiddleware from starlette.requests import Request diff --git a/app/main.py b/app/main.py index 75ee94d..b6942dc 100644 --- a/app/main.py +++ b/app/main.py @@ -2,30 +2,19 @@ from __future__ import annotations from contextlib import asynccontextmanager -from fastapi import FastAPI -from fastapi.staticfiles import StaticFiles +from fastapi import FastAPI, HTTPException from starlette.middleware.sessions import SessionMiddleware from app.api.errors import register_api_exception_handlers from app.api.router import router as api_router +from app.api.runtime_deps import get_ingestion_service, get_scholar_source +from app.db.session import check_database from app.db.session import close_engine +from app.http.middleware import RequestLoggingMiddleware, parse_skip_paths from app.logging_config import configure_logging, parse_redact_fields from app.security.csrf import CSRFMiddleware from app.services.scheduler import SchedulerService from app.settings import settings -from app.web import common as web_common -from app.web.deps import get_ingestion_service, get_scholar_source -from app.web.middleware import RequestLoggingMiddleware, parse_skip_paths -from app.web.routers import ( - admin, - auth, - dashboard, - health, - publications, - runs, - scholars, - settings as settings_router, -) configure_logging( level=settings.log_level, @@ -71,17 +60,11 @@ app.add_middleware( log_requests=settings.log_requests, skip_paths=parse_skip_paths(settings.log_request_skip_paths), ) -app.mount("/static", StaticFiles(directory="app/static"), name="static") - app.include_router(api_router) -app.include_router(auth.router) -app.include_router(admin.router) -app.include_router(scholars.router) -app.include_router(settings_router.router) -app.include_router(runs.router) -app.include_router(publications.router) -app.include_router(dashboard.router) -app.include_router(health.router) -# Backward-compatible export kept for tests and any existing local scripts. -_get_authenticated_user = web_common.get_authenticated_user + +@app.get("/healthz") +async def healthz() -> dict[str, str]: + if await check_database(): + return {"status": "ok"} + raise HTTPException(status_code=500, detail="database unavailable") diff --git a/app/presentation/__init__.py b/app/presentation/__init__.py deleted file mode 100644 index d3d8cc8..0000000 --- a/app/presentation/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Presentation-layer view models used by template routes.""" - diff --git a/app/presentation/dashboard.py b/app/presentation/dashboard.py deleted file mode 100644 index 2f5d609..0000000 --- a/app/presentation/dashboard.py +++ /dev/null @@ -1,108 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime, timezone -from typing import Sequence - -from app.db.models import CrawlRun -from app.services.publications import UnreadPublicationItem - -SECTION_TEMPLATES: tuple[str, ...] = ( - "dashboard/_run_controls.html", - "dashboard/_unread_publications.html", - "dashboard/_run_history.html", -) - - -@dataclass(frozen=True) -class RunControlsViewModel: - request_delay_seconds: int - queue_queued_count: int - queue_retrying_count: int - queue_dropped_count: int - run_manual_action: str = "/runs/manual" - mark_all_read_action: str = "/publications/mark-all-read" - - -@dataclass(frozen=True) -class UnreadPublicationViewModel: - title: str - scholar_label: str - year_display: str - citation_count: int - venue_text: str | None - pub_url: str | None - - -@dataclass(frozen=True) -class RunHistoryItemViewModel: - run_id: int | None - detail_url: str | None - started_at_display: str - status_label: str - status_badge: str - scholar_count: int - new_publication_count: int - - -@dataclass(frozen=True) -class DashboardViewModel: - section_templates: tuple[str, ...] - run_controls: RunControlsViewModel - unread_publications: list[UnreadPublicationViewModel] - run_history: list[RunHistoryItemViewModel] - - -def build_dashboard_view_model( - *, - unread_publications: Sequence[UnreadPublicationItem], - recent_runs: Sequence[CrawlRun], - request_delay_seconds: int, - queue_counts: dict[str, int] | None = None, -) -> DashboardViewModel: - queue_counts = queue_counts or {} - return DashboardViewModel( - section_templates=SECTION_TEMPLATES, - run_controls=RunControlsViewModel( - request_delay_seconds=request_delay_seconds, - queue_queued_count=int(queue_counts.get("queued", 0)), - queue_retrying_count=int(queue_counts.get("retrying", 0)), - queue_dropped_count=int(queue_counts.get("dropped", 0)), - ), - unread_publications=[ - UnreadPublicationViewModel( - title=item.title, - scholar_label=item.scholar_label, - year_display=str(item.year) if item.year is not None else "-", - citation_count=item.citation_count, - venue_text=item.venue_text, - pub_url=item.pub_url, - ) - for item in unread_publications - ], - run_history=[ - RunHistoryItemViewModel( - run_id=run.id, - detail_url=f"/runs/{run.id}" if run.id is not None else None, - started_at_display=_format_started_at(run.start_dt), - status_label=run.status.value, - status_badge=_status_badge(run.status.value), - scholar_count=run.scholar_count, - new_publication_count=run.new_pub_count, - ) - for run in recent_runs - ], - ) - - -def _status_badge(status: str) -> str: - if status == "success": - return "ok" - if status == "failed": - return "danger" - return "warn" - - -def _format_started_at(dt: datetime) -> str: - local_aware = dt.astimezone(timezone.utc) - return local_aware.strftime("%Y-%m-%d %H:%M UTC") diff --git a/app/security/csrf.py b/app/security/csrf.py index 1f5c1a1..70eecac 100644 --- a/app/security/csrf.py +++ b/app/security/csrf.py @@ -77,8 +77,6 @@ class CSRFMiddleware(BaseHTTPMiddleware): if request.method in SAFE_METHODS: return True path = request.url.path - if path.startswith("/static/"): - return True return path in self._exempt_paths def _is_form_payload(self, request: Request) -> bool: diff --git a/app/settings.py b/app/settings.py index 826fa13..79fd40c 100644 --- a/app/settings.py +++ b/app/settings.py @@ -48,7 +48,7 @@ class Settings: log_format: str = _env_str("LOG_FORMAT", "console") log_requests: bool = _env_bool("LOG_REQUESTS", True) log_uvicorn_access: bool = _env_bool("LOG_UVICORN_ACCESS", False) - log_request_skip_paths: str = _env_str("LOG_REQUEST_SKIP_PATHS", "/healthz,/static/") + log_request_skip_paths: str = _env_str("LOG_REQUEST_SKIP_PATHS", "/healthz") log_redact_fields: str = os.getenv("LOG_REDACT_FIELDS", "") scheduler_enabled: bool = _env_bool("SCHEDULER_ENABLED", True) scheduler_tick_seconds: int = _env_int("SCHEDULER_TICK_SECONDS", 60) diff --git a/app/static/app.css b/app/static/app.css deleted file mode 100644 index 3b332c5..0000000 --- a/app/static/app.css +++ /dev/null @@ -1,501 +0,0 @@ -* { - box-sizing: border-box; -} - -body { - margin: 0; - min-height: 100vh; - color: var(--md-sys-color-on-surface); - font-family: var(--font-body); - background: - radial-gradient(circle at 10% 5%, var(--md-sys-color-primary-container) 0%, transparent 30%), - radial-gradient(circle at 90% 0%, var(--md-sys-color-secondary-container) 0%, transparent 28%), - var(--md-sys-color-surface); -} - -.loading-indicator { - position: fixed; - top: 0; - left: 0; - width: 100%; - height: 3px; - z-index: 40; - transform: scaleX(0); - transform-origin: left; - background: linear-gradient(90deg, var(--md-sys-color-primary), var(--md-sys-color-tertiary)); - opacity: 0; -} - -body.is-page-loading .loading-indicator { - opacity: 1; - animation: loading-bar 1.2s ease-in-out infinite; -} - -.app-backdrop { - position: fixed; - inset: 0; - pointer-events: none; - background: - linear-gradient(130deg, var(--md-sys-color-primary-tint), transparent 40%), - linear-gradient(230deg, var(--md-sys-color-surface-tint), transparent 45%); - opacity: 0.28; -} - -.top-app-bar { - position: sticky; - top: 0; - z-index: 10; - display: flex; - align-items: center; - gap: 0.8rem; - flex-wrap: wrap; - padding: 0.8rem 1.1rem; - border-bottom: 1px solid var(--md-sys-color-outline-variant); - backdrop-filter: blur(9px); - background: color-mix(in srgb, var(--md-sys-color-surface) 88%, white 12%); -} - -.brand-wrap { - display: flex; - align-items: center; - gap: 0.65rem; -} - -.brand { - color: var(--md-sys-color-primary); - text-decoration: none; - font-family: var(--font-heading); - font-weight: 700; - font-size: 1.2rem; - letter-spacing: 0.02em; -} - -.session-user { - margin: 0; - padding: 0.24rem 0.58rem; - border-radius: 999px; - border: 1px solid var(--md-sys-color-outline-variant); - background: var(--md-sys-color-surface-container-high); - color: var(--md-sys-color-on-surface-variant); - font-size: 0.78rem; -} - -.site-nav { - display: flex; - align-items: center; - gap: 0.35rem; - flex-wrap: wrap; -} - -.nav-link { - color: var(--md-sys-color-on-surface-variant); - text-decoration: none; - padding: 0.42rem 0.78rem; - border-radius: 999px; - border: 1px solid transparent; - font-size: 0.92rem; -} - -.nav-link:hover { - background: var(--md-sys-color-surface-container); - border-color: var(--md-sys-color-outline-variant); -} - -.nav-link.is-active { - color: var(--md-sys-color-on-secondary-container); - background: var(--md-sys-color-secondary-container); - border-color: transparent; -} - -.header-actions { - margin-left: auto; - display: flex; - align-items: center; - gap: 0.7rem; -} - -.theme-control-wrap { - display: flex; - align-items: center; - gap: 0.45rem; -} - -.theme-control-wrap label { - color: var(--md-sys-color-on-surface-variant); - font-size: 0.82rem; -} - -.theme-control { - border: 1px solid var(--md-sys-color-outline); - border-radius: 999px; - background: var(--md-sys-color-surface-container-low); - color: var(--md-sys-color-on-surface); - padding: 0.32rem 0.64rem; - font: inherit; -} - -.logout-form { - margin: 0; -} - -.page-shell { - position: relative; - display: grid; - gap: 0.9rem; - width: min(1100px, calc(100vw - 1.4rem)); - margin: 0 auto; - padding: 1.1rem 0 1.6rem; -} - -.flash { - margin: 0; - padding: 0.72rem 0.82rem; - border-radius: 14px; - border: 1px solid var(--md-sys-color-outline-variant); - background: var(--md-sys-color-surface-container); -} - -.flash-notice { - color: var(--md-sys-color-on-tertiary-container); - border-color: var(--md-sys-color-tertiary-container); - background: color-mix(in srgb, var(--md-sys-color-tertiary-container) 30%, white 70%); -} - -.flash-error { - color: var(--md-sys-color-on-error-container); - border-color: var(--md-sys-color-error-container); - background: color-mix(in srgb, var(--md-sys-color-error-container) 25%, white 75%); -} - -.hero, -.sandbox { - border-radius: 24px; - border: 1px solid var(--md-sys-color-outline-variant); - background: var(--md-sys-color-surface-container-low); - box-shadow: var(--elevation-card); -} - -.hero { - padding: 1.3rem; -} - -.sandbox { - padding: 1.05rem; -} - -.eyebrow { - margin: 0; - font-size: 0.78rem; - color: var(--md-sys-color-primary); - text-transform: uppercase; - letter-spacing: 0.11em; - font-weight: 600; -} - -h1 { - margin: 0.28rem 0 0.5rem; - color: var(--md-sys-color-on-surface); - font-family: var(--font-heading); - font-size: clamp(1.45rem, 3vw, 2.05rem); - line-height: 1.15; -} - -h2 { - margin: 0; - color: var(--md-sys-color-on-surface); - font-family: var(--font-heading); - font-size: 1.08rem; -} - -.lede { - margin: 0; - color: var(--md-sys-color-on-surface-variant); - max-width: 68ch; -} - -.section-heading { - display: flex; - align-items: center; - justify-content: space-between; - gap: 0.55rem; - margin-bottom: 0.7rem; - flex-wrap: wrap; -} - -.stat-strip { - margin-top: 0.95rem; - display: grid; - gap: 0.75rem; - grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); -} - -.stat-item { - margin: 0; - padding: 0.72rem 0.78rem; - border-radius: 16px; - border: 1px solid var(--md-sys-color-outline-variant); - background: var(--md-sys-color-surface-container); -} - -.stat-label { - margin: 0; - color: var(--md-sys-color-on-surface-variant); - font-size: 0.8rem; -} - -.stat-value { - margin: 0.2rem 0 0; - color: var(--md-sys-color-on-surface); - font-size: 1.36rem; - font-weight: 700; -} - -form { - display: grid; - gap: 0.45rem; -} - -.stack-form { - max-width: 560px; -} - -label { - font-size: 0.9rem; - color: var(--md-sys-color-on-surface-variant); -} - -input, -select { - width: 100%; - border: 1px solid var(--md-sys-color-outline); - border-radius: 12px; - padding: 0.56rem 0.64rem; - font: inherit; - color: var(--md-sys-color-on-surface); - background: var(--md-sys-color-surface); -} - -input:focus, -select:focus { - outline: 2px solid color-mix(in srgb, var(--md-sys-color-primary) 35%, transparent); - border-color: var(--md-sys-color-primary); -} - -.checkbox-row { - display: flex; - align-items: center; - gap: 0.54rem; -} - -.checkbox-row input { - width: auto; -} - -button, -.button { - border: 0; - border-radius: 999px; - padding: 0.54rem 0.86rem; - font: inherit; - font-weight: 600; - color: var(--md-sys-color-on-primary); - background: var(--md-sys-color-primary); - cursor: pointer; -} - -button:hover, -.button:hover { - filter: brightness(0.97); -} - -button:disabled, -.button:disabled { - cursor: not-allowed; - opacity: 0.5; -} - -.button-text { - color: var(--md-sys-color-primary); - background: transparent; - border: 1px solid var(--md-sys-color-outline-variant); -} - -.danger-button { - color: var(--md-sys-color-on-error); - background: var(--md-sys-color-error); -} - -.inline-actions { - display: flex; - align-items: center; - flex-wrap: wrap; - gap: 0.45rem; -} - -.inline-actions form { - display: inline-flex; - gap: 0.42rem; - align-items: center; -} - -.inline-reset-form input { - min-width: 160px; -} - -form.is-loading button[type="submit"], -form.is-loading input[type="submit"] { - opacity: 0.75; -} - -table { - width: 100%; - border-collapse: collapse; - border-radius: 12px; - overflow: hidden; - background: var(--md-sys-color-surface); -} - -th, -td { - padding: 0.54rem; - text-align: left; - border-bottom: 1px solid var(--md-sys-color-outline-variant); - vertical-align: top; -} - -thead th { - color: var(--md-sys-color-on-surface-variant); - font-size: 0.84rem; - font-weight: 600; - background: var(--md-sys-color-surface-container-high); -} - -tbody tr:last-child td { - border-bottom: 0; -} - -a { - color: var(--md-sys-color-primary); -} - -.link-button { - display: inline-flex; - align-items: center; - padding: 0.45rem 0.78rem; - border: 1px solid var(--md-sys-color-outline-variant); - border-radius: 999px; - color: var(--md-sys-color-primary); - text-decoration: none; - background: var(--md-sys-color-surface-container-low); -} - -.link-button.is-active { - color: var(--md-sys-color-on-secondary-container); - background: var(--md-sys-color-secondary-container); - border-color: transparent; -} - -.badge { - display: inline-block; - border-radius: 999px; - padding: 0.16rem 0.48rem; - font-size: 0.76rem; - text-transform: lowercase; - border: 1px solid var(--md-sys-color-outline-variant); -} - -.badge.ok { - color: var(--md-sys-color-on-tertiary-container); - background: var(--md-sys-color-tertiary-container); - border-color: transparent; -} - -.badge.warn, -.badge.danger { - color: var(--md-sys-color-on-error-container); - background: var(--md-sys-color-error-container); - border-color: transparent; -} - -.helper-text { - margin: 0.24rem 0 0; - color: var(--md-sys-color-on-surface-variant); - font-size: 0.82rem; -} - -.auth-card { - max-width: 520px; -} - -.auth-form { - max-width: 360px; -} - -.form-error { - margin: 0.5rem 0 0; - color: var(--md-sys-color-error); - font-weight: 600; -} - -.form-note { - margin: 0.35rem 0 0; - color: var(--md-sys-color-on-surface-variant); -} - -code { - font-family: var(--font-code); - background: var(--md-sys-color-surface-container-high); - border-radius: 6px; - padding: 0.1rem 0.25rem; -} - -pre { - margin: 0.6rem 0 0; - border-radius: 12px; - border: 1px solid var(--md-sys-color-outline-variant); - background: var(--md-sys-color-surface-container-high); - color: var(--md-sys-color-on-surface); - padding: 0.75rem; - white-space: pre-wrap; - word-break: break-word; -} - -@media (max-width: 880px) { - .top-app-bar { - padding: 0.7rem 0.8rem; - } - - .header-actions { - width: 100%; - justify-content: space-between; - margin-left: 0; - } - - .page-shell { - width: min(100vw, calc(100vw - 0.9rem)); - padding-top: 0.8rem; - } - - .hero, - .sandbox { - border-radius: 18px; - padding: 0.9rem; - } - - .inline-actions form { - width: 100%; - justify-content: flex-start; - } -} - -@keyframes loading-bar { - 0% { - transform: scaleX(0.1); - } - 50% { - transform: scaleX(0.7); - } - 100% { - transform: scaleX(1); - } -} diff --git a/app/static/app.js b/app/static/app.js deleted file mode 100644 index a2d6d80..0000000 --- a/app/static/app.js +++ /dev/null @@ -1,76 +0,0 @@ -(() => { - const body = document.body; - if (!body) { - return; - } - - const defaultLoadingText = "Working..."; - - const shouldHandleAnchor = (anchor) => { - if (!anchor || !anchor.getAttribute) { - return false; - } - const href = anchor.getAttribute("href"); - if (!href) { - return false; - } - if (href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("javascript:")) { - return false; - } - if (anchor.hasAttribute("download")) { - return false; - } - if ((anchor.getAttribute("target") || "").toLowerCase() === "_blank") { - return false; - } - return true; - }; - - document.addEventListener("click", (event) => { - const target = event.target; - if (!(target instanceof Element)) { - return; - } - const anchor = target.closest("a"); - if (!shouldHandleAnchor(anchor)) { - return; - } - body.classList.add("is-page-loading"); - }); - - document.addEventListener("submit", (event) => { - const form = event.target; - if (!(form instanceof HTMLFormElement)) { - return; - } - if (form.dataset.noLoading === "1") { - return; - } - - body.classList.add("is-page-loading"); - form.classList.add("is-loading"); - - const submitElements = form.querySelectorAll("button[type='submit'], input[type='submit']"); - submitElements.forEach((element) => { - if (element.hasAttribute("disabled")) { - return; - } - element.setAttribute("disabled", "disabled"); - if (element instanceof HTMLInputElement) { - const loadingText = element.dataset.loadingText || defaultLoadingText; - element.dataset.originalValue = element.value; - element.value = loadingText; - return; - } - if (element instanceof HTMLButtonElement) { - const loadingText = element.dataset.loadingText || defaultLoadingText; - element.dataset.originalText = element.textContent || ""; - element.textContent = loadingText; - } - }); - }); - - window.addEventListener("pageshow", () => { - body.classList.remove("is-page-loading"); - }); -})(); diff --git a/app/static/theme.css b/app/static/theme.css deleted file mode 100644 index 8506a29..0000000 --- a/app/static/theme.css +++ /dev/null @@ -1,98 +0,0 @@ -:root { - --font-body: "Plus Jakarta Sans", "Segoe UI", sans-serif; - --font-heading: "DM Serif Text", "Georgia", serif; - --font-code: "JetBrains Mono", "Fira Code", monospace; - - --md-sys-color-primary: #73594d; - --md-sys-color-on-primary: #ffffff; - --md-sys-color-primary-container: #fbded0; - --md-sys-color-on-primary-container: #2b160f; - --md-sys-color-primary-tint: rgba(115, 89, 77, 0.18); - - --md-sys-color-secondary: #6f5b52; - --md-sys-color-on-secondary: #ffffff; - --md-sys-color-secondary-container: #f8ddd2; - --md-sys-color-on-secondary-container: #281913; - - --md-sys-color-tertiary: #5b6550; - --md-sys-color-on-tertiary: #ffffff; - --md-sys-color-tertiary-container: #ddebcf; - --md-sys-color-on-tertiary-container: #17200f; - - --md-sys-color-error: #b3261e; - --md-sys-color-on-error: #ffffff; - --md-sys-color-error-container: #f9dedc; - --md-sys-color-on-error-container: #410e0b; - - --md-sys-color-surface: #fff8f5; - --md-sys-color-surface-tint: rgba(111, 87, 74, 0.12); - --md-sys-color-surface-container-low: #fffaf8; - --md-sys-color-surface-container: #fdf2ec; - --md-sys-color-surface-container-high: #f8e9e1; - --md-sys-color-on-surface: #221a17; - --md-sys-color-on-surface-variant: #55423a; - --md-sys-color-outline: #85736b; - --md-sys-color-outline-variant: #d8c2b8; - - --elevation-card: 0 10px 30px rgba(55, 39, 28, 0.08); -} - -:root[data-theme="fjord"] { - --md-sys-color-primary: #2f6678; - --md-sys-color-on-primary: #ffffff; - --md-sys-color-primary-container: #cde9f4; - --md-sys-color-on-primary-container: #051f28; - --md-sys-color-primary-tint: rgba(47, 102, 120, 0.2); - - --md-sys-color-secondary: #4d626b; - --md-sys-color-on-secondary: #ffffff; - --md-sys-color-secondary-container: #d0e6f2; - --md-sys-color-on-secondary-container: #091f27; - - --md-sys-color-tertiary: #456179; - --md-sys-color-on-tertiary: #ffffff; - --md-sys-color-tertiary-container: #d2e5ff; - --md-sys-color-on-tertiary-container: #031d33; - - --md-sys-color-surface: #f5fbff; - --md-sys-color-surface-tint: rgba(47, 102, 120, 0.12); - --md-sys-color-surface-container-low: #fcfeff; - --md-sys-color-surface-container: #ecf5fa; - --md-sys-color-surface-container-high: #dfeef6; - --md-sys-color-on-surface: #172126; - --md-sys-color-on-surface-variant: #3f4f57; - --md-sys-color-outline: #6f7f88; - --md-sys-color-outline-variant: #becfd8; - - --elevation-card: 0 10px 30px rgba(22, 43, 53, 0.09); -} - -:root[data-theme="spruce"] { - --md-sys-color-primary: #39684c; - --md-sys-color-on-primary: #ffffff; - --md-sys-color-primary-container: #bcefc9; - --md-sys-color-on-primary-container: #062111; - --md-sys-color-primary-tint: rgba(57, 104, 76, 0.18); - - --md-sys-color-secondary: #4f6354; - --md-sys-color-on-secondary: #ffffff; - --md-sys-color-secondary-container: #d2e8d4; - --md-sys-color-on-secondary-container: #0d1f12; - - --md-sys-color-tertiary: #3f655f; - --md-sys-color-on-tertiary: #ffffff; - --md-sys-color-tertiary-container: #c2ece4; - --md-sys-color-on-tertiary-container: #021f1a; - - --md-sys-color-surface: #f5fbf5; - --md-sys-color-surface-tint: rgba(57, 104, 76, 0.1); - --md-sys-color-surface-container-low: #fcfffb; - --md-sys-color-surface-container: #ebf4ea; - --md-sys-color-surface-container-high: #deecdd; - --md-sys-color-on-surface: #172119; - --md-sys-color-on-surface-variant: #415246; - --md-sys-color-outline: #708174; - --md-sys-color-outline-variant: #c1d3c2; - - --elevation-card: 0 10px 30px rgba(24, 48, 32, 0.08); -} diff --git a/app/static/theme.js b/app/static/theme.js deleted file mode 100644 index 47b95a1..0000000 --- a/app/static/theme.js +++ /dev/null @@ -1,56 +0,0 @@ -(() => { - const STORAGE_KEY = "scholarr_theme"; - const LEGACY_STORAGE_KEY = "scholar_tracker_theme"; - const root = document.documentElement; - const themeControl = document.querySelector("[data-theme-control]"); - - if (!root) { - return; - } - - const defaultTheme = root.dataset.defaultTheme || "terracotta"; - const supportedThemes = themeControl - ? Array.from(themeControl.options).map((option) => option.value) - : []; - - const isSupported = (theme) => - supportedThemes.length === 0 || supportedThemes.includes(theme); - - const applyTheme = (theme) => { - if (!isSupported(theme)) { - return; - } - root.dataset.theme = theme; - }; - - let activeTheme = defaultTheme; - try { - const savedTheme = window.localStorage.getItem(STORAGE_KEY); - const legacyTheme = window.localStorage.getItem(LEGACY_STORAGE_KEY); - if (savedTheme && isSupported(savedTheme)) { - activeTheme = savedTheme; - } else if (legacyTheme && isSupported(legacyTheme)) { - activeTheme = legacyTheme; - window.localStorage.setItem(STORAGE_KEY, legacyTheme); - } - } catch { - activeTheme = defaultTheme; - } - - applyTheme(activeTheme); - - if (!themeControl) { - return; - } - - themeControl.value = activeTheme; - themeControl.addEventListener("change", (event) => { - const selectedTheme = event.target.value; - applyTheme(selectedTheme); - try { - window.localStorage.setItem(STORAGE_KEY, selectedTheme); - } catch { - // Ignore storage errors in locked-down browsers. - } - }); -})(); diff --git a/app/templates/account_password.html b/app/templates/account_password.html deleted file mode 100644 index 687ec98..0000000 --- a/app/templates/account_password.html +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
-

Account

-

Change Password

-

Use a strong password. This updates only your own account.

-
- -
-

Password Update

-
- - - - - - - - -
-
-{% endblock %} diff --git a/app/templates/base.html b/app/templates/base.html deleted file mode 100644 index fcdd653..0000000 --- a/app/templates/base.html +++ /dev/null @@ -1,66 +0,0 @@ - - - - - - {{ page_title }} | scholarr - - - - - - -
-
- scholarr - {% if session_user %} -

{{ session_user.email }}

- {% endif %} -
- - - -
-
- - -
- {% if session_user %} -
- - -
- {% endif %} -
-
-
- {% if notice %} -

{{ notice }}

- {% endif %} - {% if page_error %} -

{{ page_error }}

- {% endif %} - {% block content %}{% endblock %} -
- - - - diff --git a/app/templates/dashboard/_run_controls.html b/app/templates/dashboard/_run_controls.html deleted file mode 100644 index 24779cd..0000000 --- a/app/templates/dashboard/_run_controls.html +++ /dev/null @@ -1,23 +0,0 @@ -
- -

Manual runs use your request delay of {{ dashboard.run_controls.request_delay_seconds }} second(s).

-

- Queue state: - {{ dashboard.run_controls.queue_queued_count }} queued, - {{ dashboard.run_controls.queue_retrying_count }} retrying, - {{ dashboard.run_controls.queue_dropped_count }} dropped. -

-
- - -
-
diff --git a/app/templates/dashboard/_run_history.html b/app/templates/dashboard/_run_history.html deleted file mode 100644 index aa6f9f1..0000000 --- a/app/templates/dashboard/_run_history.html +++ /dev/null @@ -1,38 +0,0 @@ -
-
-

Run History

- -
- {% if dashboard.run_history %} - - - - - - - - - - - {% for run in dashboard.run_history %} - - - - - - - {% endfor %} - -
StartedStatusScholarsNew Publications
- {% if run.detail_url %} - {{ run.started_at_display }} - {% else %} - {{ run.started_at_display }} - {% endif %} - {{ run.status_label }}{{ run.scholar_count }}{{ run.new_publication_count }}
- {% else %} -

No runs yet.

- {% endif %} -
diff --git a/app/templates/dashboard/_unread_publications.html b/app/templates/dashboard/_unread_publications.html deleted file mode 100644 index 1354215..0000000 --- a/app/templates/dashboard/_unread_publications.html +++ /dev/null @@ -1,47 +0,0 @@ -
-
-

New Since Last Run

-
- View All -
- - - -
-
-
- - {% if dashboard.unread_publications %} - - - - - - - - - - - {% for item in dashboard.unread_publications %} - - - - - - - {% endfor %} - -
TitleScholarYearCitations
- {% if item.pub_url %} - {{ item.title }} - {% else %} - {{ item.title }} - {% endif %} - {% if item.venue_text %} -

{{ item.venue_text }}

- {% endif %} -
{{ item.scholar_label }}{{ item.year_display }}{{ item.citation_count }}
- {% else %} -

No new publications in the latest run.

- {% endif %} -
diff --git a/app/templates/index.html b/app/templates/index.html deleted file mode 100644 index f5fd22a..0000000 --- a/app/templates/index.html +++ /dev/null @@ -1,27 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
-

Dashboard

-

Keep up with your tracked scholars

-

Run ingestion, review fresh publications, and inspect recent outcomes from one screen.

-
-
-

New Since Last Run

-

{{ dashboard.unread_publications | length }}

-
-
-

Recent Runs

-

{{ dashboard.run_history | length }}

-
-
-

Automation Delay

-

{{ dashboard.run_controls.request_delay_seconds }}s

-
-
-
- -{% for section_template in dashboard.section_templates %} -{% include section_template %} -{% endfor %} -{% endblock %} diff --git a/app/templates/login.html b/app/templates/login.html deleted file mode 100644 index 8b0f383..0000000 --- a/app/templates/login.html +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
-

Secure Access

-

Sign in

-

Use an admin-created account to access your tracked scholars and settings.

- {% if error_message %} - - {% endif %} - {% if retry_after_seconds %} -

Retry after {{ retry_after_seconds }} seconds.

- {% endif %} -
- - - - - - -
-
-{% endblock %} diff --git a/app/templates/publications.html b/app/templates/publications.html deleted file mode 100644 index 89054a2..0000000 --- a/app/templates/publications.html +++ /dev/null @@ -1,104 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
-

Publications

-

Publications

-

- Browse publications discovered in the latest run or your complete tracked library. - {% if selected_scholar %} - Current scholar filter: {{ selected_scholar.display_name or selected_scholar.scholar_id }}. - {% endif %} -

-
- New Since Last Run ({{ new_count }}) - All ({{ total_count }}) - Manage Scholars - Run Diagnostics - {% if mode == 'new' and new_count > 0 %} -
- - - -
- {% endif %} -
-
- -
-
-

Filter

-
-
- - - - - - -
- -
-

Results

-
- {% if publications %} - - - - - - - - - - - - - {% for item in publications %} - - - - - - - - - {% endfor %} - -
TitleScholarYearCitationsNew This RunStatus
- {% if item.pub_url %} - {{ item.title }} - {% else %} - {{ item.title }} - {% endif %} - {% if item.venue_text %} -

{{ item.venue_text }}

- {% endif %} -
{{ item.scholar_label }}{{ item.year if item.year is not none else "-" }}{{ item.citation_count }} - {% if item.is_new_in_latest_run %} - new - {% else %} - older - {% endif %} - - {% if item.is_read %} - read - {% else %} - new - {% endif %} -
- {% else %} -

No publications match this mode yet.

- {% endif %} -
-{% endblock %} diff --git a/app/templates/run_detail.html b/app/templates/run_detail.html deleted file mode 100644 index 06920ee..0000000 --- a/app/templates/run_detail.html +++ /dev/null @@ -1,98 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
-

Diagnostics

-

Run #{{ run.id }}

-

Detailed state and failure context for this execution.

-
- -
-

Summary

- - - - - - - - - - - - -
Started{{ run.started_at }}
Finished{{ run.finished_at }}
Status{{ run.status }}
Trigger{{ run.trigger_type }}
Scholars{{ run.scholar_count }}
New Publications{{ run.new_publication_count }}
Succeeded{{ run_summary.succeeded_count or 0 }}
Failed{{ run_summary.failed_count or 0 }}
Partial{{ run_summary.partial_count or 0 }}
- {% if run_summary.failed_state_counts or run_summary.failed_reason_counts %} -
- Failure Breakdown -
{{ {"failed_state_counts": run_summary.failed_state_counts, "failed_reason_counts": run_summary.failed_reason_counts} | tojson(indent=2) }}
-
- {% endif %} -
- -
-

Scholar Results

- {% if scholar_results %} - - - - - - - - - - - - - - - - {% for item in scholar_results %} - - - - - - - - - - - - {% if item.debug %} - - - - {% endif %} - {% endfor %} - -
ScholarOutcomeStateReasonPublicationsPagesAttemptsContinuationWarnings
{{ item.scholar_id }}{{ item.outcome or "-" }}{{ item.state }}{{ item.state_reason or "-" }}{{ item.publication_count }}{{ item.pages_fetched or 0 }} / {{ item.pages_attempted or 0 }}{{ item.attempt_count or 1 }} - {% if item.continuation_enqueued %} - queued -

- reason: {{ item.continuation_reason or "-" }}, - cstart: {{ item.continuation_cstart or "-" }} -

- {% elif item.continuation_cleared %} - cleared - {% else %} - - - {% endif %} -
- {% if item.warnings %} - {{ item.warnings | join(", ") }} - {% else %} - - - {% endif %} -
-
- Debug Context -
{{ item.debug | tojson(indent=2) }}
-
-
- {% else %} -

No scholar result details were captured.

- {% endif %} -
-{% endblock %} diff --git a/app/templates/runs.html b/app/templates/runs.html deleted file mode 100644 index 6a153c1..0000000 --- a/app/templates/runs.html +++ /dev/null @@ -1,129 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
-

Runs

-

Run History

-

Review execution outcomes and drill into run diagnostics when needed.

-
- -
-

Filters

-
- - - {% if failed_only %} - Clear - {% endif %} -
-
- -
-

Recent Runs

- {% if runs %} - - - - - - - - - - - - - - {% for run in runs %} - - - - - - - - - - {% endfor %} - -
IDStartedStatusTriggerScholarsNew PublicationsFailures
#{{ run.id }}{{ run.started_at }}{{ run.status }}{{ run.trigger_type }}{{ run.scholar_count }}{{ run.new_publication_count }}{{ run.failed_count }} failed / {{ run.partial_count }} partial
- {% else %} -

No runs match the current filter.

- {% endif %} -
- -
-
-

Continuation Queue

- Focus Failed Runs -
- {% if queue_items %} - - - - - - - - - - - - - - {% for item in queue_items %} - - - - - - - - - - {% endfor %} - -
ScholarStatusReasonAttemptsNext AttemptLast ErrorActions
- {{ item.scholar_label }} -

resume cstart: {{ item.resume_cstart }}

-
{{ item.status }} - {% if item.status == "dropped" and item.dropped_reason %} - {{ item.dropped_reason }} - {% else %} - {{ item.reason }} - {% endif %} - {{ item.attempt_count }}{{ item.next_attempt_at }} - {% if item.last_error %} -
- show -
{{ item.last_error }}
-
- {% else %} - - - {% endif %} -
-
-
- - -
- {% if item.status != "dropped" %} -
- - -
- {% endif %} -
- - -
-
-
- {% else %} -

No queued continuation items.

- {% endif %} -
-{% endblock %} diff --git a/app/templates/scholars.html b/app/templates/scholars.html deleted file mode 100644 index 0cc7821..0000000 --- a/app/templates/scholars.html +++ /dev/null @@ -1,99 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
-

Scholars

-

Your Scholars

-

Add, disable, or remove profiles scoped to your account.

-
- -
-

Add Scholar

-
- - - - - - -
-
- -
-

Tracked Scholars

- {% if scholars %} - - - - - - - - - - - - - {% for scholar in scholars %} - - - - - - - - - {% endfor %} - -
NameScholar IDStatusLast RunPublicationsActions
{{ scholar.display_name }}{{ scholar.scholar_id }} - {% if scholar.is_enabled %} - enabled - {% else %} - disabled - {% endif %} - {% if scholar.baseline_completed %} -

baseline complete

- {% else %} -

awaiting first run

- {% endif %} -
- - {{ scholar.last_run_status }} - -

{{ scholar.last_run_dt }}

-
-
- New - All -
-
-
-
- - -
-
- - -
-
-
- {% else %} -

No scholars tracked yet.

- {% endif %} -
-{% endblock %} diff --git a/app/templates/settings.html b/app/templates/settings.html deleted file mode 100644 index c07d064..0000000 --- a/app/templates/settings.html +++ /dev/null @@ -1,44 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
-

Settings

-

Your Settings

-

Control automation cadence and request pacing for your own account.

-
- -
-

Automation Settings

-
- - - - - - - - - - -
-
-{% endblock %} diff --git a/app/templates/users.html b/app/templates/users.html deleted file mode 100644 index 34b4b7d..0000000 --- a/app/templates/users.html +++ /dev/null @@ -1,75 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
-

Admin

-

User Management

-

Admin-only controls for account creation, activation, and password resets.

-
- -
-

Create User

-
- - - - - - - -
-
- -
-

Users

- - - - - - - - - - - {% for user in users %} - - - - - - - {% endfor %} - -
EmailRoleStatusActions
{{ user.email }} - {% if user.is_admin %} - admin - {% else %} - user - {% endif %} - - {% if user.is_active %} - active - {% else %} - inactive - {% endif %} - -
-
- - -
-
- - - -
-
-
-
-{% endblock %} diff --git a/app/theme.py b/app/theme.py deleted file mode 100644 index 78e152b..0000000 --- a/app/theme.py +++ /dev/null @@ -1,24 +0,0 @@ -from dataclasses import dataclass -from typing import Final - - -@dataclass(frozen=True) -class ThemeDefinition: - slug: str - label: str - - -THEMES: Final[tuple[ThemeDefinition, ...]] = ( - ThemeDefinition(slug="terracotta", label="Terracotta"), - ThemeDefinition(slug="fjord", label="Fjord"), - ThemeDefinition(slug="spruce", label="Spruce"), -) -_THEME_SLUGS: Final[frozenset[str]] = frozenset(theme.slug for theme in THEMES) -DEFAULT_THEME: Final[str] = THEMES[0].slug - - -def resolve_theme(candidate: str | None) -> str: - if candidate and candidate in _THEME_SLUGS: - return candidate - return DEFAULT_THEME - diff --git a/app/web/__init__.py b/app/web/__init__.py deleted file mode 100644 index 4403b00..0000000 --- a/app/web/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Web-layer routers, middleware, and shared helpers.""" - diff --git a/app/web/common.py b/app/web/common.py deleted file mode 100644 index 8c03915..0000000 --- a/app/web/common.py +++ /dev/null @@ -1,140 +0,0 @@ -from __future__ import annotations - -import logging -from urllib.parse import urlencode - -from fastapi import Request, status -from fastapi.responses import HTMLResponse, RedirectResponse -from fastapi.templating import Jinja2Templates -from sqlalchemy.ext.asyncio import AsyncSession - -from app.auth.session import ( - SessionUser, - clear_session_user, - get_session_user, - set_session_user, -) -from app.db.models import User -from app.security.csrf import CSRF_SESSION_KEY, ensure_csrf_token -from app.services import users as user_service -from app.theme import THEMES - -logger = logging.getLogger(__name__) - -templates = Jinja2Templates(directory="app/templates") - - -def to_session_user(user: User | None) -> SessionUser | None: - if user is None: - return None - return SessionUser(id=user.id, email=user.email, is_admin=user.is_admin) - - -def build_template_context( - request: Request, - *, - page_title: str, - active_nav: str, - theme_name: str, - session_user: SessionUser | None, - notice: str | None = None, - page_error: str | None = None, -) -> dict[str, object]: - return { - "active_nav": active_nav, - "page_title": page_title, - "theme_name": theme_name, - "themes": THEMES, - "session_user": session_user, - "csrf_token": ensure_csrf_token(request), - "notice": notice, - "page_error": page_error, - } - - -def redirect_with_message( - path: str, - *, - notice: str | None = None, - error: str | None = None, -) -> RedirectResponse: - params: dict[str, str] = {} - if notice: - params["notice"] = notice - if error: - params["error"] = error - if params: - path = f"{path}?{urlencode(params)}" - return RedirectResponse(path, status_code=status.HTTP_303_SEE_OTHER) - - -def redirect_to_login() -> RedirectResponse: - return RedirectResponse("/login", status_code=status.HTTP_303_SEE_OTHER) - - -def invalidate_session(request: Request) -> None: - clear_session_user(request) - request.session.pop(CSRF_SESSION_KEY, None) - - -async def get_authenticated_user( - request: Request, - db_session: AsyncSession, -) -> User | None: - session_user = get_session_user(request) - if session_user is None: - return None - - user = await user_service.get_user_by_id(db_session, session_user.id) - if user is None or not user.is_active: - logger.info( - "auth.session_invalidated", - extra={ - "event": "auth.session_invalidated", - "session_user_id": session_user.id, - }, - ) - invalidate_session(request) - return None - - if user.email != session_user.email or user.is_admin != session_user.is_admin: - set_session_user( - request, - user_id=user.id, - email=user.email, - is_admin=user.is_admin, - ) - - return user - - -def login_rate_limit_key(request: Request, email: str) -> str: - client_host = request.client.host if request.client is not None else "unknown" - normalized_email = email.strip().lower() - return f"{client_host}:{normalized_email or ''}" - - -def render_login_page( - request: Request, - *, - theme_name: str, - error_message: str | None = None, - status_code: int = status.HTTP_200_OK, - retry_after_seconds: int | None = None, -) -> HTMLResponse: - context = build_template_context( - request, - page_title="Login", - active_nav="login", - theme_name=theme_name, - session_user=None, - ) - context["error_message"] = error_message - context["retry_after_seconds"] = retry_after_seconds - return templates.TemplateResponse( - request=request, - name="login.html", - context=context, - status_code=status_code, - ) - diff --git a/app/web/routers/__init__.py b/app/web/routers/__init__.py deleted file mode 100644 index cbf15d9..0000000 --- a/app/web/routers/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Application routers grouped by concern.""" - diff --git a/app/web/routers/admin.py b/app/web/routers/admin.py deleted file mode 100644 index fda4f45..0000000 --- a/app/web/routers/admin.py +++ /dev/null @@ -1,209 +0,0 @@ -from __future__ import annotations - -import logging -from typing import Annotated - -from fastapi import APIRouter, Depends, Form, HTTPException, Request, status -from fastapi.responses import HTMLResponse -from sqlalchemy.ext.asyncio import AsyncSession - -from app.auth.deps import get_auth_service -from app.auth.service import AuthService -from app.db.session import get_db_session -from app.services import users as user_service -from app.theme import resolve_theme -from app.web import common - -logger = logging.getLogger(__name__) - -router = APIRouter() - - -async def _render_users_page( - request: Request, - *, - db_session: AsyncSession, - current_user, - theme_name: str, - notice: str | None = None, - page_error: str | None = None, - status_code: int = status.HTTP_200_OK, - form_email: str = "", - form_is_admin: bool = False, -) -> HTMLResponse: - users = await user_service.list_users(db_session) - context = common.build_template_context( - request, - page_title="Users", - active_nav="users", - theme_name=theme_name, - session_user=common.to_session_user(current_user), - notice=notice, - page_error=page_error, - ) - context["users"] = users - context["current_user_id"] = current_user.id - context["form_email"] = form_email - context["form_is_admin"] = form_is_admin - return common.templates.TemplateResponse( - request=request, - name="users.html", - context=context, - status_code=status_code, - ) - - -@router.get("/users", response_class=HTMLResponse) -async def users_page( - request: Request, - theme: str | None = None, - db_session: AsyncSession = Depends(get_db_session), -) -> HTMLResponse: - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - if not current_user.is_admin: - raise HTTPException(status_code=403, detail="Admin access required.") - return await _render_users_page( - request, - db_session=db_session, - current_user=current_user, - theme_name=resolve_theme(theme), - notice=request.query_params.get("notice"), - page_error=request.query_params.get("error"), - ) - - -@router.post("/users") -async def create_user( - request: Request, - email: Annotated[str, Form()], - password: Annotated[str, Form()], - is_admin: Annotated[str | None, Form()] = None, - db_session: AsyncSession = Depends(get_db_session), - auth_service: AuthService = Depends(get_auth_service), -) -> HTMLResponse: - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - if not current_user.is_admin: - raise HTTPException(status_code=403, detail="Admin access required.") - - active_theme = resolve_theme(None) - try: - validated_email = user_service.validate_email(email) - validated_password = user_service.validate_password(password) - created_user = await user_service.create_user( - db_session, - email=validated_email, - password_hash=auth_service.hash_password(validated_password), - is_admin=is_admin == "on", - ) - except user_service.UserServiceError as exc: - logger.info( - "admin.user_create_failed", - extra={ - "event": "admin.user_create_failed", - "admin_user_id": current_user.id, - "reason": "validation_or_conflict", - }, - ) - return await _render_users_page( - request, - db_session=db_session, - current_user=current_user, - theme_name=active_theme, - page_error=str(exc), - status_code=status.HTTP_400_BAD_REQUEST, - form_email=email, - form_is_admin=is_admin == "on", - ) - - logger.info( - "admin.user_created", - extra={ - "event": "admin.user_created", - "admin_user_id": current_user.id, - "target_user_id": created_user.id, - "target_is_admin": created_user.is_admin, - }, - ) - return common.redirect_with_message( - "/users", - notice=f"User created: {created_user.email}", - ) - - -@router.post("/users/{user_id}/toggle-active") -async def toggle_user_active( - request: Request, - user_id: int, - db_session: AsyncSession = Depends(get_db_session), -): - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - if not current_user.is_admin: - raise HTTPException(status_code=403, detail="Admin access required.") - - target_user = await user_service.get_user_by_id(db_session, user_id) - if target_user is None: - return common.redirect_with_message("/users", error="User not found.") - if target_user.id == current_user.id and target_user.is_active: - return common.redirect_with_message("/users", error="You cannot deactivate your own account.") - - updated_user = await user_service.set_user_active( - db_session, - user=target_user, - is_active=not target_user.is_active, - ) - status_label = "activated" if updated_user.is_active else "deactivated" - logger.info( - "admin.user_active_toggled", - extra={ - "event": "admin.user_active_toggled", - "admin_user_id": current_user.id, - "target_user_id": updated_user.id, - "is_active": updated_user.is_active, - }, - ) - return common.redirect_with_message("/users", notice=f"User {status_label}: {updated_user.email}") - - -@router.post("/users/{user_id}/reset-password") -async def reset_user_password( - request: Request, - user_id: int, - new_password: Annotated[str, Form()], - db_session: AsyncSession = Depends(get_db_session), - auth_service: AuthService = Depends(get_auth_service), -): - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - if not current_user.is_admin: - raise HTTPException(status_code=403, detail="Admin access required.") - - target_user = await user_service.get_user_by_id(db_session, user_id) - if target_user is None: - return common.redirect_with_message("/users", error="User not found.") - try: - validated_password = user_service.validate_password(new_password) - except user_service.UserServiceError as exc: - return common.redirect_with_message("/users", error=str(exc)) - - await user_service.set_user_password_hash( - db_session, - user=target_user, - password_hash=auth_service.hash_password(validated_password), - ) - logger.info( - "admin.user_password_reset", - extra={ - "event": "admin.user_password_reset", - "admin_user_id": current_user.id, - "target_user_id": target_user.id, - }, - ) - return common.redirect_with_message("/users", notice=f"Password reset: {target_user.email}") - diff --git a/app/web/routers/auth.py b/app/web/routers/auth.py deleted file mode 100644 index 8c1c819..0000000 --- a/app/web/routers/auth.py +++ /dev/null @@ -1,214 +0,0 @@ -from __future__ import annotations - -import logging -from typing import Annotated - -from fastapi import APIRouter, Depends, Form, Request, status -from fastapi.responses import HTMLResponse, RedirectResponse -from sqlalchemy.ext.asyncio import AsyncSession - -from app.auth.deps import get_auth_service, get_login_rate_limiter -from app.auth.rate_limit import SlidingWindowRateLimiter -from app.auth.service import AuthService -from app.auth.session import get_session_user, set_session_user -from app.db.session import get_db_session -from app.security.csrf import ensure_csrf_token -from app.services import users as user_service -from app.theme import resolve_theme -from app.web import common - -logger = logging.getLogger(__name__) - -router = APIRouter() - - -@router.get("/login", response_class=HTMLResponse) -async def login_page(request: Request, theme: str | None = None) -> HTMLResponse: - active_theme = resolve_theme(theme) - ensure_csrf_token(request) - if get_session_user(request) is not None: - return RedirectResponse("/", status_code=status.HTTP_303_SEE_OTHER) - return common.render_login_page(request, theme_name=active_theme) - - -@router.post("/login", response_class=HTMLResponse) -async def login( - request: Request, - email: Annotated[str, Form()], - password: Annotated[str, Form()], - db_session: AsyncSession = Depends(get_db_session), - auth_service: AuthService = Depends(get_auth_service), - rate_limiter: SlidingWindowRateLimiter = Depends(get_login_rate_limiter), -) -> HTMLResponse: - active_theme = resolve_theme(None) - limiter_key = common.login_rate_limit_key(request, email) - decision = rate_limiter.check(limiter_key) - normalized_email = email.strip().lower() - if not decision.allowed: - logger.warning( - "auth.login_rate_limited", - extra={ - "event": "auth.login_rate_limited", - "email": normalized_email, - "retry_after_seconds": decision.retry_after_seconds, - }, - ) - response = common.render_login_page( - request, - theme_name=active_theme, - error_message="Too many login attempts. Please try again later.", - status_code=status.HTTP_429_TOO_MANY_REQUESTS, - retry_after_seconds=decision.retry_after_seconds, - ) - response.headers["Retry-After"] = str(decision.retry_after_seconds) - return response - - user = await auth_service.authenticate_user( - db_session, - email=email, - password=password, - ) - if user is None: - rate_limiter.record_failure(limiter_key) - logger.info( - "auth.login_failed", - extra={ - "event": "auth.login_failed", - "email": normalized_email, - }, - ) - return common.render_login_page( - request, - theme_name=active_theme, - error_message="Invalid email or password.", - status_code=status.HTTP_401_UNAUTHORIZED, - ) - - rate_limiter.reset(limiter_key) - set_session_user( - request, - user_id=user.id, - email=user.email, - is_admin=user.is_admin, - ) - ensure_csrf_token(request) - logger.info( - "auth.login_succeeded", - extra={ - "event": "auth.login_succeeded", - "user_id": user.id, - "is_admin": user.is_admin, - }, - ) - return RedirectResponse("/", status_code=status.HTTP_303_SEE_OTHER) - - -@router.post("/logout") -async def logout(request: Request) -> RedirectResponse: - session_user = get_session_user(request) - common.invalidate_session(request) - logger.info( - "auth.logout", - extra={ - "event": "auth.logout", - "user_id": session_user.id if session_user else None, - }, - ) - return RedirectResponse("/login", status_code=status.HTTP_303_SEE_OTHER) - - -@router.get("/account/password", response_class=HTMLResponse) -async def account_password_page( - request: Request, - theme: str | None = None, - db_session: AsyncSession = Depends(get_db_session), -) -> HTMLResponse: - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - return common.templates.TemplateResponse( - request=request, - name="account_password.html", - context=common.build_template_context( - request, - page_title="Change Password", - active_nav="account_password", - theme_name=resolve_theme(theme), - session_user=common.to_session_user(current_user), - notice=request.query_params.get("notice"), - page_error=request.query_params.get("error"), - ), - ) - - -@router.post("/account/password") -async def update_account_password( - request: Request, - current_password: Annotated[str, Form()], - new_password: Annotated[str, Form()], - confirm_password: Annotated[str, Form()], - db_session: AsyncSession = Depends(get_db_session), - auth_service: AuthService = Depends(get_auth_service), -) -> RedirectResponse: - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - if not auth_service.verify_password( - password_hash=current_user.password_hash, - password=current_password, - ): - logger.info( - "account.password_change_failed", - extra={ - "event": "account.password_change_failed", - "user_id": current_user.id, - "reason": "invalid_current_password", - }, - ) - return common.redirect_with_message( - "/account/password", - error="Current password is incorrect.", - ) - if new_password != confirm_password: - logger.info( - "account.password_change_failed", - extra={ - "event": "account.password_change_failed", - "user_id": current_user.id, - "reason": "confirmation_mismatch", - }, - ) - return common.redirect_with_message( - "/account/password", - error="New password and confirmation do not match.", - ) - try: - validated_password = user_service.validate_password(new_password) - except user_service.UserServiceError as exc: - logger.info( - "account.password_change_failed", - extra={ - "event": "account.password_change_failed", - "user_id": current_user.id, - "reason": "validation_error", - }, - ) - return common.redirect_with_message("/account/password", error=str(exc)) - - await user_service.set_user_password_hash( - db_session, - user=current_user, - password_hash=auth_service.hash_password(validated_password), - ) - logger.info( - "account.password_changed", - extra={ - "event": "account.password_changed", - "user_id": current_user.id, - }, - ) - return common.redirect_with_message( - "/account/password", - notice="Password updated successfully.", - ) - diff --git a/app/web/routers/dashboard.py b/app/web/routers/dashboard.py deleted file mode 100644 index efbff30..0000000 --- a/app/web/routers/dashboard.py +++ /dev/null @@ -1,167 +0,0 @@ -from __future__ import annotations - -import logging - -from fastapi import APIRouter, Depends, Request -from fastapi.responses import HTMLResponse -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import RunTriggerType -from app.db.session import get_db_session -from app.presentation import dashboard as dashboard_presenter -from app.services import ingestion as ingestion_service -from app.services import publications as publication_service -from app.services import runs as run_service -from app.services import user_settings as user_settings_service -from app.settings import settings -from app.theme import resolve_theme -from app.web import common -from app.web.deps import get_ingestion_service - -logger = logging.getLogger(__name__) - -router = APIRouter() - - -async def _render_dashboard_page( - request: Request, - *, - db_session: AsyncSession, - current_user, - theme_name: str, - notice: str | None = None, - page_error: str | None = None, -) -> HTMLResponse: - unread_publications = await publication_service.list_new_for_latest_run_for_user( - db_session, - user_id=current_user.id, - limit=50, - ) - recent_runs = await run_service.list_recent_runs_for_user( - db_session, - user_id=current_user.id, - limit=20, - ) - user_settings = await user_settings_service.get_or_create_settings( - db_session, - user_id=current_user.id, - ) - queue_counts = await run_service.queue_status_counts_for_user( - db_session, - user_id=current_user.id, - ) - context = common.build_template_context( - request, - page_title="Dashboard", - active_nav="home", - theme_name=theme_name, - session_user=common.to_session_user(current_user), - notice=notice, - page_error=page_error, - ) - context["dashboard"] = dashboard_presenter.build_dashboard_view_model( - unread_publications=unread_publications, - recent_runs=recent_runs, - request_delay_seconds=user_settings.request_delay_seconds, - queue_counts=queue_counts, - ) - return common.templates.TemplateResponse( - request=request, - name="index.html", - context=context, - ) - - -@router.get("/", response_class=HTMLResponse) -async def home( - request: Request, - theme: str | None = None, - db_session: AsyncSession = Depends(get_db_session), -) -> HTMLResponse: - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - return await _render_dashboard_page( - request, - db_session=db_session, - current_user=current_user, - theme_name=resolve_theme(theme), - notice=request.query_params.get("notice"), - page_error=request.query_params.get("error"), - ) - - -@router.post("/runs/manual") -async def run_manual_ingestion( - request: Request, - db_session: AsyncSession = Depends(get_db_session), - ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service), -): - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - - user_settings = await user_settings_service.get_or_create_settings( - db_session, - user_id=current_user.id, - ) - logger.info( - "runs.manual_started", - extra={ - "event": "runs.manual_started", - "user_id": current_user.id, - "request_delay_seconds": user_settings.request_delay_seconds, - "network_error_retries": settings.ingestion_network_error_retries, - "max_pages_per_scholar": settings.ingestion_max_pages_per_scholar, - "page_size": settings.ingestion_page_size, - }, - ) - try: - run_summary = await ingest_service.run_for_user( - db_session, - user_id=current_user.id, - trigger_type=RunTriggerType.MANUAL, - request_delay_seconds=user_settings.request_delay_seconds, - network_error_retries=settings.ingestion_network_error_retries, - retry_backoff_seconds=settings.ingestion_retry_backoff_seconds, - max_pages_per_scholar=settings.ingestion_max_pages_per_scholar, - page_size=settings.ingestion_page_size, - auto_queue_continuations=settings.ingestion_continuation_queue_enabled, - queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds, - ) - except ingestion_service.RunAlreadyInProgressError: - await db_session.rollback() - return common.redirect_with_message( - "/", - error="A run is already in progress for this account.", - ) - except Exception: - await db_session.rollback() - logger.exception( - "runs.manual_failed", - extra={ - "event": "runs.manual_failed", - "user_id": current_user.id, - }, - ) - return common.redirect_with_message("/", error="Manual run failed. Check logs for details.") - - logger.info( - "runs.manual_completed", - extra={ - "event": "runs.manual_completed", - "user_id": current_user.id, - "run_id": run_summary.crawl_run_id, - "status": run_summary.status.value, - "scholar_count": run_summary.scholar_count, - "new_publication_count": run_summary.new_publication_count, - }, - ) - return common.redirect_with_message( - "/", - notice=( - f"Run #{run_summary.crawl_run_id} complete ({run_summary.status.value}). " - f"Scholars: {run_summary.scholar_count}, " - f"new publications: {run_summary.new_publication_count}." - ), - ) diff --git a/app/web/routers/health.py b/app/web/routers/health.py deleted file mode 100644 index 3c7c860..0000000 --- a/app/web/routers/health.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations - -from fastapi import APIRouter, HTTPException - -from app.db.session import check_database - -router = APIRouter() - - -@router.get("/healthz") -async def healthz() -> dict[str, str]: - if await check_database(): - return {"status": "ok"} - raise HTTPException(status_code=500, detail="database unavailable") - diff --git a/app/web/routers/publications.py b/app/web/routers/publications.py deleted file mode 100644 index c4ce7a2..0000000 --- a/app/web/routers/publications.py +++ /dev/null @@ -1,165 +0,0 @@ -from __future__ import annotations - -import logging -from urllib.parse import urlencode, urlparse - -from fastapi import APIRouter, Depends, Form, Request -from fastapi.responses import HTMLResponse -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.session import get_db_session -from app.services import publications as publication_service -from app.services import scholars as scholar_service -from app.theme import resolve_theme -from app.web import common - -logger = logging.getLogger(__name__) - -router = APIRouter() - -MODE_NEW = "new" -MODE_ALL = "all" - - -def _resolve_mode(raw_mode: str | None) -> str: - return publication_service.resolve_mode(raw_mode) - - -def _parse_scholar_profile_id(value: str | None) -> int | None: - if not value: - return None - try: - parsed = int(value) - except ValueError: - return None - return parsed if parsed > 0 else None - - -def _build_publications_url(*, mode: str, scholar_profile_id: int | None) -> str: - params: dict[str, str] = {"mode": mode} - if scholar_profile_id is not None: - params["scholar_profile_id"] = str(scholar_profile_id) - return f"/publications?{urlencode(params)}" - - -def _is_safe_return_to(value: str | None) -> bool: - if not value: - return False - parsed = urlparse(value) - if parsed.scheme or parsed.netloc: - return False - if not value.startswith("/"): - return False - return value == "/" or value.startswith("/publications") - - -@router.get("/publications", response_class=HTMLResponse) -async def publications_page( - request: Request, - mode: str | None = None, - scholar_profile_id: str | None = None, - theme: str | None = None, - db_session: AsyncSession = Depends(get_db_session), -) -> HTMLResponse: - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - - resolved_mode = _resolve_mode(mode) - scholar_id_filter = _parse_scholar_profile_id(scholar_profile_id) - scholars = await scholar_service.list_scholars_for_user( - db_session, - user_id=current_user.id, - ) - scholar_lookup = {scholar.id: scholar for scholar in scholars} - if scholar_id_filter not in scholar_lookup: - scholar_id_filter = None - - publications = await publication_service.list_for_user( - db_session, - user_id=current_user.id, - mode=resolved_mode, - scholar_profile_id=scholar_id_filter, - limit=500, - ) - new_count = await publication_service.count_for_user( - db_session, - user_id=current_user.id, - mode=MODE_NEW, - scholar_profile_id=scholar_id_filter, - ) - total_count = await publication_service.count_for_user( - db_session, - user_id=current_user.id, - mode=MODE_ALL, - scholar_profile_id=scholar_id_filter, - ) - mode_new_url = _build_publications_url( - mode=MODE_NEW, - scholar_profile_id=scholar_id_filter, - ) - mode_all_url = _build_publications_url( - mode=MODE_ALL, - scholar_profile_id=scholar_id_filter, - ) - selected_scholar = scholar_lookup.get(scholar_id_filter) if scholar_id_filter else None - current_publications_url = _build_publications_url( - mode=resolved_mode, - scholar_profile_id=scholar_id_filter, - ) - - context = common.build_template_context( - request, - page_title="Publications", - active_nav="publications", - theme_name=resolve_theme(theme), - session_user=common.to_session_user(current_user), - notice=request.query_params.get("notice"), - page_error=request.query_params.get("error"), - ) - context["mode"] = resolved_mode - context["publications"] = publications - context["new_count"] = new_count - context["total_count"] = total_count - context["mode_new_url"] = mode_new_url - context["mode_all_url"] = mode_all_url - context["scholars"] = scholars - context["selected_scholar_id"] = scholar_id_filter - context["selected_scholar"] = selected_scholar - context["current_publications_url"] = current_publications_url - return common.templates.TemplateResponse( - request=request, - name="publications.html", - context=context, - ) - - -@router.post("/publications/mark-all-read") -async def mark_all_publications_read( - request: Request, - return_to: str | None = Form(default=None), - db_session: AsyncSession = Depends(get_db_session), -): - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - - updated_count = await publication_service.mark_all_unread_as_read_for_user( - db_session, - user_id=current_user.id, - ) - logger.info( - "publications.mark_all_read", - extra={ - "event": "publications.mark_all_read", - "user_id": current_user.id, - "updated_count": updated_count, - }, - ) - redirect_target = "/publications?mode=new" - if _is_safe_return_to(return_to): - redirect_target = return_to - return common.redirect_with_message( - redirect_target, - notice=f"Marked {updated_count} publication(s) as read.", - ) diff --git a/app/web/routers/runs.py b/app/web/routers/runs.py deleted file mode 100644 index c6e395f..0000000 --- a/app/web/routers/runs.py +++ /dev/null @@ -1,274 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timezone - -from fastapi import APIRouter, Depends, HTTPException, Request -from fastapi.responses import HTMLResponse -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.session import get_db_session -from app.services import runs as run_service -from app.theme import resolve_theme -from app.web import common - -router = APIRouter() - - -def _as_bool(value: str | None) -> bool: - if value is None: - return False - return value.strip().lower() in {"1", "true", "yes", "on"} - - -def _status_badge(status_value: str) -> str: - if status_value == "success": - return "ok" - if status_value == "failed": - return "danger" - return "warn" - - -def _queue_status_badge(status_value: str) -> str: - if status_value == "dropped": - return "danger" - if status_value == "retrying": - return "ok" - return "warn" - - -def _format_dt(value: datetime | None) -> str: - if value is None: - return "-" - return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") - - -def _summary_dict(error_log: object) -> dict[str, object]: - if not isinstance(error_log, dict): - return {} - summary = error_log.get("summary") - if not isinstance(summary, dict): - return {} - return summary - - -@router.get("/runs", response_class=HTMLResponse) -async def runs_page( - request: Request, - theme: str | None = None, - failed_only: str | None = None, - db_session: AsyncSession = Depends(get_db_session), -) -> HTMLResponse: - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - - failed_only_enabled = _as_bool(failed_only) - runs = await run_service.list_runs_for_user( - db_session, - user_id=current_user.id, - limit=200, - failed_only=failed_only_enabled, - ) - - run_items = [] - for run in runs: - summary = _summary_dict(run.error_log) - failed_count = summary.get("failed_count", 0) - partial_count = summary.get("partial_count", 0) - run_items.append( - { - "id": run.id, - "started_at": _format_dt(run.start_dt), - "finished_at": _format_dt(run.end_dt), - "status": run.status.value, - "status_badge": _status_badge(run.status.value), - "trigger_type": run.trigger_type.value, - "scholar_count": run.scholar_count, - "new_publication_count": run.new_pub_count, - "failed_count": failed_count, - "partial_count": partial_count, - } - ) - - queue_entries = await run_service.list_queue_items_for_user( - db_session, - user_id=current_user.id, - limit=200, - ) - queue_items = [] - for item in queue_entries: - queue_items.append( - { - "id": item.id, - "scholar_profile_id": item.scholar_profile_id, - "scholar_label": item.scholar_label, - "status": item.status, - "status_badge": _queue_status_badge(item.status), - "reason": item.reason, - "dropped_reason": item.dropped_reason, - "attempt_count": item.attempt_count, - "resume_cstart": item.resume_cstart, - "next_attempt_at": _format_dt(item.next_attempt_dt), - "updated_at": _format_dt(item.updated_at), - "last_error": item.last_error, - "last_run_id": item.last_run_id, - } - ) - - context = common.build_template_context( - request, - page_title="Runs", - active_nav="runs", - theme_name=resolve_theme(theme), - session_user=common.to_session_user(current_user), - notice=request.query_params.get("notice"), - page_error=request.query_params.get("error"), - ) - context["runs"] = run_items - context["failed_only"] = failed_only_enabled - context["queue_items"] = queue_items - return common.templates.TemplateResponse( - request=request, - name="runs.html", - context=context, - ) - - -@router.get("/runs/{run_id}", response_class=HTMLResponse) -async def run_detail_page( - request: Request, - run_id: int, - theme: str | None = None, - db_session: AsyncSession = Depends(get_db_session), -) -> HTMLResponse: - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - - run = await run_service.get_run_for_user( - db_session, - user_id=current_user.id, - run_id=run_id, - ) - if run is None: - raise HTTPException(status_code=404, detail="Run not found.") - - error_log = run.error_log if isinstance(run.error_log, dict) else {} - scholar_results = error_log.get("scholar_results", []) - if not isinstance(scholar_results, list): - scholar_results = [] - - context = common.build_template_context( - request, - page_title=f"Run #{run.id}", - active_nav="runs", - theme_name=resolve_theme(theme), - session_user=common.to_session_user(current_user), - ) - context["run"] = { - "id": run.id, - "started_at": _format_dt(run.start_dt), - "finished_at": _format_dt(run.end_dt), - "status": run.status.value, - "status_badge": _status_badge(run.status.value), - "trigger_type": run.trigger_type.value, - "scholar_count": run.scholar_count, - "new_publication_count": run.new_pub_count, - } - context["run_summary"] = _summary_dict(error_log) - context["scholar_results"] = scholar_results - return common.templates.TemplateResponse( - request=request, - name="run_detail.html", - context=context, - ) - - -@router.post("/runs/queue/{queue_item_id}/drop") -async def drop_queue_item( - request: Request, - queue_item_id: int, - db_session: AsyncSession = Depends(get_db_session), -): - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - - try: - dropped = await run_service.drop_queue_item_for_user( - db_session, - user_id=current_user.id, - queue_item_id=queue_item_id, - ) - except run_service.QueueTransitionError as exc: - return common.redirect_with_message( - "/runs", - error=f"Queue item #{queue_item_id}: {exc.message}", - ) - if dropped is None: - raise HTTPException(status_code=404, detail="Queue item not found.") - return common.redirect_with_message( - "/runs", - notice=f"Queue item #{queue_item_id} marked as dropped.", - ) - - -@router.post("/runs/queue/{queue_item_id}/clear") -async def clear_queue_item( - request: Request, - queue_item_id: int, - db_session: AsyncSession = Depends(get_db_session), -): - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - - try: - cleared = await run_service.clear_queue_item_for_user( - db_session, - user_id=current_user.id, - queue_item_id=queue_item_id, - ) - except run_service.QueueTransitionError as exc: - return common.redirect_with_message( - "/runs", - error=f"Queue item #{queue_item_id}: {exc.message}", - ) - if cleared is None: - raise HTTPException(status_code=404, detail="Queue item not found.") - return common.redirect_with_message( - "/runs", - notice=f"Queue item #{queue_item_id} cleared.", - ) - - -@router.post("/runs/queue/{queue_item_id}/retry") -async def retry_queue_item( - request: Request, - queue_item_id: int, - db_session: AsyncSession = Depends(get_db_session), -): - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - - try: - queue_item = await run_service.retry_queue_item_for_user( - db_session, - user_id=current_user.id, - queue_item_id=queue_item_id, - ) - except run_service.QueueTransitionError as exc: - return common.redirect_with_message( - "/runs", - error=f"Queue item #{queue_item_id}: {exc.message}", - ) - if queue_item is None: - raise HTTPException(status_code=404, detail="Queue item not found.") - return common.redirect_with_message( - "/runs", - notice=( - f"Queue item #{queue_item_id} queued for retry " - f"(scholar: {queue_item.scholar_label})." - ), - ) diff --git a/app/web/routers/scholars.py b/app/web/routers/scholars.py deleted file mode 100644 index a761f20..0000000 --- a/app/web/routers/scholars.py +++ /dev/null @@ -1,212 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timezone -import logging -from typing import Annotated - -from fastapi import APIRouter, Depends, Form, HTTPException, Request, status -from fastapi.responses import HTMLResponse -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.session import get_db_session -from app.services import scholars as scholar_service -from app.theme import resolve_theme -from app.web import common - -logger = logging.getLogger(__name__) - -router = APIRouter() - - -def _format_dt(value: datetime | None) -> str: - if value is None: - return "-" - return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") - - -async def _render_scholars_page( - request: Request, - *, - db_session: AsyncSession, - current_user, - theme_name: str, - notice: str | None = None, - page_error: str | None = None, - status_code: int = status.HTTP_200_OK, - form_scholar_id: str = "", - form_display_name: str = "", -) -> HTMLResponse: - scholars = await scholar_service.list_scholars_for_user( - db_session, - user_id=current_user.id, - ) - scholar_items = [] - for scholar in scholars: - scholar_items.append( - { - "id": scholar.id, - "scholar_id": scholar.scholar_id, - "display_name": scholar.display_name or "Unnamed", - "is_enabled": scholar.is_enabled, - "baseline_completed": scholar.baseline_completed, - "last_run_status": ( - scholar.last_run_status.value - if scholar.last_run_status is not None - else "never" - ), - "last_run_dt": _format_dt(scholar.last_run_dt), - } - ) - context = common.build_template_context( - request, - page_title="Scholars", - active_nav="scholars", - theme_name=theme_name, - session_user=common.to_session_user(current_user), - notice=notice, - page_error=page_error, - ) - context["scholars"] = scholar_items - context["form_scholar_id"] = form_scholar_id - context["form_display_name"] = form_display_name - return common.templates.TemplateResponse( - request=request, - name="scholars.html", - context=context, - status_code=status_code, - ) - - -@router.get("/scholars", response_class=HTMLResponse) -async def scholars_page( - request: Request, - theme: str | None = None, - db_session: AsyncSession = Depends(get_db_session), -) -> HTMLResponse: - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - return await _render_scholars_page( - request, - db_session=db_session, - current_user=current_user, - theme_name=resolve_theme(theme), - notice=request.query_params.get("notice"), - page_error=request.query_params.get("error"), - ) - - -@router.post("/scholars") -async def create_scholar( - request: Request, - scholar_id: Annotated[str, Form()], - display_name: Annotated[str, Form()] = "", - db_session: AsyncSession = Depends(get_db_session), -) -> HTMLResponse: - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - active_theme = resolve_theme(None) - try: - created_profile = await scholar_service.create_scholar_for_user( - db_session, - user_id=current_user.id, - scholar_id=scholar_id, - display_name=display_name, - ) - except scholar_service.ScholarServiceError as exc: - logger.info( - "scholars.create_failed", - extra={ - "event": "scholars.create_failed", - "user_id": current_user.id, - "scholar_id": scholar_id.strip(), - }, - ) - return await _render_scholars_page( - request, - db_session=db_session, - current_user=current_user, - theme_name=active_theme, - page_error=str(exc), - status_code=status.HTTP_400_BAD_REQUEST, - form_scholar_id=scholar_id, - form_display_name=display_name, - ) - label = created_profile.display_name or created_profile.scholar_id - logger.info( - "scholars.created", - extra={ - "event": "scholars.created", - "user_id": current_user.id, - "scholar_profile_id": created_profile.id, - }, - ) - return common.redirect_with_message("/scholars", notice=f"Scholar added: {label}") - - -@router.post("/scholars/{scholar_profile_id}/toggle") -async def toggle_scholar( - request: Request, - scholar_profile_id: int, - db_session: AsyncSession = Depends(get_db_session), -): - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - - profile = await scholar_service.get_user_scholar_by_id( - db_session, - user_id=current_user.id, - scholar_profile_id=scholar_profile_id, - ) - if profile is None: - raise HTTPException(status_code=404, detail="Scholar not found.") - updated_profile = await scholar_service.toggle_scholar_enabled( - db_session, - profile=profile, - ) - status_label = "enabled" if updated_profile.is_enabled else "disabled" - logger.info( - "scholars.toggled", - extra={ - "event": "scholars.toggled", - "user_id": current_user.id, - "scholar_profile_id": updated_profile.id, - "is_enabled": updated_profile.is_enabled, - }, - ) - return common.redirect_with_message( - "/scholars", - notice=f"Scholar {status_label}: {updated_profile.scholar_id}", - ) - - -@router.post("/scholars/{scholar_profile_id}/delete") -async def delete_scholar( - request: Request, - scholar_profile_id: int, - db_session: AsyncSession = Depends(get_db_session), -): - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - - profile = await scholar_service.get_user_scholar_by_id( - db_session, - user_id=current_user.id, - scholar_profile_id=scholar_profile_id, - ) - if profile is None: - raise HTTPException(status_code=404, detail="Scholar not found.") - deleted_label = profile.display_name or profile.scholar_id - await scholar_service.delete_scholar(db_session, profile=profile) - logger.info( - "scholars.deleted", - extra={ - "event": "scholars.deleted", - "user_id": current_user.id, - "scholar_profile_id": scholar_profile_id, - }, - ) - return common.redirect_with_message("/scholars", notice=f"Scholar removed: {deleted_label}") diff --git a/app/web/routers/settings.py b/app/web/routers/settings.py deleted file mode 100644 index e5b7ba5..0000000 --- a/app/web/routers/settings.py +++ /dev/null @@ -1,94 +0,0 @@ -from __future__ import annotations - -import logging -from typing import Annotated - -from fastapi import APIRouter, Depends, Form, Request -from fastapi.responses import HTMLResponse -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.session import get_db_session -from app.services import user_settings as user_settings_service -from app.theme import resolve_theme -from app.web import common - -logger = logging.getLogger(__name__) - -router = APIRouter() - - -@router.get("/settings", response_class=HTMLResponse) -async def settings_page( - request: Request, - theme: str | None = None, - db_session: AsyncSession = Depends(get_db_session), -) -> HTMLResponse: - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - - user_settings = await user_settings_service.get_or_create_settings( - db_session, - user_id=current_user.id, - ) - context = common.build_template_context( - request, - page_title="Settings", - active_nav="settings", - theme_name=resolve_theme(theme), - session_user=common.to_session_user(current_user), - notice=request.query_params.get("notice"), - page_error=request.query_params.get("error"), - ) - context["user_settings"] = user_settings - return common.templates.TemplateResponse( - request=request, - name="settings.html", - context=context, - ) - - -@router.post("/settings") -async def update_settings( - request: Request, - run_interval_minutes: Annotated[str, Form()], - request_delay_seconds: Annotated[str, Form()], - auto_run_enabled: Annotated[str | None, Form()] = None, - db_session: AsyncSession = Depends(get_db_session), -): - current_user = await common.get_authenticated_user(request, db_session) - if current_user is None: - return common.redirect_to_login() - try: - parsed_interval = user_settings_service.parse_run_interval_minutes( - run_interval_minutes - ) - parsed_delay = user_settings_service.parse_request_delay_seconds( - request_delay_seconds - ) - except user_settings_service.UserSettingsServiceError as exc: - return common.redirect_with_message("/settings", error=str(exc)) - - user_settings = await user_settings_service.get_or_create_settings( - db_session, - user_id=current_user.id, - ) - await user_settings_service.update_settings( - db_session, - settings=user_settings, - auto_run_enabled=auto_run_enabled == "on", - run_interval_minutes=parsed_interval, - request_delay_seconds=parsed_delay, - ) - logger.info( - "settings.updated", - extra={ - "event": "settings.updated", - "user_id": current_user.id, - "auto_run_enabled": auto_run_enabled == "on", - "run_interval_minutes": parsed_interval, - "request_delay_seconds": parsed_delay, - }, - ) - return common.redirect_with_message("/settings", notice="Settings updated.") - diff --git a/docker-compose.yml b/docker-compose.yml index a6d4cda..b764f1c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: LOG_FORMAT: ${LOG_FORMAT:-console} LOG_REQUESTS: ${LOG_REQUESTS:-1} LOG_UVICORN_ACCESS: ${LOG_UVICORN_ACCESS:-0} - LOG_REQUEST_SKIP_PATHS: ${LOG_REQUEST_SKIP_PATHS:-/healthz,/static/} + LOG_REQUEST_SKIP_PATHS: ${LOG_REQUEST_SKIP_PATHS:-/healthz} LOG_REDACT_FIELDS: ${LOG_REDACT_FIELDS:-} SCHEDULER_ENABLED: ${SCHEDULER_ENABLED:-1} SCHEDULER_TICK_SECONDS: ${SCHEDULER_TICK_SECONDS:-60} @@ -45,6 +45,11 @@ services: INGESTION_RETRY_BACKOFF_SECONDS: ${INGESTION_RETRY_BACKOFF_SECONDS:-1.0} INGESTION_MAX_PAGES_PER_SCHOLAR: ${INGESTION_MAX_PAGES_PER_SCHOLAR:-30} INGESTION_PAGE_SIZE: ${INGESTION_PAGE_SIZE:-100} + INGESTION_CONTINUATION_QUEUE_ENABLED: ${INGESTION_CONTINUATION_QUEUE_ENABLED:-1} + INGESTION_CONTINUATION_BASE_DELAY_SECONDS: ${INGESTION_CONTINUATION_BASE_DELAY_SECONDS:-120} + INGESTION_CONTINUATION_MAX_DELAY_SECONDS: ${INGESTION_CONTINUATION_MAX_DELAY_SECONDS:-3600} + INGESTION_CONTINUATION_MAX_ATTEMPTS: ${INGESTION_CONTINUATION_MAX_ATTEMPTS:-6} + SCHEDULER_QUEUE_BATCH_SIZE: ${SCHEDULER_QUEUE_BATCH_SIZE:-10} BOOTSTRAP_ADMIN_ON_START: ${BOOTSTRAP_ADMIN_ON_START:-0} BOOTSTRAP_ADMIN_EMAIL: ${BOOTSTRAP_ADMIN_EMAIL:-} BOOTSTRAP_ADMIN_PASSWORD: ${BOOTSTRAP_ADMIN_PASSWORD:-} diff --git a/planning/mvp_implementation_reminders.md b/planning/mvp_implementation_reminders.md deleted file mode 100644 index 87bc41f..0000000 --- a/planning/mvp_implementation_reminders.md +++ /dev/null @@ -1,92 +0,0 @@ -# MVP Implementation Reminders (From Scholar Probe) - -Last validated probe run: `planning/scholar_probe_tmp/notes/probe_report_run_20260216T182334Z.md`. - -## Decision: Are we ready? - -Yes, for the scoped MVP. - -- We have stable selectors and field coverage for first-page profile parsing. -- We have explicit failure modes for blocked/inaccessible pages. -- We have enough fixtures to start parser + ingestion tests immediately. - -Not guaranteed (and intentionally out of MVP): - -- Full historical completeness from profile pagination. -- Robots currently disallows `citations?*cstart=`; treat deep pagination as out of scope unless policy changes. - -## Non-negotiables - -- Keep app container-first (`docker compose`) and test while developing. -- Keep feature scope small; prefer deterministic behavior over clever scraping. -- Never crash a whole run because one scholar page fails. -- Preserve strict tenant isolation for all run/output/read-state data. - -## Scraping Contract - -Target URL: - -- `https://scholar.google.com/citations?hl=en&user=` - -Primary parse markers: - -- Row: `tr.gsc_a_tr` -- Title/link: `a.gsc_a_at` -- Citation count: `a.gsc_a_ac` -- Year: `span.gsc_a_h` (fallback year regex) -- Metadata lines: first/second `div.gs_gray` => authors/venue -- Cluster id from `citation_for_view=:` - -## Required Parse States - -Use and persist one of: - -- `ok` -- `no_results` -- `blocked_or_captcha` -- `layout_changed` -- `network_error` - -Plus these page-level flags: - -- `has_show_more_button` -- `articles_range` - -## Data Quality Expectations - -Observed from probe sample: - -- `title`: 100% -- `cluster_id`: 100% -- `citation_count`: 100% -- `authors_text`: 100% -- `year`: 98.2% -- `venue_text`: 94.7% - -Implications: - -- `year` and `venue_text` must remain nullable. -- Dedupe must not depend solely on `year`/`venue_text` presence. - -## Run Semantics - -- First successful run per scholar = baseline. -- If `has_show_more_button=true`, label result as partial in run status/UI. -- Invalid/inaccessible scholar IDs are expected and should become structured run errors, not exceptions. - -## Testing Priorities (Do Before Feature Expansion) - -- Fixture parser unit tests using probe HTML snapshots. -- Parse-state classification tests (ok/blocked/layout/no_results). -- Dedupe integration tests (`cluster_id` first, fingerprint fallback). -- Tenant isolation tests for run records and read-state. -- Smoke test for manual run path through Docker. - -## Stop Conditions - -Pause implementation and re-probe if: - -- Selector markers drop out unexpectedly. -- Login/redirect pages become frequent for valid IDs. -- Robots policy for profile endpoints changes. - diff --git a/planning/scholar_probe_tmp/README.md b/planning/scholar_probe_tmp/README.md deleted file mode 100644 index b6c6ad6..0000000 --- a/planning/scholar_probe_tmp/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Scholar Scrape Probe (Temporary) - -Purpose: short-lived workspace for information gathering and planning. - -Rules: -- No production code changes here. -- Use this for HTML samples, parsing experiments, and extraction notes. -- Delete this directory after we lock the scrape contract and parser plan. - -Structure: -- `fixtures/` saved HTML samples for parser validation -- `notes/` extraction decisions, edge cases, and risk log -- `scripts/` one-off probe scripts used only during planning diff --git a/planning/scholar_probe_tmp/notes/findings_phase1_input.md b/planning/scholar_probe_tmp/notes/findings_phase1_input.md deleted file mode 100644 index 4a404e1..0000000 --- a/planning/scholar_probe_tmp/notes/findings_phase1_input.md +++ /dev/null @@ -1,71 +0,0 @@ -# Probe Findings -> Phase 1 Input - -Generated from live probe run: `run_20260216T182334Z`. - -## What is stable enough to build on - -- Public profile endpoint works for anonymous access: - - `GET /citations?hl=en&user=` -- Core row structure is present and parseable: - - row: `tr.gsc_a_tr` - - title + detail URL: `a.gsc_a_at` - - citation count: `a.gsc_a_ac` - - year: `span.gsc_a_h` (fallback: `td.gsc_a_y` text regex) - - metadata lines: first/second `div.gs_gray` => authors/venue -- `cluster_id` can be extracted reliably from `citation_for_view=:` in title URLs. - -## What can break / where to degrade gracefully - -- Invalid or inaccessible profile IDs may redirect to Google sign-in page. - - Treat as `blocked_or_captcha` / `inaccessible` state, not parser crash. -- `Show more` is present for tested profiles. - - We currently parse first page only. - - Because robots currently disallows `citations?*cstart=`, deep pagination should not be assumed. -- Some rows are missing year or venue text. - - Keep nullable fields and avoid failing dedupe for these gaps. - -## Observed quality from probe - -- Parsed publication rows: 57 across 4 accessible profiles. -- Field coverage: - - title: 100% - - cluster_id: 100% - - citation_count: 100% - - authors_text: 100% - - year: 98.2% - - venue_text: 94.7% - -## Recommended parser contract for implementation - -- Input -> `PublicationCandidate` - - `title` (required) - - `cluster_id` (required when present in URL; expected high coverage) - - `year` (nullable) - - `citation_count` (nullable/int default fallback 0) - - `authors_text` (nullable) - - `venue_text` (nullable) - - `title_url` (nullable) -- Page-level parse status enum: - - `ok`, `no_results`, `blocked_or_captcha`, `layout_changed`, `network_error` -- Page-level flags: - - `has_show_more_button` - - `articles_range` string (e.g. `Articles 1–20`) - -## Immediate test plan unlocked by this probe - -- Add fixture-driven unit tests using captured HTML from `fixtures/run_20260216T182334Z`. -- Add assertions for: - - row count > 0 on known-good fixtures - - profile name extraction - - cluster_id extraction from title URLs - - nullable handling for year/venue - - blocked/inaccessible page classification - - show-more partial warning classification - -## Phase 1 implementation guardrails - -- Keep requests low-rate with jitter and explicit timeout. -- Persist parser status per scholar run. -- If `has_show_more_button=True`, mark run as partial and show this in UI. -- Never fail entire run because one scholar page is blocked or malformed. - diff --git a/pyproject.toml b/pyproject.toml index 0669801..40166f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,8 +14,6 @@ dependencies = [ "asyncpg>=0.30,<0.31", "fastapi>=0.116,<0.117", "itsdangerous>=2.2,<3.0", - "jinja2>=3.1,<3.2", - "python-multipart>=0.0.20,<0.0.21", "sqlalchemy>=2.0,<2.1", "uvicorn[standard]>=0.34,<0.35", ] diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 49cf32f..d1f5c68 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -1,36 +1,29 @@ from __future__ import annotations -import re - from fastapi.testclient import TestClient from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession from app.auth.security import PasswordService -CSRF_TOKEN_PATTERN = re.compile(r'name="csrf_token" value="([^"]+)"') - - -def extract_csrf_token(html: str) -> str: - match = CSRF_TOKEN_PATTERN.search(html) - assert match is not None - return match.group(1) - - def login_user(client: TestClient, *, email: str, password: str) -> None: - login_page = client.get("/login") - csrf_token = extract_csrf_token(login_page.text) + bootstrap_response = client.get("/api/v1/auth/csrf") + assert bootstrap_response.status_code == 200 + csrf_token = bootstrap_response.json()["data"]["csrf_token"] + assert isinstance(csrf_token, str) and csrf_token + response = client.post( - "/login", - data={ + "/api/v1/auth/login", + json={ "email": email, "password": password, - "csrf_token": csrf_token, }, + headers={"X-CSRF-Token": csrf_token}, follow_redirects=False, ) - assert response.status_code == 303 - assert response.headers["location"] == "/" + assert response.status_code == 200 + payload = response.json() + assert payload["data"]["authenticated"] is True async def insert_user( @@ -60,4 +53,3 @@ async def insert_user( user_id = int(result.scalar_one()) await db_session.commit() return user_id - diff --git a/tests/integration/test_admin_user_management.py b/tests/integration/test_admin_user_management.py deleted file mode 100644 index 7553317..0000000 --- a/tests/integration/test_admin_user_management.py +++ /dev/null @@ -1,200 +0,0 @@ -import pytest -from fastapi.testclient import TestClient -from sqlalchemy import text -from sqlalchemy.ext.asyncio import AsyncSession - -from app.main import app -from tests.integration.helpers import extract_csrf_token, insert_user, login_user - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_users_page_is_admin_only(db_session: AsyncSession) -> None: - await insert_user( - db_session, - email="member@example.com", - password="member-pass", - is_admin=False, - ) - - client = TestClient(app) - login_user(client, email="member@example.com", password="member-pass") - - response = client.get("/users") - - assert response.status_code == 403 - assert response.json()["detail"] == "Admin access required." - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_non_admin_dashboard_hides_users_nav(db_session: AsyncSession) -> None: - await insert_user( - db_session, - email="member@example.com", - password="member-pass", - is_admin=False, - ) - - client = TestClient(app) - login_user(client, email="member@example.com", password="member-pass") - dashboard = client.get("/") - - assert dashboard.status_code == 200 - assert ">Users<" not in dashboard.text - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_admin_dashboard_shows_users_nav(db_session: AsyncSession) -> None: - await insert_user( - db_session, - email="admin@example.com", - password="admin-pass", - is_admin=True, - ) - - client = TestClient(app) - login_user(client, email="admin@example.com", password="admin-pass") - dashboard = client.get("/") - - assert dashboard.status_code == 200 - assert ">Users<" in dashboard.text - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_admin_can_create_and_deactivate_user(db_session: AsyncSession) -> None: - await insert_user( - db_session, - email="admin@example.com", - password="admin-pass", - is_admin=True, - ) - - client = TestClient(app) - login_user(client, email="admin@example.com", password="admin-pass") - - users_page = client.get("/users") - csrf_token = extract_csrf_token(users_page.text) - create_response = client.post( - "/users", - data={ - "email": "new-user@example.com", - "password": "new-user-pass", - "csrf_token": csrf_token, - }, - follow_redirects=False, - ) - assert create_response.status_code == 303 - assert create_response.headers["location"].startswith("/users") - - created_user_id_result = await db_session.execute( - text("SELECT id FROM users WHERE email = 'new-user@example.com'") - ) - created_user_id = int(created_user_id_result.scalar_one()) - - users_page_after_create = client.get("/users") - csrf_after_create = extract_csrf_token(users_page_after_create.text) - toggle_response = client.post( - f"/users/{created_user_id}/toggle-active", - data={"csrf_token": csrf_after_create}, - follow_redirects=False, - ) - assert toggle_response.status_code == 303 - - status_result = await db_session.execute( - text("SELECT is_active FROM users WHERE id = :user_id"), - {"user_id": created_user_id}, - ) - assert status_result.scalar_one() is False - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_admin_can_reset_user_password(db_session: AsyncSession) -> None: - target_user_id = await insert_user( - db_session, - email="target@example.com", - password="old-password", - is_admin=False, - ) - await insert_user( - db_session, - email="admin@example.com", - password="admin-pass", - is_admin=True, - ) - - client = TestClient(app) - login_user(client, email="admin@example.com", password="admin-pass") - - users_page = client.get("/users") - csrf_token = extract_csrf_token(users_page.text) - reset_response = client.post( - f"/users/{target_user_id}/reset-password", - data={ - "new_password": "new-password", - "csrf_token": csrf_token, - }, - follow_redirects=False, - ) - assert reset_response.status_code == 303 - - users_page_after_reset = client.get("/users") - logout_csrf = extract_csrf_token(users_page_after_reset.text) - client.post( - "/logout", - data={"csrf_token": logout_csrf}, - follow_redirects=False, - ) - - failed_login_page = client.get("/login") - failed_login_csrf = extract_csrf_token(failed_login_page.text) - failed_login = client.post( - "/login", - data={ - "email": "target@example.com", - "password": "old-password", - "csrf_token": failed_login_csrf, - }, - follow_redirects=False, - ) - assert failed_login.status_code == 401 - - login_user(client, email="target@example.com", password="new-password") - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_admin_cannot_deactivate_self(db_session: AsyncSession) -> None: - admin_user_id = await insert_user( - db_session, - email="admin@example.com", - password="admin-pass", - is_admin=True, - ) - client = TestClient(app) - login_user(client, email="admin@example.com", password="admin-pass") - - users_page = client.get("/users") - csrf_token = extract_csrf_token(users_page.text) - response = client.post( - f"/users/{admin_user_id}/toggle-active", - data={"csrf_token": csrf_token}, - follow_redirects=False, - ) - - assert response.status_code == 303 - assert response.headers["location"].startswith("/users") - result = await db_session.execute( - text("SELECT is_active FROM users WHERE id = :user_id"), - {"user_id": admin_user_id}, - ) - assert result.scalar_one() is True diff --git a/tests/integration/test_auth_flow.py b/tests/integration/test_auth_flow.py deleted file mode 100644 index 2455b80..0000000 --- a/tests/integration/test_auth_flow.py +++ /dev/null @@ -1,109 +0,0 @@ -import re - -import pytest -from sqlalchemy import text -from sqlalchemy.ext.asyncio import AsyncSession -from fastapi.testclient import TestClient - -from app.auth.security import PasswordService -from app.main import app - -CSRF_TOKEN_PATTERN = re.compile(r'name="csrf_token" value="([^"]+)"') - - -def _extract_csrf_token(html: str) -> str: - match = CSRF_TOKEN_PATTERN.search(html) - assert match is not None - return match.group(1) - - -@pytest.mark.integration -def test_dashboard_requires_authentication() -> None: - client = TestClient(app) - - response = client.get("/", follow_redirects=False) - - assert response.status_code == 303 - assert response.headers["location"] == "/login" - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_login_with_valid_credentials_allows_access(db_session: AsyncSession) -> None: - password_service = PasswordService() - await db_session.execute( - text( - """ - INSERT INTO users (email, password_hash, is_active, is_admin) - VALUES (:email, :password_hash, :is_active, :is_admin) - """ - ), - { - "email": "reader@example.com", - "password_hash": password_service.hash_password("correct-password"), - "is_active": True, - "is_admin": False, - }, - ) - await db_session.commit() - - client = TestClient(app) - login_page = client.get("/login") - csrf_token = _extract_csrf_token(login_page.text) - login_response = client.post( - "/login", - data={ - "email": "reader@example.com", - "password": "correct-password", - "csrf_token": csrf_token, - }, - follow_redirects=False, - ) - - assert login_response.status_code == 303 - assert login_response.headers["location"] == "/" - - dashboard_response = client.get("/") - assert dashboard_response.status_code == 200 - assert "reader@example.com" in dashboard_response.text - assert 'data-test="home-hero"' in dashboard_response.text - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_login_rejects_inactive_user(db_session: AsyncSession) -> None: - password_service = PasswordService() - await db_session.execute( - text( - """ - INSERT INTO users (email, password_hash, is_active, is_admin) - VALUES (:email, :password_hash, :is_active, :is_admin) - """ - ), - { - "email": "inactive@example.com", - "password_hash": password_service.hash_password("correct-password"), - "is_active": False, - "is_admin": False, - }, - ) - await db_session.commit() - - client = TestClient(app) - login_page = client.get("/login") - csrf_token = _extract_csrf_token(login_page.text) - login_response = client.post( - "/login", - data={ - "email": "inactive@example.com", - "password": "correct-password", - "csrf_token": csrf_token, - }, - follow_redirects=False, - ) - - assert login_response.status_code == 401 - assert "Invalid email or password." in login_response.text - diff --git a/tests/integration/test_manual_ingestion_flow.py b/tests/integration/test_manual_ingestion_flow.py deleted file mode 100644 index 25de49a..0000000 --- a/tests/integration/test_manual_ingestion_flow.py +++ /dev/null @@ -1,1151 +0,0 @@ -from __future__ import annotations - -import pytest -from fastapi.testclient import TestClient -from sqlalchemy import text -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine - -from app.main import app, get_scholar_source -from app.db.models import RunTriggerType -from app.services.ingestion import RUN_LOCK_NAMESPACE, RunAlreadyInProgressError, ScholarIngestionService -from app.services.scheduler import SchedulerService -from app.services.scholar_source import FetchResult -from tests.integration.helpers import extract_csrf_token, insert_user, login_user - -HTML_BASELINE = """ - -
Fixture Scholar
- Articles 1-1 - - - - - - -
- Paper One -
A Author
-
Venue One, 2024
-
32024
- -""" - -HTML_INCREMENTAL = """ - -
Fixture Scholar
- Articles 1-2 - - - - - - - - - - - -
- Paper One -
A Author
-
Venue One, 2024
-
42024
- Paper Two -
B Author
-
Venue Two, 2025
-
12025
- -""" - -HTML_PAGED_ONE = """ - -
Paged Scholar
- Articles 1-1 - - - - - - -
- Paged Paper One -
P Author
-
Paged Venue, 2023
-
52023
-
- -
- -""" - -HTML_PAGED_TWO = """ - -
Paged Scholar
- Articles 2-2 - - - - - - -
- Paged Paper Two -
P Author
-
Paged Venue, 2024
-
22024
- -""" - -HTML_STALLED_TAIL_ONE = """ - -
Tail Scholar
- Articles 1-1 - - - - - - -
- Tail Paper One -
T Author
-
Tail Venue, 2022
-
92022
-
- -""" - -HTML_STALLED_TAIL_TWO = """ - -
Tail Scholar
-
No documents. Your search didn't match any articles.
-
- -""" - - -class StubScholarSource: - def __init__(self, html_bodies: list[str]) -> None: - self._html_bodies = html_bodies - self._index = 0 - - async def fetch_profile_html(self, scholar_id: str) -> FetchResult: - if self._index >= len(self._html_bodies): - body = self._html_bodies[-1] - else: - body = self._html_bodies[self._index] - self._index += 1 - - url = f"https://scholar.google.com/citations?hl=en&user={scholar_id}" - return FetchResult( - requested_url=url, - status_code=200, - final_url=url, - body=body, - error=None, - ) - - -class StubScholarSourceResults: - def __init__(self, results: list[FetchResult]) -> None: - self._results = results - self._index = 0 - - async def fetch_profile_html(self, scholar_id: str) -> FetchResult: - if self._index >= len(self._results): - result = self._results[-1] - else: - result = self._results[self._index] - self._index += 1 - return result - - -class CountingScholarSourceResults(StubScholarSourceResults): - def __init__(self, results: list[FetchResult]) -> None: - super().__init__(results) - self.calls = 0 - - async def fetch_profile_html(self, scholar_id: str) -> FetchResult: - self.calls += 1 - return await super().fetch_profile_html(scholar_id) - - -class StubPagedScholarSource: - def __init__(self, pages: dict[int, str]) -> None: - self._pages = pages - self.calls: list[tuple[int, int]] = [] - - async def fetch_profile_html(self, scholar_id: str) -> FetchResult: - return await self.fetch_profile_page_html( - scholar_id, - cstart=0, - pagesize=100, - ) - - async def fetch_profile_page_html( - self, - scholar_id: str, - *, - cstart: int, - pagesize: int, - ) -> FetchResult: - self.calls.append((cstart, pagesize)) - body = self._pages.get(cstart, "") - url = ( - "https://scholar.google.com/citations" - f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={pagesize}" - ) - return FetchResult( - requested_url=url, - status_code=200, - final_url=url, - body=body, - error=None, - ) - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_manual_ingestion_sets_baseline_then_adds_unread(db_session: AsyncSession) -> None: - user_id = await insert_user( - db_session, - email="ingest@example.com", - password="ingest-password", - ) - await db_session.execute( - text( - """ - INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) - VALUES (:user_id, :scholar_id, :display_name, :is_enabled) - """ - ), - { - "user_id": user_id, - "scholar_id": "abcDEF123456", - "display_name": "Fixture Scholar", - "is_enabled": True, - }, - ) - await db_session.commit() - - stub_source = StubScholarSource([HTML_BASELINE, HTML_INCREMENTAL]) - app.dependency_overrides[get_scholar_source] = lambda: stub_source - - client = TestClient(app) - try: - login_user(client, email="ingest@example.com", password="ingest-password") - - dashboard = client.get("/") - csrf = extract_csrf_token(dashboard.text) - run_one = client.post( - "/runs/manual", - data={"csrf_token": csrf}, - follow_redirects=False, - ) - assert run_one.status_code == 303 - assert run_one.headers["location"].startswith("/") - - baseline_status = await db_session.execute( - text("SELECT baseline_completed FROM scholar_profiles WHERE user_id = :user_id"), - {"user_id": user_id}, - ) - assert baseline_status.scalar_one() is True - - run_one_stats = await db_session.execute( - text( - """ - SELECT status::text, new_pub_count - FROM crawl_runs - WHERE user_id = :user_id - ORDER BY id DESC - LIMIT 1 - """ - ), - {"user_id": user_id}, - ) - assert run_one_stats.one() == ("success", 1) - - unread_after_baseline = await db_session.execute( - text( - """ - SELECT COUNT(*) - FROM scholar_publications sp - JOIN scholar_profiles s ON s.id = sp.scholar_profile_id - WHERE s.user_id = :user_id AND sp.is_read = false - """ - ), - {"user_id": user_id}, - ) - assert unread_after_baseline.scalar_one() == 1 - - dashboard_again = client.get("/") - csrf_two = extract_csrf_token(dashboard_again.text) - run_two = client.post( - "/runs/manual", - data={"csrf_token": csrf_two}, - follow_redirects=False, - ) - assert run_two.status_code == 303 - - run_two_stats = await db_session.execute( - text( - """ - SELECT status::text, new_pub_count - FROM crawl_runs - WHERE user_id = :user_id - ORDER BY id DESC - LIMIT 1 - """ - ), - {"user_id": user_id}, - ) - assert run_two_stats.one() == ("success", 1) - - unread_after_second = await db_session.execute( - text( - """ - SELECT COUNT(*) - FROM scholar_publications sp - JOIN scholar_profiles s ON s.id = sp.scholar_profile_id - WHERE s.user_id = :user_id AND sp.is_read = false - """ - ), - {"user_id": user_id}, - ) - assert unread_after_second.scalar_one() == 2 - - publications_new_page = client.get("/publications?mode=new") - assert publications_new_page.status_code == 200 - assert "Paper Two" in publications_new_page.text - assert "Paper One" not in publications_new_page.text - - publications_all_page = client.get("/publications?mode=all") - assert publications_all_page.status_code == 200 - assert "Paper One" in publications_all_page.text - assert "Paper Two" in publications_all_page.text - - dashboard_mark = client.get("/") - csrf_mark = extract_csrf_token(dashboard_mark.text) - mark_response = client.post( - "/publications/mark-all-read", - data={"csrf_token": csrf_mark}, - follow_redirects=False, - ) - assert mark_response.status_code == 303 - - unread_after_mark = await db_session.execute( - text( - """ - SELECT COUNT(*) - FROM scholar_publications sp - JOIN scholar_profiles s ON s.id = sp.scholar_profile_id - WHERE s.user_id = :user_id AND sp.is_read = false - """ - ), - {"user_id": user_id}, - ) - assert unread_after_mark.scalar_one() == 0 - - publications_new_after_mark = client.get("/publications?mode=new") - assert publications_new_after_mark.status_code == 200 - assert "Paper Two" in publications_new_after_mark.text - assert "Paper One" not in publications_new_after_mark.text - finally: - app.dependency_overrides.clear() - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_publications_page_supports_scholar_filtering_navigation( - db_session: AsyncSession, -) -> None: - user_id = await insert_user( - db_session, - email="filtering@example.com", - password="filter-password", - ) - - scholar_one_result = await db_session.execute( - text( - """ - INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) - VALUES (:user_id, :scholar_id, :display_name, :is_enabled) - RETURNING id - """ - ), - { - "user_id": user_id, - "scholar_id": "abcDEF123456", - "display_name": "Scholar One", - "is_enabled": True, - }, - ) - scholar_one_id = int(scholar_one_result.scalar_one()) - - scholar_two_result = await db_session.execute( - text( - """ - INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) - VALUES (:user_id, :scholar_id, :display_name, :is_enabled) - RETURNING id - """ - ), - { - "user_id": user_id, - "scholar_id": "uvwXYZ987654", - "display_name": "Scholar Two", - "is_enabled": True, - }, - ) - scholar_two_id = int(scholar_two_result.scalar_one()) - - run_result = await db_session.execute( - text( - """ - INSERT INTO crawl_runs ( - user_id, - trigger_type, - status, - start_dt, - end_dt, - scholar_count, - new_pub_count, - error_log - ) - VALUES ( - :user_id, - 'manual', - 'success', - NOW(), - NOW(), - 2, - 2, - '{}'::jsonb - ) - RETURNING id - """ - ), - {"user_id": user_id}, - ) - run_id = int(run_result.scalar_one()) - - publication_one_result = await db_session.execute( - text( - """ - INSERT INTO publications ( - cluster_id, - fingerprint_sha256, - title_raw, - title_normalized, - year, - citation_count, - author_text, - venue_text, - pub_url, - pdf_url - ) - VALUES ( - NULL, - :fingerprint, - :title_raw, - :title_normalized, - :year, - 0, - NULL, - NULL, - NULL, - NULL - ) - RETURNING id - """ - ), - { - "fingerprint": "f" * 64, - "title_raw": "Scholar One Paper", - "title_normalized": "scholar one paper", - "year": 2024, - }, - ) - publication_one_id = int(publication_one_result.scalar_one()) - - publication_two_result = await db_session.execute( - text( - """ - INSERT INTO publications ( - cluster_id, - fingerprint_sha256, - title_raw, - title_normalized, - year, - citation_count, - author_text, - venue_text, - pub_url, - pdf_url - ) - VALUES ( - NULL, - :fingerprint, - :title_raw, - :title_normalized, - :year, - 0, - NULL, - NULL, - NULL, - NULL - ) - RETURNING id - """ - ), - { - "fingerprint": "e" * 64, - "title_raw": "Scholar Two Paper", - "title_normalized": "scholar two paper", - "year": 2025, - }, - ) - publication_two_id = int(publication_two_result.scalar_one()) - - await db_session.execute( - text( - """ - INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, first_seen_run_id) - VALUES (:scholar_profile_id, :publication_id, :is_read, :first_seen_run_id) - """ - ), - { - "scholar_profile_id": scholar_one_id, - "publication_id": publication_one_id, - "is_read": False, - "first_seen_run_id": run_id, - }, - ) - await db_session.execute( - text( - """ - INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, first_seen_run_id) - VALUES (:scholar_profile_id, :publication_id, :is_read, :first_seen_run_id) - """ - ), - { - "scholar_profile_id": scholar_two_id, - "publication_id": publication_two_id, - "is_read": False, - "first_seen_run_id": run_id, - }, - ) - await db_session.commit() - - client = TestClient(app) - login_user(client, email="filtering@example.com", password="filter-password") - - scholars_page = client.get("/scholars") - assert scholars_page.status_code == 200 - assert f"/publications?mode=new&scholar_profile_id={scholar_one_id}" in scholars_page.text - assert f"/publications?mode=all&scholar_profile_id={scholar_two_id}" in scholars_page.text - - filtered_all = client.get(f"/publications?mode=all&scholar_profile_id={scholar_one_id}") - assert filtered_all.status_code == 200 - assert "Scholar One Paper" in filtered_all.text - assert "Scholar Two Paper" not in filtered_all.text - - filtered_new = client.get(f"/publications?mode=new&scholar_profile_id={scholar_two_id}") - assert filtered_new.status_code == 200 - assert "Scholar Two Paper" in filtered_new.text - assert "Scholar One Paper" not in filtered_new.text - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_manual_ingestion_persists_failure_debug_context(db_session: AsyncSession) -> None: - user_id = await insert_user( - db_session, - email="ingest-failure@example.com", - password="ingest-password", - ) - await db_session.execute( - text( - """ - INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) - VALUES (:user_id, :scholar_id, :display_name, :is_enabled) - """ - ), - { - "user_id": user_id, - "scholar_id": "abcDEF123456", - "display_name": "Failure Fixture", - "is_enabled": True, - }, - ) - await db_session.commit() - - stub_source = StubScholarSourceResults( - [ - FetchResult( - requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", - status_code=None, - final_url=None, - body="", - error="timed out", - ) - ] - ) - app.dependency_overrides[get_scholar_source] = lambda: stub_source - - client = TestClient(app) - try: - login_user(client, email="ingest-failure@example.com", password="ingest-password") - dashboard = client.get("/") - csrf = extract_csrf_token(dashboard.text) - run_response = client.post( - "/runs/manual", - data={"csrf_token": csrf}, - follow_redirects=False, - ) - assert run_response.status_code == 303 - - run_result = await db_session.execute( - text( - """ - SELECT status::text, error_log - FROM crawl_runs - WHERE user_id = :user_id - ORDER BY id DESC - LIMIT 1 - """ - ), - {"user_id": user_id}, - ) - status_text, error_log = run_result.one() - assert status_text == "failed" - assert error_log["summary"]["failed_count"] == 1 - assert error_log["summary"]["failed_state_counts"] == {"network_error": 1} - assert error_log["summary"]["failed_reason_counts"] == { - "network_error_missing_status_code": 1 - } - - scholar_result = error_log["scholar_results"][0] - assert scholar_result["state"] == "network_error" - assert scholar_result["state_reason"] == "network_error_missing_status_code" - debug = scholar_result["debug"] - assert debug["status_code"] is None - assert debug["fetch_error"] == "timed out" - assert debug["requested_url"].endswith("user=abcDEF123456") - assert debug["body_excerpt"] is None - - runs_page = client.get("/runs") - assert runs_page.status_code == 200 - assert "Run History" in runs_page.text - assert "failed /" in runs_page.text - - failed_only_page = client.get("/runs?failed_only=1") - assert failed_only_page.status_code == 200 - assert "Run History" in failed_only_page.text - - run_id_result = await db_session.execute( - text( - """ - SELECT id - FROM crawl_runs - WHERE user_id = :user_id - ORDER BY id DESC - LIMIT 1 - """ - ), - {"user_id": user_id}, - ) - run_id = run_id_result.scalar_one() - - detail_page = client.get(f"/runs/{run_id}") - assert detail_page.status_code == 200 - assert f"Run #{run_id}" in detail_page.text - assert "network_error_missing_status_code" in detail_page.text - finally: - app.dependency_overrides.clear() - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_manual_ingestion_fetches_all_pages_for_scholar(db_session: AsyncSession) -> None: - user_id = await insert_user( - db_session, - email="paged@example.com", - password="ingest-password", - ) - await db_session.execute( - text( - """ - INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) - VALUES (:user_id, :scholar_id, :display_name, :is_enabled) - """ - ), - { - "user_id": user_id, - "scholar_id": "abcDEF123456", - "display_name": "Paged Scholar", - "is_enabled": True, - }, - ) - await db_session.commit() - - source = StubPagedScholarSource( - { - 0: HTML_PAGED_ONE, - 1: HTML_PAGED_TWO, - } - ) - app.dependency_overrides[get_scholar_source] = lambda: source - - client = TestClient(app) - try: - login_user(client, email="paged@example.com", password="ingest-password") - dashboard = client.get("/") - csrf = extract_csrf_token(dashboard.text) - run_response = client.post( - "/runs/manual", - data={"csrf_token": csrf}, - follow_redirects=False, - ) - assert run_response.status_code == 303 - - assert source.calls[0][0] == 0 - assert source.calls[1][0] == 1 - - publications_count = await db_session.execute( - text( - """ - SELECT COUNT(*) - FROM scholar_publications sp - JOIN scholar_profiles s ON s.id = sp.scholar_profile_id - WHERE s.user_id = :user_id - """ - ), - {"user_id": user_id}, - ) - assert publications_count.scalar_one() == 2 - - run_result = await db_session.execute( - text( - """ - SELECT status::text, new_pub_count, error_log - FROM crawl_runs - WHERE user_id = :user_id - ORDER BY id DESC - LIMIT 1 - """ - ), - {"user_id": user_id}, - ) - status_text, new_pub_count, error_log = run_result.one() - assert status_text == "success" - assert new_pub_count == 2 - scholar_result = error_log["scholar_results"][0] - assert scholar_result["publication_count"] == 2 - assert scholar_result["pages_fetched"] == 2 - assert scholar_result["has_more_remaining"] is False - assert scholar_result["pagination_truncated_reason"] is None - finally: - app.dependency_overrides.clear() - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_manual_ingestion_handles_empty_no_results_tail_without_partial( - db_session: AsyncSession, -) -> None: - user_id = await insert_user( - db_session, - email="tail@example.com", - password="ingest-password", - ) - await db_session.execute( - text( - """ - INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) - VALUES (:user_id, :scholar_id, :display_name, :is_enabled) - """ - ), - { - "user_id": user_id, - "scholar_id": "abcDEF123456", - "display_name": "Tail Scholar", - "is_enabled": True, - }, - ) - await db_session.commit() - - source = StubPagedScholarSource( - { - 0: HTML_STALLED_TAIL_ONE, - 1: HTML_STALLED_TAIL_TWO, - } - ) - app.dependency_overrides[get_scholar_source] = lambda: source - - client = TestClient(app) - try: - login_user(client, email="tail@example.com", password="ingest-password") - dashboard = client.get("/") - csrf = extract_csrf_token(dashboard.text) - run_response = client.post( - "/runs/manual", - data={"csrf_token": csrf}, - follow_redirects=False, - ) - assert run_response.status_code == 303 - assert source.calls[0][0] == 0 - assert source.calls[1][0] == 1 - - run_result = await db_session.execute( - text( - """ - SELECT status::text, new_pub_count, error_log - FROM crawl_runs - WHERE user_id = :user_id - ORDER BY id DESC - LIMIT 1 - """ - ), - {"user_id": user_id}, - ) - status_text, new_pub_count, error_log = run_result.one() - assert status_text == "success" - assert new_pub_count == 1 - scholar_result = error_log["scholar_results"][0] - assert scholar_result["outcome"] == "success" - assert scholar_result["pagination_truncated_reason"] is None - - queue_count_result = await db_session.execute( - text( - """ - SELECT COUNT(*) - FROM ingestion_queue_items - WHERE user_id = :user_id - """ - ), - {"user_id": user_id}, - ) - assert queue_count_result.scalar_one() == 0 - finally: - app.dependency_overrides.clear() - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_ingestion_enqueues_continuation_when_max_pages_reached( - db_session: AsyncSession, -) -> None: - user_id = await insert_user( - db_session, - email="queued@example.com", - password="ingest-password", - ) - await db_session.execute( - text( - """ - INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) - VALUES (:user_id, :scholar_id, :display_name, :is_enabled) - """ - ), - { - "user_id": user_id, - "scholar_id": "abcDEF123456", - "display_name": "Queued Scholar", - "is_enabled": True, - }, - ) - await db_session.commit() - - source = StubPagedScholarSource({0: HTML_PAGED_ONE, 1: HTML_PAGED_TWO}) - service = ScholarIngestionService(source=source) - - summary = await service.run_for_user( - db_session, - user_id=user_id, - trigger_type=RunTriggerType.MANUAL, - request_delay_seconds=0, - network_error_retries=0, - retry_backoff_seconds=0, - max_pages_per_scholar=1, - page_size=100, - auto_queue_continuations=True, - queue_delay_seconds=0, - ) - - assert summary.status.value == "partial_failure" - queue_result = await db_session.execute( - text( - """ - SELECT reason, resume_cstart, attempt_count - FROM ingestion_queue_items - WHERE user_id = :user_id - """ - ), - {"user_id": user_id}, - ) - reason, resume_cstart, attempt_count = queue_result.one() - assert reason == "max_pages_reached" - assert int(resume_cstart) == 1 - assert int(attempt_count) == 0 - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_scheduler_processes_queued_continuation_items( - db_session: AsyncSession, -) -> None: - user_id = await insert_user( - db_session, - email="queue-scheduler@example.com", - password="ingest-password", - ) - scholar_result = await db_session.execute( - text( - """ - INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) - VALUES (:user_id, :scholar_id, :display_name, :is_enabled) - RETURNING id - """ - ), - { - "user_id": user_id, - "scholar_id": "abcDEF123456", - "display_name": "Queue Scheduler Scholar", - "is_enabled": True, - }, - ) - scholar_profile_id = int(scholar_result.scalar_one()) - await db_session.execute( - text( - """ - INSERT INTO ingestion_queue_items ( - user_id, - scholar_profile_id, - resume_cstart, - reason, - attempt_count, - next_attempt_dt - ) - VALUES ( - :user_id, - :scholar_profile_id, - :resume_cstart, - :reason, - :attempt_count, - NOW() - INTERVAL '1 minute' - ) - """ - ), - { - "user_id": user_id, - "scholar_profile_id": scholar_profile_id, - "resume_cstart": 1, - "reason": "max_pages_reached", - "attempt_count": 0, - }, - ) - await db_session.commit() - - source = StubPagedScholarSource({1: HTML_PAGED_TWO}) - scheduler = SchedulerService( - enabled=True, - tick_seconds=60, - network_error_retries=0, - retry_backoff_seconds=0, - max_pages_per_scholar=30, - page_size=100, - continuation_queue_enabled=True, - continuation_base_delay_seconds=1, - continuation_max_delay_seconds=60, - continuation_max_attempts=4, - queue_batch_size=10, - ) - scheduler._source = source - - await scheduler._tick_once() - - queue_count_result = await db_session.execute( - text( - """ - SELECT COUNT(*) - FROM ingestion_queue_items - WHERE user_id = :user_id - """ - ), - {"user_id": user_id}, - ) - assert queue_count_result.scalar_one() == 0 - - publications_count = await db_session.execute( - text( - """ - SELECT COUNT(*) - FROM scholar_publications sp - JOIN scholar_profiles s ON s.id = sp.scholar_profile_id - WHERE s.user_id = :user_id - """ - ), - {"user_id": user_id}, - ) - assert publications_count.scalar_one() == 1 - assert source.calls and source.calls[0][0] == 1 - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_ingestion_retries_network_error_and_recovers(db_session: AsyncSession) -> None: - user_id = await insert_user( - db_session, - email="retry@example.com", - password="ingest-password", - ) - await db_session.execute( - text( - """ - INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) - VALUES (:user_id, :scholar_id, :display_name, :is_enabled) - """ - ), - { - "user_id": user_id, - "scholar_id": "abcDEF123456", - "display_name": "Retry Fixture", - "is_enabled": True, - }, - ) - await db_session.commit() - - source = CountingScholarSourceResults( - [ - FetchResult( - requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", - status_code=None, - final_url=None, - body="", - error="temporary timeout", - ), - FetchResult( - requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", - status_code=200, - final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", - body=HTML_BASELINE, - error=None, - ), - ] - ) - service = ScholarIngestionService(source=source) - - summary = await service.run_for_user( - db_session, - user_id=user_id, - trigger_type=RunTriggerType.MANUAL, - request_delay_seconds=0, - network_error_retries=1, - retry_backoff_seconds=0, - ) - - assert source.calls == 2 - assert summary.status.value == "success" - assert summary.failed_count == 0 - assert summary.succeeded_count == 1 - - run_error_log_result = await db_session.execute( - text( - """ - SELECT error_log - FROM crawl_runs - WHERE user_id = :user_id - ORDER BY id DESC - LIMIT 1 - """ - ), - {"user_id": user_id}, - ) - run_error_log = run_error_log_result.scalar_one() - scholar_result = run_error_log["scholar_results"][0] - assert scholar_result["attempt_count"] == 2 - assert scholar_result["state"] == "ok" - assert scholar_result["state_reason"] == "publications_extracted" - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_ingestion_rejects_overlapping_run_lock( - db_session: AsyncSession, - database_url: str, -) -> None: - user_id = await insert_user( - db_session, - email="lock@example.com", - password="ingest-password", - ) - await db_session.execute( - text( - """ - INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) - VALUES (:user_id, :scholar_id, :display_name, :is_enabled) - """ - ), - { - "user_id": user_id, - "scholar_id": "abcDEF123456", - "display_name": "Lock Fixture", - "is_enabled": True, - }, - ) - await db_session.commit() - - engine = create_async_engine(database_url, pool_pre_ping=True) - factory = async_sessionmaker(engine, expire_on_commit=False) - lock_session = factory() - try: - await lock_session.execute( - text("SELECT pg_advisory_lock(:namespace, :user_key)"), - { - "namespace": RUN_LOCK_NAMESPACE, - "user_key": user_id, - }, - ) - - service = ScholarIngestionService( - source=CountingScholarSourceResults( - [ - FetchResult( - requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", - status_code=200, - final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456", - body=HTML_BASELINE, - error=None, - ) - ] - ) - ) - with pytest.raises(RunAlreadyInProgressError): - await service.run_for_user( - db_session, - user_id=user_id, - trigger_type=RunTriggerType.MANUAL, - request_delay_seconds=0, - network_error_retries=0, - retry_backoff_seconds=0, - ) - finally: - await lock_session.execute( - text("SELECT pg_advisory_unlock(:namespace, :user_key)"), - { - "namespace": RUN_LOCK_NAMESPACE, - "user_key": user_id, - }, - ) - await lock_session.close() - await engine.dispose() diff --git a/tests/integration/test_queue_ui.py b/tests/integration/test_queue_ui.py deleted file mode 100644 index 602ed60..0000000 --- a/tests/integration/test_queue_ui.py +++ /dev/null @@ -1,227 +0,0 @@ -from __future__ import annotations - -import pytest -from fastapi.testclient import TestClient -from sqlalchemy import text -from sqlalchemy.ext.asyncio import AsyncSession - -from app.main import app -from tests.integration.helpers import extract_csrf_token, insert_user, login_user - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_runs_page_queue_actions_lifecycle(db_session: AsyncSession) -> None: - user_id = await insert_user( - db_session, - email="queue-ui@example.com", - password="queue-ui-password", - ) - scholar_result = await db_session.execute( - text( - """ - INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) - VALUES (:user_id, :scholar_id, :display_name, true) - RETURNING id - """ - ), - { - "user_id": user_id, - "scholar_id": "abcDEF123456", - "display_name": "Queue UI Scholar", - }, - ) - scholar_profile_id = int(scholar_result.scalar_one()) - - queue_result = await db_session.execute( - text( - """ - INSERT INTO ingestion_queue_items ( - user_id, - scholar_profile_id, - resume_cstart, - reason, - status, - attempt_count, - next_attempt_dt, - last_error, - dropped_reason, - dropped_at - ) - VALUES ( - :user_id, - :scholar_profile_id, - 200, - 'dropped', - 'dropped', - 3, - NOW() + INTERVAL '30 minutes', - 'captcha challenge', - 'max_attempts_after_run', - NOW() - INTERVAL '1 minute' - ) - RETURNING id - """ - ), - { - "user_id": user_id, - "scholar_profile_id": scholar_profile_id, - }, - ) - queue_item_id = int(queue_result.scalar_one()) - await db_session.commit() - - client = TestClient(app) - login_user(client, email="queue-ui@example.com", password="queue-ui-password") - - runs_page = client.get("/runs") - assert runs_page.status_code == 200 - assert "Continuation Queue" in runs_page.text - assert "Queue UI Scholar" in runs_page.text - assert "dropped" in runs_page.text - assert "max_attempts_after_run" in runs_page.text - - csrf_retry = extract_csrf_token(runs_page.text) - retry_response = client.post( - f"/runs/queue/{queue_item_id}/retry", - data={"csrf_token": csrf_retry}, - follow_redirects=False, - ) - assert retry_response.status_code == 303 - - queue_after_retry = await db_session.execute( - text( - """ - SELECT status, reason, attempt_count - FROM ingestion_queue_items - WHERE id = :queue_item_id - """ - ), - {"queue_item_id": queue_item_id}, - ) - assert queue_after_retry.one() == ("queued", "manual_retry", 0) - - runs_page_after_retry = client.get("/runs") - csrf_drop = extract_csrf_token(runs_page_after_retry.text) - drop_response = client.post( - f"/runs/queue/{queue_item_id}/drop", - data={"csrf_token": csrf_drop}, - follow_redirects=False, - ) - assert drop_response.status_code == 303 - - queue_after_drop = await db_session.execute( - text( - """ - SELECT status, reason, dropped_reason - FROM ingestion_queue_items - WHERE id = :queue_item_id - """ - ), - {"queue_item_id": queue_item_id}, - ) - assert queue_after_drop.one() == ("dropped", "dropped", "manual_drop") - - runs_page_after_drop = client.get("/runs") - csrf_clear = extract_csrf_token(runs_page_after_drop.text) - clear_response = client.post( - f"/runs/queue/{queue_item_id}/clear", - data={"csrf_token": csrf_clear}, - follow_redirects=False, - ) - assert clear_response.status_code == 303 - - queue_after_clear = await db_session.execute( - text("SELECT COUNT(*) FROM ingestion_queue_items WHERE id = :queue_item_id"), - {"queue_item_id": queue_item_id}, - ) - assert queue_after_clear.scalar_one() == 0 - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_runs_queue_actions_are_tenant_scoped(db_session: AsyncSession) -> None: - user_a_id = await insert_user( - db_session, - email="queue-owner-a@example.com", - password="queue-owner-a-password", - ) - user_b_id = await insert_user( - db_session, - email="queue-owner-b@example.com", - password="queue-owner-b-password", - ) - - scholar_result = await db_session.execute( - text( - """ - INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) - VALUES (:user_id, :scholar_id, :display_name, true) - RETURNING id - """ - ), - { - "user_id": user_b_id, - "scholar_id": "zxyWVU654321", - "display_name": "Owner B Queue Scholar", - }, - ) - scholar_profile_id = int(scholar_result.scalar_one()) - queue_result = await db_session.execute( - text( - """ - INSERT INTO ingestion_queue_items ( - user_id, - scholar_profile_id, - resume_cstart, - reason, - status, - attempt_count, - next_attempt_dt - ) - VALUES ( - :user_id, - :scholar_profile_id, - 0, - 'max_pages_reached', - 'queued', - 0, - NOW() - ) - RETURNING id - """ - ), - { - "user_id": user_b_id, - "scholar_profile_id": scholar_profile_id, - }, - ) - queue_item_id = int(queue_result.scalar_one()) - await db_session.commit() - - client = TestClient(app) - login_user(client, email="queue-owner-a@example.com", password="queue-owner-a-password") - - runs_page = client.get("/runs") - csrf_token = extract_csrf_token(runs_page.text) - forbidden_retry = client.post( - f"/runs/queue/{queue_item_id}/retry", - data={"csrf_token": csrf_token}, - follow_redirects=False, - ) - assert forbidden_retry.status_code == 404 - - unchanged = await db_session.execute( - text( - """ - SELECT status, reason, user_id - FROM ingestion_queue_items - WHERE id = :queue_item_id - """ - ), - {"queue_item_id": queue_item_id}, - ) - assert unchanged.one() == ("queued", "max_pages_reached", user_b_id) - assert user_a_id != user_b_id diff --git a/tests/integration/test_tenant_routes.py b/tests/integration/test_tenant_routes.py deleted file mode 100644 index e5f7d59..0000000 --- a/tests/integration/test_tenant_routes.py +++ /dev/null @@ -1,197 +0,0 @@ -import pytest -from fastapi.testclient import TestClient -from sqlalchemy import text -from sqlalchemy.ext.asyncio import AsyncSession - -from app.main import app -from tests.integration.helpers import extract_csrf_token, insert_user, login_user - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_user_can_change_own_password(db_session: AsyncSession) -> None: - await insert_user( - db_session, - email="reader@example.com", - password="old-password", - ) - - client = TestClient(app) - login_user(client, email="reader@example.com", password="old-password") - - account_page = client.get("/account/password") - csrf_token = extract_csrf_token(account_page.text) - wrong_current = client.post( - "/account/password", - data={ - "current_password": "wrong-password", - "new_password": "new-password", - "confirm_password": "new-password", - "csrf_token": csrf_token, - }, - follow_redirects=False, - ) - assert wrong_current.status_code == 303 - assert wrong_current.headers["location"].startswith("/account/password") - - account_page_retry = client.get("/account/password") - retry_csrf = extract_csrf_token(account_page_retry.text) - success = client.post( - "/account/password", - data={ - "current_password": "old-password", - "new_password": "new-password", - "confirm_password": "new-password", - "csrf_token": retry_csrf, - }, - follow_redirects=False, - ) - assert success.status_code == 303 - - account_page_after = client.get("/account/password") - logout_csrf = extract_csrf_token(account_page_after.text) - client.post("/logout", data={"csrf_token": logout_csrf}, follow_redirects=False) - - failed_login_page = client.get("/login") - failed_login_csrf = extract_csrf_token(failed_login_page.text) - failed_login = client.post( - "/login", - data={ - "email": "reader@example.com", - "password": "old-password", - "csrf_token": failed_login_csrf, - }, - follow_redirects=False, - ) - assert failed_login.status_code == 401 - - login_user(client, email="reader@example.com", password="new-password") - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_scholar_routes_are_tenant_scoped(db_session: AsyncSession) -> None: - user_a_id = await insert_user( - db_session, - email="owner-a@example.com", - password="owner-a-password", - ) - user_b_id = await insert_user( - db_session, - email="owner-b@example.com", - password="owner-b-password", - ) - - client = TestClient(app) - login_user(client, email="owner-a@example.com", password="owner-a-password") - - scholars_page = client.get("/scholars") - csrf_token = extract_csrf_token(scholars_page.text) - create_response = client.post( - "/scholars", - data={ - "scholar_id": "abcDEF123456", - "display_name": "Owner A Scholar", - "csrf_token": csrf_token, - }, - follow_redirects=False, - ) - assert create_response.status_code == 303 - - owner_a_row = await db_session.execute( - text( - """ - SELECT user_id - FROM scholar_profiles - WHERE scholar_id = 'abcDEF123456' - """ - ) - ) - assert owner_a_row.scalar_one() == user_a_id - - owner_b_profile = await db_session.execute( - text( - """ - INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled) - VALUES (:user_id, :scholar_id, :display_name, :is_enabled) - RETURNING id - """ - ), - { - "user_id": user_b_id, - "scholar_id": "zxcvbn654321", - "display_name": "Owner B Scholar", - "is_enabled": True, - }, - ) - owner_b_profile_id = int(owner_b_profile.scalar_one()) - await db_session.commit() - - scholars_page_after_insert = client.get("/scholars") - csrf_after_insert = extract_csrf_token(scholars_page_after_insert.text) - forbidden_toggle = client.post( - f"/scholars/{owner_b_profile_id}/toggle", - data={"csrf_token": csrf_after_insert}, - follow_redirects=False, - ) - assert forbidden_toggle.status_code == 404 - - owner_b_status = await db_session.execute( - text("SELECT is_enabled FROM scholar_profiles WHERE id = :profile_id"), - {"profile_id": owner_b_profile_id}, - ) - assert owner_b_status.scalar_one() is True - - -@pytest.mark.integration -@pytest.mark.db -@pytest.mark.asyncio -async def test_settings_updates_are_user_scoped(db_session: AsyncSession) -> None: - user_a_id = await insert_user( - db_session, - email="settings-a@example.com", - password="settings-a-password", - ) - user_b_id = await insert_user( - db_session, - email="settings-b@example.com", - password="settings-b-password", - ) - - client = TestClient(app) - login_user(client, email="settings-a@example.com", password="settings-a-password") - - settings_page = client.get("/settings") - csrf_token = extract_csrf_token(settings_page.text) - response = client.post( - "/settings", - data={ - "auto_run_enabled": "on", - "run_interval_minutes": "45", - "request_delay_seconds": "7", - "csrf_token": csrf_token, - }, - follow_redirects=False, - ) - assert response.status_code == 303 - - user_a_settings = await db_session.execute( - text( - """ - SELECT auto_run_enabled, run_interval_minutes, request_delay_seconds - FROM user_settings - WHERE user_id = :user_id - """ - ), - {"user_id": user_a_id}, - ) - assert user_a_settings.one() == (True, 45, 7) - - user_b_settings = await db_session.execute( - text("SELECT COUNT(*) FROM user_settings WHERE user_id = :user_id"), - {"user_id": user_b_id}, - ) - assert user_b_settings.scalar_one() == 0 - diff --git a/tests/unit/test_auth.py b/tests/unit/test_auth.py deleted file mode 100644 index d471996..0000000 --- a/tests/unit/test_auth.py +++ /dev/null @@ -1,204 +0,0 @@ -import re -from collections.abc import AsyncIterator -from types import SimpleNamespace -from unittest.mock import AsyncMock - -from fastapi.testclient import TestClient -import pytest - -from app.auth.deps import get_auth_service, get_login_rate_limiter -from app.auth.rate_limit import SlidingWindowRateLimiter -from app.db.session import get_db_session -from app.main import app - -CSRF_TOKEN_PATTERN = re.compile(r'name="csrf_token" value="([^"]+)"') - - -class StubAuthService: - def __init__(self, *, user: object | None) -> None: - self._user = user - - async def authenticate_user(self, _db_session, *, email: str, password: str): - if self._user is None: - return None - if email.strip().lower() != str(self._user.email): - return None - if password != "correct-password": - return None - return self._user - - -def _extract_csrf_token(html: str) -> str: - match = CSRF_TOKEN_PATTERN.search(html) - assert match is not None - return match.group(1) - - -async def _override_db_session() -> AsyncIterator[object]: - yield object() - - -@pytest.fixture(autouse=True) -def clear_dependency_overrides() -> AsyncIterator[None]: - app.dependency_overrides.clear() - yield - app.dependency_overrides.clear() - - -def test_login_requires_csrf_token() -> None: - client = TestClient(app) - - response = client.post( - "/login", - data={"email": "user@example.com", "password": "correct-password"}, - follow_redirects=False, - ) - - assert response.status_code == 403 - assert response.text == "CSRF token missing." - - -def test_successful_login_creates_session_and_allows_dashboard(monkeypatch) -> None: - limiter = SlidingWindowRateLimiter(max_attempts=5, window_seconds=60) - app.dependency_overrides[get_db_session] = _override_db_session - app.dependency_overrides[get_auth_service] = lambda: StubAuthService( - user=SimpleNamespace(id=1, email="user@example.com", is_admin=False) - ) - app.dependency_overrides[get_login_rate_limiter] = lambda: limiter - client = TestClient(app) - monkeypatch.setattr( - "app.web.common.get_authenticated_user", - AsyncMock( - return_value=SimpleNamespace( - id=1, - email="user@example.com", - is_admin=False, - ) - ), - ) - monkeypatch.setattr( - "app.web.routers.dashboard.publication_service.list_new_for_latest_run_for_user", - AsyncMock(return_value=[]), - ) - monkeypatch.setattr( - "app.web.routers.dashboard.run_service.list_recent_runs_for_user", - AsyncMock(return_value=[]), - ) - monkeypatch.setattr( - "app.web.routers.dashboard.run_service.queue_status_counts_for_user", - AsyncMock(return_value={"queued": 0, "retrying": 0, "dropped": 0}), - ) - monkeypatch.setattr( - "app.web.routers.dashboard.user_settings_service.get_or_create_settings", - AsyncMock(return_value=SimpleNamespace(request_delay_seconds=0)), - ) - - login_page = client.get("/login") - csrf_token = _extract_csrf_token(login_page.text) - - login_response = client.post( - "/login", - data={ - "email": "user@example.com", - "password": "correct-password", - "csrf_token": csrf_token, - }, - follow_redirects=False, - ) - - assert login_response.status_code == 303 - assert login_response.headers["location"] == "/" - - dashboard_response = client.get("/") - assert dashboard_response.status_code == 200 - assert 'data-test="home-hero"' in dashboard_response.text - assert 'data-test="session-user"' in dashboard_response.text - assert "user@example.com" in dashboard_response.text - - -def test_login_rate_limiting_returns_429_after_threshold() -> None: - limiter = SlidingWindowRateLimiter(max_attempts=2, window_seconds=60) - app.dependency_overrides[get_db_session] = _override_db_session - app.dependency_overrides[get_auth_service] = lambda: StubAuthService(user=None) - app.dependency_overrides[get_login_rate_limiter] = lambda: limiter - client = TestClient(app) - - login_page = client.get("/login") - csrf_token = _extract_csrf_token(login_page.text) - payload = { - "email": "user@example.com", - "password": "wrong-password", - "csrf_token": csrf_token, - } - - first = client.post("/login", data=payload, follow_redirects=False) - second = client.post("/login", data=payload, follow_redirects=False) - third = client.post("/login", data=payload, follow_redirects=False) - - assert first.status_code == 401 - assert second.status_code == 401 - assert third.status_code == 429 - assert third.headers["Retry-After"] == "60" - - -def test_logout_requires_csrf_token_and_clears_session(monkeypatch) -> None: - limiter = SlidingWindowRateLimiter(max_attempts=5, window_seconds=60) - app.dependency_overrides[get_db_session] = _override_db_session - app.dependency_overrides[get_auth_service] = lambda: StubAuthService( - user=SimpleNamespace(id=1, email="user@example.com", is_admin=False) - ) - app.dependency_overrides[get_login_rate_limiter] = lambda: limiter - client = TestClient(app) - monkeypatch.setattr( - "app.web.common.get_authenticated_user", - AsyncMock( - side_effect=[ - SimpleNamespace(id=1, email="user@example.com", is_admin=False), - None, - ] - ), - ) - monkeypatch.setattr( - "app.web.routers.dashboard.publication_service.list_new_for_latest_run_for_user", - AsyncMock(return_value=[]), - ) - monkeypatch.setattr( - "app.web.routers.dashboard.run_service.list_recent_runs_for_user", - AsyncMock(return_value=[]), - ) - monkeypatch.setattr( - "app.web.routers.dashboard.run_service.queue_status_counts_for_user", - AsyncMock(return_value={"queued": 0, "retrying": 0, "dropped": 0}), - ) - monkeypatch.setattr( - "app.web.routers.dashboard.user_settings_service.get_or_create_settings", - AsyncMock(return_value=SimpleNamespace(request_delay_seconds=0)), - ) - - login_page = client.get("/login") - csrf_token = _extract_csrf_token(login_page.text) - client.post( - "/login", - data={ - "email": "user@example.com", - "password": "correct-password", - "csrf_token": csrf_token, - }, - follow_redirects=False, - ) - - failed_logout = client.post("/logout", data={}, follow_redirects=False) - assert failed_logout.status_code == 403 - assert failed_logout.text == "CSRF token invalid." - - dashboard_page = client.get("/") - logout_token = _extract_csrf_token(dashboard_page.text) - successful_logout = client.post( - "/logout", - data={"csrf_token": logout_token}, - follow_redirects=False, - ) - - assert successful_logout.status_code == 303 - assert successful_logout.headers["location"] == "/login" - assert client.get("/", follow_redirects=False).headers["location"] == "/login" diff --git a/tests/unit/test_dashboard_view_model.py b/tests/unit/test_dashboard_view_model.py deleted file mode 100644 index 70bb34a..0000000 --- a/tests/unit/test_dashboard_view_model.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timezone - -from app.db.models import CrawlRun, RunStatus, RunTriggerType -from app.presentation.dashboard import SECTION_TEMPLATES, build_dashboard_view_model -from app.services.publications import UnreadPublicationItem - - -def test_build_dashboard_view_model_maps_sections_and_unread_items() -> None: - unread_items = [ - UnreadPublicationItem( - publication_id=10, - scholar_profile_id=3, - scholar_label="Ada Lovelace", - title="Analytical Engine Notes", - year=None, - citation_count=42, - venue_text="Computing Letters", - pub_url="https://example.test/pub-10", - ) - ] - runs = [ - CrawlRun( - id=77, - user_id=1, - trigger_type=RunTriggerType.MANUAL, - status=RunStatus.SUCCESS, - start_dt=datetime(2026, 2, 16, 18, 0, tzinfo=timezone.utc), - scholar_count=1, - new_pub_count=1, - error_log={}, - ) - ] - - vm = build_dashboard_view_model( - unread_publications=unread_items, - recent_runs=runs, - request_delay_seconds=7, - queue_counts={"queued": 2, "retrying": 1, "dropped": 3}, - ) - - assert vm.section_templates == SECTION_TEMPLATES - assert vm.run_controls.request_delay_seconds == 7 - assert vm.run_controls.queue_queued_count == 2 - assert vm.run_controls.queue_retrying_count == 1 - assert vm.run_controls.queue_dropped_count == 3 - assert vm.unread_publications[0].title == "Analytical Engine Notes" - assert vm.unread_publications[0].year_display == "-" - assert vm.unread_publications[0].citation_count == 42 - assert vm.run_history[0].status_label == "success" - assert vm.run_history[0].status_badge == "ok" - assert vm.run_history[0].started_at_display == "2026-02-16 18:00 UTC" - assert vm.run_history[0].detail_url == "/runs/77" - - -def test_build_dashboard_view_model_maps_failed_and_partial_statuses() -> None: - runs = [ - CrawlRun( - id=11, - user_id=1, - trigger_type=RunTriggerType.MANUAL, - status=RunStatus.FAILED, - start_dt=datetime(2026, 2, 16, 19, 0, tzinfo=timezone.utc), - scholar_count=2, - new_pub_count=0, - error_log={}, - ), - CrawlRun( - id=12, - user_id=1, - trigger_type=RunTriggerType.MANUAL, - status=RunStatus.PARTIAL_FAILURE, - start_dt=datetime(2026, 2, 16, 20, 0, tzinfo=timezone.utc), - scholar_count=2, - new_pub_count=1, - error_log={}, - ), - ] - - vm = build_dashboard_view_model( - unread_publications=[], - recent_runs=runs, - request_delay_seconds=1, - ) - - assert [item.status_badge for item in vm.run_history] == ["danger", "warn"] - assert [item.status_label for item in vm.run_history] == ["failed", "partial_failure"] diff --git a/tests/unit/test_healthz.py b/tests/unit/test_healthz.py index a1776c3..32a993c 100644 --- a/tests/unit/test_healthz.py +++ b/tests/unit/test_healthz.py @@ -6,7 +6,7 @@ from app.main import app def test_healthz_returns_200_when_database_is_available(monkeypatch) -> None: - monkeypatch.setattr("app.web.routers.health.check_database", AsyncMock(return_value=True)) + monkeypatch.setattr("app.main.check_database", AsyncMock(return_value=True)) client = TestClient(app) response = client.get("/healthz") @@ -16,7 +16,7 @@ def test_healthz_returns_200_when_database_is_available(monkeypatch) -> None: def test_healthz_returns_500_when_database_is_unavailable(monkeypatch) -> None: - monkeypatch.setattr("app.web.routers.health.check_database", AsyncMock(return_value=False)) + monkeypatch.setattr("app.main.check_database", AsyncMock(return_value=False)) client = TestClient(app) response = client.get("/healthz") diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index ccd9293..84cf882 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -3,12 +3,13 @@ from __future__ import annotations import json import logging import re +from unittest.mock import AsyncMock from fastapi.testclient import TestClient from app.logging_config import JsonLogFormatter, parse_redact_fields from app.main import app -from app.web.middleware import REQUEST_ID_HEADER, parse_skip_paths +from app.http.middleware import REQUEST_ID_HEADER, parse_skip_paths def test_json_log_formatter_redacts_sensitive_fields() -> None: @@ -39,14 +40,16 @@ def test_json_log_formatter_redacts_sensitive_fields() -> None: assert "color_message" not in payload -def test_request_logging_middleware_sets_request_id_header() -> None: +def test_request_logging_middleware_sets_request_id_header(monkeypatch) -> None: + monkeypatch.setattr("app.main.check_database", AsyncMock(return_value=True)) client = TestClient(app) - - response = client.get("/login", headers={REQUEST_ID_HEADER: "request-123"}) - + response = client.get("/healthz", headers={REQUEST_ID_HEADER: "request-123"}) assert response.status_code == 200 assert response.headers[REQUEST_ID_HEADER] == "request-123" def test_parse_skip_paths_trims_and_discards_empty_segments() -> None: - assert parse_skip_paths(" /healthz , , /static/ ") == ("/healthz", "/static/") + assert parse_skip_paths(" /healthz , , /api/v1/metrics ") == ( + "/healthz", + "/api/v1/metrics", + ) diff --git a/tests/unit/test_pages.py b/tests/unit/test_pages.py deleted file mode 100644 index 2f0a003..0000000 --- a/tests/unit/test_pages.py +++ /dev/null @@ -1,54 +0,0 @@ -from fastapi.testclient import TestClient - -from app.main import app - - -def test_home_page_redirects_to_login_when_unauthenticated() -> None: - client = TestClient(app) - - response = client.get("/", follow_redirects=False) - - assert response.status_code == 303 - assert response.headers["location"] == "/login" - - -def test_dev_ui_page_is_removed() -> None: - client = TestClient(app) - - response = client.get("/dev/ui", follow_redirects=False) - - assert response.status_code == 404 - - -def test_login_page_renders_html() -> None: - client = TestClient(app) - - response = client.get("/login") - - assert response.status_code == 200 - assert response.headers["content-type"].startswith("text/html") - assert "scholarr" in response.text - assert 'data-test="login-form"' in response.text - assert 'name="csrf_token"' in response.text - assert 'data-theme-control' in response.text - set_cookie = response.headers["set-cookie"].lower() - assert "httponly" in set_cookie - assert "samesite=lax" in set_cookie - - -def test_theme_query_parameter_accepts_supported_theme() -> None: - client = TestClient(app) - - response = client.get("/login?theme=spruce") - - assert response.status_code == 200 - assert 'data-theme="spruce"' in response.text - - -def test_theme_query_parameter_falls_back_for_unknown_theme() -> None: - client = TestClient(app) - - response = client.get("/login?theme=not-a-theme") - - assert response.status_code == 200 - assert 'data-theme="terracotta"' in response.text diff --git a/tests/unit/test_phase2_pages.py b/tests/unit/test_phase2_pages.py deleted file mode 100644 index cf2a308..0000000 --- a/tests/unit/test_phase2_pages.py +++ /dev/null @@ -1,20 +0,0 @@ -from fastapi.testclient import TestClient - -from app.main import app - - -def test_phase2_pages_redirect_to_login_when_unauthenticated() -> None: - client = TestClient(app) - - for path in ( - "/users", - "/runs", - "/runs/1", - "/publications", - "/scholars", - "/settings", - "/account/password", - ): - response = client.get(path, follow_redirects=False) - assert response.status_code == 303 - assert response.headers["location"] == "/login" diff --git a/uv.lock b/uv.lock index 571b45e..8fde75a 100644 --- a/uv.lock +++ b/uv.lock @@ -346,18 +346,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234 }, ] -[[package]] -name = "jinja2" -version = "3.1.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899 }, -] - [[package]] name = "mako" version = "1.3.10" @@ -588,15 +576,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230 }, ] -[[package]] -name = "python-multipart" -version = "0.0.20" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/87/f44d7c9f274c7ee665a29b885ec97089ec5dc034c7f3fafa03da9e39a09e/python_multipart-0.0.20.tar.gz", hash = "sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13", size = 37158 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546 }, -] - [[package]] name = "pyyaml" version = "6.0.3" @@ -653,8 +632,6 @@ dependencies = [ { name = "asyncpg" }, { name = "fastapi" }, { name = "itsdangerous" }, - { name = "jinja2" }, - { name = "python-multipart" }, { name = "sqlalchemy" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -674,10 +651,8 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.116,<0.117" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.28,<0.29" }, { name = "itsdangerous", specifier = ">=2.2,<3.0" }, - { name = "jinja2", specifier = ">=3.1,<3.2" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3,<9.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.25,<0.26" }, - { name = "python-multipart", specifier = ">=0.0.20,<0.0.21" }, { name = "sqlalchemy", specifier = ">=2.0,<2.1" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.34,<0.35" }, ]