justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

test

Browse source
This commit is contained in:
Justin Visser 2026-02-17 20:24:12 +01:00
parent efd21a7297
commit 441676be27
97 changed files with 10764 additions and 223 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.dockerignore
View file

@ -12,3 +12,6 @@ __pycache__
*.log
.env
htmlcov
frontend/node_modules
frontend/dist
planning

25
.env.example
View file

@ -1,21 +1,26 @@
POSTGRES_DB=scholar
POSTGRES_USER=scholar
POSTGRES_PASSWORD=scholar
POSTGRES_PORT=5432
POSTGRES_PASSWORD=change-me
DATABASE_URL=postgresql+asyncpg://scholar:scholar@db:5432/scholar
# Optional override. If empty, tests derive "<DATABASE_URL db name>_test".
TEST_DATABASE_URL=
SCHOLARR_IMAGE=justinzeus/scholarr:latest
APP_NAME=scholarr
APP_HOST=0.0.0.0
APP_PORT=8000
APP_HOST_PORT=8000
APP_RELOAD=1
APP_RELOAD=0
FRONTEND_ENABLED=1
FRONTEND_DIST_DIR=/app/frontend/dist
FRONTEND_HOST_PORT=5173
CHOKIDAR_USEPOLLING=1
VITE_DEV_API_PROXY_TARGET=http://app:8000
MIGRATE_ON_START=1
SESSION_SECRET_KEY=replace-with-a-long-random-secret
SESSION_COOKIE_SECURE=0
SESSION_SECRET_KEY=replace-with-a-long-random-secret-at-least-32-characters
SESSION_COOKIE_SECURE=1
LOGIN_RATE_LIMIT_ATTEMPTS=5
LOGIN_RATE_LIMIT_WINDOW_SECONDS=60
LOG_LEVEL=INFO
@ -35,6 +40,16 @@ INGESTION_CONTINUATION_BASE_DELAY_SECONDS=120
INGESTION_CONTINUATION_MAX_DELAY_SECONDS=3600
INGESTION_CONTINUATION_MAX_ATTEMPTS=6
SCHEDULER_QUEUE_BATCH_SIZE=10
SCHOLAR_IMAGE_UPLOAD_DIR=/var/lib/scholarr/uploads
SCHOLAR_IMAGE_UPLOAD_MAX_BYTES=2000000
SCHOLAR_NAME_SEARCH_ENABLED=1
SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS=21600
SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS=300
SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES=512
SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS=8.0
SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS=2.0
SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD=1
SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS=1800
BOOTSTRAP_ADMIN_ON_START=0
BOOTSTRAP_ADMIN_EMAIL=

82
.github/workflows/ci.yml vendored
View file

@ -51,3 +51,85 @@ jobs:
- name: Integration tests
run: uv run pytest -m integration
frontend-quality:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: API contract drift check
run: python3 scripts/check_frontend_api_contract.py
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Install frontend dependencies
working-directory: frontend
run: npm install
- name: Frontend typecheck
working-directory: frontend
run: npm run typecheck
- name: Frontend unit tests
working-directory: frontend
run: npm run test:run
- name: Frontend build
working-directory: frontend
run: npm run build
docker-publish:
runs-on: ubuntu-latest
needs:
- test
- frontend-quality
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Derive image metadata
id: meta
uses: docker/metadata-action@v5
with:
images: justinzeus/scholarr
tags: |
type=raw,value=latest
type=sha,format=short,prefix=sha-
- name: Build and push multi-arch image
uses: docker/build-push-action@v6
with:
context: .
file: ./Dockerfile
target: prod
push: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max

3
.gitignore vendored
View file

@ -10,3 +10,6 @@ htmlcov/
.uv-cache/
.env
*.log
planning/
frontend/node_modules/
frontend/dist/

104
AGENTS.MD
View file

@ -2,8 +2,6 @@
This file is the consolidated source of truth for project scope, constraints, scrape contract, and UI implementation flow.
Raw probe artifacts in `planning/scholar_probe_tmp/notes/*.json` and `planning/scholar_probe_tmp/notes/*.md` remain historical evidence and fixture references.
## 1. Product Objective
- Build a self-hosted scholar tracking system ("scholarr") with reliable, low-cost scraping and clear, actionable UI.
@ -94,7 +92,12 @@ Admin users:
Scholars:
- `GET /api/v1/scholars`
- `POST /api/v1/scholars`
- `GET /api/v1/scholars/search`
- `PATCH /api/v1/scholars/{id}/toggle`
- `PUT /api/v1/scholars/{id}/image/url`
- `POST /api/v1/scholars/{id}/image/upload`
- `GET /api/v1/scholars/{id}/image/upload`
- `DELETE /api/v1/scholars/{id}/image`
- `DELETE /api/v1/scholars/{id}`
Settings:
@ -112,11 +115,42 @@ Runs/diagnostics/queue:
Publications:
- `GET /api/v1/publications`
- `POST /api/v1/publications/mark-read`
- `POST /api/v1/publications/mark-all-read`
Ops:
- `GET /healthz`
## 5.1 Runtime Semantics (Locked)
Manual run idempotency:
- `Idempotency-Key` is optional on `POST /api/v1/runs/manual`.
- Idempotency scope is `(user_id, idempotency_key)`.
- Replaying the same key for the same user returns the same run payload after completion.
- If the keyed run is still in progress, respond with `409 run_in_progress` and include `run_id`.
- Enforcement is database-backed, not best-effort in-memory logic.
- Frontend triggers generate keys internally; key input is not user-facing.
Continuation queue attempt policy:
- `attempt_count` is a failed-attempt budget, not a total execution counter.
- Increment attempts only when a continuation execution fails to make progress.
- Reset attempts to `0` after successful progress (success or resumable partial progress).
- Drop queue items only after failed-attempt threshold is reached.
Scholar no-change skip policy:
- Persist first-page fingerprint snapshots per scholar.
- If the first-page fingerprint is unchanged for a normal run (`cstart=0`), skip deep pagination/upsert work.
- Record run scholar result `state_reason` as `no_change_initial_page_signature`.
Scholar create naming policy:
- Do not accept custom display names during scholar create.
- Hydrate names from scraped profile metadata.
Scholar name-search safety policy:
- Treat name search as best-effort only.
- Apply single-flight execution, pacing/jitter, cache, and cooldown circuit breaker after repeated blocked responses.
- Prefer Scholar URL/ID adds for reliable ingestion onboarding.
## 6. Required UI Behavior (MVP)
Core UX entities:
@ -132,6 +166,7 @@ Critical semantics:
- First-time ingestion should not imply user read-state.
- Surface partial runs and warnings clearly.
- Show loading, empty, and error states explicitly.
- Support both light and dark modes with consistent status semantics.
## 7. UI Information Architecture
@ -160,19 +195,48 @@ Authenticated:
3. Logout calls `POST /api/v1/auth/logout`.
## 8.3 Dashboard
1. Fetch latest publications summary from `GET /api/v1/publications?mode=new&limit=20`.
2. Fetch recent runs from `GET /api/v1/runs?limit=5`.
3. Fetch queue status from `GET /api/v1/runs/queue/items?limit=200` (aggregate in UI).
4. Show:
- new publications count (`new_count`)
- total tracked publications (`total_count`)
- latest run status/started time
- queue badges (`queued`, `retrying`, `dropped`)
5. Admin users additionally see a diagnostics shortcut block to full runs/queue page.
6. Every dashboard error panel must display request-id for support.
## 8.4 Scholars
1. List via `GET /api/v1/scholars`.
2. Add/toggle/delete actions mapped to scholar endpoints.
3. Per-row quick links:
3. Search by name via `GET /api/v1/scholars/search` with candidate add actions.
4. Profile image controls:
- scraped image fallback from Scholar metadata
- custom URL override
- image upload + reset to scraped/default
5. Per-row quick links:
- all publications filtered by scholar
- new publications filtered by scholar
## 8.5 Publications
1. Default view mode is `new` (`GET /api/v1/publications?mode=new`).
2. Support mode toggle:
- `new`: scoped to latest completed run
- `all`: full history
3. Support scholar filter via `scholar_profile_id` query param.
4. Render both badges independently:
- `is_new_in_latest_run`
- `is_read`
5. "Mark all read" action calls `POST /api/v1/publications/mark-all-read` with CSRF.
6. Show explicit empty states:
- no publications tracked
- no new publications in latest run
- filtered scholar has no items
7. Surface run recency context when mode is `new` (latest completed run semantics).
## 8.6 Runs and diagnostics (admin)
1. List runs from `GET /api/v1/runs`.
@ -215,6 +279,36 @@ Authenticated:
## 11. Open Items (Known Future Work)
- Scholar discovery by real name (search Google Scholar candidates, select one, then add scholar ID).
- API contract freeze/changelog discipline for external UI clients.
- Additional API-first smoke coverage for end-to-end UI critical flows.
- Optional background image enrichment/refresh jobs for already-tracked scholars.
## 12. UI Rebuild Program Artifacts
- Phase 0 through Phase 3 decisions are consolidated into this handbook and `README.md`.
- Local scratch notes, probes, and temporary planning artifacts are intentionally not tracked in git.
## 13. Frontend Scaffold Status (Implemented)
- Frontend scaffold path: `frontend/`
- Implemented core:
- App shell + routing + auth/role route guards
- Shared API client with envelope parsing, credentials, and CSRF header injection
- Pinia stores for auth, theme, and global UI errors
- Tailwind-first design foundation with light/dark themes and pre-paint theme initialization
- Page shells for login, dashboard, scholars, publications, runs, run detail, settings, and admin users
- Initial feature modules under `frontend/src/features/*` mapped to current API endpoints
- Deployment boundary:
- No in-repo reverse-proxy implementation
- Frontend edge/proxy routing is managed externally (for example, Caddy)
## 14. Frontend Quality Gates (Phase 3 Implemented)
- CI now includes frontend quality gates in `.github/workflows/ci.yml`:
- API contract drift check (`scripts/check_frontend_api_contract.py`)
- frontend typecheck (`npm run typecheck`)
- frontend unit tests (`npm run test:run`)
- frontend build (`npm run build`)
- Frontend test harness:
- `vitest` configured in `frontend/vitest.config.ts`
- baseline tests in `frontend/src/lib/api/envelope.test.ts` and `frontend/src/lib/api/errors.test.ts`

15
Dockerfile
View file

@ -1,3 +1,13 @@
FROM node:20-alpine AS frontend-builder
WORKDIR /frontend
COPY frontend/package.json frontend/package-lock.json ./
RUN npm ci
COPY frontend/ ./
RUN npm run build
FROM python:3.12-slim AS base
ENV PYTHONDONTWRITEBYTECODE=1 \
@ -33,7 +43,9 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
UV_PROJECT_ENVIRONMENT=/opt/venv \
PATH="/opt/venv/bin:$PATH" \
APP_RELOAD=0 \
UVICORN_WORKERS=1
UVICORN_WORKERS=1 \
FRONTEND_ENABLED=1 \
FRONTEND_DIST_DIR=/app/frontend/dist
WORKDIR /app
@ -51,4 +63,5 @@ COPY alembic ./alembic
RUN uv sync --frozen
COPY scripts ./scripts
COPY --from=frontend-builder /frontend/dist /app/frontend/dist
ENTRYPOINT ["/bin/sh", "/app/scripts/entrypoint.sh"]

262
README.md
View file

@ -1,30 +1,131 @@
# scholarr
API-first, self-hosted scholar tracking backend in the spirit of the `*arr` ecosystem.
<div align="center">
The legacy server-rendered UI has been removed. This repository now focuses on core ingestion, scheduling, and multi-user API functionality.
API-first, self-hosted scholar tracking in the spirit of the `*arr` ecosystem.
## Current Scope
[![CI](https://img.shields.io/github/actions/workflow/status/justinzeus/scholarr/ci.yml?style=for-the-badge)](https://github.com/justinzeus/scholarr/actions/workflows/ci.yml)
[![Docker Pulls](https://img.shields.io/docker/pulls/justinzeus/scholarr?style=for-the-badge&logo=docker)](https://hub.docker.com/r/justinzeus/scholarr)
[![Docker Image](https://img.shields.io/badge/docker-justinzeus%2Fscholarr-2496ED?style=for-the-badge&logo=docker&logoColor=white)](https://hub.docker.com/r/justinzeus/scholarr)
- Multi-user accounts with admin-managed user lifecycle
- Same-origin cookie sessions with CSRF enforcement
- API auth flows (login, logout, me, password change)
- Admin user management API
- Scholar CRUD per user
- Per-user ingestion settings
- Manual runs with idempotency support
- Run history, run detail diagnostics, and continuation queue actions
- Publication listing (`new` / `all`) and mark-all-read
- Ingestion scheduler + continuation queue retries
- Structured logging with request IDs + redaction
- PostgreSQL + Alembic migrations
- Container-first development workflow with `uv`
</div>
## Functionality Tracking
## What This Includes
Planned and supported backend functionality is tracked in:
- Multi-user accounts with admin-managed lifecycle
- Cookie session auth with CSRF enforcement
- Scholar CRUD + name-search discovery + profile image management
- Per-user ingestion settings and scheduler
- Manual runs with idempotency behavior
- Run history and continuation queue diagnostics/actions
- Publications workflow (`new` / `all`, mark-selected-read, mark-all-read)
- Vue 3 + Vite frontend served from the same container image as the API
- `AGENTS.MD`
## Docker Image
- Image: `justinzeus/scholarr`
- Published by GitHub Actions on every push to `main`
- Architectures: `linux/amd64`, `linux/arm64`
- Tags:
- `latest`
- `sha-<short_commit>`
## Quick Deploy (Copy Compose + Fill Env)
1. Copy `docker-compose.yml` and `.env.example` into your deployment directory.
2. Create `.env` from `.env.example`.
3. Set at minimum:
- `SESSION_SECRET_KEY`
- `POSTGRES_PASSWORD`
4. Pull and start:
```bash
docker compose pull
docker compose up -d
```
5. Open:
- App + API: `http://localhost:8000`
- Health check: `http://localhost:8000/healthz`
The SPA and API are same-origin by default in this deployment model.
## Required Environment Variables
| Variable | Required | Purpose |
| --- | --- | --- |
| `SESSION_SECRET_KEY` | Yes | Session signing key. Use a long random value. |
| `POSTGRES_PASSWORD` | Yes | Database password for the bundled Postgres service. |
Optional startup bootstrap:
| Variable | Default | Purpose |
| --- | --- | --- |
| `BOOTSTRAP_ADMIN_ON_START` | `0` | Auto-create admin on app start. |
| `BOOTSTRAP_ADMIN_EMAIL` | empty | Admin email for bootstrap. |
| `BOOTSTRAP_ADMIN_PASSWORD` | empty | Admin password for bootstrap. |
| `BOOTSTRAP_ADMIN_FORCE_PASSWORD` | `0` | Force-reset bootstrap admin password if exists. |
## Ports
| Port | Service | Description |
| --- | --- | --- |
| `8000` | `app` | scholarr API + frontend |
## Volumes
| Volume | Container Path | Purpose |
| --- | --- | --- |
| `postgres_data` | `/var/lib/postgresql/data` | Postgres persistence |
| `scholar_uploads` | `/var/lib/scholarr/uploads` | Scholar image upload persistence |
## Upgrade
```bash
docker compose pull
docker compose up -d
```
## Development Workflow
Default `docker-compose.yml` is deployment-oriented (prebuilt image).
For local development with source mounts + Vite dev server:
```bash
docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
```
Then open:
- API: `http://localhost:8000`
- Frontend dev server: `http://localhost:5173`
## Test and Quality Commands
Backend:
```bash
docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest tests/unit
docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest -m integration
```
Frontend:
```bash
cd frontend
npm install
npm run typecheck
npm run test:run
npm run build
```
Contract drift check:
```bash
python3 scripts/check_frontend_api_contract.py
```
## API Base
@ -34,122 +135,13 @@ Planned and supported backend functionality is tracked in:
- Error envelope:
- `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}`
## Auth & Session Model
- Session transport: same-origin cookie session (`HttpOnly`, `SameSite=Lax`)
- CSRF:
- Required for unsafe methods (`POST`, `PUT`, `PATCH`, `DELETE`) via `X-CSRF-Token`
- Bootstrap token via `GET /api/v1/auth/csrf`
## API Surface
- Auth:
- `GET /api/v1/auth/csrf`
- `POST /api/v1/auth/login`
- `GET /api/v1/auth/me`
- `POST /api/v1/auth/change-password`
- `POST /api/v1/auth/logout`
- Admin users:
- `GET /api/v1/admin/users`
- `POST /api/v1/admin/users`
- `PATCH /api/v1/admin/users/{id}/active`
- `POST /api/v1/admin/users/{id}/reset-password`
- Scholars:
- `GET /api/v1/scholars`
- `POST /api/v1/scholars`
- `PATCH /api/v1/scholars/{id}/toggle`
- `DELETE /api/v1/scholars/{id}`
- Settings:
- `GET /api/v1/settings`
- `PUT /api/v1/settings`
- Runs:
- `GET /api/v1/runs`
- `GET /api/v1/runs/{id}`
- `POST /api/v1/runs/manual` (`Idempotency-Key` supported)
- `GET /api/v1/runs/queue/items`
- `POST /api/v1/runs/queue/{id}/retry`
- `POST /api/v1/runs/queue/{id}/drop`
- `DELETE /api/v1/runs/queue/{id}`
- Publications:
- `GET /api/v1/publications`
- `POST /api/v1/publications/mark-all-read`
- Ops:
- `GET /healthz`
## Quick Start
1. Copy environment defaults:
```bash
cp .env.example .env
```
2. Optional bootstrap admin on startup:
```bash
export BOOTSTRAP_ADMIN_ON_START=1
export BOOTSTRAP_ADMIN_EMAIL=admin@example.com
export BOOTSTRAP_ADMIN_PASSWORD=change-me-now
```
3. Start stack:
```bash
docker compose up --build
```
4. Health check:
```text
http://localhost:8000/healthz
```
## Admin Bootstrap (Manual)
```bash
docker compose run --rm app uv run python scripts/bootstrap_admin.py \
--email admin@example.com \
--password change-me-now \
--force-password
```
## Test Workflow
- Unit tests:
```bash
docker compose run --rm app uv run pytest tests/unit
```
- Integration tests:
```bash
docker compose run --rm app uv run pytest -m integration
```
- Smoke checks:
```bash
./scripts/smoke_compose.sh
```
## Logging
Configurable env vars:
Important envs:
- `LOG_LEVEL` (default `INFO`)
- `LOG_FORMAT` (`console` or `json`, default `console`)
- `LOG_REQUESTS` (`1`/`0`, default `1`)
- `LOG_UVICORN_ACCESS` (`1`/`0`, default `0`)
- `LOG_REQUEST_SKIP_PATHS` (default `/healthz`)
- `LOG_REDACT_FIELDS` (additional redact keys)
## Project Layout
```text
app/ FastAPI app (API routers, auth, services, db, middleware)
alembic/ Migration environment and versions
scripts/ Entrypoint, db wait/bootstrap, smoke automation
tests/ Unit, integration, and smoke suites
planning/ Scope and implementation planning notes
```
- `LOG_LEVEL`
- `LOG_FORMAT` (`console` or `json`)
- `LOG_REQUESTS`
- `LOG_UVICORN_ACCESS`
- `LOG_REQUEST_SKIP_PATHS`
- `LOG_REDACT_FIELDS`

88
alembic/versions/20260217_0005_manual_run_idempotency.py Normal file
View file

@ -0,0 +1,88 @@
"""Add database-backed manual run idempotency key
Revision ID: 20260217_0005
Revises: 20260217_0004
Create Date: 2026-02-17 16:20:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "20260217_0005"
down_revision: str | Sequence[str] | None = "20260217_0004"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
INDEX_NAME = "uq_crawl_runs_user_manual_idempotency_key"
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns("crawl_runs")}
indexes = {index["name"] for index in inspector.get_indexes("crawl_runs")}
if "idempotency_key" not in columns:
op.add_column(
"crawl_runs",
sa.Column("idempotency_key", sa.String(length=128), nullable=True),
)
columns.add("idempotency_key")
if "idempotency_key" in columns:
op.execute(
"""
UPDATE crawl_runs
SET idempotency_key = NULLIF(BTRIM(error_log #>> '{meta,idempotency_key}'), '')
WHERE trigger_type = 'manual'
AND idempotency_key IS NULL
"""
)
# Preserve one winning row per (user_id, idempotency_key) before creating uniqueness.
op.execute(
"""
WITH ranked AS (
SELECT
id,
ROW_NUMBER() OVER (
PARTITION BY user_id, idempotency_key
ORDER BY start_dt DESC, id DESC
) AS rn
FROM crawl_runs
WHERE trigger_type = 'manual'
AND idempotency_key IS NOT NULL
)
UPDATE crawl_runs AS runs
SET idempotency_key = NULL
FROM ranked
WHERE runs.id = ranked.id
AND ranked.rn > 1
"""
)
if INDEX_NAME not in indexes:
op.create_index(
INDEX_NAME,
"crawl_runs",
["user_id", "idempotency_key"],
unique=True,
postgresql_where=sa.text(
"idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"
),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns("crawl_runs")}
indexes = {index["name"] for index in inspector.get_indexes("crawl_runs")}
if INDEX_NAME in indexes:
op.drop_index(INDEX_NAME, table_name="crawl_runs")
if "idempotency_key" in columns:
op.drop_column("crawl_runs", "idempotency_key")

59
alembic/versions/20260217_0006_scholar_profile_images.py Normal file
View file

@ -0,0 +1,59 @@
"""Add scholar profile image metadata and overrides.
Revision ID: 20260217_0006
Revises: 20260217_0005
Create Date: 2026-02-17 18:30:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "20260217_0006"
down_revision: str | Sequence[str] | None = "20260217_0005"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
TABLE_NAME = "scholar_profiles"
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns(TABLE_NAME)}
if "profile_image_url" not in columns:
op.add_column(
TABLE_NAME,
sa.Column("profile_image_url", sa.Text(), nullable=True),
)
if "profile_image_override_url" not in columns:
op.add_column(
TABLE_NAME,
sa.Column("profile_image_override_url", sa.Text(), nullable=True),
)
if "profile_image_upload_path" not in columns:
op.add_column(
TABLE_NAME,
sa.Column("profile_image_upload_path", sa.Text(), nullable=True),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns(TABLE_NAME)}
if "profile_image_upload_path" in columns:
op.drop_column(TABLE_NAME, "profile_image_upload_path")
if "profile_image_override_url" in columns:
op.drop_column(TABLE_NAME, "profile_image_override_url")
if "profile_image_url" in columns:
op.drop_column(TABLE_NAME, "profile_image_url")

50
alembic/versions/20260217_0007_scholar_initial_page_fingerprint.py Normal file
View file

@ -0,0 +1,50 @@
"""Add per-scholar initial page fingerprint snapshot columns.
Revision ID: 20260217_0007
Revises: 20260217_0006
Create Date: 2026-02-17 18:55:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "20260217_0007"
down_revision: str | Sequence[str] | None = "20260217_0006"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
TABLE_NAME = "scholar_profiles"
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns(TABLE_NAME)}
if "last_initial_page_fingerprint_sha256" not in columns:
op.add_column(
TABLE_NAME,
sa.Column("last_initial_page_fingerprint_sha256", sa.String(length=64), nullable=True),
)
if "last_initial_page_checked_at" not in columns:
op.add_column(
TABLE_NAME,
sa.Column("last_initial_page_checked_at", sa.DateTime(timezone=True), nullable=True),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
columns = {column["name"] for column in inspector.get_columns(TABLE_NAME)}
if "last_initial_page_checked_at" in columns:
op.drop_column(TABLE_NAME, "last_initial_page_checked_at")
if "last_initial_page_fingerprint_sha256" in columns:
op.drop_column(TABLE_NAME, "last_initial_page_fingerprint_sha256")

47
app/api/routers/publications.py
View file

@ -9,7 +9,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_current_user
from app.api.errors import ApiException
from app.api.responses import success_payload
from app.api.schemas import MarkAllReadEnvelope, PublicationsListEnvelope
from app.api.schemas import (
MarkAllReadEnvelope,
MarkSelectedReadEnvelope,
MarkSelectedReadRequest,
PublicationsListEnvelope,
)
from app.db.models import User
from app.db.session import get_db_session
from app.services import publications as publication_service
@ -121,3 +126,43 @@ async def mark_all_publications_read(
"updated_count": updated_count,
},
)
@router.post(
"/mark-read",
response_model=MarkSelectedReadEnvelope,
)
async def mark_selected_publications_read(
payload: MarkSelectedReadRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
selection_pairs = sorted(
{
(int(item.scholar_profile_id), int(item.publication_id))
for item in payload.selections
}
)
updated_count = await publication_service.mark_selected_as_read_for_user(
db_session,
user_id=current_user.id,
selections=selection_pairs,
)
logger.info(
"api.publications.mark_selected_read",
extra={
"event": "api.publications.mark_selected_read",
"user_id": current_user.id,
"requested_count": len(selection_pairs),
"updated_count": updated_count,
},
)
return success_payload(
request,
data={
"message": "Marked selected publications as read.",
"requested_count": len(selection_pairs),
"updated_count": updated_count,
},
)

49
app/api/routers/runs.py
View file

@ -5,6 +5,7 @@ import re
from typing import Any
from fastapi import APIRouter, Depends, Query, Request
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_current_user
@ -380,6 +381,7 @@ async def run_manual(
page_size=settings.ingestion_page_size,
auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
idempotency_key=idempotency_key,
)
except ingestion_service.RunAlreadyInProgressError as exc:
await db_session.rollback()
@ -388,6 +390,45 @@ async def run_manual(
code="run_in_progress",
message="A run is already in progress for this account.",
) from exc
except IntegrityError as exc:
await db_session.rollback()
if idempotency_key is not None:
existing_run = await run_service.get_manual_run_by_idempotency_key(
db_session,
user_id=current_user.id,
idempotency_key=idempotency_key,
)
if existing_run is not None:
if existing_run.status == RunStatus.RUNNING:
raise ApiException(
status_code=409,
code="run_in_progress",
message="A run with this idempotency key is still in progress.",
details={
"run_id": int(existing_run.id),
"idempotency_key": idempotency_key,
},
) from exc
return success_payload(
request,
data=_manual_run_payload_from_run(
existing_run,
idempotency_key=idempotency_key,
reused_existing_run=True,
),
)
logger.exception(
"api.runs.manual_integrity_error",
extra={
"event": "api.runs.manual_integrity_error",
"user_id": current_user.id,
},
)
raise ApiException(
status_code=500,
code="manual_run_failed",
message="Manual run failed.",
) from exc
except Exception as exc:
await db_session.rollback()
logger.exception(
@ -403,14 +444,6 @@ async def run_manual(
message="Manual run failed.",
) from exc
if idempotency_key is not None:
await run_service.set_manual_run_idempotency_key(
db_session,
user_id=current_user.id,
run_id=run_summary.crawl_run_id,
idempotency_key=idempotency_key,
)
return success_payload(
request,
data={

326
app/api/routers/scholars.py
View file

@ -1,33 +1,56 @@
from __future__ import annotations
import asyncio
import logging
import mimetypes
from fastapi import APIRouter, Depends, Request
from fastapi import APIRouter, Depends, File, Query, Request, UploadFile
from fastapi.responses import FileResponse
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_current_user
from app.api.errors import ApiException
from app.api.responses import success_payload
from app.api.runtime_deps import get_scholar_source
from app.api.schemas import (
MessageEnvelope,
ScholarCreateRequest,
ScholarEnvelope,
ScholarImageUrlUpdateRequest,
ScholarSearchEnvelope,
ScholarsListEnvelope,
)
from app.db.models import User
from app.db.session import get_db_session
from app.services import scholars as scholar_service
from app.services.scholar_source import ScholarSource
from app.settings import settings
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/scholars", tags=["api-scholars"])
def _uploaded_image_api_path(scholar_profile_id: int) -> str:
return f"/api/v1/scholars/{scholar_profile_id}/image/upload"
def _serialize_scholar(profile) -> dict[str, object]:
uploaded_image_url = None
if profile.profile_image_upload_path:
uploaded_image_url = _uploaded_image_api_path(int(profile.id))
profile_image_url, profile_image_source = scholar_service.resolve_profile_image(
profile,
uploaded_image_url=uploaded_image_url,
)
return {
"id": int(profile.id),
"scholar_id": profile.scholar_id,
"display_name": profile.display_name,
"profile_image_url": profile_image_url,
"profile_image_source": profile_image_source,
"is_enabled": bool(profile.is_enabled),
"baseline_completed": bool(profile.baseline_completed),
"last_run_dt": profile.last_run_dt,
@ -67,6 +90,7 @@ async def create_scholar(
payload: ScholarCreateRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
source: ScholarSource = Depends(get_scholar_source),
current_user: User = Depends(get_api_current_user),
):
try:
@ -74,7 +98,8 @@ async def create_scholar(
db_session,
user_id=current_user.id,
scholar_id=payload.scholar_id,
display_name=payload.display_name or "",
display_name="",
profile_image_url=payload.profile_image_url,
)
except scholar_service.ScholarServiceError as exc:
raise ApiException(
@ -90,12 +115,104 @@ async def create_scholar(
"scholar_profile_id": created.id,
},
)
try:
if not created.profile_image_url or not (created.display_name or "").strip():
created = await asyncio.wait_for(
scholar_service.hydrate_profile_metadata(
db_session,
profile=created,
source=source,
),
timeout=5.0,
)
except Exception:
logger.warning(
"api.scholars.create_metadata_hydration_failed",
extra={
"event": "api.scholars.create_metadata_hydration_failed",
"user_id": current_user.id,
"scholar_profile_id": created.id,
},
)
return success_payload(
request,
data=_serialize_scholar(created),
)
@router.get(
"/search",
response_model=ScholarSearchEnvelope,
)
async def search_scholars(
request: Request,
query: str = Query(..., min_length=2, max_length=120),
limit: int = Query(10, ge=1, le=25),
source: ScholarSource = Depends(get_scholar_source),
current_user: User = Depends(get_api_current_user),
):
try:
parsed = await scholar_service.search_author_candidates(
source=source,
query=query,
limit=limit,
network_error_retries=settings.ingestion_network_error_retries,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
search_enabled=settings.scholar_name_search_enabled,
cache_ttl_seconds=settings.scholar_name_search_cache_ttl_seconds,
blocked_cache_ttl_seconds=settings.scholar_name_search_blocked_cache_ttl_seconds,
cache_max_entries=settings.scholar_name_search_cache_max_entries,
min_interval_seconds=settings.scholar_name_search_min_interval_seconds,
interval_jitter_seconds=settings.scholar_name_search_interval_jitter_seconds,
cooldown_block_threshold=settings.scholar_name_search_cooldown_block_threshold,
cooldown_seconds=settings.scholar_name_search_cooldown_seconds,
)
except scholar_service.ScholarServiceError as exc:
raise ApiException(
status_code=400,
code="invalid_scholar_search",
message=str(exc),
) from exc
logger.info(
"api.scholars.search.completed",
extra={
"event": "api.scholars.search.completed",
"user_id": current_user.id,
"query": query.strip(),
"candidate_count": len(parsed.candidates),
"state": parsed.state.value,
},
)
return success_payload(
request,
data={
"query": query.strip(),
"state": parsed.state.value,
"state_reason": parsed.state_reason,
"action_hint": scholar_service.scrape_state_hint(
state=parsed.state,
state_reason=parsed.state_reason,
),
"candidates": [
{
"scholar_id": item.scholar_id,
"display_name": item.display_name,
"affiliation": item.affiliation,
"email_domain": item.email_domain,
"cited_by_count": item.cited_by_count,
"interests": item.interests,
"profile_url": item.profile_url,
"profile_image_url": item.profile_image_url,
}
for item in parsed.candidates
],
"warnings": parsed.warnings,
},
)
@router.patch(
"/{scholar_profile_id}/toggle",
response_model=ScholarEnvelope,
@ -154,7 +271,11 @@ async def delete_scholar(
code="scholar_not_found",
message="Scholar not found.",
)
await scholar_service.delete_scholar(db_session, profile=profile)
await scholar_service.delete_scholar(
db_session,
profile=profile,
upload_dir=settings.scholar_image_upload_dir,
)
logger.info(
"api.scholars.deleted",
extra={
@ -167,3 +288,202 @@ async def delete_scholar(
request,
data={"message": "Scholar deleted."},
)
@router.put(
"/{scholar_profile_id}/image/url",
response_model=ScholarEnvelope,
)
async def update_scholar_image_url(
scholar_profile_id: int,
payload: ScholarImageUrlUpdateRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
profile = await scholar_service.get_user_scholar_by_id(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
)
if profile is None:
raise ApiException(
status_code=404,
code="scholar_not_found",
message="Scholar not found.",
)
try:
updated = await scholar_service.set_profile_image_override_url(
db_session,
profile=profile,
image_url=payload.image_url,
upload_dir=settings.scholar_image_upload_dir,
)
except scholar_service.ScholarServiceError as exc:
raise ApiException(
status_code=400,
code="invalid_scholar_image",
message=str(exc),
) from exc
logger.info(
"api.scholars.image_url_updated",
extra={
"event": "api.scholars.image_url_updated",
"user_id": current_user.id,
"scholar_profile_id": updated.id,
},
)
return success_payload(
request,
data=_serialize_scholar(updated),
)
@router.post(
"/{scholar_profile_id}/image/upload",
response_model=ScholarEnvelope,
)
async def upload_scholar_image(
scholar_profile_id: int,
request: Request,
image: UploadFile = File(...),
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
profile = await scholar_service.get_user_scholar_by_id(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
)
if profile is None:
raise ApiException(
status_code=404,
code="scholar_not_found",
message="Scholar not found.",
)
try:
image_bytes = await image.read()
updated = await scholar_service.set_profile_image_upload(
db_session,
profile=profile,
content_type=image.content_type,
image_bytes=image_bytes,
upload_dir=settings.scholar_image_upload_dir,
max_upload_bytes=settings.scholar_image_upload_max_bytes,
)
except scholar_service.ScholarServiceError as exc:
raise ApiException(
status_code=400,
code="invalid_scholar_image",
message=str(exc),
) from exc
finally:
await image.close()
logger.info(
"api.scholars.image_uploaded",
extra={
"event": "api.scholars.image_uploaded",
"user_id": current_user.id,
"scholar_profile_id": updated.id,
"content_type": image.content_type,
"size_bytes": len(image_bytes),
},
)
return success_payload(
request,
data=_serialize_scholar(updated),
)
@router.delete(
"/{scholar_profile_id}/image",
response_model=ScholarEnvelope,
)
async def clear_scholar_image_customization(
scholar_profile_id: int,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
profile = await scholar_service.get_user_scholar_by_id(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
)
if profile is None:
raise ApiException(
status_code=404,
code="scholar_not_found",
message="Scholar not found.",
)
updated = await scholar_service.clear_profile_image_customization(
db_session,
profile=profile,
upload_dir=settings.scholar_image_upload_dir,
)
logger.info(
"api.scholars.image_customization_cleared",
extra={
"event": "api.scholars.image_customization_cleared",
"user_id": current_user.id,
"scholar_profile_id": updated.id,
},
)
return success_payload(
request,
data=_serialize_scholar(updated),
)
@router.get(
"/{scholar_profile_id}/image/upload",
)
async def get_uploaded_scholar_image(
scholar_profile_id: int,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
profile = await scholar_service.get_user_scholar_by_id(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
)
if profile is None:
raise ApiException(
status_code=404,
code="scholar_not_found",
message="Scholar not found.",
)
if not profile.profile_image_upload_path:
raise ApiException(
status_code=404,
code="scholar_image_not_found",
message="Scholar image not found.",
)
try:
image_path = scholar_service.resolve_upload_file_path(
upload_dir=settings.scholar_image_upload_dir,
relative_path=profile.profile_image_upload_path,
)
except scholar_service.ScholarServiceError as exc:
raise ApiException(
status_code=404,
code="scholar_image_not_found",
message="Scholar image not found.",
) from exc
if not image_path.exists() or not image_path.is_file():
raise ApiException(
status_code=404,
code="scholar_image_not_found",
message="Scholar image not found.",
)
media_type = mimetypes.guess_type(str(image_path))[0] or "application/octet-stream"
return FileResponse(path=image_path, media_type=media_type)

69
app/api/schemas.py
View file

@ -112,6 +112,8 @@ class ScholarItemData(BaseModel):
id: int
scholar_id: str
display_name: str | None
profile_image_url: str | None
profile_image_source: str
is_enabled: bool
baseline_completed: bool
last_run_dt: datetime | None
@ -135,7 +137,7 @@ class ScholarsListEnvelope(BaseModel):
class ScholarCreateRequest(BaseModel):
scholar_id: str
display_name: str | None = None
profile_image_url: str | None = None
model_config = ConfigDict(extra="forbid")
@ -147,6 +149,43 @@ class ScholarEnvelope(BaseModel):
model_config = ConfigDict(extra="forbid")
class ScholarSearchCandidateData(BaseModel):
scholar_id: str
display_name: str
affiliation: str | None
email_domain: str | None
cited_by_count: int | None
interests: list[str] = Field(default_factory=list)
profile_url: str
profile_image_url: str | None
model_config = ConfigDict(extra="forbid")
class ScholarSearchData(BaseModel):
query: str
state: str
state_reason: str
action_hint: str | None = None
candidates: list[ScholarSearchCandidateData]
warnings: list[str] = Field(default_factory=list)
model_config = ConfigDict(extra="forbid")
class ScholarSearchEnvelope(BaseModel):
data: ScholarSearchData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class ScholarImageUrlUpdateRequest(BaseModel):
image_url: str
model_config = ConfigDict(extra="forbid")
class RunListItemData(BaseModel):
id: int
trigger_type: str
@ -454,3 +493,31 @@ class MarkAllReadEnvelope(BaseModel):
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class PublicationSelectionItem(BaseModel):
scholar_profile_id: int
publication_id: int
model_config = ConfigDict(extra="forbid")
class MarkSelectedReadRequest(BaseModel):
selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
model_config = ConfigDict(extra="forbid")
class MarkSelectedReadData(BaseModel):
message: str
requested_count: int
updated_count: int
model_config = ConfigDict(extra="forbid")
class MarkSelectedReadEnvelope(BaseModel):
data: MarkSelectedReadData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")

11
app/db/base.py
View file

@ -1,4 +1,6 @@
from sqlalchemy import MetaData
from datetime import datetime, timezone
from sqlalchemy import MetaData, event
from sqlalchemy.orm import DeclarativeBase
@ -15,4 +17,11 @@ class Base(DeclarativeBase):
metadata = MetaData(naming_convention=NAMING_CONVENTION)
@event.listens_for(Base, "before_update", propagate=True)
def _set_updated_at_before_update(_mapper, _connection, target) -> None:
# Keep audit timestamps current for ORM-managed updates.
if hasattr(target, "updated_at"):
target.updated_at = datetime.now(timezone.utc)
metadata = Base.metadata

15
app/db/models.py
View file

@ -119,6 +119,11 @@ class ScholarProfile(Base):
)
scholar_id: Mapped[str] = mapped_column(String(64), nullable=False)
display_name: Mapped[str | None] = mapped_column(String(255))
profile_image_url: Mapped[str | None] = mapped_column(Text)
profile_image_override_url: Mapped[str | None] = mapped_column(Text)
profile_image_upload_path: Mapped[str | None] = mapped_column(Text)
last_initial_page_fingerprint_sha256: Mapped[str | None] = mapped_column(String(64))
last_initial_page_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
is_enabled: Mapped[bool] = mapped_column(
Boolean, nullable=False, server_default=text("true")
)
@ -141,6 +146,15 @@ class CrawlRun(Base):
__tablename__ = "crawl_runs"
__table_args__ = (
Index("ix_crawl_runs_user_start", "user_id", "start_dt"),
Index(
"uq_crawl_runs_user_manual_idempotency_key",
"user_id",
"idempotency_key",
unique=True,
postgresql_where=text(
"idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"
),
),
)
id: Mapped[int] = mapped_column(primary_key=True)
@ -163,6 +177,7 @@ class CrawlRun(Base):
new_pub_count: Mapped[int] = mapped_column(
Integer, nullable=False, server_default=text("0")
)
idempotency_key: Mapped[str | None] = mapped_column(String(128))
error_log: Mapped[dict] = mapped_column(
JSONB, nullable=False, server_default=text("'{}'::jsonb")
)

42
app/main.py
View file

@ -1,8 +1,11 @@
from __future__ import annotations
from contextlib import asynccontextmanager
from pathlib import Path
from fastapi import FastAPI, HTTPException
from fastapi.responses import FileResponse
from fastapi.staticfiles import StaticFiles
from starlette.middleware.sessions import SessionMiddleware
from app.api.errors import register_api_exception_handlers
@ -68,3 +71,42 @@ async def healthz() -> dict[str, str]:
if await check_database():
return {"status": "ok"}
raise HTTPException(status_code=500, detail="database unavailable")
def _configure_frontend_routes(application: FastAPI) -> None:
if not settings.frontend_enabled:
return
dist_dir = Path(settings.frontend_dist_dir)
index_file = dist_dir / "index.html"
if not index_file.is_file():
return
assets_dir = dist_dir / "assets"
if assets_dir.is_dir():
application.mount("/assets", StaticFiles(directory=assets_dir), name="frontend-assets")
@application.get("/", include_in_schema=False)
async def frontend_index() -> FileResponse:
return FileResponse(index_file)
@application.get("/{full_path:path}", include_in_schema=False)
async def frontend_entry(full_path: str) -> FileResponse:
normalized = full_path.lstrip("/")
if normalized in {"healthz", "api"} or normalized.startswith("api/"):
raise HTTPException(status_code=404, detail="Not Found")
candidate = (dist_dir / normalized).resolve()
try:
candidate.relative_to(dist_dir.resolve())
except ValueError as error:
raise HTTPException(status_code=404, detail="Not Found") from error
if candidate.is_file():
return FileResponse(candidate)
return FileResponse(index_file)
_configure_frontend_routes(app)

17
app/services/continuation_queue.py
View file

@ -181,6 +181,23 @@ async def increment_attempt_count(
return item
async def reset_attempt_count(
db_session: AsyncSession,
*,
job_id: int,
) -> IngestionQueueItem | None:
now = datetime.now(timezone.utc)
result = await db_session.execute(
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
)
item = result.scalar_one_or_none()
if item is None:
return None
item.attempt_count = 0
item.updated_at = now
return item
async def reschedule_job(
db_session: AsyncSession,
*,

144
app/services/ingestion.py
View file

@ -4,6 +4,7 @@ import asyncio
from dataclasses import dataclass
from datetime import datetime, timezone
import hashlib
import json
import logging
import re
from typing import Any
@ -45,6 +46,7 @@ RESUMABLE_PARTIAL_REASONS = {
"pagination_cursor_stalled",
}
RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",)
INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS = 30
logger = logging.getLogger(__name__)
@ -63,6 +65,9 @@ class RunExecutionSummary:
class PagedParseResult:
fetch_result: FetchResult
parsed_page: ParsedProfilePage
first_page_fetch_result: FetchResult
first_page_parsed_page: ParsedProfilePage
first_page_fingerprint_sha256: str | None
publications: list[PublicationCandidate]
attempt_log: list[dict[str, Any]]
page_logs: list[dict[str, Any]]
@ -71,6 +76,7 @@ class PagedParseResult:
has_more_remaining: bool
pagination_truncated_reason: str | None
continuation_cstart: int | None
skipped_no_change: bool
class RunAlreadyInProgressError(RuntimeError):
@ -96,6 +102,7 @@ class ScholarIngestionService:
start_cstart_by_scholar_id: dict[int, int] | None = None,
auto_queue_continuations: bool = True,
queue_delay_seconds: int = 60,
idempotency_key: str | None = None,
) -> RunExecutionSummary:
lock_acquired = await self._try_acquire_user_lock(
db_session,
@ -153,6 +160,7 @@ class ScholarIngestionService:
"retry_backoff_seconds": retry_backoff_seconds,
"max_pages_per_scholar": max_pages_per_scholar,
"page_size": page_size,
"idempotency_key": idempotency_key,
},
)
@ -162,6 +170,7 @@ class ScholarIngestionService:
status=RunStatus.RUNNING,
scholar_count=len(scholars),
new_pub_count=0,
idempotency_key=idempotency_key,
error_log={},
)
db_session.add(run)
@ -187,6 +196,7 @@ class ScholarIngestionService:
retry_backoff_seconds=retry_backoff_seconds,
max_pages=max_pages_per_scholar,
page_size=page_size,
previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256,
)
fetch_result = paged_parse_result.fetch_result
parsed_page = paged_parse_result.parsed_page
@ -194,6 +204,17 @@ class ScholarIngestionService:
attempt_log = paged_parse_result.attempt_log
page_logs = paged_parse_result.page_logs
first_page = paged_parse_result.first_page_parsed_page
if first_page.profile_name and not (scholar.display_name or "").strip():
scholar.display_name = first_page.profile_name
if first_page.profile_image_url:
scholar.profile_image_url = first_page.profile_image_url
if paged_parse_result.first_page_fingerprint_sha256:
scholar.last_initial_page_fingerprint_sha256 = (
paged_parse_result.first_page_fingerprint_sha256
)
scholar.last_initial_page_checked_at = run_dt
logger.info(
"ingestion.scholar_parsed",
extra={
@ -210,6 +231,7 @@ class ScholarIngestionService:
"has_more_remaining": paged_parse_result.has_more_remaining,
"pagination_truncated_reason": paged_parse_result.pagination_truncated_reason,
"warning_count": len(parsed_page.warnings),
"skipped_no_change": paged_parse_result.skipped_no_change,
},
)
@ -230,7 +252,55 @@ class ScholarIngestionService:
"has_more_remaining": paged_parse_result.has_more_remaining,
"pagination_truncated_reason": paged_parse_result.pagination_truncated_reason,
"continuation_cstart": paged_parse_result.continuation_cstart,
"skipped_no_change": paged_parse_result.skipped_no_change,
"initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256,
}
if paged_parse_result.skipped_no_change:
scholar.last_run_status = RunStatus.SUCCESS
scholar.last_run_dt = run_dt
succeeded_count += 1
result_entry["state"] = first_page.state.value
result_entry["state_reason"] = "no_change_initial_page_signature"
result_entry["outcome"] = "success"
result_entry["publication_count"] = 0
result_entry["warnings"] = first_page.warnings
result_entry["debug"] = {
"state_reason": "no_change_initial_page_signature",
"first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256,
"attempt_log": attempt_log,
"page_logs": page_logs,
}
scholar_results.append(result_entry)
queue_reason, queue_cstart = self._resolve_continuation_queue_target(
outcome=str(result_entry.get("outcome", "")),
state=str(result_entry.get("state", "")),
pagination_truncated_reason=paged_parse_result.pagination_truncated_reason,
continuation_cstart=paged_parse_result.continuation_cstart,
fallback_cstart=start_cstart,
)
if auto_queue_continuations and queue_reason is not None:
await queue_service.upsert_job(
db_session,
user_id=user_id,
scholar_profile_id=scholar.id,
resume_cstart=queue_cstart,
reason=queue_reason,
run_id=run.id,
delay_seconds=queue_delay_seconds,
)
result_entry["continuation_enqueued"] = True
result_entry["continuation_reason"] = queue_reason
result_entry["continuation_cstart"] = queue_cstart
else:
cleared = await queue_service.clear_job_for_scholar(
db_session,
user_id=user_id,
scholar_profile_id=scholar.id,
)
if cleared:
result_entry["continuation_cleared"] = True
continue
had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}
has_partial_publication_set = len(publications) > 0 and had_page_failure
is_partial_due_to_pagination = (
@ -373,6 +443,11 @@ class ScholarIngestionService:
"failed_state_counts": failed_state_counts,
"failed_reason_counts": failed_reason_counts,
},
"meta": {
"idempotency_key": idempotency_key,
}
if idempotency_key
else {},
}
await db_session.commit()
@ -519,6 +594,7 @@ class ScholarIngestionService:
retry_backoff_seconds: float,
max_pages: int,
page_size: int,
previous_initial_page_fingerprint_sha256: str | None = None,
) -> PagedParseResult:
bounded_max_pages = max(1, int(max_pages))
bounded_page_size = max(1, int(page_size))
@ -534,6 +610,9 @@ class ScholarIngestionService:
network_error_retries=network_error_retries,
retry_backoff_seconds=retry_backoff_seconds,
)
first_page_fetch_result = fetch_result
first_page_parsed_page = parsed_page
first_page_fingerprint_sha256 = build_initial_page_fingerprint(parsed_page)
attempt_log: list[dict[str, Any]] = list(first_attempt_log)
page_logs: list[dict[str, Any]] = []
@ -553,11 +632,39 @@ class ScholarIngestionService:
)
pages_attempted = 1
should_skip_no_change = (
start_cstart <= 0
and first_page_fingerprint_sha256 is not None
and previous_initial_page_fingerprint_sha256 is not None
and first_page_fingerprint_sha256 == previous_initial_page_fingerprint_sha256
and parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS}
)
if should_skip_no_change:
return PagedParseResult(
fetch_result=fetch_result,
parsed_page=parsed_page,
first_page_fetch_result=first_page_fetch_result,
first_page_parsed_page=first_page_parsed_page,
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
publications=[],
attempt_log=attempt_log,
page_logs=page_logs,
pages_fetched=1,
pages_attempted=pages_attempted,
has_more_remaining=False,
pagination_truncated_reason=None,
continuation_cstart=None,
skipped_no_change=True,
)
# Immediate hard failure: nothing to salvage.
if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
return PagedParseResult(
fetch_result=fetch_result,
parsed_page=parsed_page,
first_page_fetch_result=first_page_fetch_result,
first_page_parsed_page=first_page_parsed_page,
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
publications=[],
attempt_log=attempt_log,
page_logs=page_logs,
@ -568,6 +675,7 @@ class ScholarIngestionService:
continuation_cstart=(
start_cstart if parsed_page.state == ParseState.NETWORK_ERROR else None
),
skipped_no_change=False,
)
publications = list(parsed_page.publications)
@ -649,6 +757,9 @@ class ScholarIngestionService:
return PagedParseResult(
fetch_result=fetch_result,
parsed_page=parsed_page,
first_page_fetch_result=first_page_fetch_result,
first_page_parsed_page=first_page_parsed_page,
first_page_fingerprint_sha256=first_page_fingerprint_sha256,
publications=_dedupe_publication_candidates(publications),
attempt_log=attempt_log,
page_logs=page_logs,
@ -657,6 +768,7 @@ class ScholarIngestionService:
has_more_remaining=has_more_remaining,
pagination_truncated_reason=pagination_truncated_reason,
continuation_cstart=continuation_cstart,
skipped_no_change=False,
)
async def _upsert_profile_publications(
@ -838,6 +950,7 @@ class ScholarIngestionService:
"fetch_error": fetch_result.error,
"state_reason": parsed_page.state_reason,
"profile_name": parsed_page.profile_name,
"profile_image_url": parsed_page.profile_image_url,
"articles_range": parsed_page.articles_range,
"has_show_more_button": parsed_page.has_show_more_button,
"has_operation_error_banner": parsed_page.has_operation_error_banner,
@ -914,6 +1027,37 @@ def build_publication_fingerprint(candidate: PublicationCandidate) -> str:
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def build_initial_page_fingerprint(parsed_page: ParsedProfilePage) -> str | None:
if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
return None
normalized_rows: list[dict[str, Any]] = []
for publication in parsed_page.publications[:INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS]:
normalized_rows.append(
{
"cluster_id": publication.cluster_id or "",
"title_normalized": normalize_title(publication.title),
"year": publication.year,
"citation_count": publication.citation_count,
}
)
payload = {
"state": parsed_page.state.value,
"articles_range": parsed_page.articles_range or "",
"has_show_more_button": parsed_page.has_show_more_button,
"profile_name": parsed_page.profile_name or "",
"publications": normalized_rows,
}
canonical = json.dumps(
payload,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def build_publication_url(path_or_url: str | None) -> str | None:
if not path_or_url:
return None

38
app/services/publications.py
View file

@ -3,7 +3,7 @@ from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from sqlalchemy import Select, func, select, update
from sqlalchemy import Select, func, select, tuple_, update
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import (
@ -295,3 +295,39 @@ async def mark_all_unread_as_read_for_user(
rowcount = result.rowcount
return int(rowcount or 0)
async def mark_selected_as_read_for_user(
db_session: AsyncSession,
*,
user_id: int,
selections: list[tuple[int, int]],
) -> int:
normalized_pairs = {
(int(scholar_profile_id), int(publication_id))
for scholar_profile_id, publication_id in selections
if int(scholar_profile_id) > 0 and int(publication_id) > 0
}
if not normalized_pairs:
return 0
scholar_ids = (
select(ScholarProfile.id)
.where(ScholarProfile.user_id == user_id)
.scalar_subquery()
)
stmt = (
update(ScholarPublication)
.where(
ScholarPublication.scholar_profile_id.in_(scholar_ids),
tuple_(
ScholarPublication.scholar_profile_id,
ScholarPublication.publication_id,
).in_(list(normalized_pairs)),
ScholarPublication.is_read.is_(False),
)
.values(is_read=True)
)
result = await db_session.execute(stmt)
await db_session.commit()
return int(result.rowcount or 0)

34
app/services/runs.py
View file

@ -4,7 +4,7 @@ from dataclasses import dataclass
from datetime import datetime
from typing import Any
from sqlalchemy import and_, case, func, select
from sqlalchemy import and_, case, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import (
@ -152,7 +152,10 @@ async def get_manual_run_by_idempotency_key(
.where(
CrawlRun.user_id == user_id,
CrawlRun.trigger_type == RunTriggerType.MANUAL,
CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key,
or_(
CrawlRun.idempotency_key == idempotency_key,
CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key,
),
)
.order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
.limit(1)
@ -160,33 +163,6 @@ async def get_manual_run_by_idempotency_key(
return result.scalar_one_or_none()
async def set_manual_run_idempotency_key(
db_session: AsyncSession,
*,
user_id: int,
run_id: int,
idempotency_key: str,
) -> bool:
run = await get_run_for_user(
db_session,
user_id=user_id,
run_id=run_id,
)
if run is None:
return False
if run.trigger_type != RunTriggerType.MANUAL:
return False
payload = dict(run.error_log) if isinstance(run.error_log, dict) else {}
meta = payload.get("meta")
meta_dict = dict(meta) if isinstance(meta, dict) else {}
meta_dict["idempotency_key"] = idempotency_key
payload["meta"] = meta_dict
run.error_log = payload
await db_session.commit()
return True
async def list_queue_items_for_user(
db_session: AsyncSession,
*,

37
app/services/scheduler.py
View file

@ -399,6 +399,39 @@ class SchedulerService:
return
async with session_factory() as session:
# Failed-attempt budget should advance only when continuation execution fails.
if int(run_summary.failed_count) <= 0:
queue_item = await queue_service.reset_attempt_count(
session,
job_id=job.id,
)
if queue_item is None:
await session.commit()
logger.info(
"scheduler.queue_item_resolved",
extra={
"event": "scheduler.queue_item_resolved",
"queue_item_id": job.id,
"user_id": job.user_id,
"run_id": run_summary.crawl_run_id,
"status": run_summary.status.value,
},
)
return
await session.commit()
logger.info(
"scheduler.queue_item_progressed",
extra={
"event": "scheduler.queue_item_progressed",
"queue_item_id": job.id,
"user_id": job.user_id,
"attempt_count": int(queue_item.attempt_count),
"run_id": run_summary.crawl_run_id,
"status": run_summary.status.value,
},
)
return
queue_item = await queue_service.increment_attempt_count(
session,
job_id=job.id,
@ -451,9 +484,9 @@ class SchedulerService:
)
await session.commit()
logger.info(
"scheduler.queue_item_rescheduled",
"scheduler.queue_item_rescheduled_failed",
extra={
"event": "scheduler.queue_item_rescheduled",
"event": "scheduler.queue_item_rescheduled_failed",
"queue_item_id": job.id,
"user_id": job.user_id,
"attempt_count": queue_item.attempt_count,

408
app/services/scholar_parser.py
View file

@ -6,7 +6,7 @@ from enum import StrEnum
from html import unescape
from html.parser import HTMLParser
from typing import Any
from urllib.parse import parse_qs, urlparse
from urllib.parse import parse_qs, urljoin, urlparse
from app.services.scholar_source import FetchResult
@ -16,6 +16,8 @@ BLOCKED_KEYWORDS = [
"not a robot",
"our systems have detected",
"automated queries",
"recaptcha",
"captcha",
]
NO_RESULTS_KEYWORDS = [
@ -25,6 +27,14 @@ NO_RESULTS_KEYWORDS = [
"no documents",
]
NO_AUTHOR_RESULTS_KEYWORDS = [
"didn't match any user profiles",
"did not match any user profiles",
"didn't match any scholars",
"did not match any scholars",
"no user profiles",
]
MARKER_KEYS = [
"gsc_a_tr",
"gsc_a_at",
@ -36,6 +46,33 @@ MARKER_KEYS = [
"gsc_rsb_st",
]
AUTHOR_SEARCH_MARKER_KEYS = [
"gsc_1usr",
"gs_ai_name",
"gs_ai_aff",
"gs_ai_eml",
"gs_ai_cby",
"gs_ai_one_int",
]
NETWORK_DNS_ERROR_KEYWORDS = [
"temporary failure in name resolution",
"name or service not known",
"nodename nor servname provided",
"getaddrinfo failed",
]
NETWORK_TIMEOUT_KEYWORDS = [
"timed out",
"timeout",
]
NETWORK_TLS_ERROR_KEYWORDS = [
"ssl",
"tls",
"certificate verify failed",
]
TAG_RE = re.compile(r"<[^>]+>", re.S)
SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.I | re.S)
SHOW_MORE_BUTTON_RE = re.compile(
@ -63,11 +100,24 @@ class PublicationCandidate:
venue_text: str | None
@dataclass(frozen=True)
class ScholarSearchCandidate:
scholar_id: str
display_name: str
affiliation: str | None
email_domain: str | None
cited_by_count: int | None
interests: list[str]
profile_url: str
profile_image_url: str | None
@dataclass(frozen=True)
class ParsedProfilePage:
state: ParseState
state_reason: str
profile_name: str | None
profile_image_url: str | None
publications: list[PublicationCandidate]
marker_counts: dict[str, int]
warnings: list[str]
@ -76,6 +126,15 @@ class ParsedProfilePage:
articles_range: str | None
@dataclass(frozen=True)
class ParsedAuthorSearchPage:
state: ParseState
state_reason: str
candidates: list[ScholarSearchCandidate]
marker_counts: dict[str, int]
warnings: list[str]
def normalize_space(value: str) -> str:
return " ".join(unescape(value).split())
@ -98,6 +157,19 @@ def attr_href(attrs: list[tuple[str, str | None]]) -> str | None:
return None
def attr_src(attrs: list[tuple[str, str | None]]) -> str | None:
for name, raw_value in attrs:
if name.lower() == "src":
return raw_value
return None
def build_absolute_scholar_url(path_or_url: str | None) -> str | None:
if not path_or_url:
return None
return urljoin("https://scholar.google.com", path_or_url)
class ScholarRowParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
@ -197,6 +269,18 @@ def parse_cluster_id_from_href(href: str | None) -> str | None:
return None
def parse_scholar_id_from_href(href: str | None) -> str | None:
if not href:
return None
parsed = urlparse(href)
query = parse_qs(parsed.query)
user_values = query.get("user")
if not user_values:
return None
candidate = user_values[0].strip()
return candidate or None
def parse_year(parts: list[str]) -> int | None:
text = normalize_space(" ".join(parts))
match = re.search(r"\b(19|20)\d{2}\b", text)
@ -265,6 +349,30 @@ def extract_profile_name(html: str) -> str | None:
return value or None
def extract_profile_image_url(html: str) -> str | None:
og_image_pattern = re.compile(
r"<meta[^>]+property=['\"]og:image['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>",
re.I | re.S,
)
og_match = og_image_pattern.search(html)
if og_match:
value = normalize_space(og_match.group(1))
absolute = build_absolute_scholar_url(value)
if absolute:
return absolute
image_pattern = re.compile(
r"<img[^>]*\bid=['\"]gsc_prf_pup-img['\"][^>]*\bsrc=['\"]([^'\"]+)['\"][^>]*>",
re.I | re.S,
)
image_match = image_pattern.search(html)
if not image_match:
return None
value = normalize_space(image_match.group(1))
return build_absolute_scholar_url(value)
def extract_articles_range(html: str) -> str | None:
pattern = re.compile(
r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)</[^>]+>",
@ -304,6 +412,229 @@ def count_markers(html: str) -> dict[str, int]:
return {key: lowered.count(key.lower()) for key in MARKER_KEYS}
def count_author_search_markers(html: str) -> dict[str, int]:
lowered = html.lower()
return {key: lowered.count(key.lower()) for key in AUTHOR_SEARCH_MARKER_KEYS}
def _extract_verified_email_domain(value: str | None) -> str | None:
if not value:
return None
match = re.search(r"verified email at\s+(.+)$", value.strip(), re.I)
if not match:
return None
domain = normalize_space(match.group(1))
return domain or None
class ScholarAuthorSearchParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.candidates: list[ScholarSearchCandidate] = []
self._candidate: dict[str, Any] | None = None
def _begin_candidate(self) -> None:
self._candidate = {
"depth": 1,
"name_href": None,
"name_parts": [],
"aff_depth": 0,
"aff_parts": [],
"name_depth": 0,
"eml_depth": 0,
"eml_parts": [],
"cby_depth": 0,
"cby_parts": [],
"interest_depth": 0,
"interest_parts": [],
"interests": [],
"image_src": None,
}
def _increment_capture_depths(self) -> None:
if self._candidate is None:
return
for key in ("name_depth", "aff_depth", "eml_depth", "cby_depth", "interest_depth"):
if self._candidate[key] > 0:
self._candidate[key] += 1
def _finalize_candidate(self) -> None:
if self._candidate is None:
return
name = normalize_space("".join(self._candidate["name_parts"]))
scholar_id = parse_scholar_id_from_href(self._candidate["name_href"])
if not name or not scholar_id:
return
affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None
email_domain = _extract_verified_email_domain(
normalize_space("".join(self._candidate["eml_parts"])) or None
)
cited_by_text = normalize_space("".join(self._candidate["cby_parts"]))
cited_by_match = re.search(r"\d+", cited_by_text)
cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None
seen_interests: set[str] = set()
interests: list[str] = []
for interest in self._candidate["interests"]:
normalized = normalize_space(interest)
if not normalized or normalized in seen_interests:
continue
seen_interests.add(normalized)
interests.append(normalized)
profile_url = build_absolute_scholar_url(self._candidate["name_href"])
if not profile_url:
profile_url = (
"https://scholar.google.com/citations"
f"?hl=en&user={scholar_id}"
)
self.candidates.append(
ScholarSearchCandidate(
scholar_id=scholar_id,
display_name=name,
affiliation=affiliation,
email_domain=email_domain,
cited_by_count=cited_by_count,
interests=interests,
profile_url=profile_url,
profile_image_url=build_absolute_scholar_url(self._candidate["image_src"]),
)
)
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
classes = attr_class(attrs)
if self._candidate is None:
if tag == "div" and "gsc_1usr" in classes:
self._begin_candidate()
return
self._candidate["depth"] += 1
self._increment_capture_depths()
if tag == "a" and "gs_ai_name" in classes:
self._candidate["name_depth"] = 1
self._candidate["name_href"] = attr_href(attrs)
return
if tag == "div" and "gs_ai_aff" in classes:
self._candidate["aff_depth"] = 1
return
if tag == "div" and "gs_ai_eml" in classes:
self._candidate["eml_depth"] = 1
return
if tag == "div" and "gs_ai_cby" in classes:
self._candidate["cby_depth"] = 1
return
if tag == "a" and "gs_ai_one_int" in classes:
self._candidate["interest_depth"] = 1
self._candidate["interest_parts"] = []
return
if tag == "img" and self._candidate["image_src"] is None:
self._candidate["image_src"] = attr_src(attrs)
def handle_data(self, data: str) -> None:
if self._candidate is None:
return
if self._candidate["name_depth"] > 0:
self._candidate["name_parts"].append(data)
if self._candidate["aff_depth"] > 0:
self._candidate["aff_parts"].append(data)
if self._candidate["eml_depth"] > 0:
self._candidate["eml_parts"].append(data)
if self._candidate["cby_depth"] > 0:
self._candidate["cby_parts"].append(data)
if self._candidate["interest_depth"] > 0:
self._candidate["interest_parts"].append(data)
def _decrement_capture_depth(self, key: str) -> bool:
if self._candidate is None:
return False
if self._candidate[key] <= 0:
return False
self._candidate[key] -= 1
return self._candidate[key] == 0
def handle_endtag(self, _tag: str) -> None:
if self._candidate is None:
return
interest_closed = self._decrement_capture_depth("interest_depth")
self._decrement_capture_depth("name_depth")
self._decrement_capture_depth("aff_depth")
self._decrement_capture_depth("eml_depth")
self._decrement_capture_depth("cby_depth")
if interest_closed:
interest_text = normalize_space("".join(self._candidate["interest_parts"]))
if interest_text:
self._candidate["interests"].append(interest_text)
self._candidate["interest_parts"] = []
self._candidate["depth"] -= 1
if self._candidate["depth"] > 0:
return
self._finalize_candidate()
self._candidate = None
def classify_network_error_reason(fetch_error: str | None) -> str:
lowered = (fetch_error or "").lower()
if lowered:
if any(keyword in lowered for keyword in NETWORK_DNS_ERROR_KEYWORDS):
return "network_dns_resolution_failed"
if any(keyword in lowered for keyword in NETWORK_TIMEOUT_KEYWORDS):
return "network_timeout"
if any(keyword in lowered for keyword in NETWORK_TLS_ERROR_KEYWORDS):
return "network_tls_error"
if "connection reset" in lowered:
return "network_connection_reset"
if "connection refused" in lowered:
return "network_connection_refused"
if "network is unreachable" in lowered:
return "network_unreachable"
return "network_error_missing_status_code"
def classify_block_or_captcha_reason(
*,
status_code: int,
final_url: str,
body_lowered: str,
) -> str | None:
if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url):
return "blocked_accounts_redirect"
if status_code == 429:
return "blocked_http_429_rate_limited"
if status_code == 403:
if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url:
return "blocked_http_403_captcha_challenge"
return "blocked_http_403_forbidden"
if "sorry/index" in final_url or "sorry/index" in body_lowered:
return "blocked_google_sorry_challenge"
if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered:
return "blocked_unusual_traffic_detected"
if "automated queries" in body_lowered:
return "blocked_automated_queries_detected"
if "not a robot" in body_lowered:
return "blocked_not_a_robot_challenge"
if "recaptcha" in body_lowered:
return "blocked_recaptcha_challenge"
if "captcha" in body_lowered:
return "blocked_captcha_challenge"
if any(keyword in body_lowered for keyword in BLOCKED_KEYWORDS):
return "blocked_keyword_detected"
return None
def detect_state(
fetch_result: FetchResult,
publications: list[PublicationCandidate],
@ -312,15 +643,19 @@ def detect_state(
visible_text: str,
) -> tuple[ParseState, str]:
if fetch_result.status_code is None:
return ParseState.NETWORK_ERROR, "network_error_missing_status_code"
return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
lowered = fetch_result.body.lower()
final = (fetch_result.final_url or "").lower()
status_code = int(fetch_result.status_code)
if "accounts.google.com" in final and ("signin" in final or "servicelogin" in final):
return ParseState.BLOCKED_OR_CAPTCHA, "blocked_accounts_redirect"
if any(keyword in lowered for keyword in BLOCKED_KEYWORDS) or "sorry/index" in final:
return ParseState.BLOCKED_OR_CAPTCHA, "blocked_keyword_detected"
block_reason = classify_block_or_captcha_reason(
status_code=status_code,
final_url=final,
body_lowered=lowered,
)
if block_reason is not None:
return ParseState.BLOCKED_OR_CAPTCHA, block_reason
if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS):
return ParseState.NO_RESULTS, "no_results_keyword_detected"
@ -335,6 +670,40 @@ def detect_state(
return ParseState.OK, "publications_extracted"
def detect_author_search_state(
fetch_result: FetchResult,
candidates: list[ScholarSearchCandidate],
marker_counts: dict[str, int],
*,
visible_text: str,
) -> tuple[ParseState, str]:
if fetch_result.status_code is None:
return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error)
lowered = fetch_result.body.lower()
final = (fetch_result.final_url or "").lower()
status_code = int(fetch_result.status_code)
block_reason = classify_block_or_captcha_reason(
status_code=status_code,
final_url=final,
body_lowered=lowered,
)
if block_reason is not None:
return ParseState.BLOCKED_OR_CAPTCHA, block_reason
if not candidates and any(keyword in visible_text for keyword in NO_AUTHOR_RESULTS_KEYWORDS):
return ParseState.NO_RESULTS, "no_results_keyword_detected"
if not candidates:
has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0
if not has_search_markers:
return ParseState.NO_RESULTS, "no_search_candidates_detected"
return ParseState.OK, "search_markers_present_with_empty_results"
return ParseState.OK, "author_candidates_extracted"
def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
publications, warnings = parse_publications(fetch_result.body)
marker_counts = count_markers(fetch_result.body)
@ -361,6 +730,7 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
state=state,
state_reason=state_reason,
profile_name=extract_profile_name(fetch_result.body),
profile_image_url=extract_profile_image_url(fetch_result.body),
publications=publications,
marker_counts=marker_counts,
warnings=warnings,
@ -368,3 +738,29 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
has_operation_error_banner=operation_error_banner,
articles_range=extract_articles_range(fetch_result.body),
)
def parse_author_search_page(fetch_result: FetchResult) -> ParsedAuthorSearchPage:
parser = ScholarAuthorSearchParser()
parser.feed(fetch_result.body)
marker_counts = count_author_search_markers(fetch_result.body)
visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
warnings: list[str] = []
if not parser.candidates:
warnings.append("no_author_candidates_detected")
state, state_reason = detect_author_search_state(
fetch_result,
parser.candidates,
marker_counts,
visible_text=visible_text,
)
return ParsedAuthorSearchPage(
state=state,
state_reason=state_reason,
candidates=parser.candidates,
marker_counts=marker_counts,
warnings=warnings,
)

40
app/services/scholar_source.py
View file

@ -52,6 +52,14 @@ class ScholarSource(Protocol):
) -> FetchResult:
...
async def fetch_author_search_html(
self,
query: str,
*,
start: int,
) -> FetchResult:
...
class LiveScholarSource:
def __init__(
@ -94,6 +102,27 @@ class LiveScholarSource:
)
return await asyncio.to_thread(self._fetch_sync, requested_url)
async def fetch_author_search_html(
self,
query: str,
*,
start: int = 0,
) -> FetchResult:
requested_url = _build_author_search_url(
query=query,
start=start,
)
logger.debug(
"scholar_source.search_fetch_started",
extra={
"event": "scholar_source.search_fetch_started",
"query": query,
"requested_url": requested_url,
"start": start,
},
)
return await asyncio.to_thread(self._fetch_sync, requested_url)
def _fetch_sync(self, requested_url: str) -> FetchResult:
request = Request(
requested_url,
@ -169,3 +198,14 @@ def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:
if pagesize > 0 and cstart > 0:
query["pagesize"] = int(pagesize)
return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}"
def _build_author_search_url(*, query: str, start: int) -> str:
params: dict[str, int | str] = {
"hl": "en",
"view_op": "search_authors",
"mauthors": query,
}
if start > 0:
params["astart"] = int(start)
return f"{SCHOLAR_PROFILE_URL}?{urlencode(params)}"

548
app/services/scholars.py
View file

@ -1,14 +1,106 @@
from __future__ import annotations
import asyncio
from collections import OrderedDict
from dataclasses import dataclass
from dataclasses import replace
from datetime import datetime
from datetime import timedelta
from datetime import timezone
import logging
import os
import random
import re
import time
from pathlib import Path
from urllib.parse import urlparse
from uuid import uuid4
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import ScholarProfile
from app.services.scholar_parser import (
ParseState,
ParsedAuthorSearchPage,
parse_author_search_page,
parse_profile_page,
)
from app.services.scholar_source import ScholarSource
SCHOLAR_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]{12}$")
MAX_IMAGE_URL_LENGTH = 2048
MAX_AUTHOR_SEARCH_LIMIT = 25
DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES = 512
DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS = 300
DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD = 1
DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS = 1800
DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS = 3.0
DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS = 1.0
ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES = {
"image/jpeg": ".jpg",
"image/png": ".png",
"image/webp": ".webp",
"image/gif": ".gif",
}
SEARCH_DISABLED_REASON = "search_disabled_by_configuration"
SEARCH_COOLDOWN_REASON = "search_temporarily_disabled_due_to_repeated_blocks"
SEARCH_CACHED_BLOCK_REASON = "search_temporarily_disabled_from_cached_blocked_response"
STATE_REASON_HINTS: dict[str, str] = {
SEARCH_DISABLED_REASON: (
"Scholar name search is currently disabled by service policy. "
"Add scholars by profile URL or Scholar ID."
),
SEARCH_COOLDOWN_REASON: (
"Scholar name search is temporarily paused after repeated block responses. "
"Use Scholar URL/ID adds until cooldown expires."
),
SEARCH_CACHED_BLOCK_REASON: (
"A recent blocked response was cached to reduce traffic. "
"Retry later or add by Scholar URL/ID."
),
"network_dns_resolution_failed": (
"DNS resolution failed while reaching scholar.google.com. "
"Verify container DNS/network and retry."
),
"network_timeout": (
"Request timed out before Google Scholar responded. "
"Increase delay/backoff and retry."
),
"network_tls_error": (
"TLS handshake/certificate validation failed. "
"Verify outbound TLS/network configuration."
),
"blocked_http_429_rate_limited": (
"Google Scholar rate-limited the request. "
"Slow request cadence and retry later."
),
"blocked_unusual_traffic_detected": (
"Google Scholar flagged traffic as unusual. "
"Increase delay/jitter and reduce concurrent scraping."
),
"blocked_accounts_redirect": (
"Request was redirected to Google Account sign-in. "
"Treat as access block and retry later."
),
}
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class _AuthorSearchCacheEntry:
parsed: ParsedAuthorSearchPage
expires_at_monotonic: float
cached_at_utc: datetime
_AUTHOR_SEARCH_EXECUTION_LOCK = asyncio.Lock()
_AUTHOR_SEARCH_CACHE: OrderedDict[str, _AuthorSearchCacheEntry] = OrderedDict()
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = 0.0
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC: datetime | None = None
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
class ScholarServiceError(ValueError):
@ -27,6 +119,174 @@ def normalize_display_name(value: str) -> str | None:
return normalized if normalized else None
def normalize_profile_image_url(value: str | None) -> str | None:
if value is None:
return None
candidate = value.strip()
if not candidate:
return None
if len(candidate) > MAX_IMAGE_URL_LENGTH:
raise ScholarServiceError(
f"Image URL must be {MAX_IMAGE_URL_LENGTH} characters or fewer."
)
parsed = urlparse(candidate)
if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc:
raise ScholarServiceError("Image URL must be an absolute http(s) URL.")
return candidate
def _ensure_upload_root(upload_dir: str, *, create: bool) -> Path:
root = Path(upload_dir).expanduser().resolve()
if create:
root.mkdir(parents=True, exist_ok=True)
return root
def _resolve_upload_path(upload_root: Path, relative_path: str) -> Path:
candidate = (upload_root / relative_path).resolve()
if upload_root != candidate and upload_root not in candidate.parents:
raise ScholarServiceError("Invalid scholar image path.")
return candidate
def _safe_remove_upload(upload_root: Path, relative_path: str | None) -> None:
if not relative_path:
return
try:
file_path = _resolve_upload_path(upload_root, relative_path)
except ScholarServiceError:
return
try:
if file_path.exists() and file_path.is_file():
file_path.unlink()
except OSError:
return
def resolve_profile_image(
profile: ScholarProfile,
*,
uploaded_image_url: str | None,
) -> tuple[str | None, str]:
if profile.profile_image_upload_path and uploaded_image_url:
return uploaded_image_url, "upload"
if profile.profile_image_override_url:
return profile.profile_image_override_url, "override"
if profile.profile_image_url:
return profile.profile_image_url, "scraped"
return None, "none"
def resolve_upload_file_path(*, upload_dir: str, relative_path: str) -> Path:
root = _ensure_upload_root(upload_dir, create=False)
return _resolve_upload_path(root, relative_path)
def scrape_state_hint(*, state: ParseState, state_reason: str) -> str | None:
if state not in {ParseState.NETWORK_ERROR, ParseState.BLOCKED_OR_CAPTCHA}:
return None
return STATE_REASON_HINTS.get(state_reason)
def _merge_warnings(base: list[str], extra: list[str]) -> list[str]:
if not extra:
return sorted(set(base))
return sorted(set(base + extra))
def _trim_author_search_result(
parsed: ParsedAuthorSearchPage,
*,
limit: int,
extra_warnings: list[str] | None = None,
state_reason_override: str | None = None,
) -> ParsedAuthorSearchPage:
return ParsedAuthorSearchPage(
state=parsed.state,
state_reason=state_reason_override or parsed.state_reason,
candidates=parsed.candidates[: max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))],
marker_counts=parsed.marker_counts,
warnings=_merge_warnings(parsed.warnings, extra_warnings or []),
)
def _policy_blocked_author_search_result(
*,
reason: str,
warning_codes: list[str],
limit: int,
) -> ParsedAuthorSearchPage:
_ = limit
return ParsedAuthorSearchPage(
state=ParseState.BLOCKED_OR_CAPTCHA,
state_reason=reason,
candidates=[],
marker_counts={},
warnings=_merge_warnings([], warning_codes),
)
def _cache_get_author_search_result(query_key: str, now_monotonic: float) -> _AuthorSearchCacheEntry | None:
entry = _AUTHOR_SEARCH_CACHE.get(query_key)
if entry is None:
return None
if entry.expires_at_monotonic <= now_monotonic:
_AUTHOR_SEARCH_CACHE.pop(query_key, None)
return None
_AUTHOR_SEARCH_CACHE.move_to_end(query_key)
return entry
def _cache_set_author_search_result(
*,
query_key: str,
parsed: ParsedAuthorSearchPage,
ttl_seconds: float,
max_entries: int,
) -> None:
ttl = max(float(ttl_seconds), 0.0)
if ttl <= 0.0:
_AUTHOR_SEARCH_CACHE.pop(query_key, None)
return
_AUTHOR_SEARCH_CACHE[query_key] = _AuthorSearchCacheEntry(
parsed=parsed,
expires_at_monotonic=time.monotonic() + ttl,
cached_at_utc=datetime.now(timezone.utc),
)
_AUTHOR_SEARCH_CACHE.move_to_end(query_key)
bounded_max_entries = max(1, int(max_entries))
while len(_AUTHOR_SEARCH_CACHE) > bounded_max_entries:
_AUTHOR_SEARCH_CACHE.popitem(last=False)
def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool:
return parsed.state == ParseState.BLOCKED_OR_CAPTCHA
def _author_search_cooldown_remaining_seconds(now_utc: datetime) -> int:
if _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC is None:
return 0
remaining_seconds = int((_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC - now_utc).total_seconds())
return max(0, remaining_seconds)
def _reset_author_search_runtime_state_for_tests() -> None:
global _AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC
global _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
global _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT
_AUTHOR_SEARCH_CACHE.clear()
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = 0.0
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC = None
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
async def list_scholars_for_user(
db_session: AsyncSession,
*,
@ -46,11 +306,13 @@ async def create_scholar_for_user(
user_id: int,
scholar_id: str,
display_name: str,
profile_image_url: str | None = None,
) -> ScholarProfile:
profile = ScholarProfile(
user_id=user_id,
scholar_id=validate_scholar_id(scholar_id),
display_name=normalize_display_name(display_name),
profile_image_url=normalize_profile_image_url(profile_image_url),
)
db_session.add(profile)
try:
@ -92,7 +354,293 @@ async def delete_scholar(
db_session: AsyncSession,
*,
profile: ScholarProfile,
upload_dir: str | None = None,
) -> None:
if upload_dir:
upload_root = _ensure_upload_root(upload_dir, create=True)
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
await db_session.delete(profile)
await db_session.commit()
async def search_author_candidates(
*,
source: ScholarSource,
query: str,
limit: int,
network_error_retries: int = 1,
retry_backoff_seconds: float = 1.0,
search_enabled: bool = True,
cache_ttl_seconds: int = 21_600,
blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS,
cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES,
min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS,
interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS,
cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD,
cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS,
) -> ParsedAuthorSearchPage:
global _AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC
global _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
global _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT
normalized_query = query.strip()
if len(normalized_query) < 2:
raise ScholarServiceError("Search query must be at least 2 characters.")
bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))
query_key = normalized_query.casefold()
if not search_enabled:
logger.warning(
"scholar_search.disabled_by_configuration",
extra={
"event": "scholar_search.disabled_by_configuration",
"query": normalized_query,
},
)
return _policy_blocked_author_search_result(
reason=SEARCH_DISABLED_REASON,
warning_codes=["author_search_disabled_by_configuration"],
limit=bounded_limit,
)
async with _AUTHOR_SEARCH_EXECUTION_LOCK:
now_utc = datetime.now(timezone.utc)
now_monotonic = time.monotonic()
cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(now_utc)
if cooldown_remaining_seconds > 0:
logger.warning(
"scholar_search.cooldown_active",
extra={
"event": "scholar_search.cooldown_active",
"query": normalized_query,
"cooldown_remaining_seconds": cooldown_remaining_seconds,
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat()
if _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
else None,
},
)
return _policy_blocked_author_search_result(
reason=SEARCH_COOLDOWN_REASON,
warning_codes=[
"author_search_cooldown_active",
f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
],
limit=bounded_limit,
)
cached_entry = _cache_get_author_search_result(query_key, now_monotonic)
if cached_entry is not None:
cached = cached_entry.parsed
state_reason_override = (
SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None
)
logger.info(
"scholar_search.cache_hit",
extra={
"event": "scholar_search.cache_hit",
"query": normalized_query,
"state": cached.state.value,
"state_reason": cached.state_reason,
},
)
return _trim_author_search_result(
cached,
limit=bounded_limit,
extra_warnings=["author_search_served_from_cache"],
state_reason_override=state_reason_override,
)
enforced_wait_seconds = (
(_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC + max(float(min_interval_seconds), 0.0))
- now_monotonic
)
jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0))
sleep_seconds = max(0.0, enforced_wait_seconds) + jitter_seconds
if sleep_seconds > 0.0:
logger.info(
"scholar_search.throttle_wait",
extra={
"event": "scholar_search.throttle_wait",
"query": normalized_query,
"sleep_seconds": round(sleep_seconds, 3),
},
)
await asyncio.sleep(sleep_seconds)
max_attempts = max(1, int(network_error_retries) + 1)
parsed: ParsedAuthorSearchPage | None = None
retry_warnings: list[str] = []
for attempt_index in range(max_attempts):
fetch_result = await source.fetch_author_search_html(normalized_query, start=0)
parsed = parse_author_search_page(fetch_result)
if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1:
break
retry_warnings.append("network_retry_scheduled_for_author_search")
sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index)
if sleep_seconds > 0:
await asyncio.sleep(sleep_seconds)
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = time.monotonic()
if parsed is None:
raise ScholarServiceError("Unable to complete scholar author search.")
merged_parsed = replace(
parsed,
warnings=_merge_warnings(parsed.warnings, retry_warnings),
)
if _is_author_search_block_state(merged_parsed):
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT += 1
logger.warning(
"scholar_search.block_detected",
extra={
"event": "scholar_search.block_detected",
"query": normalized_query,
"state_reason": merged_parsed.state_reason,
"consecutive_blocked_count": _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT,
},
)
if _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT >= max(1, int(cooldown_block_threshold)):
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC = datetime.now(timezone.utc) + timedelta(
seconds=max(60, int(cooldown_seconds))
)
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
merged_parsed = replace(
merged_parsed,
warnings=_merge_warnings(
merged_parsed.warnings,
["author_search_circuit_breaker_armed"],
),
)
logger.error(
"scholar_search.cooldown_activated",
extra={
"event": "scholar_search.cooldown_activated",
"query": normalized_query,
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat(),
},
)
else:
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
ttl_seconds = (
min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds)))
if _is_author_search_block_state(merged_parsed)
else max(1, int(cache_ttl_seconds))
)
_cache_set_author_search_result(
query_key=query_key,
parsed=merged_parsed,
ttl_seconds=float(ttl_seconds),
max_entries=cache_max_entries,
)
return _trim_author_search_result(
merged_parsed,
limit=bounded_limit,
)
async def hydrate_profile_metadata(
db_session: AsyncSession,
*,
profile: ScholarProfile,
source: ScholarSource,
) -> ScholarProfile:
fetch_result = await source.fetch_profile_html(profile.scholar_id)
parsed_page = parse_profile_page(fetch_result)
if parsed_page.profile_name and not (profile.display_name or "").strip():
profile.display_name = parsed_page.profile_name
if parsed_page.profile_image_url and not profile.profile_image_url:
profile.profile_image_url = parsed_page.profile_image_url
await db_session.commit()
await db_session.refresh(profile)
return profile
async def set_profile_image_override_url(
db_session: AsyncSession,
*,
profile: ScholarProfile,
image_url: str | None,
upload_dir: str,
) -> ScholarProfile:
upload_root = _ensure_upload_root(upload_dir, create=True)
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
profile.profile_image_upload_path = None
profile.profile_image_override_url = normalize_profile_image_url(image_url)
await db_session.commit()
await db_session.refresh(profile)
return profile
async def clear_profile_image_customization(
db_session: AsyncSession,
*,
profile: ScholarProfile,
upload_dir: str,
) -> ScholarProfile:
upload_root = _ensure_upload_root(upload_dir, create=True)
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
profile.profile_image_upload_path = None
profile.profile_image_override_url = None
await db_session.commit()
await db_session.refresh(profile)
return profile
async def set_profile_image_upload(
db_session: AsyncSession,
*,
profile: ScholarProfile,
content_type: str | None,
image_bytes: bytes,
upload_dir: str,
max_upload_bytes: int,
) -> ScholarProfile:
normalized_content_type = (content_type or "").strip().lower()
extension = ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES.get(normalized_content_type)
if extension is None:
raise ScholarServiceError(
"Unsupported image type. Use JPEG, PNG, WEBP, or GIF."
)
if not image_bytes:
raise ScholarServiceError("Uploaded image file is empty.")
if len(image_bytes) > max_upload_bytes:
raise ScholarServiceError(
f"Uploaded image exceeds {max_upload_bytes} bytes."
)
upload_root = _ensure_upload_root(upload_dir, create=True)
user_dir = upload_root / str(profile.user_id)
user_dir.mkdir(parents=True, exist_ok=True)
filename = f"{profile.id}_{uuid4().hex}{extension}"
relative_path = os.path.join(str(profile.user_id), filename)
absolute_path = _resolve_upload_path(upload_root, relative_path)
absolute_path.write_bytes(image_bytes)
old_path = profile.profile_image_upload_path
profile.profile_image_upload_path = relative_path
profile.profile_image_override_url = None
await db_session.commit()
await db_session.refresh(profile)
if old_path and old_path != relative_path:
_safe_remove_upload(upload_root, old_path)
return profile

39
app/settings.py
View file

@ -79,6 +79,45 @@ class Settings:
6,
)
scheduler_queue_batch_size: int = _env_int("SCHEDULER_QUEUE_BATCH_SIZE", 10)
frontend_enabled: bool = _env_bool("FRONTEND_ENABLED", True)
frontend_dist_dir: str = _env_str("FRONTEND_DIST_DIR", "/app/frontend/dist")
scholar_image_upload_dir: str = _env_str(
"SCHOLAR_IMAGE_UPLOAD_DIR",
"/var/lib/scholarr/uploads",
)
scholar_image_upload_max_bytes: int = _env_int(
"SCHOLAR_IMAGE_UPLOAD_MAX_BYTES",
2_000_000,
)
scholar_name_search_enabled: bool = _env_bool("SCHOLAR_NAME_SEARCH_ENABLED", True)
scholar_name_search_cache_ttl_seconds: int = _env_int(
"SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS",
21_600,
)
scholar_name_search_blocked_cache_ttl_seconds: int = _env_int(
"SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS",
300,
)
scholar_name_search_cache_max_entries: int = _env_int(
"SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES",
512,
)
scholar_name_search_min_interval_seconds: float = _env_float(
"SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS",
3.0,
)
scholar_name_search_interval_jitter_seconds: float = _env_float(
"SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS",
1.0,
)
scholar_name_search_cooldown_block_threshold: int = _env_int(
"SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD",
1,
)
scholar_name_search_cooldown_seconds: int = _env_int(
"SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS",
1800,
)
settings = Settings()

31
docker-compose.dev.yml Normal file
View file

@ -0,0 +1,31 @@
services:
app:
image: scholarr-dev:local
build:
context: .
target: dev
environment:
APP_RELOAD: ${APP_RELOAD:-1}
SESSION_COOKIE_SECURE: ${SESSION_COOKIE_SECURE:-0}
FRONTEND_ENABLED: ${FRONTEND_ENABLED:-0}
volumes:
- ./:/app
frontend:
image: node:20-alpine
working_dir: /frontend
command: ["sh", "-lc", "npm install && npm run dev"]
environment:
CHOKIDAR_USEPOLLING: ${CHOKIDAR_USEPOLLING:-1}
VITE_DEV_API_PROXY_TARGET: ${VITE_DEV_API_PROXY_TARGET:-http://app:8000}
ports:
- "${FRONTEND_HOST_PORT:-5173}:5173"
volumes:
- ./frontend:/frontend
- frontend_node_modules:/frontend/node_modules
depends_on:
app:
condition: service_healthy
volumes:
frontend_node_modules:

34
docker-compose.yml
View file

@ -3,12 +3,11 @@ name: scholarr
services:
db:
image: postgres:15-alpine
restart: unless-stopped
environment:
POSTGRES_DB: ${POSTGRES_DB:-scholar}
POSTGRES_USER: ${POSTGRES_USER:-scholar}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-scholar}
ports:
- "${POSTGRES_PORT:-5432}:5432"
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
@ -18,19 +17,19 @@ services:
retries: 20
app:
build:
context: .
target: dev
image: ${SCHOLARR_IMAGE:-justinzeus/scholarr:latest}
restart: unless-stopped
environment:
DATABASE_URL: ${DATABASE_URL:-postgresql+asyncpg://scholar:scholar@db:5432/scholar}
TEST_DATABASE_URL: ${TEST_DATABASE_URL:-}
DATABASE_URL: ${DATABASE_URL:-postgresql+asyncpg://${POSTGRES_USER:-scholar}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-scholar}}
MIGRATE_ON_START: ${MIGRATE_ON_START:-1}
APP_NAME: ${APP_NAME:-scholarr}
APP_HOST: ${APP_HOST:-0.0.0.0}
APP_PORT: ${APP_PORT:-8000}
APP_RELOAD: ${APP_RELOAD:-1}
SESSION_SECRET_KEY: ${SESSION_SECRET_KEY:-dev-insecure-session-key}
SESSION_COOKIE_SECURE: ${SESSION_COOKIE_SECURE:-0}
APP_RELOAD: ${APP_RELOAD:-0}
SESSION_SECRET_KEY: ${SESSION_SECRET_KEY:?set SESSION_SECRET_KEY}
SESSION_COOKIE_SECURE: ${SESSION_COOKIE_SECURE:-1}
FRONTEND_ENABLED: ${FRONTEND_ENABLED:-1}
FRONTEND_DIST_DIR: ${FRONTEND_DIST_DIR:-/app/frontend/dist}
LOGIN_RATE_LIMIT_ATTEMPTS: ${LOGIN_RATE_LIMIT_ATTEMPTS:-5}
LOGIN_RATE_LIMIT_WINDOW_SECONDS: ${LOGIN_RATE_LIMIT_WINDOW_SECONDS:-60}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
@ -50,6 +49,16 @@ services:
INGESTION_CONTINUATION_MAX_DELAY_SECONDS: ${INGESTION_CONTINUATION_MAX_DELAY_SECONDS:-3600}
INGESTION_CONTINUATION_MAX_ATTEMPTS: ${INGESTION_CONTINUATION_MAX_ATTEMPTS:-6}
SCHEDULER_QUEUE_BATCH_SIZE: ${SCHEDULER_QUEUE_BATCH_SIZE:-10}
SCHOLAR_IMAGE_UPLOAD_DIR: ${SCHOLAR_IMAGE_UPLOAD_DIR:-/var/lib/scholarr/uploads}
SCHOLAR_IMAGE_UPLOAD_MAX_BYTES: ${SCHOLAR_IMAGE_UPLOAD_MAX_BYTES:-2000000}
SCHOLAR_NAME_SEARCH_ENABLED: ${SCHOLAR_NAME_SEARCH_ENABLED:-1}
SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS: ${SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS:-21600}
SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS: ${SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS:-300}
SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES: ${SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES:-512}
SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS: ${SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS:-8.0}
SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS: ${SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS:-2.0}
SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD: ${SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD:-1}
SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS: ${SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS:-1800}
BOOTSTRAP_ADMIN_ON_START: ${BOOTSTRAP_ADMIN_ON_START:-0}
BOOTSTRAP_ADMIN_EMAIL: ${BOOTSTRAP_ADMIN_EMAIL:-}
BOOTSTRAP_ADMIN_PASSWORD: ${BOOTSTRAP_ADMIN_PASSWORD:-}
@ -59,7 +68,7 @@ services:
ports:
- "${APP_HOST_PORT:-8000}:8000"
volumes:
- ./:/app
- scholar_uploads:/var/lib/scholarr/uploads
depends_on:
db:
condition: service_healthy
@ -71,3 +80,4 @@ services:
volumes:
postgres_data:
scholar_uploads:

3
frontend/.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
node_modules/
dist/
.vite/

31
frontend/index.html Normal file
View file

@ -0,0 +1,31 @@
<!doctype html>
<html lang="en" class="h-full">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>scholarr</title>
<script>
(function () {
var storageKey = "scholarr-theme-preference";
var root = document.documentElement;
var stored = null;
try {
stored = localStorage.getItem(storageKey);
} catch (_err) {
stored = null;
}
var preference = stored === "light" || stored === "dark" || stored === "system" ? stored : "system";
var systemDark = window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches;
var effective = preference === "system" ? (systemDark ? "dark" : "light") : preference;
root.classList.toggle("dark", effective === "dark");
root.setAttribute("data-theme-preference", preference);
})();
</script>
</head>
<body class="h-full">
<div id="app"></div>
<script type="module" src="/src/main.ts"></script>
</body>
</html>

2985
frontend/package-lock.json generated Normal file
View file

File diff suppressed because it is too large Load diff

30
frontend/package.json Normal file
View file

@ -0,0 +1,30 @@
{
"name": "scholarr-frontend",
"private": true,
"version": "0.1.0",
"type": "module",
"scripts": {
"dev": "vite --host 0.0.0.0 --port 5173",
"build": "vue-tsc --noEmit && vite build",
"preview": "vite preview --host 0.0.0.0 --port 4173",
"typecheck": "vue-tsc --noEmit",
"test": "vitest",
"test:run": "vitest run"
},
"dependencies": {
"pinia": "^2.1.7",
"vue": "^3.5.13",
"vue-router": "^4.5.0"
},
"devDependencies": {
"@types/node": "^22.10.2",
"@vitejs/plugin-vue": "^5.2.1",
"autoprefixer": "^10.4.21",
"postcss": "^8.5.3",
"tailwindcss": "^3.4.17",
"typescript": "^5.7.2",
"vite": "^5.4.11",
"vitest": "^2.1.8",
"vue-tsc": "^2.1.10"
}
}

6
frontend/postcss.config.cjs Normal file
View file

@ -0,0 +1,6 @@
module.exports = {
plugins: {
tailwindcss: {},
autoprefixer: {},
},
};

35
frontend/src/app/AppShell.vue Normal file
View file

@ -0,0 +1,35 @@
<script setup lang="ts">
import { computed } from "vue";
import { RouterView } from "vue-router";
import AppHeader from "@/components/layout/AppHeader.vue";
import AppNav from "@/components/layout/AppNav.vue";
import RequestErrorPanel from "@/components/patterns/RequestErrorPanel.vue";
import { useAuthStore } from "@/stores/auth";
const auth = useAuthStore();
const showChrome = computed(() => auth.isAuthenticated);
</script>
<template>
<div class="min-h-screen overflow-x-clip">
<a href="#app-main" class="skip-link">Skip to main content</a>
<RequestErrorPanel />
<template v-if="showChrome">
<AppHeader />
<div
class="grid min-h-[calc(100dvh-4.5rem)] grid-cols-1 lg:h-[calc(100dvh-4.5rem)] lg:min-h-[calc(100dvh-4.5rem)] lg:grid-cols-[17rem_minmax(0,1fr)]"
>
<AppNav class="lg:min-h-0 lg:overflow-y-auto" />
<main id="app-main" class="min-w-0 px-4 py-6 sm:px-6 lg:min-h-0 lg:overflow-y-auto lg:px-8">
<RouterView />
</main>
</div>
</template>
<main id="app-main" v-else class="min-h-screen overflow-x-clip">
<RouterView />
</main>
</div>
</template>

23
frontend/src/app/guards.ts Normal file
View file

@ -0,0 +1,23 @@
import type { Router } from "vue-router";
import { useAuthStore } from "@/stores/auth";
export function applyRouteGuards(router: Router): void {
router.beforeEach((to) => {
const auth = useAuthStore();
if (to.meta.requiresAuth && !auth.isAuthenticated) {
return { name: "login" };
}
if (to.meta.requiresAdmin && !auth.isAdmin) {
return { name: "dashboard" };
}
if (to.meta.guestOnly && auth.isAuthenticated) {
return { name: "dashboard" };
}
return true;
});
}

12
frontend/src/app/providers.ts Normal file
View file

@ -0,0 +1,12 @@
import { setCsrfTokenProvider } from "@/lib/api/client";
import { useAuthStore } from "@/stores/auth";
import { useThemeStore } from "@/stores/theme";
export async function bootstrapAppProviders(): Promise<void> {
const theme = useThemeStore();
theme.initialize();
const auth = useAuthStore();
setCsrfTokenProvider(() => auth.csrfToken);
await auth.bootstrapSession();
}

84
frontend/src/app/router.ts Normal file
View file

@ -0,0 +1,84 @@
import { createRouter, createWebHistory } from "vue-router";
import { applyRouteGuards } from "@/app/guards";
import LoginPage from "@/pages/LoginPage.vue";
import DashboardPage from "@/pages/DashboardPage.vue";
import ScholarsPage from "@/pages/ScholarsPage.vue";
import PublicationsPage from "@/pages/PublicationsPage.vue";
import RunsPage from "@/pages/RunsPage.vue";
import RunDetailPage from "@/pages/RunDetailPage.vue";
import SettingsPage from "@/pages/SettingsPage.vue";
import AdminUsersPage from "@/pages/AdminUsersPage.vue";
import StyleGuidePage from "@/pages/StyleGuidePage.vue";
export const router = createRouter({
history: createWebHistory(),
routes: [
{
path: "/login",
name: "login",
component: LoginPage,
meta: { guestOnly: true },
},
{
path: "/",
redirect: "/dashboard",
},
{
path: "/dashboard",
name: "dashboard",
component: DashboardPage,
meta: { requiresAuth: true },
},
{
path: "/scholars",
name: "scholars",
component: ScholarsPage,
meta: { requiresAuth: true },
},
{
path: "/publications",
name: "publications",
component: PublicationsPage,
meta: { requiresAuth: true },
},
{
path: "/settings",
name: "settings",
component: SettingsPage,
meta: { requiresAuth: true },
},
{
path: "/admin/style-guide",
name: "style-guide",
component: StyleGuidePage,
meta: { requiresAuth: true, requiresAdmin: true },
},
{
path: "/admin/runs",
name: "runs",
component: RunsPage,
meta: { requiresAuth: true, requiresAdmin: true },
},
{
path: "/admin/runs/:id",
name: "run-detail",
component: RunDetailPage,
meta: { requiresAuth: true, requiresAdmin: true },
},
{
path: "/admin/users",
name: "admin-users",
component: AdminUsersPage,
meta: { requiresAuth: true, requiresAdmin: true },
},
{
path: "/:pathMatch(.*)*",
redirect: "/dashboard",
},
],
});
applyRouteGuards(router);
export default router;

71
frontend/src/components/layout/AppHeader.vue Normal file
View file

@ -0,0 +1,71 @@
<script setup lang="ts">
import { computed } from "vue";
import AppButton from "@/components/ui/AppButton.vue";
import { useThemeStore } from "@/stores/theme";
const theme = useThemeStore();
const isDarkTheme = computed(() => theme.active === "dark");
const toggleThemeLabel = computed(() =>
isDarkTheme.value ? "Switch to light theme" : "Switch to dark theme",
);
function onToggleTheme(): void {
theme.setPreference(isDarkTheme.value ? "light" : "dark");
}
</script>
<template>
<header class="sticky top-0 z-30 border-b border-zinc-200/70 bg-white/90 backdrop-blur dark:border-zinc-800 dark:bg-zinc-900/90">
<div class="flex min-h-[4.5rem] w-full items-center justify-between gap-4 px-4 sm:px-6 lg:px-8">
<div class="flex min-w-0 items-center gap-2">
<RouterLink
to="/dashboard"
class="rounded-sm font-display text-lg tracking-tight text-zinc-900 transition hover:text-zinc-700 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-brand-500 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 dark:text-zinc-100 dark:hover:text-zinc-300 dark:focus-visible:ring-brand-400 dark:focus-visible:ring-offset-zinc-950"
>
scholarr
</RouterLink>
</div>
<div class="flex items-center justify-end">
<AppButton
variant="ghost"
class="h-10 w-10 rounded-full p-0"
:aria-label="toggleThemeLabel"
:title="toggleThemeLabel"
@click="onToggleTheme"
>
<span class="sr-only">{{ toggleThemeLabel }}</span>
<svg
v-if="isDarkTheme"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="1.8"
class="h-5 w-5"
aria-hidden="true"
>
<circle cx="12" cy="12" r="4" />
<path
d="M12 2v2.5M12 19.5V22M4.9 4.9l1.8 1.8M17.3 17.3l1.8 1.8M2 12h2.5M19.5 12H22M4.9 19.1l1.8-1.8M17.3 6.7l1.8-1.8"
/>
</svg>
<svg
v-else
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="1.8"
class="h-5 w-5"
aria-hidden="true"
>
<path d="M21 14.5A8.5 8.5 0 1 1 9.5 3 7 7 0 0 0 21 14.5z" />
</svg>
</AppButton>
</div>
</div>
</header>
</template>

56
frontend/src/components/layout/AppNav.vue Normal file
View file

@ -0,0 +1,56 @@
<script setup lang="ts">
import { computed } from "vue";
import { useRouter } from "vue-router";
import AppButton from "@/components/ui/AppButton.vue";
import { useAuthStore } from "@/stores/auth";
const auth = useAuthStore();
const router = useRouter();
const links = computed(() => {
const base = [
{ to: "/dashboard", label: "Dashboard" },
{ to: "/scholars", label: "Scholars" },
{ to: "/publications", label: "Publications" },
{ to: "/settings", label: "Settings" },
];
if (auth.isAdmin) {
base.push({ to: "/admin/runs", label: "Runs" });
base.push({ to: "/admin/users", label: "Users" });
}
return base;
});
async function onLogout(): Promise<void> {
await auth.logout();
await router.replace({ name: "login" });
}
</script>
<template>
<aside
class="min-w-0 border-b border-zinc-200 bg-white/70 px-4 py-4 dark:border-zinc-800 dark:bg-zinc-900/70 lg:h-full lg:min-h-0 lg:overflow-y-auto lg:border-b-0 lg:border-r lg:px-5 lg:py-6"
>
<div class="flex h-full min-h-0 flex-col gap-4">
<nav class="grid grid-cols-2 content-start gap-2 sm:grid-cols-3 lg:grid-cols-1" aria-label="Primary">
<RouterLink
v-for="link in links"
:key="link.to"
:to="link.to"
class="inline-flex min-h-10 min-w-0 items-center rounded-lg border border-transparent px-3 py-2 text-sm font-medium text-zinc-600 transition hover:border-zinc-300 hover:bg-zinc-50 hover:text-zinc-900 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-brand-500 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 dark:text-zinc-300 dark:hover:border-zinc-700 dark:hover:bg-zinc-800 dark:hover:text-zinc-100 dark:focus-visible:ring-brand-400 dark:focus-visible:ring-offset-zinc-950"
active-class="border-zinc-900 bg-zinc-900 text-white hover:border-zinc-900 hover:bg-zinc-900 hover:text-white dark:border-brand-400 dark:bg-brand-400 dark:text-zinc-950 dark:hover:border-brand-300 dark:hover:bg-brand-300"
>
{{ link.label }}
</RouterLink>
</nav>
<div class="mt-auto grid gap-2 border-t border-zinc-200 pt-3 dark:border-zinc-800">
<p class="truncate text-xs text-zinc-500 dark:text-zinc-400">{{ auth.user?.email }}</p>
<AppButton variant="ghost" class="w-full justify-start" @click="onLogout">Logout</AppButton>
</div>
</div>
</aside>
</template>

16
frontend/src/components/layout/AppPage.vue Normal file
View file

@ -0,0 +1,16 @@
<script setup lang="ts">
defineProps<{
title: string;
subtitle?: string;
}>();
</script>
<template>
<section class="page-stack min-w-0">
<header class="space-y-2">
<h1 class="page-title">{{ title }}</h1>
<p v-if="subtitle" class="page-subtitle">{{ subtitle }}</p>
</header>
<slot />
</section>
</template>

22
frontend/src/components/patterns/QueueHealthBadge.vue Normal file
View file

@ -0,0 +1,22 @@
<script setup lang="ts">
const props = defineProps<{
queued: number;
retrying: number;
dropped: number;
}>();
</script>
<template>
<span
class="inline-flex flex-wrap items-center gap-2 rounded-full border border-zinc-300 bg-zinc-100 px-3 py-1 text-xs text-zinc-700 dark:border-zinc-700 dark:bg-zinc-800 dark:text-zinc-300"
>
<span class="font-medium">Queue</span>
<span class="rounded-full bg-zinc-200 px-2 py-0.5 dark:bg-zinc-700">{{ props.queued }} queued</span>
<span class="rounded-full bg-amber-100 px-2 py-0.5 text-amber-800 dark:bg-amber-950/50 dark:text-amber-300">
{{ props.retrying }} retrying
</span>
<span class="rounded-full bg-rose-100 px-2 py-0.5 text-rose-700 dark:bg-rose-950/50 dark:text-rose-300">
{{ props.dropped }} dropped
</span>
</span>
</template>

18
frontend/src/components/patterns/RequestErrorPanel.vue Normal file
View file

@ -0,0 +1,18 @@
<script setup lang="ts">
import AppAlert from "@/components/ui/AppAlert.vue";
import { useUiStore } from "@/stores/ui";
const ui = useUiStore();
</script>
<template>
<div v-if="ui.globalError" class="sticky top-[4.5rem] z-20 px-4 pb-2 sm:px-6 lg:px-8">
<div class="mx-auto w-full max-w-7xl">
<AppAlert tone="danger" :dismissible="true" @dismiss="ui.clearGlobalError()">
<template #title>Request failed</template>
<p>{{ ui.globalError.message }}</p>
<p class="text-secondary">Request ID: {{ ui.globalError.requestId || "n/a" }}</p>
</AppAlert>
</div>
</div>
</template>

34
frontend/src/components/patterns/RunStatusBadge.vue Normal file
View file

@ -0,0 +1,34 @@
<script setup lang="ts">
import { computed } from "vue";
const props = defineProps<{
status: string;
}>();
const label = computed(() => props.status.replaceAll("_", " "));
const toneClass = computed(() => {
if (props.status === "success") {
return "border-emerald-300 bg-emerald-50 text-emerald-700 dark:border-emerald-800 dark:bg-emerald-950/50 dark:text-emerald-300";
}
if (props.status === "partial_failure") {
return "border-amber-300 bg-amber-50 text-amber-800 dark:border-amber-800 dark:bg-amber-950/50 dark:text-amber-300";
}
if (props.status === "failed") {
return "border-rose-300 bg-rose-50 text-rose-700 dark:border-rose-800 dark:bg-rose-950/50 dark:text-rose-300";
}
if (props.status === "running") {
return "border-brand-300 bg-brand-50 text-brand-700 dark:border-brand-800 dark:bg-brand-950/50 dark:text-brand-300";
}
return "border-zinc-300 bg-zinc-100 text-zinc-700 dark:border-zinc-700 dark:bg-zinc-800 dark:text-zinc-300";
});
</script>
<template>
<span
class="inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-semibold capitalize"
:class="toneClass"
>
{{ label }}
</span>
</template>

49
frontend/src/components/ui/AppAlert.vue Normal file
View file

@ -0,0 +1,49 @@
<script setup lang="ts">
import { computed } from "vue";
import AppButton from "@/components/ui/AppButton.vue";
const props = withDefaults(
defineProps<{
tone?: "danger" | "warning" | "info" | "success";
dismissible?: boolean;
}>(),
{
tone: "info",
dismissible: false,
},
);
const emit = defineEmits<{ dismiss: [] }>();
const toneClass = computed(() => {
if (props.tone === "danger") {
return "border-rose-300 bg-rose-50 text-rose-900 dark:border-rose-800 dark:bg-rose-950/45 dark:text-rose-200";
}
if (props.tone === "warning") {
return "border-amber-300 bg-amber-50 text-amber-900 dark:border-amber-800 dark:bg-amber-950/45 dark:text-amber-200";
}
if (props.tone === "success") {
return "border-emerald-300 bg-emerald-50 text-emerald-900 dark:border-emerald-800 dark:bg-emerald-950/45 dark:text-emerald-200";
}
return "border-brand-300 bg-brand-50 text-brand-900 dark:border-brand-800 dark:bg-brand-950/45 dark:text-brand-200";
});
const alertRole = computed(() => (props.tone === "danger" || props.tone === "warning" ? "alert" : "status"));
const alertLive = computed(() => (props.tone === "danger" || props.tone === "warning" ? "assertive" : "polite"));
</script>
<template>
<div
class="flex items-start justify-between gap-3 rounded-xl border px-4 py-3 text-sm"
:class="toneClass"
:role="alertRole"
:aria-live="alertLive"
>
<div class="space-y-1">
<strong v-if="$slots.title" class="block font-semibold"><slot name="title" /></strong>
<slot />
</div>
<AppButton v-if="props.dismissible" variant="ghost" @click="emit('dismiss')">Dismiss</AppButton>
</div>
</template>

33
frontend/src/components/ui/AppBadge.vue Normal file
View file

@ -0,0 +1,33 @@
<script setup lang="ts">
import { computed } from "vue";
const props = defineProps<{
tone?: "neutral" | "success" | "warning" | "danger" | "info";
}>();
const toneClass = computed(() => {
if (props.tone === "success") {
return "border-emerald-300 bg-emerald-50 text-emerald-700 dark:border-emerald-800 dark:bg-emerald-950/50 dark:text-emerald-300";
}
if (props.tone === "warning") {
return "border-amber-300 bg-amber-50 text-amber-800 dark:border-amber-800 dark:bg-amber-950/50 dark:text-amber-300";
}
if (props.tone === "danger") {
return "border-rose-300 bg-rose-50 text-rose-700 dark:border-rose-800 dark:bg-rose-950/50 dark:text-rose-300";
}
if (props.tone === "info") {
return "border-brand-300 bg-brand-50 text-brand-700 dark:border-brand-800 dark:bg-brand-950/50 dark:text-brand-300";
}
return "border-zinc-300 bg-zinc-100 text-zinc-700 dark:border-zinc-700 dark:bg-zinc-800 dark:text-zinc-300";
});
</script>
<template>
<span class="inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-medium" :class="toneClass">
<slot />
</span>
</template>

43
frontend/src/components/ui/AppButton.vue Normal file
View file

@ -0,0 +1,43 @@
<script setup lang="ts">
import { computed } from "vue";
const props = withDefaults(
defineProps<{
variant?: "primary" | "secondary" | "ghost" | "danger";
type?: "button" | "submit" | "reset";
disabled?: boolean;
}>(),
{
variant: "primary",
type: "button",
disabled: false,
},
);
const variantClass = computed(() => {
if (props.variant === "secondary") {
return "border-zinc-300 bg-zinc-100 text-zinc-900 hover:bg-zinc-200 dark:border-zinc-700 dark:bg-zinc-800 dark:text-zinc-100 dark:hover:bg-zinc-700";
}
if (props.variant === "ghost") {
return "border-zinc-300 bg-transparent text-zinc-700 hover:bg-zinc-100 dark:border-zinc-700 dark:text-zinc-200 dark:hover:bg-zinc-800";
}
if (props.variant === "danger") {
return "border-rose-300 bg-rose-100 text-rose-800 hover:bg-rose-200 dark:border-rose-800 dark:bg-rose-950/60 dark:text-rose-200 dark:hover:bg-rose-900/70";
}
return "border-brand-700 bg-brand-700 text-white hover:bg-brand-600 dark:border-brand-400 dark:bg-brand-400 dark:text-zinc-950 dark:hover:bg-brand-300";
});
</script>
<template>
<button
class="inline-flex min-h-10 items-center justify-center rounded-lg border px-3 py-2 text-sm font-semibold transition focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-brand-500 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 disabled:cursor-not-allowed disabled:opacity-60 dark:focus-visible:ring-brand-400 dark:focus-visible:ring-offset-zinc-950"
:class="variantClass"
:type="props.type"
:disabled="props.disabled"
>
<slot />
</button>
</template>

5
frontend/src/components/ui/AppCard.vue Normal file
View file

@ -0,0 +1,5 @@
<template>
<article class="min-w-0 rounded-2xl border border-zinc-200 bg-white p-5 shadow-panel dark:border-zinc-800 dark:bg-zinc-900">
<slot />
</article>
</template>

22
frontend/src/components/ui/AppCheckbox.vue Normal file
View file

@ -0,0 +1,22 @@
<script setup lang="ts">
const model = defineModel<boolean>({ default: false });
defineProps<{
id?: string;
disabled?: boolean;
label?: string;
}>();
</script>
<template>
<label class="inline-flex min-h-10 items-center gap-2 text-sm text-zinc-800 dark:text-zinc-200" :for="id">
<input
:id="id"
v-model="model"
class="h-4 w-4 rounded border-zinc-400 text-brand-600 focus-visible:ring-2 focus-visible:ring-brand-500 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 dark:border-zinc-600 dark:bg-zinc-900 dark:focus-visible:ring-brand-400 dark:focus-visible:ring-offset-zinc-950"
type="checkbox"
:disabled="disabled"
/>
<span><slot>{{ label }}</slot></span>
</label>
</template>

16
frontend/src/components/ui/AppEmptyState.vue Normal file
View file

@ -0,0 +1,16 @@
<script setup lang="ts">
defineProps<{
title: string;
body: string;
}>();
</script>
<template>
<section class="rounded-xl border border-dashed border-zinc-300 bg-zinc-50 p-5 dark:border-zinc-700 dark:bg-zinc-900/60">
<h3 class="font-display text-lg font-semibold text-zinc-900 dark:text-zinc-100">{{ title }}</h3>
<p class="mt-1 text-sm text-zinc-600 dark:text-zinc-400">{{ body }}</p>
<div v-if="$slots.default" class="mt-4">
<slot />
</div>
</section>
</template>

21
frontend/src/components/ui/AppInput.vue Normal file
View file

@ -0,0 +1,21 @@
<script setup lang="ts">
const model = defineModel<string>({ default: "" });
defineProps<{
id?: string;
type?: string;
placeholder?: string;
disabled?: boolean;
}>();
</script>
<template>
<input
v-model="model"
class="w-full rounded-lg border border-zinc-300 bg-white px-3 py-2 text-sm text-zinc-900 outline-none ring-brand-300 transition placeholder:text-zinc-400 focus-visible:ring-2 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 disabled:cursor-not-allowed disabled:opacity-60 dark:border-zinc-700 dark:bg-zinc-900 dark:text-zinc-100 dark:ring-brand-400 dark:focus-visible:ring-offset-zinc-950 dark:placeholder:text-zinc-500"
:id="id"
:type="type || 'text'"
:placeholder="placeholder"
:disabled="disabled"
/>
</template>

17
frontend/src/components/ui/AppModal.vue Normal file
View file

@ -0,0 +1,17 @@
<script setup lang="ts">
const props = defineProps<{
open: boolean;
title: string;
}>();
const emit = defineEmits<{ close: [] }>();
</script>
<template>
<div v-if="props.open" class="fixed inset-0 z-50 grid place-items-center bg-zinc-950/60 p-4" @click.self="emit('close')">
<div class="w-full max-w-lg rounded-2xl border border-zinc-200 bg-white p-6 shadow-panel dark:border-zinc-800 dark:bg-zinc-900">
<h3 class="mb-4 font-display text-xl font-semibold text-zinc-900 dark:text-zinc-100">{{ props.title }}</h3>
<slot />
</div>
</div>
</template>

19
frontend/src/components/ui/AppSelect.vue Normal file
View file

@ -0,0 +1,19 @@
<script setup lang="ts">
const model = defineModel<string>({ default: "" });
defineProps<{
id?: string;
disabled?: boolean;
}>();
</script>
<template>
<select
v-model="model"
class="w-full rounded-lg border border-zinc-300 bg-white px-3 py-2 text-sm text-zinc-900 outline-none ring-brand-300 transition focus-visible:ring-2 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 disabled:cursor-not-allowed disabled:opacity-60 dark:border-zinc-700 dark:bg-zinc-900 dark:text-zinc-100 dark:ring-brand-400 dark:focus-visible:ring-offset-zinc-950"
:id="id"
:disabled="disabled"
>
<slot />
</select>
</template>

15
frontend/src/components/ui/AppSkeleton.vue Normal file
View file

@ -0,0 +1,15 @@
<script setup lang="ts">
defineProps<{
lines?: number;
}>();
</script>
<template>
<div class="space-y-2" role="status" aria-live="polite">
<span
v-for="line in lines || 3"
:key="line"
class="block h-3 animate-pulse rounded-full bg-zinc-200 dark:bg-zinc-800"
/>
</div>
</template>

31
frontend/src/components/ui/AppTable.vue Normal file
View file

@ -0,0 +1,31 @@
<script setup lang="ts">
defineProps<{
label?: string;
}>();
</script>
<template>
<div
class="max-w-full overflow-x-auto rounded-xl border border-zinc-200 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-brand-500 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 dark:border-zinc-800 dark:focus-visible:ring-brand-400 dark:focus-visible:ring-offset-zinc-950"
tabindex="0"
>
<table class="min-w-full border-collapse bg-white text-sm dark:bg-zinc-900" :aria-label="label || 'Data table'">
<slot />
</table>
</div>
</template>
<style scoped>
:deep(th),
:deep(td) {
@apply border-b border-zinc-200 px-3 py-2 text-left align-top dark:border-zinc-800;
}
:deep(th) {
@apply sticky top-0 z-10 bg-zinc-100 font-semibold text-zinc-800 dark:bg-zinc-900 dark:text-zinc-200;
}
:deep(td) {
@apply text-zinc-700 dark:text-zinc-300;
}
</style>

1
frontend/src/env.d.ts vendored Normal file
View file

@ -0,0 +1 @@
/// <reference types="vite/client" />

49
frontend/src/features/admin_users/index.ts Normal file
View file

@ -0,0 +1,49 @@
import { apiRequest } from "@/lib/api/client";
export interface AdminUser {
id: number;
email: string;
is_active: boolean;
is_admin: boolean;
created_at: string;
updated_at: string;
}
interface UsersListData {
users: AdminUser[];
}
export interface CreateAdminUserPayload {
email: string;
password: string;
is_admin: boolean;
}
export async function listAdminUsers(): Promise<AdminUser[]> {
const response = await apiRequest<UsersListData>("/admin/users", { method: "GET" });
return response.data.users;
}
export async function createAdminUser(payload: CreateAdminUserPayload): Promise<AdminUser> {
const response = await apiRequest<AdminUser>("/admin/users", {
method: "POST",
body: payload,
});
return response.data;
}
export async function setAdminUserActive(userId: number, isActive: boolean): Promise<AdminUser> {
const response = await apiRequest<AdminUser>(`/admin/users/${userId}/active`, {
method: "PATCH",
body: { is_active: isActive },
});
return response.data;
}
export async function resetAdminUserPassword(userId: number, newPassword: string): Promise<{ message: string }> {
const response = await apiRequest<{ message: string }>(`/admin/users/${userId}/reset-password`, {
method: "POST",
body: { new_password: newPassword },
});
return response.data;
}

9
frontend/src/features/auth/index.ts Normal file
View file

@ -0,0 +1,9 @@
export { fetchCsrfBootstrap, type CsrfBootstrapData } from "@/lib/api/csrf";
export {
fetchMe,
loginSession,
logoutSession,
type AuthSessionData,
type MessageData,
type SessionUser,
} from "@/lib/auth/session";

58
frontend/src/features/dashboard/index.ts Normal file
View file

@ -0,0 +1,58 @@
import {
listPublications,
type PublicationItem,
type PublicationMode,
} from "@/features/publications";
import { listQueueItems, listRuns, type RunListItem } from "@/features/runs";
export interface QueueHealth {
queued: number;
retrying: number;
dropped: number;
}
export interface DashboardSnapshot {
newCount: number;
totalCount: number;
mode: PublicationMode;
latestRun: RunListItem | null;
recentRuns: RunListItem[];
recentPublications: PublicationItem[];
queue: QueueHealth;
}
function countQueueStatuses(statuses: string[]): QueueHealth {
return statuses.reduce<QueueHealth>(
(acc, status) => {
if (status === "queued") {
acc.queued += 1;
} else if (status === "retrying") {
acc.retrying += 1;
} else if (status === "dropped") {
acc.dropped += 1;
}
return acc;
},
{ queued: 0, retrying: 0, dropped: 0 },
);
}
export async function fetchDashboardSnapshot(): Promise<DashboardSnapshot> {
const [publications, runs, queueItems] = await Promise.all([
listPublications({ mode: "new", limit: 20 }),
listRuns({ limit: 5 }),
listQueueItems(200),
]);
const queueHealth = countQueueStatuses(queueItems.map((item) => item.status));
return {
newCount: publications.new_count,
totalCount: publications.total_count,
mode: publications.mode,
latestRun: runs[0] ?? null,
recentRuns: runs,
recentPublications: publications.publications,
queue: queueHealth,
};
}

81
frontend/src/features/publications/index.ts Normal file
View file

@ -0,0 +1,81 @@
import { apiRequest } from "@/lib/api/client";
export type PublicationMode = "all" | "new";
export interface PublicationItem {
publication_id: number;
scholar_profile_id: number;
scholar_label: string;
title: string;
year: number | null;
citation_count: number;
venue_text: string | null;
pub_url: string | null;
is_read: boolean;
first_seen_at: string;
is_new_in_latest_run: boolean;
}
export interface PublicationsResult {
mode: PublicationMode;
selected_scholar_profile_id: number | null;
new_count: number;
total_count: number;
publications: PublicationItem[];
}
export interface PublicationsQuery {
mode?: PublicationMode;
scholarProfileId?: number;
limit?: number;
}
export interface PublicationSelection {
scholar_profile_id: number;
publication_id: number;
}
export async function listPublications(query: PublicationsQuery = {}): Promise<PublicationsResult> {
const params = new URLSearchParams();
if (query.mode) {
params.set("mode", query.mode);
}
if (query.scholarProfileId) {
params.set("scholar_profile_id", String(query.scholarProfileId));
}
if (query.limit) {
params.set("limit", String(query.limit));
}
const suffix = params.toString();
const response = await apiRequest<PublicationsResult>(
`/publications${suffix ? `?${suffix}` : ""}`,
{ method: "GET" },
);
return response.data;
}
export async function markAllRead(): Promise<{ message: string; updated_count: number }> {
const response = await apiRequest<{ message: string; updated_count: number }>(
"/publications/mark-all-read",
{ method: "POST" },
);
return response.data;
}
export async function markSelectedRead(selections: PublicationSelection[]): Promise<{
message: string;
requested_count: number;
updated_count: number;
}> {
const response = await apiRequest<{
message: string;
requested_count: number;
updated_count: number;
}>("/publications/mark-read", {
method: "POST",
body: { selections },
});
return response.data;
}

171
frontend/src/features/runs/index.ts Normal file
View file

@ -0,0 +1,171 @@
import { apiRequest } from "@/lib/api/client";
export interface RunListItem {
id: number;
trigger_type: string;
status: string;
start_dt: string;
end_dt: string | null;
scholar_count: number;
new_publication_count: number;
failed_count: number;
partial_count: number;
}
export interface RunSummary {
succeeded_count: number;
failed_count: number;
partial_count: number;
failed_state_counts: Record<string, number>;
failed_reason_counts: Record<string, number>;
}
export interface RunScholarResult {
scholar_profile_id: number;
scholar_id: string;
state: string;
state_reason: string | null;
outcome: string;
attempt_count: number;
publication_count: number;
start_cstart: number;
continuation_cstart: number | null;
continuation_enqueued: boolean;
continuation_cleared: boolean;
warnings: string[];
error: string | null;
debug: Record<string, unknown> | null;
}
export interface RunDetail {
run: RunListItem;
summary: RunSummary;
scholar_results: RunScholarResult[];
}
export interface QueueItem {
id: number;
scholar_profile_id: number;
scholar_label: string;
status: string;
reason: string;
dropped_reason: string | null;
attempt_count: number;
resume_cstart: number;
next_attempt_dt: string | null;
updated_at: string;
last_error: string | null;
last_run_id: number | null;
}
interface RunsListData {
runs: RunListItem[];
}
interface QueueListData {
queue_items: QueueItem[];
}
export interface RunsListQuery {
failedOnly?: boolean;
limit?: number;
}
export async function listRuns(query: RunsListQuery = {}): Promise<RunListItem[]> {
const params = new URLSearchParams();
if (query.failedOnly) {
params.set("failed_only", "true");
}
if (query.limit) {
params.set("limit", String(query.limit));
}
const suffix = params.toString();
const response = await apiRequest<RunsListData>(`/runs${suffix ? `?${suffix}` : ""}`, {
method: "GET",
});
return response.data.runs;
}
export async function getRunDetail(runId: number): Promise<RunDetail> {
const response = await apiRequest<RunDetail>(`/runs/${runId}`, { method: "GET" });
return response.data;
}
function generateIdempotencyKey(): string {
const randomUuid = globalThis.crypto?.randomUUID?.();
if (randomUuid) {
return randomUuid;
}
return `${Date.now()}-${Math.random().toString(16).slice(2)}`;
}
export async function triggerManualRun(): Promise<{
run_id: number;
status: string;
scholar_count: number;
succeeded_count: number;
failed_count: number;
partial_count: number;
new_publication_count: number;
reused_existing_run: boolean;
idempotency_key: string | null;
}> {
const headers: Record<string, string> = {
"Idempotency-Key": generateIdempotencyKey(),
};
const response = await apiRequest<{
run_id: number;
status: string;
scholar_count: number;
succeeded_count: number;
failed_count: number;
partial_count: number;
new_publication_count: number;
reused_existing_run: boolean;
idempotency_key: string | null;
}>("/runs/manual", {
method: "POST",
headers,
});
return response.data;
}
export async function listQueueItems(limit = 200): Promise<QueueItem[]> {
const response = await apiRequest<QueueListData>(`/runs/queue/items?limit=${limit}`, {
method: "GET",
});
return response.data.queue_items;
}
export async function retryQueueItem(queueItemId: number): Promise<QueueItem> {
const response = await apiRequest<QueueItem>(`/runs/queue/${queueItemId}/retry`, {
method: "POST",
});
return response.data;
}
export async function dropQueueItem(queueItemId: number): Promise<QueueItem> {
const response = await apiRequest<QueueItem>(`/runs/queue/${queueItemId}/drop`, {
method: "POST",
});
return response.data;
}
export async function clearQueueItem(queueItemId: number): Promise<{
queue_item_id: number;
previous_status: string;
status: string;
message: string;
}> {
const response = await apiRequest<{
queue_item_id: number;
previous_status: string;
status: string;
message: string;
}>(`/runs/queue/${queueItemId}`, {
method: "DELETE",
});
return response.data;
}

117
frontend/src/features/scholars/index.ts Normal file
View file

@ -0,0 +1,117 @@
import { apiRequest } from "@/lib/api/client";
export interface ScholarProfile {
id: number;
scholar_id: string;
display_name: string | null;
profile_image_url: string | null;
profile_image_source: "upload" | "override" | "scraped" | "none";
is_enabled: boolean;
baseline_completed: boolean;
last_run_dt: string | null;
last_run_status: string | null;
}
export interface ScholarCreatePayload {
scholar_id: string;
profile_image_url?: string;
}
export interface ScholarSearchCandidate {
scholar_id: string;
display_name: string;
affiliation: string | null;
email_domain: string | null;
cited_by_count: number | null;
interests: string[];
profile_url: string;
profile_image_url: string | null;
}
export interface ScholarSearchResult {
query: string;
state: "ok" | "no_results" | "blocked_or_captcha" | "layout_changed" | "network_error";
state_reason: string;
action_hint: string | null;
candidates: ScholarSearchCandidate[];
warnings: string[];
}
interface ScholarsListData {
scholars: ScholarProfile[];
}
interface ScholarSearchData extends ScholarSearchResult {}
export async function listScholars(): Promise<ScholarProfile[]> {
const response = await apiRequest<ScholarsListData>("/scholars", { method: "GET" });
return response.data.scholars;
}
export async function createScholar(payload: ScholarCreatePayload): Promise<ScholarProfile> {
const response = await apiRequest<ScholarProfile>("/scholars", {
method: "POST",
body: payload,
});
return response.data;
}
export async function toggleScholar(scholarProfileId: number): Promise<ScholarProfile> {
const response = await apiRequest<ScholarProfile>(`/scholars/${scholarProfileId}/toggle`, {
method: "PATCH",
});
return response.data;
}
export async function deleteScholar(scholarProfileId: number): Promise<void> {
await apiRequest<{ message: string }>(`/scholars/${scholarProfileId}`, {
method: "DELETE",
});
}
export async function searchScholarsByName(
query: string,
limit = 10,
): Promise<ScholarSearchResult> {
const searchParams = new URLSearchParams({
query,
limit: String(limit),
});
const response = await apiRequest<ScholarSearchData>(`/scholars/search?${searchParams.toString()}`, {
method: "GET",
});
return response.data;
}
export async function setScholarImageUrl(
scholarProfileId: number,
imageUrl: string,
): Promise<ScholarProfile> {
const response = await apiRequest<ScholarProfile>(`/scholars/${scholarProfileId}/image/url`, {
method: "PUT",
body: { image_url: imageUrl },
});
return response.data;
}
export async function uploadScholarImage(
scholarProfileId: number,
file: File,
): Promise<ScholarProfile> {
const form = new FormData();
form.append("image", file);
const response = await apiRequest<ScholarProfile>(`/scholars/${scholarProfileId}/image/upload`, {
method: "POST",
body: form,
});
return response.data;
}
export async function clearScholarImage(
scholarProfileId: number,
): Promise<ScholarProfile> {
const response = await apiRequest<ScholarProfile>(`/scholars/${scholarProfileId}/image`, {
method: "DELETE",
});
return response.data;
}

34
frontend/src/features/settings/index.ts Normal file
View file

@ -0,0 +1,34 @@
import { apiRequest } from "@/lib/api/client";
export interface UserSettings {
auto_run_enabled: boolean;
run_interval_minutes: number;
request_delay_seconds: number;
}
export interface ChangePasswordPayload {
current_password: string;
new_password: string;
confirm_password: string;
}
export async function fetchSettings(): Promise<UserSettings> {
const response = await apiRequest<UserSettings>("/settings", { method: "GET" });
return response.data;
}
export async function updateSettings(payload: UserSettings): Promise<UserSettings> {
const response = await apiRequest<UserSettings>("/settings", {
method: "PUT",
body: payload,
});
return response.data;
}
export async function changePassword(payload: ChangePasswordPayload): Promise<{ message: string }> {
const response = await apiRequest<{ message: string }>("/auth/change-password", {
method: "POST",
body: payload,
});
return response.data;
}

102
frontend/src/lib/api/client.ts Normal file
View file

@ -0,0 +1,102 @@
import {
isErrorEnvelope,
isSuccessEnvelope,
readRequestId,
type ApiSuccessEnvelope,
} from "@/lib/api/envelope";
import { ApiRequestError } from "@/lib/api/errors";
export type HttpMethod = "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
export interface ApiRequestOptions {
method?: HttpMethod;
body?: unknown | FormData;
headers?: Record<string, string>;
}
const UNSAFE_METHODS = new Set<HttpMethod>(["POST", "PUT", "PATCH", "DELETE"]);
const API_BASE = "/api/v1";
let csrfTokenProvider: (() => string | null) | null = null;
export function setCsrfTokenProvider(provider: () => string | null): void {
csrfTokenProvider = provider;
}
export async function apiRequest<T>(path: string, options: ApiRequestOptions = {}): Promise<ApiSuccessEnvelope<T>> {
const method = options.method ?? "GET";
const headers = new Headers(options.headers ?? {});
headers.set("Accept", "application/json");
const hasBody = options.body !== undefined;
const isFormData = typeof FormData !== "undefined" && options.body instanceof FormData;
let requestBody: BodyInit | undefined;
if (hasBody && !isFormData && !headers.has("Content-Type")) {
headers.set("Content-Type", "application/json");
}
if (hasBody) {
requestBody = isFormData ? (options.body as FormData) : JSON.stringify(options.body);
}
if (UNSAFE_METHODS.has(method) && csrfTokenProvider) {
const csrfToken = csrfTokenProvider();
if (csrfToken) {
headers.set("X-CSRF-Token", csrfToken);
}
}
const response = await fetch(`${API_BASE}${path}`, {
method,
headers,
credentials: "include",
body: requestBody,
});
const raw = await parseResponseBody(response);
const requestId = readRequestId(raw) ?? response.headers.get("X-Request-ID");
if (!response.ok) {
if (isErrorEnvelope(raw)) {
throw new ApiRequestError({
status: response.status,
code: raw.error.code || "error",
message: raw.error.message || "Request failed.",
details: raw.error.details,
requestId,
});
}
throw new ApiRequestError({
status: response.status,
code: "http_error",
message: `Request failed with status ${response.status}.`,
requestId,
});
}
if (!isSuccessEnvelope<T>(raw)) {
throw new ApiRequestError({
status: response.status,
code: "invalid_envelope",
message: "Server returned an unexpected response format.",
requestId,
details: raw,
});
}
return raw;
}
async function parseResponseBody(response: Response): Promise<unknown> {
const contentType = response.headers.get("Content-Type") || "";
if (!contentType.includes("application/json")) {
return null;
}
try {
return await response.json();
} catch (_err) {
return null;
}
}

10
frontend/src/lib/api/csrf.ts Normal file
View file

@ -0,0 +1,10 @@
import { apiRequest } from "@/lib/api/client";
export interface CsrfBootstrapData {
csrf_token: string;
authenticated: boolean;
}
export async function fetchCsrfBootstrap() {
return apiRequest<CsrfBootstrapData>("/auth/csrf", { method: "GET" });
}

43
frontend/src/lib/api/envelope.test.ts Normal file
View file

@ -0,0 +1,43 @@
import { describe, expect, it } from "vitest";
import {
isErrorEnvelope,
isSuccessEnvelope,
readRequestId,
type ApiSuccessEnvelope,
} from "@/lib/api/envelope";
describe("api envelope helpers", () => {
it("recognizes a success envelope", () => {
const payload: ApiSuccessEnvelope<{ ok: boolean }> = {
data: { ok: true },
meta: { request_id: "req_123" },
};
expect(isSuccessEnvelope(payload)).toBe(true);
expect(isErrorEnvelope(payload)).toBe(false);
expect(readRequestId(payload)).toBe("req_123");
});
it("recognizes an error envelope", () => {
const payload = {
error: {
code: "invalid",
message: "Invalid request",
details: null,
},
meta: { request_id: "req_456" },
};
expect(isErrorEnvelope(payload)).toBe(true);
expect(isSuccessEnvelope(payload)).toBe(false);
expect(readRequestId(payload)).toBe("req_456");
});
it("returns null when request id is missing or invalid", () => {
expect(readRequestId(null)).toBeNull();
expect(readRequestId({})).toBeNull();
expect(readRequestId({ meta: {} })).toBeNull();
expect(readRequestId({ meta: { request_id: "" } })).toBeNull();
});
});

44
frontend/src/lib/api/envelope.ts Normal file
View file

@ -0,0 +1,44 @@
import type { ApiErrorPayload } from "@/lib/api/errors";
export interface ApiMeta {
request_id: string | null;
}
export interface ApiSuccessEnvelope<T> {
data: T;
meta: ApiMeta;
}
export interface ApiErrorEnvelope {
error: ApiErrorPayload;
meta: ApiMeta;
}
export function isSuccessEnvelope<T>(value: unknown): value is ApiSuccessEnvelope<T> {
if (typeof value !== "object" || value === null) {
return false;
}
return "data" in value && "meta" in value;
}
export function isErrorEnvelope(value: unknown): value is ApiErrorEnvelope {
if (typeof value !== "object" || value === null) {
return false;
}
return "error" in value && "meta" in value;
}
export function readRequestId(payload: unknown): string | null {
if (typeof payload !== "object" || payload === null) {
return null;
}
const meta = (payload as { meta?: unknown }).meta;
if (typeof meta !== "object" || meta === null) {
return null;
}
const requestId = (meta as { request_id?: unknown }).request_id;
if (typeof requestId !== "string" || !requestId.trim()) {
return null;
}
return requestId;
}

22
frontend/src/lib/api/errors.test.ts Normal file
View file

@ -0,0 +1,22 @@
import { describe, expect, it } from "vitest";
import { ApiRequestError } from "@/lib/api/errors";
describe("ApiRequestError", () => {
it("preserves structured metadata", () => {
const error = new ApiRequestError({
status: 409,
code: "run_in_progress",
message: "A run is already in progress",
details: { run_id: 42 },
requestId: "req_789",
});
expect(error.name).toBe("ApiRequestError");
expect(error.status).toBe(409);
expect(error.code).toBe("run_in_progress");
expect(error.message).toBe("A run is already in progress");
expect(error.details).toEqual({ run_id: 42 });
expect(error.requestId).toBe("req_789");
});
});

27
frontend/src/lib/api/errors.ts Normal file
View file

@ -0,0 +1,27 @@
export interface ApiErrorPayload {
code: string;
message: string;
details: unknown;
}
export class ApiRequestError extends Error {
readonly status: number;
readonly code: string;
readonly details: unknown;
readonly requestId: string | null;
constructor(params: {
status: number;
code: string;
message: string;
details?: unknown;
requestId?: string | null;
}) {
super(params.message);
this.name = "ApiRequestError";
this.status = params.status;
this.code = params.code;
this.details = params.details ?? null;
this.requestId = params.requestId ?? null;
}
}

35
frontend/src/lib/auth/session.ts Normal file
View file

@ -0,0 +1,35 @@
import { apiRequest } from "@/lib/api/client";
export interface SessionUser {
id: number;
email: string;
is_admin: boolean;
is_active: boolean;
}
export interface AuthSessionData {
authenticated: boolean;
csrf_token: string;
user: SessionUser;
}
export interface MessageData {
message: string;
}
export async function fetchMe() {
return apiRequest<AuthSessionData>("/auth/me", { method: "GET" });
}
export async function loginSession(params: { email: string; password: string }) {
return apiRequest<AuthSessionData>("/auth/login", {
method: "POST",
body: params,
});
}
export async function logoutSession() {
return apiRequest<MessageData>("/auth/logout", {
method: "POST",
});
}

8
frontend/src/lib/utils/request_id.ts Normal file
View file

@ -0,0 +1,8 @@
import { ApiRequestError } from "@/lib/api/errors";
export function toRequestId(value: unknown): string | null {
if (value instanceof ApiRequestError) {
return value.requestId;
}
return null;
}

22
frontend/src/main.ts Normal file
View file

@ -0,0 +1,22 @@
import { createApp } from "vue";
import { createPinia } from "pinia";
import AppShell from "@/app/AppShell.vue";
import router from "@/app/router";
import { bootstrapAppProviders } from "@/app/providers";
import "@/styles.css";
async function main(): Promise<void> {
const app = createApp(AppShell);
const pinia = createPinia();
app.use(pinia);
await bootstrapAppProviders();
app.use(router);
await router.isReady();
app.mount("#app");
}
void main();

199
frontend/src/pages/AdminUsersPage.vue Normal file
View file

@ -0,0 +1,199 @@
<script setup lang="ts">
import { onMounted, ref } from "vue";
import AppPage from "@/components/layout/AppPage.vue";
import AppAlert from "@/components/ui/AppAlert.vue";
import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue";
import AppCheckbox from "@/components/ui/AppCheckbox.vue";
import AppEmptyState from "@/components/ui/AppEmptyState.vue";
import AppInput from "@/components/ui/AppInput.vue";
import AppSkeleton from "@/components/ui/AppSkeleton.vue";
import AppTable from "@/components/ui/AppTable.vue";
import {
createAdminUser,
listAdminUsers,
setAdminUserActive,
type AdminUser,
} from "@/features/admin_users";
import { ApiRequestError } from "@/lib/api/errors";
const loading = ref(true);
const creating = ref(false);
const togglingUserId = ref<number | null>(null);
const users = ref<AdminUser[]>([]);
const email = ref("");
const password = ref("");
const createIsAdmin = ref(false);
const errorMessage = ref<string | null>(null);
const errorRequestId = ref<string | null>(null);
const successMessage = ref<string | null>(null);
function formatDate(value: string): string {
const asDate = new Date(value);
if (Number.isNaN(asDate.getTime())) {
return value;
}
return asDate.toLocaleString();
}
async function loadUsers(): Promise<void> {
loading.value = true;
errorMessage.value = null;
errorRequestId.value = null;
try {
users.value = await listAdminUsers();
} catch (error) {
users.value = [];
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to load users.";
}
} finally {
loading.value = false;
}
}
async function onCreateUser(): Promise<void> {
creating.value = true;
successMessage.value = null;
errorMessage.value = null;
errorRequestId.value = null;
try {
if (!email.value.trim() || !password.value) {
throw new Error("Email and password are required.");
}
const created = await createAdminUser({
email: email.value.trim(),
password: password.value,
is_admin: createIsAdmin.value,
});
email.value = "";
password.value = "";
createIsAdmin.value = false;
successMessage.value = `User created: ${created.email}`;
await loadUsers();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else if (error instanceof Error) {
errorMessage.value = error.message;
} else {
errorMessage.value = "Unable to create user.";
}
} finally {
creating.value = false;
}
}
async function onToggleUser(user: AdminUser): Promise<void> {
togglingUserId.value = user.id;
successMessage.value = null;
try {
const updated = await setAdminUserActive(user.id, !user.is_active);
successMessage.value = `${updated.email} is now ${updated.is_active ? "active" : "inactive"}.`;
await loadUsers();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to update user.";
}
} finally {
togglingUserId.value = null;
}
}
onMounted(() => {
void loadUsers();
});
</script>
<template>
<AppPage title="Admin Users" subtitle="Create and manage user access for this instance.">
<AppAlert v-if="successMessage" tone="success" dismissible @dismiss="successMessage = null">
<template #title>Operation complete</template>
<p>{{ successMessage }}</p>
</AppAlert>
<AppAlert v-if="errorMessage" tone="danger">
<template #title>User management request failed</template>
<p>{{ errorMessage }}</p>
<p class="text-secondary">Request ID: {{ errorRequestId || "n/a" }}</p>
</AppAlert>
<section class="grid gap-4 lg:grid-cols-2">
<AppCard class="space-y-4">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Create User</h2>
<form class="grid gap-3" @submit.prevent="onCreateUser">
<label class="grid gap-2 text-sm font-medium text-zinc-700 dark:text-zinc-300">
<span>Email</span>
<AppInput id="admin-user-email" v-model="email" type="email" autocomplete="off" />
</label>
<label class="grid gap-2 text-sm font-medium text-zinc-700 dark:text-zinc-300">
<span>Password</span>
<AppInput id="admin-user-password" v-model="password" type="password" autocomplete="new-password" />
</label>
<AppCheckbox id="admin-user-is-admin" v-model="createIsAdmin" label="Grant admin privileges" />
<AppButton type="submit" :disabled="creating">
{{ creating ? "Creating..." : "Create user" }}
</AppButton>
</form>
</AppCard>
<AppCard class="space-y-4">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Users</h2>
<AppSkeleton v-if="loading" :lines="5" />
<AppEmptyState
v-else-if="users.length === 0"
title="No users available"
body="Create an account to begin assigning access."
/>
<AppTable v-else label="Admin users table">
<thead>
<tr>
<th scope="col">Email</th>
<th scope="col">Role</th>
<th scope="col">Active</th>
<th scope="col">Updated</th>
<th scope="col">Action</th>
</tr>
</thead>
<tbody>
<tr v-for="user in users" :key="user.id">
<td>{{ user.email }}</td>
<td>{{ user.is_admin ? "Admin" : "User" }}</td>
<td>{{ user.is_active ? "Yes" : "No" }}</td>
<td>{{ formatDate(user.updated_at) }}</td>
<td>
<AppButton
variant="secondary"
:disabled="togglingUserId === user.id"
@click="onToggleUser(user)"
>
{{ user.is_active ? "Deactivate" : "Activate" }}
</AppButton>
</td>
</tr>
</tbody>
</AppTable>
</AppCard>
</section>
</AppPage>
</template>

187
frontend/src/pages/DashboardPage.vue Normal file
View file

@ -0,0 +1,187 @@
<script setup lang="ts">
import { onMounted, ref } from "vue";
import { fetchDashboardSnapshot, type DashboardSnapshot } from "@/features/dashboard";
import { ApiRequestError } from "@/lib/api/errors";
import AppPage from "@/components/layout/AppPage.vue";
import QueueHealthBadge from "@/components/patterns/QueueHealthBadge.vue";
import RunStatusBadge from "@/components/patterns/RunStatusBadge.vue";
import AppAlert from "@/components/ui/AppAlert.vue";
import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue";
import AppEmptyState from "@/components/ui/AppEmptyState.vue";
import AppSkeleton from "@/components/ui/AppSkeleton.vue";
import { useAuthStore } from "@/stores/auth";
const loading = ref(true);
const errorMessage = ref<string | null>(null);
const errorRequestId = ref<string | null>(null);
const snapshot = ref<DashboardSnapshot | null>(null);
const auth = useAuthStore();
function formatDate(value: string | null): string {
if (!value) {
return "n/a";
}
const asDate = new Date(value);
if (Number.isNaN(asDate.getTime())) {
return value;
}
return asDate.toLocaleString();
}
async function loadSnapshot(): Promise<void> {
loading.value = true;
errorMessage.value = null;
errorRequestId.value = null;
try {
snapshot.value = await fetchDashboardSnapshot();
} catch (error) {
snapshot.value = null;
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to load dashboard data.";
}
} finally {
loading.value = false;
}
}
onMounted(() => {
void loadSnapshot();
});
</script>
<template>
<AppPage title="Dashboard" subtitle="What changed since the latest run.">
<div class="flex flex-wrap items-center justify-between gap-3">
<AppButton variant="secondary" @click="loadSnapshot" :disabled="loading">
{{ loading ? "Refreshing..." : "Refresh" }}
</AppButton>
<QueueHealthBadge
v-if="snapshot"
:queued="snapshot.queue.queued"
:retrying="snapshot.queue.retrying"
:dropped="snapshot.queue.dropped"
/>
</div>
<AppAlert v-if="errorMessage" tone="danger">
<template #title>Dashboard request failed</template>
<p>{{ errorMessage }}</p>
<p class="text-secondary">Request ID: {{ errorRequestId || "n/a" }}</p>
</AppAlert>
<AppSkeleton v-if="loading" :lines="6" />
<template v-else-if="snapshot">
<section class="grid gap-4 xl:grid-cols-[minmax(0,1fr)_22rem]">
<AppCard class="space-y-4">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Snapshot Summary</h2>
<p class="text-sm text-secondary">Latest view mode: {{ snapshot.mode }}</p>
<ul class="grid gap-2 text-sm text-zinc-700 dark:text-zinc-300">
<li class="rounded-xl border border-zinc-200 bg-zinc-50 px-3 py-2 dark:border-zinc-800 dark:bg-zinc-900/60">
Latest run introduced
<span class="font-semibold text-zinc-900 dark:text-zinc-100">{{ snapshot.newCount }}</span>
publications.
</li>
<li class="rounded-xl border border-zinc-200 bg-zinc-50 px-3 py-2 dark:border-zinc-800 dark:bg-zinc-900/60">
Total tracked publications:
<span class="font-semibold text-zinc-900 dark:text-zinc-100">{{ snapshot.totalCount }}</span>
</li>
</ul>
</AppCard>
<AppCard class="space-y-4">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Latest Run</h2>
<div v-if="snapshot.latestRun" class="space-y-2 text-sm">
<div class="flex flex-wrap items-center justify-between gap-2">
<RunStatusBadge :status="snapshot.latestRun.status" />
<RouterLink
v-if="auth.isAdmin"
:to="`/admin/runs/${snapshot.latestRun.id}`"
class="link-inline"
>
Run #{{ snapshot.latestRun.id }}
</RouterLink>
<span v-else class="text-secondary">Run #{{ snapshot.latestRun.id }}</span>
</div>
<p class="text-secondary">Started: {{ formatDate(snapshot.latestRun.start_dt) }}</p>
<p class="text-muted">
{{ snapshot.latestRun.scholar_count }} scholars, {{ snapshot.latestRun.new_publication_count }} new
publications
</p>
</div>
<AppEmptyState
v-else
title="No runs recorded"
body="Trigger a manual run from the Runs screen to begin tracking."
/>
</AppCard>
</section>
<section class="grid gap-4 xl:grid-cols-2">
<AppCard class="space-y-4">
<div class="flex flex-wrap items-center justify-between gap-2">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Recent Publications</h2>
<RouterLink
to="/publications"
class="link-inline text-sm"
>
Open publications
</RouterLink>
</div>
<AppEmptyState
v-if="snapshot.recentPublications.length === 0"
title="No new publications"
body="When a completed run discovers updates, they will appear here."
/>
<ul v-else class="grid gap-3">
<li
v-for="item in snapshot.recentPublications.slice(0, 6)"
:key="item.publication_id"
class="grid gap-1 rounded-xl border border-zinc-200 bg-zinc-50 px-3 py-2 dark:border-zinc-800 dark:bg-zinc-900/60"
>
<strong class="text-zinc-900 dark:text-zinc-100">{{ item.title }}</strong>
<span class="text-secondary">{{ item.scholar_label }}</span>
</li>
</ul>
</AppCard>
<AppCard class="space-y-4">
<div class="flex flex-wrap items-center justify-between gap-2">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Recent Runs</h2>
<RouterLink
v-if="auth.isAdmin"
to="/admin/runs"
class="link-inline text-sm"
>
Open runs
</RouterLink>
</div>
<AppEmptyState
v-if="snapshot.recentRuns.length === 0"
title="No runs yet"
body="Trigger a manual run from the Runs screen to begin tracking."
/>
<ul v-else class="grid gap-3">
<li
v-for="run in snapshot.recentRuns"
:key="run.id"
class="grid gap-1 rounded-xl border border-zinc-200 bg-zinc-50 px-3 py-2 dark:border-zinc-800 dark:bg-zinc-900/60"
>
<div class="flex items-center gap-2">
<RunStatusBadge :status="run.status" />
<span class="text-secondary">Run #{{ run.id }}</span>
</div>
<span class="text-secondary">{{ formatDate(run.start_dt) }}</span>
</li>
</ul>
</AppCard>
</section>
</template>
</AppPage>
</template>

129
frontend/src/pages/LoginPage.vue Normal file
View file

@ -0,0 +1,129 @@
<script setup lang="ts">
import { computed, ref } from "vue";
import { useRouter } from "vue-router";
import AppAlert from "@/components/ui/AppAlert.vue";
import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue";
import AppInput from "@/components/ui/AppInput.vue";
import { ApiRequestError } from "@/lib/api/errors";
import { useAuthStore } from "@/stores/auth";
const router = useRouter();
const auth = useAuthStore();
const email = ref("");
const password = ref("");
const pending = ref(false);
const errorMessage = ref<string | null>(null);
const errorRequestId = ref<string | null>(null);
const retryAfterSeconds = ref<number | null>(null);
const canSubmit = computed(
() => !pending.value && email.value.trim().length > 0 && password.value.length > 0,
);
async function onSubmit(): Promise<void> {
if (!canSubmit.value) {
return;
}
pending.value = true;
errorMessage.value = null;
errorRequestId.value = null;
retryAfterSeconds.value = null;
try {
await auth.login(email.value.trim(), password.value);
await router.replace({ name: "dashboard" });
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
const details = error.details as { retry_after_seconds?: unknown } | null;
if (typeof details?.retry_after_seconds === "number") {
retryAfterSeconds.value = details.retry_after_seconds;
}
} else {
errorMessage.value = "Unable to sign in. Please try again.";
}
} finally {
pending.value = false;
}
}
</script>
<template>
<div class="relative min-h-screen overflow-hidden bg-zinc-100 dark:bg-zinc-950">
<div class="pointer-events-none absolute inset-0">
<div class="absolute -top-20 left-[-8rem] h-72 w-72 rounded-full bg-brand-300/35 blur-3xl dark:bg-brand-900/35" />
<div class="absolute bottom-[-7rem] right-[-6rem] h-80 w-80 rounded-full bg-emerald-300/25 blur-3xl dark:bg-emerald-900/25" />
<div class="absolute left-1/3 top-1/3 h-52 w-52 rounded-full bg-amber-300/20 blur-3xl dark:bg-amber-900/20" />
</div>
<div
class="relative mx-auto grid min-h-screen w-full max-w-6xl items-center gap-8 px-4 py-10 sm:px-6 lg:grid-cols-[minmax(0,1fr)_26rem] lg:px-8"
>
<section class="hidden space-y-5 lg:block">
<p class="inline-flex items-center rounded-full border border-zinc-300 bg-white/70 px-3 py-1 text-xs font-medium uppercase tracking-[0.12em] text-zinc-600 dark:border-zinc-700 dark:bg-zinc-900/70 dark:text-zinc-300">
Scholarr Control Center
</p>
<h1 class="max-w-xl font-display text-4xl font-semibold tracking-tight text-zinc-900 dark:text-zinc-100">
Scholar tracking with reliable operational controls.
</h1>
<p class="max-w-xl text-base leading-relaxed text-zinc-600 dark:text-zinc-400">
Use your account to review ingestion runs, publication changes, and continuation queue diagnostics from a
single workspace.
</p>
<ul class="grid max-w-xl gap-2 text-sm text-zinc-600 dark:text-zinc-400">
<li class="rounded-lg border border-zinc-200 bg-white/60 px-3 py-2 dark:border-zinc-800 dark:bg-zinc-900/60">
Session-based authentication with CSRF enforcement.
</li>
<li class="rounded-lg border border-zinc-200 bg-white/60 px-3 py-2 dark:border-zinc-800 dark:bg-zinc-900/60">
Run and queue diagnostics are available for support workflows.
</li>
<li class="rounded-lg border border-zinc-200 bg-white/60 px-3 py-2 dark:border-zinc-800 dark:bg-zinc-900/60">
Theme-aware interface with light and dark mode support.
</li>
</ul>
</section>
<AppCard class="w-full max-w-md justify-self-end space-y-6 border-zinc-200/80 bg-white/90 backdrop-blur dark:border-zinc-800 dark:bg-zinc-900/85">
<div class="space-y-2">
<h1 class="font-display text-3xl font-semibold tracking-tight text-zinc-900 dark:text-zinc-100">Sign In</h1>
<p class="text-sm text-secondary">
Sign in to access scholar tracking, publication updates, and run diagnostics.
</p>
</div>
<AppAlert v-if="errorMessage" tone="danger">
<template #title>Login failed</template>
<p>{{ errorMessage }}</p>
<p v-if="retryAfterSeconds !== null" class="text-secondary">Retry after {{ retryAfterSeconds }} seconds.</p>
<p class="text-secondary">Request ID: {{ errorRequestId || "n/a" }}</p>
</AppAlert>
<form class="grid gap-4" @submit.prevent="onSubmit">
<label class="grid gap-2 text-sm font-medium text-zinc-700 dark:text-zinc-300">
<span>Email</span>
<AppInput id="login-email" v-model="email" type="email" autocomplete="email" />
</label>
<label class="grid gap-2 text-sm font-medium text-zinc-700 dark:text-zinc-300">
<span>Password</span>
<AppInput
id="login-password"
v-model="password"
type="password"
autocomplete="current-password"
/>
</label>
<AppButton type="submit" :disabled="!canSubmit" class="w-full justify-center">
{{ pending ? "Signing in..." : "Sign in" }}
</AppButton>
</form>
</AppCard>
</div>
</div>
</template>

449
frontend/src/pages/PublicationsPage.vue Normal file
View file

@ -0,0 +1,449 @@
<script setup lang="ts">
import { computed, onMounted, ref, watch } from "vue";
import { useRoute, useRouter } from "vue-router";
import AppPage from "@/components/layout/AppPage.vue";
import AppAlert from "@/components/ui/AppAlert.vue";
import AppBadge from "@/components/ui/AppBadge.vue";
import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue";
import AppEmptyState from "@/components/ui/AppEmptyState.vue";
import AppInput from "@/components/ui/AppInput.vue";
import AppSelect from "@/components/ui/AppSelect.vue";
import AppSkeleton from "@/components/ui/AppSkeleton.vue";
import AppTable from "@/components/ui/AppTable.vue";
import {
listPublications,
markAllRead,
markSelectedRead,
type PublicationItem,
type PublicationMode,
type PublicationsResult,
} from "@/features/publications";
import { listScholars, type ScholarProfile } from "@/features/scholars";
import { ApiRequestError } from "@/lib/api/errors";
const loading = ref(true);
const publishingAll = ref(false);
const publishingSelected = ref(false);
const mode = ref<PublicationMode>("new");
const selectedScholarFilter = ref("");
const searchQuery = ref("");
const scholars = ref<ScholarProfile[]>([]);
const listState = ref<PublicationsResult | null>(null);
const selectedPublicationKeys = ref<Set<string>>(new Set());
const errorMessage = ref<string | null>(null);
const errorRequestId = ref<string | null>(null);
const successMessage = ref<string | null>(null);
const route = useRoute();
const router = useRouter();
function normalizeScholarFilterQuery(value: unknown): string {
if (Array.isArray(value)) {
return normalizeScholarFilterQuery(value[0]);
}
if (typeof value !== "string") {
return "";
}
const parsed = Number(value);
return Number.isInteger(parsed) && parsed > 0 ? String(parsed) : "";
}
function syncScholarFilterFromRoute(): boolean {
const nextValue = normalizeScholarFilterQuery(route.query.scholar);
if (selectedScholarFilter.value === nextValue) {
return false;
}
selectedScholarFilter.value = nextValue;
return true;
}
async function syncScholarFilterToRoute(): Promise<void> {
const nextValue = selectedScholarFilter.value.trim();
const currentValue = normalizeScholarFilterQuery(route.query.scholar);
if (nextValue === currentValue) {
return;
}
const nextQuery = {
...route.query,
scholar: nextValue || undefined,
};
await router.replace({ query: nextQuery });
}
function formatDate(value: string): string {
const asDate = new Date(value);
if (Number.isNaN(asDate.getTime())) {
return value;
}
return asDate.toLocaleDateString();
}
function publicationKey(item: PublicationItem): string {
return `${item.scholar_profile_id}:${item.publication_id}`;
}
const filteredPublications = computed(() => {
const base = listState.value?.publications ?? [];
const normalized = searchQuery.value.trim().toLowerCase();
if (!normalized) {
return base;
}
return base.filter((item) => {
const year = item.year === null ? "" : String(item.year);
return [item.title, item.scholar_label, item.venue_text || "", year]
.join(" ")
.toLowerCase()
.includes(normalized);
});
});
const visibleUnreadKeys = computed(() => {
const keys = new Set<string>();
for (const item of filteredPublications.value) {
if (!item.is_read) {
keys.add(publicationKey(item));
}
}
return keys;
});
const selectedCount = computed(() => selectedPublicationKeys.value.size);
const allVisibleUnreadSelected = computed(() => {
if (visibleUnreadKeys.value.size === 0) {
return false;
}
for (const key of visibleUnreadKeys.value) {
if (!selectedPublicationKeys.value.has(key)) {
return false;
}
}
return true;
});
watch(filteredPublications, (items) => {
const validKeys = new Set(items.filter((item) => !item.is_read).map((item) => publicationKey(item)));
const next = new Set<string>();
for (const key of selectedPublicationKeys.value) {
if (validKeys.has(key)) {
next.add(key);
}
}
if (next.size !== selectedPublicationKeys.value.size) {
selectedPublicationKeys.value = next;
}
});
async function loadScholarFilters(): Promise<void> {
try {
scholars.value = await listScholars();
} catch {
scholars.value = [];
}
}
function selectedScholarId(): number | undefined {
const parsed = Number(selectedScholarFilter.value);
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
}
async function loadPublications(): Promise<void> {
loading.value = true;
errorMessage.value = null;
errorRequestId.value = null;
try {
listState.value = await listPublications({
mode: mode.value,
scholarProfileId: selectedScholarId(),
limit: 400,
});
selectedPublicationKeys.value = new Set();
} catch (error) {
listState.value = null;
selectedPublicationKeys.value = new Set();
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to load publications.";
}
} finally {
loading.value = false;
}
}
async function setMode(nextMode: PublicationMode): Promise<void> {
if (mode.value === nextMode) {
return;
}
mode.value = nextMode;
await loadPublications();
}
async function onScholarFilterChanged(): Promise<void> {
await syncScholarFilterToRoute();
await loadPublications();
}
function onToggleAllVisible(event: Event): void {
const checked = (event.target as HTMLInputElement).checked;
const next = new Set(selectedPublicationKeys.value);
for (const key of visibleUnreadKeys.value) {
if (checked) {
next.add(key);
} else {
next.delete(key);
}
}
selectedPublicationKeys.value = next;
}
function onToggleRowSelection(item: PublicationItem, event: Event): void {
const checked = (event.target as HTMLInputElement).checked;
const key = publicationKey(item);
const next = new Set(selectedPublicationKeys.value);
if (checked) {
next.add(key);
} else {
next.delete(key);
}
selectedPublicationKeys.value = next;
}
async function onMarkSelectedRead(): Promise<void> {
if (selectedPublicationKeys.value.size === 0 || !listState.value) {
return;
}
const selectedLookup = new Set(selectedPublicationKeys.value);
const selections = listState.value.publications
.filter((item) => selectedLookup.has(publicationKey(item)))
.map((item) => ({
scholar_profile_id: item.scholar_profile_id,
publication_id: item.publication_id,
}));
publishingSelected.value = true;
successMessage.value = null;
errorMessage.value = null;
errorRequestId.value = null;
try {
const response = await markSelectedRead(selections);
successMessage.value = `${response.updated_count} publication${response.updated_count === 1 ? "" : "s"} marked as read.`;
await loadPublications();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to mark selected publications as read.";
}
} finally {
publishingSelected.value = false;
}
}
async function onMarkAllRead(): Promise<void> {
publishingAll.value = true;
successMessage.value = null;
errorMessage.value = null;
errorRequestId.value = null;
try {
const response = await markAllRead();
successMessage.value = response.message;
await loadPublications();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to mark publications as read.";
}
} finally {
publishingAll.value = false;
}
}
onMounted(() => {
syncScholarFilterFromRoute();
void Promise.all([loadScholarFilters(), loadPublications()]);
});
watch(
() => route.query.scholar,
async () => {
const changed = syncScholarFilterFromRoute();
if (!changed) {
return;
}
await loadPublications();
},
);
</script>
<template>
<AppPage title="Publications" subtitle="Filter, search, and triage publication status quickly.">
<div class="grid gap-3 xl:grid-cols-[minmax(0,1fr)_auto] xl:items-end">
<div class="grid gap-3 md:grid-cols-[auto_auto_minmax(0,1fr)] md:items-end">
<div class="grid gap-1 text-xs text-secondary">
<span>Scope</span>
<div class="flex min-h-10 flex-wrap items-center gap-2">
<AppButton :variant="mode === 'new' ? 'primary' : 'secondary'" @click="setMode('new')">
New
</AppButton>
<AppButton :variant="mode === 'all' ? 'primary' : 'secondary'" @click="setMode('all')">
All
</AppButton>
</div>
</div>
<label class="grid gap-1 text-xs text-secondary" for="publications-scholar-filter">
<span>Scholar</span>
<AppSelect
id="publications-scholar-filter"
v-model="selectedScholarFilter"
:disabled="loading || publishingAll || publishingSelected"
@change="onScholarFilterChanged"
>
<option value="">All scholars</option>
<option v-for="scholar in scholars" :key="scholar.id" :value="String(scholar.id)">
{{ scholar.display_name || scholar.scholar_id }}
</option>
</AppSelect>
</label>
<label class="grid gap-1 text-xs text-secondary" for="publications-search-input">
<span>Search</span>
<AppInput
id="publications-search-input"
v-model="searchQuery"
placeholder="Search title, scholar, venue, year"
:disabled="loading"
/>
</label>
</div>
<div class="flex flex-wrap items-center gap-2">
<AppButton
variant="secondary"
:disabled="selectedCount === 0 || loading || publishingSelected || publishingAll"
@click="onMarkSelectedRead"
>
{{ publishingSelected ? "Marking..." : `Mark selected read (${selectedCount})` }}
</AppButton>
<AppButton variant="secondary" :disabled="publishingAll || loading || publishingSelected" @click="onMarkAllRead">
{{ publishingAll ? "Marking..." : "Mark all unread as read" }}
</AppButton>
<AppButton variant="ghost" :disabled="loading" @click="loadPublications">
{{ loading ? "Refreshing..." : "Refresh" }}
</AppButton>
</div>
</div>
<AppAlert v-if="successMessage" tone="success" dismissible @dismiss="successMessage = null">
<template #title>Update complete</template>
<p>{{ successMessage }}</p>
</AppAlert>
<AppAlert v-if="errorMessage" tone="danger">
<template #title>Publication request failed</template>
<p>{{ errorMessage }}</p>
<p class="text-secondary">Request ID: {{ errorRequestId || "n/a" }}</p>
</AppAlert>
<AppSkeleton v-if="loading" :lines="8" />
<template v-else-if="listState">
<AppCard class="space-y-4">
<div class="flex flex-wrap items-center justify-between gap-2">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Publication list</h2>
<div class="flex flex-wrap items-center gap-2">
<AppBadge tone="info">Mode: {{ listState.mode }}</AppBadge>
<AppBadge tone="neutral">Visible: {{ filteredPublications.length }}</AppBadge>
</div>
</div>
<AppEmptyState
v-if="filteredPublications.length === 0"
title="No publications found"
body="Try changing mode, scholar filter, or search terms."
/>
<AppTable v-else label="Publication list table">
<thead>
<tr>
<th scope="col" class="w-12">
<input
type="checkbox"
class="h-4 w-4 rounded border-zinc-400 text-brand-600 focus-visible:ring-2 focus-visible:ring-brand-500 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 dark:border-zinc-600 dark:bg-zinc-900 dark:focus-visible:ring-brand-400 dark:focus-visible:ring-offset-zinc-950"
:checked="allVisibleUnreadSelected"
:disabled="visibleUnreadKeys.size === 0"
aria-label="Select all visible unread publications"
@change="onToggleAllVisible"
/>
</th>
<th scope="col">Title</th>
<th scope="col">Scholar</th>
<th scope="col">Year</th>
<th scope="col">Citations</th>
<th scope="col">Status</th>
<th scope="col">First seen</th>
</tr>
</thead>
<tbody>
<tr v-for="item in filteredPublications" :key="publicationKey(item)">
<td>
<input
type="checkbox"
class="h-4 w-4 rounded border-zinc-400 text-brand-600 focus-visible:ring-2 focus-visible:ring-brand-500 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 dark:border-zinc-600 dark:bg-zinc-900 dark:focus-visible:ring-brand-400 dark:focus-visible:ring-offset-zinc-950"
:checked="selectedPublicationKeys.has(publicationKey(item))"
:disabled="item.is_read"
:aria-label="`Select publication ${item.title}`"
@change="onToggleRowSelection(item, $event)"
/>
</td>
<td>
<a
v-if="item.pub_url"
:href="item.pub_url"
target="_blank"
rel="noreferrer"
class="link-inline"
>
{{ item.title }}
</a>
<span v-else>{{ item.title }}</span>
</td>
<td>{{ item.scholar_label }}</td>
<td>{{ item.year ?? "n/a" }}</td>
<td>{{ item.citation_count }}</td>
<td>
<div class="flex flex-wrap items-center gap-2">
<AppBadge :tone="item.is_new_in_latest_run ? 'info' : 'neutral'">
{{ item.is_new_in_latest_run ? "New" : "Existing" }}
</AppBadge>
<AppBadge :tone="item.is_read ? 'success' : 'warning'">
{{ item.is_read ? "Read" : "Unread" }}
</AppBadge>
</div>
</td>
<td>{{ formatDate(item.first_seen_at) }}</td>
</tr>
</tbody>
</AppTable>
</AppCard>
</template>
</AppPage>
</template>

134
frontend/src/pages/RunDetailPage.vue Normal file
View file

@ -0,0 +1,134 @@
<script setup lang="ts">
import { computed, onMounted, ref } from "vue";
import { useRoute } from "vue-router";
import AppPage from "@/components/layout/AppPage.vue";
import RunStatusBadge from "@/components/patterns/RunStatusBadge.vue";
import AppAlert from "@/components/ui/AppAlert.vue";
import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue";
import AppEmptyState from "@/components/ui/AppEmptyState.vue";
import AppSkeleton from "@/components/ui/AppSkeleton.vue";
import AppTable from "@/components/ui/AppTable.vue";
import { getRunDetail, type RunDetail } from "@/features/runs";
import { ApiRequestError } from "@/lib/api/errors";
const route = useRoute();
const loading = ref(true);
const detail = ref<RunDetail | null>(null);
const errorMessage = ref<string | null>(null);
const errorRequestId = ref<string | null>(null);
const runId = computed(() => Number(route.params.id));
function formatDate(value: string | null): string {
if (!value) {
return "n/a";
}
const asDate = new Date(value);
if (Number.isNaN(asDate.getTime())) {
return value;
}
return asDate.toLocaleString();
}
async function loadDetail(): Promise<void> {
loading.value = true;
errorMessage.value = null;
errorRequestId.value = null;
if (!Number.isFinite(runId.value) || runId.value <= 0) {
detail.value = null;
errorMessage.value = "Invalid run id.";
loading.value = false;
return;
}
try {
detail.value = await getRunDetail(runId.value);
} catch (error) {
detail.value = null;
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to load run details.";
}
} finally {
loading.value = false;
}
}
onMounted(() => {
void loadDetail();
});
</script>
<template>
<AppPage title="Run Detail" subtitle="Per-scholar diagnostics for the selected run.">
<div class="flex justify-end">
<AppButton variant="secondary" @click="loadDetail" :disabled="loading">
{{ loading ? "Refreshing..." : "Refresh" }}
</AppButton>
</div>
<AppAlert v-if="errorMessage" tone="danger">
<template #title>Run detail request failed</template>
<p>{{ errorMessage }}</p>
<p class="text-secondary">Request ID: {{ errorRequestId || "n/a" }}</p>
</AppAlert>
<AppSkeleton v-if="loading" :lines="8" />
<template v-else-if="detail">
<AppCard class="space-y-4">
<div class="flex flex-wrap items-start justify-between gap-3">
<div class="space-y-1">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Run #{{ detail.run.id }}</h2>
<p class="text-sm text-secondary">Started: {{ formatDate(detail.run.start_dt) }}</p>
<p class="text-sm text-secondary">Ended: {{ formatDate(detail.run.end_dt) }}</p>
</div>
<RunStatusBadge :status="detail.run.status" />
</div>
<p class="text-sm text-zinc-700 dark:text-zinc-300">
Outcome summary: {{ detail.summary.succeeded_count }} succeeded, {{ detail.summary.partial_count }} partial,
{{ detail.summary.failed_count }} failed.
</p>
</AppCard>
<AppCard class="space-y-4">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Scholar Results</h2>
<AppEmptyState
v-if="detail.scholar_results.length === 0"
title="No scholar diagnostics"
body="This run did not include per-scholar diagnostics payloads."
/>
<AppTable v-else label="Run detail scholar diagnostics table">
<thead>
<tr>
<th scope="col">Scholar ID</th>
<th scope="col">Outcome</th>
<th scope="col">State</th>
<th scope="col">Publications</th>
<th scope="col">Attempts</th>
<th scope="col">Warnings</th>
</tr>
</thead>
<tbody>
<tr v-for="result in detail.scholar_results" :key="`${result.scholar_profile_id}-${result.scholar_id}`">
<td><code>{{ result.scholar_id }}</code></td>
<td>{{ result.outcome }}</td>
<td>{{ result.state }}<span v-if="result.state_reason"> ({{ result.state_reason }})</span></td>
<td>{{ result.publication_count }}</td>
<td>{{ result.attempt_count }}</td>
<td>{{ result.warnings.length ? result.warnings.join(", ") : "n/a" }}</td>
</tr>
</tbody>
</AppTable>
</AppCard>
</template>
</AppPage>
</template>

266
frontend/src/pages/RunsPage.vue Normal file
View file

@ -0,0 +1,266 @@
<script setup lang="ts">
import { computed, onMounted, ref } from "vue";
import AppPage from "@/components/layout/AppPage.vue";
import QueueHealthBadge from "@/components/patterns/QueueHealthBadge.vue";
import RunStatusBadge from "@/components/patterns/RunStatusBadge.vue";
import AppAlert from "@/components/ui/AppAlert.vue";
import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue";
import AppEmptyState from "@/components/ui/AppEmptyState.vue";
import AppSkeleton from "@/components/ui/AppSkeleton.vue";
import AppTable from "@/components/ui/AppTable.vue";
import {
clearQueueItem,
dropQueueItem,
listQueueItems,
listRuns,
retryQueueItem,
triggerManualRun,
type QueueItem,
type RunListItem,
} from "@/features/runs";
import { ApiRequestError } from "@/lib/api/errors";
const loading = ref(true);
const pendingRun = ref(false);
const runs = ref<RunListItem[]>([]);
const queueItems = ref<QueueItem[]>([]);
const errorMessage = ref<string | null>(null);
const errorRequestId = ref<string | null>(null);
const successMessage = ref<string | null>(null);
const activeQueueItemId = ref<number | null>(null);
function formatDate(value: string | null): string {
if (!value) {
return "n/a";
}
const asDate = new Date(value);
if (Number.isNaN(asDate.getTime())) {
return value;
}
return asDate.toLocaleString();
}
function queueHealth() {
const counts = { queued: 0, retrying: 0, dropped: 0 };
for (const item of queueItems.value) {
if (item.status === "queued") {
counts.queued += 1;
} else if (item.status === "retrying") {
counts.retrying += 1;
} else if (item.status === "dropped") {
counts.dropped += 1;
}
}
return counts;
}
const queueCounts = computed(() => queueHealth());
async function loadData(): Promise<void> {
loading.value = true;
errorMessage.value = null;
errorRequestId.value = null;
try {
const [loadedRuns, loadedQueue] = await Promise.all([listRuns({ limit: 100 }), listQueueItems(200)]);
runs.value = loadedRuns;
queueItems.value = loadedQueue;
} catch (error) {
runs.value = [];
queueItems.value = [];
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to load runs and queue diagnostics.";
}
} finally {
loading.value = false;
}
}
async function onTriggerManualRun(): Promise<void> {
pendingRun.value = true;
successMessage.value = null;
try {
const result = await triggerManualRun();
successMessage.value = `Manual run queued as #${result.run_id} (${result.status}).`;
await loadData();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to trigger manual run.";
}
} finally {
pendingRun.value = false;
}
}
async function runQueueAction(itemId: number, action: "retry" | "drop" | "clear"): Promise<void> {
activeQueueItemId.value = itemId;
successMessage.value = null;
try {
if (action === "retry") {
await retryQueueItem(itemId);
successMessage.value = `Queue item #${itemId} moved to retry.`;
} else if (action === "drop") {
await dropQueueItem(itemId);
successMessage.value = `Queue item #${itemId} dropped.`;
} else {
await clearQueueItem(itemId);
successMessage.value = `Queue item #${itemId} cleared.`;
}
await loadData();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to update queue item.";
}
} finally {
activeQueueItemId.value = null;
}
}
onMounted(() => {
void loadData();
});
</script>
<template>
<AppPage title="Runs" subtitle="Manual run controls and continuation queue diagnostics.">
<div class="flex flex-wrap items-center justify-between gap-3">
<div class="flex flex-wrap items-center gap-2">
<AppButton :disabled="pendingRun" @click="onTriggerManualRun">
{{ pendingRun ? "Triggering..." : "Run now" }}
</AppButton>
<AppButton variant="secondary" :disabled="loading" @click="loadData">
{{ loading ? "Refreshing..." : "Refresh" }}
</AppButton>
</div>
<QueueHealthBadge
:queued="queueCounts.queued"
:retrying="queueCounts.retrying"
:dropped="queueCounts.dropped"
/>
</div>
<AppAlert v-if="successMessage" tone="success" dismissible @dismiss="successMessage = null">
<template #title>Operation complete</template>
<p>{{ successMessage }}</p>
</AppAlert>
<AppAlert v-if="errorMessage" tone="danger">
<template #title>Run diagnostics request failed</template>
<p>{{ errorMessage }}</p>
<p class="text-secondary">Request ID: {{ errorRequestId || "n/a" }}</p>
</AppAlert>
<AppSkeleton v-if="loading" :lines="8" />
<template v-else>
<AppCard class="space-y-4">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Recent Runs</h2>
<AppEmptyState
v-if="runs.length === 0"
title="No runs found"
body="Manual or scheduled ingestion runs will appear here."
/>
<AppTable v-else label="Recent runs table">
<thead>
<tr>
<th scope="col">Run</th>
<th scope="col">Status</th>
<th scope="col">Started</th>
<th scope="col">Scholars</th>
<th scope="col">New pubs</th>
<th scope="col">Failures</th>
</tr>
</thead>
<tbody>
<tr v-for="run in runs" :key="run.id">
<td>
<RouterLink
:to="`/admin/runs/${run.id}`"
class="link-inline"
>
Run #{{ run.id }}
</RouterLink>
</td>
<td><RunStatusBadge :status="run.status" /></td>
<td>{{ formatDate(run.start_dt) }}</td>
<td>{{ run.scholar_count }}</td>
<td>{{ run.new_publication_count }}</td>
<td>{{ run.failed_count + run.partial_count }}</td>
</tr>
</tbody>
</AppTable>
</AppCard>
<AppCard class="space-y-4">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Continuation Queue</h2>
<AppEmptyState
v-if="queueItems.length === 0"
title="Queue is empty"
body="Retrying and dropped continuation items appear here for admin action."
/>
<AppTable v-else label="Continuation queue table">
<thead>
<tr>
<th scope="col">ID</th>
<th scope="col">Scholar</th>
<th scope="col">Status</th>
<th scope="col">Attempts</th>
<th scope="col">Next attempt</th>
<th scope="col">Actions</th>
</tr>
</thead>
<tbody>
<tr v-for="item in queueItems" :key="item.id">
<td>{{ item.id }}</td>
<td>{{ item.scholar_label }}</td>
<td>{{ item.status }}</td>
<td>{{ item.attempt_count }}</td>
<td>{{ formatDate(item.next_attempt_dt) }}</td>
<td>
<div class="flex flex-wrap items-center gap-2">
<AppButton
variant="ghost"
:disabled="activeQueueItemId === item.id"
@click="runQueueAction(item.id, 'retry')"
>
Retry
</AppButton>
<AppButton
variant="danger"
:disabled="activeQueueItemId === item.id"
@click="runQueueAction(item.id, 'drop')"
>
Drop
</AppButton>
<AppButton
variant="secondary"
:disabled="activeQueueItemId === item.id"
@click="runQueueAction(item.id, 'clear')"
>
Clear
</AppButton>
</div>
</td>
</tr>
</tbody>
</AppTable>
</AppCard>
</template>
</AppPage>
</template>

921
frontend/src/pages/ScholarsPage.vue Normal file
View file

@ -0,0 +1,921 @@
<script setup lang="ts">
import { computed, onMounted, ref } from "vue";
import AppPage from "@/components/layout/AppPage.vue";
import RunStatusBadge from "@/components/patterns/RunStatusBadge.vue";
import AppAlert from "@/components/ui/AppAlert.vue";
import AppBadge from "@/components/ui/AppBadge.vue";
import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue";
import AppEmptyState from "@/components/ui/AppEmptyState.vue";
import AppInput from "@/components/ui/AppInput.vue";
import AppModal from "@/components/ui/AppModal.vue";
import AppSkeleton from "@/components/ui/AppSkeleton.vue";
import AppTable from "@/components/ui/AppTable.vue";
import {
clearScholarImage,
createScholar,
deleteScholar,
listScholars,
searchScholarsByName,
setScholarImageUrl,
toggleScholar,
uploadScholarImage,
type ScholarProfile,
type ScholarSearchCandidate,
type ScholarSearchResult,
} from "@/features/scholars";
import { ApiRequestError } from "@/lib/api/errors";
const loading = ref(true);
const saving = ref(false);
const searchingByName = ref(false);
const activeScholarId = ref<number | null>(null);
const imageSavingScholarId = ref<number | null>(null);
const imageUploadingScholarId = ref<number | null>(null);
const addingCandidateScholarId = ref<string | null>(null);
const activeScholarSettingsId = ref<number | null>(null);
const scholars = ref<ScholarProfile[]>([]);
const imageUrlDraftByScholarId = ref<Record<number, string>>({});
const failedImageByKey = ref<Record<string, boolean>>({});
const scholarBatchInput = ref("");
const searchQuery = ref("");
const searchResult = ref<ScholarSearchResult | null>(null);
const searchErrorMessage = ref<string | null>(null);
const searchErrorRequestId = ref<string | null>(null);
const errorMessage = ref<string | null>(null);
const errorRequestId = ref<string | null>(null);
const successMessage = ref<string | null>(null);
const SCHOLAR_ID_PATTERN = /^[a-zA-Z0-9_-]{12}$/;
const URL_USER_PARAM_PATTERN = /(?:\?|&)user=([a-zA-Z0-9_-]{12})(?:&|#|$)/i;
const trackedScholarIds = computed(() => new Set(scholars.value.map((item) => item.scholar_id)));
const activeScholarSettings = computed(
() => scholars.value.find((item) => item.id === activeScholarSettingsId.value) ?? null,
);
const searchIsDegraded = computed(() => {
if (!searchResult.value) {
return false;
}
return searchResult.value.state === "blocked_or_captcha" || searchResult.value.state === "network_error";
});
function formatDate(value: string | null, compact = false): string {
if (!value) {
return "n/a";
}
const asDate = new Date(value);
if (Number.isNaN(asDate.getTime())) {
return value;
}
if (compact) {
return asDate.toLocaleString(undefined, {
dateStyle: "short",
timeStyle: "short",
});
}
return asDate.toLocaleString();
}
function scholarProfileUrl(scholarId: string): string {
return `https://scholar.google.com/citations?hl=en&user=${encodeURIComponent(scholarId)}`;
}
function scholarPublicationsRoute(profile: ScholarProfile): { name: string; query: { scholar: string } } {
return {
name: "publications",
query: { scholar: String(profile.id) },
};
}
function parseScholarIds(raw: string): string[] {
const ordered: string[] = [];
const seen = new Set<string>();
const tokens = raw
.split(/[\s,;]+/)
.map((value) => value.trim())
.filter((value) => value.length > 0);
for (const token of tokens) {
let candidate: string | null = null;
if (SCHOLAR_ID_PATTERN.test(token)) {
candidate = token;
}
if (!candidate) {
const directParamMatch = token.match(URL_USER_PARAM_PATTERN);
if (directParamMatch) {
candidate = directParamMatch[1];
}
}
if (!candidate && token.includes("scholar.google")) {
try {
const parsed = new URL(token);
const userParam = parsed.searchParams.get("user");
if (userParam && SCHOLAR_ID_PATTERN.test(userParam)) {
candidate = userParam;
}
} catch (_error) {
// Ignore non-URL tokens.
}
}
if (!candidate || seen.has(candidate)) {
continue;
}
seen.add(candidate);
ordered.push(candidate);
}
return ordered;
}
function formatImageSource(value: ScholarProfile["profile_image_source"]): string {
if (value === "upload") {
return "Uploaded";
}
if (value === "override") {
return "Custom URL";
}
if (value === "scraped") {
return "Scraped";
}
return "Fallback";
}
function sourceTone(value: ScholarProfile["profile_image_source"]): "neutral" | "info" | "success" {
if (value === "upload") {
return "success";
}
if (value === "override" || value === "scraped") {
return "info";
}
return "neutral";
}
function makeInitials(label: string | null | undefined, scholarId: string): string {
const source = (label || "").trim();
if (source.length === 0) {
return scholarId.slice(0, 2).toUpperCase();
}
const tokens = source.split(/\s+/).filter(Boolean);
if (tokens.length === 1) {
return tokens[0].slice(0, 2).toUpperCase();
}
return `${tokens[0].charAt(0)}${tokens[1].charAt(0)}`.toUpperCase();
}
function imageKey(prefix: string, id: string | number, imageUrl: string | null): string {
return `${prefix}:${String(id)}:${imageUrl || "none"}`;
}
function canRenderImage(prefix: string, id: string | number, imageUrl: string | null): boolean {
if (!imageUrl) {
return false;
}
return !failedImageByKey.value[imageKey(prefix, id, imageUrl)];
}
function markImageFailed(prefix: string, id: string | number, imageUrl: string | null): void {
failedImageByKey.value[imageKey(prefix, id, imageUrl)] = true;
}
function scholarLabel(profile: ScholarProfile): string {
return profile.display_name || profile.scholar_id;
}
function isImageBusy(scholarProfileId: number): boolean {
return (
imageSavingScholarId.value === scholarProfileId ||
imageUploadingScholarId.value === scholarProfileId ||
activeScholarId.value === scholarProfileId
);
}
function openScholarSettings(profile: ScholarProfile): void {
activeScholarSettingsId.value = profile.id;
}
function closeScholarSettings(): void {
activeScholarSettingsId.value = null;
}
function syncImageDrafts(): void {
const next: Record<number, string> = {};
for (const item of scholars.value) {
const existing = imageUrlDraftByScholarId.value[item.id];
if (typeof existing === "string" && existing.length > 0) {
next[item.id] = existing;
continue;
}
next[item.id] = item.profile_image_source === "override" ? (item.profile_image_url ?? "") : "";
}
imageUrlDraftByScholarId.value = next;
}
async function loadScholars(): Promise<void> {
loading.value = true;
try {
scholars.value = await listScholars();
syncImageDrafts();
} catch (error) {
scholars.value = [];
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to load scholars.";
}
} finally {
loading.value = false;
}
}
async function onAddScholars(): Promise<void> {
saving.value = true;
errorMessage.value = null;
errorRequestId.value = null;
successMessage.value = null;
try {
const scholarIds = parseScholarIds(scholarBatchInput.value);
if (scholarIds.length === 0) {
throw new Error("Provide at least one valid Scholar ID or profile URL.");
}
const settled = await Promise.allSettled(
scholarIds.map((scholarId) => createScholar({ scholar_id: scholarId })),
);
const failures: string[] = [];
let requestIdFromFailures: string | null = null;
let createdCount = 0;
settled.forEach((result, index) => {
if (result.status === "fulfilled") {
createdCount += 1;
return;
}
const scholarId = scholarIds[index];
if (result.reason instanceof ApiRequestError) {
failures.push(`${scholarId}: ${result.reason.message}`);
if (!requestIdFromFailures && result.reason.requestId) {
requestIdFromFailures = result.reason.requestId;
}
} else if (result.reason instanceof Error) {
failures.push(`${scholarId}: ${result.reason.message}`);
} else {
failures.push(`${scholarId}: Unable to create scholar.`);
}
});
if (createdCount > 0) {
const total = scholarIds.length;
successMessage.value = `Added ${createdCount} of ${total} scholar${total === 1 ? "" : "s"}.`;
scholarBatchInput.value = "";
}
if (failures.length > 0) {
const preview = failures.slice(0, 3).join(" | ");
const remainder = failures.length > 3 ? ` (+${failures.length - 3} more)` : "";
errorMessage.value = `Failed to add ${failures.length} scholar${failures.length === 1 ? "" : "s"}: ${preview}${remainder}`;
errorRequestId.value = requestIdFromFailures;
}
await loadScholars();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else if (error instanceof Error) {
errorMessage.value = error.message;
} else {
errorMessage.value = "Unable to create scholar records.";
}
} finally {
saving.value = false;
}
}
async function onSearchByName(): Promise<void> {
searchingByName.value = true;
searchErrorMessage.value = null;
searchErrorRequestId.value = null;
try {
const normalized = searchQuery.value.trim();
if (normalized.length < 2) {
throw new Error("Enter at least 2 characters to search.");
}
searchResult.value = await searchScholarsByName(normalized, 12);
} catch (error) {
searchResult.value = null;
if (error instanceof ApiRequestError) {
searchErrorMessage.value = error.message;
searchErrorRequestId.value = error.requestId;
} else if (error instanceof Error) {
searchErrorMessage.value = error.message;
} else {
searchErrorMessage.value = "Unable to search scholars by name.";
}
} finally {
searchingByName.value = false;
}
}
async function onAddCandidate(candidate: ScholarSearchCandidate): Promise<void> {
addingCandidateScholarId.value = candidate.scholar_id;
errorMessage.value = null;
errorRequestId.value = null;
successMessage.value = null;
try {
await createScholar({
scholar_id: candidate.scholar_id,
profile_image_url: candidate.profile_image_url ?? undefined,
});
successMessage.value = `${candidate.display_name} added.`;
await loadScholars();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to add scholar from search result.";
}
} finally {
addingCandidateScholarId.value = null;
}
}
async function onToggleScholar(profile: ScholarProfile): Promise<void> {
activeScholarId.value = profile.id;
errorMessage.value = null;
errorRequestId.value = null;
successMessage.value = null;
try {
await toggleScholar(profile.id);
successMessage.value = `${scholarLabel(profile)} ${profile.is_enabled ? "disabled" : "enabled"}.`;
await loadScholars();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to update scholar status.";
}
} finally {
activeScholarId.value = null;
}
}
async function onDeleteScholar(profile: ScholarProfile): Promise<void> {
const label = scholarLabel(profile);
const shouldDelete = window.confirm(`Delete scholar ${label}? This removes all linked publications and queue data.`);
if (!shouldDelete) {
return;
}
activeScholarId.value = profile.id;
errorMessage.value = null;
errorRequestId.value = null;
successMessage.value = null;
try {
await deleteScholar(profile.id);
successMessage.value = `${label} deleted.`;
if (activeScholarSettingsId.value === profile.id) {
closeScholarSettings();
}
await loadScholars();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to delete scholar.";
}
} finally {
activeScholarId.value = null;
}
}
async function onSaveImageUrl(profile: ScholarProfile): Promise<void> {
const candidate = (imageUrlDraftByScholarId.value[profile.id] || "").trim();
if (!candidate) {
errorMessage.value = "Enter an image URL before saving, or use Reset image.";
return;
}
imageSavingScholarId.value = profile.id;
errorMessage.value = null;
errorRequestId.value = null;
successMessage.value = null;
try {
await setScholarImageUrl(profile.id, candidate);
successMessage.value = `Image URL updated for ${scholarLabel(profile)}.`;
await loadScholars();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to update scholar image URL.";
}
} finally {
imageSavingScholarId.value = null;
}
}
async function onUploadImage(profile: ScholarProfile, event: Event): Promise<void> {
const input = event.target as HTMLInputElement | null;
const file = input?.files?.[0] ?? null;
if (!file) {
return;
}
imageUploadingScholarId.value = profile.id;
errorMessage.value = null;
errorRequestId.value = null;
successMessage.value = null;
try {
await uploadScholarImage(profile.id, file);
successMessage.value = `Uploaded image for ${scholarLabel(profile)}.`;
await loadScholars();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to upload scholar image.";
}
} finally {
imageUploadingScholarId.value = null;
if (input) {
input.value = "";
}
}
}
async function onResetImage(profile: ScholarProfile): Promise<void> {
imageSavingScholarId.value = profile.id;
errorMessage.value = null;
errorRequestId.value = null;
successMessage.value = null;
try {
await clearScholarImage(profile.id);
successMessage.value = `Image reset for ${scholarLabel(profile)}.`;
await loadScholars();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to reset scholar image.";
}
} finally {
imageSavingScholarId.value = null;
}
}
onMounted(() => {
void loadScholars();
});
</script>
<template>
<AppPage title="Scholars" subtitle="Track scholars and manage profile behavior with less noise.">
<div class="flex justify-end">
<AppButton variant="secondary" @click="loadScholars" :disabled="loading || saving || searchingByName">
{{ loading ? "Refreshing..." : "Refresh" }}
</AppButton>
</div>
<AppAlert v-if="successMessage" tone="success" dismissible @dismiss="successMessage = null">
<template #title>Scholar update complete</template>
<p>{{ successMessage }}</p>
</AppAlert>
<AppAlert v-if="errorMessage" tone="danger">
<template #title>Scholar request failed</template>
<p>{{ errorMessage }}</p>
<p class="text-secondary">Request ID: {{ errorRequestId || "n/a" }}</p>
</AppAlert>
<section class="grid gap-4 xl:grid-cols-[minmax(0,34rem)_minmax(0,1fr)]">
<div class="grid content-start gap-4">
<AppCard class="space-y-4">
<div class="space-y-1">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Add Scholars</h2>
<p class="text-sm text-secondary">
Paste one or many Scholar IDs or profile URLs. Duplicate and already-tracked IDs are ignored per response.
</p>
</div>
<form class="grid gap-3" @submit.prevent="onAddScholars">
<label class="grid gap-2 text-sm font-medium text-zinc-700 dark:text-zinc-300">
<span>Scholar IDs or profile URLs</span>
<textarea
id="scholar-batch-input"
v-model="scholarBatchInput"
rows="5"
placeholder="A-UbBTPM15wL\nhttps://scholar.google.com/citations?hl=en&user=A-UbBTPM15wL"
class="w-full rounded-lg border border-zinc-300 bg-white px-3 py-2 text-sm text-zinc-900 outline-none ring-brand-300 transition placeholder:text-zinc-400 focus-visible:ring-2 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 disabled:cursor-not-allowed disabled:opacity-60 dark:border-zinc-700 dark:bg-zinc-900 dark:text-zinc-100 dark:ring-brand-400 dark:focus-visible:ring-offset-zinc-950 dark:placeholder:text-zinc-500"
/>
</label>
<div class="flex flex-wrap items-center justify-between gap-2">
<AppButton type="submit" :disabled="saving || loading">
{{ saving ? "Adding..." : "Add scholars" }}
</AppButton>
<span class="text-xs text-secondary">Accepted pattern: <code>[a-zA-Z0-9_-]{12}</code></span>
</div>
</form>
</AppCard>
<AppCard class="space-y-4">
<div class="space-y-1">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Search by Name</h2>
<p class="text-sm text-secondary">
Best-effort helper. For reliable adds, use Scholar URL/ID above.
</p>
</div>
<form class="flex flex-wrap items-center gap-2" @submit.prevent="onSearchByName">
<div class="min-w-0 flex-1">
<AppInput
id="scholar-search-name"
v-model="searchQuery"
placeholder="e.g. Geoffrey Hinton"
:disabled="searchingByName"
/>
</div>
<AppButton type="submit" :disabled="searchingByName || searchQuery.trim().length < 2">
{{ searchingByName ? "Searching..." : "Search" }}
</AppButton>
</form>
<AppAlert v-if="searchErrorMessage" tone="danger">
<template #title>Search failed</template>
<p>{{ searchErrorMessage }}</p>
<p class="text-secondary">Request ID: {{ searchErrorRequestId || "n/a" }}</p>
</AppAlert>
<AppSkeleton v-else-if="searchingByName" :lines="4" />
<template v-else-if="searchResult">
<div class="flex flex-wrap items-center justify-between gap-2">
<p class="text-sm text-secondary">
{{ searchResult.candidates.length }} candidate{{ searchResult.candidates.length === 1 ? "" : "s" }} for
<strong class="text-zinc-900 dark:text-zinc-100">{{ searchResult.query }}</strong>
</p>
<AppBadge
:tone="
searchResult.state === 'ok'
? 'success'
: searchResult.state === 'blocked_or_captcha' || searchResult.state === 'network_error'
? 'warning'
: 'neutral'
"
>
{{ searchResult.state }}
</AppBadge>
</div>
<p
v-if="searchResult.state !== 'ok' || searchResult.warnings.length > 0"
class="text-xs text-secondary"
>
<span>State reason: <code>{{ searchResult.state_reason }}</code></span>
<span v-if="searchResult.action_hint">. {{ searchResult.action_hint }}</span>
<span v-if="searchResult.warnings.length > 0">
. Warnings: {{ searchResult.warnings.join(", ") }}
</span>
</p>
<AppAlert v-if="searchIsDegraded" tone="warning">
<template #title>Name search is degraded</template>
<p>
To avoid blocks, this feature is throttled and may temporarily pause itself. Use Scholar URL/ID adds
for dependable tracking.
</p>
</AppAlert>
<AppEmptyState
v-if="searchResult.candidates.length === 0"
title="No scholar matches returned"
body="Try again later or paste Scholar profile URLs/IDs to continue safely."
/>
<ul v-else class="grid gap-3">
<li
v-for="candidate in searchResult.candidates"
:key="candidate.scholar_id"
class="rounded-xl border border-zinc-200 bg-zinc-50/70 p-3 dark:border-zinc-800 dark:bg-zinc-900/50"
>
<div class="flex items-start gap-3">
<div
class="flex h-12 w-12 shrink-0 items-center justify-center overflow-hidden rounded-full border border-zinc-200 bg-zinc-100 text-xs font-semibold text-zinc-700 dark:border-zinc-700 dark:bg-zinc-800 dark:text-zinc-200"
>
<img
v-if="canRenderImage('candidate', candidate.scholar_id, candidate.profile_image_url)"
:src="candidate.profile_image_url || ''"
:alt="`${candidate.display_name} profile image`"
class="h-full w-full object-cover"
loading="lazy"
referrerpolicy="no-referrer"
@error="markImageFailed('candidate', candidate.scholar_id, candidate.profile_image_url)"
/>
<span v-else>{{ makeInitials(candidate.display_name, candidate.scholar_id) }}</span>
</div>
<div class="min-w-0 flex-1 space-y-1">
<div class="flex flex-wrap items-center gap-2">
<strong class="text-sm text-zinc-900 dark:text-zinc-100">{{ candidate.display_name }}</strong>
<code class="text-xs text-secondary">{{ candidate.scholar_id }}</code>
<AppBadge v-if="trackedScholarIds.has(candidate.scholar_id)" tone="success">Tracked</AppBadge>
</div>
<p class="truncate text-xs text-secondary">
{{ candidate.affiliation || "No affiliation provided" }}
</p>
<div class="flex flex-wrap items-center gap-2 text-xs text-secondary">
<span v-if="candidate.email_domain">Email: {{ candidate.email_domain }}</span>
<span v-if="candidate.cited_by_count !== null">Cited by: {{ candidate.cited_by_count }}</span>
</div>
<div class="flex flex-wrap items-center gap-1">
<AppBadge v-for="interest in candidate.interests.slice(0, 3)" :key="interest" tone="neutral">
{{ interest }}
</AppBadge>
</div>
</div>
<div class="grid shrink-0 gap-2">
<AppButton
:disabled="trackedScholarIds.has(candidate.scholar_id) || addingCandidateScholarId === candidate.scholar_id"
@click="onAddCandidate(candidate)"
>
{{ addingCandidateScholarId === candidate.scholar_id ? "Adding..." : "Add" }}
</AppButton>
<a :href="candidate.profile_url" target="_blank" rel="noreferrer" class="link-inline text-xs text-center">
Open profile
</a>
</div>
</div>
</li>
</ul>
</template>
<p v-else class="text-sm text-secondary">Run a name search to see matching scholar candidates.</p>
</AppCard>
</div>
<AppCard class="space-y-4">
<div class="flex flex-wrap items-center justify-between gap-2">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Tracked Scholars</h2>
<span class="text-sm text-secondary">{{ scholars.length }} total</span>
</div>
<AppSkeleton v-if="loading" :lines="6" />
<AppEmptyState
v-else-if="scholars.length === 0"
title="No scholars tracked"
body="Add a Scholar ID directly or search by name to start ingestion tracking."
/>
<div v-else class="space-y-3">
<ul class="grid gap-3 lg:hidden">
<li
v-for="item in scholars"
:key="item.id"
class="rounded-xl border border-zinc-200 bg-zinc-50/70 p-3 dark:border-zinc-800 dark:bg-zinc-900/50"
>
<div class="flex items-start gap-3">
<div
class="flex h-12 w-12 shrink-0 items-center justify-center overflow-hidden rounded-full border border-zinc-200 bg-zinc-100 text-xs font-semibold text-zinc-700 dark:border-zinc-700 dark:bg-zinc-800 dark:text-zinc-200"
>
<img
v-if="canRenderImage('scholar', item.id, item.profile_image_url)"
:src="item.profile_image_url || ''"
:alt="`${scholarLabel(item)} profile image`"
class="h-full w-full object-cover"
loading="lazy"
referrerpolicy="no-referrer"
@error="markImageFailed('scholar', item.id, item.profile_image_url)"
/>
<span v-else>{{ makeInitials(item.display_name, item.scholar_id) }}</span>
</div>
<div class="min-w-0 flex-1 space-y-1">
<p class="truncate text-sm font-semibold text-zinc-900 dark:text-zinc-100">{{ scholarLabel(item) }}</p>
<p class="text-xs text-secondary"><code>{{ item.scholar_id }}</code></p>
<div class="flex flex-wrap items-center gap-2">
<AppBadge :tone="item.is_enabled ? 'success' : 'warning'">
{{ item.is_enabled ? "Enabled" : "Disabled" }}
</AppBadge>
<RunStatusBadge :status="item.last_run_status || 'unknown'" />
<span class="text-xs text-secondary">{{ formatDate(item.last_run_dt, true) }}</span>
</div>
<RouterLink :to="scholarPublicationsRoute(item)" class="link-inline text-xs">
View publications
</RouterLink>
</div>
<AppButton variant="secondary" :disabled="saving" @click="openScholarSettings(item)">Manage</AppButton>
</div>
</li>
</ul>
<AppTable class="hidden lg:block" label="Scholars table">
<thead>
<tr>
<th scope="col">Scholar</th>
<th scope="col">Scholar ID</th>
<th scope="col">Enabled</th>
<th scope="col" class="w-[15rem]">Last run</th>
<th scope="col">Manage</th>
</tr>
</thead>
<tbody>
<tr v-for="item in scholars" :key="item.id">
<td>
<div class="flex items-start gap-3">
<div
class="flex h-12 w-12 shrink-0 items-center justify-center overflow-hidden rounded-full border border-zinc-200 bg-zinc-100 text-xs font-semibold text-zinc-700 dark:border-zinc-700 dark:bg-zinc-800 dark:text-zinc-200"
>
<img
v-if="canRenderImage('scholar', item.id, item.profile_image_url)"
:src="item.profile_image_url || ''"
:alt="`${scholarLabel(item)} profile image`"
class="h-full w-full object-cover"
loading="lazy"
referrerpolicy="no-referrer"
@error="markImageFailed('scholar', item.id, item.profile_image_url)"
/>
<span v-else>{{ makeInitials(item.display_name, item.scholar_id) }}</span>
</div>
<div class="grid min-w-0 gap-1">
<strong class="truncate text-zinc-900 dark:text-zinc-100">{{ scholarLabel(item) }}</strong>
<div class="flex flex-wrap items-center gap-2">
<RouterLink :to="scholarPublicationsRoute(item)" class="link-inline text-xs">
Publications
</RouterLink>
<a :href="scholarProfileUrl(item.scholar_id)" target="_blank" rel="noreferrer" class="link-inline text-xs">
Open profile
</a>
<AppBadge :tone="sourceTone(item.profile_image_source)">
{{ formatImageSource(item.profile_image_source) }}
</AppBadge>
</div>
</div>
</div>
</td>
<td><code class="text-xs">{{ item.scholar_id }}</code></td>
<td>
<AppBadge :tone="item.is_enabled ? 'success' : 'warning'">
{{ item.is_enabled ? "Enabled" : "Disabled" }}
</AppBadge>
</td>
<td>
<div class="grid gap-1 text-xs">
<RunStatusBadge :status="item.last_run_status || 'unknown'" />
<span class="whitespace-nowrap text-secondary">{{ formatDate(item.last_run_dt, true) }}</span>
</div>
</td>
<td>
<AppButton variant="secondary" :disabled="saving" @click="openScholarSettings(item)">
Manage
</AppButton>
</td>
</tr>
</tbody>
</AppTable>
</div>
</AppCard>
</section>
<AppModal :open="activeScholarSettings !== null" title="Scholar settings" @close="closeScholarSettings">
<div v-if="activeScholarSettings" class="grid gap-4">
<div class="flex items-start gap-3">
<div
class="flex h-14 w-14 shrink-0 items-center justify-center overflow-hidden rounded-full border border-zinc-200 bg-zinc-100 text-xs font-semibold text-zinc-700 dark:border-zinc-700 dark:bg-zinc-800 dark:text-zinc-200"
>
<img
v-if="canRenderImage('scholar', activeScholarSettings.id, activeScholarSettings.profile_image_url)"
:src="activeScholarSettings.profile_image_url || ''"
:alt="`${scholarLabel(activeScholarSettings)} profile image`"
class="h-full w-full object-cover"
loading="lazy"
referrerpolicy="no-referrer"
@error="markImageFailed('scholar', activeScholarSettings.id, activeScholarSettings.profile_image_url)"
/>
<span v-else>{{ makeInitials(activeScholarSettings.display_name, activeScholarSettings.scholar_id) }}</span>
</div>
<div class="min-w-0 space-y-1">
<p class="truncate text-sm font-semibold text-zinc-900 dark:text-zinc-100">
{{ scholarLabel(activeScholarSettings) }}
</p>
<p class="text-xs text-secondary">
ID: <code>{{ activeScholarSettings.scholar_id }}</code>
</p>
<div class="flex flex-wrap items-center gap-3">
<RouterLink :to="scholarPublicationsRoute(activeScholarSettings)" class="link-inline text-xs">
View publications
</RouterLink>
<a
:href="scholarProfileUrl(activeScholarSettings.scholar_id)"
target="_blank"
rel="noreferrer"
class="link-inline text-xs"
>
Open Google Scholar profile
</a>
</div>
</div>
</div>
<div class="grid gap-2">
<label class="text-sm font-medium text-zinc-700 dark:text-zinc-300" :for="`scholar-image-url-${activeScholarSettings.id}`">
Profile image URL override
</label>
<div class="flex flex-wrap items-center gap-2">
<div class="min-w-0 flex-1">
<AppInput
:id="`scholar-image-url-${activeScholarSettings.id}`"
v-model="imageUrlDraftByScholarId[activeScholarSettings.id]"
placeholder="https://example.com/avatar.jpg"
:disabled="isImageBusy(activeScholarSettings.id)"
/>
</div>
<AppButton
variant="secondary"
:disabled="isImageBusy(activeScholarSettings.id)"
@click="onSaveImageUrl(activeScholarSettings)"
>
{{ imageSavingScholarId === activeScholarSettings.id ? "Saving..." : "Save URL" }}
</AppButton>
</div>
<div class="flex flex-wrap items-center gap-2">
<label
:for="`scholar-image-upload-${activeScholarSettings.id}`"
class="inline-flex min-h-10 cursor-pointer items-center justify-center rounded-lg border border-zinc-300 bg-zinc-100 px-3 py-2 text-sm font-semibold text-zinc-900 transition hover:bg-zinc-200 focus-within:outline-none focus-within:ring-2 focus-within:ring-brand-500 focus-within:ring-offset-2 focus-within:ring-offset-zinc-100 dark:border-zinc-700 dark:bg-zinc-800 dark:text-zinc-100 dark:hover:bg-zinc-700 dark:focus-within:ring-brand-400 dark:focus-within:ring-offset-zinc-950"
:class="{ 'pointer-events-none opacity-60': isImageBusy(activeScholarSettings.id) }"
>
{{ imageUploadingScholarId === activeScholarSettings.id ? "Uploading..." : "Upload image" }}
</label>
<input
:id="`scholar-image-upload-${activeScholarSettings.id}`"
type="file"
class="sr-only"
accept="image/jpeg,image/png,image/webp,image/gif"
:disabled="isImageBusy(activeScholarSettings.id)"
@change="onUploadImage(activeScholarSettings, $event)"
/>
<AppButton variant="ghost" :disabled="isImageBusy(activeScholarSettings.id)" @click="onResetImage(activeScholarSettings)">
Reset image
</AppButton>
</div>
</div>
<div class="flex flex-wrap items-center justify-between gap-2 border-t border-zinc-200 pt-3 dark:border-zinc-800">
<AppButton
variant="secondary"
:disabled="isImageBusy(activeScholarSettings.id) || saving"
@click="onToggleScholar(activeScholarSettings)"
>
{{ activeScholarSettings.is_enabled ? "Disable scholar" : "Enable scholar" }}
</AppButton>
<AppButton
variant="danger"
:disabled="isImageBusy(activeScholarSettings.id) || saving"
@click="onDeleteScholar(activeScholarSettings)"
>
Delete scholar
</AppButton>
</div>
</div>
</AppModal>
</AppPage>
</template>

255
frontend/src/pages/SettingsPage.vue Normal file
View file

@ -0,0 +1,255 @@
<script setup lang="ts">
import { onMounted, ref } from "vue";
import AppPage from "@/components/layout/AppPage.vue";
import AppAlert from "@/components/ui/AppAlert.vue";
import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue";
import AppCheckbox from "@/components/ui/AppCheckbox.vue";
import AppInput from "@/components/ui/AppInput.vue";
import AppModal from "@/components/ui/AppModal.vue";
import AppSkeleton from "@/components/ui/AppSkeleton.vue";
import {
changePassword,
fetchSettings,
updateSettings,
type UserSettings,
} from "@/features/settings";
import { ApiRequestError } from "@/lib/api/errors";
const loading = ref(true);
const saving = ref(false);
const updatingPassword = ref(false);
const autoRunEnabled = ref(false);
const runIntervalMinutes = ref("60");
const requestDelaySeconds = ref("2");
const currentPassword = ref("");
const newPassword = ref("");
const confirmPassword = ref("");
const errorMessage = ref<string | null>(null);
const errorRequestId = ref<string | null>(null);
const successMessage = ref<string | null>(null);
const showIngestionModal = ref(false);
const showPasswordModal = ref(false);
function hydrateSettings(settings: UserSettings): void {
autoRunEnabled.value = settings.auto_run_enabled;
runIntervalMinutes.value = String(settings.run_interval_minutes);
requestDelaySeconds.value = String(settings.request_delay_seconds);
}
function parsePositiveInteger(value: string, label: string): number {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed <= 0) {
throw new Error(`${label} must be a positive integer.`);
}
return parsed;
}
async function loadSettings(): Promise<void> {
loading.value = true;
errorMessage.value = null;
errorRequestId.value = null;
try {
const settings = await fetchSettings();
hydrateSettings(settings);
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to load user settings.";
}
} finally {
loading.value = false;
}
}
async function onSaveSettings(): Promise<void> {
saving.value = true;
errorMessage.value = null;
errorRequestId.value = null;
successMessage.value = null;
try {
const payload: UserSettings = {
auto_run_enabled: autoRunEnabled.value,
run_interval_minutes: parsePositiveInteger(runIntervalMinutes.value, "Run interval"),
request_delay_seconds: parsePositiveInteger(requestDelaySeconds.value, "Request delay"),
};
const saved = await updateSettings(payload);
hydrateSettings(saved);
successMessage.value = "Settings updated.";
showIngestionModal.value = false;
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else if (error instanceof Error) {
errorMessage.value = error.message;
} else {
errorMessage.value = "Unable to save settings.";
}
} finally {
saving.value = false;
}
}
async function onChangePassword(): Promise<void> {
updatingPassword.value = true;
errorMessage.value = null;
errorRequestId.value = null;
successMessage.value = null;
try {
if (!currentPassword.value || !newPassword.value || !confirmPassword.value) {
throw new Error("All password fields are required.");
}
const response = await changePassword({
current_password: currentPassword.value,
new_password: newPassword.value,
confirm_password: confirmPassword.value,
});
currentPassword.value = "";
newPassword.value = "";
confirmPassword.value = "";
successMessage.value = response.message;
showPasswordModal.value = false;
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else if (error instanceof Error) {
errorMessage.value = error.message;
} else {
errorMessage.value = "Unable to change password.";
}
} finally {
updatingPassword.value = false;
}
}
onMounted(() => {
void loadSettings();
});
</script>
<template>
<AppPage title="Settings" subtitle="Configure ingestion and account controls.">
<AppAlert v-if="successMessage" tone="success" dismissible @dismiss="successMessage = null">
<template #title>Saved</template>
<p>{{ successMessage }}</p>
</AppAlert>
<AppAlert v-if="errorMessage" tone="danger">
<template #title>Settings request failed</template>
<p>{{ errorMessage }}</p>
<p class="text-secondary">Request ID: {{ errorRequestId || "n/a" }}</p>
</AppAlert>
<AppSkeleton v-if="loading" :lines="7" />
<template v-else>
<section class="grid gap-4 lg:grid-cols-2">
<AppCard class="space-y-4">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Scholar ingestion defaults</h2>
<dl class="grid gap-2 text-sm text-secondary">
<div class="flex items-center justify-between gap-2">
<dt>Scheduler</dt>
<dd class="font-semibold text-zinc-900 dark:text-zinc-100">
{{ autoRunEnabled ? "Enabled" : "Disabled" }}
</dd>
</div>
<div class="flex items-center justify-between gap-2">
<dt>Run interval</dt>
<dd class="font-semibold text-zinc-900 dark:text-zinc-100">{{ runIntervalMinutes }} min</dd>
</div>
<div class="flex items-center justify-between gap-2">
<dt>Request delay</dt>
<dd class="font-semibold text-zinc-900 dark:text-zinc-100">{{ requestDelaySeconds }} sec</dd>
</div>
</dl>
<AppButton variant="secondary" @click="showIngestionModal = true">
Manage ingestion settings
</AppButton>
</AppCard>
<AppCard class="space-y-4">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Account security</h2>
<p class="text-sm text-secondary">Password changes are handled in a dedicated overlay to reduce clutter.</p>
<AppButton variant="secondary" @click="showPasswordModal = true">Manage password</AppButton>
</AppCard>
</section>
</template>
<AppModal :open="showIngestionModal" title="Ingestion settings" @close="showIngestionModal = false">
<form class="grid gap-3" @submit.prevent="onSaveSettings">
<AppCheckbox id="auto-run-enabled" v-model="autoRunEnabled" label="Enable scheduler auto-run" />
<label class="grid gap-2 text-sm font-medium text-zinc-700 dark:text-zinc-300">
<span>Run interval (minutes)</span>
<AppInput id="run-interval" v-model="runIntervalMinutes" type="number" min="1" />
</label>
<label class="grid gap-2 text-sm font-medium text-zinc-700 dark:text-zinc-300">
<span>Request delay (seconds)</span>
<AppInput id="request-delay" v-model="requestDelaySeconds" type="number" min="1" />
</label>
<div class="mt-2 flex flex-wrap justify-end gap-2">
<AppButton
variant="ghost"
type="button"
:disabled="saving"
@click="showIngestionModal = false"
>
Cancel
</AppButton>
<AppButton type="submit" :disabled="saving">
{{ saving ? "Saving..." : "Save settings" }}
</AppButton>
</div>
</form>
</AppModal>
<AppModal :open="showPasswordModal" title="Change password" @close="showPasswordModal = false">
<form class="grid gap-3" @submit.prevent="onChangePassword">
<label class="grid gap-2 text-sm font-medium text-zinc-700 dark:text-zinc-300">
<span>Current password</span>
<AppInput v-model="currentPassword" type="password" autocomplete="current-password" />
</label>
<label class="grid gap-2 text-sm font-medium text-zinc-700 dark:text-zinc-300">
<span>New password</span>
<AppInput v-model="newPassword" type="password" autocomplete="new-password" />
</label>
<label class="grid gap-2 text-sm font-medium text-zinc-700 dark:text-zinc-300">
<span>Confirm new password</span>
<AppInput v-model="confirmPassword" type="password" autocomplete="new-password" />
</label>
<div class="mt-2 flex flex-wrap justify-end gap-2">
<AppButton
variant="ghost"
type="button"
:disabled="updatingPassword"
@click="showPasswordModal = false"
>
Cancel
</AppButton>
<AppButton type="submit" :disabled="updatingPassword">
{{ updatingPassword ? "Updating..." : "Change password" }}
</AppButton>
</div>
</form>
</AppModal>
</AppPage>
</template>

138
frontend/src/pages/StyleGuidePage.vue Normal file
View file

@ -0,0 +1,138 @@
<script setup lang="ts">
import { ref } from "vue";
import AppPage from "@/components/layout/AppPage.vue";
import QueueHealthBadge from "@/components/patterns/QueueHealthBadge.vue";
import RunStatusBadge from "@/components/patterns/RunStatusBadge.vue";
import AppAlert from "@/components/ui/AppAlert.vue";
import AppBadge from "@/components/ui/AppBadge.vue";
import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue";
import AppCheckbox from "@/components/ui/AppCheckbox.vue";
import AppInput from "@/components/ui/AppInput.vue";
import AppSelect from "@/components/ui/AppSelect.vue";
import AppTable from "@/components/ui/AppTable.vue";
const sampleName = ref("Ada Lovelace");
const sampleEmail = ref("ada@example.com");
const sampleRole = ref("operator");
const sampleEnabled = ref(true);
const sampleRows = [
{ id: 101, title: "Entity Resolution Improvements", status: "running", owner: "Scheduler" },
{ id: 102, title: "Retry Queue Drain", status: "partial_failure", owner: "Ops" },
{ id: 103, title: "Weekly Publication Sync", status: "success", owner: "Ingestion" },
];
</script>
<template>
<AppPage
title="Style Guide"
subtitle="Reference page for Tailwind component patterns used across the application."
>
<section class="grid gap-4 xl:grid-cols-2">
<AppCard class="space-y-4">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Buttons</h2>
<div class="flex flex-wrap items-center gap-2">
<AppButton>Primary</AppButton>
<AppButton variant="secondary">Secondary</AppButton>
<AppButton variant="ghost">Ghost</AppButton>
<AppButton variant="danger">Danger</AppButton>
<AppButton :disabled="true">Disabled</AppButton>
</div>
<p class="text-sm text-secondary">Use `primary` for main actions and `danger` for destructive actions.</p>
</AppCard>
<AppCard class="space-y-4">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Status System</h2>
<div class="flex flex-wrap items-center gap-2">
<AppBadge tone="neutral">Neutral</AppBadge>
<AppBadge tone="info">Info</AppBadge>
<AppBadge tone="success">Success</AppBadge>
<AppBadge tone="warning">Warning</AppBadge>
<AppBadge tone="danger">Danger</AppBadge>
</div>
<div class="flex flex-wrap items-center gap-2">
<RunStatusBadge status="running" />
<RunStatusBadge status="success" />
<RunStatusBadge status="partial_failure" />
<RunStatusBadge status="failed" />
</div>
<QueueHealthBadge :queued="2" :retrying="1" :dropped="1" />
</AppCard>
</section>
<section class="grid gap-4 xl:grid-cols-2">
<AppCard class="space-y-4">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Forms</h2>
<form class="grid gap-3" @submit.prevent>
<label class="grid gap-2 text-sm font-medium text-zinc-700 dark:text-zinc-300">
<span>Name</span>
<AppInput v-model="sampleName" autocomplete="name" />
</label>
<label class="grid gap-2 text-sm font-medium text-zinc-700 dark:text-zinc-300">
<span>Email</span>
<AppInput v-model="sampleEmail" type="email" autocomplete="email" />
</label>
<label class="grid gap-2 text-sm font-medium text-zinc-700 dark:text-zinc-300">
<span>Role</span>
<AppSelect v-model="sampleRole">
<option value="viewer">Viewer</option>
<option value="operator">Operator</option>
<option value="admin">Admin</option>
</AppSelect>
</label>
<AppCheckbox id="style-guide-enabled" v-model="sampleEnabled" label="Enabled" />
<div class="flex flex-wrap items-center gap-2">
<AppButton>Submit</AppButton>
<AppButton variant="secondary" type="reset">Reset</AppButton>
</div>
</form>
</AppCard>
<AppCard class="space-y-4">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Alerts</h2>
<AppAlert tone="info">
<template #title>Informational</template>
<p>Use for contextual guidance and non-critical notices.</p>
</AppAlert>
<AppAlert tone="success">
<template #title>Success</template>
<p>Use after completed mutations to confirm outcomes.</p>
</AppAlert>
<AppAlert tone="warning">
<template #title>Warning</template>
<p>Use when user intervention may be required soon.</p>
</AppAlert>
</AppCard>
</section>
<AppCard class="space-y-4">
<h2 class="text-lg font-semibold text-zinc-900 dark:text-zinc-100">Table Pattern</h2>
<AppTable label="Style guide sample run table">
<thead>
<tr>
<th scope="col">Run</th>
<th scope="col">Title</th>
<th scope="col">Owner</th>
<th scope="col">Status</th>
<th scope="col">Action</th>
</tr>
</thead>
<tbody>
<tr v-for="row in sampleRows" :key="row.id">
<td>#{{ row.id }}</td>
<td>{{ row.title }}</td>
<td>{{ row.owner }}</td>
<td><RunStatusBadge :status="row.status" /></td>
<td><AppButton variant="ghost">Inspect</AppButton></td>
</tr>
</tbody>
</AppTable>
</AppCard>
</AppPage>
</template>

71
frontend/src/stores/auth.ts Normal file
View file

@ -0,0 +1,71 @@
import { defineStore } from "pinia";
import { ApiRequestError } from "@/lib/api/errors";
import { fetchCsrfBootstrap } from "@/lib/api/csrf";
import { fetchMe, loginSession, logoutSession, type SessionUser } from "@/lib/auth/session";
import { useUiStore } from "@/stores/ui";
export type AuthState = "unknown" | "authenticated" | "anonymous";
export const useAuthStore = defineStore("auth", {
state: () => ({
state: "unknown" as AuthState,
csrfToken: null as string | null,
user: null as SessionUser | null,
}),
getters: {
isAuthenticated: (state) => state.state === "authenticated",
isAdmin: (state) => Boolean(state.user?.is_admin),
},
actions: {
async bootstrapSession(): Promise<void> {
const ui = useUiStore();
ui.clearGlobalError();
try {
const csrf = await fetchCsrfBootstrap();
this.csrfToken = csrf.data.csrf_token;
if (!csrf.data.authenticated) {
this.state = "anonymous";
this.user = null;
return;
}
const me = await fetchMe();
this.state = "authenticated";
this.user = me.data.user;
this.csrfToken = me.data.csrf_token;
} catch (error) {
this.state = "anonymous";
this.user = null;
if (error instanceof ApiRequestError) {
ui.setGlobalError({
message: error.message,
requestId: error.requestId,
});
}
}
},
async login(email: string, password: string): Promise<void> {
const response = await loginSession({ email, password });
this.state = "authenticated";
this.user = response.data.user;
this.csrfToken = response.data.csrf_token;
},
async logout(): Promise<void> {
await logoutSession();
this.state = "anonymous";
this.user = null;
this.csrfToken = null;
try {
const csrf = await fetchCsrfBootstrap();
this.csrfToken = csrf.data.csrf_token;
} catch (_err) {
// No-op: app can still function by refreshing.
}
},
},
});

51
frontend/src/stores/theme.ts Normal file
View file

@ -0,0 +1,51 @@
import { defineStore } from "pinia";
export type ThemePreference = "system" | "light" | "dark";
export type ThemeValue = "light" | "dark";
const STORAGE_KEY = "scholarr-theme-preference";
function systemTheme(): ThemeValue {
return window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light";
}
function applyTheme(theme: ThemeValue, preference: ThemePreference): void {
const root = document.documentElement;
root.classList.toggle("dark", theme === "dark");
root.setAttribute("data-theme-preference", preference);
}
export const useThemeStore = defineStore("theme", {
state: () => ({
preference: "system" as ThemePreference,
active: "light" as ThemeValue,
}),
actions: {
initialize(): void {
let stored: string | null = null;
try {
stored = localStorage.getItem(STORAGE_KEY);
} catch (_err) {
stored = null;
}
if (stored === "light" || stored === "dark" || stored === "system") {
this.preference = stored;
}
this.active = this.preference === "system" ? systemTheme() : this.preference;
applyTheme(this.active, this.preference);
},
setPreference(preference: ThemePreference): void {
this.preference = preference;
this.active = preference === "system" ? systemTheme() : preference;
applyTheme(this.active, this.preference);
try {
localStorage.setItem(STORAGE_KEY, preference);
} catch (_err) {
// Ignore storage failures in private contexts.
}
},
},
});

20
frontend/src/stores/ui.ts Normal file
View file

@ -0,0 +1,20 @@
import { defineStore } from "pinia";
export interface GlobalErrorState {
message: string;
requestId: string | null;
}
export const useUiStore = defineStore("ui", {
state: () => ({
globalError: null as GlobalErrorState | null,
}),
actions: {
setGlobalError(error: GlobalErrorState): void {
this.globalError = error;
},
clearGlobalError(): void {
this.globalError = null;
},
},
});

69
frontend/src/styles.css Normal file
View file

@ -0,0 +1,69 @@
@import url("https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;600;700;800&family=Space+Grotesk:wght@500;600;700&display=swap");
@tailwind base;
@tailwind components;
@tailwind utilities;
@layer base {
html,
body,
#app {
@apply min-h-full overflow-x-clip;
}
body {
@apply bg-zinc-100 text-zinc-900 antialiased transition-colors duration-300 dark:bg-zinc-950 dark:text-zinc-100;
}
h1,
h2,
h3,
h4 {
@apply font-display;
}
a {
@apply text-inherit no-underline;
}
input,
select,
textarea,
button {
font-family: inherit;
}
}
@layer components {
.skip-link {
@apply sr-only focus:not-sr-only focus:absolute focus:left-4 focus:top-3 focus:z-50 focus:rounded-md focus:bg-zinc-900 focus:px-3 focus:py-2 focus:text-sm focus:font-medium focus:text-white focus:outline-none dark:focus:bg-brand-400 dark:focus:text-zinc-950;
}
.page-stack {
@apply space-y-6;
}
.page-title {
@apply font-display text-3xl font-semibold tracking-tight text-zinc-900 sm:text-4xl dark:text-zinc-100;
}
.page-subtitle {
@apply text-sm leading-relaxed text-zinc-600 dark:text-zinc-400;
}
.text-secondary {
@apply text-zinc-600 dark:text-zinc-400;
}
.text-muted {
@apply text-zinc-500 dark:text-zinc-500;
}
.row {
@apply flex items-center gap-3;
}
.link-inline {
@apply rounded-sm text-brand-700 underline-offset-4 transition hover:underline focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-brand-500 focus-visible:ring-offset-2 focus-visible:ring-offset-zinc-100 dark:text-brand-300 dark:focus-visible:ring-brand-400 dark:focus-visible:ring-offset-zinc-950;
}
}

32
frontend/tailwind.config.js Normal file
View file

@ -0,0 +1,32 @@
/** @type {import('tailwindcss').Config} */
export default {
darkMode: "class",
content: ["./index.html", "./src/**/*.{vue,ts,tsx,js,jsx}"],
theme: {
extend: {
fontFamily: {
sans: ["Manrope", "ui-sans-serif", "system-ui", "sans-serif"],
display: ["Space Grotesk", "Manrope", "ui-sans-serif", "system-ui", "sans-serif"],
},
boxShadow: {
panel: "0 10px 30px -20px rgba(8, 15, 30, 0.45)",
},
colors: {
brand: {
50: "#ecfeff",
100: "#cffafe",
200: "#a5f3fc",
300: "#67e8f9",
400: "#22d3ee",
500: "#06b6d4",
600: "#0891b2",
700: "#0e7490",
800: "#155e75",
900: "#164e63",
950: "#083344",
},
},
},
},
plugins: [],
};

19
frontend/tsconfig.json Normal file
View file

@ -0,0 +1,19 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "ESNext",
"moduleResolution": "Bundler",
"strict": true,
"jsx": "preserve",
"resolveJsonModule": true,
"isolatedModules": true,
"esModuleInterop": true,
"lib": ["ES2022", "ESNext", "DOM", "DOM.Iterable"],
"types": ["vite/client", "node"],
"baseUrl": ".",
"paths": {
"@/*": ["src/*"]
}
},
"include": ["src/**/*.ts", "src/**/*.d.ts", "src/**/*.tsx", "src/**/*.vue"]
}

26
frontend/vite.config.ts Normal file
View file

@ -0,0 +1,26 @@
import { defineConfig } from "vite";
import vue from "@vitejs/plugin-vue";
import path from "node:path";
const devApiProxyTarget = process.env.VITE_DEV_API_PROXY_TARGET ?? "http://localhost:8000";
export default defineConfig({
plugins: [vue()],
server: {
proxy: {
"/api": {
target: devApiProxyTarget,
changeOrigin: true,
},
"/healthz": {
target: devApiProxyTarget,
changeOrigin: true,
},
},
},
resolve: {
alias: {
"@": path.resolve(__dirname, "src"),
},
},
});

16
frontend/vitest.config.ts Normal file
View file

@ -0,0 +1,16 @@
import { defineConfig } from "vitest/config";
import vue from "@vitejs/plugin-vue";
import path from "node:path";
export default defineConfig({
plugins: [vue()],
resolve: {
alias: {
"@": path.resolve(__dirname, "src"),
},
},
test: {
environment: "node",
include: ["src/**/*.test.ts"],
},
});

1
pyproject.toml
View file

@ -14,6 +14,7 @@ dependencies = [
"asyncpg>=0.30,<0.31",
"fastapi>=0.116,<0.117",
"itsdangerous>=2.2,<3.0",
"python-multipart>=0.0.9,<0.1",
"sqlalchemy>=2.0,<2.1",
"uvicorn[standard]>=0.34,<0.35",
]

113
scripts/check_frontend_api_contract.py Executable file
View file

@ -0,0 +1,113 @@
#!/usr/bin/env python3
from __future__ import annotations
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BACKEND_ROUTER_DIR = ROOT / "app" / "api" / "routers"
FRONTEND_API_DIRS = [
ROOT / "frontend" / "src" / "features",
ROOT / "frontend" / "src" / "lib" / "auth",
ROOT / "frontend" / "src" / "lib" / "api",
]
BACKEND_PREFIX = "/api/v1"
BACKEND_ROUTE_PREFIX_RE = re.compile(r'router\s*=\s*APIRouter\(\s*prefix\s*=\s*"([^"]+)"')
BACKEND_DECORATOR_RE = re.compile(r'@router\.(get|post|put|patch|delete)\(\s*"([^"]*)"')
FRONTEND_CALL_RE = re.compile(
r'apiRequest(?:<[\s\S]*?>)?\(\s*("[^"]+"|\'[^\']+\'|`[^`]+`)\s*,\s*(\{[\s\S]{0,280}?\})',
re.MULTILINE,
)
FRONTEND_METHOD_RE = re.compile(r'method\s*:\s*"([A-Z]+)"')
def normalize_path(path: str) -> str:
cleaned = path.strip()
if not cleaned:
return "/"
if "?" in cleaned:
cleaned = cleaned.split("?", 1)[0]
cleaned = re.sub(r"\$\{[^}]+\}", "{}", cleaned)
cleaned = re.sub(r"\{[^}]+\}", "{}", cleaned)
if not cleaned.startswith("/"):
cleaned = f"/{cleaned}"
if len(cleaned) > 1 and cleaned.endswith("/"):
cleaned = cleaned.rstrip("/")
return cleaned
def read_backend_routes() -> set[tuple[str, str]]:
routes: set[tuple[str, str]] = set()
for path in sorted(BACKEND_ROUTER_DIR.glob("*.py")):
text = path.read_text(encoding="utf-8")
prefix_match = BACKEND_ROUTE_PREFIX_RE.search(text)
if not prefix_match:
continue
router_prefix = prefix_match.group(1)
for method, route_path in BACKEND_DECORATOR_RE.findall(text):
if route_path:
full_path = f"{BACKEND_PREFIX}{router_prefix}{route_path}"
else:
full_path = f"{BACKEND_PREFIX}{router_prefix}"
routes.add((method.upper(), normalize_path(full_path)))
return routes
def read_frontend_routes() -> set[tuple[str, str]]:
routes: set[tuple[str, str]] = set()
files: list[Path] = []
for directory in FRONTEND_API_DIRS:
if directory.exists():
files.extend(sorted(directory.rglob("*.ts")))
for path in files:
text = path.read_text(encoding="utf-8")
for path_literal, options_block in FRONTEND_CALL_RE.findall(text):
raw_path = path_literal.strip("`\"'")
if not raw_path.startswith("/"):
continue
method_match = FRONTEND_METHOD_RE.search(options_block)
method = method_match.group(1) if method_match else "GET"
normalized = normalize_path(f"{BACKEND_PREFIX}{raw_path}")
routes.add((method, normalized))
return routes
def main() -> int:
backend_routes = read_backend_routes()
frontend_routes = read_frontend_routes()
missing = sorted(frontend_routes - backend_routes)
if missing:
print("Frontend API contract drift detected. Missing backend routes:")
for method, route in missing:
print(f"- {method} {route}")
return 1
print(
f"Frontend API contract check passed: {len(frontend_routes)} frontend routes match backend definitions."
)
return 0
if __name__ == "__main__":
sys.exit(main())

7
scripts/smoke_compose.sh
View file

@ -4,15 +4,14 @@ set -euo pipefail
export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-scholarr-smoke}"
export APP_PORT="${APP_PORT:-8000}"
export APP_HOST_PORT="${APP_HOST_PORT:-18000}"
export POSTGRES_PORT="${POSTGRES_PORT:-15432}"
cleanup() {
docker compose down -v --remove-orphans
docker compose -f docker-compose.yml -f docker-compose.dev.yml down -v --remove-orphans
}
trap cleanup EXIT
docker compose up -d --build
docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
echo "Waiting for application health check..."
for _ in {1..45}; do
@ -24,4 +23,4 @@ done
curl -fsS "http://localhost:${APP_HOST_PORT}/healthz" >/dev/null
docker compose run --rm app uv run pytest tests/smoke -m "integration and smoke"
docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest tests/smoke -m "integration and smoke"

355
tests/integration/test_api_v1.py
View file

@ -1,11 +1,16 @@
from __future__ import annotations
from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.runtime_deps import get_scholar_source
from app.main import app
from app.services.scholar_source import FetchResult
from app.settings import settings
from tests.integration.helpers import insert_user, login_user
@ -299,14 +304,14 @@ async def test_api_scholars_crud_flow(db_session: AsyncSession) -> None:
missing_csrf = client.post(
"/api/v1/scholars",
json={"scholar_id": "abcDEF123456", "display_name": "Ada"},
json={"scholar_id": "abcDEF123456"},
)
assert missing_csrf.status_code == 403
assert missing_csrf.json()["error"]["code"] == "csrf_invalid"
create_response = client.post(
"/api/v1/scholars",
json={"scholar_id": "abcDEF123456", "display_name": "Ada"},
json={"scholar_id": "abcDEF123456"},
headers=headers,
)
assert create_response.status_code == 201
@ -334,6 +339,243 @@ async def test_api_scholars_crud_flow(db_session: AsyncSession) -> None:
assert delete_response.json()["data"]["message"] == "Scholar deleted."
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_scholars_search_and_profile_image_management(
db_session: AsyncSession,
tmp_path: Path,
) -> None:
await insert_user(
db_session,
email="api-scholar-images@example.com",
password="api-password",
)
class StubScholarSource:
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
assert scholar_id == "abcDEF123456"
return FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
status_code=200,
final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
body=(
"<html><head>"
'<meta property="og:image" content="https://images.example.com/ada.png" />'
"</head><body>"
'<div id="gsc_prf_in">Ada Lovelace</div>'
"</body></html>"
),
error=None,
)
async def fetch_author_search_html(self, query: str, *, start: int) -> FetchResult:
assert query == "Ada Lovelace"
assert start == 0
return FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
status_code=200,
final_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
body=(
'<div class="gsc_1usr">'
'<img src="/citations/images/avatar_scholar_256.png" />'
'<a class="gs_ai_name" href="/citations?hl=en&user=abcDEF123456">Ada Lovelace</a>'
'<div class="gs_ai_aff">Analytical Engine</div>'
'<div class="gs_ai_eml">Verified email at computing.example</div>'
'<div class="gs_ai_cby">Cited by 42</div>'
'<a class="gs_ai_one_int">Mathematics</a>'
"</div>"
),
error=None,
)
previous_upload_dir = settings.scholar_image_upload_dir
previous_upload_max_bytes = settings.scholar_image_upload_max_bytes
app.dependency_overrides[get_scholar_source] = lambda: StubScholarSource()
object.__setattr__(settings, "scholar_image_upload_dir", str(tmp_path / "scholar_images"))
object.__setattr__(settings, "scholar_image_upload_max_bytes", 1_000_000)
try:
client = TestClient(app)
login_user(client, email="api-scholar-images@example.com", password="api-password")
headers = _api_csrf_headers(client)
search_response = client.get("/api/v1/scholars/search", params={"query": "Ada Lovelace", "limit": 5})
assert search_response.status_code == 200
search_payload = search_response.json()["data"]
assert search_payload["state"] == "ok"
assert len(search_payload["candidates"]) == 1
candidate = search_payload["candidates"][0]
assert candidate["scholar_id"] == "abcDEF123456"
assert candidate["profile_image_url"] == "https://scholar.google.com/citations/images/avatar_scholar_256.png"
create_response = client.post(
"/api/v1/scholars",
json={
"scholar_id": candidate["scholar_id"],
},
headers=headers,
)
assert create_response.status_code == 201
created = create_response.json()["data"]
scholar_profile_id = int(created["id"])
assert created["profile_image_source"] == "scraped"
assert created["profile_image_url"] == "https://images.example.com/ada.png"
set_url_response = client.put(
f"/api/v1/scholars/{scholar_profile_id}/image/url",
json={"image_url": "https://cdn.example.com/custom-avatar.png"},
headers=headers,
)
assert set_url_response.status_code == 200
set_url_data = set_url_response.json()["data"]
assert set_url_data["profile_image_source"] == "override"
assert set_url_data["profile_image_url"] == "https://cdn.example.com/custom-avatar.png"
uploaded_bytes = b"\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIHDR\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x01"
upload_response = client.post(
f"/api/v1/scholars/{scholar_profile_id}/image/upload",
files={"image": ("avatar.png", uploaded_bytes, "image/png")},
headers=headers,
)
assert upload_response.status_code == 200
upload_data = upload_response.json()["data"]
assert upload_data["profile_image_source"] == "upload"
assert upload_data["profile_image_url"] == f"/api/v1/scholars/{scholar_profile_id}/image/upload"
uploaded_image_response = client.get(f"/api/v1/scholars/{scholar_profile_id}/image/upload")
assert uploaded_image_response.status_code == 200
assert uploaded_image_response.headers["content-type"].startswith("image/png")
assert uploaded_image_response.content == uploaded_bytes
clear_response = client.delete(
f"/api/v1/scholars/{scholar_profile_id}/image",
headers=headers,
)
assert clear_response.status_code == 200
clear_data = clear_response.json()["data"]
assert clear_data["profile_image_source"] == "scraped"
assert clear_data["profile_image_url"] == "https://images.example.com/ada.png"
finally:
app.dependency_overrides.pop(get_scholar_source, None)
object.__setattr__(settings, "scholar_image_upload_dir", previous_upload_dir)
object.__setattr__(
settings,
"scholar_image_upload_max_bytes",
previous_upload_max_bytes,
)
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_manual_run_skips_unchanged_initial_page_for_scholar(
db_session: AsyncSession,
) -> None:
await insert_user(
db_session,
email="api-skip-unchanged@example.com",
password="api-password",
)
profile_html = """
<html>
<head>
<meta property="og:image" content="https://images.example.com/skip.png" />
</head>
<body>
<div id="gsc_prf_in">Skip Candidate</div>
<span id="gsc_a_nn">Articles 1-1</span>
<table>
<tbody id="gsc_a_b">
<tr class="gsc_a_tr">
<td class="gsc_a_t">
<a class="gsc_a_at" href="/citations?view_op=view_citation&citation_for_view=abcDEF123456:xyz123">Stable Paper</a>
<div class="gs_gray">A Author</div>
<div class="gs_gray">Stable Venue</div>
</td>
<td class="gsc_a_c"><a class="gsc_a_ac">5</a></td>
<td class="gsc_a_y"><span class="gsc_a_h">2024</span></td>
</tr>
</tbody>
</table>
</body>
</html>
"""
class StubScholarSource:
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
assert scholar_id == "abcDEF123456"
return FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
status_code=200,
final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
body=profile_html,
error=None,
)
async def fetch_profile_page_html(
self,
scholar_id: str,
*,
cstart: int,
pagesize: int,
) -> FetchResult:
assert scholar_id == "abcDEF123456"
assert cstart == 0
assert pagesize == settings.ingestion_page_size
return FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
status_code=200,
final_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
body=profile_html,
error=None,
)
app.dependency_overrides[get_scholar_source] = lambda: StubScholarSource()
try:
client = TestClient(app)
login_user(client, email="api-skip-unchanged@example.com", password="api-password")
headers = _api_csrf_headers(client)
create_response = client.post(
"/api/v1/scholars",
json={"scholar_id": "abcDEF123456"},
headers=headers,
)
assert create_response.status_code == 201
first_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "skip-unchanged-run-001"},
)
assert first_run_response.status_code == 200
first_run_id = int(first_run_response.json()["data"]["run_id"])
first_run_detail = client.get(f"/api/v1/runs/{first_run_id}")
assert first_run_detail.status_code == 200
first_results = first_run_detail.json()["data"]["scholar_results"]
assert len(first_results) == 1
assert first_results[0]["state_reason"] != "no_change_initial_page_signature"
second_run_response = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "skip-unchanged-run-002"},
)
assert second_run_response.status_code == 200
second_run_id = int(second_run_response.json()["data"]["run_id"])
second_run_detail = client.get(f"/api/v1/runs/{second_run_id}")
assert second_run_detail.status_code == 200
second_results = second_run_detail.json()["data"]["scholar_results"]
assert len(second_results) == 1
assert second_results[0]["state_reason"] == "no_change_initial_page_signature"
assert second_results[0]["publication_count"] == 0
assert second_results[0]["outcome"] == "success"
finally:
app.dependency_overrides.pop(get_scholar_source, None)
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
@ -392,6 +634,13 @@ async def test_api_runs_manual_and_queue_actions(db_session: AsyncSession) -> No
assert run_payload["status"] in {"success", "partial_failure", "failed"}
assert run_payload["reused_existing_run"] is False
assert run_payload["idempotency_key"] == "manual-run-0001"
run_id = int(run_payload["run_id"])
stored_key = await db_session.execute(
text("SELECT idempotency_key FROM crawl_runs WHERE id = :run_id"),
{"run_id": run_id},
)
assert stored_key.scalar_one() == "manual-run-0001"
replay_response = client.post(
"/api/v1/runs/manual",
@ -405,7 +654,6 @@ async def test_api_runs_manual_and_queue_actions(db_session: AsyncSession) -> No
runs_response = client.get("/api/v1/runs")
assert runs_response.status_code == 200
assert len(runs_response.json()["data"]["runs"]) >= 1
run_id = int(run_payload["run_id"])
run_detail_response = client.get(f"/api/v1/runs/{run_id}")
assert run_detail_response.status_code == 200
@ -497,12 +745,77 @@ async def test_api_runs_manual_and_queue_actions(db_session: AsyncSession) -> No
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_publications_list_and_mark_read(db_session: AsyncSession) -> None:
await insert_user(
user_id = await insert_user(
db_session,
email="api-pubs@example.com",
password="api-password",
)
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, :scholar_id, :display_name, true)
RETURNING id
"""
),
{
"user_id": user_id,
"scholar_id": "abcDEF123456",
"display_name": "Publication Owner",
},
)
scholar_profile_id = int(scholar_result.scalar_one())
publication_a = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 10)
RETURNING id
"""
),
{
"fingerprint": f"{user_id:064x}",
"title_raw": "Paper A",
"title_normalized": "paper a",
},
)
publication_a_id = int(publication_a.scalar_one())
publication_b = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 4)
RETURNING id
"""
),
{
"fingerprint": f"{(user_id + 1):064x}",
"title_raw": "Paper B",
"title_normalized": "paper b",
},
)
publication_b_id = int(publication_b.scalar_one())
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
VALUES
(:scholar_profile_id, :publication_a_id, false),
(:scholar_profile_id, :publication_b_id, false)
"""
),
{
"scholar_profile_id": scholar_profile_id,
"publication_a_id": publication_a_id,
"publication_b_id": publication_b_id,
},
)
await db_session.commit()
client = TestClient(app)
login_user(client, email="api-pubs@example.com", password="api-password")
headers = _api_csrf_headers(client)
@ -512,7 +825,39 @@ async def test_api_publications_list_and_mark_read(db_session: AsyncSession) ->
data = list_response.json()["data"]
assert data["mode"] == "all"
assert isinstance(data["publications"], list)
assert len(data["publications"]) == 2
mark_selected_response = client.post(
"/api/v1/publications/mark-read",
json={
"selections": [
{
"scholar_profile_id": scholar_profile_id,
"publication_id": publication_a_id,
}
]
},
headers=headers,
)
assert mark_selected_response.status_code == 200
assert mark_selected_response.json()["data"]["requested_count"] == 1
assert mark_selected_response.json()["data"]["updated_count"] == 1
read_state = await db_session.execute(
text(
"""
SELECT publication_id, is_read
FROM scholar_publications
WHERE scholar_profile_id = :scholar_profile_id
ORDER BY publication_id
"""
),
{"scholar_profile_id": scholar_profile_id},
)
states = {int(row[0]): bool(row[1]) for row in read_state.all()}
assert states[publication_a_id] is True
assert states[publication_b_id] is False
mark_response = client.post("/api/v1/publications/mark-all-read", headers=headers)
assert mark_response.status_code == 200
assert "updated_count" in mark_response.json()["data"]
assert mark_response.json()["data"]["updated_count"] == 1

93
tests/integration/test_migrations.py
View file

@ -14,7 +14,7 @@ EXPECTED_TABLES = {
}
EXPECTED_ENUMS = {"run_status", "run_trigger_type"}
EXPECTED_REVISION = "20260217_0004"
EXPECTED_REVISION = "20260217_0007"
@pytest.mark.integration
@ -91,3 +91,94 @@ async def test_ingestion_queue_table_has_status_and_drop_columns(db_session: Asy
)
columns = {row[0] for row in result}
assert columns == {"status", "dropped_reason", "dropped_at"}
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.migrations
@pytest.mark.asyncio
async def test_crawl_runs_table_has_idempotency_key_column(db_session: AsyncSession) -> None:
result = await db_session.execute(
text(
"""
SELECT 1
FROM information_schema.columns
WHERE table_name = 'crawl_runs' AND column_name = 'idempotency_key'
"""
)
)
assert result.scalar_one() == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.migrations
@pytest.mark.asyncio
async def test_crawl_runs_has_manual_idempotency_unique_index(db_session: AsyncSession) -> None:
result = await db_session.execute(
text(
"""
SELECT indexdef
FROM pg_indexes
WHERE schemaname = 'public'
AND tablename = 'crawl_runs'
AND indexname = 'uq_crawl_runs_user_manual_idempotency_key'
"""
)
)
indexdef = result.scalar_one()
assert "UNIQUE INDEX" in indexdef
assert "(user_id, idempotency_key)" in indexdef
assert "WHERE" in indexdef
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.migrations
@pytest.mark.asyncio
async def test_scholar_profiles_has_profile_image_columns(db_session: AsyncSession) -> None:
result = await db_session.execute(
text(
"""
SELECT column_name
FROM information_schema.columns
WHERE table_name = 'scholar_profiles'
AND column_name IN (
'profile_image_url',
'profile_image_override_url',
'profile_image_upload_path'
)
"""
)
)
columns = {row[0] for row in result}
assert columns == {
"profile_image_url",
"profile_image_override_url",
"profile_image_upload_path",
}
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.migrations
@pytest.mark.asyncio
async def test_scholar_profiles_has_initial_page_snapshot_columns(db_session: AsyncSession) -> None:
result = await db_session.execute(
text(
"""
SELECT column_name
FROM information_schema.columns
WHERE table_name = 'scholar_profiles'
AND column_name IN (
'last_initial_page_fingerprint_sha256',
'last_initial_page_checked_at'
)
"""
)
)
columns = {row[0] for row in result}
assert columns == {
"last_initial_page_fingerprint_sha256",
"last_initial_page_checked_at",
}

2
tests/smoke/test_migration_schema.py
View file

@ -9,7 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
@pytest.mark.asyncio
async def test_schema_head_revision_is_available(db_session: AsyncSession) -> None:
result = await db_session.execute(text("SELECT version_num FROM alembic_version"))
assert result.scalar_one() == "20260217_0004"
assert result.scalar_one() == "20260217_0007"
@pytest.mark.integration

97
tests/unit/test_scholar_parser.py
View file

@ -2,7 +2,11 @@ from __future__ import annotations
from pathlib import Path
from app.services.scholar_parser import ParseState, parse_profile_page
from app.services.scholar_parser import (
ParseState,
parse_author_search_page,
parse_profile_page,
)
from app.services.scholar_source import FetchResult
@ -30,6 +34,8 @@ def test_parse_profile_page_extracts_core_fields_from_fixture() -> None:
assert parsed.state == ParseState.OK
assert parsed.state_reason == "publications_extracted"
assert parsed.profile_name == "Bangar Raju Cherukuri"
assert parsed.profile_image_url
assert parsed.profile_image_url.startswith("http")
assert len(parsed.publications) >= 10
assert parsed.has_show_more_button is True
assert parsed.articles_range is not None
@ -120,7 +126,22 @@ def test_parse_profile_page_reports_network_reason_when_status_missing() -> None
parsed = parse_profile_page(fetch_result)
assert parsed.state == ParseState.NETWORK_ERROR
assert parsed.state_reason == "network_error_missing_status_code"
assert parsed.state_reason == "network_timeout"
def test_parse_profile_page_reports_dns_network_reason_when_status_missing() -> None:
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&user=abcDEF123456",
status_code=None,
final_url=None,
body="",
error="<urlopen error [Errno -3] Temporary failure in name resolution>",
)
parsed = parse_profile_page(fetch_result)
assert parsed.state == ParseState.NETWORK_ERROR
assert parsed.state_reason == "network_dns_resolution_failed"
def test_parse_profile_page_ignores_no_results_keyword_inside_script_blocks() -> None:
@ -238,3 +259,75 @@ def test_parse_profile_page_regression_fixture_blocked_redirect() -> None:
assert parsed.state_reason == "blocked_accounts_redirect"
assert parsed.profile_name is None
assert len(parsed.publications) == 0
def test_parse_author_search_page_extracts_candidates_with_image() -> None:
html = """
<html>
<body>
<div class="gsc_1usr">
<img src="/citations/images/avatar_scholar_256.png" />
<a class="gs_ai_name" href="/citations?hl=en&user=abcDEF123456">Ada Lovelace</a>
<div class="gs_ai_aff">Analytical Engine Lab</div>
<div class="gs_ai_eml">Verified email at computing.example</div>
<div class="gs_ai_cby">Cited by 128</div>
<a class="gs_ai_one_int">Algorithms</a>
<a class="gs_ai_one_int">Mathematics</a>
</div>
</body>
</html>
"""
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
status_code=200,
final_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
body=html,
error=None,
)
parsed = parse_author_search_page(fetch_result)
assert parsed.state == ParseState.OK
assert parsed.state_reason == "author_candidates_extracted"
assert len(parsed.candidates) == 1
candidate = parsed.candidates[0]
assert candidate.scholar_id == "abcDEF123456"
assert candidate.display_name == "Ada Lovelace"
assert candidate.affiliation == "Analytical Engine Lab"
assert candidate.email_domain == "computing.example"
assert candidate.cited_by_count == 128
assert candidate.interests == ["Algorithms", "Mathematics"]
assert candidate.profile_url.startswith("https://scholar.google.com/citations")
assert candidate.profile_image_url == "https://scholar.google.com/citations/images/avatar_scholar_256.png"
def test_parse_author_search_page_detects_no_results_keyword() -> None:
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=nope",
status_code=200,
final_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=nope",
body="<html><body>Your search didn't match any user profiles.</body></html>",
error=None,
)
parsed = parse_author_search_page(fetch_result)
assert parsed.state == ParseState.NO_RESULTS
assert parsed.state_reason == "no_results_keyword_detected"
assert len(parsed.candidates) == 0
def test_parse_author_search_page_classifies_http_429_as_blocked() -> None:
fetch_result = FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
status_code=429,
final_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
body="<html><body>Too many requests</body></html>",
error=None,
)
parsed = parse_author_search_page(fetch_result)
assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA
assert parsed.state_reason == "blocked_http_429_rate_limited"

176
tests/unit/test_scholar_search_safety.py Normal file
View file

@ -0,0 +1,176 @@
from __future__ import annotations
import pytest
from app.services import scholars as scholar_service
from app.services.scholar_parser import ParseState
from app.services.scholar_source import FetchResult
class StubScholarSource:
def __init__(self, fetch_results: list[FetchResult]) -> None:
self._fetch_results = list(fetch_results)
self.calls = 0
async def fetch_author_search_html(self, query: str, *, start: int = 0) -> FetchResult:
assert start == 0
self.calls += 1
if not self._fetch_results:
raise RuntimeError("No stub fetch results configured.")
index = min(self.calls - 1, len(self._fetch_results) - 1)
return self._fetch_results[index]
@pytest.fixture(autouse=True)
def reset_author_search_runtime_state() -> None:
scholar_service._reset_author_search_runtime_state_for_tests()
yield
scholar_service._reset_author_search_runtime_state_for_tests()
def _ok_author_search_fetch() -> FetchResult:
body = (
"<html><body>"
'<div class="gsc_1usr">'
'<img src="/citations/images/avatar_scholar_256.png" />'
'<a class="gs_ai_name" href="/citations?hl=en&user=abcDEF123456">Ada Lovelace</a>'
'<div class="gs_ai_aff">Analytical Engine</div>'
'<div class="gs_ai_eml">Verified email at computing.example</div>'
'<div class="gs_ai_cby">Cited by 42</div>'
'<a class="gs_ai_one_int">Mathematics</a>'
"</div>"
"</body></html>"
)
return FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
status_code=200,
final_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
body=body,
error=None,
)
def _blocked_author_search_fetch() -> FetchResult:
return FetchResult(
requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
status_code=200,
final_url=(
"https://accounts.google.com/v3/signin/identifier"
"?continue=https%3A%2F%2Fscholar.google.com%2Fcitations"
),
body="<html><body>Sign in</body></html>",
error=None,
)
@pytest.mark.asyncio
async def test_search_author_candidates_serves_cached_response_for_same_query() -> None:
source = StubScholarSource([_ok_author_search_fetch()])
first = await scholar_service.search_author_candidates(
source=source,
query="Ada Lovelace",
limit=10,
network_error_retries=0,
retry_backoff_seconds=0.0,
search_enabled=True,
cache_ttl_seconds=600,
blocked_cache_ttl_seconds=60,
cache_max_entries=64,
min_interval_seconds=0.0,
interval_jitter_seconds=0.0,
cooldown_block_threshold=3,
cooldown_seconds=300,
)
second = await scholar_service.search_author_candidates(
source=source,
query="Ada Lovelace",
limit=10,
network_error_retries=0,
retry_backoff_seconds=0.0,
search_enabled=True,
cache_ttl_seconds=600,
blocked_cache_ttl_seconds=60,
cache_max_entries=64,
min_interval_seconds=0.0,
interval_jitter_seconds=0.0,
cooldown_block_threshold=3,
cooldown_seconds=300,
)
assert first.state == ParseState.OK
assert second.state == ParseState.OK
assert source.calls == 1
assert len(second.candidates) == 1
assert second.candidates[0].scholar_id == "abcDEF123456"
assert "author_search_served_from_cache" in second.warnings
@pytest.mark.asyncio
async def test_search_author_candidates_trips_cooldown_after_blocked_responses() -> None:
source = StubScholarSource([_blocked_author_search_fetch()])
first = await scholar_service.search_author_candidates(
source=source,
query="Blocked Query",
limit=10,
network_error_retries=0,
retry_backoff_seconds=0.0,
search_enabled=True,
cache_ttl_seconds=0,
blocked_cache_ttl_seconds=0,
cache_max_entries=64,
min_interval_seconds=0.0,
interval_jitter_seconds=0.0,
cooldown_block_threshold=1,
cooldown_seconds=300,
)
second = await scholar_service.search_author_candidates(
source=source,
query="Blocked Query",
limit=10,
network_error_retries=0,
retry_backoff_seconds=0.0,
search_enabled=True,
cache_ttl_seconds=0,
blocked_cache_ttl_seconds=0,
cache_max_entries=64,
min_interval_seconds=0.0,
interval_jitter_seconds=0.0,
cooldown_block_threshold=1,
cooldown_seconds=300,
)
assert first.state == ParseState.BLOCKED_OR_CAPTCHA
assert "author_search_circuit_breaker_armed" in first.warnings
assert source.calls == 1
assert second.state == ParseState.BLOCKED_OR_CAPTCHA
assert second.state_reason == scholar_service.SEARCH_COOLDOWN_REASON
assert "author_search_cooldown_active" in second.warnings
@pytest.mark.asyncio
async def test_search_author_candidates_can_be_disabled_by_configuration() -> None:
source = StubScholarSource([_ok_author_search_fetch()])
parsed = await scholar_service.search_author_candidates(
source=source,
query="Ada Lovelace",
limit=5,
network_error_retries=0,
retry_backoff_seconds=0.0,
search_enabled=False,
cache_ttl_seconds=600,
blocked_cache_ttl_seconds=60,
cache_max_entries=64,
min_interval_seconds=0.0,
interval_jitter_seconds=0.0,
cooldown_block_threshold=1,
cooldown_seconds=300,
)
assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA
assert parsed.state_reason == scholar_service.SEARCH_DISABLED_REASON
assert source.calls == 0
assert "author_search_disabled_by_configuration" in parsed.warnings

11
uv.lock generated
View file

@ -576,6 +576,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230 },
]
[[package]]
name = "python-multipart"
version = "0.0.22"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/94/01/979e98d542a70714b0cb2b6728ed0b7c46792b695e3eaec3e20711271ca3/python_multipart-0.0.22.tar.gz", hash = "sha256:7340bef99a7e0032613f56dc36027b959fd3b30a787ed62d310e951f7c3a3a58", size = 37612 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/1b/d0/397f9626e711ff749a95d96b7af99b9c566a9bb5129b8e4c10fc4d100304/python_multipart-0.0.22-py3-none-any.whl", hash = "sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155", size = 24579 },
]
[[package]]
name = "pyyaml"
version = "6.0.3"
@ -632,6 +641,7 @@ dependencies = [
{ name = "asyncpg" },
{ name = "fastapi" },
{ name = "itsdangerous" },
{ name = "python-multipart" },
{ name = "sqlalchemy" },
{ name = "uvicorn", extra = ["standard"] },
]
@ -653,6 +663,7 @@ requires-dist = [
{ name = "itsdangerous", specifier = ">=2.2,<3.0" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3,<9.0" },
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.25,<0.26" },
{ name = "python-multipart", specifier = ">=0.0.9,<0.1" },
{ name = "sqlalchemy", specifier = ">=2.0,<2.1" },
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.34,<0.35" },
]