Big changes

This commit is contained in:
Justin Visser 2026-02-21 17:33:05 +01:00
parent 4240ad38e2
commit e3f0d63fec
99 changed files with 8804 additions and 1731 deletions

View file

@ -1,26 +1,52 @@
# ------------------------------
# Compose + Database
# ------------------------------
POSTGRES_DB=scholar POSTGRES_DB=scholar
POSTGRES_USER=scholar POSTGRES_USER=scholar
POSTGRES_PASSWORD=change-me POSTGRES_PASSWORD=change-me
DATABASE_URL=postgresql+asyncpg://scholar:scholar@db:5432/scholar DATABASE_URL=postgresql+asyncpg://scholar:scholar@db:5432/scholar
# Optional override. If empty, tests derive "<DATABASE_URL db name>_test". # Optional override. If empty, tests derive "<DATABASE_URL db name>_test".
TEST_DATABASE_URL= TEST_DATABASE_URL=
SCHOLARR_IMAGE=justinzeus/scholarr:latest SCHOLARR_IMAGE=justinzeus/scholarr:latest
# ------------------------------
# App Runtime + Networking
# ------------------------------
APP_NAME=scholarr APP_NAME=scholarr
APP_HOST=0.0.0.0 APP_HOST=0.0.0.0
APP_PORT=8000 APP_PORT=8000
APP_HOST_PORT=8000 APP_HOST_PORT=8000
APP_RELOAD=0 APP_RELOAD=0
MIGRATE_ON_START=1
FRONTEND_ENABLED=1 FRONTEND_ENABLED=1
FRONTEND_DIST_DIR=/app/frontend/dist FRONTEND_DIST_DIR=/app/frontend/dist
# ------------------------------
# Database Pool
# ------------------------------
DATABASE_POOL_MODE=auto
DATABASE_POOL_SIZE=5
DATABASE_POOL_MAX_OVERFLOW=10
DATABASE_POOL_TIMEOUT_SECONDS=30
# ------------------------------
# Frontend Dev Overrides
# ------------------------------
FRONTEND_HOST_PORT=5173 FRONTEND_HOST_PORT=5173
CHOKIDAR_USEPOLLING=1 CHOKIDAR_USEPOLLING=1
VITE_DEV_API_PROXY_TARGET=http://app:8000 VITE_DEV_API_PROXY_TARGET=http://app:8000
MIGRATE_ON_START=1
# ------------------------------
# Auth + Session
# ------------------------------
SESSION_SECRET_KEY=replace-with-a-long-random-secret-at-least-32-characters SESSION_SECRET_KEY=replace-with-a-long-random-secret-at-least-32-characters
SESSION_COOKIE_SECURE=1 SESSION_COOKIE_SECURE=1
LOGIN_RATE_LIMIT_ATTEMPTS=5
LOGIN_RATE_LIMIT_WINDOW_SECONDS=60
# ------------------------------
# HTTP Security Headers + CSP
# ------------------------------
SECURITY_HEADERS_ENABLED=1 SECURITY_HEADERS_ENABLED=1
SECURITY_X_CONTENT_TYPE_OPTIONS=nosniff SECURITY_X_CONTENT_TYPE_OPTIONS=nosniff
SECURITY_X_FRAME_OPTIONS=DENY SECURITY_X_FRAME_OPTIONS=DENY
@ -36,16 +62,23 @@ SECURITY_STRICT_TRANSPORT_SECURITY_ENABLED=0
SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE=31536000 SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE=31536000
SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS=1 SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS=1
SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD=0 SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD=0
LOGIN_RATE_LIMIT_ATTEMPTS=5
LOGIN_RATE_LIMIT_WINDOW_SECONDS=60 # ------------------------------
# Logging
# ------------------------------
LOG_LEVEL=INFO LOG_LEVEL=INFO
LOG_FORMAT=console LOG_FORMAT=console
LOG_REQUESTS=1 LOG_REQUESTS=1
LOG_UVICORN_ACCESS=0 LOG_UVICORN_ACCESS=0
LOG_REQUEST_SKIP_PATHS=/healthz LOG_REQUEST_SKIP_PATHS=/healthz
LOG_REDACT_FIELDS= LOG_REDACT_FIELDS=
# ------------------------------
# Scheduler + Ingestion Safety
# ------------------------------
SCHEDULER_ENABLED=1 SCHEDULER_ENABLED=1
SCHEDULER_TICK_SECONDS=60 SCHEDULER_TICK_SECONDS=60
SCHEDULER_QUEUE_BATCH_SIZE=10
INGESTION_AUTOMATION_ALLOWED=1 INGESTION_AUTOMATION_ALLOWED=1
INGESTION_MANUAL_RUN_ALLOWED=1 INGESTION_MANUAL_RUN_ALLOWED=1
INGESTION_MIN_RUN_INTERVAL_MINUTES=15 INGESTION_MIN_RUN_INTERVAL_MINUTES=15
@ -63,7 +96,10 @@ INGESTION_CONTINUATION_QUEUE_ENABLED=1
INGESTION_CONTINUATION_BASE_DELAY_SECONDS=120 INGESTION_CONTINUATION_BASE_DELAY_SECONDS=120
INGESTION_CONTINUATION_MAX_DELAY_SECONDS=3600 INGESTION_CONTINUATION_MAX_DELAY_SECONDS=3600
INGESTION_CONTINUATION_MAX_ATTEMPTS=6 INGESTION_CONTINUATION_MAX_ATTEMPTS=6
SCHEDULER_QUEUE_BATCH_SIZE=10
# ------------------------------
# Scholar Images + Name Search Safety
# ------------------------------
SCHOLAR_IMAGE_UPLOAD_DIR=/var/lib/scholarr/uploads SCHOLAR_IMAGE_UPLOAD_DIR=/var/lib/scholarr/uploads
SCHOLAR_IMAGE_UPLOAD_MAX_BYTES=2000000 SCHOLAR_IMAGE_UPLOAD_MAX_BYTES=2000000
SCHOLAR_NAME_SEARCH_ENABLED=1 SCHOLAR_NAME_SEARCH_ENABLED=1
@ -76,21 +112,34 @@ SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD=1
SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS=1800 SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS=1800
SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD=2 SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD=2
SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD=3 SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD=3
# ------------------------------
# OA Enrichment + PDF Resolution
# ------------------------------
UNPAYWALL_ENABLED=1 UNPAYWALL_ENABLED=1
UNPAYWALL_EMAIL= UNPAYWALL_EMAIL=
UNPAYWALL_TIMEOUT_SECONDS=4.0 UNPAYWALL_TIMEOUT_SECONDS=4.0
UNPAYWALL_MIN_INTERVAL_SECONDS=0.6
UNPAYWALL_MAX_ITEMS_PER_REQUEST=20 UNPAYWALL_MAX_ITEMS_PER_REQUEST=20
UNPAYWALL_RETRY_COOLDOWN_SECONDS=1800 UNPAYWALL_RETRY_COOLDOWN_SECONDS=1800
UNPAYWALL_PDF_DISCOVERY_ENABLED=1
UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES=5
UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES=500000
PDF_AUTO_RETRY_INTERVAL_SECONDS=86400
PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS=3600
PDF_AUTO_RETRY_MAX_ATTEMPTS=3
CROSSREF_ENABLED=1 CROSSREF_ENABLED=1
CROSSREF_MAX_ROWS=10 CROSSREF_MAX_ROWS=10
CROSSREF_TIMEOUT_SECONDS=8.0 CROSSREF_TIMEOUT_SECONDS=8.0
CROSSREF_MIN_INTERVAL_SECONDS=0.6 CROSSREF_MIN_INTERVAL_SECONDS=0.6
CROSSREF_MAX_LOOKUPS_PER_REQUEST=8 CROSSREF_MAX_LOOKUPS_PER_REQUEST=8
# ------------------------------
# Startup Bootstrap + DB Wait
# ------------------------------
BOOTSTRAP_ADMIN_ON_START=0 BOOTSTRAP_ADMIN_ON_START=0
BOOTSTRAP_ADMIN_EMAIL= BOOTSTRAP_ADMIN_EMAIL=
BOOTSTRAP_ADMIN_PASSWORD= BOOTSTRAP_ADMIN_PASSWORD=
BOOTSTRAP_ADMIN_FORCE_PASSWORD=0 BOOTSTRAP_ADMIN_FORCE_PASSWORD=0
DB_WAIT_TIMEOUT_SECONDS=60 DB_WAIT_TIMEOUT_SECONDS=60
DB_WAIT_INTERVAL_SECONDS=2 DB_WAIT_INTERVAL_SECONDS=2

View file

@ -17,6 +17,9 @@ jobs:
- name: Enforce no generated artifacts - name: Enforce no generated artifacts
run: ./scripts/check_no_generated_artifacts.sh run: ./scripts/check_no_generated_artifacts.sh
- name: Enforce env contract parity
run: python3 scripts/check_env_contract.py
test: test:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: needs:
@ -106,12 +109,33 @@ jobs:
working-directory: frontend working-directory: frontend
run: npm run build run: npm run build
docs-quality:
runs-on: ubuntu-latest
needs:
- repo-hygiene
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Install docs dependencies
run: npm install --prefix website
- name: Docs build
run: npm --prefix website run build
docker-publish: docker-publish:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: needs:
- repo-hygiene - repo-hygiene
- test - test
- frontend-quality - frontend-quality
- docs-quality
if: github.event_name == 'push' && github.ref == 'refs/heads/main' if: github.event_name == 'push' && github.ref == 'refs/heads/main'
permissions: permissions:
contents: read contents: read

58
.github/workflows/docs-pages.yml vendored Normal file
View file

@ -0,0 +1,58 @@
name: Docs Pages
on:
workflow_dispatch:
push:
branches:
- main
paths:
- docs/**
- website/**
- .github/workflows/docs-pages.yml
permissions:
contents: read
concurrency:
group: docs-pages
cancel-in-progress: true
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Pages
uses: actions/configure-pages@v5
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Install docs dependencies
run: npm install --prefix website
- name: Build docs site
run: npm --prefix website run build
- name: Upload pages artifact
uses: actions/upload-pages-artifact@v3
with:
path: website/build
deploy:
needs: build
runs-on: ubuntu-latest
permissions:
pages: write
id-token: write
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4

2
.gitignore vendored
View file

@ -15,4 +15,6 @@ planning/
frontend/node_modules/ frontend/node_modules/
frontend/dist/ frontend/dist/
frontend/.vite/ frontend/.vite/
website/node_modules/
website/build/
AGENTS.MD AGENTS.MD

291
README.md
View file

@ -22,11 +22,7 @@ cp .env.example .env
- `POSTGRES_PASSWORD` - `POSTGRES_PASSWORD`
- `SESSION_SECRET_KEY` - `SESSION_SECRET_KEY`
3. Choose a deploy method below. 3. Start stack:
## Deploy Method A: Prebuilt Image (Recommended)
Use this for normal self-hosted deployment.
```bash ```bash
docker compose pull docker compose pull
@ -37,234 +33,36 @@ Open:
- App/API: `http://localhost:8000` - App/API: `http://localhost:8000`
- Health: `http://localhost:8000/healthz` - Health: `http://localhost:8000/healthz`
Upgrade: ## Documentation
Primary docs live under `docs/`.
- Index: `docs/README.md`
- Deploy and dev workflow: `docs/deploy/quickstart.md`
- Environment reference: `docs/reference/environment.md`
- API contract and payload conventions: `docs/reference/api_contract.md`
- Architecture and domain boundaries: `docs/architecture/domain_boundaries.md`
- Scrape safety runbook: `docs/ops/scrape_safety_runbook.md`
- DB backup/restore/integrity runbook: `docs/ops/db_runbook.md`
- Migration rollout checklist: `docs/ops/migration_checklist.md`
- Frontend theme inventory: `docs/frontend/theme_phase0_inventory.md`
- Contributing policy: `docs/contributing.md`
## Docs Site (Docusaurus)
Docusaurus source lives in `website/` and consumes markdown from `docs/`.
Local build:
```bash ```bash
docker compose pull cd website
docker compose up -d npm install
npm run build
``` ```
## Deploy Method B: Local Source Build + Dev Frontend GitHub Pages deploy is automated via `.github/workflows/docs-pages.yml`.
Use this for development on this repository. ## Quality Gates
```bash
docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
```
Open:
- API (FastAPI): `http://localhost:8000`
- Frontend dev server (Vite): `http://localhost:5173`
Stop:
```bash
docker compose -f docker-compose.yml -f docker-compose.dev.yml down
```
## Essential Files
- `docker-compose.yml`: deployment compose (prebuilt image).
- `docker-compose.dev.yml`: dev override (source mounts + Vite dev server).
- `.env.example`: full env variable template.
- `Dockerfile`: multi-stage build (frontend + backend runtime).
- `README.md`: deployment and operations reference.
- `CONTRIBUTING.md`: contribution policy and merge checklist.
- `docs/ops/scrape_safety_runbook.md`: scrape cooldown and blocked-IP operations guide.
- `scripts/check_no_generated_artifacts.sh`: tracked-artifact guard used by CI.
## Data Model Notes
- Scholar tracking is user-scoped: each account can track the same Scholar ID independently.
- Publications are shared/global records deduplicated by Scholar cluster ID and normalized fingerprint.
- Per-account visibility and read state is stored on scholar-publication links, not on the global publication row.
- Publication records include canonical Scholar detail URLs (`pub_url`), normalized DOI (`doi`), and resolved OA PDF links (`pdf_url`) when available.
## Backend Architecture (Refactored)
- Canonical service logic lives in `app/services/domains/*`; root-level `app/services/*.py` business-logic modules are removed.
- `app/services/domains/ingestion/*`: run orchestration (`application.py`), continuation queue (`queue.py`), scrape safety cooldown policy (`safety.py`), scheduler/queue draining (`scheduler.py`), plus shared constants/types/fingerprint helpers.
- `app/services/domains/scholar/*`: fail-fast Google Scholar parsing and source access. Parser flow is modularized across row extractors, state detection, constants, and parser types.
- `app/services/domains/scholars/*`: scholar CRUD, name-search safety/caching, profile-image validation/upload handling, and shared scholar validation helpers.
- `app/services/domains/publications/*`: publication listing/read-state query modules, lazy/background OA enrichment scheduling (non-blocking for list responses), and per-publication PDF retry support.
- `app/services/domains/doi/*`: DOI normalization helpers used by ingestion and enrichment.
- `app/services/domains/crossref/*`: DOI discovery fallback using Crossref metadata queries with bounded/paced request behavior.
- `app/services/domains/unpaywall/*`: OA resolution by DOI (best OA location + PDF URL extraction), using per-user email for API calls.
- `app/services/domains/runs/*`: run history summaries plus continuation queue status/retry/drop/clear operations.
- `app/services/domains/portability/*`: import/export of tracked scholars and publication-link state while preserving global publication deduplication.
- `app/services/domains/settings/*` and `app/services/domains/users/*`: user settings and user management services.
## Frontend Navigation and Layout Notes
- On mobile (`< lg`), primary navigation is behind a header hamburger and opens as a left-side drawer with backdrop.
- Mobile nav closes on route change, nav link click, and logout.
- Long lists are constrained to internal scroll containers in fill-layout pages (including dashboard recent publications and tracked scholars table view).
## Name Search Status
- Scholar name search is intentionally WIP in the UI.
- Current Google Scholar behavior often redirects name-search traffic to login/challenge flows, so production onboarding should use direct Scholar ID/profile URL adds.
## Environment Variables (Complete Reference)
Notes:
- Boolean envs accept: `1/0`, `true/false`, `yes/no`, `on/off`.
- Values shown are deployment defaults from `.env.example` and `docker-compose.yml`.
- Some internal app fallbacks may differ for local test/dev safety when env vars are omitted.
- `deploy` means used in regular deployment.
- `dev` means used in local dev workflow.
### Core Compose and Database
| Variable | Default | Options | Scope | Description |
| --- | --- | --- | --- | --- |
| `POSTGRES_DB` | `scholar` | any valid DB name | deploy, dev | PostgreSQL database name. |
| `POSTGRES_USER` | `scholar` | any valid DB user | deploy, dev | PostgreSQL user. |
| `POSTGRES_PASSWORD` | `change-me` | strong password | deploy, dev | PostgreSQL password. Required. |
| `DATABASE_URL` | `postgresql+asyncpg://scholar:scholar@db:5432/scholar` | valid SQLAlchemy asyncpg URL | deploy, dev, test | App database connection URL. |
| `TEST_DATABASE_URL` | empty | valid SQLAlchemy asyncpg URL | test | Optional explicit integration test DB URL. |
| `SCHOLARR_IMAGE` | `justinzeus/scholarr:latest` | any image ref | deploy | App image tag used by deployment compose. |
### App Runtime and Networking
| Variable | Default | Options | Scope | Description |
| --- | --- | --- | --- | --- |
| `APP_NAME` | `scholarr` | any string | deploy, dev | FastAPI app name/title. |
| `APP_HOST` | `0.0.0.0` | bind host | deploy, dev | Uvicorn bind host inside container. |
| `APP_PORT` | `8000` | integer | deploy, dev | Uvicorn bind port inside container. |
| `APP_HOST_PORT` | `8000` | integer | deploy, dev | Host port mapped to app container port 8000. |
| `APP_RELOAD` | `0` | boolean | deploy, dev | Enable uvicorn reload mode. Recommended `0` in deploy, `1` in dev. |
| `MIGRATE_ON_START` | `1` | boolean | deploy, dev | Run Alembic migrations during app startup. |
| `FRONTEND_ENABLED` | `1` | boolean | deploy, dev | Serve bundled frontend assets from FastAPI. |
| `FRONTEND_DIST_DIR` | `/app/frontend/dist` | absolute path | deploy, dev | Path to built frontend assets inside app container. |
### Dev Frontend Overrides
| Variable | Default | Options | Scope | Description |
| --- | --- | --- | --- | --- |
| `FRONTEND_HOST_PORT` | `5173` | integer | dev | Host port mapped to Vite dev server. |
| `CHOKIDAR_USEPOLLING` | `1` | boolean | dev | File watching mode for containerized Vite. |
| `VITE_DEV_API_PROXY_TARGET` | `http://app:8000` | URL | dev | Vite proxy target for `/api` and `/healthz`. |
### Auth and Session Security
| Variable | Default | Options | Scope | Description |
| --- | --- | --- | --- | --- |
| `SESSION_SECRET_KEY` | `replace-with-a-long-random-secret-at-least-32-characters` | long random string | deploy, dev | Session signing secret. Required in deploy. |
| `SESSION_COOKIE_SECURE` | `1` | boolean | deploy, dev | Mark session cookie as HTTPS-only. Set `1` behind HTTPS. |
| `LOGIN_RATE_LIMIT_ATTEMPTS` | `5` | integer >= 1 | deploy, dev | Login attempts allowed per window. |
| `LOGIN_RATE_LIMIT_WINDOW_SECONDS` | `60` | integer >= 1 | deploy, dev | Login rate limit window in seconds. |
### HTTP Security Headers and CSP
| Variable | Default | Options | Scope | Description |
| --- | --- | --- | --- | --- |
| `SECURITY_HEADERS_ENABLED` | `1` | boolean | deploy, dev | Global switch for response security headers middleware. |
| `SECURITY_X_CONTENT_TYPE_OPTIONS` | `nosniff` | header value | deploy, dev | `X-Content-Type-Options` response header value. |
| `SECURITY_X_FRAME_OPTIONS` | `DENY` | header value | deploy, dev | `X-Frame-Options` response header value. |
| `SECURITY_REFERRER_POLICY` | `strict-origin-when-cross-origin` | header value | deploy, dev | `Referrer-Policy` response header value. |
| `SECURITY_PERMISSIONS_POLICY` | `accelerometer=(), autoplay=(), camera=(), display-capture=(), geolocation=(), gyroscope=(), microphone=(), payment=(), usb=()` | permissions policy string | deploy, dev | `Permissions-Policy` response header value. |
| `SECURITY_CROSS_ORIGIN_OPENER_POLICY` | `same-origin` | header value | deploy, dev | `Cross-Origin-Opener-Policy` response header value. |
| `SECURITY_CROSS_ORIGIN_RESOURCE_POLICY` | `same-origin` | header value | deploy, dev | `Cross-Origin-Resource-Policy` response header value. |
| `SECURITY_CSP_ENABLED` | `1` | boolean | deploy, dev | Enable Content Security Policy headers. |
| `SECURITY_CSP_POLICY` | strict SPA/API default | CSP policy string | deploy, dev | CSP applied to app/API routes (excluding docs paths). |
| `SECURITY_CSP_DOCS_POLICY` | relaxed docs default | CSP policy string | deploy, dev | CSP override for `/docs` and `/redoc` to keep Swagger/ReDoc usable. |
| `SECURITY_CSP_REPORT_ONLY` | `0` | boolean | deploy, dev | Emit CSP in report-only mode instead of enforcement mode. |
| `SECURITY_STRICT_TRANSPORT_SECURITY_ENABLED` | `0` | boolean | deploy, dev | Enable `Strict-Transport-Security` header. |
| `SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE` | `31536000` | integer >= 0 | deploy, dev | `max-age` for HSTS header. |
| `SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS` | `1` | boolean | deploy, dev | Add `includeSubDomains` directive to HSTS header. |
| `SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD` | `0` | boolean | deploy, dev | Add `preload` directive to HSTS header. |
### Logging
| Variable | Default | Options | Scope | Description |
| --- | --- | --- | --- | --- |
| `LOG_LEVEL` | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` | deploy, dev | Application log level. |
| `LOG_FORMAT` | `console` | `console`, `json` | deploy, dev | Log output format. |
| `LOG_REQUESTS` | `1` | boolean | deploy, dev | Enable request start/completion logs. |
| `LOG_UVICORN_ACCESS` | `0` | boolean | deploy, dev | Enable uvicorn access logs. |
| `LOG_REQUEST_SKIP_PATHS` | `/healthz` | comma-separated path prefixes | deploy, dev | Request log skip list. |
| `LOG_REDACT_FIELDS` | empty | comma-separated keys | deploy, dev | Extra fields to redact in structured logs. |
### Scheduler and Ingestion
| Variable | Default | Options | Scope | Description |
| --- | --- | --- | --- | --- |
| `SCHEDULER_ENABLED` | `1` | boolean | deploy, dev | Enable background scheduler loop. |
| `SCHEDULER_TICK_SECONDS` | `60` | integer >= 1 | deploy, dev | Scheduler interval in seconds. |
| `INGESTION_AUTOMATION_ALLOWED` | `1` | boolean | deploy, dev | Global safety switch for scheduled/automatic checks. When disabled, auto-run settings are forced off. |
| `INGESTION_MANUAL_RUN_ALLOWED` | `1` | boolean | deploy, dev | Global safety switch for manual checks from API/UI. |
| `INGESTION_MIN_RUN_INTERVAL_MINUTES` | `15` | integer >= 15 | deploy, dev | Server-enforced minimum for user-configured automatic check interval. |
| `INGESTION_MIN_REQUEST_DELAY_SECONDS` | `2` | integer >= 2 | deploy, dev | Server-enforced minimum delay between scholar requests. |
| `SCHEDULER_QUEUE_BATCH_SIZE` | `10` | integer >= 1 | deploy, dev | Queue items processed per scheduler tick. |
| `INGESTION_NETWORK_ERROR_RETRIES` | `1` | integer >= 0 | deploy, dev | Retries for transient ingestion network errors. |
| `INGESTION_RETRY_BACKOFF_SECONDS` | `1.0` | float >= 0 | deploy, dev | Backoff delay for retry attempts. |
| `INGESTION_MAX_PAGES_PER_SCHOLAR` | `30` | integer >= 1 | deploy, dev | Upper bound of pages fetched per scholar run. |
| `INGESTION_PAGE_SIZE` | `100` | integer >= 1 | deploy, dev | Requested Scholar page size. |
| `INGESTION_ALERT_BLOCKED_FAILURE_THRESHOLD` | `1` | integer >= 1 | deploy, dev | Trigger blocked/captcha scrape alert flag when this many blocked failures occur in a run. |
| `INGESTION_ALERT_NETWORK_FAILURE_THRESHOLD` | `2` | integer >= 1 | deploy, dev | Trigger network scrape alert flag when this many network failures occur in a run. |
| `INGESTION_ALERT_RETRY_SCHEDULED_THRESHOLD` | `3` | integer >= 1 | deploy, dev | Trigger retry alert flag when scheduled retry count reaches this threshold in a run. |
| `INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS` | `1800` | integer >= 60 | deploy, dev | Cooldown duration applied when blocked-failure threshold is exceeded. |
| `INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS` | `900` | integer >= 60 | deploy, dev | Cooldown duration applied when network-failure threshold is exceeded. |
| `INGESTION_CONTINUATION_QUEUE_ENABLED` | `1` | boolean | deploy, dev | Enable continuation queue for long runs. |
| `INGESTION_CONTINUATION_BASE_DELAY_SECONDS` | `120` | integer >= 0 | deploy, dev | Initial delay before retrying continuation queue items. |
| `INGESTION_CONTINUATION_MAX_DELAY_SECONDS` | `3600` | integer >= 0 | deploy, dev | Maximum continuation retry delay. |
| `INGESTION_CONTINUATION_MAX_ATTEMPTS` | `6` | integer >= 1 | deploy, dev | Max failed continuation attempts before dropping item. |
### Scholar Images and Name Search Safety
| Variable | Default | Options | Scope | Description |
| --- | --- | --- | --- | --- |
| `SCHOLAR_IMAGE_UPLOAD_DIR` | `/var/lib/scholarr/uploads` | writable absolute path | deploy, dev | Storage path for uploaded scholar images in compose deployments. App fallback without env is `/tmp/scholarr_uploads/scholar_images`. |
| `SCHOLAR_IMAGE_UPLOAD_MAX_BYTES` | `2000000` | integer >= 1 | deploy, dev | Max uploaded image size in bytes. |
| `SCHOLAR_NAME_SEARCH_ENABLED` | `1` | boolean | deploy, dev | Enable name-search helper endpoint. |
| `SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS` | `21600` | integer >= 1 | deploy, dev | Cache TTL for successful name-search responses. |
| `SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS` | `300` | integer >= 1 | deploy, dev | Cache TTL for blocked/captcha responses. |
| `SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES` | `512` | integer >= 1 | deploy, dev | Max cached search entries. |
| `SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS` | `8.0` | float >= 0 | deploy, dev | Minimum interval between live name-search requests. |
| `SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS` | `2.0` | float >= 0 | deploy, dev | Added jitter to reduce request burst patterns. |
| `SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD` | `1` | integer >= 1 | deploy, dev | Consecutive blocked responses before cooldown starts. |
| `SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS` | `1800` | integer >= 1 | deploy, dev | Cooldown duration after repeated blocked responses. |
| `SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD` | `2` | integer >= 1 | deploy, dev | Emit retry-threshold observability warning when name-search retry count reaches this value. |
| `SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD` | `3` | integer >= 1 | deploy, dev | Emit cooldown-threshold observability alert after this many requests are rejected during active cooldown. |
### Open-Access Enrichment (Crossref + Unpaywall)
| Variable | Default | Options | Scope | Description |
| --- | --- | --- | --- | --- |
| `UNPAYWALL_ENABLED` | `1` | boolean | deploy, dev | Enable Unpaywall OA resolution. |
| `UNPAYWALL_EMAIL` | empty | valid email | deploy, dev | Fallback email if per-user email is unavailable. |
| `UNPAYWALL_TIMEOUT_SECONDS` | `4.0` | float >= 0.5 | deploy, dev | Timeout for Unpaywall requests. |
| `UNPAYWALL_MAX_ITEMS_PER_REQUEST` | `20` | integer >= 0 | deploy, dev | Max publications queued per lazy enrichment pass. |
| `UNPAYWALL_RETRY_COOLDOWN_SECONDS` | `1800` | integer >= 1 | deploy, dev | Per-user/per-publication cooldown before re-scheduling unresolved OA lookups. |
| `CROSSREF_ENABLED` | `1` | boolean | deploy, dev | Enable Crossref DOI discovery fallback when DOI is missing/insufficient. |
| `CROSSREF_MAX_ROWS` | `10` | integer >= 1 | deploy, dev | Max Crossref rows evaluated per lookup. |
| `CROSSREF_TIMEOUT_SECONDS` | `8.0` | float >= 0.5 | deploy, dev | Timeout budget per Crossref lookup task. |
| `CROSSREF_MIN_INTERVAL_SECONDS` | `0.6` | float >= 0 | deploy, dev | Minimum pacing interval between Crossref requests. |
| `CROSSREF_MAX_LOOKUPS_PER_REQUEST` | `8` | integer >= 0 | deploy, dev | Max Crossref DOI lookups performed during one lazy pass or manual retry. |
### Scrape Safety Operations
- Structured safety events are emitted for:
- policy/manual run blocks,
- cooldown entered/cleared transitions,
- blocked/network/retry threshold trips,
- scheduler cooldown skips/deferments.
- Use `LOG_FORMAT=json` and ship logs to your preferred collector for alerting.
- Runbook: `docs/ops/scrape_safety_runbook.md`.
### Startup Bootstrap and DB Wait
| Variable | Default | Options | Scope | Description |
| --- | --- | --- | --- | --- |
| `BOOTSTRAP_ADMIN_ON_START` | `0` | boolean | deploy, dev | Auto-create admin at startup. |
| `BOOTSTRAP_ADMIN_EMAIL` | empty | valid email | deploy, dev | Bootstrap admin email. |
| `BOOTSTRAP_ADMIN_PASSWORD` | empty | strong password | deploy, dev | Bootstrap admin password. |
| `BOOTSTRAP_ADMIN_FORCE_PASSWORD` | `0` | boolean | deploy, dev | Force-reset bootstrap admin password if user already exists. |
| `DB_WAIT_TIMEOUT_SECONDS` | `60` | integer >= 1 | deploy, dev | Max seconds to wait for DB readiness at startup. |
| `DB_WAIT_INTERVAL_SECONDS` | `2` | integer >= 1 | deploy, dev | Interval between DB readiness checks. |
## Quality and Test Commands
Backend: Backend:
@ -283,43 +81,10 @@ npm run test:run
npm run build npm run build
``` ```
Contract drift: Contract and env checks:
```bash ```bash
python3 scripts/check_frontend_api_contract.py python3 scripts/check_frontend_api_contract.py
``` python3 scripts/check_env_contract.py
Repository hygiene:
```bash
./scripts/check_no_generated_artifacts.sh ./scripts/check_no_generated_artifacts.sh
``` ```
Scheduled fixture probes run in GitHub Actions via `.github/workflows/scheduled-probes.yml`.
## API Contract
- Base path: `/api/v1`
- Success envelope: `{"data": ..., "meta": {"request_id": "..."}}`
- Error envelope: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}`
### Publications semantics
- `GET /api/v1/publications` supports `mode=all|unread|latest` (plus temporary alias `mode=new`).
- `unread` = actionable read-state (`is_read=false`).
- `latest` = discovery-state (`first seen in the latest completed check`).
- Publications include `pub_url`, `doi`, and `pdf_url` in API responses.
- OA enrichment flow is DOI-first:
- existing DOI (or explicit DOI in source metadata) -> Unpaywall lookup,
- Crossref DOI discovery fallback (paced + bounded),
- Unpaywall DOI lookup for OA status and `pdf_url`.
- `POST /api/v1/publications/{publication_id}/retry-pdf` retries enrichment for a single publication in current user scope.
- Response counters:
- `unread_count`: unread publications in current scope.
- `latest_count`: publications discovered in latest completed check.
- `new_count`: compatibility alias for `latest_count` (temporary).
### Scholar Data Portability
- `GET /api/v1/scholars/export`: exports tracked scholars plus scholar-publication link data as JSON.
- `POST /api/v1/scholars/import`: imports that JSON payload, upserting scholars, global publication records, and per-scholar read state.

View file

@ -36,3 +36,8 @@ These limits prevent IP bans and are not to be optimized away.
* **`app/services/domains/unpaywall/*`:** OA resolver by DOI only (best OA location + PDF URL extraction). Do not use Google Scholar for PDF resolution to avoid N+1 scrape amplification. * **`app/services/domains/unpaywall/*`:** OA resolver by DOI only (best OA location + PDF URL extraction). Do not use Google Scholar for PDF resolution to avoid N+1 scrape amplification.
* **`app/services/domains/portability/*`:** Handles JSON import/export for user-scoped scholars and scholar-publication link state while preserving global publication dedup rules. * **`app/services/domains/portability/*`:** Handles JSON import/export for user-scoped scholars and scholar-publication link state while preserving global publication dedup rules.
* **`app/services/domains/ingestion/scheduler.py`:** Owns automatic runs and continuation queue retries/drops; do not bypass safety gate or cooldown logic. * **`app/services/domains/ingestion/scheduler.py`:** Owns automatic runs and continuation queue retries/drops; do not bypass safety gate or cooldown logic.
## 6. UI rules
Make sure to properly integrate tailwind in combination with the preset theming
Clarity through both styling and language are a priority. all UI elements need to have a proper reason for existing.

View file

@ -0,0 +1,106 @@
"""Add data repair job audit table.
Revision ID: 20260220_0012
Revises: 20260220_0011
Create Date: 2026-02-20 23:35:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
revision: str = "20260220_0012"
down_revision: str | Sequence[str] | None = "20260220_0011"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _table_names() -> set[str]:
bind = op.get_bind()
inspector = sa.inspect(bind)
return set(inspector.get_table_names())
def _create_data_repair_jobs_table() -> None:
op.create_table(
"data_repair_jobs",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("job_name", sa.String(length=64), nullable=False),
sa.Column("requested_by", sa.String(length=255), nullable=True),
sa.Column(
"scope",
postgresql.JSONB(astext_type=sa.Text()),
nullable=False,
server_default=sa.text("'{}'::jsonb"),
),
sa.Column("dry_run", sa.Boolean(), nullable=False, server_default=sa.text("true")),
sa.Column(
"status",
sa.String(length=16),
nullable=False,
server_default=sa.text("'planned'"),
),
sa.Column(
"summary",
postgresql.JSONB(astext_type=sa.Text()),
nullable=False,
server_default=sa.text("'{}'::jsonb"),
),
sa.Column("error_text", sa.Text(), nullable=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.CheckConstraint(
"status IN ('planned', 'running', 'completed', 'failed')",
name="data_repair_jobs_status_valid",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_data_repair_jobs")),
)
def _create_data_repair_jobs_indexes() -> None:
op.create_index(
"ix_data_repair_jobs_created_at",
"data_repair_jobs",
["created_at"],
unique=False,
)
op.create_index(
"ix_data_repair_jobs_job_name_created_at",
"data_repair_jobs",
["job_name", "created_at"],
unique=False,
)
def _drop_data_repair_jobs_indexes() -> None:
op.drop_index("ix_data_repair_jobs_job_name_created_at", table_name="data_repair_jobs")
op.drop_index("ix_data_repair_jobs_created_at", table_name="data_repair_jobs")
def upgrade() -> None:
if "data_repair_jobs" in _table_names():
return
_create_data_repair_jobs_table()
_create_data_repair_jobs_indexes()
def downgrade() -> None:
if "data_repair_jobs" not in _table_names():
return
_drop_data_repair_jobs_indexes()
op.drop_table("data_repair_jobs")

View file

@ -0,0 +1,179 @@
"""Add persistent publication PDF job tracking tables.
Revision ID: 20260221_0013
Revises: 20260220_0012
Create Date: 2026-02-21 12:25:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
revision: str = "20260221_0013"
down_revision: str | Sequence[str] | None = "20260220_0012"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _table_names() -> set[str]:
bind = op.get_bind()
inspector = sa.inspect(bind)
return set(inspector.get_table_names())
def _create_publication_pdf_jobs_table() -> None:
op.create_table(
"publication_pdf_jobs",
sa.Column("publication_id", sa.Integer(), nullable=False),
sa.Column(
"status",
sa.String(length=16),
nullable=False,
server_default=sa.text("'queued'"),
),
sa.Column("attempt_count", sa.Integer(), nullable=False, server_default=sa.text("0")),
sa.Column("queued_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_failure_reason", sa.String(length=64), nullable=True),
sa.Column("last_failure_detail", sa.Text(), nullable=True),
sa.Column("last_source", sa.String(length=32), nullable=True),
sa.Column("last_requested_by_user_id", sa.Integer(), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.CheckConstraint(
"status IN ('queued', 'running', 'resolved', 'failed')",
name="publication_pdf_jobs_status_valid",
),
sa.ForeignKeyConstraint(
["last_requested_by_user_id"],
["users.id"],
ondelete="SET NULL",
),
sa.ForeignKeyConstraint(
["publication_id"],
["publications.id"],
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("publication_id", name=op.f("pk_publication_pdf_jobs")),
)
def _create_publication_pdf_jobs_indexes() -> None:
op.create_index(
"ix_publication_pdf_jobs_status",
"publication_pdf_jobs",
["status"],
unique=False,
)
op.create_index(
"ix_publication_pdf_jobs_updated_at",
"publication_pdf_jobs",
["updated_at"],
unique=False,
)
op.create_index(
"ix_publication_pdf_jobs_queued_at",
"publication_pdf_jobs",
["queued_at"],
unique=False,
)
def _create_publication_pdf_job_events_table() -> None:
op.create_table(
"publication_pdf_job_events",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("publication_id", sa.Integer(), nullable=False),
sa.Column("user_id", sa.Integer(), nullable=True),
sa.Column("event_type", sa.String(length=32), nullable=False),
sa.Column("status", sa.String(length=16), nullable=True),
sa.Column("source", sa.String(length=32), nullable=True),
sa.Column("failure_reason", sa.String(length=64), nullable=True),
sa.Column("message", sa.Text(), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.ForeignKeyConstraint(
["publication_id"],
["publications.id"],
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["user_id"],
["users.id"],
ondelete="SET NULL",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_publication_pdf_job_events")),
)
def _create_publication_pdf_job_events_indexes() -> None:
op.create_index(
"ix_publication_pdf_job_events_publication_created",
"publication_pdf_job_events",
["publication_id", "created_at"],
unique=False,
)
op.create_index(
"ix_publication_pdf_job_events_created_at",
"publication_pdf_job_events",
["created_at"],
unique=False,
)
op.create_index(
"ix_publication_pdf_job_events_event_type",
"publication_pdf_job_events",
["event_type"],
unique=False,
)
def _drop_publication_pdf_jobs_indexes() -> None:
op.drop_index("ix_publication_pdf_jobs_queued_at", table_name="publication_pdf_jobs")
op.drop_index("ix_publication_pdf_jobs_updated_at", table_name="publication_pdf_jobs")
op.drop_index("ix_publication_pdf_jobs_status", table_name="publication_pdf_jobs")
def _drop_publication_pdf_job_events_indexes() -> None:
op.drop_index("ix_publication_pdf_job_events_event_type", table_name="publication_pdf_job_events")
op.drop_index("ix_publication_pdf_job_events_created_at", table_name="publication_pdf_job_events")
op.drop_index(
"ix_publication_pdf_job_events_publication_created",
table_name="publication_pdf_job_events",
)
def upgrade() -> None:
tables = _table_names()
if "publication_pdf_jobs" not in tables:
_create_publication_pdf_jobs_table()
_create_publication_pdf_jobs_indexes()
if "publication_pdf_job_events" not in tables:
_create_publication_pdf_job_events_table()
_create_publication_pdf_job_events_indexes()
def downgrade() -> None:
tables = _table_names()
if "publication_pdf_job_events" in tables:
_drop_publication_pdf_job_events_indexes()
op.drop_table("publication_pdf_job_events")
if "publication_pdf_jobs" in tables:
_drop_publication_pdf_jobs_indexes()
op.drop_table("publication_pdf_jobs")

View file

@ -0,0 +1,44 @@
"""add scholar publication favorite state
Revision ID: 20260221_0014
Revises: 20260221_0013
Create Date: 2026-02-21 14:35:00
"""
from __future__ import annotations
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = "20260221_0014"
down_revision = "20260221_0013"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"scholar_publications",
sa.Column(
"is_favorite",
sa.Boolean(),
nullable=False,
server_default=sa.text("false"),
),
)
op.create_index(
"ix_scholar_publications_is_favorite",
"scholar_publications",
["is_favorite"],
unique=False,
)
def downgrade() -> None:
op.drop_index(
"ix_scholar_publications_is_favorite",
table_name="scholar_publications",
)
op.drop_column("scholar_publications", "is_favorite")

View file

@ -0,0 +1,80 @@
"""enforce request delay floor at two seconds
Revision ID: 20260221_0015
Revises: 20260221_0014
Create Date: 2026-02-21 18:10:00
"""
from __future__ import annotations
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "20260221_0015"
down_revision: str | Sequence[str] | None = "20260221_0014"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
TABLE_NAME = "user_settings"
CHECK_NAME_MIN_1 = "request_delay_seconds_min_1"
CHECK_NAME_MIN_2 = "request_delay_seconds_min_2"
LEGACY_CHECK_NAME_MIN_1 = "ck_user_settings_request_delay_seconds_min_1"
LEGACY_CHECK_NAME_MIN_2 = "ck_user_settings_request_delay_seconds_min_2"
def _check_constraints() -> set[str]:
bind = op.get_bind()
inspector = sa.inspect(bind)
return {
str(item.get("name"))
for item in inspector.get_check_constraints(TABLE_NAME)
if item.get("name")
}
def _drop_check_if_exists(name: str) -> None:
if name not in _check_constraints():
return
op.drop_constraint(
name,
TABLE_NAME,
type_="check",
)
def upgrade() -> None:
op.execute(
sa.text(
"UPDATE user_settings SET request_delay_seconds = 2 "
"WHERE request_delay_seconds < 2"
)
)
_drop_check_if_exists(CHECK_NAME_MIN_1)
_drop_check_if_exists(LEGACY_CHECK_NAME_MIN_1)
existing = _check_constraints()
if CHECK_NAME_MIN_2 not in existing and LEGACY_CHECK_NAME_MIN_2 not in existing:
op.create_check_constraint(
CHECK_NAME_MIN_2,
TABLE_NAME,
"request_delay_seconds >= 2",
)
def downgrade() -> None:
_drop_check_if_exists(CHECK_NAME_MIN_2)
_drop_check_if_exists(LEGACY_CHECK_NAME_MIN_2)
existing = _check_constraints()
if CHECK_NAME_MIN_1 not in existing and LEGACY_CHECK_NAME_MIN_1 not in existing:
op.create_check_constraint(
CHECK_NAME_MIN_1,
TABLE_NAME,
"request_delay_seconds >= 1",
)

View file

@ -2,11 +2,12 @@ from __future__ import annotations
from fastapi import APIRouter from fastapi import APIRouter
from app.api.routers import admin, auth, publications, runs, scholars, settings from app.api.routers import admin, admin_dbops, auth, publications, runs, scholars, settings
router = APIRouter(prefix="/api/v1") router = APIRouter(prefix="/api/v1")
router.include_router(auth.router) router.include_router(auth.router)
router.include_router(admin.router) router.include_router(admin.router)
router.include_router(admin_dbops.router)
router.include_router(scholars.router) router.include_router(scholars.router)
router.include_router(settings.router) router.include_router(settings.router)
router.include_router(runs.router) router.include_router(runs.router)

View file

@ -0,0 +1,337 @@
from __future__ import annotations
import logging
from fastapi import APIRouter, Depends, Path, Query, Request
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_admin_user
from app.api.errors import ApiException
from app.api.responses import success_payload
from app.api.schemas import (
AdminDbIntegrityEnvelope,
AdminPdfQueueBulkEnqueueEnvelope,
AdminPdfQueueRequeueEnvelope,
AdminPdfQueueEnvelope,
AdminDbRepairJobsEnvelope,
AdminRepairPublicationLinksEnvelope,
AdminRepairPublicationLinksRequest,
)
from app.db.models import DataRepairJob, User
from app.db.session import get_db_session
from app.services.domains.dbops import (
collect_integrity_report,
list_repair_jobs,
run_publication_link_repair,
)
from app.services.domains.publications import application as publication_service
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/admin/db", tags=["api-admin-dbops"])
def _serialize_repair_job(job: DataRepairJob) -> dict[str, object]:
return {
"id": int(job.id),
"job_name": job.job_name,
"requested_by": job.requested_by,
"dry_run": bool(job.dry_run),
"status": job.status,
"scope": dict(job.scope or {}),
"summary": dict(job.summary or {}),
"error_text": job.error_text,
"started_at": job.started_at,
"finished_at": job.finished_at,
"created_at": job.created_at,
"updated_at": job.updated_at,
}
def _requested_by_value(*, payload: AdminRepairPublicationLinksRequest, admin_user: User) -> str:
from_payload = (payload.requested_by or "").strip()
return from_payload or admin_user.email
def _serialize_pdf_queue_item(item) -> dict[str, object]:
return {
"publication_id": item.publication_id,
"title": item.title,
"doi": item.doi,
"pdf_url": item.pdf_url,
"status": item.status,
"attempt_count": item.attempt_count,
"last_failure_reason": item.last_failure_reason,
"last_failure_detail": item.last_failure_detail,
"last_source": item.last_source,
"requested_by_user_id": item.requested_by_user_id,
"requested_by_email": item.requested_by_email,
"queued_at": item.queued_at,
"last_attempt_at": item.last_attempt_at,
"resolved_at": item.resolved_at,
"updated_at": item.updated_at,
}
def _requeue_response_state(*, queued: bool) -> tuple[str, str]:
if queued:
return "queued", "PDF lookup queued."
return "blocked", "PDF lookup is already queued or currently running."
def _bulk_enqueue_message(*, queued_count: int, requested_count: int) -> str:
if requested_count == 0:
return "No missing-PDF publications were found."
if queued_count == 0:
return "No publications were queued; all candidates were already in-flight."
return f"Queued {queued_count} publication(s) for PDF lookup."
def _resolve_pdf_queue_paging(
*,
page: int,
page_size: int,
limit: int | None,
offset: int | None,
) -> tuple[int, int, int]:
resolved_limit = int(limit) if limit is not None else int(page_size)
resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0)
resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1)
return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0)
def _pdf_queue_page_data(*, total_count: int, page: int, page_size: int, offset: int) -> dict[str, object]:
return {
"total_count": int(total_count),
"page": int(page),
"page_size": int(page_size),
"has_prev": int(offset) > 0,
"has_next": int(offset) + int(page_size) < int(total_count),
}
@router.get(
"/integrity",
response_model=AdminDbIntegrityEnvelope,
)
async def get_integrity_report(
request: Request,
db_session: AsyncSession = Depends(get_db_session),
admin_user: User = Depends(get_api_admin_user),
):
report = await collect_integrity_report(db_session)
logger.info(
"api.admin.db.integrity_checked",
extra={
"event": "api.admin.db.integrity_checked",
"admin_user_id": int(admin_user.id),
"status": report.get("status"),
"failure_count": len(report.get("failures", [])),
"warning_count": len(report.get("warnings", [])),
},
)
return success_payload(request, data=report)
@router.get(
"/repair-jobs",
response_model=AdminDbRepairJobsEnvelope,
)
async def get_repair_jobs(
request: Request,
limit: int = Query(default=50, ge=1, le=200),
db_session: AsyncSession = Depends(get_db_session),
admin_user: User = Depends(get_api_admin_user),
):
jobs = await list_repair_jobs(db_session, limit=limit)
logger.info(
"api.admin.db.repair_jobs_listed",
extra={
"event": "api.admin.db.repair_jobs_listed",
"admin_user_id": int(admin_user.id),
"limit": int(limit),
"job_count": len(jobs),
},
)
return success_payload(
request,
data={"jobs": [_serialize_repair_job(job) for job in jobs]},
)
@router.get(
"/pdf-queue",
response_model=AdminPdfQueueEnvelope,
)
async def get_pdf_queue(
request: Request,
page: int = Query(default=1, ge=1),
page_size: int = Query(default=100, ge=1, le=500),
limit: int | None = Query(default=None, ge=1, le=500),
offset: int | None = Query(default=None, ge=0),
status: str | None = Query(default=None),
db_session: AsyncSession = Depends(get_db_session),
admin_user: User = Depends(get_api_admin_user),
):
normalized_status = (status or "").strip().lower() or None
resolved_page, resolved_limit, resolved_offset = _resolve_pdf_queue_paging(
page=page,
page_size=page_size,
limit=limit,
offset=offset,
)
queue_page = await publication_service.list_pdf_queue_page(
db_session,
limit=resolved_limit,
offset=resolved_offset,
status=normalized_status,
)
logger.info(
"api.admin.db.pdf_queue_listed",
extra={
"event": "api.admin.db.pdf_queue_listed",
"admin_user_id": int(admin_user.id),
"page": int(resolved_page),
"page_size": int(resolved_limit),
"offset": int(resolved_offset),
"status": normalized_status,
"item_count": len(queue_page.items),
"total_count": int(queue_page.total_count),
},
)
return success_payload(
request,
data={
"items": [_serialize_pdf_queue_item(item) for item in queue_page.items],
**_pdf_queue_page_data(
total_count=queue_page.total_count,
page=resolved_page,
page_size=resolved_limit,
offset=resolved_offset,
),
},
)
@router.post(
"/pdf-queue/{publication_id}/requeue",
response_model=AdminPdfQueueRequeueEnvelope,
)
async def requeue_pdf_lookup(
request: Request,
publication_id: int = Path(ge=1),
db_session: AsyncSession = Depends(get_db_session),
admin_user: User = Depends(get_api_admin_user),
):
result = await publication_service.enqueue_retry_pdf_job_for_publication_id(
db_session,
user_id=int(admin_user.id),
request_email=admin_user.email,
publication_id=publication_id,
)
if not result.publication_exists:
raise ApiException(
status_code=404,
code="publication_not_found",
message="Publication not found.",
)
status, message = _requeue_response_state(queued=result.queued)
logger.info(
"api.admin.db.pdf_queue_requeue_requested",
extra={
"event": "api.admin.db.pdf_queue_requeue_requested",
"admin_user_id": int(admin_user.id),
"publication_id": int(publication_id),
"queued": bool(result.queued),
},
)
return success_payload(
request,
data={
"publication_id": int(publication_id),
"queued": bool(result.queued),
"status": status,
"message": message,
},
)
@router.post(
"/pdf-queue/requeue-all",
response_model=AdminPdfQueueBulkEnqueueEnvelope,
)
async def requeue_all_missing_pdfs(
request: Request,
limit: int = Query(default=1000, ge=1, le=5000),
db_session: AsyncSession = Depends(get_db_session),
admin_user: User = Depends(get_api_admin_user),
):
result = await publication_service.enqueue_all_missing_pdf_jobs(
db_session,
user_id=int(admin_user.id),
request_email=admin_user.email,
limit=limit,
)
logger.info(
"api.admin.db.pdf_queue_requeue_all",
extra={
"event": "api.admin.db.pdf_queue_requeue_all",
"admin_user_id": int(admin_user.id),
"limit": int(limit),
"requested_count": int(result.requested_count),
"queued_count": int(result.queued_count),
},
)
return success_payload(
request,
data={
"requested_count": int(result.requested_count),
"queued_count": int(result.queued_count),
"message": _bulk_enqueue_message(
queued_count=int(result.queued_count),
requested_count=int(result.requested_count),
),
},
)
@router.post(
"/repairs/publication-links",
response_model=AdminRepairPublicationLinksEnvelope,
)
async def trigger_publication_link_repair(
payload: AdminRepairPublicationLinksRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
admin_user: User = Depends(get_api_admin_user),
):
try:
result = await run_publication_link_repair(
db_session,
scope_mode=payload.scope_mode,
user_id=payload.user_id,
scholar_profile_ids=payload.scholar_profile_ids,
dry_run=bool(payload.dry_run),
gc_orphan_publications=bool(payload.gc_orphan_publications),
requested_by=_requested_by_value(payload=payload, admin_user=admin_user),
)
except ValueError as exc:
raise ApiException(
status_code=400,
code="invalid_repair_scope",
message=str(exc),
) from exc
logger.info(
"api.admin.db.publication_link_repair_triggered",
extra={
"event": "api.admin.db.publication_link_repair_triggered",
"admin_user_id": int(admin_user.id),
"scope_mode": payload.scope_mode,
"target_user_id": int(payload.user_id) if payload.user_id is not None else None,
"dry_run": bool(payload.dry_run),
"gc_orphan_publications": bool(payload.gc_orphan_publications),
"job_id": int(result["job_id"]),
"status": result["status"],
},
)
return success_payload(request, data=result)

View file

@ -16,6 +16,8 @@ from app.api.schemas import (
PublicationsListEnvelope, PublicationsListEnvelope,
RetryPublicationPdfEnvelope, RetryPublicationPdfEnvelope,
RetryPublicationPdfRequest, RetryPublicationPdfRequest,
TogglePublicationFavoriteEnvelope,
TogglePublicationFavoriteRequest,
) )
from app.db.models import User from app.db.models import User
from app.db.session import get_db_session from app.db.session import get_db_session
@ -61,7 +63,12 @@ def _serialize_publication_item(item) -> dict[str, object]:
"pub_url": item.pub_url, "pub_url": item.pub_url,
"doi": item.doi, "doi": item.doi,
"pdf_url": item.pdf_url, "pdf_url": item.pdf_url,
"pdf_status": item.pdf_status,
"pdf_attempt_count": item.pdf_attempt_count,
"pdf_failure_reason": item.pdf_failure_reason,
"pdf_failure_detail": item.pdf_failure_detail,
"is_read": item.is_read, "is_read": item.is_read,
"is_favorite": item.is_favorite,
"first_seen_at": item.first_seen_at, "first_seen_at": item.first_seen_at,
"is_new_in_latest_run": item.is_new_in_latest_run, "is_new_in_latest_run": item.is_new_in_latest_run,
} }
@ -72,46 +79,235 @@ async def _publication_counts(
*, *,
user_id: int, user_id: int,
selected_scholar_id: int | None, selected_scholar_id: int | None,
) -> tuple[int, int, int]: favorite_only: bool,
) -> tuple[int, int, int, int]:
unread_count = await publication_service.count_unread_for_user( unread_count = await publication_service.count_unread_for_user(
db_session, db_session,
user_id=user_id, user_id=user_id,
scholar_profile_id=selected_scholar_id, scholar_profile_id=selected_scholar_id,
favorite_only=favorite_only,
)
favorites_count = await publication_service.count_favorite_for_user(
db_session,
user_id=user_id,
scholar_profile_id=selected_scholar_id,
) )
latest_count = await publication_service.count_latest_for_user( latest_count = await publication_service.count_latest_for_user(
db_session, db_session,
user_id=user_id, user_id=user_id,
scholar_profile_id=selected_scholar_id, scholar_profile_id=selected_scholar_id,
favorite_only=favorite_only,
) )
total_count = await publication_service.count_for_user( total_count = await publication_service.count_for_user(
db_session, db_session,
user_id=user_id, user_id=user_id,
mode=publication_service.MODE_ALL, mode=publication_service.MODE_ALL,
scholar_profile_id=selected_scholar_id, scholar_profile_id=selected_scholar_id,
favorite_only=favorite_only,
) )
return unread_count, latest_count, total_count return unread_count, favorites_count, latest_count, total_count
async def _list_publications_for_request(
db_session: AsyncSession,
*,
current_user: User,
mode: Literal["all", "unread", "latest", "new"] | None,
favorite_only: bool,
scholar_profile_id: int | None,
limit: int,
offset: int,
) -> tuple[str, int | None, list]:
resolved_mode = publication_service.resolve_publication_view_mode(mode)
selected_scholar_id = scholar_profile_id
await _require_selected_profile(
db_session,
user_id=current_user.id,
selected_scholar_id=selected_scholar_id,
)
publications = await publication_service.list_for_user(
db_session,
user_id=current_user.id,
mode=resolved_mode,
scholar_profile_id=selected_scholar_id,
favorite_only=favorite_only,
limit=limit,
offset=offset,
)
await publication_service.schedule_missing_pdf_enrichment_for_user(
db_session,
user_id=current_user.id,
request_email=current_user.email,
items=publications,
max_items=settings.unpaywall_max_items_per_request,
)
hydrated = await publication_service.hydrate_pdf_enrichment_state(
db_session,
items=publications,
)
return resolved_mode, selected_scholar_id, hydrated
def _resolve_publications_paging(
*,
page: int,
page_size: int,
limit: int | None,
offset: int | None,
) -> tuple[int, int, int]:
resolved_limit = int(limit) if limit is not None else int(page_size)
resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0)
resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1)
return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0)
def _publications_list_data( def _publications_list_data(
*, *,
mode: str, mode: str,
favorite_only: bool,
selected_scholar_id: int | None, selected_scholar_id: int | None,
unread_count: int, unread_count: int,
favorites_count: int,
latest_count: int, latest_count: int,
total_count: int, total_count: int,
publications: list, publications: list,
page: int,
page_size: int,
offset: int,
) -> dict[str, object]: ) -> dict[str, object]:
return { return {
"mode": mode, "mode": mode,
"favorite_only": favorite_only,
"selected_scholar_profile_id": selected_scholar_id, "selected_scholar_profile_id": selected_scholar_id,
"unread_count": unread_count, "unread_count": unread_count,
"favorites_count": favorites_count,
"latest_count": latest_count, "latest_count": latest_count,
"new_count": latest_count, "new_count": latest_count,
"total_count": total_count, "total_count": total_count,
"page": int(page),
"page_size": int(page_size),
"has_prev": int(offset) > 0,
"has_next": int(offset) + int(page_size) < int(total_count),
"publications": [_serialize_publication_item(item) for item in publications], "publications": [_serialize_publication_item(item) for item in publications],
} }
def _retry_pdf_message(*, queued: bool, resolved_pdf: bool, pdf_status: str) -> str:
if resolved_pdf:
return "Open-access PDF link already resolved."
if queued:
return "PDF lookup queued."
if pdf_status in {"queued", "running"}:
return "PDF lookup is already queued."
return "No open-access PDF link found."
def _favorite_message(*, is_favorite: bool) -> str:
if is_favorite:
return "Publication marked as favorite."
return "Publication removed from favorites."
async def _retry_publication_state(
db_session: AsyncSession,
*,
current_user: User,
scholar_profile_id: int,
publication_id: int,
):
publication = await publication_service.retry_pdf_for_user(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
publication_id=publication_id,
)
if publication is None:
raise ApiException(
status_code=404,
code="publication_not_found",
message="Publication not found.",
)
queued = False
if not publication.pdf_url:
queued = await publication_service.schedule_retry_pdf_enrichment_for_row(
db_session,
user_id=current_user.id,
request_email=current_user.email,
item=publication,
)
hydrated = await publication_service.hydrate_pdf_enrichment_state(
db_session,
items=[publication],
)
current = hydrated[0] if hydrated else publication
return current, queued
async def _favorite_publication_state(
db_session: AsyncSession,
*,
current_user: User,
scholar_profile_id: int,
publication_id: int,
is_favorite: bool,
):
updated_count = await publication_service.set_publication_favorite_for_user(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
publication_id=publication_id,
is_favorite=is_favorite,
)
if updated_count <= 0:
raise ApiException(
status_code=404,
code="publication_not_found",
message="Publication not found.",
)
publication = await publication_service.get_publication_item_for_user(
db_session,
user_id=current_user.id,
scholar_profile_id=scholar_profile_id,
publication_id=publication_id,
)
if publication is None:
raise ApiException(
status_code=404,
code="publication_not_found",
message="Publication not found.",
)
hydrated = await publication_service.hydrate_pdf_enrichment_state(
db_session,
items=[publication],
)
return hydrated[0] if hydrated else publication
def _log_retry_pdf_result(
*,
current_user: User,
scholar_profile_id: int,
publication_id: int,
queued: bool,
resolved_pdf: bool,
pdf_status: str,
doi: str | None,
) -> None:
logger.info(
"api.publications.retry_pdf",
extra={
"event": "api.publications.retry_pdf",
"user_id": current_user.id,
"scholar_profile_id": scholar_profile_id,
"publication_id": publication_id,
"queued": queued,
"resolved_pdf": resolved_pdf,
"pdf_status": pdf_status,
"has_doi": bool(doi),
},
)
@router.get( @router.get(
"", "",
response_model=PublicationsListEnvelope, response_model=PublicationsListEnvelope,
@ -119,44 +315,48 @@ def _publications_list_data(
async def list_publications( async def list_publications(
request: Request, request: Request,
mode: Literal["all", "unread", "latest", "new"] | None = Query(default=None), mode: Literal["all", "unread", "latest", "new"] | None = Query(default=None),
favorite_only: bool = Query(default=False),
scholar_profile_id: int | None = Query(default=None, ge=1), scholar_profile_id: int | None = Query(default=None, ge=1),
limit: int = Query(default=300, ge=1, le=1000), page: int = Query(default=1, ge=1),
page_size: int = Query(default=100, ge=1, le=500),
limit: int | None = Query(default=None, ge=1, le=1000),
offset: int | None = Query(default=None, ge=0),
db_session: AsyncSession = Depends(get_db_session), db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user), current_user: User = Depends(get_api_current_user),
): ):
resolved_mode = publication_service.resolve_publication_view_mode(mode) resolved_page, resolved_limit, resolved_offset = _resolve_publications_paging(
selected_scholar_id = scholar_profile_id page=page,
await _require_selected_profile( page_size=page_size,
db_session,
user_id=current_user.id,
selected_scholar_id=selected_scholar_id,
)
publications = await publication_service.list_for_user(
db_session,
user_id=current_user.id,
mode=resolved_mode,
scholar_profile_id=selected_scholar_id,
limit=limit, limit=limit,
offset=offset,
) )
await publication_service.schedule_missing_pdf_enrichment_for_user( resolved_mode, selected_scholar_id, publications = await _list_publications_for_request(
user_id=current_user.id, db_session,
request_email=current_user.email, current_user=current_user,
items=publications, mode=mode,
max_items=settings.unpaywall_max_items_per_request, favorite_only=favorite_only,
scholar_profile_id=scholar_profile_id,
limit=resolved_limit,
offset=resolved_offset,
) )
unread_count, latest_count, total_count = await _publication_counts( unread_count, favorites_count, latest_count, total_count = await _publication_counts(
db_session, db_session,
user_id=current_user.id, user_id=current_user.id,
selected_scholar_id=selected_scholar_id, selected_scholar_id=selected_scholar_id,
favorite_only=favorite_only,
) )
data = _publications_list_data( data = _publications_list_data(
mode=resolved_mode, mode=resolved_mode,
favorite_only=favorite_only,
selected_scholar_id=selected_scholar_id, selected_scholar_id=selected_scholar_id,
unread_count=unread_count, unread_count=unread_count,
favorites_count=favorites_count,
latest_count=latest_count, latest_count=latest_count,
total_count=total_count, total_count=total_count,
publications=publications, publications=publications,
page=resolved_page,
page_size=resolved_limit,
offset=resolved_offset,
) )
return success_payload(request, data=data) return success_payload(request, data=data)
@ -242,37 +442,70 @@ async def retry_publication_pdf(
db_session: AsyncSession = Depends(get_db_session), db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user), current_user: User = Depends(get_api_current_user),
): ):
publication = await publication_service.retry_pdf_for_user( current, queued = await _retry_publication_state(
db_session, db_session,
user_id=current_user.id, current_user=current_user,
scholar_profile_id=payload.scholar_profile_id, scholar_profile_id=payload.scholar_profile_id,
publication_id=publication_id, publication_id=publication_id,
unpaywall_email=current_user.email,
) )
if publication is None: resolved_pdf = bool(current.pdf_url)
raise ApiException( message = _retry_pdf_message(
status_code=404, queued=queued,
code="publication_not_found", resolved_pdf=resolved_pdf,
message="Publication not found.", pdf_status=current.pdf_status,
) )
resolved_pdf = bool(publication.pdf_url) _log_retry_pdf_result(
logger.info( current_user=current_user,
"api.publications.retry_pdf", scholar_profile_id=payload.scholar_profile_id,
extra={ publication_id=publication_id,
"event": "api.publications.retry_pdf", queued=queued,
"user_id": current_user.id, resolved_pdf=resolved_pdf,
"scholar_profile_id": payload.scholar_profile_id, pdf_status=current.pdf_status,
"publication_id": publication_id, doi=current.doi,
"resolved_pdf": resolved_pdf,
"has_doi": bool(publication.doi),
},
) )
message = "Open-access PDF link resolved." if resolved_pdf else "No open-access PDF link found."
return success_payload( return success_payload(
request, request,
data={ data={
"message": message, "message": message,
"queued": queued,
"resolved_pdf": resolved_pdf, "resolved_pdf": resolved_pdf,
"publication": _serialize_publication_item(publication), "publication": _serialize_publication_item(current),
},
)
@router.post(
"/{publication_id}/favorite",
response_model=TogglePublicationFavoriteEnvelope,
)
async def toggle_publication_favorite(
payload: TogglePublicationFavoriteRequest,
request: Request,
publication_id: int = Path(ge=1),
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
current = await _favorite_publication_state(
db_session,
current_user=current_user,
scholar_profile_id=payload.scholar_profile_id,
publication_id=publication_id,
is_favorite=payload.is_favorite,
)
logger.info(
"api.publications.favorite",
extra={
"event": "api.publications.favorite",
"user_id": current_user.id,
"scholar_profile_id": payload.scholar_profile_id,
"publication_id": publication_id,
"is_favorite": bool(payload.is_favorite),
},
)
return success_payload(
request,
data={
"message": _favorite_message(is_favorite=bool(payload.is_favorite)),
"publication": _serialize_publication_item(current),
}, },
) )

View file

@ -1,9 +1,9 @@
from __future__ import annotations from __future__ import annotations
from datetime import datetime from datetime import datetime
from typing import Any from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field from pydantic import BaseModel, ConfigDict, Field, model_validator
class ApiMeta(BaseModel): class ApiMeta(BaseModel):
@ -525,6 +525,175 @@ class AdminResetPasswordRequest(BaseModel):
model_config = ConfigDict(extra="forbid") model_config = ConfigDict(extra="forbid")
class AdminDbIntegrityCheckData(BaseModel):
name: str
count: int
severity: str
message: str
model_config = ConfigDict(extra="forbid")
class AdminDbIntegrityData(BaseModel):
status: str
checked_at: datetime
failures: list[str] = Field(default_factory=list)
warnings: list[str] = Field(default_factory=list)
checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
model_config = ConfigDict(extra="forbid")
class AdminDbIntegrityEnvelope(BaseModel):
data: AdminDbIntegrityData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class AdminDbRepairJobData(BaseModel):
id: int
job_name: str
requested_by: str | None
dry_run: bool
status: str
scope: dict[str, Any] = Field(default_factory=dict)
summary: dict[str, Any] = Field(default_factory=dict)
error_text: str | None
started_at: datetime | None
finished_at: datetime | None
created_at: datetime
updated_at: datetime
model_config = ConfigDict(extra="forbid")
class AdminDbRepairJobsData(BaseModel):
jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
model_config = ConfigDict(extra="forbid")
class AdminDbRepairJobsEnvelope(BaseModel):
data: AdminDbRepairJobsData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class AdminPdfQueueItemData(BaseModel):
publication_id: int
title: str
doi: str | None
pdf_url: str | None
status: str
attempt_count: int
last_failure_reason: str | None
last_failure_detail: str | None
last_source: str | None
requested_by_user_id: int | None
requested_by_email: str | None
queued_at: datetime | None
last_attempt_at: datetime | None
resolved_at: datetime | None
updated_at: datetime
model_config = ConfigDict(extra="forbid")
class AdminPdfQueueData(BaseModel):
items: list[AdminPdfQueueItemData] = Field(default_factory=list)
total_count: int = 0
page: int = 1
page_size: int = 100
has_next: bool = False
has_prev: bool = False
model_config = ConfigDict(extra="forbid")
class AdminPdfQueueEnvelope(BaseModel):
data: AdminPdfQueueData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class AdminPdfQueueRequeueData(BaseModel):
publication_id: int
queued: bool
status: str
message: str
model_config = ConfigDict(extra="forbid")
class AdminPdfQueueRequeueEnvelope(BaseModel):
data: AdminPdfQueueRequeueData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class AdminPdfQueueBulkEnqueueData(BaseModel):
requested_count: int
queued_count: int
message: str
model_config = ConfigDict(extra="forbid")
class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
data: AdminPdfQueueBulkEnqueueData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class AdminRepairPublicationLinksRequest(BaseModel):
scope_mode: Literal["single_user", "all_users"] = "single_user"
user_id: int | None = Field(default=None, ge=1)
scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
dry_run: bool = True
gc_orphan_publications: bool = False
requested_by: str | None = None
confirmation_text: str | None = None
model_config = ConfigDict(extra="forbid")
@model_validator(mode="after")
def validate_scope(self) -> "AdminRepairPublicationLinksRequest":
if self.scope_mode == "single_user" and self.user_id is None:
raise ValueError("user_id is required when scope_mode=single_user.")
if self.scope_mode == "all_users" and self.user_id is not None:
raise ValueError("user_id must be omitted when scope_mode=all_users.")
if not self.dry_run and self.scope_mode == "all_users":
expected = "REPAIR ALL USERS"
provided = (self.confirmation_text or "").strip()
if provided != expected:
raise ValueError(
"confirmation_text must equal 'REPAIR ALL USERS' "
"when applying a repair to all users."
)
return self
class AdminRepairPublicationLinksResultData(BaseModel):
job_id: int
status: str
scope: dict[str, Any] = Field(default_factory=dict)
summary: dict[str, Any] = Field(default_factory=dict)
model_config = ConfigDict(extra="forbid")
class AdminRepairPublicationLinksEnvelope(BaseModel):
data: AdminRepairPublicationLinksResultData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class SettingsPolicyData(BaseModel): class SettingsPolicyData(BaseModel):
min_run_interval_minutes: int min_run_interval_minutes: int
min_request_delay_seconds: int min_request_delay_seconds: int
@ -576,7 +745,12 @@ class PublicationItemData(BaseModel):
pub_url: str | None pub_url: str | None
doi: str | None doi: str | None
pdf_url: str | None pdf_url: str | None
pdf_status: str = "untracked"
pdf_attempt_count: int = 0
pdf_failure_reason: str | None = None
pdf_failure_detail: str | None = None
is_read: bool is_read: bool
is_favorite: bool = False
first_seen_at: datetime first_seen_at: datetime
is_new_in_latest_run: bool is_new_in_latest_run: bool
@ -585,11 +759,17 @@ class PublicationItemData(BaseModel):
class PublicationsListData(BaseModel): class PublicationsListData(BaseModel):
mode: str mode: str
favorite_only: bool = False
selected_scholar_profile_id: int | None selected_scholar_profile_id: int | None
unread_count: int unread_count: int
favorites_count: int
latest_count: int latest_count: int
new_count: int new_count: int
total_count: int total_count: int
page: int
page_size: int
has_next: bool = False
has_prev: bool = False
publications: list[PublicationItemData] publications: list[PublicationItemData]
model_config = ConfigDict(extra="forbid") model_config = ConfigDict(extra="forbid")
@ -652,6 +832,7 @@ class RetryPublicationPdfRequest(BaseModel):
class RetryPublicationPdfData(BaseModel): class RetryPublicationPdfData(BaseModel):
message: str message: str
queued: bool
resolved_pdf: bool resolved_pdf: bool
publication: PublicationItemData publication: PublicationItemData
@ -663,3 +844,24 @@ class RetryPublicationPdfEnvelope(BaseModel):
meta: ApiMeta meta: ApiMeta
model_config = ConfigDict(extra="forbid") model_config = ConfigDict(extra="forbid")
class TogglePublicationFavoriteRequest(BaseModel):
scholar_profile_id: int = Field(ge=1)
is_favorite: bool
model_config = ConfigDict(extra="forbid")
class TogglePublicationFavoriteData(BaseModel):
message: str
publication: PublicationItemData
model_config = ConfigDict(extra="forbid")
class TogglePublicationFavoriteEnvelope(BaseModel):
data: TogglePublicationFavoriteData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")

View file

@ -81,8 +81,8 @@ class UserSetting(Base):
name="run_interval_minutes_min_15", name="run_interval_minutes_min_15",
), ),
CheckConstraint( CheckConstraint(
"request_delay_seconds >= 1", "request_delay_seconds >= 2",
name="request_delay_seconds_min_1", name="request_delay_seconds_min_2",
), ),
) )
@ -237,10 +237,80 @@ class Publication(Base):
) )
class PublicationPdfJob(Base):
__tablename__ = "publication_pdf_jobs"
__table_args__ = (
CheckConstraint(
"status IN ('queued', 'running', 'resolved', 'failed')",
name="publication_pdf_jobs_status_valid",
),
Index("ix_publication_pdf_jobs_status", "status"),
Index("ix_publication_pdf_jobs_updated_at", "updated_at"),
Index("ix_publication_pdf_jobs_queued_at", "queued_at"),
)
publication_id: Mapped[int] = mapped_column(
ForeignKey("publications.id", ondelete="CASCADE"),
primary_key=True,
)
status: Mapped[str] = mapped_column(
String(16),
nullable=False,
server_default=text("'queued'"),
)
attempt_count: Mapped[int] = mapped_column(
Integer,
nullable=False,
server_default=text("0"),
)
queued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_failure_reason: Mapped[str | None] = mapped_column(String(64))
last_failure_detail: Mapped[str | None] = mapped_column(Text)
last_source: Mapped[str | None] = mapped_column(String(32))
last_requested_by_user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="SET NULL"),
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class PublicationPdfJobEvent(Base):
__tablename__ = "publication_pdf_job_events"
__table_args__ = (
Index("ix_publication_pdf_job_events_publication_created", "publication_id", "created_at"),
Index("ix_publication_pdf_job_events_created_at", "created_at"),
Index("ix_publication_pdf_job_events_event_type", "event_type"),
)
id: Mapped[int] = mapped_column(primary_key=True)
publication_id: Mapped[int] = mapped_column(
ForeignKey("publications.id", ondelete="CASCADE"),
nullable=False,
)
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="SET NULL"),
)
event_type: Mapped[str] = mapped_column(String(32), nullable=False)
status: Mapped[str | None] = mapped_column(String(16))
source: Mapped[str | None] = mapped_column(String(32))
failure_reason: Mapped[str | None] = mapped_column(String(64))
message: Mapped[str | None] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class ScholarPublication(Base): class ScholarPublication(Base):
__tablename__ = "scholar_publications" __tablename__ = "scholar_publications"
__table_args__ = ( __table_args__ = (
Index("ix_scholar_publications_is_read", "is_read"), Index("ix_scholar_publications_is_read", "is_read"),
Index("ix_scholar_publications_is_favorite", "is_favorite"),
) )
scholar_profile_id: Mapped[int] = mapped_column( scholar_profile_id: Mapped[int] = mapped_column(
@ -254,6 +324,9 @@ class ScholarPublication(Base):
is_read: Mapped[bool] = mapped_column( is_read: Mapped[bool] = mapped_column(
Boolean, nullable=False, server_default=text("false") Boolean, nullable=False, server_default=text("false")
) )
is_favorite: Mapped[bool] = mapped_column(
Boolean, nullable=False, server_default=text("false")
)
first_seen_run_id: Mapped[int | None] = mapped_column( first_seen_run_id: Mapped[int | None] = mapped_column(
ForeignKey("crawl_runs.id", ondelete="SET NULL") ForeignKey("crawl_runs.id", ondelete="SET NULL")
) )
@ -322,6 +395,51 @@ class IngestionQueueItem(Base):
) )
class DataRepairJob(Base):
__tablename__ = "data_repair_jobs"
__table_args__ = (
CheckConstraint(
"status IN ('planned', 'running', 'completed', 'failed')",
name="data_repair_jobs_status_valid",
),
Index("ix_data_repair_jobs_created_at", "created_at"),
Index("ix_data_repair_jobs_job_name_created_at", "job_name", "created_at"),
)
id: Mapped[int] = mapped_column(primary_key=True)
job_name: Mapped[str] = mapped_column(String(64), nullable=False)
requested_by: Mapped[str | None] = mapped_column(String(255))
scope: Mapped[dict] = mapped_column(
JSONB,
nullable=False,
server_default=text("'{}'::jsonb"),
)
dry_run: Mapped[bool] = mapped_column(
Boolean,
nullable=False,
server_default=text("true"),
)
status: Mapped[str] = mapped_column(
String(16),
nullable=False,
server_default=text("'planned'"),
)
summary: Mapped[dict] = mapped_column(
JSONB,
nullable=False,
server_default=text("'{}'::jsonb"),
)
error_text: Mapped[str | None] = mapped_column(Text)
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class AuthorSearchRuntimeState(Base): class AuthorSearchRuntimeState(Base):
__tablename__ = "author_search_runtime_state" __tablename__ = "author_search_runtime_state"

View file

@ -61,7 +61,7 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
start = time.perf_counter() start = time.perf_counter()
should_log = self._log_requests and not self._is_skipped_path(request.url.path) should_log = self._log_requests and not self._is_skipped_path(request.url.path)
if should_log: if should_log:
logger.info( logger.debug(
"request.started", "request.started",
extra={ extra={
"event": "request.started", "event": "request.started",
@ -88,7 +88,7 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
duration_ms = int((time.perf_counter() - start) * 1000) duration_ms = int((time.perf_counter() - start) * 1000)
response.headers[REQUEST_ID_HEADER] = request_id response.headers[REQUEST_ID_HEADER] = request_id
if should_log: if should_log:
logger.info( logger.debug(
"request.completed", "request.completed",
extra={ extra={
"event": "request.completed", "event": "request.completed",

View file

@ -0,0 +1,5 @@
from app.services.domains.dbops.application import run_publication_link_repair
from app.services.domains.dbops.integrity import collect_integrity_report
from app.services.domains.dbops.query import list_repair_jobs
__all__ = ["collect_integrity_report", "list_repair_jobs", "run_publication_link_repair"]

View file

@ -0,0 +1,364 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from sqlalchemy import delete, exists, func, select, update
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import DataRepairJob, IngestionQueueItem, Publication, ScholarProfile, ScholarPublication
REPAIR_STATUS_PLANNED = "planned"
REPAIR_STATUS_RUNNING = "running"
REPAIR_STATUS_COMPLETED = "completed"
REPAIR_STATUS_FAILED = "failed"
SCOPE_MODE_SINGLE_USER = "single_user"
SCOPE_MODE_ALL_USERS = "all_users"
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
def _normalize_scope_mode(scope_mode: str) -> str:
normalized = scope_mode.strip().lower()
if normalized in {SCOPE_MODE_SINGLE_USER, SCOPE_MODE_ALL_USERS}:
return normalized
raise ValueError("Unknown scope mode.")
def _scope_user_id(*, scope_mode: str, user_id: int | None) -> int | None:
if scope_mode == SCOPE_MODE_SINGLE_USER:
if user_id is None:
raise ValueError("user_id is required when scope_mode=single_user.")
return int(user_id)
if user_id is not None:
raise ValueError("user_id must be omitted when scope_mode=all_users.")
return None
def _scope_payload(
*,
scope_mode: str,
user_id: int | None,
target_scholar_profile_ids: list[int],
orphan_gc: bool,
) -> dict[str, Any]:
payload: dict[str, Any] = {
"scope_mode": scope_mode,
"scholar_profile_ids": [int(value) for value in target_scholar_profile_ids],
"gc_orphan_publications": bool(orphan_gc),
}
if user_id is not None:
payload["user_id"] = int(user_id)
return payload
async def _target_scholar_profile_ids(
db_session: AsyncSession,
*,
scope_mode: str,
user_id: int | None,
scholar_profile_ids: list[int] | None,
) -> list[int]:
stmt = select(ScholarProfile.id)
if scope_mode == SCOPE_MODE_SINGLE_USER:
stmt = stmt.where(ScholarProfile.user_id == user_id)
if scholar_profile_ids:
normalized_ids = [int(value) for value in scholar_profile_ids]
stmt = stmt.where(ScholarProfile.id.in_(normalized_ids))
result = await db_session.execute(stmt.order_by(ScholarProfile.id.asc()))
ids = [int(row[0]) for row in result.all()]
if not ids:
raise ValueError("No target scholar profiles found for the requested scope.")
return ids
async def _count_scope(
db_session: AsyncSession,
*,
user_id: int | None,
target_scholar_profile_ids: list[int],
) -> dict[str, int]:
links_result = await db_session.execute(
select(func.count())
.select_from(ScholarPublication)
.where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids))
)
queue_stmt = (
select(func.count())
.select_from(IngestionQueueItem)
.where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids))
)
if user_id is not None:
queue_stmt = queue_stmt.where(IngestionQueueItem.user_id == user_id)
queue_result = await db_session.execute(queue_stmt)
return {
"target_scholar_count": len(target_scholar_profile_ids),
"links_in_scope": int(links_result.scalar_one() or 0),
"queue_items_in_scope": int(queue_result.scalar_one() or 0),
}
async def _count_orphan_publications(db_session: AsyncSession) -> int:
stmt = (
select(func.count())
.select_from(Publication)
.where(
~exists(
select(1).where(ScholarPublication.publication_id == Publication.id)
)
)
)
result = await db_session.execute(stmt)
return int(result.scalar_one() or 0)
async def _create_job(
db_session: AsyncSession,
*,
requested_by: str | None,
scope: dict[str, Any],
dry_run: bool,
) -> DataRepairJob:
job = DataRepairJob(
job_name="repair_publication_links",
requested_by=(requested_by or "").strip() or None,
scope=scope,
dry_run=dry_run,
status=REPAIR_STATUS_PLANNED,
summary={},
)
db_session.add(job)
await db_session.flush()
return job
def _job_summary(
*,
counts: dict[str, int],
dry_run: bool,
links_deleted: int,
queue_items_deleted: int,
scholars_reset: int,
orphan_publications_before: int,
orphan_publications_deleted: int,
) -> dict[str, Any]:
return {
**counts,
"dry_run": bool(dry_run),
"links_deleted": int(links_deleted),
"queue_items_deleted": int(queue_items_deleted),
"scholars_reset": int(scholars_reset),
"orphan_publications_before": int(orphan_publications_before),
"orphan_publications_deleted": int(orphan_publications_deleted),
}
async def _delete_links_for_targets(db_session: AsyncSession, *, target_scholar_profile_ids: list[int]) -> int:
result = await db_session.execute(
delete(ScholarPublication).where(
ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids)
)
)
return int(result.rowcount or 0)
async def _delete_queue_for_targets(
db_session: AsyncSession,
*,
user_id: int | None,
target_scholar_profile_ids: list[int],
) -> int:
stmt = delete(IngestionQueueItem).where(
IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids)
)
if user_id is not None:
stmt = stmt.where(IngestionQueueItem.user_id == user_id)
result = await db_session.execute(stmt)
return int(result.rowcount or 0)
async def _reset_scholar_tracking_state(
db_session: AsyncSession,
*,
user_id: int | None,
target_scholar_profile_ids: list[int],
) -> int:
stmt = update(ScholarProfile).where(ScholarProfile.id.in_(target_scholar_profile_ids))
if user_id is not None:
stmt = stmt.where(ScholarProfile.user_id == user_id)
result = await db_session.execute(
stmt.values(
baseline_completed=False,
last_initial_page_fingerprint_sha256=None,
last_initial_page_checked_at=None,
last_run_dt=None,
last_run_status=None,
)
)
return int(result.rowcount or 0)
async def _delete_orphan_publications(db_session: AsyncSession) -> int:
result = await db_session.execute(
delete(Publication).where(
~exists(select(1).where(ScholarPublication.publication_id == Publication.id))
)
)
return int(result.rowcount or 0)
async def _mutation_counts(
db_session: AsyncSession,
*,
user_id: int | None,
target_ids: list[int],
dry_run: bool,
gc_orphan_publications: bool,
) -> tuple[int, int, int, int]:
if dry_run:
return 0, 0, 0, 0
links_deleted = await _delete_links_for_targets(
db_session,
target_scholar_profile_ids=target_ids,
)
queue_deleted = await _delete_queue_for_targets(
db_session,
user_id=user_id,
target_scholar_profile_ids=target_ids,
)
scholars_reset = await _reset_scholar_tracking_state(
db_session,
user_id=user_id,
target_scholar_profile_ids=target_ids,
)
orphan_deleted = 0
if gc_orphan_publications:
orphan_deleted = await _delete_orphan_publications(db_session)
return links_deleted, queue_deleted, scholars_reset, orphan_deleted
def _result_payload(*, job: DataRepairJob, scope: dict[str, Any], summary: dict[str, Any]) -> dict[str, Any]:
return {
"job_id": int(job.id),
"status": job.status,
"scope": scope,
"summary": summary,
}
async def _complete_job(
db_session: AsyncSession,
*,
job: DataRepairJob,
summary: dict[str, Any],
scope: dict[str, Any],
) -> dict[str, Any]:
job.summary = summary
job.status = REPAIR_STATUS_COMPLETED
job.finished_at = _utcnow()
await db_session.commit()
return _result_payload(job=job, scope=scope, summary=summary)
async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None:
await db_session.rollback()
job.status = REPAIR_STATUS_FAILED
job.error_text = str(error)
job.finished_at = _utcnow()
db_session.add(job)
await db_session.commit()
async def _prepare_repair_job(
db_session: AsyncSession,
*,
scope_mode: str,
user_id: int | None,
scholar_profile_ids: list[int] | None,
dry_run: bool,
gc_orphan_publications: bool,
requested_by: str | None,
) -> tuple[int | None, list[int], dict[str, Any], DataRepairJob]:
normalized_scope = _normalize_scope_mode(scope_mode)
scope_user_id = _scope_user_id(scope_mode=normalized_scope, user_id=user_id)
target_ids = await _target_scholar_profile_ids(
db_session,
scope_mode=normalized_scope,
user_id=scope_user_id,
scholar_profile_ids=scholar_profile_ids,
)
scope = _scope_payload(
scope_mode=normalized_scope,
user_id=scope_user_id,
target_scholar_profile_ids=target_ids,
orphan_gc=gc_orphan_publications,
)
job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run)
job.status = REPAIR_STATUS_RUNNING
job.started_at = _utcnow()
return scope_user_id, target_ids, scope, job
async def _build_repair_summary(
db_session: AsyncSession,
*,
scope_user_id: int | None,
target_ids: list[int],
dry_run: bool,
gc_orphan_publications: bool,
) -> dict[str, Any]:
counts = await _count_scope(db_session, user_id=scope_user_id, target_scholar_profile_ids=target_ids)
orphan_before = await _count_orphan_publications(db_session)
links_deleted, queue_deleted, scholars_reset, orphan_deleted = await _mutation_counts(
db_session,
user_id=scope_user_id,
target_ids=target_ids,
dry_run=dry_run,
gc_orphan_publications=gc_orphan_publications,
)
return _job_summary(
counts=counts,
dry_run=dry_run,
links_deleted=links_deleted,
queue_items_deleted=queue_deleted,
scholars_reset=scholars_reset,
orphan_publications_before=orphan_before,
orphan_publications_deleted=orphan_deleted,
)
async def run_publication_link_repair(
db_session: AsyncSession,
*,
scope_mode: str = SCOPE_MODE_SINGLE_USER,
user_id: int | None = None,
scholar_profile_ids: list[int] | None = None,
dry_run: bool = True,
gc_orphan_publications: bool = False,
requested_by: str | None = None,
) -> dict[str, Any]:
scope_user_id, target_ids, scope, job = await _prepare_repair_job(
db_session,
scope_mode=scope_mode,
user_id=user_id,
scholar_profile_ids=scholar_profile_ids,
dry_run=dry_run,
gc_orphan_publications=gc_orphan_publications,
requested_by=requested_by,
)
try:
summary = await _build_repair_summary(
db_session,
scope_user_id=scope_user_id,
target_ids=target_ids,
dry_run=dry_run,
gc_orphan_publications=gc_orphan_publications,
)
return await _complete_job(db_session, job=job, summary=summary, scope=scope)
except Exception as exc:
await _fail_job(db_session, job=job, error=exc)
raise

View file

@ -0,0 +1,175 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from sqlalchemy import func, select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Publication
INTEGRITY_CHECK_DEFS = (
(
"legacy_cluster_id_format",
"warning",
"Publications with non-namespaced cluster IDs.",
),
(
"negative_citation_count",
"failure",
"Publications with negative citation counts.",
),
(
"orphan_publications",
"warning",
"Publications with no scholar links.",
),
(
"orphan_scholar_publication_links",
"failure",
"Link rows missing parent scholar/publication.",
),
(
"duplicate_fingerprint_keys",
"failure",
"Duplicate publication fingerprint keys.",
),
(
"duplicate_cluster_ids",
"failure",
"Duplicate non-null publication cluster IDs.",
),
(
"missing_pdf_url",
"metric",
"Publications without a resolved PDF URL.",
),
)
async def _legacy_cluster_id_count(db_session: AsyncSession) -> int:
stmt = (
select(func.count())
.select_from(Publication)
.where(Publication.cluster_id.is_not(None))
.where(~Publication.cluster_id.like("cfv:%"))
.where(~Publication.cluster_id.like("cluster:%"))
)
result = await db_session.execute(stmt)
return int(result.scalar_one() or 0)
async def _negative_citation_count(db_session: AsyncSession) -> int:
result = await db_session.execute(
select(func.count()).select_from(Publication).where(Publication.citation_count < 0)
)
return int(result.scalar_one() or 0)
async def _missing_pdf_url_count(db_session: AsyncSession) -> int:
result = await db_session.execute(
select(func.count()).select_from(Publication).where(Publication.pdf_url.is_(None))
)
return int(result.scalar_one() or 0)
async def _count_from_sql(db_session: AsyncSession, *, sql: str) -> int:
result = await db_session.execute(text(sql))
return int(result.scalar_one() or 0)
def _issues_for_severity(*, checks: list[dict[str, Any]], severity: str) -> list[str]:
return [row["name"] for row in checks if row["severity"] == severity and row["count"] > 0]
def _check_row(*, name: str, count: int, severity: str, message: str) -> dict[str, Any]:
return {
"name": name,
"count": int(count),
"severity": severity,
"message": message,
}
async def _collect_counts(db_session: AsyncSession) -> dict[str, int]:
orphan_publications = await _count_from_sql(
db_session,
sql=(
"SELECT count(*) FROM publications p "
"LEFT JOIN scholar_publications sp ON sp.publication_id = p.id "
"WHERE sp.publication_id IS NULL"
),
)
orphan_links = await _count_from_sql(
db_session,
sql=(
"SELECT count(*) FROM scholar_publications sp "
"LEFT JOIN publications p ON p.id = sp.publication_id "
"LEFT JOIN scholar_profiles s ON s.id = sp.scholar_profile_id "
"WHERE p.id IS NULL OR s.id IS NULL"
),
)
duplicate_fingerprints = await _count_from_sql(
db_session,
sql=(
"SELECT count(*) FROM ("
"SELECT fingerprint_sha256 FROM publications "
"GROUP BY fingerprint_sha256 HAVING count(*) > 1"
") dup"
),
)
duplicate_cluster_ids = await _count_from_sql(
db_session,
sql=(
"SELECT count(*) FROM ("
"SELECT cluster_id FROM publications "
"WHERE cluster_id IS NOT NULL "
"GROUP BY cluster_id HAVING count(*) > 1"
") dup"
),
)
return {
"legacy_cluster_id_format": await _legacy_cluster_id_count(db_session),
"negative_citation_count": await _negative_citation_count(db_session),
"orphan_publications": orphan_publications,
"orphan_scholar_publication_links": orphan_links,
"duplicate_fingerprint_keys": duplicate_fingerprints,
"duplicate_cluster_ids": duplicate_cluster_ids,
"missing_pdf_url": await _missing_pdf_url_count(db_session),
}
def _build_checks(*, counts: dict[str, int]) -> list[dict[str, Any]]:
return [
_check_row(
name=name,
count=counts[name],
severity=severity,
message=message,
)
for name, severity, message in INTEGRITY_CHECK_DEFS
]
def _status_from_issues(*, failures: list[str], warnings: list[str]) -> str:
status = "ok"
if failures:
status = "failed"
elif warnings:
status = "warning"
return status
async def collect_integrity_report(db_session: AsyncSession) -> dict[str, Any]:
counts = await _collect_counts(db_session)
checks = _build_checks(counts=counts)
failures = _issues_for_severity(checks=checks, severity="failure")
warnings = _issues_for_severity(checks=checks, severity="warning")
return {
"status": _status_from_issues(failures=failures, warnings=warnings),
"checked_at": datetime.now(timezone.utc).isoformat(),
"failures": failures,
"warnings": warnings,
"checks": checks,
}

View file

@ -0,0 +1,22 @@
from __future__ import annotations
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import DataRepairJob
def _bounded_limit(limit: int) -> int:
return max(1, min(int(limit), 200))
async def list_repair_jobs(
db_session: AsyncSession,
*,
limit: int = 50,
) -> list[DataRepairJob]:
bounded = _bounded_limit(limit)
result = await db_session.execute(
select(DataRepairJob).order_by(DataRepairJob.created_at.desc()).limit(bounded)
)
return list(result.scalars())

View file

@ -91,6 +91,35 @@ class ScholarIngestionService:
def __init__(self, *, source: ScholarSource) -> None: def __init__(self, *, source: ScholarSource) -> None:
self._source = source self._source = source
@staticmethod
def _effective_request_delay_seconds(value: int) -> int:
policy_minimum = user_settings_service.resolve_request_delay_minimum(
settings.ingestion_min_request_delay_seconds
)
return max(policy_minimum, _int_or_default(value, policy_minimum))
@staticmethod
def _log_request_delay_coercion(
*,
user_id: int,
requested_request_delay_seconds: int,
effective_request_delay_seconds: int,
) -> None:
logger.warning(
"ingestion.request_delay_coerced_to_policy_floor",
extra={
"event": "ingestion.request_delay_coerced_to_policy_floor",
"user_id": user_id,
"requested_request_delay_seconds": requested_request_delay_seconds,
"effective_request_delay_seconds": effective_request_delay_seconds,
"policy_minimum_request_delay_seconds": user_settings_service.resolve_request_delay_minimum(
settings.ingestion_min_request_delay_seconds
),
"metric_name": "ingestion_request_delay_coerced_total",
"metric_value": 1,
},
)
async def _load_user_settings_for_run( async def _load_user_settings_for_run(
self, self,
db_session: AsyncSession, db_session: AsyncSession,
@ -1278,7 +1307,14 @@ class ScholarIngestionService:
return failure_summary, alert_summary return failure_summary, alert_summary
async def run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, request_delay_seconds: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, max_pages_per_scholar: int = 30, page_size: int = 100, scholar_profile_ids: set[int] | None = None, start_cstart_by_scholar_id: dict[int, int] | None = None, auto_queue_continuations: bool = True, queue_delay_seconds: int = 60, idempotency_key: str | None = None, alert_blocked_failure_threshold: int = 1, alert_network_failure_threshold: int = 2, alert_retry_scheduled_threshold: int = 3) -> RunExecutionSummary: async def run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, request_delay_seconds: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, max_pages_per_scholar: int = 30, page_size: int = 100, scholar_profile_ids: set[int] | None = None, start_cstart_by_scholar_id: dict[int, int] | None = None, auto_queue_continuations: bool = True, queue_delay_seconds: int = 60, idempotency_key: str | None = None, alert_blocked_failure_threshold: int = 1, alert_network_failure_threshold: int = 2, alert_retry_scheduled_threshold: int = 3) -> RunExecutionSummary:
paging_kwargs = self._paging_kwargs(request_delay_seconds=request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size) effective_request_delay_seconds = self._effective_request_delay_seconds(request_delay_seconds)
if effective_request_delay_seconds != _int_or_default(request_delay_seconds, effective_request_delay_seconds):
self._log_request_delay_coercion(
user_id=user_id,
requested_request_delay_seconds=_int_or_default(request_delay_seconds, 0),
effective_request_delay_seconds=effective_request_delay_seconds,
)
paging_kwargs = self._paging_kwargs(request_delay_seconds=effective_request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size)
threshold_kwargs = self._threshold_kwargs(alert_blocked_failure_threshold=alert_blocked_failure_threshold, alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold) threshold_kwargs = self._threshold_kwargs(alert_blocked_failure_threshold=alert_blocked_failure_threshold, alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold)
user_settings, run, scholars, start_cstart_map = await self._initialize_run_for_user( user_settings, run, scholars, start_cstart_map = await self._initialize_run_for_user(
db_session, db_session,

View file

@ -23,12 +23,28 @@ from app.services.domains.ingestion.application import (
RunBlockedBySafetyPolicyError, RunBlockedBySafetyPolicyError,
ScholarIngestionService, ScholarIngestionService,
) )
from app.services.domains.settings import application as user_settings_service
from app.services.domains.scholar.source import LiveScholarSource from app.services.domains.scholar.source import LiveScholarSource
from app.settings import settings from app.settings import settings
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def _request_delay_floor_seconds() -> int:
return user_settings_service.resolve_request_delay_minimum(
settings.ingestion_min_request_delay_seconds
)
def _effective_request_delay_seconds(value: int | None) -> int:
floor = _request_delay_floor_seconds()
try:
parsed = int(value) if value is not None else floor
except (TypeError, ValueError):
parsed = floor
return max(floor, parsed)
@dataclass(frozen=True) @dataclass(frozen=True)
class _AutoRunCandidate: class _AutoRunCandidate:
user_id: int user_id: int
@ -178,7 +194,7 @@ class SchedulerService:
return _AutoRunCandidate( return _AutoRunCandidate(
user_id=int(user_id), user_id=int(user_id),
run_interval_minutes=int(run_interval_minutes), run_interval_minutes=int(run_interval_minutes),
request_delay_seconds=int(request_delay_seconds), request_delay_seconds=_effective_request_delay_seconds(request_delay_seconds),
cooldown_until=cooldown_until, cooldown_until=cooldown_until,
cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None), cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
) )
@ -577,6 +593,4 @@ class SchedulerService:
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id) select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
) )
delay = result.scalar_one_or_none() delay = result.scalar_one_or_none()
if delay is None: return _effective_request_delay_seconds(delay)
return 10
return max(1, int(delay))

View file

@ -2,6 +2,7 @@ from __future__ import annotations
from app.services.domains.publications.counts import ( from app.services.domains.publications.counts import (
count_for_user, count_for_user,
count_favorite_for_user,
count_latest_for_user, count_latest_for_user,
count_unread_for_user, count_unread_for_user,
) )
@ -12,7 +13,9 @@ from app.services.domains.publications.listing import (
list_unread_for_user, list_unread_for_user,
) )
from app.services.domains.publications.enrichment import ( from app.services.domains.publications.enrichment import (
hydrate_pdf_enrichment_state,
schedule_missing_pdf_enrichment_for_user, schedule_missing_pdf_enrichment_for_user,
schedule_retry_pdf_enrichment_for_row,
) )
from app.services.domains.publications.modes import ( from app.services.domains.publications.modes import (
MODE_ALL, MODE_ALL,
@ -30,6 +33,14 @@ from app.services.domains.publications.queries import (
from app.services.domains.publications.read_state import ( from app.services.domains.publications.read_state import (
mark_all_unread_as_read_for_user, mark_all_unread_as_read_for_user,
mark_selected_as_read_for_user, mark_selected_as_read_for_user,
set_publication_favorite_for_user,
)
from app.services.domains.publications.pdf_queue import (
count_pdf_queue_items,
enqueue_all_missing_pdf_jobs,
enqueue_retry_pdf_job_for_publication_id,
list_pdf_queue_page,
list_pdf_queue_items,
) )
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
@ -49,10 +60,19 @@ __all__ = [
"list_unread_for_user", "list_unread_for_user",
"list_new_for_latest_run_for_user", "list_new_for_latest_run_for_user",
"retry_pdf_for_user", "retry_pdf_for_user",
"hydrate_pdf_enrichment_state",
"schedule_retry_pdf_enrichment_for_row",
"list_pdf_queue_items",
"list_pdf_queue_page",
"count_pdf_queue_items",
"enqueue_all_missing_pdf_jobs",
"enqueue_retry_pdf_job_for_publication_id",
"schedule_missing_pdf_enrichment_for_user", "schedule_missing_pdf_enrichment_for_user",
"count_for_user", "count_for_user",
"count_favorite_for_user",
"count_unread_for_user", "count_unread_for_user",
"count_latest_for_user", "count_latest_for_user",
"mark_all_unread_as_read_for_user", "mark_all_unread_as_read_for_user",
"mark_selected_as_read_for_user", "mark_selected_as_read_for_user",
"set_publication_favorite_for_user",
] ]

View file

@ -19,6 +19,7 @@ async def count_for_user(
user_id: int, user_id: int,
mode: str = MODE_ALL, mode: str = MODE_ALL,
scholar_profile_id: int | None = None, scholar_profile_id: int | None = None,
favorite_only: bool = False,
) -> int: ) -> int:
resolved_mode = resolve_publication_view_mode(mode) resolved_mode = resolve_publication_view_mode(mode)
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
@ -30,6 +31,8 @@ async def count_for_user(
) )
if scholar_profile_id is not None: if scholar_profile_id is not None:
stmt = stmt.where(ScholarProfile.id == scholar_profile_id) stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
if favorite_only:
stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
if resolved_mode == MODE_UNREAD: if resolved_mode == MODE_UNREAD:
stmt = stmt.where(ScholarPublication.is_read.is_(False)) stmt = stmt.where(ScholarPublication.is_read.is_(False))
if resolved_mode == MODE_LATEST: if resolved_mode == MODE_LATEST:
@ -45,12 +48,14 @@ async def count_unread_for_user(
*, *,
user_id: int, user_id: int,
scholar_profile_id: int | None = None, scholar_profile_id: int | None = None,
favorite_only: bool = False,
) -> int: ) -> int:
return await count_for_user( return await count_for_user(
db_session, db_session,
user_id=user_id, user_id=user_id,
mode=MODE_UNREAD, mode=MODE_UNREAD,
scholar_profile_id=scholar_profile_id, scholar_profile_id=scholar_profile_id,
favorite_only=favorite_only,
) )
@ -59,10 +64,27 @@ async def count_latest_for_user(
*, *,
user_id: int, user_id: int,
scholar_profile_id: int | None = None, scholar_profile_id: int | None = None,
favorite_only: bool = False,
) -> int: ) -> int:
return await count_for_user( return await count_for_user(
db_session, db_session,
user_id=user_id, user_id=user_id,
mode=MODE_LATEST, mode=MODE_LATEST,
scholar_profile_id=scholar_profile_id, scholar_profile_id=scholar_profile_id,
favorite_only=favorite_only,
)
async def count_favorite_for_user(
db_session: AsyncSession,
*,
user_id: int,
scholar_profile_id: int | None = None,
) -> int:
return await count_for_user(
db_session,
user_id=user_id,
mode=MODE_ALL,
scholar_profile_id=scholar_profile_id,
favorite_only=True,
) )

View file

@ -1,135 +1,73 @@
from __future__ import annotations from __future__ import annotations
import asyncio
import logging import logging
import time
from app.db.session import get_session_factory from sqlalchemy.ext.asyncio import AsyncSession
from app.services.domains.publications.listing import (
missing_pdf_items, from app.services.domains.publications.pdf_queue import (
resolve_and_persist_oa_metadata, enqueue_missing_pdf_jobs,
enqueue_retry_pdf_job,
overlay_pdf_job_state,
) )
from app.services.domains.publications.types import PublicationListItem from app.services.domains.publications.types import PublicationListItem
from app.settings import settings
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_enrichment_lock = asyncio.Lock()
_inflight_publications: set[tuple[int, int]] = set()
_recent_attempt_seconds: dict[tuple[int, int], float] = {}
_scheduled_tasks: set[asyncio.Task[None]] = set()
def _cooldown_seconds() -> float:
return max(float(settings.unpaywall_retry_cooldown_seconds), 1.0)
def _prune_recent_attempts(now_seconds: float, *, cooldown_seconds: float) -> None:
expiry = cooldown_seconds * 3
stale_keys = [
key for key, attempted_seconds in _recent_attempt_seconds.items()
if now_seconds - attempted_seconds >= expiry
]
for key in stale_keys:
_recent_attempt_seconds.pop(key, None)
async def _claim_items(
*,
user_id: int,
items: list[PublicationListItem],
max_items: int,
) -> list[PublicationListItem]:
candidates = missing_pdf_items(items, limit=max_items)
if not candidates:
return []
now_seconds = time.monotonic()
cooldown_seconds = _cooldown_seconds()
claimed: list[PublicationListItem] = []
async with _enrichment_lock:
_prune_recent_attempts(now_seconds, cooldown_seconds=cooldown_seconds)
for item in candidates:
key = (user_id, item.publication_id)
attempted_seconds = _recent_attempt_seconds.get(key)
if key in _inflight_publications:
continue
if attempted_seconds is not None and now_seconds - attempted_seconds < cooldown_seconds:
continue
_inflight_publications.add(key)
_recent_attempt_seconds[key] = now_seconds
claimed.append(item)
return claimed
async def _release_claims(*, user_id: int, publication_ids: list[int]) -> None:
async with _enrichment_lock:
for publication_id in publication_ids:
_inflight_publications.discard((user_id, publication_id))
def _on_task_done(task: asyncio.Task[None]) -> None:
_scheduled_tasks.discard(task)
try:
task.result()
except Exception:
logger.exception(
"publications.enrichment.task_failed",
extra={"event": "publications.enrichment.task_failed"},
)
async def _run_enrichment(
*,
user_id: int,
request_email: str | None,
items: list[PublicationListItem],
) -> None:
publication_ids = [item.publication_id for item in items]
try:
session_factory = get_session_factory()
async with session_factory() as db_session:
await resolve_and_persist_oa_metadata(
db_session,
rows=items,
unpaywall_email=request_email,
)
finally:
await _release_claims(user_id=user_id, publication_ids=publication_ids)
logger.info(
"publications.enrichment.completed",
extra={
"event": "publications.enrichment.completed",
"user_id": user_id,
"publication_count": len(items),
},
)
async def schedule_missing_pdf_enrichment_for_user( async def schedule_missing_pdf_enrichment_for_user(
db_session: AsyncSession,
*, *,
user_id: int, user_id: int,
request_email: str | None, request_email: str | None,
items: list[PublicationListItem], items: list[PublicationListItem],
max_items: int, max_items: int,
) -> int: ) -> int:
claimed_items = await _claim_items(user_id=user_id, items=items, max_items=max_items) queued_ids = await enqueue_missing_pdf_jobs(
if not claimed_items: db_session,
return 0
task = asyncio.create_task(
_run_enrichment(
user_id=user_id, user_id=user_id,
request_email=request_email, request_email=request_email,
items=claimed_items, rows=items,
max_items=max_items,
) )
)
_scheduled_tasks.add(task)
task.add_done_callback(_on_task_done)
logger.info( logger.info(
"publications.enrichment.scheduled", "publications.enrichment.scheduled",
extra={ extra={
"event": "publications.enrichment.scheduled", "event": "publications.enrichment.scheduled",
"user_id": user_id, "user_id": user_id,
"publication_count": len(claimed_items), "publication_count": len(queued_ids),
}, },
) )
return len(claimed_items) return len(queued_ids)
async def schedule_retry_pdf_enrichment_for_row(
db_session: AsyncSession,
*,
user_id: int,
request_email: str | None,
item: PublicationListItem,
) -> bool:
queued = await enqueue_retry_pdf_job(
db_session,
user_id=user_id,
request_email=request_email,
row=item,
)
logger.info(
"publications.enrichment.retry_scheduled",
extra={
"event": "publications.enrichment.retry_scheduled",
"user_id": user_id,
"publication_id": item.publication_id,
"queued": queued,
},
)
return queued
async def hydrate_pdf_enrichment_state(
db_session: AsyncSession,
*,
items: list[PublicationListItem],
) -> list[PublicationListItem]:
return await overlay_pdf_job_state(db_session, rows=items)

View file

@ -16,94 +16,6 @@ from app.services.domains.publications.queries import (
unread_item_from_row, unread_item_from_row,
) )
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
from app.services.domains.unpaywall.application import resolve_publication_oa_metadata
from sqlalchemy import update
from app.db.models import Publication
def _with_oa_overrides(
rows: list[PublicationListItem],
oa_data: dict[int, tuple[str | None, str | None]],
) -> list[PublicationListItem]:
return [
PublicationListItem(
publication_id=row.publication_id,
scholar_profile_id=row.scholar_profile_id,
scholar_label=row.scholar_label,
title=row.title,
year=row.year,
citation_count=row.citation_count,
venue_text=row.venue_text,
pub_url=row.pub_url,
doi=(oa_data.get(row.publication_id) or (None, None))[0] or row.doi,
pdf_url=(oa_data.get(row.publication_id) or (None, None))[1] or row.pdf_url,
is_read=row.is_read,
first_seen_at=row.first_seen_at,
is_new_in_latest_run=row.is_new_in_latest_run,
)
for row in rows
]
def _resolved_fields(
*,
row: PublicationListItem,
resolved: tuple[str | None, str | None] | None,
) -> tuple[str | None, str | None]:
if resolved is None:
return row.doi, row.pdf_url
resolved_doi, resolved_pdf = resolved
return resolved_doi or row.doi, resolved_pdf or row.pdf_url
async def _persist_resolved_metadata(
db_session: AsyncSession,
*,
rows: list[PublicationListItem],
oa_data: dict[int, tuple[str | None, str | None]],
) -> None:
by_id = {row.publication_id: row for row in rows}
updates: list[tuple[int, str | None, str | None]] = []
for publication_id, resolved in oa_data.items():
existing = by_id.get(publication_id)
if existing is None:
continue
next_doi, next_pdf = _resolved_fields(row=existing, resolved=resolved)
if next_doi == existing.doi and next_pdf == existing.pdf_url:
continue
updates.append((publication_id, next_doi, next_pdf))
for publication_id, doi, pdf_url in updates:
await db_session.execute(
update(Publication)
.where(Publication.id == publication_id)
.values(doi=doi, pdf_url=pdf_url)
)
if updates:
await db_session.commit()
def missing_pdf_items(
rows: list[PublicationListItem],
*,
limit: int,
) -> list[PublicationListItem]:
bounded_limit = max(0, int(limit))
if bounded_limit == 0:
return []
return [row for row in rows if not row.pdf_url][:bounded_limit]
async def resolve_and_persist_oa_metadata(
db_session: AsyncSession,
*,
rows: list[PublicationListItem],
unpaywall_email: str | None = None,
) -> dict[int, tuple[str | None, str | None]]:
if not rows:
return {}
oa_data = await resolve_publication_oa_metadata(rows, request_email=unpaywall_email)
await _persist_resolved_metadata(db_session, rows=rows, oa_data=oa_data)
return oa_data
async def list_for_user( async def list_for_user(
@ -112,7 +24,9 @@ async def list_for_user(
user_id: int, user_id: int,
mode: str = MODE_ALL, mode: str = MODE_ALL,
scholar_profile_id: int | None = None, scholar_profile_id: int | None = None,
favorite_only: bool = False,
limit: int = 300, limit: int = 300,
offset: int = 0,
) -> list[PublicationListItem]: ) -> list[PublicationListItem]:
resolved_mode = resolve_publication_view_mode(mode) resolved_mode = resolve_publication_view_mode(mode)
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
@ -122,7 +36,9 @@ async def list_for_user(
mode=resolved_mode, mode=resolved_mode,
latest_run_id=latest_run_id, latest_run_id=latest_run_id,
scholar_profile_id=scholar_profile_id, scholar_profile_id=scholar_profile_id,
favorite_only=favorite_only,
limit=limit, limit=limit,
offset=offset,
) )
) )
rows = [ rows = [
@ -138,22 +54,13 @@ async def retry_pdf_for_user(
user_id: int, user_id: int,
scholar_profile_id: int, scholar_profile_id: int,
publication_id: int, publication_id: int,
unpaywall_email: str | None = None,
) -> PublicationListItem | None: ) -> PublicationListItem | None:
item = await get_publication_item_for_user( return await get_publication_item_for_user(
db_session, db_session,
user_id=user_id, user_id=user_id,
scholar_profile_id=scholar_profile_id, scholar_profile_id=scholar_profile_id,
publication_id=publication_id, publication_id=publication_id,
) )
if item is None:
return None
oa_data = await resolve_and_persist_oa_metadata(
db_session,
rows=[item],
unpaywall_email=unpaywall_email,
)
return _with_oa_overrides([item], oa_data)[0]
async def list_unread_for_user( async def list_unread_for_user(
@ -168,7 +75,9 @@ async def list_unread_for_user(
mode=MODE_UNREAD, mode=MODE_UNREAD,
latest_run_id=None, latest_run_id=None,
scholar_profile_id=None, scholar_profile_id=None,
favorite_only=False,
limit=limit, limit=limit,
offset=0,
) )
) )
return [unread_item_from_row(row) for row in result.all()] return [unread_item_from_row(row) for row in result.all()]

View file

@ -0,0 +1,877 @@
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from datetime import datetime, timezone
import logging
from sqlalchemy import Select, func, literal, or_, select, union_all
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import (
Publication,
PublicationPdfJob,
PublicationPdfJobEvent,
ScholarProfile,
ScholarPublication,
User,
)
from app.db.session import get_session_factory
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.unpaywall.application import (
FAILURE_RESOLUTION_EXCEPTION,
OaResolutionOutcome,
resolve_publication_oa_outcomes,
)
from app.settings import settings
PDF_STATUS_UNTRACKED = "untracked"
PDF_STATUS_QUEUED = "queued"
PDF_STATUS_RUNNING = "running"
PDF_STATUS_RESOLVED = "resolved"
PDF_STATUS_FAILED = "failed"
PDF_EVENT_QUEUED = "queued"
PDF_EVENT_ATTEMPT_STARTED = "attempt_started"
PDF_EVENT_RESOLVED = "resolved"
PDF_EVENT_FAILED = "failed"
logger = logging.getLogger(__name__)
_scheduled_tasks: set[asyncio.Task[None]] = set()
@dataclass(frozen=True)
class PdfQueueListItem:
publication_id: int
title: str
doi: str | None
pdf_url: str | None
status: str
attempt_count: int
last_failure_reason: str | None
last_failure_detail: str | None
last_source: str | None
requested_by_user_id: int | None
requested_by_email: str | None
queued_at: datetime | None
last_attempt_at: datetime | None
resolved_at: datetime | None
updated_at: datetime
@dataclass(frozen=True)
class PdfRequeueResult:
publication_exists: bool
queued: bool
@dataclass(frozen=True)
class PdfBulkQueueResult:
requested_count: int
queued_count: int
@dataclass(frozen=True)
class PdfQueuePage:
items: list[PdfQueueListItem]
total_count: int
limit: int
offset: int
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
def _publication_ids(rows: list[PublicationListItem]) -> list[int]:
return sorted({row.publication_id for row in rows})
def _status_from_job(row: PublicationListItem, job: PublicationPdfJob | None) -> str:
if row.pdf_url:
return PDF_STATUS_RESOLVED
if job is None:
return PDF_STATUS_UNTRACKED
return job.status
def _item_from_row_and_job(
row: PublicationListItem,
job: PublicationPdfJob | None,
) -> PublicationListItem:
return PublicationListItem(
publication_id=row.publication_id,
scholar_profile_id=row.scholar_profile_id,
scholar_label=row.scholar_label,
title=row.title,
year=row.year,
citation_count=row.citation_count,
venue_text=row.venue_text,
pub_url=row.pub_url,
doi=row.doi,
pdf_url=row.pdf_url,
is_read=row.is_read,
is_favorite=row.is_favorite,
first_seen_at=row.first_seen_at,
is_new_in_latest_run=row.is_new_in_latest_run,
pdf_status=_status_from_job(row, job),
pdf_attempt_count=int(job.attempt_count) if job is not None else 0,
pdf_failure_reason=job.last_failure_reason if job is not None else None,
pdf_failure_detail=job.last_failure_detail if job is not None else None,
)
def _queueable_rows(
rows: list[PublicationListItem],
*,
max_items: int,
) -> list[PublicationListItem]:
bounded = max(0, int(max_items))
if bounded == 0:
return []
candidates = [row for row in rows if not row.pdf_url]
return candidates[:bounded]
def _bounded_limit(limit: int, *, max_value: int = 500) -> int:
return max(1, min(int(limit), max_value))
def _bounded_offset(offset: int) -> int:
return max(int(offset), 0)
def _auto_retry_interval_seconds() -> int:
return max(int(settings.pdf_auto_retry_interval_seconds), 1)
def _auto_retry_first_interval_seconds() -> int:
return max(int(settings.pdf_auto_retry_first_interval_seconds), 1)
def _auto_retry_max_attempts() -> int:
return max(int(settings.pdf_auto_retry_max_attempts), 1)
def _retry_interval_seconds_for_attempt_count(attempt_count: int) -> int:
if int(attempt_count) <= 1:
return _auto_retry_first_interval_seconds()
return _auto_retry_interval_seconds()
def _cooldown_active(
*,
last_attempt_at: datetime | None,
attempt_count: int,
) -> bool:
if last_attempt_at is None:
return False
elapsed = (_utcnow() - last_attempt_at).total_seconds()
return elapsed < _retry_interval_seconds_for_attempt_count(int(attempt_count))
def _can_enqueue_job(
job: PublicationPdfJob | None,
*,
force_retry: bool,
) -> bool:
if job is None:
return True
if job.status in {PDF_STATUS_QUEUED, PDF_STATUS_RUNNING}:
return False
if force_retry:
return job.status in {PDF_STATUS_FAILED, PDF_STATUS_RESOLVED, PDF_STATUS_UNTRACKED}
if job.status == PDF_STATUS_RESOLVED:
return False
if int(job.attempt_count) >= _auto_retry_max_attempts():
return False
if _cooldown_active(
last_attempt_at=job.last_attempt_at,
attempt_count=int(job.attempt_count),
):
return False
return True
def _event_row(
*,
publication_id: int,
user_id: int | None,
event_type: str,
status: str | None,
source: str | None = None,
failure_reason: str | None = None,
message: str | None = None,
) -> PublicationPdfJobEvent:
return PublicationPdfJobEvent(
publication_id=publication_id,
user_id=user_id,
event_type=event_type,
status=status,
source=source,
failure_reason=failure_reason,
message=message,
)
def _queued_job(
*,
publication_id: int,
user_id: int,
) -> PublicationPdfJob:
now = _utcnow()
return PublicationPdfJob(
publication_id=publication_id,
status=PDF_STATUS_QUEUED,
queued_at=now,
last_requested_by_user_id=user_id,
)
def _mark_job_queued(job: PublicationPdfJob, *, user_id: int) -> None:
now = _utcnow()
job.status = PDF_STATUS_QUEUED
job.queued_at = now
job.last_requested_by_user_id = user_id
job.last_failure_reason = None
job.last_failure_detail = None
job.last_source = None
def _state_map(jobs: list[PublicationPdfJob]) -> dict[int, PublicationPdfJob]:
return {int(job.publication_id): job for job in jobs}
async def _jobs_for_publication_ids(
db_session: AsyncSession,
*,
publication_ids: list[int],
) -> dict[int, PublicationPdfJob]:
if not publication_ids:
return {}
result = await db_session.execute(
select(PublicationPdfJob).where(PublicationPdfJob.publication_id.in_(publication_ids))
)
return _state_map(list(result.scalars()))
async def overlay_pdf_job_state(
db_session: AsyncSession,
*,
rows: list[PublicationListItem],
) -> list[PublicationListItem]:
if not rows:
return []
jobs = await _jobs_for_publication_ids(
db_session,
publication_ids=_publication_ids(rows),
)
return [_item_from_row_and_job(row, jobs.get(row.publication_id)) for row in rows]
async def _enqueue_rows(
db_session: AsyncSession,
*,
user_id: int,
rows: list[PublicationListItem],
force_retry: bool,
) -> list[PublicationListItem]:
if not rows:
return []
queued: list[PublicationListItem] = []
jobs = await _jobs_for_publication_ids(
db_session,
publication_ids=_publication_ids(rows),
)
for row in rows:
job = jobs.get(row.publication_id)
if not _can_enqueue_job(job, force_retry=force_retry):
continue
if job is None:
job = _queued_job(publication_id=row.publication_id, user_id=user_id)
jobs[row.publication_id] = job
db_session.add(job)
else:
_mark_job_queued(job, user_id=user_id)
db_session.add(
_event_row(
publication_id=row.publication_id,
user_id=user_id,
event_type=PDF_EVENT_QUEUED,
status=PDF_STATUS_QUEUED,
)
)
queued.append(row)
if queued:
await db_session.commit()
return queued
def _register_task(task: asyncio.Task[None]) -> None:
_scheduled_tasks.add(task)
def _drop_finished_task(task: asyncio.Task[None]) -> None:
_scheduled_tasks.discard(task)
try:
task.result()
except Exception:
logger.exception(
"publications.pdf_queue.task_failed",
extra={"event": "publications.pdf_queue.task_failed"},
)
async def _mark_attempt_started(
*,
publication_id: int,
user_id: int,
) -> None:
session_factory = get_session_factory()
async with session_factory() as db_session:
job = await db_session.get(PublicationPdfJob, publication_id)
if job is None:
job = _queued_job(publication_id=publication_id, user_id=user_id)
db_session.add(job)
job.status = PDF_STATUS_RUNNING
job.last_attempt_at = _utcnow()
job.attempt_count = int(job.attempt_count) + 1
db_session.add(
_event_row(
publication_id=publication_id,
user_id=user_id,
event_type=PDF_EVENT_ATTEMPT_STARTED,
status=PDF_STATUS_RUNNING,
)
)
await db_session.commit()
def _failed_outcome(
*,
row: PublicationListItem,
) -> OaResolutionOutcome:
return OaResolutionOutcome(
publication_id=row.publication_id,
doi=row.doi,
pdf_url=None,
failure_reason=FAILURE_RESOLUTION_EXCEPTION,
source=None,
used_crossref=False,
)
async def _fetch_outcome_for_row(
*,
row: PublicationListItem,
request_email: str | None,
) -> OaResolutionOutcome:
outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email)
outcome = outcomes.get(row.publication_id)
if outcome is not None:
return outcome
return _failed_outcome(row=row)
def _apply_publication_update(
publication: Publication,
*,
doi: str | None,
pdf_url: str | None,
) -> None:
if doi and publication.doi != doi:
publication.doi = doi
if pdf_url and publication.pdf_url != pdf_url:
publication.pdf_url = pdf_url
def _apply_job_outcome(job: PublicationPdfJob, *, outcome: OaResolutionOutcome) -> None:
job.last_source = outcome.source
if outcome.pdf_url:
job.status = PDF_STATUS_RESOLVED
job.resolved_at = _utcnow()
job.last_failure_reason = None
job.last_failure_detail = None
return
job.status = PDF_STATUS_FAILED
job.last_failure_reason = outcome.failure_reason
job.last_failure_detail = outcome.failure_reason
def _result_event(outcome: OaResolutionOutcome) -> tuple[str, str]:
if outcome.pdf_url:
return PDF_EVENT_RESOLVED, PDF_STATUS_RESOLVED
return PDF_EVENT_FAILED, PDF_STATUS_FAILED
async def _persist_outcome(
*,
publication_id: int,
user_id: int,
outcome: OaResolutionOutcome,
) -> None:
session_factory = get_session_factory()
async with session_factory() as db_session:
publication = await db_session.get(Publication, publication_id)
job = await db_session.get(PublicationPdfJob, publication_id)
if publication is None or job is None:
return
_apply_publication_update(publication, doi=outcome.doi, pdf_url=outcome.pdf_url)
_apply_job_outcome(job, outcome=outcome)
event_type, status = _result_event(outcome)
db_session.add(
_event_row(
publication_id=publication_id,
user_id=user_id,
event_type=event_type,
status=status,
source=outcome.source,
failure_reason=outcome.failure_reason,
message=outcome.failure_reason,
)
)
await db_session.commit()
async def _resolve_publication_row(
*,
user_id: int,
request_email: str | None,
row: PublicationListItem,
) -> None:
await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
try:
outcome = await _fetch_outcome_for_row(row=row, request_email=request_email)
except Exception as exc: # pragma: no cover - defensive network boundary
logger.warning(
"publications.pdf_queue.resolve_failed",
extra={
"event": "publications.pdf_queue.resolve_failed",
"publication_id": row.publication_id,
"error": str(exc),
},
)
outcome = _failed_outcome(row=row)
await _persist_outcome(
publication_id=row.publication_id,
user_id=user_id,
outcome=outcome,
)
async def _run_resolution_task(
*,
user_id: int,
request_email: str | None,
rows: list[PublicationListItem],
) -> None:
for row in rows:
await _resolve_publication_row(
user_id=user_id,
request_email=request_email,
row=row,
)
def _schedule_rows(
*,
user_id: int,
request_email: str | None,
rows: list[PublicationListItem],
) -> None:
if not rows:
return
task = asyncio.create_task(
_run_resolution_task(
user_id=user_id,
request_email=request_email,
rows=rows,
)
)
_register_task(task)
task.add_done_callback(_drop_finished_task)
async def enqueue_missing_pdf_jobs(
db_session: AsyncSession,
*,
user_id: int,
request_email: str | None,
rows: list[PublicationListItem],
max_items: int,
) -> list[int]:
queueable = _queueable_rows(rows, max_items=max_items)
queued_rows = await _enqueue_rows(
db_session,
user_id=user_id,
rows=queueable,
force_retry=False,
)
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
return [row.publication_id for row in queued_rows]
async def enqueue_retry_pdf_job(
db_session: AsyncSession,
*,
user_id: int,
request_email: str | None,
row: PublicationListItem,
) -> bool:
queued_rows = await _enqueue_rows(
db_session,
user_id=user_id,
rows=[row],
force_retry=True,
)
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
return bool(queued_rows)
def _retry_item_label(display_name: str | None, scholar_id: str | None) -> str:
return str(display_name or scholar_id or "unknown")
def _retry_item_from_publication(
publication: Publication,
*,
link_row: tuple | None,
) -> PublicationListItem:
if link_row is None:
scholar_profile_id = 0
scholar_label = "unknown"
is_read = True
first_seen_at = publication.created_at or _utcnow()
else:
scholar_profile_id = int(link_row[0])
scholar_label = _retry_item_label(link_row[1], link_row[2])
is_read = bool(link_row[3])
first_seen_at = link_row[4] or publication.created_at or _utcnow()
return PublicationListItem(
publication_id=int(publication.id),
scholar_profile_id=scholar_profile_id,
scholar_label=scholar_label,
title=publication.title_raw,
year=publication.year,
citation_count=int(publication.citation_count or 0),
venue_text=publication.venue_text,
pub_url=publication.pub_url,
doi=publication.doi,
pdf_url=publication.pdf_url,
is_read=is_read,
first_seen_at=first_seen_at,
is_new_in_latest_run=False,
)
async def _retry_item_for_publication_id(
db_session: AsyncSession,
*,
publication_id: int,
) -> PublicationListItem | None:
publication = await db_session.get(Publication, publication_id)
if publication is None:
return None
result = await db_session.execute(
select(
ScholarProfile.id,
ScholarProfile.display_name,
ScholarProfile.scholar_id,
ScholarPublication.is_read,
ScholarPublication.created_at,
)
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
.where(ScholarPublication.publication_id == publication_id)
.order_by(ScholarPublication.created_at.asc())
.limit(1)
)
return _retry_item_from_publication(publication, link_row=result.one_or_none())
async def enqueue_retry_pdf_job_for_publication_id(
db_session: AsyncSession,
*,
user_id: int,
request_email: str | None,
publication_id: int,
) -> PdfRequeueResult:
row = await _retry_item_for_publication_id(
db_session,
publication_id=publication_id,
)
if row is None:
return PdfRequeueResult(publication_exists=False, queued=False)
queued = await enqueue_retry_pdf_job(
db_session,
user_id=user_id,
request_email=request_email,
row=row,
)
return PdfRequeueResult(publication_exists=True, queued=queued)
def _queue_candidate_from_publication(publication: Publication) -> PublicationListItem:
return PublicationListItem(
publication_id=int(publication.id),
scholar_profile_id=0,
scholar_label="admin",
title=publication.title_raw,
year=publication.year,
citation_count=int(publication.citation_count or 0),
venue_text=publication.venue_text,
pub_url=publication.pub_url,
doi=publication.doi,
pdf_url=publication.pdf_url,
is_read=True,
first_seen_at=publication.created_at or _utcnow(),
is_new_in_latest_run=False,
)
async def _missing_pdf_candidates(
db_session: AsyncSession,
*,
limit: int,
) -> list[PublicationListItem]:
bounded_limit = max(1, min(int(limit), 5000))
result = await db_session.execute(
select(Publication)
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
.where(Publication.pdf_url.is_(None))
.where(
or_(
PublicationPdfJob.publication_id.is_(None),
PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
)
)
.order_by(Publication.updated_at.desc(), Publication.id.desc())
.limit(bounded_limit)
)
return [
_queue_candidate_from_publication(publication)
for publication in result.scalars()
]
async def enqueue_all_missing_pdf_jobs(
db_session: AsyncSession,
*,
user_id: int,
request_email: str | None,
limit: int = 1000,
) -> PdfBulkQueueResult:
candidates = await _missing_pdf_candidates(db_session, limit=limit)
queued_rows = await _enqueue_rows(
db_session,
user_id=user_id,
rows=candidates,
force_retry=True,
)
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
return PdfBulkQueueResult(
requested_count=len(candidates),
queued_count=len(queued_rows),
)
def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]:
stmt = (
select(
PublicationPdfJob.publication_id,
Publication.title_raw,
Publication.doi,
Publication.pdf_url,
PublicationPdfJob.status,
PublicationPdfJob.attempt_count,
PublicationPdfJob.last_failure_reason,
PublicationPdfJob.last_failure_detail,
PublicationPdfJob.last_source,
PublicationPdfJob.last_requested_by_user_id,
User.email,
PublicationPdfJob.queued_at,
PublicationPdfJob.last_attempt_at,
PublicationPdfJob.resolved_at,
PublicationPdfJob.updated_at,
)
.join(Publication, Publication.id == PublicationPdfJob.publication_id)
.outerjoin(User, User.id == PublicationPdfJob.last_requested_by_user_id)
)
if status:
stmt = stmt.where(PublicationPdfJob.status == status)
return stmt
def _tracked_queue_select(*, limit: int, offset: int, status: str | None) -> Select[tuple]:
return (
_tracked_queue_select_base(status=status)
.order_by(PublicationPdfJob.updated_at.desc())
.limit(_bounded_limit(limit))
.offset(_bounded_offset(offset))
)
def _untracked_queue_select_base() -> Select[tuple]:
return (
select(
Publication.id,
Publication.title_raw,
Publication.doi,
Publication.pdf_url,
literal(PDF_STATUS_UNTRACKED),
literal(0),
literal(None),
literal(None),
literal(None),
literal(None),
literal(None),
literal(None),
literal(None),
literal(None),
Publication.updated_at,
)
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
.where(Publication.pdf_url.is_(None))
.where(PublicationPdfJob.publication_id.is_(None))
)
def _untracked_queue_select(*, limit: int, offset: int) -> Select[tuple]:
return (
_untracked_queue_select_base()
.order_by(Publication.updated_at.desc(), Publication.id.desc())
.limit(_bounded_limit(limit))
.offset(_bounded_offset(offset))
)
def _all_queue_select(*, limit: int, offset: int) -> Select[tuple]:
union_stmt = union_all(
_tracked_queue_select_base(status=None),
_untracked_queue_select_base(),
).subquery()
return (
select(union_stmt)
.order_by(union_stmt.c.updated_at.desc())
.limit(_bounded_limit(limit))
.offset(_bounded_offset(offset))
)
def _tracked_queue_count_select(*, status: str | None) -> Select[tuple]:
stmt = select(func.count()).select_from(PublicationPdfJob)
if status:
stmt = stmt.where(PublicationPdfJob.status == status)
return stmt
def _untracked_queue_count_select() -> Select[tuple]:
return (
select(func.count())
.select_from(Publication)
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
.where(Publication.pdf_url.is_(None))
.where(PublicationPdfJob.publication_id.is_(None))
)
def _queue_item_from_row(row: tuple) -> PdfQueueListItem:
return PdfQueueListItem(
publication_id=int(row[0]),
title=str(row[1] or ""),
doi=row[2],
pdf_url=row[3],
status=str(row[4] or PDF_STATUS_UNTRACKED),
attempt_count=int(row[5] or 0),
last_failure_reason=row[6],
last_failure_detail=row[7],
last_source=row[8],
requested_by_user_id=int(row[9]) if row[9] is not None else None,
requested_by_email=row[10],
queued_at=row[11],
last_attempt_at=row[12],
resolved_at=row[13],
updated_at=row[14],
)
async def list_pdf_queue_items(
db_session: AsyncSession,
*,
limit: int = 100,
offset: int = 0,
status: str | None = None,
) -> list[PdfQueueListItem]:
bounded_limit = _bounded_limit(limit)
bounded_offset = _bounded_offset(offset)
normalized_status = (status or "").strip().lower() or None
if normalized_status == PDF_STATUS_UNTRACKED:
result = await db_session.execute(
_untracked_queue_select(
limit=bounded_limit,
offset=bounded_offset,
)
)
return [_queue_item_from_row(row) for row in result.all()]
if normalized_status is None:
result = await db_session.execute(
_all_queue_select(
limit=bounded_limit,
offset=bounded_offset,
)
)
return [_queue_item_from_row(row) for row in result.all()]
result = await db_session.execute(
_tracked_queue_select(
limit=bounded_limit,
offset=bounded_offset,
status=normalized_status,
)
)
return [_queue_item_from_row(row) for row in result.all()]
async def count_pdf_queue_items(
db_session: AsyncSession,
*,
status: str | None = None,
) -> int:
normalized_status = (status or "").strip().lower() or None
if normalized_status == PDF_STATUS_UNTRACKED:
result = await db_session.execute(_untracked_queue_count_select())
return int(result.scalar_one() or 0)
tracked_result = await db_session.execute(
_tracked_queue_count_select(status=normalized_status)
)
tracked_count = int(tracked_result.scalar_one() or 0)
if normalized_status is not None:
return tracked_count
untracked_result = await db_session.execute(_untracked_queue_count_select())
untracked_count = int(untracked_result.scalar_one() or 0)
return tracked_count + untracked_count
async def list_pdf_queue_page(
db_session: AsyncSession,
*,
limit: int = 100,
offset: int = 0,
status: str | None = None,
) -> PdfQueuePage:
bounded_limit = _bounded_limit(limit)
bounded_offset = _bounded_offset(offset)
items = await list_pdf_queue_items(
db_session,
limit=bounded_limit,
offset=bounded_offset,
status=status,
)
total_count = await count_pdf_queue_items(
db_session,
status=status,
)
return PdfQueuePage(
items=items,
total_count=total_count,
limit=bounded_limit,
offset=bounded_offset,
)

View file

@ -36,7 +36,9 @@ def publications_query(
mode: str, mode: str,
latest_run_id: int | None, latest_run_id: int | None,
scholar_profile_id: int | None, scholar_profile_id: int | None,
favorite_only: bool,
limit: int, limit: int,
offset: int = 0,
) -> Select[tuple]: ) -> Select[tuple]:
scholar_label = ScholarProfile.display_name scholar_label = ScholarProfile.display_name
stmt = ( stmt = (
@ -53,6 +55,7 @@ def publications_query(
Publication.doi, Publication.doi,
Publication.pdf_url, Publication.pdf_url,
ScholarPublication.is_read, ScholarPublication.is_read,
ScholarPublication.is_favorite,
ScholarPublication.first_seen_run_id, ScholarPublication.first_seen_run_id,
ScholarPublication.created_at, ScholarPublication.created_at,
) )
@ -60,10 +63,13 @@ def publications_query(
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
.where(ScholarProfile.user_id == user_id) .where(ScholarProfile.user_id == user_id)
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) .order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
.offset(max(int(offset), 0))
.limit(limit) .limit(limit)
) )
if scholar_profile_id is not None: if scholar_profile_id is not None:
stmt = stmt.where(ScholarProfile.id == scholar_profile_id) stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
if favorite_only:
stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
if mode == MODE_UNREAD: if mode == MODE_UNREAD:
stmt = stmt.where(ScholarPublication.is_read.is_(False)) stmt = stmt.where(ScholarPublication.is_read.is_(False))
if mode == MODE_LATEST: if mode == MODE_LATEST:
@ -93,6 +99,7 @@ def publication_query_for_user(
Publication.doi, Publication.doi,
Publication.pdf_url, Publication.pdf_url,
ScholarPublication.is_read, ScholarPublication.is_read,
ScholarPublication.is_favorite,
ScholarPublication.first_seen_run_id, ScholarPublication.first_seen_run_id,
ScholarPublication.created_at, ScholarPublication.created_at,
) )
@ -146,6 +153,7 @@ def publication_list_item_from_row(
doi, doi,
pdf_url, pdf_url,
is_read, is_read,
is_favorite,
first_seen_run_id, first_seen_run_id,
created_at, created_at,
) = row ) = row
@ -161,6 +169,7 @@ def publication_list_item_from_row(
doi=doi, doi=doi,
pdf_url=pdf_url, pdf_url=pdf_url,
is_read=bool(is_read), is_read=bool(is_read),
is_favorite=bool(is_favorite),
first_seen_at=created_at, first_seen_at=created_at,
is_new_in_latest_run=( is_new_in_latest_run=(
latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
@ -182,6 +191,7 @@ def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
doi, doi,
pdf_url, pdf_url,
_is_read, _is_read,
_is_favorite,
_first_seen_run_id, _first_seen_run_id,
_created_at, _created_at,
) = row ) = row

View file

@ -16,16 +16,20 @@ def _normalized_selection_pairs(selections: list[tuple[int, int]]) -> set[tuple[
return pairs return pairs
def _scoped_scholar_ids_query(*, user_id: int):
return (
select(ScholarProfile.id)
.where(ScholarProfile.user_id == user_id)
.scalar_subquery()
)
async def mark_all_unread_as_read_for_user( async def mark_all_unread_as_read_for_user(
db_session: AsyncSession, db_session: AsyncSession,
*, *,
user_id: int, user_id: int,
) -> int: ) -> int:
scholar_ids = ( scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
select(ScholarProfile.id)
.where(ScholarProfile.user_id == user_id)
.scalar_subquery()
)
stmt = ( stmt = (
update(ScholarPublication) update(ScholarPublication)
.where( .where(
@ -49,11 +53,7 @@ async def mark_selected_as_read_for_user(
if not normalized_pairs: if not normalized_pairs:
return 0 return 0
scholar_ids = ( scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
select(ScholarProfile.id)
.where(ScholarProfile.user_id == user_id)
.scalar_subquery()
)
stmt = ( stmt = (
update(ScholarPublication) update(ScholarPublication)
.where( .where(
@ -69,3 +69,26 @@ async def mark_selected_as_read_for_user(
result = await db_session.execute(stmt) result = await db_session.execute(stmt)
await db_session.commit() await db_session.commit()
return int(result.rowcount or 0) return int(result.rowcount or 0)
async def set_publication_favorite_for_user(
db_session: AsyncSession,
*,
user_id: int,
scholar_profile_id: int,
publication_id: int,
is_favorite: bool,
) -> int:
scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
stmt = (
update(ScholarPublication)
.where(
ScholarPublication.scholar_profile_id.in_(scholar_ids),
ScholarPublication.scholar_profile_id == int(scholar_profile_id),
ScholarPublication.publication_id == int(publication_id),
)
.values(is_favorite=bool(is_favorite))
)
result = await db_session.execute(stmt)
await db_session.commit()
return int(result.rowcount or 0)

View file

@ -19,6 +19,11 @@ class PublicationListItem:
is_read: bool is_read: bool
first_seen_at: datetime first_seen_at: datetime
is_new_in_latest_run: bool is_new_in_latest_run: bool
is_favorite: bool = False
pdf_status: str = "untracked"
pdf_attempt_count: int = 0
pdf_failure_reason: str | None = None
pdf_failure_detail: str | None = None
@dataclass(frozen=True) @dataclass(frozen=True)

View file

@ -1,5 +1,10 @@
from __future__ import annotations from __future__ import annotations
from app.services.domains.unpaywall.application import resolve_publication_pdf_urls
async def resolve_publication_pdf_urls(*args, **kwargs):
from app.services.domains.unpaywall.application import resolve_publication_pdf_urls as _impl
return await _impl(*args, **kwargs)
__all__ = ["resolve_publication_pdf_urls"] __all__ = ["resolve_publication_pdf_urls"]

View file

@ -1,21 +1,44 @@
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass
import logging import logging
import re import re
from typing import TYPE_CHECKING
from urllib.parse import unquote from urllib.parse import unquote
from app.services.domains.crossref.application import discover_doi_for_publication from app.services.domains.crossref.application import discover_doi_for_publication
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
from app.services.domains.doi.normalize import normalize_doi from app.services.domains.doi.normalize import normalize_doi
from app.services.domains.unpaywall.pdf_discovery import (
looks_like_pdf_url,
resolve_pdf_from_landing_page,
)
from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
from app.settings import settings from app.settings import settings
if TYPE_CHECKING:
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I) DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I) DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
DOI_URL_RE = re.compile(r"(?:https?://)?(?:dx\.)?doi\.org/(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I) DOI_URL_RE = re.compile(r"(?:https?://)?(?:dx\.)?doi\.org/(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
UNPAYWALL_URL_TEMPLATE = "https://api.unpaywall.org/v2/{doi}" UNPAYWALL_URL_TEMPLATE = "https://api.unpaywall.org/v2/{doi}"
FAILURE_MISSING_DOI = "missing_doi"
FAILURE_NO_RECORD = "no_unpaywall_record"
FAILURE_NO_PDF = "no_pdf_found"
FAILURE_RESOLUTION_EXCEPTION = "resolution_exception"
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class OaResolutionOutcome:
publication_id: int
doi: str | None
pdf_url: str | None
failure_reason: str | None
source: str | None
used_crossref: bool
def _extract_doi_candidate(text: str | None) -> str | None: def _extract_doi_candidate(text: str | None) -> str | None:
if not text: if not text:
return None return None
@ -57,21 +80,89 @@ def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str |
) )
def _payload_pdf_url(payload: dict) -> str | None: def _payload_locations(payload: dict) -> list[dict]:
locations: list[dict] = []
best = payload.get("best_oa_location") best = payload.get("best_oa_location")
if isinstance(best, dict): if isinstance(best, dict):
pdf_url = best.get("url_for_pdf") locations.append(best)
if isinstance(pdf_url, str) and pdf_url.strip(): oa_locations = payload.get("oa_locations")
return pdf_url.strip() if isinstance(oa_locations, list):
locations = payload.get("oa_locations") locations.extend(location for location in oa_locations if isinstance(location, dict))
if not isinstance(locations, list): return locations
def _location_value(location: dict, key: str) -> str | None:
value = location.get(key)
if not isinstance(value, str):
return None return None
for location in locations: normalized = value.strip()
if not isinstance(location, dict): return normalized or None
def _payload_pdf_candidates(payload: dict) -> list[str]:
candidates: list[str] = []
seen: set[str] = set()
for location in _payload_locations(payload):
candidate = _location_value(location, "url_for_pdf")
if candidate is None or candidate in seen:
continue continue
pdf_url = location.get("url_for_pdf") seen.add(candidate)
if isinstance(pdf_url, str) and pdf_url.strip(): candidates.append(candidate)
return pdf_url.strip() return candidates
def _payload_landing_candidates(payload: dict) -> list[str]:
candidates: list[str] = []
seen: set[str] = set()
for location in _payload_locations(payload):
candidate = _location_value(location, "url")
if candidate is None or candidate in seen:
continue
seen.add(candidate)
candidates.append(candidate)
return candidates
def _crawl_targets(
*,
payload: dict,
pdf_candidates: list[str],
) -> list[str]:
targets = _payload_landing_candidates(payload)
seen = set(targets)
for candidate in pdf_candidates:
if candidate in seen:
continue
targets.append(candidate)
seen.add(candidate)
doi = normalize_doi(str(payload.get("doi") or ""))
doi_landing_url = f"https://doi.org/{doi}" if doi else None
if doi_landing_url and doi_landing_url not in seen:
targets.append(doi_landing_url)
return targets
def _has_direct_payload_pdf(payload: dict) -> bool:
return any(looks_like_pdf_url(candidate) for candidate in _payload_pdf_candidates(payload))
async def _resolved_pdf_url_from_payload(
payload: dict,
*,
client,
) -> str | None:
pdf_candidates = _payload_pdf_candidates(payload)
for candidate in pdf_candidates:
if looks_like_pdf_url(candidate):
return candidate
for page_url in _crawl_targets(payload=payload, pdf_candidates=pdf_candidates)[:3]:
discovered = await resolve_pdf_from_landing_page(client, page_url=page_url)
if discovered:
logger.info(
"unpaywall.pdf_discovered_from_landing",
extra={"event": "unpaywall.pdf_discovered_from_landing", "landing_url": page_url},
)
return discovered
return None return None
@ -81,6 +172,7 @@ async def _fetch_unpaywall_payload_by_doi(
doi: str, doi: str,
email: str, email: str,
) -> dict | None: ) -> dict | None:
await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
response = await client.get( response = await client.get(
UNPAYWALL_URL_TEMPLATE.format(doi=doi), UNPAYWALL_URL_TEMPLATE.format(doi=doi),
params={"email": email}, params={"email": email},
@ -130,7 +222,7 @@ async def _resolve_item_payload(
payload: dict | None = None payload: dict | None = None
if doi: if doi:
payload = await _fetch_unpaywall_payload_by_doi(client=client, doi=doi, email=email) payload = await _fetch_unpaywall_payload_by_doi(client=client, doi=doi, email=email)
if payload is not None and _payload_pdf_url(payload): if payload is not None and _has_direct_payload_pdf(payload):
return payload, False return payload, False
if not allow_crossref or not settings.crossref_enabled: if not allow_crossref or not settings.crossref_enabled:
return payload, False return payload, False
@ -151,9 +243,13 @@ async def _resolve_item_payload(
return payload, True return payload, True
def _doi_and_pdf_from_payload(payload: dict) -> tuple[str | None, str | None]: async def _doi_and_pdf_from_payload(
payload: dict,
*,
client,
) -> tuple[str | None, str | None]:
doi = normalize_doi(str(payload.get("doi") or "")) doi = normalize_doi(str(payload.get("doi") or ""))
return doi, _payload_pdf_url(payload) return doi, await _resolved_pdf_url_from_payload(payload, client=client)
def _resolution_targets(items: list[PublicationListItem]) -> list[PublicationListItem]: def _resolution_targets(items: list[PublicationListItem]) -> list[PublicationListItem]:
@ -164,11 +260,165 @@ def _crossref_budget_value() -> int:
return max(int(settings.crossref_max_lookups_per_request), 0) return max(int(settings.crossref_max_lookups_per_request), 0)
def _outcome_with_failure(
*,
item: PublicationListItem,
failure_reason: str,
used_crossref: bool,
) -> OaResolutionOutcome:
return OaResolutionOutcome(
publication_id=item.publication_id,
doi=_publication_doi(item),
pdf_url=None,
failure_reason=failure_reason,
source=None,
used_crossref=used_crossref,
)
def _missing_payload_failure_reason(item: PublicationListItem, *, used_crossref: bool) -> str:
if _publication_doi(item):
return FAILURE_NO_RECORD
if used_crossref:
return FAILURE_NO_RECORD
return FAILURE_MISSING_DOI
def _source_name(*, used_crossref: bool) -> str:
return "crossref+unpaywall" if used_crossref else "unpaywall"
def _outcome_from_payload(
*,
item: PublicationListItem,
doi: str | None,
pdf_url: str | None,
used_crossref: bool,
) -> OaResolutionOutcome:
return OaResolutionOutcome(
publication_id=item.publication_id,
doi=doi,
pdf_url=pdf_url,
failure_reason=None if pdf_url else FAILURE_NO_PDF,
source=_source_name(used_crossref=used_crossref),
used_crossref=used_crossref,
)
def _resolved_pdf_count(outcomes: dict[int, OaResolutionOutcome]) -> int:
return sum(1 for outcome in outcomes.values() if outcome.pdf_url)
def _outcome_metadata(outcomes: dict[int, OaResolutionOutcome]) -> dict[int, tuple[str | None, str | None]]:
return {
publication_id: (outcome.doi, outcome.pdf_url)
for publication_id, outcome in outcomes.items()
}
async def _resolve_outcome_for_item(
*,
client,
item: PublicationListItem,
email: str,
allow_crossref: bool,
) -> OaResolutionOutcome:
payload, used_crossref = await _resolve_item_payload(
client=client,
item=item,
email=email,
allow_crossref=allow_crossref,
)
if not isinstance(payload, dict):
return _outcome_with_failure(
item=item,
failure_reason=_missing_payload_failure_reason(item, used_crossref=used_crossref),
used_crossref=used_crossref,
)
doi, pdf_url = await _doi_and_pdf_from_payload(payload, client=client)
return _outcome_from_payload(
item=item,
doi=doi,
pdf_url=pdf_url,
used_crossref=used_crossref,
)
def _doi_input_count(items: list[PublicationListItem]) -> int:
return len([item for item in items if _publication_doi(item)])
def _search_attempt_count(*, targets: list[PublicationListItem]) -> int:
return len([item for item in targets if not _publication_doi(item)])
async def _safe_outcome_for_item(
*,
client,
item: PublicationListItem,
email: str,
allow_crossref: bool,
) -> OaResolutionOutcome:
try:
return await _resolve_outcome_for_item(
client=client,
item=item,
email=email,
allow_crossref=allow_crossref,
)
except Exception as exc: # pragma: no cover - defensive network boundary
logger.warning(
"unpaywall.resolve_item_failed",
extra={
"event": "unpaywall.resolve_item_failed",
"publication_id": item.publication_id,
"error": str(exc),
},
)
return _outcome_with_failure(
item=item,
failure_reason=FAILURE_RESOLUTION_EXCEPTION,
used_crossref=allow_crossref and settings.crossref_enabled,
)
async def _resolve_outcomes_with_client(
*,
client,
targets: list[PublicationListItem],
email: str,
) -> dict[int, OaResolutionOutcome]:
outcomes: dict[int, OaResolutionOutcome] = {}
crossref_budget = _crossref_budget_value()
crossref_lookups = 0
for item in targets:
allow_crossref = crossref_budget > 0 and crossref_lookups < crossref_budget
outcome = await _safe_outcome_for_item(
client=client,
item=item,
email=email,
allow_crossref=allow_crossref,
)
if outcome.used_crossref:
crossref_lookups += 1
outcomes[item.publication_id] = outcome
return outcomes
async def resolve_publication_oa_metadata( async def resolve_publication_oa_metadata(
items: list[PublicationListItem], items: list[PublicationListItem],
*, *,
request_email: str | None = None, request_email: str | None = None,
) -> dict[int, tuple[str | None, str | None]]: ) -> dict[int, tuple[str | None, str | None]]:
outcomes = await resolve_publication_oa_outcomes(items, request_email=request_email)
return _outcome_metadata(outcomes)
async def resolve_publication_oa_outcomes(
items: list[PublicationListItem],
*,
request_email: str | None = None,
) -> dict[int, OaResolutionOutcome]:
if not settings.unpaywall_enabled: if not settings.unpaywall_enabled:
return {} return {}
email = _email_for_request(request_email) email = _email_for_request(request_email)
@ -178,35 +428,21 @@ async def resolve_publication_oa_metadata(
import httpx import httpx
timeout_seconds = max(float(settings.unpaywall_timeout_seconds), 0.5) timeout_seconds = max(float(settings.unpaywall_timeout_seconds), 0.5)
resolved: dict[int, tuple[str | None, str | None]] = {}
crossref_budget = _crossref_budget_value()
crossref_lookups = 0
targets = _resolution_targets(items)[: max(int(settings.unpaywall_max_items_per_request), 0)] targets = _resolution_targets(items)[: max(int(settings.unpaywall_max_items_per_request), 0)]
async with httpx.AsyncClient(timeout=timeout_seconds) as client: async with httpx.AsyncClient(timeout=timeout_seconds) as client:
for item in targets: outcomes = await _resolve_outcomes_with_client(
allow_crossref = crossref_budget > 0 and crossref_lookups < crossref_budget
payload, used_crossref = await _resolve_item_payload(
client=client, client=client,
item=item, targets=targets,
email=email, email=email,
allow_crossref=allow_crossref,
) )
if used_crossref:
crossref_lookups += 1
if not isinstance(payload, dict):
continue
resolved[item.publication_id] = _doi_and_pdf_from_payload(payload)
resolved_count = sum(1 for _doi, pdf in resolved.values() if pdf)
doi_input_count = len([item for item in items if _publication_doi(item)])
target_doi_count = len([item for item in targets if _publication_doi(item)])
_log_resolution_summary( _log_resolution_summary(
publication_count=len(items), publication_count=len(items),
doi_input_count=doi_input_count, doi_input_count=_doi_input_count(items),
search_attempt_count=max(0, len(targets) - target_doi_count), search_attempt_count=_search_attempt_count(targets=targets),
resolved_pdf_count=resolved_count, resolved_pdf_count=_resolved_pdf_count(outcomes),
email=email, email=email,
) )
return resolved return outcomes
async def resolve_publication_pdf_urls( async def resolve_publication_pdf_urls(

View file

@ -0,0 +1,156 @@
from __future__ import annotations
from html.parser import HTMLParser
import re
from urllib.parse import urljoin, urlparse
from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
from app.settings import settings
PDF_MIME = "application/pdf"
URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.I)
class _LandingPdfParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.base_href: str | None = None
self.candidates: list[str] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
attrs_map = {key.lower(): (value or "").strip() for key, value in attrs}
if tag == "base":
self.base_href = attrs_map.get("href") or self.base_href
return
if tag == "meta":
self._append_meta_candidate(attrs_map)
return
if tag == "link":
self._append_link_candidate(attrs_map)
return
if tag == "a":
self._append_anchor_candidate(attrs_map)
def _append_meta_candidate(self, attrs_map: dict[str, str]) -> None:
meta_name = (attrs_map.get("name") or attrs_map.get("property") or "").lower()
if meta_name != "citation_pdf_url":
return
content = attrs_map.get("content")
if content:
self.candidates.append(content)
def _append_link_candidate(self, attrs_map: dict[str, str]) -> None:
href = attrs_map.get("href")
link_type = (attrs_map.get("type") or "").lower()
if href and "pdf" in link_type:
self.candidates.append(href)
def _append_anchor_candidate(self, attrs_map: dict[str, str]) -> None:
href = attrs_map.get("href")
if href:
self.candidates.append(href)
def looks_like_pdf_url(url: str | None) -> bool:
if not isinstance(url, str):
return False
value = url.strip()
if not value:
return False
parsed = urlparse(value)
path = (parsed.path or "").lower()
query = (parsed.query or "").lower()
return path.endswith(".pdf") or ".pdf" in query
def _normalized_candidate_urls(*, page_url: str, html: str) -> list[str]:
parser = _LandingPdfParser()
parser.feed(html)
parser.close()
base_url = urljoin(page_url, parser.base_href) if parser.base_href else page_url
raw_candidates = [*parser.candidates, *_text_url_candidates(html)]
deduped: list[str] = []
seen: set[str] = set()
for raw in raw_candidates:
absolute = urljoin(base_url, raw.strip())
parsed = urlparse(absolute)
if parsed.scheme not in {"http", "https"}:
continue
if absolute in seen:
continue
seen.add(absolute)
deduped.append(absolute)
return sorted(deduped, key=_candidate_sort_key)
def _text_url_candidates(html: str) -> list[str]:
candidates: list[str] = []
for match in URL_RE.findall(html):
cleaned = match.rstrip(".,);]}>")
if "pdf" not in cleaned.lower():
continue
candidates.append(cleaned)
return candidates
def _candidate_sort_key(candidate: str) -> tuple[int, str]:
lowered = candidate.lower()
if looks_like_pdf_url(candidate):
return (0, lowered)
if any(token in lowered for token in ("doi", "full", "article", "download")):
return (1, lowered)
return (2, lowered)
def _is_html_response(response) -> bool:
content_type = str(response.headers.get("content-type") or "").lower()
return "text/html" in content_type or "application/xhtml+xml" in content_type
async def _fetch_page_html(client, *, page_url: str) -> str | None:
await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
response = await client.get(page_url, follow_redirects=True)
if response.status_code != 200 or not _is_html_response(response):
return None
text = response.text or ""
return text[: max(int(settings.unpaywall_pdf_discovery_max_html_bytes), 0)]
async def _candidate_is_pdf(client, *, candidate_url: str) -> bool:
if looks_like_pdf_url(candidate_url):
return True
await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
response = await client.get(candidate_url, follow_redirects=True)
content_type = str(response.headers.get("content-type") or "").lower()
return response.status_code == 200 and PDF_MIME in content_type
def _candidate_limit() -> int:
return max(int(settings.unpaywall_pdf_discovery_max_candidates), 1)
async def _resolve_pdf_from_candidate_page(client, *, candidate_url: str) -> str | None:
html = await _fetch_page_html(client, page_url=candidate_url)
if not html:
return None
nested_candidates = _normalized_candidate_urls(page_url=candidate_url, html=html)
for nested in nested_candidates[: _candidate_limit()]:
if await _candidate_is_pdf(client, candidate_url=nested):
return nested
return None
async def resolve_pdf_from_landing_page(client, *, page_url: str) -> str | None:
if not settings.unpaywall_pdf_discovery_enabled:
return None
html = await _fetch_page_html(client, page_url=page_url)
if not html:
return None
candidates = _normalized_candidate_urls(page_url=page_url, html=html)
for candidate in candidates[: _candidate_limit()]:
if await _candidate_is_pdf(client, candidate_url=candidate):
return candidate
nested_pdf = await _resolve_pdf_from_candidate_page(client, candidate_url=candidate)
if nested_pdf:
return nested_pdf
return None

View file

@ -0,0 +1,18 @@
from __future__ import annotations
import asyncio
import time
_REQUEST_LOCK = asyncio.Lock()
_LAST_REQUEST_AT = 0.0
async def wait_for_unpaywall_slot(*, min_interval_seconds: float) -> None:
global _LAST_REQUEST_AT
interval = max(float(min_interval_seconds), 0.0)
async with _REQUEST_LOCK:
elapsed = time.monotonic() - _LAST_REQUEST_AT
remaining = interval - elapsed
if remaining > 0:
await asyncio.sleep(remaining)
_LAST_REQUEST_AT = time.monotonic()

View file

@ -232,8 +232,21 @@ class Settings:
unpaywall_enabled: bool = _env_bool("UNPAYWALL_ENABLED", True) unpaywall_enabled: bool = _env_bool("UNPAYWALL_ENABLED", True)
unpaywall_email: str = _env_str("UNPAYWALL_EMAIL", "") unpaywall_email: str = _env_str("UNPAYWALL_EMAIL", "")
unpaywall_timeout_seconds: float = _env_float("UNPAYWALL_TIMEOUT_SECONDS", 4.0) unpaywall_timeout_seconds: float = _env_float("UNPAYWALL_TIMEOUT_SECONDS", 4.0)
unpaywall_min_interval_seconds: float = _env_float("UNPAYWALL_MIN_INTERVAL_SECONDS", 0.6)
unpaywall_max_items_per_request: int = _env_int("UNPAYWALL_MAX_ITEMS_PER_REQUEST", 20) unpaywall_max_items_per_request: int = _env_int("UNPAYWALL_MAX_ITEMS_PER_REQUEST", 20)
unpaywall_retry_cooldown_seconds: int = _env_int("UNPAYWALL_RETRY_COOLDOWN_SECONDS", 1800) unpaywall_retry_cooldown_seconds: int = _env_int("UNPAYWALL_RETRY_COOLDOWN_SECONDS", 1800)
pdf_auto_retry_interval_seconds: int = _env_int(
"PDF_AUTO_RETRY_INTERVAL_SECONDS",
86_400,
)
pdf_auto_retry_first_interval_seconds: int = _env_int(
"PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS",
3_600,
)
pdf_auto_retry_max_attempts: int = _env_int("PDF_AUTO_RETRY_MAX_ATTEMPTS", 3)
unpaywall_pdf_discovery_enabled: bool = _env_bool("UNPAYWALL_PDF_DISCOVERY_ENABLED", True)
unpaywall_pdf_discovery_max_candidates: int = _env_int("UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES", 5)
unpaywall_pdf_discovery_max_html_bytes: int = _env_int("UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES", 500_000)
crossref_enabled: bool = _env_bool("CROSSREF_ENABLED", True) crossref_enabled: bool = _env_bool("CROSSREF_ENABLED", True)
crossref_max_rows: int = _env_int("CROSSREF_MAX_ROWS", 10) crossref_max_rows: int = _env_int("CROSSREF_MAX_ROWS", 10)
crossref_timeout_seconds: float = _env_float("CROSSREF_TIMEOUT_SECONDS", 8.0) crossref_timeout_seconds: float = _env_float("CROSSREF_TIMEOUT_SECONDS", 8.0)

30
docs/README.md Normal file
View file

@ -0,0 +1,30 @@
# Documentation Index
Use this directory for project documentation outside `README.md` and `agents.md`.
## Deploy
- `docs/deploy/quickstart.md`: deployment modes, local dev startup, and quality command reference.
## Reference
- `docs/reference/environment.md`: complete environment-variable catalog and contract/audit notes.
- `docs/reference/api_contract.md`: API envelope contract and endpoint semantics.
## Architecture
- `docs/architecture/domain_boundaries.md`: domain boundaries, user-scoped data rules, and frontend behavior notes.
## Operations
- `docs/ops/scrape_safety_runbook.md`: scrape cooldown and blocked-IP operations.
- `docs/ops/db_runbook.md`: backup, restore, and verification workflows.
- `docs/ops/migration_checklist.md`: migration planning and rollout checklist.
## Frontend
- `docs/frontend/theme_phase0_inventory.md`: theme-token inventory and adoption notes.
## Contribution
- `docs/contributing.md`: contribution policy and merge checklist.

View file

@ -0,0 +1,25 @@
# Domain Boundaries
## Data Model Rules
- Scholar tracking is user-scoped.
- Publications are global/deduplicated records.
- Read/favorite/visibility state stays on scholar-publication link rows.
## Service Boundaries
Canonical business logic belongs in `app/services/domains/*`.
- `app/services/domains/ingestion/*`: run orchestration, continuation queue, scheduler, and scrape safety.
- `app/services/domains/scholar/*`: fail-fast scholar parsing and source fetch adapters.
- `app/services/domains/scholars/*`: scholar CRUD, profile image, and name-search controls.
- `app/services/domains/publications/*`: listing/read-state, favorite toggles, enrichment scheduling, and retry paths.
- `app/services/domains/crossref/*` + `app/services/domains/unpaywall/*`: DOI/OA enrichment with bounded pacing.
- `app/services/domains/runs/*`: run history and continuation queue operations.
- `app/services/domains/portability/*`: import/export workflows.
## Frontend Behavior Notes
- Mobile primary nav is in a left drawer and closes on route change or logout.
- Long list views use internal scroll containers.
- Name search remains intentionally constrained due upstream anti-bot behavior; production onboarding should prefer scholar ID/profile URL.

68
docs/deploy/quickstart.md Normal file
View file

@ -0,0 +1,68 @@
# Deploy and Dev Quickstart
## Required Setup
1. Copy `.env.example` to `.env`.
2. Set `POSTGRES_PASSWORD`.
3. Set `SESSION_SECRET_KEY`.
## Deploy Method A: Prebuilt Image
```bash
docker compose pull
docker compose up -d
```
Open:
- App/API: `http://localhost:8000`
- Health endpoint: `http://localhost:8000/healthz`
Upgrade:
```bash
docker compose pull
docker compose up -d
```
## Deploy Method B: Local Source + Dev Frontend
```bash
docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
```
Open:
- API: `http://localhost:8000`
- Frontend dev server: `http://localhost:5173`
Stop:
```bash
docker compose -f docker-compose.yml -f docker-compose.dev.yml down
```
## Quality Commands
Backend:
```bash
docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest tests/unit
docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest -m integration
```
Frontend:
```bash
cd frontend
npm install
npm run typecheck
npm run test:run
npm run build
```
Repository checks:
```bash
python3 scripts/check_frontend_api_contract.py
python3 scripts/check_env_contract.py
./scripts/check_no_generated_artifacts.sh
```

12
docs/index.md Normal file
View file

@ -0,0 +1,12 @@
---
slug: /
---
# scholarr Documentation
- Start here: `docs/deploy/quickstart.md`
- Runtime config: `docs/reference/environment.md`
- API contract: `docs/reference/api_contract.md`
- Domain boundaries: `docs/architecture/domain_boundaries.md`
- Ops runbooks: `docs/ops/`
- Frontend theme inventory: `docs/frontend/theme_phase0_inventory.md`

100
docs/ops/db_runbook.md Normal file
View file

@ -0,0 +1,100 @@
# Database Operations Runbook
This runbook defines backup, restore, and verification procedures for the
PostgreSQL database used by `scholarr`.
## Objectives
- Keep data loss bounded via scheduled backups.
- Provide repeatable restore procedures.
- Verify backups by running regular restore drills.
Recommended starting targets:
- `RPO`: 24 hours (maximum acceptable data loss).
- `RTO`: 60 minutes (maximum acceptable restore time).
## Backup Strategy
Use logical backups from the running `db` compose service.
- Custom-format backup (recommended for restore flexibility):
```bash
scripts/db/backup_full.sh
```
- Plain SQL backup:
```bash
scripts/db/backup_full.sh --plain
```
Optional environment variables:
- `BACKUP_DIR`: destination directory (default: `<repo>/backups`)
- `BACKUP_PREFIX`: backup filename prefix (default: `scholarr`)
- `USE_DEV_COMPOSE=1`: include `docker-compose.dev.yml`
## Restore Strategy
Restore from a `.dump` (custom format) or `.sql` file into the running `db`
compose service.
- Restore without schema wipe:
```bash
scripts/db/restore_dump.sh --file backups/scholarr_20260220T120000Z.dump
```
- Restore with full `public` schema reset:
```bash
scripts/db/restore_dump.sh --file backups/scholarr_20260220T120000Z.dump --wipe-public
```
## Safety Checklist Before Restore
1. Pause writes (stop app/scheduler or set maintenance mode).
2. Create a fresh pre-restore backup.
3. Validate target dump checksum/size.
4. Confirm operator, scope, and rollback plan.
## Post-Restore Verification
1. Run migrations head check.
2. Run health endpoint checks.
3. Verify table row counts for core tables:
- `users`
- `scholar_profiles`
- `publications`
- `scholar_publications`
- `crawl_runs`
4. Run integrity report and require zero failures:
```bash
python3 scripts/db/check_integrity.py
```
5. Run API smoke checks for scholars/publications/runs pages.
## Data Cleanup and Repair
Use the audited repair job for scholar-publication relinking cleanup:
```bash
python3 scripts/db/repair_publication_links.py --user-id <id> --requested-by "<operator>" --apply
```
Dry-run first, then re-run with `--apply` once scope and summary counts match expectation.
If cleanup changes schema assumptions, follow `docs/ops/migration_checklist.md` before any migration rollout.
## Restore Drill Cadence
- Run at least one full restore drill per month.
- Record:
- backup artifact used,
- restore start/end timestamps,
- issues encountered,
- achieved `RTO`.

View file

@ -0,0 +1,48 @@
# Migration Checklist
Use this checklist for every schema/data migration.
## 1. Design Review
- Define change type: `expand`, `backfill`, `contract`.
- Document expected lock behavior and index impact.
- Confirm backward compatibility with currently deployed app version.
- Define rollback strategy before implementation.
## 2. Pre-Migration Controls
- Capture a fresh backup before applying migration.
- Confirm migration head revision and working tree cleanliness.
- Prepare validation queries for new/changed tables and indexes.
- Identify high-risk tables (large row count, hot write paths).
## 3. Implementation Standards
- Keep migrations idempotent when feasible.
- Prefer additive steps first (`nullable`, new index, new table).
- For destructive changes, separate into later contract migration.
- Avoid large blocking rewrites in a single deployment step.
## 4. Verification
- Apply migration in staging against production-like snapshot.
- Verify:
- expected tables/columns/indexes,
- app startup and health endpoint,
- affected API flows,
- data consistency queries.
## 5. Rollout and Recovery
- Apply migration during planned window.
- Monitor logs/errors and DB metrics during rollout.
- If rollback needed:
- follow downgrade/recovery runbook,
- restore from backup if downgrade is unsafe,
- document incident timeline.
## 6. Post-Migration Tasks
- Update `README.md` / `.env.example` / ops docs.
- Add/update integration tests for new schema assumptions.
- Record migration notes in changelog.

View file

@ -0,0 +1,29 @@
# API Contract
## Envelope Invariant
All API responses under `/api/v1` use one of these envelopes:
- Success: `{"data": ..., "meta": {"request_id": "..."}}`
- Error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}`
`meta.request_id` must be present for success and error responses.
## Publications Semantics
- `GET /api/v1/publications` supports `mode=all|unread|latest`.
- `mode=new` is currently accepted as a compatibility alias for `latest`.
- Pagination controls: `page`, `page_size` (with backward-compatible `limit`/`offset` support).
- Pagination fields in response: `page`, `page_size`, `has_prev`, `has_next`, `total_count`.
- `unread` represents read-state (`is_read=false`).
- `latest` represents discovery-state (`first seen in the latest completed run`).
Publication payloads expose:
- `pub_url` (canonical scholar detail URL)
- `doi` (normalized DOI)
- `pdf_url` (resolved OA PDF when available)
## Scholar Portability
- `GET /api/v1/scholars/export` exports tracked scholars and scholar-publication link state.
- `POST /api/v1/scholars/import` imports that payload while preserving global publication deduplication.

View file

@ -0,0 +1,68 @@
# Environment Reference and Audit
This document is the source-of-truth audit for what remains configurable versus internal defaults.
## Decision Summary
- Keep in `.env.example`: all runtime and compose controls currently exposed by `app/settings.py` plus compose/bootstrap extras.
- Move to internal defaults only: none at this time.
- Internal fallback behavior still exists in code to keep local/test startup resilient when values are omitted.
## Configurable Variables (By Section)
### Compose + Database
`POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `DATABASE_URL`, `TEST_DATABASE_URL`, `SCHOLARR_IMAGE`
### App Runtime + Networking
`APP_NAME`, `APP_HOST`, `APP_PORT`, `APP_HOST_PORT`, `APP_RELOAD`, `MIGRATE_ON_START`, `FRONTEND_ENABLED`, `FRONTEND_DIST_DIR`
### Database Pool
`DATABASE_POOL_MODE`, `DATABASE_POOL_SIZE`, `DATABASE_POOL_MAX_OVERFLOW`, `DATABASE_POOL_TIMEOUT_SECONDS`
### Frontend Dev Overrides
`FRONTEND_HOST_PORT`, `CHOKIDAR_USEPOLLING`, `VITE_DEV_API_PROXY_TARGET`
### Auth + Session
`SESSION_SECRET_KEY`, `SESSION_COOKIE_SECURE`, `LOGIN_RATE_LIMIT_ATTEMPTS`, `LOGIN_RATE_LIMIT_WINDOW_SECONDS`
### HTTP Security Headers + CSP
`SECURITY_HEADERS_ENABLED`, `SECURITY_X_CONTENT_TYPE_OPTIONS`, `SECURITY_X_FRAME_OPTIONS`, `SECURITY_REFERRER_POLICY`, `SECURITY_PERMISSIONS_POLICY`, `SECURITY_CROSS_ORIGIN_OPENER_POLICY`, `SECURITY_CROSS_ORIGIN_RESOURCE_POLICY`, `SECURITY_CSP_ENABLED`, `SECURITY_CSP_POLICY`, `SECURITY_CSP_DOCS_POLICY`, `SECURITY_CSP_REPORT_ONLY`, `SECURITY_STRICT_TRANSPORT_SECURITY_ENABLED`, `SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE`, `SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS`, `SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD`
### Logging
`LOG_LEVEL`, `LOG_FORMAT`, `LOG_REQUESTS`, `LOG_UVICORN_ACCESS`, `LOG_REQUEST_SKIP_PATHS`, `LOG_REDACT_FIELDS`
### Scheduler + Ingestion Safety
`SCHEDULER_ENABLED`, `SCHEDULER_TICK_SECONDS`, `SCHEDULER_QUEUE_BATCH_SIZE`, `INGESTION_AUTOMATION_ALLOWED`, `INGESTION_MANUAL_RUN_ALLOWED`, `INGESTION_MIN_RUN_INTERVAL_MINUTES`, `INGESTION_MIN_REQUEST_DELAY_SECONDS`, `INGESTION_NETWORK_ERROR_RETRIES`, `INGESTION_RETRY_BACKOFF_SECONDS`, `INGESTION_MAX_PAGES_PER_SCHOLAR`, `INGESTION_PAGE_SIZE`, `INGESTION_ALERT_BLOCKED_FAILURE_THRESHOLD`, `INGESTION_ALERT_NETWORK_FAILURE_THRESHOLD`, `INGESTION_ALERT_RETRY_SCHEDULED_THRESHOLD`, `INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS`, `INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS`, `INGESTION_CONTINUATION_QUEUE_ENABLED`, `INGESTION_CONTINUATION_BASE_DELAY_SECONDS`, `INGESTION_CONTINUATION_MAX_DELAY_SECONDS`, `INGESTION_CONTINUATION_MAX_ATTEMPTS`
### Scholar Images + Name Search Safety
`SCHOLAR_IMAGE_UPLOAD_DIR`, `SCHOLAR_IMAGE_UPLOAD_MAX_BYTES`, `SCHOLAR_NAME_SEARCH_ENABLED`, `SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS`, `SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS`, `SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES`, `SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS`, `SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS`, `SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD`, `SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS`, `SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD`, `SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD`
### OA Enrichment + PDF Resolution
`UNPAYWALL_ENABLED`, `UNPAYWALL_EMAIL`, `UNPAYWALL_TIMEOUT_SECONDS`, `UNPAYWALL_MIN_INTERVAL_SECONDS`, `UNPAYWALL_MAX_ITEMS_PER_REQUEST`, `UNPAYWALL_RETRY_COOLDOWN_SECONDS`, `UNPAYWALL_PDF_DISCOVERY_ENABLED`, `UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES`, `UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES`, `PDF_AUTO_RETRY_INTERVAL_SECONDS`, `PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS`, `PDF_AUTO_RETRY_MAX_ATTEMPTS`, `CROSSREF_ENABLED`, `CROSSREF_MAX_ROWS`, `CROSSREF_TIMEOUT_SECONDS`, `CROSSREF_MIN_INTERVAL_SECONDS`, `CROSSREF_MAX_LOOKUPS_PER_REQUEST`
### Startup Bootstrap + DB Wait
`BOOTSTRAP_ADMIN_ON_START`, `BOOTSTRAP_ADMIN_EMAIL`, `BOOTSTRAP_ADMIN_PASSWORD`, `BOOTSTRAP_ADMIN_FORCE_PASSWORD`, `DB_WAIT_TIMEOUT_SECONDS`, `DB_WAIT_INTERVAL_SECONDS`
## Drift Guard
CI enforces env parity with:
```bash
python3 scripts/check_env_contract.py
```
The check fails when:
- `app/settings.py` references a variable missing from `.env.example`.
- `.env.example` contains an unknown key (outside the approved compose/bootstrap extras).
- `.env.example` contains duplicate keys.

View file

@ -3,6 +3,12 @@
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="theme-color" content="#495e6f" />
<link rel="icon" type="image/x-icon" href="/favicon.ico" />
<link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png" />
<link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png" />
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png" />
<link rel="manifest" href="/site.webmanifest" />
<title>scholarr</title> <title>scholarr</title>
<script> <script>
(function () { (function () {

View file

@ -8,6 +8,7 @@
"build": "vue-tsc --noEmit && vite build", "build": "vue-tsc --noEmit && vite build",
"preview": "vite preview --host 0.0.0.0 --port 4173", "preview": "vite preview --host 0.0.0.0 --port 4173",
"typecheck": "vue-tsc --noEmit", "typecheck": "vue-tsc --noEmit",
"generate:brand-assets": "./scripts/generate_brand_assets.sh",
"check:theme-tokens": "node ./scripts/check_theme_tokens.mjs", "check:theme-tokens": "node ./scripts/check_theme_tokens.mjs",
"test": "vitest", "test": "vitest",
"test:run": "vitest run" "test:run": "vitest run"

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 40 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 580 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 990 B

BIN
frontend/public/favicon.ico Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

View file

@ -0,0 +1,21 @@
{
"name": "scholarr",
"short_name": "scholarr",
"description": "Track scholar publications and open-access PDF availability.",
"icons": [
{
"src": "/android-chrome-192x192.png",
"sizes": "192x192",
"type": "image/png"
},
{
"src": "/android-chrome-512x512.png",
"sizes": "512x512",
"type": "image/png",
"purpose": "any maskable"
}
],
"theme_color": "#495e6f",
"background_color": "#f2eee9",
"display": "standalone"
}

View file

@ -0,0 +1,56 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
FRONTEND_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
SRC_LOGO="${FRONTEND_DIR}/src/logo.png"
PUBLIC_DIR="${FRONTEND_DIR}/public"
TMP_DIR="$(mktemp -d)"
TMP_ALPHA="${TMP_DIR}/logo-alpha-mask.png"
TMP_GLYPH="${TMP_DIR}/logo-glyph-white.png"
TMP_CANVAS="${TMP_DIR}/logo-canvas.png"
TMP_ICON="${TMP_DIR}/logo-icon.png"
BRAND_COLOR="${1:-#495e6f}"
if ! command -v magick >/dev/null 2>&1; then
echo "ImageMagick is required (missing 'magick' command)." >&2
exit 1
fi
if [[ ! -f "${SRC_LOGO}" ]]; then
echo "Missing source logo at ${SRC_LOGO}" >&2
exit 1
fi
mkdir -p "${PUBLIC_DIR}"
# Build a high-contrast icon: solid brand background + white logo.
magick "${SRC_LOGO}" -alpha extract "${TMP_ALPHA}"
magick -size 1024x1024 xc:white "${TMP_ALPHA}" \
-compose CopyOpacity \
-composite \
"${TMP_GLYPH}"
magick -size 1024x1024 "xc:${BRAND_COLOR}" "${TMP_CANVAS}"
magick "${TMP_GLYPH}" -resize 760x760 "${TMP_GLYPH}"
magick "${TMP_CANVAS}" "${TMP_GLYPH}" \
-gravity center \
-compose over \
-composite \
"${TMP_ICON}"
magick "${TMP_ICON}" -resize 16x16 "${PUBLIC_DIR}/favicon-16x16.png"
magick "${TMP_ICON}" -resize 32x32 "${PUBLIC_DIR}/favicon-32x32.png"
magick "${TMP_ICON}" -resize 180x180 "${PUBLIC_DIR}/apple-touch-icon.png"
magick "${TMP_ICON}" -resize 192x192 "${PUBLIC_DIR}/android-chrome-192x192.png"
magick "${TMP_ICON}" -resize 512x512 "${PUBLIC_DIR}/android-chrome-512x512.png"
magick \
\( "${TMP_ICON}" -resize 16x16 \) \
\( "${TMP_ICON}" -resize 32x32 \) \
\( "${TMP_ICON}" -resize 48x48 \) \
"${PUBLIC_DIR}/favicon.ico"
rm -rf "${TMP_DIR}"
echo "Generated favicon + app icon assets in ${PUBLIC_DIR}"

View file

@ -8,7 +8,6 @@ import PublicationsPage from "@/pages/PublicationsPage.vue";
import RunsPage from "@/pages/RunsPage.vue"; import RunsPage from "@/pages/RunsPage.vue";
import RunDetailPage from "@/pages/RunDetailPage.vue"; import RunDetailPage from "@/pages/RunDetailPage.vue";
import SettingsPage from "@/pages/SettingsPage.vue"; import SettingsPage from "@/pages/SettingsPage.vue";
import AdminUsersPage from "@/pages/AdminUsersPage.vue";
import StyleGuidePage from "@/pages/StyleGuidePage.vue"; import StyleGuidePage from "@/pages/StyleGuidePage.vue";
export const router = createRouter({ export const router = createRouter({
@ -66,12 +65,6 @@ export const router = createRouter({
component: RunDetailPage, component: RunDetailPage,
meta: { requiresAuth: true, requiresAdmin: true }, meta: { requiresAuth: true, requiresAdmin: true },
}, },
{
path: "/admin/users",
name: "admin-users",
component: AdminUsersPage,
meta: { requiresAuth: true, requiresAdmin: true },
},
{ {
path: "/:pathMatch(.*)*", path: "/:pathMatch(.*)*",
redirect: "/dashboard", redirect: "/dashboard",

View file

@ -2,9 +2,8 @@
import { computed } from "vue"; import { computed } from "vue";
import AppButton from "@/components/ui/AppButton.vue"; import AppButton from "@/components/ui/AppButton.vue";
import AppSelect from "@/components/ui/AppSelect.vue"; import AppBrandMark from "@/components/ui/AppBrandMark.vue";
import { useThemeStore } from "@/stores/theme"; import AppThemePicker from "@/components/ui/AppThemePicker.vue";
import type { ThemePresetId } from "@/theme/presets";
const props = withDefaults( const props = withDefaults(
defineProps<{ defineProps<{
@ -20,25 +19,9 @@ const emit = defineEmits<{
(e: "toggle-menu"): void; (e: "toggle-menu"): void;
}>(); }>();
const theme = useThemeStore();
const isDarkTheme = computed(() => theme.active === "dark");
const toggleThemeLabel = computed(() =>
isDarkTheme.value ? "Switch to light theme" : "Switch to dark theme",
);
const menuButtonLabel = computed(() => const menuButtonLabel = computed(() =>
props.menuOpen ? "Close navigation menu" : "Open navigation menu", props.menuOpen ? "Close navigation menu" : "Open navigation menu",
); );
const selectedPreset = computed<ThemePresetId>({
get: () => theme.preset,
set: (value) => theme.setPreset(value),
});
const presetOptions = computed(() => theme.availablePresets);
const themePresetLabel = computed(() => theme.presetLabel);
function onToggleTheme(): void {
theme.setPreference(isDarkTheme.value ? "light" : "dark");
}
function onToggleMenu(): void { function onToggleMenu(): void {
emit("toggle-menu"); emit("toggle-menu");
@ -75,64 +58,14 @@ function onToggleMenu(): void {
</AppButton> </AppButton>
<RouterLink <RouterLink
to="/dashboard" to="/dashboard"
class="rounded-sm font-display text-lg tracking-tight text-ink-primary transition hover:text-ink-secondary focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset" class="inline-flex items-center gap-2.5 rounded-sm font-display text-xl tracking-tight text-ink-primary transition hover:text-ink-secondary focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset"
> >
scholarr <AppBrandMark size="lg" />
<span>scholarr</span>
</RouterLink> </RouterLink>
</div> </div>
<div class="flex items-center justify-end gap-2"> <AppThemePicker id-prefix="header-theme" />
<div class="w-36 sm:w-44">
<label for="theme-preset-select" class="sr-only">Theme preset</label>
<AppSelect
id="theme-preset-select"
v-model="selectedPreset"
:disabled="presetOptions.length <= 1"
:title="`Theme preset: ${themePresetLabel}`"
class="py-1.5"
>
<option v-for="preset in presetOptions" :key="preset.id" :value="preset.id">
{{ preset.label }}
</option>
</AppSelect>
</div>
<AppButton
variant="ghost"
class="h-10 w-10 rounded-full p-0"
:aria-label="toggleThemeLabel"
:title="toggleThemeLabel"
@click="onToggleTheme"
>
<span class="sr-only">{{ toggleThemeLabel }}</span>
<svg
v-if="isDarkTheme"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="1.8"
class="h-5 w-5"
aria-hidden="true"
>
<circle cx="12" cy="12" r="4" />
<path
d="M12 2v2.5M12 19.5V22M4.9 4.9l1.8 1.8M17.3 17.3l1.8 1.8M2 12h2.5M19.5 12H22M4.9 19.1l1.8-1.8M17.3 6.7l1.8-1.8"
/>
</svg>
<svg
v-else
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="1.8"
class="h-5 w-5"
aria-hidden="true"
>
<path d="M21 14.5A8.5 8.5 0 1 1 9.5 3 7 7 0 0 0 21 14.5z" />
</svg>
</AppButton>
</div>
</div> </div>
</header> </header>
</template> </template>

View file

@ -19,22 +19,12 @@ const emit = defineEmits<{
}>(); }>();
const links = computed(() => { const links = computed(() => {
const base = [ return [
{ id: "dashboard", to: "/dashboard", label: "Dashboard", adminOnly: false }, { id: "dashboard", to: "/dashboard", label: "Dashboard" },
{ id: "scholars", to: "/scholars", label: "Scholars", adminOnly: false }, { id: "scholars", to: "/scholars", label: "Scholars" },
{ id: "publications", to: "/publications", label: "Publications", adminOnly: false }, { id: "publications", to: "/publications", label: "Publications" },
{ id: "settings", to: "/settings", label: "Settings", adminOnly: false }, { id: "settings", to: "/settings", label: "Settings" },
{ id: "style-guide", to: "/admin/style-guide", label: "Style Guide", adminOnly: true },
{ id: "runs", to: "/admin/runs", label: "Runs", adminOnly: true },
{ id: "users", to: "/admin/users", label: "Users", adminOnly: true },
]; ];
return base.filter((item) => {
if (item.adminOnly && !auth.isAdmin) {
return false;
}
return userSettings.isPageVisible(item.id);
});
}); });
const navSafetyText = computed(() => { const navSafetyText = computed(() => {
if (!userSettings.manualRunAllowed) { if (!userSettings.manualRunAllowed) {

View file

@ -0,0 +1,55 @@
<script setup lang="ts">
import { computed } from "vue";
import brandLogo from "@/logo.png";
const props = withDefaults(
defineProps<{
size?: "sm" | "md" | "lg" | "xl";
}>(),
{
size: "md",
},
);
const sizeClass = computed(() => {
if (props.size === "sm") {
return "h-5 w-5";
}
if (props.size === "lg") {
return "h-8 w-8";
}
if (props.size === "xl") {
return "h-12 w-12";
}
return "h-6 w-6";
});
const logoMaskStyle: Record<string, string> = {
"--brand-logo-mask": `url(${brandLogo})`,
};
</script>
<template>
<span
:class="['brand-logo-mark', sizeClass]"
:style="logoMaskStyle"
aria-hidden="true"
/>
</template>
<style scoped>
.brand-logo-mark {
display: inline-block;
flex-shrink: 0;
background-color: rgb(var(--color-brand-700));
-webkit-mask-image: var(--brand-logo-mask);
mask-image: var(--brand-logo-mask);
-webkit-mask-position: center;
mask-position: center;
-webkit-mask-repeat: no-repeat;
mask-repeat: no-repeat;
-webkit-mask-size: contain;
mask-size: contain;
}
</style>

View file

@ -0,0 +1,61 @@
<script setup lang="ts">
import { computed } from "vue";
import AppButton from "@/components/ui/AppButton.vue";
const props = withDefaults(
defineProps<{
disabled?: boolean;
loading?: boolean;
title?: string;
loadingTitle?: string;
variant?: "primary" | "secondary" | "ghost" | "danger";
size?: "md" | "sm";
}>(),
{
disabled: false,
loading: false,
title: "Refresh",
loadingTitle: "Refreshing",
variant: "secondary",
size: "md",
},
);
const resolvedTitle = computed(() => (props.loading ? props.loadingTitle : props.title));
const isDisabled = computed(() => props.disabled || props.loading);
const buttonSizeClass = computed(() => {
if (props.size === "sm") {
return "min-h-8 h-8 w-8";
}
return "min-h-10 h-10 w-10";
});
const iconSizeClass = computed(() => (props.size === "sm" ? "h-3.5 w-3.5" : "h-4 w-4"));
</script>
<template>
<AppButton
:variant="props.variant"
:disabled="isDisabled"
class="rounded-full p-0"
:class="buttonSizeClass"
:title="resolvedTitle"
:aria-label="resolvedTitle"
>
<span class="sr-only">{{ resolvedTitle }}</span>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="1.8"
:class="[iconSizeClass, props.loading ? 'animate-spin' : '']"
aria-hidden="true"
>
<path d="M20 4v6h-6" />
<path d="M4 20v-6h6" />
<path d="M6.5 9A7.5 7.5 0 0 1 19 7.5L20 10" />
<path d="M17.5 15A7.5 7.5 0 0 1 5 16.5L4 14" />
</svg>
</AppButton>
</template>

View file

@ -0,0 +1,51 @@
<script setup lang="ts">
export interface AppTabItem {
id: string;
label: string;
disabled?: boolean;
}
const props = defineProps<{
modelValue: string;
items: AppTabItem[];
ariaLabel?: string;
}>();
const emit = defineEmits<{
(e: "update:modelValue", value: string): void;
}>();
function onSelect(tabId: string, disabled: boolean | undefined): void {
if (disabled || tabId === props.modelValue) {
return;
}
emit("update:modelValue", tabId);
}
</script>
<template>
<div
class="flex flex-wrap gap-2 rounded-lg border border-stroke-default bg-surface-card-muted p-2"
role="tablist"
:aria-label="props.ariaLabel || 'Tabs'"
>
<button
v-for="item in props.items"
:key="item.id"
type="button"
role="tab"
:aria-selected="item.id === props.modelValue"
:tabindex="item.id === props.modelValue ? 0 : -1"
:disabled="item.disabled"
class="inline-flex min-h-9 items-center rounded-md border px-3 py-1.5 text-sm font-medium transition focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset disabled:cursor-not-allowed disabled:opacity-60"
:class="
item.id === props.modelValue
? 'border-stroke-interactive bg-surface-nav-active text-ink-primary'
: 'border-stroke-default bg-surface-card text-ink-secondary hover:border-stroke-interactive hover:text-ink-primary'
"
@click="onSelect(item.id, item.disabled)"
>
{{ item.label }}
</button>
</div>
</template>

View file

@ -0,0 +1,95 @@
<script setup lang="ts">
import { computed } from "vue";
import AppButton from "@/components/ui/AppButton.vue";
import AppSelect from "@/components/ui/AppSelect.vue";
import { useThemeStore } from "@/stores/theme";
import type { ThemePresetId } from "@/theme/presets";
const props = withDefaults(
defineProps<{
compact?: boolean;
idPrefix?: string;
}>(),
{
compact: false,
idPrefix: "theme",
},
);
const theme = useThemeStore();
const isDarkTheme = computed(() => theme.active === "dark");
const toggleThemeLabel = computed(() =>
isDarkTheme.value ? "Switch to light theme" : "Switch to dark theme",
);
const selectedPreset = computed<ThemePresetId>({
get: () => theme.preset,
set: (value) => theme.setPreset(value),
});
const presetOptions = computed(() => theme.availablePresets);
const themePresetLabel = computed(() => theme.presetLabel);
const selectId = computed(() => `${props.idPrefix}-preset-select`);
const selectWidthClass = computed(() => (props.compact ? "w-32 sm:w-36" : "w-36 sm:w-44"));
const toggleButtonClass = computed(() =>
props.compact ? "h-9 w-9 rounded-full p-0" : "h-10 w-10 rounded-full p-0",
);
function onToggleTheme(): void {
theme.setPreference(isDarkTheme.value ? "light" : "dark");
}
</script>
<template>
<div class="flex items-center justify-end gap-2">
<div :class="selectWidthClass">
<label :for="selectId" class="sr-only">Theme preset</label>
<AppSelect
:id="selectId"
v-model="selectedPreset"
:disabled="presetOptions.length <= 1"
:title="`Theme preset: ${themePresetLabel}`"
class="py-1.5"
>
<option v-for="preset in presetOptions" :key="preset.id" :value="preset.id">
{{ preset.label }}
</option>
</AppSelect>
</div>
<AppButton
variant="ghost"
:class="toggleButtonClass"
:aria-label="toggleThemeLabel"
:title="toggleThemeLabel"
@click="onToggleTheme"
>
<span class="sr-only">{{ toggleThemeLabel }}</span>
<svg
v-if="isDarkTheme"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="1.8"
class="h-5 w-5"
aria-hidden="true"
>
<circle cx="12" cy="12" r="4" />
<path
d="M12 2v2.5M12 19.5V22M4.9 4.9l1.8 1.8M17.3 17.3l1.8 1.8M2 12h2.5M19.5 12H22M4.9 19.1l1.8-1.8M17.3 6.7l1.8-1.8"
/>
</svg>
<svg
v-else
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="1.8"
class="h-5 w-5"
aria-hidden="true"
>
<path d="M21 14.5A8.5 8.5 0 1 1 9.5 3 7 7 0 0 0 21 14.5z" />
</svg>
</AppButton>
</div>
</template>

View file

@ -0,0 +1,153 @@
import { apiRequest } from "@/lib/api/client";
export interface AdminDbIntegrityCheck {
name: string;
count: number;
severity: "warning" | "failure" | "metric";
message: string;
}
export interface AdminDbIntegrityReport {
status: "ok" | "warning" | "failed";
checked_at: string;
failures: string[];
warnings: string[];
checks: AdminDbIntegrityCheck[];
}
export interface AdminDbRepairJob {
id: number;
job_name: string;
requested_by: string | null;
dry_run: boolean;
status: string;
scope: Record<string, unknown>;
summary: Record<string, unknown>;
error_text: string | null;
started_at: string | null;
finished_at: string | null;
created_at: string;
updated_at: string;
}
export interface AdminPdfQueueItem {
publication_id: number;
title: string;
doi: string | null;
pdf_url: string | null;
status: string;
attempt_count: number;
last_failure_reason: string | null;
last_failure_detail: string | null;
last_source: string | null;
requested_by_user_id: number | null;
requested_by_email: string | null;
queued_at: string | null;
last_attempt_at: string | null;
resolved_at: string | null;
updated_at: string;
}
export interface AdminPdfQueuePage {
items: AdminPdfQueueItem[];
total_count: number;
page: number;
page_size: number;
has_next: boolean;
has_prev: boolean;
}
export interface AdminPdfQueueRequeueResult {
publication_id: number;
queued: boolean;
status: string;
message: string;
}
export interface AdminPdfQueueBulkEnqueueResult {
requested_count: number;
queued_count: number;
message: string;
}
export interface TriggerPublicationLinkRepairPayload {
scope_mode?: "single_user" | "all_users";
user_id?: number;
scholar_profile_ids?: number[];
dry_run?: boolean;
gc_orphan_publications?: boolean;
requested_by?: string;
confirmation_text?: string;
}
export interface TriggerPublicationLinkRepairResult {
job_id: number;
status: string;
scope: Record<string, unknown>;
summary: Record<string, unknown>;
}
export async function getAdminDbIntegrityReport(): Promise<AdminDbIntegrityReport> {
const response = await apiRequest<AdminDbIntegrityReport>("/admin/db/integrity", { method: "GET" });
return response.data;
}
export async function listAdminDbRepairJobs(limit = 30): Promise<AdminDbRepairJob[]> {
const parsedLimit = Number.isFinite(limit) ? Math.max(1, Math.min(200, Math.trunc(limit))) : 30;
const response = await apiRequest<{ jobs: AdminDbRepairJob[] }>(
`/admin/db/repair-jobs?limit=${parsedLimit}`,
{ method: "GET" },
);
return response.data.jobs;
}
export async function triggerPublicationLinkRepair(
payload: TriggerPublicationLinkRepairPayload,
): Promise<TriggerPublicationLinkRepairResult> {
const response = await apiRequest<TriggerPublicationLinkRepairResult>(
"/admin/db/repairs/publication-links",
{
method: "POST",
body: payload,
},
);
return response.data;
}
export async function listAdminPdfQueue(
page = 1,
pageSize = 100,
status: string | null = null,
): Promise<AdminPdfQueuePage> {
const parsedPage = Number.isFinite(page) ? Math.max(1, Math.trunc(page)) : 1;
const parsedPageSize = Number.isFinite(pageSize) ? Math.max(1, Math.min(500, Math.trunc(pageSize))) : 100;
const params = new URLSearchParams();
params.set("page", String(parsedPage));
params.set("page_size", String(parsedPageSize));
if (status && status.trim().length > 0) {
params.set("status", status.trim().toLowerCase());
}
const response = await apiRequest<AdminPdfQueuePage>(
`/admin/db/pdf-queue?${params.toString()}`,
{ method: "GET" },
);
return response.data;
}
export async function requeueAdminPdfLookup(publicationId: number): Promise<AdminPdfQueueRequeueResult> {
const id = Number.isFinite(publicationId) ? Math.trunc(publicationId) : 0;
const response = await apiRequest<AdminPdfQueueRequeueResult>(
`/admin/db/pdf-queue/${Math.max(1, id)}/requeue`,
{ method: "POST" },
);
return response.data;
}
export async function requeueAllAdminPdfLookups(limit = 1000): Promise<AdminPdfQueueBulkEnqueueResult> {
const parsedLimit = Number.isFinite(limit) ? Math.max(1, Math.min(5000, Math.trunc(limit))) : 1000;
const response = await apiRequest<AdminPdfQueueBulkEnqueueResult>(
`/admin/db/pdf-queue/requeue-all?limit=${parsedLimit}`,
{ method: "POST" },
);
return response.data;
}

View file

@ -41,7 +41,7 @@ function countQueueStatuses(statuses: string[]): QueueHealth {
export async function fetchDashboardSnapshot(): Promise<DashboardSnapshot> { export async function fetchDashboardSnapshot(): Promise<DashboardSnapshot> {
const [publications, runsPayload, queueItems] = await Promise.all([ const [publications, runsPayload, queueItems] = await Promise.all([
listPublications({ mode: "latest", limit: 20 }), listPublications({ mode: "latest", page: 1, pageSize: 20 }),
listRuns({ limit: 20 }), listRuns({ limit: 20 }),
listQueueItems(200), listQueueItems(200),
]); ]);

View file

@ -13,26 +13,39 @@ export interface PublicationItem {
pub_url: string | null; pub_url: string | null;
doi: string | null; doi: string | null;
pdf_url: string | null; pdf_url: string | null;
pdf_status: "untracked" | "queued" | "running" | "resolved" | "failed";
pdf_attempt_count: number;
pdf_failure_reason: string | null;
pdf_failure_detail: string | null;
is_read: boolean; is_read: boolean;
is_favorite: boolean;
first_seen_at: string; first_seen_at: string;
is_new_in_latest_run: boolean; is_new_in_latest_run: boolean;
} }
export interface PublicationsResult { export interface PublicationsResult {
mode: PublicationMode; mode: PublicationMode;
favorite_only: boolean;
selected_scholar_profile_id: number | null; selected_scholar_profile_id: number | null;
unread_count: number; unread_count: number;
favorites_count: number;
latest_count: number; latest_count: number;
// Compatibility alias for latest_count; retained while API clients migrate. // Compatibility alias for latest_count; retained while API clients migrate.
new_count: number; new_count: number;
total_count: number; total_count: number;
page: number;
page_size: number;
has_next: boolean;
has_prev: boolean;
publications: PublicationItem[]; publications: PublicationItem[];
} }
export interface PublicationsQuery { export interface PublicationsQuery {
mode?: PublicationMode; mode?: PublicationMode;
favoriteOnly?: boolean;
scholarProfileId?: number; scholarProfileId?: number;
limit?: number; page?: number;
pageSize?: number;
} }
export interface PublicationSelection { export interface PublicationSelection {
@ -46,12 +59,18 @@ export async function listPublications(query: PublicationsQuery = {}): Promise<P
if (query.mode) { if (query.mode) {
params.set("mode", query.mode); params.set("mode", query.mode);
} }
if (query.favoriteOnly) {
params.set("favorite_only", "true");
}
if (query.scholarProfileId) { if (query.scholarProfileId) {
params.set("scholar_profile_id", String(query.scholarProfileId)); params.set("scholar_profile_id", String(query.scholarProfileId));
} }
if (query.limit) { const parsedPage = Number.isFinite(query.page) ? Math.max(1, Math.trunc(Number(query.page))) : 1;
params.set("limit", String(query.limit)); const parsedPageSize = Number.isFinite(query.pageSize)
} ? Math.max(1, Math.min(500, Math.trunc(Number(query.pageSize))))
: 100;
params.set("page", String(parsedPage));
params.set("page_size", String(parsedPageSize));
const suffix = params.toString(); const suffix = params.toString();
const response = await apiRequest<PublicationsResult>( const response = await apiRequest<PublicationsResult>(
@ -87,10 +106,16 @@ export async function markSelectedRead(selections: PublicationSelection[]): Prom
export interface RetryPublicationPdfResult { export interface RetryPublicationPdfResult {
message: string; message: string;
queued: boolean;
resolved_pdf: boolean; resolved_pdf: boolean;
publication: PublicationItem; publication: PublicationItem;
} }
export interface TogglePublicationFavoriteResult {
message: string;
publication: PublicationItem;
}
export async function retryPublicationPdf( export async function retryPublicationPdf(
publicationId: number, publicationId: number,
scholarProfileId: number, scholarProfileId: number,
@ -104,3 +129,21 @@ export async function retryPublicationPdf(
); );
return response.data; return response.data;
} }
export async function togglePublicationFavorite(
publicationId: number,
scholarProfileId: number,
isFavorite: boolean,
): Promise<TogglePublicationFavoriteResult> {
const response = await apiRequest<TogglePublicationFavoriteResult>(
`/publications/${publicationId}/favorite`,
{
method: "POST",
body: {
scholar_profile_id: scholarProfileId,
is_favorite: isFavorite,
},
},
);
return response.data;
}

View file

@ -0,0 +1,826 @@
<script setup lang="ts">
import { computed, onMounted, ref, watch } from "vue";
import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue";
import AppBadge from "@/components/ui/AppBadge.vue";
import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue";
import AppCheckbox from "@/components/ui/AppCheckbox.vue";
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
import AppInput from "@/components/ui/AppInput.vue";
import AppModal from "@/components/ui/AppModal.vue";
import AppRefreshButton from "@/components/ui/AppRefreshButton.vue";
import AppSelect from "@/components/ui/AppSelect.vue";
import AppTable from "@/components/ui/AppTable.vue";
import {
getAdminDbIntegrityReport,
listAdminDbRepairJobs,
listAdminPdfQueue,
requeueAdminPdfLookup,
requeueAllAdminPdfLookups,
triggerPublicationLinkRepair,
type AdminDbIntegrityCheck,
type AdminDbIntegrityReport,
type AdminDbRepairJob,
type AdminPdfQueueItem,
type TriggerPublicationLinkRepairResult,
} from "@/features/admin_dbops";
import {
createAdminUser,
listAdminUsers,
resetAdminUserPassword,
setAdminUserActive,
type AdminUser,
} from "@/features/admin_users";
import { ApiRequestError } from "@/lib/api/errors";
const SECTION_USERS = "users";
const SECTION_INTEGRITY = "integrity";
const SECTION_REPAIRS = "repairs";
const SECTION_PDF = "pdf";
const SCOPE_SINGLE_USER = "single_user";
const SCOPE_ALL_USERS = "all_users";
const APPLY_ALL_USERS_CONFIRM_TEXT = "REPAIR ALL USERS";
type RepairScopeMode = typeof SCOPE_SINGLE_USER | typeof SCOPE_ALL_USERS;
const props = defineProps<{
section: "users" | "integrity" | "repairs" | "pdf";
}>();
const loading = ref(true);
const errorMessage = ref<string | null>(null);
const errorRequestId = ref<string | null>(null);
const successMessage = ref<string | null>(null);
const refreshingUsers = ref(false);
const creating = ref(false);
const togglingUserId = ref<number | null>(null);
const resettingPassword = ref(false);
const users = ref<AdminUser[]>([]);
const email = ref("");
const password = ref("");
const createIsAdmin = ref(false);
const activeUserId = ref<number | null>(null);
const resetPassword = ref("");
const refreshingIntegrity = ref(false);
const integrityReport = ref<AdminDbIntegrityReport | null>(null);
const refreshingJobs = ref(false);
const runningRepair = ref(false);
const repairJobs = ref<AdminDbRepairJob[]>([]);
const lastRepairResult = ref<TriggerPublicationLinkRepairResult | null>(null);
const repairScopeMode = ref<RepairScopeMode>(SCOPE_SINGLE_USER);
const repairUserId = ref("");
const repairScholarIds = ref("");
const repairRequestedBy = ref("");
const repairDryRun = ref(true);
const repairGcOrphans = ref(false);
const repairConfirmationText = ref("");
const refreshingPdfQueue = ref(false);
const requeueingPublicationId = ref<number | null>(null);
const requeueingAllPdfs = ref(false);
const pdfQueueItems = ref<AdminPdfQueueItem[]>([]);
const pdfQueueStatusFilter = ref("");
const pdfQueuePage = ref(1);
const pdfQueuePageSize = ref("50");
const pdfQueueTotalCount = ref(0);
const pdfQueueHasNext = ref(false);
const pdfQueueHasPrev = ref(false);
const activeUser = computed(() => users.value.find((user) => user.id === activeUserId.value) ?? null);
const typedConfirmationRequired = computed(
() => repairScopeMode.value === SCOPE_ALL_USERS && !repairDryRun.value,
);
const pdfQueuePageSizeValue = computed(() => {
const parsed = Number(pdfQueuePageSize.value);
if (!Number.isFinite(parsed)) {
return 50;
}
return Math.max(1, Math.min(500, Math.trunc(parsed)));
});
const pdfQueueTotalPages = computed(() =>
Math.max(1, Math.ceil(pdfQueueTotalCount.value / Math.max(pdfQueuePageSizeValue.value, 1))),
);
const pdfQueueSummary = computed(() =>
`${pdfQueueTotalCount.value} item${pdfQueueTotalCount.value === 1 ? "" : "s"} total`,
);
function clearAlerts(): void {
errorMessage.value = null;
errorRequestId.value = null;
successMessage.value = null;
}
function formatTimestamp(value: string | null): string {
if (!value) {
return "n/a";
}
const parsed = new Date(value);
return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString();
}
function statusTone(status: string): "success" | "warning" | "danger" | "info" | "neutral" {
if (status === "ok" || status === "completed" || status === "resolved") {
return "success";
}
if (status === "warning" || status === "running" || status === "queued") {
return "warning";
}
if (status === "failed") {
return "danger";
}
return "info";
}
function checkTone(check: AdminDbIntegrityCheck): "warning" | "danger" | "neutral" | "info" {
if (check.severity === "metric") {
return "info";
}
if (check.count <= 0) {
return "neutral";
}
return check.severity === "failure" ? "danger" : "warning";
}
function userRoleLabel(user: AdminUser): string {
return user.is_admin ? "Admin" : "User";
}
function canRequeuePdf(item: AdminPdfQueueItem): boolean {
return item.status !== "queued" && item.status !== "running";
}
function statusDotClass(user: AdminUser): string {
return user.is_active ? "bg-success-500 ring-success-200" : "bg-ink-muted/70 ring-stroke-default";
}
function assignError(error: unknown, fallback: string): void {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
return;
}
if (error instanceof Error && error.message) {
errorMessage.value = error.message;
return;
}
errorMessage.value = fallback;
}
function parseScholarIds(raw: string): number[] {
const tokens = raw
.split(/[,\s]+/)
.map((token) => token.trim())
.filter((token) => token.length > 0);
const deduped = new Set<number>();
for (const token of tokens) {
const parsed = Number(token);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error("Scholar profile IDs must be positive integers.");
}
deduped.add(parsed);
}
return [...deduped];
}
function parseRepairUserIdOrThrow(raw: string): number {
const parsed = Number((raw || "").trim());
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error("Select a valid target user.");
}
return parsed;
}
function validateTypedConfirmation(): string {
const normalized = repairConfirmationText.value.trim();
if (typedConfirmationRequired.value && normalized !== APPLY_ALL_USERS_CONFIRM_TEXT) {
throw new Error(`Type '${APPLY_ALL_USERS_CONFIRM_TEXT}' to apply repair for all users.`);
}
return normalized;
}
function summaryCount(job: AdminDbRepairJob, key: string): string {
const value = job.summary[key];
return typeof value === "number" ? String(value) : "n/a";
}
function openUserModal(user: AdminUser): void {
activeUserId.value = user.id;
resetPassword.value = "";
}
function closeUserModal(): void {
activeUserId.value = null;
resetPassword.value = "";
}
function ensureRepairUserSelected(): void {
if (repairScopeMode.value !== SCOPE_SINGLE_USER || repairUserId.value || users.value.length === 0) {
return;
}
repairUserId.value = String(users.value[0].id);
}
async function refreshUsers(): Promise<void> {
refreshingUsers.value = true;
try {
users.value = await listAdminUsers();
ensureRepairUserSelected();
if (activeUserId.value !== null && !users.value.some((item) => item.id === activeUserId.value)) {
closeUserModal();
}
} finally {
refreshingUsers.value = false;
}
}
async function refreshIntegrity(): Promise<void> {
refreshingIntegrity.value = true;
try {
integrityReport.value = await getAdminDbIntegrityReport();
} finally {
refreshingIntegrity.value = false;
}
}
async function refreshRepairJobs(): Promise<void> {
refreshingJobs.value = true;
try {
repairJobs.value = await listAdminDbRepairJobs(30);
} finally {
refreshingJobs.value = false;
}
}
async function refreshPdfQueue(): Promise<void> {
refreshingPdfQueue.value = true;
try {
const status = pdfQueueStatusFilter.value.trim() || null;
const page = await listAdminPdfQueue(pdfQueuePage.value, pdfQueuePageSizeValue.value, status);
pdfQueueItems.value = page.items;
pdfQueueTotalCount.value = page.total_count;
pdfQueueHasNext.value = page.has_next;
pdfQueueHasPrev.value = page.has_prev;
pdfQueuePage.value = page.page;
pdfQueuePageSize.value = String(page.page_size);
} finally {
refreshingPdfQueue.value = false;
}
}
async function refreshForSection(): Promise<void> {
if (props.section === SECTION_USERS) {
await refreshUsers();
return;
}
if (props.section === SECTION_INTEGRITY) {
await refreshIntegrity();
return;
}
if (props.section === SECTION_REPAIRS) {
await Promise.all([refreshUsers(), refreshRepairJobs()]);
return;
}
await refreshPdfQueue();
}
async function onCreateUser(): Promise<void> {
creating.value = true;
clearAlerts();
try {
if (!email.value.trim() || !password.value) {
throw new Error("Email and password are required.");
}
const created = await createAdminUser({
email: email.value.trim(),
password: password.value,
is_admin: createIsAdmin.value,
});
email.value = "";
password.value = "";
createIsAdmin.value = false;
successMessage.value = `User created: ${created.email}`;
await refreshUsers();
} catch (error) {
assignError(error, "Unable to create user.");
} finally {
creating.value = false;
}
}
async function onToggleUser(user: AdminUser): Promise<void> {
togglingUserId.value = user.id;
clearAlerts();
try {
const updated = await setAdminUserActive(user.id, !user.is_active);
successMessage.value = `${updated.email} is now ${updated.is_active ? "active" : "inactive"}.`;
await refreshUsers();
} catch (error) {
assignError(error, "Unable to update user.");
} finally {
togglingUserId.value = null;
}
}
async function onResetPassword(): Promise<void> {
const user = activeUser.value;
if (!user) {
return;
}
resettingPassword.value = true;
clearAlerts();
try {
const candidate = resetPassword.value.trim();
if (candidate.length < 12) {
throw new Error("New password must be at least 12 characters.");
}
const result = await resetAdminUserPassword(user.id, candidate);
resetPassword.value = "";
successMessage.value = result.message || `Password reset for ${user.email}.`;
} catch (error) {
assignError(error, "Unable to reset password.");
} finally {
resettingPassword.value = false;
}
}
async function onRunRepair(): Promise<void> {
runningRepair.value = true;
clearAlerts();
try {
const scopeMode = repairScopeMode.value;
const payload = {
scope_mode: scopeMode,
scholar_profile_ids: parseScholarIds(repairScholarIds.value),
dry_run: repairDryRun.value,
gc_orphan_publications: repairGcOrphans.value,
requested_by: repairRequestedBy.value.trim() || undefined,
confirmation_text: validateTypedConfirmation() || undefined,
user_id: scopeMode === SCOPE_SINGLE_USER ? parseRepairUserIdOrThrow(repairUserId.value) : undefined,
};
const result = await triggerPublicationLinkRepair(payload);
lastRepairResult.value = result;
successMessage.value = `Repair job #${result.job_id} completed (${result.status}).`;
repairConfirmationText.value = "";
await Promise.all([refreshIntegrity(), refreshRepairJobs(), refreshPdfQueue()]);
} catch (error) {
assignError(error, "Unable to run publication link repair.");
} finally {
runningRepair.value = false;
}
}
function onScopeModeChange(): void {
ensureRepairUserSelected();
}
async function onPdfQueueFilterChanged(): Promise<void> {
pdfQueuePage.value = 1;
await refreshPdfQueue();
}
async function onPdfQueuePageSizeChanged(): Promise<void> {
pdfQueuePage.value = 1;
await refreshPdfQueue();
}
async function onPdfQueuePrevPage(): Promise<void> {
if (!pdfQueueHasPrev.value || pdfQueuePage.value <= 1) {
return;
}
pdfQueuePage.value = Math.max(pdfQueuePage.value - 1, 1);
await refreshPdfQueue();
}
async function onPdfQueueNextPage(): Promise<void> {
if (!pdfQueueHasNext.value) {
return;
}
pdfQueuePage.value += 1;
await refreshPdfQueue();
}
async function onRequeuePdf(item: AdminPdfQueueItem): Promise<void> {
requeueingPublicationId.value = item.publication_id;
clearAlerts();
try {
const result = await requeueAdminPdfLookup(item.publication_id);
successMessage.value = result.message;
await refreshPdfQueue();
} catch (error) {
assignError(error, "Unable to requeue PDF lookup.");
} finally {
requeueingPublicationId.value = null;
}
}
async function onRequeueAllPdfs(): Promise<void> {
requeueingAllPdfs.value = true;
clearAlerts();
try {
const result = await requeueAllAdminPdfLookups(5000);
successMessage.value = result.message;
await refreshPdfQueue();
} catch (error) {
assignError(error, "Unable to queue missing PDF lookups.");
} finally {
requeueingAllPdfs.value = false;
}
}
onMounted(async () => {
loading.value = true;
clearAlerts();
try {
await refreshForSection();
} catch (error) {
assignError(error, "Unable to load admin data.");
} finally {
loading.value = false;
}
});
watch(
() => props.section,
async () => {
loading.value = true;
clearAlerts();
try {
await refreshForSection();
} catch (error) {
assignError(error, "Unable to load admin data.");
} finally {
loading.value = false;
}
},
);
</script>
<template>
<section class="grid gap-4">
<RequestStateAlerts
:success-message="successMessage"
:error-message="errorMessage"
:error-request-id="errorRequestId"
success-title="Operation complete"
error-title="Operation failed"
@dismiss-success="successMessage = null"
/>
<AsyncStateGate :loading="loading" :loading-lines="10" :show-empty="false">
<AppCard v-if="props.section === SECTION_USERS" class="space-y-4">
<div class="flex flex-wrap items-center justify-between gap-2">
<div class="flex items-center gap-1">
<h2 class="text-lg font-semibold text-ink-primary">Users</h2>
<AppHelpHint text="Create accounts, toggle active status, and reset passwords." />
</div>
<AppRefreshButton
variant="secondary"
:loading="refreshingUsers"
title="Refresh users"
loading-title="Refreshing users"
@click="refreshUsers"
/>
</div>
<form class="grid gap-3 md:grid-cols-3" @submit.prevent="onCreateUser">
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
<span>Email</span>
<AppInput v-model="email" type="email" autocomplete="off" />
</label>
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
<span>Password</span>
<AppInput v-model="password" type="password" autocomplete="new-password" />
</label>
<div class="grid gap-2 text-sm font-medium text-ink-secondary">
<span>Role</span>
<div class="flex items-center gap-3">
<AppCheckbox id="admin-create-user-is-admin" v-model="createIsAdmin" label="Grant admin" />
<AppButton type="submit" :disabled="creating">
{{ creating ? "Creating..." : "Create user" }}
</AppButton>
</div>
</div>
</form>
<AppTable label="Users table">
<thead>
<tr>
<th scope="col">Email</th>
<th scope="col">Role</th>
<th scope="col">Status</th>
<th scope="col">Actions</th>
</tr>
</thead>
<tbody>
<tr v-for="user in users" :key="user.id">
<td class="align-middle">
<button
type="button"
class="group inline-flex items-center gap-2 rounded-md px-1 py-0.5 text-left text-ink-primary transition hover:bg-surface-card-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset"
@click="openUserModal(user)"
>
<span
:class="statusDotClass(user)"
class="h-2.5 w-2.5 rounded-full ring-2"
:aria-label="user.is_active ? 'Active user' : 'Inactive user'"
/>
<span class="underline-offset-2 group-hover:underline">{{ user.email }}</span>
</button>
</td>
<td>{{ userRoleLabel(user) }}</td>
<td>{{ user.is_active ? "active" : "inactive" }}</td>
<td class="flex flex-wrap items-center gap-2">
<AppButton variant="secondary" :disabled="togglingUserId === user.id" @click="onToggleUser(user)">
{{ togglingUserId === user.id ? "Updating..." : user.is_active ? "Deactivate" : "Activate" }}
</AppButton>
<AppButton variant="ghost" @click="openUserModal(user)">Manage</AppButton>
</td>
</tr>
</tbody>
</AppTable>
</AppCard>
<AppCard v-if="props.section === SECTION_INTEGRITY" class="space-y-3">
<div class="flex flex-wrap items-center justify-between gap-2">
<div class="flex items-center gap-1">
<h2 class="text-lg font-semibold text-ink-primary">Integrity Report</h2>
<AppHelpHint text="Read-only checks for known corruption patterns and data drift." />
</div>
<AppRefreshButton
variant="secondary"
:loading="refreshingIntegrity"
title="Refresh integrity report"
loading-title="Refreshing integrity report"
@click="refreshIntegrity"
/>
</div>
<div v-if="integrityReport" class="flex flex-wrap items-center gap-2 text-xs">
<AppBadge :tone="statusTone(integrityReport.status)">Status: {{ integrityReport.status }}</AppBadge>
<AppBadge tone="warning">Warnings: {{ integrityReport.warnings.length }}</AppBadge>
<AppBadge tone="danger">Failures: {{ integrityReport.failures.length }}</AppBadge>
<span class="text-secondary">Checked: {{ formatTimestamp(integrityReport.checked_at) }}</span>
</div>
<AppTable v-if="integrityReport" label="Integrity checks">
<thead>
<tr>
<th scope="col">Check</th>
<th scope="col">Count</th>
<th scope="col">Severity</th>
<th scope="col">Message</th>
</tr>
</thead>
<tbody>
<tr v-for="check in integrityReport.checks" :key="check.name">
<td>{{ check.name }}</td>
<td>{{ check.count }}</td>
<td><AppBadge :tone="checkTone(check)">{{ check.severity }}</AppBadge></td>
<td>{{ check.message }}</td>
</tr>
</tbody>
</AppTable>
</AppCard>
<section v-if="props.section === SECTION_REPAIRS" class="grid gap-4">
<AppCard class="space-y-3">
<div class="flex items-center gap-1">
<h2 class="text-lg font-semibold text-ink-primary">Publication Link Repair</h2>
<AppHelpHint text="Dry-run first. For all-users apply mode, typed confirmation is required." />
</div>
<form class="grid gap-3 md:grid-cols-2" @submit.prevent="onRunRepair">
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
<span>Scope</span>
<AppSelect v-model="repairScopeMode" @change="onScopeModeChange">
<option :value="SCOPE_SINGLE_USER">Single user</option>
<option :value="SCOPE_ALL_USERS">All users</option>
</AppSelect>
</label>
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
<span>Target user</span>
<AppSelect v-model="repairUserId" :disabled="repairScopeMode === SCOPE_ALL_USERS || users.length === 0">
<option value="" disabled>Select user</option>
<option v-for="user in users" :key="user.id" :value="String(user.id)">
{{ user.email }} (ID {{ user.id }})
</option>
</AppSelect>
</label>
<label class="grid gap-1 text-sm font-medium text-ink-secondary md:col-span-2">
<span>Scholar profile IDs (optional)</span>
<AppInput v-model="repairScholarIds" placeholder="e.g. 12,13,14" />
</label>
<label class="grid gap-1 text-sm font-medium text-ink-secondary md:col-span-2">
<span>Requested by (optional)</span>
<AppInput v-model="repairRequestedBy" placeholder="email/name/ticket id" />
</label>
<div class="flex flex-wrap items-center gap-4 md:col-span-2">
<AppCheckbox id="repair-dry-run" v-model="repairDryRun" label="Dry-run (no writes)" />
<AppCheckbox id="repair-gc-orphans" v-model="repairGcOrphans" label="Delete orphan publications" />
</div>
<label v-if="typedConfirmationRequired" class="grid gap-1 text-sm font-medium text-ink-secondary md:col-span-2">
<span>Type '{{ APPLY_ALL_USERS_CONFIRM_TEXT }}' to confirm</span>
<AppInput v-model="repairConfirmationText" :placeholder="APPLY_ALL_USERS_CONFIRM_TEXT" />
</label>
<p v-if="repairScopeMode === SCOPE_ALL_USERS" class="text-xs text-secondary md:col-span-2">
All-users scope includes every scholar profile across all accounts.
</p>
<div class="md:col-span-2">
<AppButton type="submit" :disabled="runningRepair">
{{ runningRepair ? "Running..." : "Run repair job" }}
</AppButton>
</div>
</form>
<div v-if="lastRepairResult" class="rounded-lg border border-stroke-default bg-surface-card-muted p-3 text-xs">
<div class="mb-2 flex flex-wrap items-center gap-2">
<AppBadge :tone="statusTone(lastRepairResult.status)">Job #{{ lastRepairResult.job_id }}</AppBadge>
<span class="text-secondary">Status: {{ lastRepairResult.status }}</span>
</div>
<pre class="overflow-x-auto text-secondary">{{ JSON.stringify(lastRepairResult.summary, null, 2) }}</pre>
</div>
</AppCard>
<AppCard class="space-y-3">
<div class="flex flex-wrap items-center justify-between gap-2">
<div class="flex items-center gap-1">
<h2 class="text-lg font-semibold text-ink-primary">Recent Repair Jobs</h2>
<AppHelpHint text="Audit history and summary counters for each repair job." />
</div>
<AppRefreshButton
variant="secondary"
:loading="refreshingJobs"
title="Refresh repair jobs"
loading-title="Refreshing repair jobs"
@click="refreshRepairJobs"
/>
</div>
<AppTable label="Repair jobs table">
<thead>
<tr>
<th scope="col">Job</th>
<th scope="col">Status</th>
<th scope="col">Mode</th>
<th scope="col">Requested by</th>
<th scope="col">Created</th>
<th scope="col">Links in scope</th>
<th scope="col">Links deleted</th>
</tr>
</thead>
<tbody>
<tr v-for="job in repairJobs" :key="job.id">
<td>#{{ job.id }} · {{ job.job_name }}</td>
<td><AppBadge :tone="statusTone(job.status)">{{ job.status }}</AppBadge></td>
<td>{{ job.dry_run ? "dry-run" : "apply" }}</td>
<td>{{ job.requested_by || "n/a" }}</td>
<td>{{ formatTimestamp(job.created_at) }}</td>
<td>{{ summaryCount(job, "links_in_scope") }}</td>
<td>{{ summaryCount(job, "links_deleted") }}</td>
</tr>
</tbody>
</AppTable>
</AppCard>
</section>
<AppCard v-if="props.section === SECTION_PDF" class="space-y-3">
<div class="flex flex-wrap items-center justify-between gap-2">
<div class="flex items-center gap-1">
<h2 class="text-lg font-semibold text-ink-primary">PDF Gathering Queue</h2>
<AppHelpHint text="Live queue and outcome history for PDF acquisition across all publications." />
</div>
<div class="flex items-center gap-2">
<AppSelect v-model="pdfQueueStatusFilter" class="min-w-[12rem] !py-1.5 !text-sm" @change="onPdfQueueFilterChanged">
<option value="">All statuses</option>
<option value="untracked">untracked</option>
<option value="queued">queued</option>
<option value="running">running</option>
<option value="failed">failed</option>
<option value="resolved">resolved</option>
</AppSelect>
<AppSelect v-model="pdfQueuePageSize" class="min-w-[10rem] !py-1.5 !text-sm" @change="onPdfQueuePageSizeChanged">
<option value="25">25 / page</option>
<option value="50">50 / page</option>
<option value="100">100 / page</option>
</AppSelect>
<AppButton
variant="secondary"
class="!min-h-8 whitespace-nowrap !px-2.5 !py-1 !text-xs"
:disabled="requeueingAllPdfs"
title="Queue all missing PDFs"
@click="onRequeueAllPdfs"
>
{{ requeueingAllPdfs ? "Queueing..." : "Queue all" }}
</AppButton>
<AppRefreshButton
variant="secondary"
size="sm"
:loading="refreshingPdfQueue"
title="Refresh PDF queue"
loading-title="Refreshing PDF queue"
@click="refreshPdfQueue"
/>
</div>
</div>
<AppTable label="PDF queue table">
<thead>
<tr>
<th scope="col">Publication</th>
<th scope="col">Status</th>
<th scope="col">Attempts</th>
<th scope="col">Failure reason</th>
<th scope="col">Source</th>
<th scope="col">Requested by</th>
<th scope="col">Queued</th>
<th scope="col">Last attempt</th>
<th scope="col">Resolved</th>
<th scope="col">Actions</th>
</tr>
</thead>
<tbody>
<tr v-for="item in pdfQueueItems" :key="item.publication_id">
<td>
<div class="grid gap-1">
<span class="font-medium text-ink-primary">{{ item.title }}</span>
<a v-if="item.doi" :href="`https://doi.org/${item.doi}`" target="_blank" rel="noreferrer" class="link-inline text-xs">
DOI: {{ item.doi }}
</a>
</div>
</td>
<td><AppBadge :tone="statusTone(item.status)">{{ item.status }}</AppBadge></td>
<td>{{ item.attempt_count }}</td>
<td>{{ item.last_failure_reason || "n/a" }}</td>
<td>{{ item.last_source || "n/a" }}</td>
<td>{{ item.requested_by_email || "n/a" }}</td>
<td>{{ formatTimestamp(item.queued_at) }}</td>
<td>{{ formatTimestamp(item.last_attempt_at) }}</td>
<td>{{ formatTimestamp(item.resolved_at) }}</td>
<td>
<AppButton
variant="ghost"
:disabled="requeueingPublicationId === item.publication_id || !canRequeuePdf(item)"
@click="onRequeuePdf(item)"
>
{{ requeueingPublicationId === item.publication_id ? "Requeueing..." : "Requeue" }}
</AppButton>
</td>
</tr>
</tbody>
</AppTable>
<div class="flex flex-wrap items-center justify-between gap-2 text-xs text-secondary">
<span>{{ pdfQueueSummary }}</span>
<div class="flex items-center gap-2">
<span>Page {{ pdfQueuePage }} / {{ pdfQueueTotalPages }}</span>
<AppButton variant="ghost" :disabled="!pdfQueueHasPrev" @click="onPdfQueuePrevPage">Prev</AppButton>
<AppButton variant="ghost" :disabled="!pdfQueueHasNext" @click="onPdfQueueNextPage">Next</AppButton>
</div>
</div>
</AppCard>
</AsyncStateGate>
<AppModal :open="activeUser !== null" title="User settings" @close="closeUserModal">
<div v-if="activeUser" class="grid gap-4">
<div class="space-y-1">
<div class="flex items-center gap-2">
<span
:class="statusDotClass(activeUser)"
class="h-2.5 w-2.5 rounded-full ring-2"
:aria-label="activeUser.is_active ? 'Active user' : 'Inactive user'"
/>
<p class="truncate text-sm font-semibold text-ink-primary">{{ activeUser.email }}</p>
</div>
<p class="text-sm text-secondary">Role: {{ userRoleLabel(activeUser) }}</p>
<p class="text-xs text-secondary">Last updated: {{ formatTimestamp(activeUser.updated_at) }}</p>
</div>
<div class="grid gap-2">
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
<span>New password</span>
<AppInput
id="admin-reset-password"
v-model="resetPassword"
type="password"
autocomplete="new-password"
placeholder="At least 12 characters"
/>
</label>
<div class="flex flex-wrap gap-2">
<AppButton :disabled="resettingPassword" @click="onResetPassword">
{{ resettingPassword ? "Resetting..." : "Reset password" }}
</AppButton>
<AppButton variant="secondary" :disabled="togglingUserId === activeUser.id" @click="onToggleUser(activeUser)">
{{ togglingUserId === activeUser.id ? "Updating..." : activeUser.is_active ? "Deactivate user" : "Activate user" }}
</AppButton>
</div>
</div>
</div>
</AppModal>
</section>
</template>

BIN
frontend/src/logo.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 104 KiB

View file

@ -1,316 +0,0 @@
<script setup lang="ts">
import { computed, onMounted, ref } from "vue";
import AppPage from "@/components/layout/AppPage.vue";
import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue";
import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue";
import AppCheckbox from "@/components/ui/AppCheckbox.vue";
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
import AppInput from "@/components/ui/AppInput.vue";
import AppModal from "@/components/ui/AppModal.vue";
import AppTable from "@/components/ui/AppTable.vue";
import {
createAdminUser,
listAdminUsers,
resetAdminUserPassword,
setAdminUserActive,
type AdminUser,
} from "@/features/admin_users";
import { ApiRequestError } from "@/lib/api/errors";
const loading = ref(true);
const creating = ref(false);
const togglingUserId = ref<number | null>(null);
const resettingPassword = ref(false);
const activeUserId = ref<number | null>(null);
const users = ref<AdminUser[]>([]);
const email = ref("");
const password = ref("");
const createIsAdmin = ref(false);
const resetPassword = ref("");
const errorMessage = ref<string | null>(null);
const errorRequestId = ref<string | null>(null);
const successMessage = ref<string | null>(null);
const activeUser = computed(() => users.value.find((item) => item.id === activeUserId.value) ?? null);
function formatDate(value: string): string {
const asDate = new Date(value);
if (Number.isNaN(asDate.getTime())) {
return value;
}
return asDate.toLocaleString();
}
function userRoleLabel(user: AdminUser): string {
return user.is_admin ? "Admin" : "User";
}
function statusDotClass(user: AdminUser): string {
return user.is_active
? "bg-success-500 ring-success-200"
: "bg-ink-muted/70 ring-stroke-default";
}
function openUserModal(user: AdminUser): void {
activeUserId.value = user.id;
resetPassword.value = "";
}
function closeUserModal(): void {
activeUserId.value = null;
resetPassword.value = "";
}
async function loadUsers(): Promise<void> {
loading.value = true;
errorMessage.value = null;
errorRequestId.value = null;
try {
users.value = await listAdminUsers();
if (activeUserId.value !== null && !users.value.some((item) => item.id === activeUserId.value)) {
closeUserModal();
}
} catch (error) {
users.value = [];
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to load users.";
}
} finally {
loading.value = false;
}
}
async function onCreateUser(): Promise<void> {
creating.value = true;
successMessage.value = null;
errorMessage.value = null;
errorRequestId.value = null;
try {
if (!email.value.trim() || !password.value) {
throw new Error("Email and password are required.");
}
const created = await createAdminUser({
email: email.value.trim(),
password: password.value,
is_admin: createIsAdmin.value,
});
email.value = "";
password.value = "";
createIsAdmin.value = false;
successMessage.value = `User created: ${created.email}`;
await loadUsers();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else if (error instanceof Error) {
errorMessage.value = error.message;
} else {
errorMessage.value = "Unable to create user.";
}
} finally {
creating.value = false;
}
}
async function onToggleUser(user: AdminUser): Promise<void> {
togglingUserId.value = user.id;
successMessage.value = null;
try {
const updated = await setAdminUserActive(user.id, !user.is_active);
successMessage.value = `${updated.email} is now ${updated.is_active ? "active" : "inactive"}.`;
await loadUsers();
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to update user.";
}
} finally {
togglingUserId.value = null;
}
}
async function onResetPassword(): Promise<void> {
const user = activeUser.value;
if (!user) {
return;
}
resettingPassword.value = true;
successMessage.value = null;
errorMessage.value = null;
errorRequestId.value = null;
try {
const candidate = resetPassword.value.trim();
if (candidate.length < 12) {
throw new Error("New password must be at least 12 characters.");
}
const result = await resetAdminUserPassword(user.id, candidate);
successMessage.value = result.message || `Password reset for ${user.email}.`;
resetPassword.value = "";
} catch (error) {
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else if (error instanceof Error) {
errorMessage.value = error.message;
} else {
errorMessage.value = "Unable to reset password.";
}
} finally {
resettingPassword.value = false;
}
}
onMounted(() => {
void loadUsers();
});
</script>
<template>
<AppPage title="Admin Users" subtitle="Create and manage user access for this instance.">
<RequestStateAlerts
:success-message="successMessage"
:error-message="errorMessage"
:error-request-id="errorRequestId"
error-title="User management request failed"
@dismiss-success="successMessage = null"
/>
<section class="grid gap-4 lg:grid-cols-2">
<AppCard class="space-y-4">
<div class="flex items-center gap-1">
<h2 class="text-lg font-semibold text-ink-primary">Create User</h2>
<AppHelpHint text="Create local accounts for this Scholarr instance and optionally grant admin rights." />
</div>
<form class="grid gap-3" @submit.prevent="onCreateUser">
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
<span>Email</span>
<AppInput id="admin-user-email" v-model="email" type="email" autocomplete="off" />
</label>
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
<span>Password</span>
<AppInput id="admin-user-password" v-model="password" type="password" autocomplete="new-password" />
</label>
<AppCheckbox id="admin-user-is-admin" v-model="createIsAdmin" label="Grant admin privileges" />
<AppButton type="submit" :disabled="creating">
{{ creating ? "Creating..." : "Create user" }}
</AppButton>
</form>
</AppCard>
<AppCard class="space-y-4">
<div class="flex items-center gap-1">
<h2 class="text-lg font-semibold text-ink-primary">Users</h2>
<AppHelpHint text="Review account roles and active status. Inactive users cannot sign in." />
</div>
<AsyncStateGate
:loading="loading"
:loading-lines="5"
:empty="users.length === 0"
:show-empty="!errorMessage"
empty-title="No users available"
empty-body="Create an account to begin assigning access."
>
<AppTable label="Admin users table">
<thead>
<tr>
<th scope="col">Email</th>
<th scope="col">Role</th>
</tr>
</thead>
<tbody>
<tr v-for="user in users" :key="user.id">
<td>
<button
type="button"
class="group inline-flex items-center gap-2 rounded-md px-1 py-0.5 text-left text-ink-primary transition hover:bg-surface-card-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset"
@click="openUserModal(user)"
>
<span
:class="statusDotClass(user)"
class="h-2.5 w-2.5 rounded-full ring-2"
:aria-label="user.is_active ? 'Active user' : 'Inactive user'"
/>
<span class="underline-offset-2 group-hover:underline">{{ user.email }}</span>
</button>
</td>
<td>{{ userRoleLabel(user) }}</td>
</tr>
</tbody>
</AppTable>
<p class="text-xs text-secondary">Status dot: green is active, gray is inactive. Click an email to manage.</p>
</AsyncStateGate>
</AppCard>
</section>
<AppModal :open="activeUser !== null" title="User settings" @close="closeUserModal">
<div v-if="activeUser" class="grid gap-4">
<div class="space-y-1">
<div class="flex items-center gap-2">
<span
:class="statusDotClass(activeUser)"
class="h-2.5 w-2.5 rounded-full ring-2"
:aria-label="activeUser.is_active ? 'Active user' : 'Inactive user'"
/>
<p class="truncate text-sm font-semibold text-ink-primary">{{ activeUser.email }}</p>
</div>
<p class="text-sm text-secondary">Role: {{ userRoleLabel(activeUser) }}</p>
<p class="text-xs text-secondary">Last updated: {{ formatDate(activeUser.updated_at) }}</p>
</div>
<div class="grid gap-2 border-t border-stroke-default pt-3">
<label class="grid gap-2 text-sm font-medium text-ink-secondary" for="admin-user-reset-password">
<span>Reset password</span>
<AppInput
id="admin-user-reset-password"
v-model="resetPassword"
type="password"
autocomplete="new-password"
placeholder="Minimum 12 characters"
:disabled="resettingPassword"
/>
</label>
<div class="flex flex-wrap justify-end gap-2">
<AppButton variant="secondary" :disabled="resettingPassword" @click="onResetPassword">
{{ resettingPassword ? "Resetting..." : "Reset password" }}
</AppButton>
</div>
</div>
<div class="flex flex-wrap items-center justify-between gap-2 border-t border-stroke-default pt-3">
<AppButton
variant="secondary"
:disabled="togglingUserId === activeUser.id"
@click="onToggleUser(activeUser)"
>
{{ activeUser.is_active ? "Deactivate user" : "Activate user" }}
</AppButton>
<AppButton variant="ghost" :disabled="togglingUserId === activeUser.id || resettingPassword" @click="closeUserModal">
Close
</AppButton>
</div>
</div>
</AppModal>
</AppPage>
</template>

View file

@ -1,5 +1,5 @@
<script setup lang="ts"> <script setup lang="ts">
import { computed, onMounted, onUnmounted, ref, watch } from "vue"; import { computed, onMounted, ref, watch } from "vue";
import { fetchDashboardSnapshot, type DashboardSnapshot } from "@/features/dashboard"; import { fetchDashboardSnapshot, type DashboardSnapshot } from "@/features/dashboard";
import { ApiRequestError } from "@/lib/api/errors"; import { ApiRequestError } from "@/lib/api/errors";
@ -14,7 +14,7 @@ import AppCard from "@/components/ui/AppCard.vue";
import AppEmptyState from "@/components/ui/AppEmptyState.vue"; import AppEmptyState from "@/components/ui/AppEmptyState.vue";
import AppHelpHint from "@/components/ui/AppHelpHint.vue"; import AppHelpHint from "@/components/ui/AppHelpHint.vue";
import { useAuthStore } from "@/stores/auth"; import { useAuthStore } from "@/stores/auth";
import { RUN_STATUS_POLL_INTERVAL_MS, useRunStatusStore } from "@/stores/run_status"; import { useRunStatusStore } from "@/stores/run_status";
import { useUserSettingsStore } from "@/stores/user_settings"; import { useUserSettingsStore } from "@/stores/user_settings";
const loading = ref(true); const loading = ref(true);
@ -26,7 +26,6 @@ const refreshingAfterCompletion = ref(false);
const auth = useAuthStore(); const auth = useAuthStore();
const runStatus = useRunStatusStore(); const runStatus = useRunStatusStore();
const userSettings = useUserSettingsStore(); const userSettings = useUserSettingsStore();
let latestRunSyncTimer: ReturnType<typeof setInterval> | null = null;
const isStartBlocked = computed( const isStartBlocked = computed(
() => () =>
@ -135,23 +134,6 @@ function shouldRefreshAfterRunChange(
return previousRun.status === "running"; return previousRun.status === "running";
} }
function startLatestRunSyncLoop(): void {
if (latestRunSyncTimer !== null) {
return;
}
latestRunSyncTimer = setInterval(() => {
void runStatus.syncLatest();
}, RUN_STATUS_POLL_INTERVAL_MS);
}
function stopLatestRunSyncLoop(): void {
if (latestRunSyncTimer === null) {
return;
}
clearInterval(latestRunSyncTimer);
latestRunSyncTimer = null;
}
async function loadSnapshot(): Promise<void> { async function loadSnapshot(): Promise<void> {
loading.value = true; loading.value = true;
errorMessage.value = null; errorMessage.value = null;
@ -207,11 +189,6 @@ async function onTriggerRun(): Promise<void> {
onMounted(() => { onMounted(() => {
void loadSnapshot(); void loadSnapshot();
void runStatus.syncLatest(); void runStatus.syncLatest();
startLatestRunSyncLoop();
});
onUnmounted(() => {
stopLatestRunSyncLoop();
}); });
watch( watch(
@ -266,7 +243,12 @@ watch(
</div> </div>
<p class="text-sm text-secondary">Newest papers found while checking your tracked scholar profiles.</p> <p class="text-sm text-secondary">Newest papers found while checking your tracked scholar profiles.</p>
</div> </div>
<RouterLink to="/publications" class="link-inline text-sm">Open full publications view</RouterLink> <RouterLink
to="/publications"
class="inline-flex min-h-10 items-center justify-center rounded-lg border border-action-secondary-border bg-action-secondary-bg px-3 py-2 text-sm font-semibold text-action-secondary-text shadow-sm transition hover:border-action-secondary-hover-border hover:bg-action-secondary-hover-bg hover:text-action-secondary-hover-text focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset"
>
view all
</RouterLink>
</div> </div>
<AppEmptyState <AppEmptyState

View file

@ -3,9 +3,11 @@ import { computed, ref } from "vue";
import { useRouter } from "vue-router"; import { useRouter } from "vue-router";
import AppAlert from "@/components/ui/AppAlert.vue"; import AppAlert from "@/components/ui/AppAlert.vue";
import AppBrandMark from "@/components/ui/AppBrandMark.vue";
import AppButton from "@/components/ui/AppButton.vue"; import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue"; import AppCard from "@/components/ui/AppCard.vue";
import AppInput from "@/components/ui/AppInput.vue"; import AppInput from "@/components/ui/AppInput.vue";
import AppThemePicker from "@/components/ui/AppThemePicker.vue";
import { ApiRequestError } from "@/lib/api/errors"; import { ApiRequestError } from "@/lib/api/errors";
import { useAuthStore } from "@/stores/auth"; import { useAuthStore } from "@/stores/auth";
@ -54,46 +56,22 @@ async function onSubmit(): Promise<void> {
</script> </script>
<template> <template>
<div class="relative h-[100dvh] max-h-[100dvh] overflow-y-auto overflow-x-hidden bg-surface-app"> <div class="relative h-[100dvh] max-h-[100dvh] overflow-hidden bg-surface-app">
<div class="pointer-events-none absolute inset-0"> <div class="absolute right-4 top-4 z-10 sm:right-6 sm:top-6">
<div class="absolute -top-20 left-[-8rem] h-72 w-72 rounded-full bg-brand-300/30 blur-3xl" /> <AppThemePicker compact id-prefix="login-theme" />
<div class="absolute bottom-[-7rem] right-[-6rem] h-80 w-80 rounded-full bg-success-300/25 blur-3xl" />
<div class="absolute left-1/3 top-1/3 h-52 w-52 rounded-full bg-warning-300/20 blur-3xl" />
</div> </div>
<div <div class="pointer-events-none absolute inset-0">
class="relative mx-auto grid min-h-full w-full max-w-6xl items-center gap-8 px-4 py-6 sm:px-6 lg:grid-cols-[minmax(0,1fr)_26rem] lg:px-8" <div class="absolute -top-28 right-[-8rem] h-72 w-72 rounded-full bg-brand-300/25 blur-3xl" />
> <div class="absolute bottom-[-8rem] left-[-7rem] h-80 w-80 rounded-full bg-info-300/20 blur-3xl" />
<section class="hidden space-y-5 lg:block"> </div>
<p class="inline-flex items-center rounded-full border border-stroke-strong bg-surface-card/70 px-3 py-1 text-xs font-medium uppercase tracking-[0.12em] text-ink-muted">
Scholarr Control Center
</p>
<h1 class="max-w-xl font-display text-4xl font-semibold tracking-tight text-ink-primary">
Scholar tracking with reliable operational controls.
</h1>
<p class="max-w-xl text-base leading-relaxed text-ink-muted">
Use your account to review ingestion runs, publication changes, and continuation queue diagnostics from a
single workspace.
</p>
<ul class="grid max-w-xl gap-2 text-sm text-ink-muted">
<li class="rounded-lg border border-stroke-default bg-surface-card/60 px-3 py-2">
Session-based authentication with CSRF enforcement.
</li>
<li class="rounded-lg border border-stroke-default bg-surface-card/60 px-3 py-2">
Run and queue diagnostics are available for support workflows.
</li>
<li class="rounded-lg border border-stroke-default bg-surface-card/60 px-3 py-2">
Theme-aware interface with light and dark mode support.
</li>
</ul>
</section>
<AppCard class="w-full max-w-md justify-self-end space-y-6 border-stroke-default/80 bg-surface-card/90 backdrop-blur"> <div class="relative mx-auto grid h-full w-full max-w-md items-center px-4 py-8 sm:px-6">
<div class="space-y-2"> <AppCard class="space-y-6 border-stroke-default/80 bg-surface-card/90 p-6 backdrop-blur sm:p-7">
<h1 class="font-display text-3xl font-semibold tracking-tight text-ink-primary">Sign In</h1> <div class="grid justify-items-center gap-2 text-center">
<p class="text-sm text-secondary"> <AppBrandMark size="xl" />
Sign in to access scholar tracking, publication updates, and run diagnostics. <p class="font-display text-2xl font-semibold tracking-tight text-ink-primary">scholarr</p>
</p> <h1 class="text-sm font-medium uppercase tracking-[0.12em] text-ink-secondary">Sign in</h1>
</div> </div>
<AppAlert v-if="errorMessage" tone="danger"> <AppAlert v-if="errorMessage" tone="danger">
@ -106,7 +84,7 @@ async function onSubmit(): Promise<void> {
<form class="grid gap-4" @submit.prevent="onSubmit"> <form class="grid gap-4" @submit.prevent="onSubmit">
<label class="grid gap-2 text-sm font-medium text-ink-secondary"> <label class="grid gap-2 text-sm font-medium text-ink-secondary">
<span>Email</span> <span>Email</span>
<AppInput id="login-email" v-model="email" type="email" autocomplete="email" /> <AppInput id="login-email" v-model="email" type="email" autocomplete="email" autofocus />
</label> </label>
<label class="grid gap-2 text-sm font-medium text-ink-secondary"> <label class="grid gap-2 text-sm font-medium text-ink-secondary">
@ -123,6 +101,10 @@ async function onSubmit(): Promise<void> {
{{ pending ? "Signing in..." : "Sign in" }} {{ pending ? "Signing in..." : "Sign in" }}
</AppButton> </AppButton>
</form> </form>
<p class="text-center text-xs text-secondary">
Use your assigned account credentials.
</p>
</AppCard> </AppCard>
</div> </div>
</div> </div>

View file

@ -10,6 +10,7 @@ import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue"; import AppCard from "@/components/ui/AppCard.vue";
import AppHelpHint from "@/components/ui/AppHelpHint.vue"; import AppHelpHint from "@/components/ui/AppHelpHint.vue";
import AppInput from "@/components/ui/AppInput.vue"; import AppInput from "@/components/ui/AppInput.vue";
import AppRefreshButton from "@/components/ui/AppRefreshButton.vue";
import AppSelect from "@/components/ui/AppSelect.vue"; import AppSelect from "@/components/ui/AppSelect.vue";
import AppTable from "@/components/ui/AppTable.vue"; import AppTable from "@/components/ui/AppTable.vue";
import { import {
@ -17,28 +18,50 @@ import {
markAllRead, markAllRead,
markSelectedRead, markSelectedRead,
retryPublicationPdf, retryPublicationPdf,
togglePublicationFavorite,
type PublicationItem, type PublicationItem,
type PublicationMode, type PublicationMode,
type PublicationsResult, type PublicationsResult,
} from "@/features/publications"; } from "@/features/publications";
import { listScholars, type ScholarProfile } from "@/features/scholars"; import { listScholars, type ScholarProfile } from "@/features/scholars";
import { ApiRequestError } from "@/lib/api/errors"; import { ApiRequestError } from "@/lib/api/errors";
import { useRunStatusStore } from "@/stores/run_status";
import { useUserSettingsStore } from "@/stores/user_settings";
type PublicationSortKey = "title" | "scholar" | "year" | "citations" | "status" | "first_seen"; type PublicationSortKey =
| "title"
| "favorite"
| "scholar"
| "year"
| "citations"
| "status"
| "first_seen";
type BulkAction =
| "mark_selected_read"
| "mark_all_unread_read"
| "select_all_visible_unread"
| "clear_selection";
type BulkActionOption = { value: BulkAction; label: string };
const loading = ref(true); const loading = ref(true);
const publishingAll = ref(false); const publishingAll = ref(false);
const publishingSelected = ref(false); const publishingSelected = ref(false);
const mode = ref<PublicationMode>("all"); const mode = ref<PublicationMode>("all");
const favoriteOnly = ref(false);
const bulkAction = ref<BulkAction>("mark_all_unread_read");
const selectedScholarFilter = ref(""); const selectedScholarFilter = ref("");
const searchQuery = ref(""); const searchQuery = ref("");
const sortKey = ref<PublicationSortKey>("first_seen"); const sortKey = ref<PublicationSortKey>("first_seen");
const sortDirection = ref<"asc" | "desc">("desc"); const sortDirection = ref<"asc" | "desc">("desc");
const currentPage = ref(1);
const pageSize = ref("50");
const scholars = ref<ScholarProfile[]>([]); const scholars = ref<ScholarProfile[]>([]);
const listState = ref<PublicationsResult | null>(null); const listState = ref<PublicationsResult | null>(null);
const selectedPublicationKeys = ref<Set<string>>(new Set()); const selectedPublicationKeys = ref<Set<string>>(new Set());
const retryingPublicationKeys = ref<Set<string>>(new Set()); const retryingPublicationKeys = ref<Set<string>>(new Set());
const favoriteUpdatingKeys = ref<Set<string>>(new Set());
const errorMessage = ref<string | null>(null); const errorMessage = ref<string | null>(null);
const errorRequestId = ref<string | null>(null); const errorRequestId = ref<string | null>(null);
@ -46,43 +69,84 @@ const successMessage = ref<string | null>(null);
const route = useRoute(); const route = useRoute();
const router = useRouter(); const router = useRouter();
const textCollator = new Intl.Collator(undefined, { sensitivity: "base", numeric: true }); const textCollator = new Intl.Collator(undefined, { sensitivity: "base", numeric: true });
const runStatus = useRunStatusStore();
const userSettings = useUserSettingsStore();
function normalizeScholarFilterQuery(value: unknown): string { function normalizeScholarFilterQuery(value: unknown): string {
if (Array.isArray(value)) { if (Array.isArray(value)) {
return normalizeScholarFilterQuery(value[0]); return normalizeScholarFilterQuery(value[0]);
} }
if (typeof value !== "string") { if (typeof value !== "string") {
return ""; return "";
} }
const parsed = Number(value); const parsed = Number(value);
return Number.isInteger(parsed) && parsed > 0 ? String(parsed) : ""; return Number.isInteger(parsed) && parsed > 0 ? String(parsed) : "";
} }
function syncScholarFilterFromRoute(): boolean { function normalizeFavoriteOnlyQuery(value: unknown): boolean {
const nextValue = normalizeScholarFilterQuery(route.query.scholar); if (Array.isArray(value)) {
if (selectedScholarFilter.value === nextValue) { return normalizeFavoriteOnlyQuery(value[0]);
}
if (typeof value !== "string") {
return false; return false;
} }
const normalized = value.trim().toLowerCase();
selectedScholarFilter.value = nextValue; return normalized === "1" || normalized === "true" || normalized === "yes";
return true;
} }
async function syncScholarFilterToRoute(): Promise<void> { function normalizePageQuery(value: unknown): number {
const nextValue = selectedScholarFilter.value.trim(); if (Array.isArray(value)) {
const currentValue = normalizeScholarFilterQuery(route.query.scholar); return normalizePageQuery(value[0]);
if (nextValue === currentValue) { }
if (typeof value !== "string") {
return 1;
}
const parsed = Number(value);
return Number.isInteger(parsed) && parsed > 0 ? parsed : 1;
}
function syncFiltersFromRoute(): boolean {
let changed = false;
const nextScholar = normalizeScholarFilterQuery(route.query.scholar);
const nextFavoriteOnly = normalizeFavoriteOnlyQuery(route.query.favorite);
const nextPage = normalizePageQuery(route.query.page);
if (selectedScholarFilter.value !== nextScholar) {
selectedScholarFilter.value = nextScholar;
changed = true;
}
if (favoriteOnly.value !== nextFavoriteOnly) {
favoriteOnly.value = nextFavoriteOnly;
changed = true;
}
if (currentPage.value !== nextPage) {
currentPage.value = nextPage;
changed = true;
}
return changed;
}
async function syncFiltersToRoute(): Promise<void> {
const nextScholar = selectedScholarFilter.value.trim();
const currentScholar = normalizeScholarFilterQuery(route.query.scholar);
const currentFavoriteOnly = normalizeFavoriteOnlyQuery(route.query.favorite);
const currentPageQuery = normalizePageQuery(route.query.page);
if (
nextScholar === currentScholar
&& favoriteOnly.value === currentFavoriteOnly
&& currentPage.value === currentPageQuery
) {
return; return;
} }
const nextQuery = { await router.replace({
query: {
...route.query, ...route.query,
scholar: nextValue || undefined, scholar: nextScholar || undefined,
}; favorite: favoriteOnly.value ? "1" : undefined,
page: currentPage.value > 1 ? String(currentPage.value) : undefined,
await router.replace({ query: nextQuery }); },
});
} }
function formatDate(value: string): string { function formatDate(value: string): string {
@ -127,7 +191,6 @@ const filteredPublications = computed(() => {
if (!normalized) { if (!normalized) {
return base; return base;
} }
return base.filter((item) => { return base.filter((item) => {
const year = item.year === null ? "" : String(item.year); const year = item.year === null ? "" : String(item.year);
return [item.title, item.scholar_label, item.venue_text || "", year] return [item.title, item.scholar_label, item.venue_text || "", year]
@ -141,6 +204,9 @@ function publicationSortValue(item: PublicationItem, key: PublicationSortKey): n
if (key === "title") { if (key === "title") {
return item.title; return item.title;
} }
if (key === "favorite") {
return item.is_favorite ? 1 : 0;
}
if (key === "scholar") { if (key === "scholar") {
return item.scholar_label; return item.scholar_label;
} }
@ -194,12 +260,119 @@ const visibleUnreadKeys = computed(() => {
return keys; return keys;
}); });
const pageSizeValue = computed(() => {
const parsed = Number(pageSize.value);
if (!Number.isInteger(parsed)) {
return 50;
}
return Math.max(10, Math.min(200, parsed));
});
const hasNextPage = computed(() => Boolean(listState.value?.has_next));
const hasPrevPage = computed(() => Boolean(listState.value?.has_prev));
const totalPages = computed(() => {
if (!listState.value) {
return 1;
}
return Math.max(1, Math.ceil(listState.value.total_count / Math.max(listState.value.page_size, 1)));
});
const selectedCount = computed(() => selectedPublicationKeys.value.size); const selectedCount = computed(() => selectedPublicationKeys.value.size);
const visibleCount = computed(() => sortedPublications.value.length); const visibleCount = computed(() => sortedPublications.value.length);
const visibleUnreadCount = computed(() => visibleUnreadKeys.value.size); const visibleUnreadCount = computed(() => visibleUnreadKeys.value.size);
const visibleFavoriteCount = computed(
() => sortedPublications.value.filter((item) => item.is_favorite).length,
);
const actionBusy = computed(() => loading.value || publishingAll.value || publishingSelected.value); const actionBusy = computed(() => loading.value || publishingAll.value || publishingSelected.value);
const startRunDisabledReason = computed(() => {
if (!userSettings.manualRunAllowed) {
return "Manual runs are disabled by policy.";
}
if (runStatus.safetyState.cooldown_active) {
return runStatus.safetyState.cooldown_reason_label || "Safety cooldown is active.";
}
if (runStatus.isRunActive) {
return "A run is already in progress.";
}
return null;
});
const startRunButtonLabel = computed(() => {
if (runStatus.isSubmitting) {
return "Starting...";
}
if (!userSettings.manualRunAllowed) {
return "Manual runs disabled";
}
if (runStatus.safetyState.cooldown_active) {
return "Safety cooldown";
}
if (runStatus.isLikelyRunning) {
return "Run active";
}
return "Start run";
});
const startRunDisabled = computed(() => Boolean(startRunDisabledReason.value) || runStatus.isSubmitting);
const hasSelection = computed(() => selectedCount.value > 0);
const bulkActionOptions = computed<BulkActionOption[]>(() => {
if (hasSelection.value) {
return [
{
value: "mark_selected_read",
label: `Mark selected as read (${selectedCount.value})`,
},
{
value: "clear_selection",
label: `Clear selection (${selectedCount.value})`,
},
];
}
return [
{ value: "mark_all_unread_read", label: "Mark all unread as read" },
{ value: "select_all_visible_unread", label: "Select all visible unread" },
];
});
const bulkApplyDisabled = computed(() => {
if (actionBusy.value) {
return true;
}
if (bulkAction.value === "mark_selected_read" || bulkAction.value === "clear_selection") {
return selectedCount.value === 0;
}
if (bulkAction.value === "mark_all_unread_read" || bulkAction.value === "select_all_visible_unread") {
return visibleUnreadCount.value === 0;
}
return false;
});
const bulkApplyLabel = computed(() => {
if (publishingSelected.value || publishingAll.value) {
return "Applying...";
}
if (bulkAction.value === "select_all_visible_unread") {
return "Select";
}
if (bulkAction.value === "clear_selection") {
return "Clear";
}
return "Apply";
});
const showingEmptyList = computed(() => Boolean(listState.value) && sortedPublications.value.length === 0); const showingEmptyList = computed(() => Boolean(listState.value) && sortedPublications.value.length === 0);
const modeLabel = computed(() => (mode.value === "unread" ? "Unread" : "All records")); const scopeLabel = computed(() => {
if (mode.value === "latest" && favoriteOnly.value) {
return "New + Favorites";
}
if (mode.value === "latest") {
return "New";
}
if (mode.value === "unread" && favoriteOnly.value) {
return "Unread + Favorites";
}
if (mode.value === "unread") {
return "Unread";
}
if (favoriteOnly.value) {
return "All + Favorites";
}
return "All";
});
const emptyTitle = computed(() => const emptyTitle = computed(() =>
searchQuery.value.trim().length > 0 ? "No publications match this search" : "No publications found", searchQuery.value.trim().length > 0 ? "No publications match this search" : "No publications found",
@ -209,6 +382,18 @@ const emptyBody = computed(() => {
if (searchQuery.value.trim().length > 0) { if (searchQuery.value.trim().length > 0) {
return "Try another title, scholar, venue, or year."; return "Try another title, scholar, venue, or year.";
} }
if (favoriteOnly.value && mode.value === "latest") {
return `No new favorites for ${selectedScholarName.value}.`;
}
if (mode.value === "latest") {
return `No new publications in the latest run for ${selectedScholarName.value}.`;
}
if (favoriteOnly.value && mode.value === "unread") {
return `No unread favorites for ${selectedScholarName.value}.`;
}
if (favoriteOnly.value) {
return `No favorites for ${selectedScholarName.value}.`;
}
if (mode.value === "unread") { if (mode.value === "unread") {
return `No unread publications for ${selectedScholarName.value}.`; return `No unread publications for ${selectedScholarName.value}.`;
} }
@ -240,6 +425,14 @@ watch(sortedPublications, (items) => {
} }
}); });
watch(hasSelection, (nextHasSelection) => {
const validValues = new Set(bulkActionOptions.value.map((item) => item.value));
if (validValues.has(bulkAction.value)) {
return;
}
bulkAction.value = nextHasSelection ? "mark_selected_read" : "mark_all_unread_read";
});
function toggleSort(nextKey: PublicationSortKey): void { function toggleSort(nextKey: PublicationSortKey): void {
if (sortKey.value === nextKey) { if (sortKey.value === nextKey) {
sortDirection.value = sortDirection.value === "asc" ? "desc" : "asc"; sortDirection.value = sortDirection.value === "asc" ? "desc" : "asc";
@ -277,9 +470,13 @@ async function loadPublications(): Promise<void> {
try { try {
listState.value = await listPublications({ listState.value = await listPublications({
mode: mode.value, mode: mode.value,
favoriteOnly: favoriteOnly.value,
scholarProfileId: selectedScholarId(), scholarProfileId: selectedScholarId(),
limit: 400, page: currentPage.value,
pageSize: pageSizeValue.value,
}); });
currentPage.value = listState.value.page;
pageSize.value = String(listState.value.page_size);
selectedPublicationKeys.value = new Set(); selectedPublicationKeys.value = new Set();
} catch (error) { } catch (error) {
listState.value = null; listState.value = null;
@ -295,19 +492,76 @@ async function loadPublications(): Promise<void> {
} }
} }
async function setMode(nextMode: PublicationMode): Promise<void> { async function onModeChanged(): Promise<void> {
if (mode.value === nextMode) { currentPage.value = 1;
return; await syncFiltersToRoute();
}
mode.value = nextMode;
await loadPublications(); await loadPublications();
} }
async function onScholarFilterChanged(): Promise<void> { async function onScholarFilterChanged(): Promise<void> {
await syncScholarFilterToRoute(); currentPage.value = 1;
await syncFiltersToRoute();
await loadPublications(); await loadPublications();
} }
async function onFavoriteOnlyChanged(): Promise<void> {
favoriteOnly.value = !favoriteOnly.value;
currentPage.value = 1;
await syncFiltersToRoute();
await loadPublications();
}
async function onPageSizeChanged(): Promise<void> {
currentPage.value = 1;
await syncFiltersToRoute();
await loadPublications();
}
async function onPrevPage(): Promise<void> {
if (!hasPrevPage.value || currentPage.value <= 1) {
return;
}
currentPage.value -= 1;
await syncFiltersToRoute();
await loadPublications();
}
async function onNextPage(): Promise<void> {
if (!hasNextPage.value) {
return;
}
currentPage.value += 1;
await syncFiltersToRoute();
await loadPublications();
}
function selectAllVisibleUnread(): void {
selectedPublicationKeys.value = new Set(visibleUnreadKeys.value);
}
function clearSelection(): void {
selectedPublicationKeys.value = new Set();
}
async function onApplyBulkAction(): Promise<void> {
if (bulkApplyDisabled.value) {
return;
}
if (bulkAction.value === "select_all_visible_unread") {
selectAllVisibleUnread();
return;
}
if (bulkAction.value === "clear_selection") {
clearSelection();
return;
}
if (bulkAction.value === "mark_selected_read") {
await onMarkSelectedRead();
return;
}
await onMarkAllRead();
}
function onToggleAllVisible(event: Event): void { function onToggleAllVisible(event: Event): void {
const checked = (event.target as HTMLInputElement).checked; const checked = (event.target as HTMLInputElement).checked;
const next = new Set(selectedPublicationKeys.value); const next = new Set(selectedPublicationKeys.value);
@ -337,17 +591,84 @@ function isRetryingPublication(item: PublicationItem): boolean {
return retryingPublicationKeys.value.has(publicationKey(item)); return retryingPublicationKeys.value.has(publicationKey(item));
} }
function isFavoriteUpdating(item: PublicationItem): boolean {
return favoriteUpdatingKeys.value.has(publicationKey(item));
}
function canRetryPublicationPdf(item: PublicationItem): boolean {
return !item.pdf_url && item.pdf_status === "failed" && item.pdf_attempt_count > 0;
}
function pdfPendingLabel(item: PublicationItem): string {
if (item.pdf_status === "queued") {
return "queued";
}
if (item.pdf_status === "running") {
return "resolving";
}
if (item.pdf_status === "failed") {
return "failed";
}
return "untracked";
}
function replacePublication(updated: PublicationItem): void { function replacePublication(updated: PublicationItem): void {
if (!listState.value) { if (!listState.value) {
return; return;
} }
listState.value.publications = listState.value.publications.map((item) => listState.value.publications = listState.value.publications.map((item) => {
publicationKey(item) === publicationKey(updated) ? updated : item, if (publicationKey(item) !== publicationKey(updated)) {
return item;
}
return updated;
});
}
function withFavorite(item: PublicationItem, isFavorite: boolean): PublicationItem {
return {
...item,
is_favorite: isFavorite,
};
}
async function onToggleFavorite(item: PublicationItem): Promise<void> {
if (isFavoriteUpdating(item)) {
return;
}
const key = publicationKey(item);
const nextFavorite = !item.is_favorite;
const nextUpdating = new Set(favoriteUpdatingKeys.value);
nextUpdating.add(key);
favoriteUpdatingKeys.value = nextUpdating;
errorMessage.value = null;
errorRequestId.value = null;
replacePublication(withFavorite(item, nextFavorite));
try {
const response = await togglePublicationFavorite(
item.publication_id,
item.scholar_profile_id,
nextFavorite,
); );
replacePublication(response.publication);
} catch (error) {
replacePublication(withFavorite(item, item.is_favorite));
if (error instanceof ApiRequestError) {
errorMessage.value = error.message;
errorRequestId.value = error.requestId;
} else {
errorMessage.value = "Unable to update favorite state.";
}
} finally {
const cleared = new Set(favoriteUpdatingKeys.value);
cleared.delete(key);
favoriteUpdatingKeys.value = cleared;
}
} }
async function onRetryPdf(item: PublicationItem): Promise<void> { async function onRetryPdf(item: PublicationItem): Promise<void> {
if (item.pdf_url) { if (item.pdf_url || !canRetryPublicationPdf(item)) {
return; return;
} }
const key = publicationKey(item); const key = publicationKey(item);
@ -432,19 +753,42 @@ async function onMarkAllRead(): Promise<void> {
} }
} }
async function onStartRun(): Promise<void> {
if (startRunDisabled.value) {
return;
}
successMessage.value = null;
errorMessage.value = null;
errorRequestId.value = null;
const result = await runStatus.startManualCheck();
if (result.kind === "started") {
successMessage.value = `Run #${result.runId} started.`;
return;
}
if (result.kind === "already_running") {
successMessage.value = result.runId
? `Run #${result.runId} is already in progress.`
: "A run is already in progress.";
return;
}
errorMessage.value = result.message;
errorRequestId.value = result.requestId;
}
function resetSearchQuery(): void { function resetSearchQuery(): void {
searchQuery.value = ""; searchQuery.value = "";
} }
onMounted(() => { onMounted(() => {
syncScholarFilterFromRoute(); syncFiltersFromRoute();
void Promise.all([loadScholarFilters(), loadPublications()]); void Promise.all([loadScholarFilters(), loadPublications(), runStatus.syncLatest()]);
}); });
watch( watch(
() => route.query.scholar, () => [route.query.scholar, route.query.favorite, route.query.page],
async () => { async () => {
const changed = syncScholarFilterFromRoute(); const changed = syncFiltersFromRoute();
if (!changed) { if (!changed) {
return; return;
} }
@ -454,59 +798,25 @@ watch(
</script> </script>
<template> <template>
<AppPage <AppPage title="Publications" subtitle="Review discoveries, open PDFs, and update read/favorite state in one workspace.">
title="Publications"
subtitle="Review discoveries, open PDFs, and keep read state current."
>
<AppCard class="space-y-4"> <AppCard class="space-y-4">
<div class="flex flex-wrap items-center justify-between gap-2"> <div class="grid gap-3 xl:grid-cols-[minmax(0,13rem)_minmax(0,18rem)_minmax(0,1fr)_auto] xl:items-end">
<div class="space-y-1">
<div class="flex items-center gap-1">
<h2 class="text-lg font-semibold text-ink-primary">Publications workspace</h2>
<AppHelpHint
text="Use this workspace to filter discoveries, open OA PDF links, and mark reviewed papers as read so future checks stay focused."
/>
</div>
<p class="text-sm text-secondary">
Choose scope, triage results, and update read state in one place.
</p>
</div>
<AppButton variant="ghost" :disabled="loading" @click="loadPublications">
{{ loading ? "Refreshing..." : "Refresh" }}
</AppButton>
</div>
<div class="grid gap-3 xl:grid-cols-[minmax(0,15rem)_minmax(0,18rem)_minmax(0,1fr)] xl:items-end">
<div class="grid gap-1 text-xs text-secondary"> <div class="grid gap-1 text-xs text-secondary">
<div class="flex items-center gap-1"> <div class="flex items-center gap-1">
<span>View mode</span> <span>Status</span>
<AppHelpHint text="Unread shows only unreviewed records; All records includes the full publication history." /> <AppHelpHint text="All shows full history. Unread and New narrow the dataset before search." />
</div>
<div class="flex min-h-10 flex-wrap items-center gap-2">
<AppButton
:variant="mode === 'unread' ? 'primary' : 'secondary'"
class="min-w-16 justify-center"
:disabled="actionBusy"
@click="setMode('unread')"
>
Unread
</AppButton>
<AppButton
:variant="mode === 'all' ? 'primary' : 'secondary'"
class="min-w-16 justify-center"
:disabled="actionBusy"
@click="setMode('all')"
>
All records
</AppButton>
</div> </div>
<AppSelect v-model="mode" :disabled="actionBusy" @change="onModeChanged">
<option value="all">All records</option>
<option value="unread">Unread</option>
<option value="latest">New (latest run)</option>
</AppSelect>
</div> </div>
<label class="grid gap-1 text-xs text-secondary" for="publications-scholar-filter"> <label class="grid gap-1 text-xs text-secondary" for="publications-scholar-filter">
<span class="inline-flex items-center gap-1"> <span class="inline-flex items-center gap-1">
Scholar Scholar
<AppHelpHint text="Filter results to one tracked scholar profile. The selected scholar is synced to the URL query." /> <AppHelpHint text="Filter to one tracked scholar profile. Filter is synced to URL query." />
</span> </span>
<AppSelect <AppSelect
id="publications-scholar-filter" id="publications-scholar-filter"
@ -523,8 +833,8 @@ watch(
<label class="grid gap-1 text-xs text-secondary" for="publications-search-input"> <label class="grid gap-1 text-xs text-secondary" for="publications-search-input">
<span class="inline-flex items-center gap-1"> <span class="inline-flex items-center gap-1">
Search current results Search
<AppHelpHint text="Search matches title, scholar, venue, and year within the currently loaded scope." /> <AppHelpHint text="Searches title, scholar, venue, and year within currently loaded results." />
</span> </span>
<div class="flex min-w-0 items-center gap-2"> <div class="flex min-w-0 items-center gap-2">
<AppInput <AppInput
@ -544,27 +854,36 @@ watch(
</AppButton> </AppButton>
</div> </div>
</label> </label>
</div>
<div class="flex flex-wrap items-center justify-between gap-2 border-t border-stroke-default pt-3"> <div class="flex flex-wrap items-end justify-end gap-2">
<div class="flex flex-wrap items-center gap-2"> <button
<AppBadge tone="info">Mode: {{ modeLabel }}</AppBadge> type="button"
<AppBadge tone="neutral">Visible: {{ visibleCount }}</AppBadge> class="favorite-filter-button"
<AppBadge tone="warning">Unread: {{ visibleUnreadCount }}</AppBadge> :class="favoriteOnly ? 'favorite-filter-on' : 'favorite-filter-off'"
<AppBadge tone="success">Selected: {{ selectedCount }}</AppBadge> :disabled="actionBusy"
</div> :title="favoriteOnly ? 'Favorites-only filter is active' : 'Show only favorites'"
:aria-pressed="favoriteOnly"
<div class="flex flex-wrap items-center gap-2"> :aria-label="favoriteOnly ? 'Disable favorites-only filter' : 'Enable favorites-only filter'"
@click="onFavoriteOnlyChanged"
>
<span aria-hidden="true">{{ favoriteOnly ? "★" : "☆" }}</span>
</button>
<AppButton <AppButton
variant="secondary" variant="secondary"
:disabled="selectedCount === 0 || actionBusy" :disabled="startRunDisabled"
@click="onMarkSelectedRead" :title="startRunDisabledReason || undefined"
@click="onStartRun"
> >
{{ publishingSelected ? "Updating..." : `Mark selected as read (${selectedCount})` }} {{ startRunButtonLabel }}
</AppButton>
<AppButton variant="secondary" :disabled="actionBusy || visibleUnreadCount === 0" @click="onMarkAllRead">
{{ publishingAll ? "Updating..." : "Mark all unread as read" }}
</AppButton> </AppButton>
<AppRefreshButton
variant="ghost"
:disabled="loading"
:loading="loading"
title="Refresh publications"
loading-title="Refreshing publications"
@click="loadPublications"
/>
</div> </div>
</div> </div>
@ -577,15 +896,53 @@ watch(
@dismiss-success="successMessage = null" @dismiss-success="successMessage = null"
/> />
<section class="space-y-3 border-t border-stroke-default pt-3"> <div class="flex flex-wrap items-center justify-between gap-2 border-t border-stroke-default pt-3">
<div class="flex flex-wrap items-center justify-between gap-2">
<div class="flex items-center gap-1"> <div class="flex items-center gap-1">
<h3 class="text-base font-semibold text-ink-primary">Results</h3> <h3 class="text-base font-semibold text-ink-primary">Results</h3>
<AppHelpHint <AppHelpHint
text="Sort columns to prioritize review. PDF opens a resolved open-access link, while retry triggers a new lookup for that row." text="Star to favorite, open PDF when available, and retry only after failed PDF lookup attempts."
/> />
</div> </div>
<span class="text-xs text-secondary">Currently showing {{ selectedScholarName }}</span> <div class="flex flex-wrap items-center justify-end gap-2 text-[11px] text-secondary">
<span>Scope: {{ scopeLabel }} | Showing {{ selectedScholarName }}</span>
<div class="flex items-center gap-1">
<label for="publications-page-size" class="sr-only">Page size</label>
<AppSelect
id="publications-page-size"
v-model="pageSize"
:disabled="loading"
class="max-w-[8rem] !py-1.5 !text-xs"
@change="onPageSizeChanged"
>
<option value="25">25 / page</option>
<option value="50">50 / page</option>
<option value="100">100 / page</option>
<option value="200">200 / page</option>
</AppSelect>
<AppHelpHint
text="When no rows are selected, actions target all unread results. After selecting rows, the menu switches to selected-row actions."
/>
<label for="publications-bulk-action" class="sr-only">Bulk task</label>
<AppSelect
id="publications-bulk-action"
v-model="bulkAction"
:disabled="actionBusy"
class="max-w-[14rem] !py-1.5 !text-xs"
>
<option v-for="option in bulkActionOptions" :key="option.value" :value="option.value">
{{ option.label }}
</option>
</AppSelect>
<AppButton
variant="secondary"
class="h-8 min-h-8 shrink-0 px-2 text-xs"
:disabled="bulkApplyDisabled"
@click="onApplyBulkAction"
>
{{ bulkApplyLabel }}
</AppButton>
</div>
</div>
</div> </div>
<AsyncStateGate <AsyncStateGate
@ -599,7 +956,7 @@ watch(
<AppTable v-if="listState" label="Publication list table"> <AppTable v-if="listState" label="Publication list table">
<thead> <thead>
<tr> <tr>
<th scope="col" class="w-12"> <th scope="col" class="w-10">
<input <input
type="checkbox" type="checkbox"
class="h-4 w-4 rounded border-stroke-interactive bg-surface-input text-brand-600 focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset" class="h-4 w-4 rounded border-stroke-interactive bg-surface-input text-brand-600 focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset"
@ -609,35 +966,40 @@ watch(
@change="onToggleAllVisible" @change="onToggleAllVisible"
/> />
</th> </th>
<th scope="col"> <th scope="col" class="w-12">
<button type="button" class="table-sort" @click="toggleSort('favorite')">
<span aria-hidden="true" class="sort-marker">{{ sortMarker('favorite') }}</span>
</button>
</th>
<th scope="col" class="w-[44%] min-w-[24rem]">
<button type="button" class="table-sort" @click="toggleSort('title')"> <button type="button" class="table-sort" @click="toggleSort('title')">
Title <span aria-hidden="true">{{ sortMarker("title") }}</span> Title <span aria-hidden="true" class="sort-marker">{{ sortMarker('title') }}</span>
</button> </button>
</th> </th>
<th scope="col"> <th scope="col" class="w-[16%] min-w-[12rem]">
<button type="button" class="table-sort" @click="toggleSort('scholar')"> <button type="button" class="table-sort" @click="toggleSort('scholar')">
Scholar <span aria-hidden="true">{{ sortMarker("scholar") }}</span> Scholar <span aria-hidden="true" class="sort-marker">{{ sortMarker('scholar') }}</span>
</button> </button>
</th> </th>
<th scope="col">PDF</th> <th scope="col" class="w-[8.5rem] whitespace-nowrap">PDF</th>
<th scope="col"> <th scope="col" class="w-16 whitespace-nowrap">
<button type="button" class="table-sort" @click="toggleSort('year')"> <button type="button" class="table-sort" @click="toggleSort('year')">
Year <span aria-hidden="true">{{ sortMarker("year") }}</span> Year <span aria-hidden="true" class="sort-marker">{{ sortMarker('year') }}</span>
</button> </button>
</th> </th>
<th scope="col"> <th scope="col" class="w-24 whitespace-nowrap">
<button type="button" class="table-sort" @click="toggleSort('citations')"> <button type="button" class="table-sort" @click="toggleSort('citations')">
Citations <span aria-hidden="true">{{ sortMarker("citations") }}</span> Citations <span aria-hidden="true" class="sort-marker">{{ sortMarker('citations') }}</span>
</button> </button>
</th> </th>
<th scope="col"> <th scope="col" class="w-44 whitespace-nowrap">
<button type="button" class="table-sort" @click="toggleSort('status')"> <button type="button" class="table-sort" @click="toggleSort('status')">
Read status <span aria-hidden="true">{{ sortMarker("status") }}</span> Read status <span aria-hidden="true" class="sort-marker">{{ sortMarker('status') }}</span>
</button> </button>
</th> </th>
<th scope="col"> <th scope="col" class="w-32 whitespace-nowrap">
<button type="button" class="table-sort" @click="toggleSort('first_seen')"> <button type="button" class="table-sort" @click="toggleSort('first_seen')">
First seen <span aria-hidden="true">{{ sortMarker("first_seen") }}</span> First seen <span aria-hidden="true" class="sort-marker">{{ sortMarker('first_seen') }}</span>
</button> </button>
</th> </th>
</tr> </tr>
@ -655,30 +1017,47 @@ watch(
/> />
</td> </td>
<td> <td>
<div class="grid gap-1"> <button
type="button"
class="favorite-star-button"
:class="item.is_favorite ? 'favorite-star-on' : 'favorite-star-off'"
:aria-label="item.is_favorite ? `Remove ${item.title} from favorites` : `Add ${item.title} to favorites`"
:aria-pressed="item.is_favorite"
:disabled="isFavoriteUpdating(item)"
@click="onToggleFavorite(item)"
>
{{ item.is_favorite ? "★" : "☆" }}
</button>
</td>
<td class="max-w-0">
<div class="grid min-w-0 gap-1">
<a <a
v-if="publicationPrimaryUrl(item)" v-if="publicationPrimaryUrl(item)"
:href="publicationPrimaryUrl(item) || ''" :href="publicationPrimaryUrl(item) || ''"
target="_blank" target="_blank"
rel="noreferrer" rel="noreferrer"
class="link-inline" class="link-inline block truncate font-medium"
:title="item.title"
> >
{{ item.title }} {{ item.title }}
</a> </a>
<span v-else>{{ item.title }}</span> <span v-else class="block truncate font-medium" :title="item.title">{{ item.title }}</span>
<a <a
v-if="publicationDoiUrl(item)" v-if="publicationDoiUrl(item)"
:href="publicationDoiUrl(item) || ''" :href="publicationDoiUrl(item) || ''"
target="_blank" target="_blank"
rel="noreferrer" rel="noreferrer"
class="link-inline text-xs" class="link-inline block truncate text-xs"
:title="`DOI: ${item.doi}`"
> >
DOI: {{ item.doi }} DOI: {{ item.doi }}
</a> </a>
</div> </div>
</td> </td>
<td>{{ item.scholar_label }}</td>
<td> <td>
<span class="block truncate" :title="item.scholar_label">{{ item.scholar_label }}</span>
</td>
<td class="whitespace-nowrap">
<a <a
v-if="item.pdf_url" v-if="item.pdf_url"
:href="item.pdf_url" :href="item.pdf_url"
@ -689,7 +1068,7 @@ watch(
PDF PDF
</a> </a>
<button <button
v-else v-else-if="canRetryPublicationPdf(item)"
type="button" type="button"
class="pdf-retry-button" class="pdf-retry-button"
:disabled="isRetryingPublication(item)" :disabled="isRetryingPublication(item)"
@ -697,32 +1076,76 @@ watch(
> >
{{ isRetryingPublication(item) ? "..." : "retry" }} {{ isRetryingPublication(item) ? "..." : "retry" }}
</button> </button>
<span v-else class="pdf-state-label">{{ pdfPendingLabel(item) }}</span>
</td> </td>
<td>{{ item.year ?? "n/a" }}</td> <td class="whitespace-nowrap">{{ item.year ?? "n/a" }}</td>
<td>{{ item.citation_count }}</td> <td class="whitespace-nowrap">{{ item.citation_count }}</td>
<td> <td>
<div class="flex flex-wrap items-center gap-2"> <div class="status-badges-row">
<AppBadge :tone="item.is_new_in_latest_run ? 'info' : 'neutral'"> <AppBadge :tone="item.is_new_in_latest_run ? 'info' : 'neutral'">
{{ item.is_new_in_latest_run ? "New" : "Seen before" }} {{ item.is_new_in_latest_run ? "New" : "Seen" }}
</AppBadge> </AppBadge>
<AppBadge :tone="item.is_read ? 'success' : 'warning'"> <AppBadge :tone="item.is_read ? 'success' : 'warning'">
{{ item.is_read ? "Read" : "Unread" }} {{ item.is_read ? "Read" : "Unread" }}
</AppBadge> </AppBadge>
</div> </div>
</td> </td>
<td>{{ formatDate(item.first_seen_at) }}</td> <td class="whitespace-nowrap">{{ formatDate(item.first_seen_at) }}</td>
</tr> </tr>
</tbody> </tbody>
</AppTable> </AppTable>
</AsyncStateGate> </AsyncStateGate>
</section>
<div class="flex flex-wrap items-center justify-between gap-2 border-t border-stroke-default pt-2 text-xs text-secondary">
<span>
visible {{ visibleCount }} · unread {{ visibleUnreadCount }} · favorites {{ visibleFavoriteCount }}
· selected {{ selectedCount }}
</span>
<div class="flex items-center gap-2">
<span>page {{ currentPage }} / {{ totalPages }}</span>
<AppButton variant="ghost" class="h-8 min-h-8 px-2" :disabled="!hasPrevPage || loading" @click="onPrevPage">
Prev
</AppButton>
<AppButton variant="ghost" class="h-8 min-h-8 px-2" :disabled="!hasNextPage || loading" @click="onNextPage">
Next
</AppButton>
</div>
</div>
</AppCard> </AppCard>
</AppPage> </AppPage>
</template> </template>
<style scoped> <style scoped>
.table-sort { .table-sort {
@apply inline-flex items-center gap-1 text-left font-semibold text-ink-primary transition hover:text-ink-link focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset; @apply flex w-full items-center justify-between gap-2 text-left font-semibold text-ink-primary transition hover:text-ink-link focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset;
}
.sort-marker {
@apply text-[11px] leading-none text-ink-muted;
}
.favorite-filter-button {
@apply inline-flex min-h-10 h-10 w-10 items-center justify-center rounded-full border text-base leading-none transition focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset disabled:cursor-not-allowed disabled:opacity-50;
}
.favorite-filter-on {
@apply border-warning-300 bg-warning-100 text-warning-700 hover:bg-warning-200;
}
.favorite-filter-off {
@apply border-stroke-default bg-surface-card-muted text-ink-muted hover:border-stroke-interactive hover:text-ink-secondary;
}
.favorite-star-button {
@apply inline-flex h-7 w-7 items-center justify-center rounded-full border text-sm leading-none transition focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset disabled:cursor-not-allowed disabled:opacity-50;
}
.favorite-star-on {
@apply border-warning-300 bg-warning-100 text-warning-700 hover:bg-warning-200;
}
.favorite-star-off {
@apply border-stroke-default bg-surface-card-muted text-ink-muted hover:border-stroke-interactive hover:text-ink-secondary;
} }
.pdf-link-button { .pdf-link-button {
@ -732,4 +1155,12 @@ watch(
.pdf-retry-button { .pdf-retry-button {
@apply inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-medium border-state-warning-border bg-state-warning-bg text-state-warning-text transition hover:brightness-95 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset disabled:cursor-not-allowed disabled:opacity-50; @apply inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-medium border-state-warning-border bg-state-warning-bg text-state-warning-text transition hover:brightness-95 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset disabled:cursor-not-allowed disabled:opacity-50;
} }
.pdf-state-label {
@apply inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-medium border-stroke-default bg-surface-card-muted text-secondary;
}
.status-badges-row {
@apply inline-flex items-center gap-1 whitespace-nowrap;
}
</style> </style>

View file

@ -6,10 +6,10 @@ import AppPage from "@/components/layout/AppPage.vue";
import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue"; import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue"; import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue";
import RunStatusBadge from "@/components/patterns/RunStatusBadge.vue"; import RunStatusBadge from "@/components/patterns/RunStatusBadge.vue";
import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue"; import AppCard from "@/components/ui/AppCard.vue";
import AppEmptyState from "@/components/ui/AppEmptyState.vue"; import AppEmptyState from "@/components/ui/AppEmptyState.vue";
import AppHelpHint from "@/components/ui/AppHelpHint.vue"; import AppHelpHint from "@/components/ui/AppHelpHint.vue";
import AppRefreshButton from "@/components/ui/AppRefreshButton.vue";
import AppTable from "@/components/ui/AppTable.vue"; import AppTable from "@/components/ui/AppTable.vue";
import { getRunDetail, type RunDetail } from "@/features/runs"; import { getRunDetail, type RunDetail } from "@/features/runs";
import { ApiRequestError } from "@/lib/api/errors"; import { ApiRequestError } from "@/lib/api/errors";
@ -108,9 +108,13 @@ onMounted(() => {
</div> </div>
<p class="text-sm text-secondary">Refresh this run detail view after queue/run updates.</p> <p class="text-sm text-secondary">Refresh this run detail view after queue/run updates.</p>
</div> </div>
<AppButton variant="secondary" @click="loadDetail" :disabled="loading"> <AppRefreshButton
{{ loading ? "Refreshing..." : "Refresh" }} variant="secondary"
</AppButton> :loading="loading"
title="Refresh run detail"
loading-title="Refreshing run detail"
@click="loadDetail"
/>
</div> </div>
</AppCard> </AppCard>

View file

@ -11,6 +11,7 @@ import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue"; import AppCard from "@/components/ui/AppCard.vue";
import AppEmptyState from "@/components/ui/AppEmptyState.vue"; import AppEmptyState from "@/components/ui/AppEmptyState.vue";
import AppHelpHint from "@/components/ui/AppHelpHint.vue"; import AppHelpHint from "@/components/ui/AppHelpHint.vue";
import AppRefreshButton from "@/components/ui/AppRefreshButton.vue";
import AppTable from "@/components/ui/AppTable.vue"; import AppTable from "@/components/ui/AppTable.vue";
import { import {
clearQueueItem, clearQueueItem,
@ -210,9 +211,13 @@ onMounted(() => {
<AppButton :disabled="isStartBlocked" :title="startCheckDisabledReason || undefined" @click="onTriggerManualRun"> <AppButton :disabled="isStartBlocked" :title="startCheckDisabledReason || undefined" @click="onTriggerManualRun">
{{ runButtonLabel }} {{ runButtonLabel }}
</AppButton> </AppButton>
<AppButton variant="secondary" :disabled="loading" @click="loadData"> <AppRefreshButton
{{ loading ? "Refreshing..." : "Refresh" }} variant="secondary"
</AppButton> :loading="loading"
title="Refresh runs and queue"
loading-title="Refreshing runs and queue"
@click="loadData"
/>
</div> </div>
<div <div
v-if="runStatus.isLikelyRunning" v-if="runStatus.isLikelyRunning"

View file

@ -12,6 +12,7 @@ import AppEmptyState from "@/components/ui/AppEmptyState.vue";
import AppHelpHint from "@/components/ui/AppHelpHint.vue"; import AppHelpHint from "@/components/ui/AppHelpHint.vue";
import AppInput from "@/components/ui/AppInput.vue"; import AppInput from "@/components/ui/AppInput.vue";
import AppModal from "@/components/ui/AppModal.vue"; import AppModal from "@/components/ui/AppModal.vue";
import AppRefreshButton from "@/components/ui/AppRefreshButton.vue";
import AppSelect from "@/components/ui/AppSelect.vue"; import AppSelect from "@/components/ui/AppSelect.vue";
import AppTable from "@/components/ui/AppTable.vue"; import AppTable from "@/components/ui/AppTable.vue";
import { import {
@ -812,9 +813,14 @@ onMounted(() => {
<AppButton variant="secondary" :disabled="loading || importingData" @click="onOpenImportPicker"> <AppButton variant="secondary" :disabled="loading || importingData" @click="onOpenImportPicker">
{{ importingData ? "Importing..." : "Import" }} {{ importingData ? "Importing..." : "Import" }}
</AppButton> </AppButton>
<AppButton variant="secondary" :disabled="loading || saving" @click="loadScholars"> <AppRefreshButton
{{ loading ? "Refreshing..." : "Refresh" }} variant="secondary"
</AppButton> :disabled="saving"
:loading="loading"
title="Refresh scholars"
loading-title="Refreshing scholars"
@click="loadScholars"
/>
</div> </div>
</div> </div>
<input <input

View file

@ -1,5 +1,6 @@
<script setup lang="ts"> <script setup lang="ts">
import { computed, onMounted, ref } from "vue"; import { computed, onMounted, ref, watch } from "vue";
import { useRoute, useRouter } from "vue-router";
import AppPage from "@/components/layout/AppPage.vue"; import AppPage from "@/components/layout/AppPage.vue";
import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue"; import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
@ -9,7 +10,8 @@ import AppCard from "@/components/ui/AppCard.vue";
import AppCheckbox from "@/components/ui/AppCheckbox.vue"; import AppCheckbox from "@/components/ui/AppCheckbox.vue";
import AppHelpHint from "@/components/ui/AppHelpHint.vue"; import AppHelpHint from "@/components/ui/AppHelpHint.vue";
import AppInput from "@/components/ui/AppInput.vue"; import AppInput from "@/components/ui/AppInput.vue";
import AppModal from "@/components/ui/AppModal.vue"; import AppTabs, { type AppTabItem } from "@/components/ui/AppTabs.vue";
import SettingsAdminPanel from "@/features/settings/SettingsAdminPanel.vue";
import { import {
changePassword, changePassword,
fetchSettings, fetchSettings,
@ -22,65 +24,18 @@ import { useAuthStore } from "@/stores/auth";
import { useRunStatusStore } from "@/stores/run_status"; import { useRunStatusStore } from "@/stores/run_status";
import { normalizeUserNavVisiblePages, useUserSettingsStore } from "@/stores/user_settings"; import { normalizeUserNavVisiblePages, useUserSettingsStore } from "@/stores/user_settings";
interface NavPageOption { const TAB_CHECKING = "checking";
id: string; const TAB_ACCOUNT = "account";
label: string; const TAB_ADMIN_USERS = "admin-users";
description: string; const TAB_ADMIN_INTEGRITY = "admin-integrity";
required: boolean; const TAB_ADMIN_REPAIRS = "admin-repairs";
adminOnly?: boolean; const TAB_ADMIN_PDF = "admin-pdf";
}
const NAV_PAGE_OPTIONS: NavPageOption[] = [
{
id: "dashboard",
label: "Dashboard",
description: "Overview and latest publication updates.",
required: true,
},
{
id: "scholars",
label: "Scholars",
description: "Tracked scholar profiles and profile management.",
required: true,
},
{
id: "publications",
label: "Publications",
description: "Review and search discovered publication records.",
required: false,
},
{
id: "settings",
label: "Settings",
description: "Configuration and account controls.",
required: true,
},
{
id: "style-guide",
label: "Style Guide",
description: "Admin-only visual reference for theme and component tokens.",
required: false,
adminOnly: true,
},
{
id: "runs",
label: "Runs",
description: "Admin-only diagnostics and queue operations.",
required: false,
adminOnly: true,
},
{
id: "users",
label: "Users",
description: "Admin-only user management.",
required: false,
adminOnly: true,
},
];
const auth = useAuthStore(); const auth = useAuthStore();
const userSettings = useUserSettingsStore(); const userSettings = useUserSettingsStore();
const runStatus = useRunStatusStore(); const runStatus = useRunStatusStore();
const route = useRoute();
const router = useRouter();
const loading = ref(true); const loading = ref(true);
const saving = ref(false); const saving = ref(false);
@ -98,55 +53,82 @@ const confirmPassword = ref("");
const errorMessage = ref<string | null>(null); const errorMessage = ref<string | null>(null);
const errorRequestId = ref<string | null>(null); const errorRequestId = ref<string | null>(null);
const successMessage = ref<string | null>(null); const successMessage = ref<string | null>(null);
const showIngestionModal = ref(false);
const showPasswordModal = ref(false);
const showNavigationModal = ref(false);
const minCheckIntervalMinutes = ref(15); const minCheckIntervalMinutes = ref(15);
const minRequestDelaySeconds = ref(2); const minRequestDelaySeconds = ref(2);
const automationAllowed = ref(true); const automationAllowed = ref(true);
const manualRunAllowed = ref(true); const manualRunAllowed = ref(true);
const blockedFailureThreshold = ref(1);
const networkFailureThreshold = ref(2);
const cooldownBlockedSeconds = ref(1800);
const cooldownNetworkSeconds = ref(900);
const visibleNavOptions = computed(() => const activeTab = ref(TAB_CHECKING);
NAV_PAGE_OPTIONS.filter((option) => !option.adminOnly || auth.isAdmin),
); const tabItems = computed<AppTabItem[]>(() => {
const visibleNavLabels = computed(() => const tabs: AppTabItem[] = [
visibleNavOptions.value { id: TAB_CHECKING, label: "Checking" },
.filter((option) => navVisiblePages.value.includes(option.id)) { id: TAB_ACCOUNT, label: "Account" },
.map((option) => option.label), ];
if (auth.isAdmin) {
tabs.push(
{ id: TAB_ADMIN_USERS, label: "Users" },
{ id: TAB_ADMIN_INTEGRITY, label: "Integrity" },
{ id: TAB_ADMIN_REPAIRS, label: "Repairs" },
{ id: TAB_ADMIN_PDF, label: "PDF Queue" },
); );
}
return tabs;
});
const activeAdminSection = computed<"users" | "integrity" | "repairs" | "pdf" | null>(() => {
if (activeTab.value === TAB_ADMIN_USERS) {
return "users";
}
if (activeTab.value === TAB_ADMIN_INTEGRITY) {
return "integrity";
}
if (activeTab.value === TAB_ADMIN_REPAIRS) {
return "repairs";
}
if (activeTab.value === TAB_ADMIN_PDF) {
return "pdf";
}
return null;
});
function isKnownTab(tabId: string): boolean {
return tabItems.value.some((item) => item.id === tabId);
}
async function syncRouteTab(candidate: string): Promise<void> {
const fallback = tabItems.value[0]?.id || TAB_CHECKING;
const nextTab = isKnownTab(candidate) ? candidate : fallback;
activeTab.value = nextTab;
const currentQueryTab = typeof route.query.tab === "string" ? route.query.tab : "";
if (currentQueryTab === nextTab) {
return;
}
await router.replace({ query: { ...route.query, tab: nextTab } });
}
function hydrateSettings(settings: UserSettings): void { function hydrateSettings(settings: UserSettings): void {
const parsedMinRunInterval = Number(settings.policy?.min_run_interval_minutes); const parsedMinRunInterval = Number(settings.policy?.min_run_interval_minutes);
minCheckIntervalMinutes.value = Number.isFinite(parsedMinRunInterval) minCheckIntervalMinutes.value = Number.isFinite(parsedMinRunInterval)
? Math.max(15, parsedMinRunInterval) ? Math.max(15, parsedMinRunInterval)
: 15; : 15;
const parsedMinRequestDelay = Number(settings.policy?.min_request_delay_seconds); const parsedMinRequestDelay = Number(settings.policy?.min_request_delay_seconds);
minRequestDelaySeconds.value = Number.isFinite(parsedMinRequestDelay) minRequestDelaySeconds.value = Number.isFinite(parsedMinRequestDelay)
? Math.max(2, parsedMinRequestDelay) ? Math.max(2, parsedMinRequestDelay)
: 2; : 2;
automationAllowed.value = Boolean(settings.policy?.automation_allowed ?? true); automationAllowed.value = Boolean(settings.policy?.automation_allowed ?? true);
manualRunAllowed.value = Boolean(settings.policy?.manual_run_allowed ?? true); manualRunAllowed.value = Boolean(settings.policy?.manual_run_allowed ?? true);
blockedFailureThreshold.value = Number.isFinite(settings.policy?.blocked_failure_threshold)
? Math.max(1, settings.policy.blocked_failure_threshold)
: 1;
networkFailureThreshold.value = Number.isFinite(settings.policy?.network_failure_threshold)
? Math.max(1, settings.policy.network_failure_threshold)
: 2;
cooldownBlockedSeconds.value = Number.isFinite(settings.policy?.cooldown_blocked_seconds)
? Math.max(60, settings.policy.cooldown_blocked_seconds)
: 1800;
cooldownNetworkSeconds.value = Number.isFinite(settings.policy?.cooldown_network_seconds)
? Math.max(60, settings.policy.cooldown_network_seconds)
: 900;
autoRunEnabled.value = Boolean(settings.auto_run_enabled) && automationAllowed.value; autoRunEnabled.value = Boolean(settings.auto_run_enabled) && automationAllowed.value;
runIntervalMinutes.value = String(settings.run_interval_minutes); runIntervalMinutes.value = String(settings.run_interval_minutes);
requestDelaySeconds.value = String(settings.request_delay_seconds); requestDelaySeconds.value = String(settings.request_delay_seconds);
navVisiblePages.value = normalizeUserNavVisiblePages(settings.nav_visible_pages); navVisiblePages.value = normalizeUserNavVisiblePages(settings.nav_visible_pages);
userSettings.applySettings(settings); userSettings.applySettings(settings);
runStatus.setSafetyState(settings.safety_state); runStatus.setSafetyState(settings.safety_state);
} }
@ -162,31 +144,10 @@ function parseBoundedInteger(value: string, label: string, minimum: number): num
return parsed; return parsed;
} }
function isNavPageVisible(pageId: string): boolean {
return navVisiblePages.value.includes(pageId);
}
function onToggleNavPage(page: NavPageOption, event: Event): void {
const input = event.target as HTMLInputElement;
if (page.required && !input.checked) {
return;
}
if (input.checked) {
navVisiblePages.value = normalizeUserNavVisiblePages([...navVisiblePages.value, page.id]);
return;
}
navVisiblePages.value = normalizeUserNavVisiblePages(
navVisiblePages.value.filter((pageId) => pageId !== page.id),
);
}
async function loadSettings(): Promise<void> { async function loadSettings(): Promise<void> {
loading.value = true; loading.value = true;
errorMessage.value = null; errorMessage.value = null;
errorRequestId.value = null; errorRequestId.value = null;
try { try {
const settings = await fetchSettings(); const settings = await fetchSettings();
hydrateSettings(settings); hydrateSettings(settings);
@ -227,8 +188,6 @@ async function onSaveSettings(): Promise<void> {
const saved = await updateSettings(payload); const saved = await updateSettings(payload);
hydrateSettings(saved); hydrateSettings(saved);
successMessage.value = "Settings updated."; successMessage.value = "Settings updated.";
showIngestionModal.value = false;
showNavigationModal.value = false;
} catch (error) { } catch (error) {
if (error instanceof ApiRequestError) { if (error instanceof ApiRequestError) {
errorMessage.value = error.message; errorMessage.value = error.message;
@ -264,7 +223,6 @@ async function onChangePassword(): Promise<void> {
newPassword.value = ""; newPassword.value = "";
confirmPassword.value = ""; confirmPassword.value = "";
successMessage.value = response.message; successMessage.value = response.message;
showPasswordModal.value = false;
} catch (error) { } catch (error) {
if (error instanceof ApiRequestError) { if (error instanceof ApiRequestError) {
errorMessage.value = error.message; errorMessage.value = error.message;
@ -279,16 +237,35 @@ async function onChangePassword(): Promise<void> {
} }
} }
onMounted(() => { async function onSelectTab(tabId: string): Promise<void> {
void loadSettings(); await syncRouteTab(tabId);
}
watch(
() => route.query.tab,
async (value) => {
const requestedTab = typeof value === "string" ? value : TAB_CHECKING;
await syncRouteTab(requestedTab);
},
);
watch(
() => auth.isAdmin,
async () => {
const requestedTab = typeof route.query.tab === "string" ? route.query.tab : TAB_CHECKING;
await syncRouteTab(requestedTab);
},
);
onMounted(async () => {
await loadSettings();
const requestedTab = typeof route.query.tab === "string" ? route.query.tab : TAB_CHECKING;
await syncRouteTab(requestedTab);
}); });
</script> </script>
<template> <template>
<AppPage <AppPage title="Settings" subtitle="Configuration and account controls in one place.">
title="Settings"
subtitle="Control how often Scholarr checks profiles and how cautiously it sends requests."
>
<RequestStateAlerts <RequestStateAlerts
:success-message="successMessage" :success-message="successMessage"
success-title="Saved" success-title="Saved"
@ -298,189 +275,91 @@ onMounted(() => {
@dismiss-success="successMessage = null" @dismiss-success="successMessage = null"
/> />
<AsyncStateGate :loading="loading" :loading-lines="7" :show-empty="false"> <AsyncStateGate :loading="loading" :loading-lines="8" :show-empty="false">
<section class="grid gap-4 xl:grid-cols-3"> <AppCard class="grid gap-4">
<AppCard class="flex h-full flex-col gap-4"> <AppTabs :model-value="activeTab" :items="tabItems" aria-label="Settings sections" @update:model-value="onSelectTab" />
<section v-if="activeTab === TAB_CHECKING" class="grid gap-4">
<div class="flex items-center gap-1"> <div class="flex items-center gap-1">
<h2 class="text-lg font-semibold text-ink-primary">Automatic Checking</h2> <h2 class="text-lg font-semibold text-ink-primary">Automatic Checking</h2>
<AppHelpHint text="Controls when Scholarr runs automatic profile checks and how cautiously it scrapes." /> <AppHelpHint text="Controls when Scholarr runs automatic profile checks and how cautiously it scrapes." />
</div> </div>
<p class="text-sm text-secondary"> <p class="text-sm text-secondary">Configure background checks and request pacing for ingestion.</p>
Configure the background checker that looks for new publications on your tracked profiles.
</p>
<AppButton variant="secondary" class="mt-auto self-start" @click="showIngestionModal = true">
Edit checking rules
</AppButton>
</AppCard>
<AppCard class="flex h-full flex-col gap-4"> <div class="grid gap-3 md:grid-cols-2 md:items-start">
<div class="flex items-center gap-1"> <div class="grid gap-2 rounded-lg border border-stroke-default bg-surface-card-muted p-3 md:col-span-2">
<h2 class="text-lg font-semibold text-ink-primary">Account Access</h2>
<AppHelpHint text="Manage credentials for your current signed-in account on this instance." />
</div>
<p class="text-sm text-secondary">
Change your sign-in password from a focused view. This does not affect other users.
</p>
<AppButton variant="secondary" class="mt-auto self-start" @click="showPasswordModal = true">
Change password
</AppButton>
</AppCard>
<AppCard class="flex h-full flex-col gap-4">
<div class="flex items-center gap-1">
<h2 class="text-lg font-semibold text-ink-primary">Navigation</h2>
<AppHelpHint text="Choose which pages appear in the left sidebar. Dashboard, Scholars, and Settings are always visible." />
</div>
<p class="text-sm text-secondary">
Visible now: {{ visibleNavLabels.length > 0 ? visibleNavLabels.join(", ") : "none" }}.
</p>
<AppButton variant="secondary" class="mt-auto self-start" @click="showNavigationModal = true">
Customize sidebar pages
</AppButton>
</AppCard>
</section>
</AsyncStateGate>
<AppModal
:open="showIngestionModal"
title="Automatic Checking Settings"
@close="showIngestionModal = false"
>
<form class="grid gap-3" @submit.prevent="onSaveSettings">
<AppCheckbox <AppCheckbox
id="auto-run-enabled" id="auto-run-enabled"
v-model="autoRunEnabled" v-model="autoRunEnabled"
:disabled="!automationAllowed" :disabled="!automationAllowed"
label="Enable automatic background checks" label="Enable automatic background checks"
/> />
<p v-if="!automationAllowed" class="text-xs text-secondary"> <p class="text-xs text-secondary">
Automatic checks are disabled by server safety policy. Automatic checks are {{ automationAllowed ? "enabled by policy." : "disabled by server safety policy." }}
</p> </p>
</div>
<label class="grid gap-2 text-sm font-medium text-ink-secondary"> <label class="grid gap-2 text-sm font-medium text-ink-secondary">
<span class="inline-flex items-center gap-1"> <span class="inline-flex items-center gap-1">
Check interval (minutes) Check interval (minutes)
<AppHelpHint text="How often Scholarr starts a background update check." /> <AppHelpHint text="Minimum is controlled by server policy." />
</span> </span>
<AppInput id="run-interval" v-model="runIntervalMinutes" type="number" :min="minCheckIntervalMinutes" /> <AppInput v-model="runIntervalMinutes" inputmode="numeric" />
<span class="text-xs text-secondary">Minimum: {{ minCheckIntervalMinutes }} minutes.</span> <span class="text-xs text-secondary">Minimum: {{ minCheckIntervalMinutes }}</span>
</label> </label>
<label class="grid gap-2 text-sm font-medium text-ink-secondary"> <label class="grid gap-2 text-sm font-medium text-ink-secondary">
<span class="inline-flex items-center gap-1"> <span class="inline-flex items-center gap-1">
Delay between requests (seconds) Delay between requests (seconds)
<AppHelpHint text="Pause between profile requests during a check. Higher values are slower but safer." /> <AppHelpHint text="Minimum is controlled by safety policy to reduce blocks." />
</span> </span>
<AppInput <AppInput v-model="requestDelaySeconds" inputmode="numeric" />
id="request-delay" <span class="text-xs text-secondary">Minimum: {{ minRequestDelaySeconds }}</span>
v-model="requestDelaySeconds"
type="number"
:min="minRequestDelaySeconds"
/>
<span class="text-xs text-secondary">Minimum: {{ minRequestDelaySeconds }} seconds.</span>
</label> </label>
<div class="grid gap-1 rounded-lg border border-stroke-default bg-surface-card-muted px-3 py-2 text-xs text-secondary">
<p class="font-medium text-ink-primary">Server-enforced scrape safety policy</p>
<p>Blocked failures trigger cooldown at {{ blockedFailureThreshold }} failures.</p>
<p>Network failures trigger cooldown at {{ networkFailureThreshold }} failures.</p>
<p>Blocked cooldown: {{ cooldownBlockedSeconds }}s. Network cooldown: {{ cooldownNetworkSeconds }}s.</p>
</div> </div>
<div class="mt-2 flex flex-wrap justify-end gap-2"> <div class="flex flex-wrap items-center gap-3 text-xs text-secondary">
<AppButton <span>Automation allowed: {{ automationAllowed ? "yes" : "no" }}</span>
variant="ghost" <span>Manual runs allowed: {{ manualRunAllowed ? "yes" : "no" }}</span>
type="button" </div>
:disabled="saving"
@click="showIngestionModal = false" <div>
> <AppButton :disabled="saving" @click="onSaveSettings">
Cancel {{ saving ? "Saving..." : "Save checking settings" }}
</AppButton>
<AppButton type="submit" :disabled="saving">
{{ saving ? "Saving..." : "Save settings" }}
</AppButton> </AppButton>
</div> </div>
</form> </section>
</AppModal>
<AppModal :open="showPasswordModal" title="Change Sign-in Password" @close="showPasswordModal = false"> <section v-if="activeTab === TAB_ACCOUNT" class="grid gap-4">
<form class="grid gap-3" @submit.prevent="onChangePassword"> <div class="flex items-center gap-1">
<label class="grid gap-2 text-sm font-medium text-ink-secondary"> <h2 class="text-lg font-semibold text-ink-primary">Account Access</h2>
<AppHelpHint text="Manage credentials for your current signed-in account on this instance." />
</div>
<p class="text-sm text-secondary">Change your sign-in password directly from this tab.</p>
<form class="grid gap-3 rounded-lg border border-stroke-default bg-surface-card-muted p-3" @submit.prevent="onChangePassword">
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
<span>Current password</span> <span>Current password</span>
<AppInput v-model="currentPassword" type="password" autocomplete="current-password" /> <AppInput v-model="currentPassword" type="password" autocomplete="current-password" />
</label> </label>
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
<span>New password</span> <span>New password</span>
<AppInput v-model="newPassword" type="password" autocomplete="new-password" /> <AppInput v-model="newPassword" type="password" autocomplete="new-password" />
</label> </label>
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
<span>Confirm new password</span> <span>Confirm new password</span>
<AppInput v-model="confirmPassword" type="password" autocomplete="new-password" /> <AppInput v-model="confirmPassword" type="password" autocomplete="new-password" />
</label> </label>
<div class="flex gap-2">
<div class="mt-2 flex flex-wrap justify-end gap-2">
<AppButton
variant="ghost"
type="button"
:disabled="updatingPassword"
@click="showPasswordModal = false"
>
Cancel
</AppButton>
<AppButton type="submit" :disabled="updatingPassword"> <AppButton type="submit" :disabled="updatingPassword">
{{ updatingPassword ? "Updating..." : "Change password" }} {{ updatingPassword ? "Updating..." : "Update password" }}
</AppButton> </AppButton>
</div> </div>
</form> </form>
</AppModal> </section>
<AppModal :open="showNavigationModal" title="Sidebar Page Visibility" @close="showNavigationModal = false"> <SettingsAdminPanel v-if="activeAdminSection" :section="activeAdminSection" />
<form class="grid gap-3" @submit.prevent="onSaveSettings"> </AppCard>
<p class="text-sm text-secondary"> </AsyncStateGate>
Turn optional pages on or off in the sidebar for your account.
</p>
<ul class="grid gap-2">
<li
v-for="option in visibleNavOptions"
:key="option.id"
class="rounded-lg border border-stroke-default bg-surface-card-muted/70 px-3 py-2"
>
<label class="flex items-start gap-3">
<input
type="checkbox"
class="mt-0.5 h-4 w-4 rounded border-stroke-interactive bg-surface-input text-brand-600 focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset"
:checked="isNavPageVisible(option.id)"
:disabled="saving || option.required"
@change="onToggleNavPage(option, $event)"
/>
<span class="grid gap-0.5">
<span class="text-sm font-medium text-ink-primary">
{{ option.label }}
<span v-if="option.required" class="ml-1 text-xs text-ink-muted">(required)</span>
</span>
<span class="text-xs text-secondary">{{ option.description }}</span>
</span>
</label>
</li>
</ul>
<div class="mt-2 flex flex-wrap justify-end gap-2">
<AppButton
variant="ghost"
type="button"
:disabled="saving"
@click="showNavigationModal = false"
>
Cancel
</AppButton>
<AppButton type="submit" :disabled="saving">
{{ saving ? "Saving..." : "Save sidebar settings" }}
</AppButton>
</div>
</form>
</AppModal>
</AppPage> </AppPage>
</template> </template>

View file

@ -0,0 +1,313 @@
export default {
"id": "canopy",
"label": "Canopy",
"description": "Vibrant forest green palette with crisp mint contrast.",
"modes": {
"light": {
"scale": {
"brand": {
"50": "#ecfdf5",
"100": "#d1fae5",
"200": "#a7f3d0",
"300": "#6ee7b7",
"400": "#34d399",
"500": "#10b981",
"600": "#059669",
"700": "#047857",
"800": "#065f46",
"900": "#064e3b",
"950": "#022c22"
},
"info": {
"50": "#ecfeff",
"100": "#cffafe",
"200": "#a5f3fc",
"300": "#67e8f9",
"400": "#22d3ee",
"500": "#06b6d4",
"600": "#0891b2",
"700": "#0e7490",
"800": "#155e75",
"900": "#164e63",
"950": "#083344"
},
"success": {
"50": "#f0fdf4",
"100": "#dcfce7",
"200": "#bbf7d0",
"300": "#86efac",
"400": "#4ade80",
"500": "#22c55e",
"600": "#16a34a",
"700": "#15803d",
"800": "#166534",
"900": "#14532d",
"950": "#052e16"
},
"warning": {
"50": "#fffbeb",
"100": "#fef3c7",
"200": "#fde68a",
"300": "#fcd34d",
"400": "#fbbf24",
"500": "#f59e0b",
"600": "#d97706",
"700": "#b45309",
"800": "#92400e",
"900": "#78350f",
"950": "#451a03"
},
"danger": {
"50": "#fff1f2",
"100": "#ffe4e6",
"200": "#fecdd3",
"300": "#fda4af",
"400": "#fb7185",
"500": "#f43f5e",
"600": "#e11d48",
"700": "#be123c",
"800": "#9f1239",
"900": "#881337",
"950": "#4c0519"
}
},
"surface": {
"app": "#eefbf4",
"nav": "#ffffff",
"nav_active": "#d1fae5",
"card": "#ffffff",
"card_muted": "#e8f7ef",
"table": "#ffffff",
"table_header": "#eefbf4",
"input": "#ffffff",
"overlay": "#0f172a"
},
"text": {
"primary": "#123524",
"secondary": "#166534",
"muted": "#4b7c67",
"inverse": "#ffffff",
"link": "#047857"
},
"border": {
"default": "#ccefe0",
"strong": "#9dd8bf",
"subtle": "#eaf8f1",
"interactive": "#34d399"
},
"focus": {
"ring": "#10b981",
"ring_offset": "#eefbf4"
},
"action": {
"primary": {
"bg": "#10b981",
"border": "#10b981",
"text": "#ffffff",
"hover_bg": "#059669",
"hover_border": "#059669",
"hover_text": "#ffffff"
},
"secondary": {
"bg": "#e8f7ef",
"border": "#ccefe0",
"text": "#123524",
"hover_bg": "#d1fae5",
"hover_border": "#9dd8bf",
"hover_text": "#052e16"
},
"ghost": {
"bg": "#eefbf4",
"border": "#ccefe0",
"text": "#047857",
"hover_bg": "#e2f4ea",
"hover_border": "#9dd8bf",
"hover_text": "#0b3f2b"
},
"danger": {
"bg": "#ffe4e6",
"border": "#fecdd3",
"text": "#be123c",
"hover_bg": "#fecdd3",
"hover_border": "#fda4af",
"hover_text": "#9f1239"
}
},
"state": {
"info": {
"bg": "#cffafe",
"border": "#a5f3fc",
"text": "#155e75"
},
"success": {
"bg": "#dcfce7",
"border": "#bbf7d0",
"text": "#166534"
},
"warning": {
"bg": "#fef3c7",
"border": "#fde68a",
"text": "#92400e"
},
"danger": {
"bg": "#ffe4e6",
"border": "#fecdd3",
"text": "#be123c"
}
}
},
"dark": {
"scale": {
"brand": {
"50": "#ecfdf5",
"100": "#d1fae5",
"200": "#a7f3d0",
"300": "#6ee7b7",
"400": "#34d399",
"500": "#10b981",
"600": "#059669",
"700": "#047857",
"800": "#065f46",
"900": "#064e3b",
"950": "#022c22"
},
"info": {
"50": "#ecfeff",
"100": "#cffafe",
"200": "#a5f3fc",
"300": "#67e8f9",
"400": "#22d3ee",
"500": "#06b6d4",
"600": "#0891b2",
"700": "#0e7490",
"800": "#155e75",
"900": "#164e63",
"950": "#083344"
},
"success": {
"50": "#f0fdf4",
"100": "#dcfce7",
"200": "#bbf7d0",
"300": "#86efac",
"400": "#4ade80",
"500": "#22c55e",
"600": "#16a34a",
"700": "#15803d",
"800": "#166534",
"900": "#14532d",
"950": "#052e16"
},
"warning": {
"50": "#fffbeb",
"100": "#fef3c7",
"200": "#fde68a",
"300": "#fcd34d",
"400": "#fbbf24",
"500": "#f59e0b",
"600": "#d97706",
"700": "#b45309",
"800": "#92400e",
"900": "#78350f",
"950": "#451a03"
},
"danger": {
"50": "#fff1f2",
"100": "#ffe4e6",
"200": "#fecdd3",
"300": "#fda4af",
"400": "#fb7185",
"500": "#f43f5e",
"600": "#e11d48",
"700": "#be123c",
"800": "#9f1239",
"900": "#881337",
"950": "#4c0519"
}
},
"surface": {
"app": "#0b1f17",
"nav": "#10271d",
"nav_active": "#047857",
"card": "#10271d",
"card_muted": "#163427",
"table": "#10271d",
"table_header": "#163427",
"input": "#0a1a14",
"overlay": "#030712"
},
"text": {
"primary": "#d9f9e8",
"secondary": "#9ad8b8",
"muted": "#6ea58a",
"inverse": "#0b1f17",
"link": "#6ee7b7"
},
"border": {
"default": "#1f4635",
"strong": "#2a5b45",
"subtle": "#163427",
"interactive": "#10b981"
},
"focus": {
"ring": "#34d399",
"ring_offset": "#0b1f17"
},
"action": {
"primary": {
"bg": "#10b981",
"border": "#10b981",
"text": "#052e16",
"hover_bg": "#34d399",
"hover_border": "#34d399",
"hover_text": "#052e16"
},
"secondary": {
"bg": "#163427",
"border": "#2a5b45",
"text": "#d9f9e8",
"hover_bg": "#214c38",
"hover_border": "#2f654d",
"hover_text": "#ffffff"
},
"ghost": {
"bg": "#0b1f17",
"border": "#1f4635",
"text": "#9ad8b8",
"hover_bg": "#163427",
"hover_border": "#2a5b45",
"hover_text": "#d9f9e8"
},
"danger": {
"bg": "#4c0519",
"border": "#9f1239",
"text": "#fecdd3",
"hover_bg": "#5b0b24",
"hover_border": "#be123c",
"hover_text": "#ffe4e6"
}
},
"state": {
"info": {
"bg": "#164e63",
"border": "#0e7490",
"text": "#cffafe"
},
"success": {
"bg": "#14532d",
"border": "#15803d",
"text": "#bbf7d0"
},
"warning": {
"bg": "#78350f",
"border": "#b45309",
"text": "#fde68a"
},
"danger": {
"bg": "#4c0519",
"border": "#be123c",
"text": "#fecdd3"
}
}
}
}
};

View file

@ -1,22 +1,22 @@
export default { export default {
"id": "dune", "id": "dune",
"label": "Dune", "label": "Dune",
"description": "Sandy beige environments featuring soft mauve highlights.", "description": "Sandy clay surfaces with warm terracotta accents.",
"modes": { "modes": {
"light": { "light": {
"scale": { "scale": {
"brand": { "brand": {
"50": "#f7f5f6", "50": "#fff5ef",
"100": "#ece7e9", "100": "#fee8da",
"200": "#d7cccd", "200": "#fbcfb2",
"300": "#baabb2", "300": "#f6ae81",
"400": "#9e8893", "400": "#ef8753",
"500": "#856b77", "500": "#e2672f",
"600": "#6c5460", "600": "#bf4f25",
"700": "#58424d", "700": "#9a3f21",
"800": "#4a3841", "800": "#7d3420",
"900": "#3d3036", "900": "#662d1d",
"950": "#21181d" "950": "#38140e"
}, },
"info": { "info": {
"50": "#f4f5f8", "50": "#f4f5f8",
@ -74,7 +74,7 @@ export default {
"surface": { "surface": {
"app": "#efece5", "app": "#efece5",
"nav": "#f7f5f0", "nav": "#f7f5f0",
"nav_active": "#d7cccd", "nav_active": "#fbcfb2",
"card": "#f7f5f0", "card": "#f7f5f0",
"card_muted": "#e6e3dd", "card_muted": "#e6e3dd",
"table": "#f7f5f0", "table": "#f7f5f0",
@ -87,25 +87,25 @@ export default {
"secondary": "#757072", "secondary": "#757072",
"muted": "#9c9699", "muted": "#9c9699",
"inverse": "#f7f5f0", "inverse": "#f7f5f0",
"link": "#6c5460" "link": "#bf4f25"
}, },
"border": { "border": {
"default": "#e6e3dd", "default": "#e6e3dd",
"strong": "#d1cdc6", "strong": "#d1cdc6",
"subtle": "#f2f0eb", "subtle": "#f2f0eb",
"interactive": "#baabb2" "interactive": "#f6ae81"
}, },
"focus": { "focus": {
"ring": "#9e8893", "ring": "#ef8753",
"ring_offset": "#efece5" "ring_offset": "#efece5"
}, },
"action": { "action": {
"primary": { "primary": {
"bg": "#9e8893", "bg": "#ef8753",
"border": "#9e8893", "border": "#ef8753",
"text": "#f7f5f0", "text": "#f7f5f0",
"hover_bg": "#856b77", "hover_bg": "#e2672f",
"hover_border": "#856b77", "hover_border": "#e2672f",
"hover_text": "#ffffff" "hover_text": "#ffffff"
}, },
"secondary": { "secondary": {
@ -119,7 +119,7 @@ export default {
"ghost": { "ghost": {
"bg": "#efece5", "bg": "#efece5",
"border": "#e6e3dd", "border": "#e6e3dd",
"text": "#6c5460", "text": "#bf4f25",
"hover_bg": "#e6e3dd", "hover_bg": "#e6e3dd",
"hover_border": "#d1cdc6", "hover_border": "#d1cdc6",
"hover_text": "#3d3a3b" "hover_text": "#3d3a3b"
@ -159,17 +159,17 @@ export default {
"dark": { "dark": {
"scale": { "scale": {
"brand": { "brand": {
"50": "#f7f5f6", "50": "#fff5ef",
"100": "#ece7e9", "100": "#fee8da",
"200": "#d7cccd", "200": "#fbcfb2",
"300": "#baabb2", "300": "#f6ae81",
"400": "#9e8893", "400": "#ef8753",
"500": "#856b77", "500": "#e2672f",
"600": "#6c5460", "600": "#bf4f25",
"700": "#58424d", "700": "#9a3f21",
"800": "#4a3841", "800": "#7d3420",
"900": "#3d3036", "900": "#662d1d",
"950": "#21181d" "950": "#38140e"
}, },
"info": { "info": {
"50": "#f4f5f8", "50": "#f4f5f8",
@ -227,7 +227,7 @@ export default {
"surface": { "surface": {
"app": "#3d3a3b", "app": "#3d3a3b",
"nav": "#4a4748", "nav": "#4a4748",
"nav_active": "#6c5460", "nav_active": "#bf4f25",
"card": "#4a4748", "card": "#4a4748",
"card_muted": "#575354", "card_muted": "#575354",
"table": "#4a4748", "table": "#4a4748",
@ -240,25 +240,25 @@ export default {
"secondary": "#b8b4b6", "secondary": "#b8b4b6",
"muted": "#918e8f", "muted": "#918e8f",
"inverse": "#3d3a3b", "inverse": "#3d3a3b",
"link": "#baabb2" "link": "#f6ae81"
}, },
"border": { "border": {
"default": "#575354", "default": "#575354",
"strong": "#696466", "strong": "#696466",
"subtle": "#474345", "subtle": "#474345",
"interactive": "#9e8893" "interactive": "#ef8753"
}, },
"focus": { "focus": {
"ring": "#9e8893", "ring": "#ef8753",
"ring_offset": "#3d3a3b" "ring_offset": "#3d3a3b"
}, },
"action": { "action": {
"primary": { "primary": {
"bg": "#9e8893", "bg": "#ef8753",
"border": "#9e8893", "border": "#ef8753",
"text": "#1f1d1e", "text": "#1f1d1e",
"hover_bg": "#baabb2", "hover_bg": "#f6ae81",
"hover_border": "#baabb2", "hover_border": "#f6ae81",
"hover_text": "#1f1d1e" "hover_text": "#1f1d1e"
}, },
"secondary": { "secondary": {

View file

@ -1,22 +1,22 @@
export default { export default {
"id": "oatmeal", "id": "oatmeal",
"label": "Oatmeal", "label": "Oatmeal",
"description": "Warm beige foundation with muted sage green interactions.", "description": "Cream backdrop with vivid olive-green accents.",
"modes": { "modes": {
"light": { "light": {
"scale": { "scale": {
"brand": { "brand": {
"50": "#f4f7f5", "50": "#effbf4",
"100": "#e5ece7", "100": "#d8f4e5",
"200": "#ccd9cf", "200": "#b4e8cd",
"300": "#a8bdae", "300": "#7fd8ab",
"400": "#8ba390", "400": "#4ec788",
"500": "#6c8471", "500": "#2faa6e",
"600": "#546958", "600": "#248a58",
"700": "#435446", "700": "#1f6f48",
"800": "#364338", "800": "#1b593c",
"900": "#2d372f", "900": "#174a32",
"950": "#161d18" "950": "#0c2a1d"
}, },
"info": { "info": {
"50": "#f0f5f6", "50": "#f0f5f6",
@ -74,7 +74,7 @@ export default {
"surface": { "surface": {
"app": "#f5f2eb", "app": "#f5f2eb",
"nav": "#fcfaf5", "nav": "#fcfaf5",
"nav_active": "#e5e1d8", "nav_active": "#d8f4e5",
"card": "#fcfaf5", "card": "#fcfaf5",
"card_muted": "#ece9e1", "card_muted": "#ece9e1",
"table": "#fcfaf5", "table": "#fcfaf5",
@ -87,25 +87,25 @@ export default {
"secondary": "#716d68", "secondary": "#716d68",
"muted": "#96918a", "muted": "#96918a",
"inverse": "#fcfaf5", "inverse": "#fcfaf5",
"link": "#546958" "link": "#248a58"
}, },
"border": { "border": {
"default": "#e5e1d8", "default": "#e5e1d8",
"strong": "#cecabf", "strong": "#cecabf",
"subtle": "#f0ede6", "subtle": "#f0ede6",
"interactive": "#a8bdae" "interactive": "#7fd8ab"
}, },
"focus": { "focus": {
"ring": "#8ba390", "ring": "#4ec788",
"ring_offset": "#f5f2eb" "ring_offset": "#f5f2eb"
}, },
"action": { "action": {
"primary": { "primary": {
"bg": "#8ba390", "bg": "#4ec788",
"border": "#8ba390", "border": "#4ec788",
"text": "#fcfaf5", "text": "#fcfaf5",
"hover_bg": "#6c8471", "hover_bg": "#2faa6e",
"hover_border": "#6c8471", "hover_border": "#2faa6e",
"hover_text": "#ffffff" "hover_text": "#ffffff"
}, },
"secondary": { "secondary": {
@ -119,7 +119,7 @@ export default {
"ghost": { "ghost": {
"bg": "#f5f2eb", "bg": "#f5f2eb",
"border": "#e5e1d8", "border": "#e5e1d8",
"text": "#546958", "text": "#248a58",
"hover_bg": "#ece9e1", "hover_bg": "#ece9e1",
"hover_border": "#cecabf", "hover_border": "#cecabf",
"hover_text": "#3d3b38" "hover_text": "#3d3b38"
@ -159,17 +159,17 @@ export default {
"dark": { "dark": {
"scale": { "scale": {
"brand": { "brand": {
"50": "#f4f7f5", "50": "#effbf4",
"100": "#e5ece7", "100": "#d8f4e5",
"200": "#ccd9cf", "200": "#b4e8cd",
"300": "#a8bdae", "300": "#7fd8ab",
"400": "#8ba390", "400": "#4ec788",
"500": "#6c8471", "500": "#2faa6e",
"600": "#546958", "600": "#248a58",
"700": "#435446", "700": "#1f6f48",
"800": "#364338", "800": "#1b593c",
"900": "#2d372f", "900": "#174a32",
"950": "#161d18" "950": "#0c2a1d"
}, },
"info": { "info": {
"50": "#f0f5f6", "50": "#f0f5f6",
@ -227,7 +227,7 @@ export default {
"surface": { "surface": {
"app": "#3a3937", "app": "#3a3937",
"nav": "#454442", "nav": "#454442",
"nav_active": "#546958", "nav_active": "#248a58",
"card": "#454442", "card": "#454442",
"card_muted": "#504e4c", "card_muted": "#504e4c",
"table": "#454442", "table": "#454442",
@ -240,25 +240,25 @@ export default {
"secondary": "#bebcb8", "secondary": "#bebcb8",
"muted": "#969490", "muted": "#969490",
"inverse": "#3a3937", "inverse": "#3a3937",
"link": "#a8bdae" "link": "#7fd8ab"
}, },
"border": { "border": {
"default": "#504e4c", "default": "#504e4c",
"strong": "#62605d", "strong": "#62605d",
"subtle": "#403f3d", "subtle": "#403f3d",
"interactive": "#8ba390" "interactive": "#4ec788"
}, },
"focus": { "focus": {
"ring": "#8ba390", "ring": "#4ec788",
"ring_offset": "#3a3937" "ring_offset": "#3a3937"
}, },
"action": { "action": {
"primary": { "primary": {
"bg": "#8ba390", "bg": "#4ec788",
"border": "#8ba390", "border": "#4ec788",
"text": "#1a1918", "text": "#1a1918",
"hover_bg": "#a8bdae", "hover_bg": "#7fd8ab",
"hover_border": "#a8bdae", "hover_border": "#7fd8ab",
"hover_text": "#1a1918" "hover_text": "#1a1918"
}, },
"secondary": { "secondary": {

View file

@ -1,22 +1,22 @@
export default { export default {
"id": "parchment", "id": "parchment",
"label": "Parchment", "label": "Parchment",
"description": "Dusty beige canvas paired with slate blue accents.", "description": "Cool parchment base with bold indigo-blue accents.",
"modes": { "modes": {
"light": { "light": {
"scale": { "scale": {
"brand": { "brand": {
"50": "#f5f7f9", "50": "#f2f6ff",
"100": "#eaf0f4", "100": "#e3edff",
"200": "#d5e0e9", "200": "#c6d8ff",
"300": "#b3c9d7", "300": "#9db9ff",
"400": "#8daabc", "400": "#7596f4",
"500": "#7d93a4", "500": "#5878da",
"600": "#5c7487", "600": "#4362bc",
"700": "#495e6f", "700": "#384f95",
"800": "#3f4d5b", "800": "#31427b",
"900": "#36414e", "900": "#2b3965",
"950": "#232b35" "950": "#1b223f"
}, },
"info": { "info": {
"50": "#f2f6fa", "50": "#f2f6fa",
@ -74,7 +74,7 @@ export default {
"surface": { "surface": {
"app": "#f2eee9", "app": "#f2eee9",
"nav": "#faf8f5", "nav": "#faf8f5",
"nav_active": "#d5e0e9", "nav_active": "#c6d8ff",
"card": "#faf8f5", "card": "#faf8f5",
"card_muted": "#eae5df", "card_muted": "#eae5df",
"table": "#faf8f5", "table": "#faf8f5",
@ -87,25 +87,25 @@ export default {
"secondary": "#6b737c", "secondary": "#6b737c",
"muted": "#929ba5", "muted": "#929ba5",
"inverse": "#faf8f5", "inverse": "#faf8f5",
"link": "#5c7487" "link": "#4362bc"
}, },
"border": { "border": {
"default": "#eae5df", "default": "#eae5df",
"strong": "#d4cec6", "strong": "#d4cec6",
"subtle": "#f6f3ef", "subtle": "#f6f3ef",
"interactive": "#b3c9d7" "interactive": "#9db9ff"
}, },
"focus": { "focus": {
"ring": "#7d93a4", "ring": "#5878da",
"ring_offset": "#f2eee9" "ring_offset": "#f2eee9"
}, },
"action": { "action": {
"primary": { "primary": {
"bg": "#7d93a4", "bg": "#5878da",
"border": "#7d93a4", "border": "#5878da",
"text": "#faf8f5", "text": "#faf8f5",
"hover_bg": "#5c7487", "hover_bg": "#4362bc",
"hover_border": "#5c7487", "hover_border": "#4362bc",
"hover_text": "#ffffff" "hover_text": "#ffffff"
}, },
"secondary": { "secondary": {
@ -119,7 +119,7 @@ export default {
"ghost": { "ghost": {
"bg": "#f2eee9", "bg": "#f2eee9",
"border": "#eae5df", "border": "#eae5df",
"text": "#5c7487", "text": "#4362bc",
"hover_bg": "#eae5df", "hover_bg": "#eae5df",
"hover_border": "#d4cec6", "hover_border": "#d4cec6",
"hover_text": "#333638" "hover_text": "#333638"
@ -159,17 +159,17 @@ export default {
"dark": { "dark": {
"scale": { "scale": {
"brand": { "brand": {
"50": "#f5f7f9", "50": "#f2f6ff",
"100": "#eaf0f4", "100": "#e3edff",
"200": "#d5e0e9", "200": "#c6d8ff",
"300": "#b3c9d7", "300": "#9db9ff",
"400": "#8daabc", "400": "#7596f4",
"500": "#7d93a4", "500": "#5878da",
"600": "#5c7487", "600": "#4362bc",
"700": "#495e6f", "700": "#384f95",
"800": "#3f4d5b", "800": "#31427b",
"900": "#36414e", "900": "#2b3965",
"950": "#232b35" "950": "#1b223f"
}, },
"info": { "info": {
"50": "#f2f6fa", "50": "#f2f6fa",
@ -227,7 +227,7 @@ export default {
"surface": { "surface": {
"app": "#35373a", "app": "#35373a",
"nav": "#424549", "nav": "#424549",
"nav_active": "#5c7487", "nav_active": "#4362bc",
"card": "#424549", "card": "#424549",
"card_muted": "#4d5156", "card_muted": "#4d5156",
"table": "#424549", "table": "#424549",
@ -240,25 +240,25 @@ export default {
"secondary": "#aeb5bc", "secondary": "#aeb5bc",
"muted": "#889098", "muted": "#889098",
"inverse": "#35373a", "inverse": "#35373a",
"link": "#b3c9d7" "link": "#9db9ff"
}, },
"border": { "border": {
"default": "#4d5156", "default": "#4d5156",
"strong": "#5d6268", "strong": "#5d6268",
"subtle": "#3e4145", "subtle": "#3e4145",
"interactive": "#7d93a4" "interactive": "#5878da"
}, },
"focus": { "focus": {
"ring": "#7d93a4", "ring": "#5878da",
"ring_offset": "#35373a" "ring_offset": "#35373a"
}, },
"action": { "action": {
"primary": { "primary": {
"bg": "#7d93a4", "bg": "#5878da",
"border": "#7d93a4", "border": "#5878da",
"text": "#1a1b1d", "text": "#1a1b1d",
"hover_bg": "#b3c9d7", "hover_bg": "#9db9ff",
"hover_border": "#b3c9d7", "hover_border": "#9db9ff",
"hover_text": "#1a1b1d" "hover_text": "#1a1b1d"
}, },
"secondary": { "secondary": {

View file

@ -0,0 +1,313 @@
export default {
"id": "sunset",
"label": "Sunset",
"description": "High-contrast amber and ember tones for a warmer visual rhythm.",
"modes": {
"light": {
"scale": {
"brand": {
"50": "#fff7ed",
"100": "#ffedd5",
"200": "#fed7aa",
"300": "#fdba74",
"400": "#fb923c",
"500": "#f97316",
"600": "#ea580c",
"700": "#c2410c",
"800": "#9a3412",
"900": "#7c2d12",
"950": "#431407"
},
"info": {
"50": "#eef2ff",
"100": "#e0e7ff",
"200": "#c7d2fe",
"300": "#a5b4fc",
"400": "#818cf8",
"500": "#6366f1",
"600": "#4f46e5",
"700": "#4338ca",
"800": "#3730a3",
"900": "#312e81",
"950": "#1e1b4b"
},
"success": {
"50": "#f0fdf4",
"100": "#dcfce7",
"200": "#bbf7d0",
"300": "#86efac",
"400": "#4ade80",
"500": "#22c55e",
"600": "#16a34a",
"700": "#15803d",
"800": "#166534",
"900": "#14532d",
"950": "#052e16"
},
"warning": {
"50": "#fffbeb",
"100": "#fef3c7",
"200": "#fde68a",
"300": "#fcd34d",
"400": "#fbbf24",
"500": "#f59e0b",
"600": "#d97706",
"700": "#b45309",
"800": "#92400e",
"900": "#78350f",
"950": "#451a03"
},
"danger": {
"50": "#fff1f2",
"100": "#ffe4e6",
"200": "#fecdd3",
"300": "#fda4af",
"400": "#fb7185",
"500": "#f43f5e",
"600": "#e11d48",
"700": "#be123c",
"800": "#9f1239",
"900": "#881337",
"950": "#4c0519"
}
},
"surface": {
"app": "#fff5ef",
"nav": "#ffffff",
"nav_active": "#fed7aa",
"card": "#ffffff",
"card_muted": "#fff0e5",
"table": "#ffffff",
"table_header": "#fff5ef",
"input": "#ffffff",
"overlay": "#2b1a12"
},
"text": {
"primary": "#3b2217",
"secondary": "#7c3f1d",
"muted": "#a05d37",
"inverse": "#ffffff",
"link": "#c2410c"
},
"border": {
"default": "#f4d8c6",
"strong": "#e9bc9f",
"subtle": "#fff0e5",
"interactive": "#fb923c"
},
"focus": {
"ring": "#f97316",
"ring_offset": "#fff5ef"
},
"action": {
"primary": {
"bg": "#f97316",
"border": "#f97316",
"text": "#ffffff",
"hover_bg": "#ea580c",
"hover_border": "#ea580c",
"hover_text": "#ffffff"
},
"secondary": {
"bg": "#fff0e5",
"border": "#f4d8c6",
"text": "#3b2217",
"hover_bg": "#fee2cf",
"hover_border": "#e9bc9f",
"hover_text": "#2a170f"
},
"ghost": {
"bg": "#fff5ef",
"border": "#f4d8c6",
"text": "#c2410c",
"hover_bg": "#fff0e5",
"hover_border": "#e9bc9f",
"hover_text": "#7c2d12"
},
"danger": {
"bg": "#ffe4e6",
"border": "#fecdd3",
"text": "#be123c",
"hover_bg": "#fecdd3",
"hover_border": "#fda4af",
"hover_text": "#9f1239"
}
},
"state": {
"info": {
"bg": "#e0e7ff",
"border": "#c7d2fe",
"text": "#4338ca"
},
"success": {
"bg": "#dcfce7",
"border": "#bbf7d0",
"text": "#166534"
},
"warning": {
"bg": "#ffedd5",
"border": "#fed7aa",
"text": "#9a3412"
},
"danger": {
"bg": "#ffe4e6",
"border": "#fecdd3",
"text": "#be123c"
}
}
},
"dark": {
"scale": {
"brand": {
"50": "#fff7ed",
"100": "#ffedd5",
"200": "#fed7aa",
"300": "#fdba74",
"400": "#fb923c",
"500": "#f97316",
"600": "#ea580c",
"700": "#c2410c",
"800": "#9a3412",
"900": "#7c2d12",
"950": "#431407"
},
"info": {
"50": "#eef2ff",
"100": "#e0e7ff",
"200": "#c7d2fe",
"300": "#a5b4fc",
"400": "#818cf8",
"500": "#6366f1",
"600": "#4f46e5",
"700": "#4338ca",
"800": "#3730a3",
"900": "#312e81",
"950": "#1e1b4b"
},
"success": {
"50": "#f0fdf4",
"100": "#dcfce7",
"200": "#bbf7d0",
"300": "#86efac",
"400": "#4ade80",
"500": "#22c55e",
"600": "#16a34a",
"700": "#15803d",
"800": "#166534",
"900": "#14532d",
"950": "#052e16"
},
"warning": {
"50": "#fffbeb",
"100": "#fef3c7",
"200": "#fde68a",
"300": "#fcd34d",
"400": "#fbbf24",
"500": "#f59e0b",
"600": "#d97706",
"700": "#b45309",
"800": "#92400e",
"900": "#78350f",
"950": "#451a03"
},
"danger": {
"50": "#fff1f2",
"100": "#ffe4e6",
"200": "#fecdd3",
"300": "#fda4af",
"400": "#fb7185",
"500": "#f43f5e",
"600": "#e11d48",
"700": "#be123c",
"800": "#9f1239",
"900": "#881337",
"950": "#4c0519"
}
},
"surface": {
"app": "#24140d",
"nav": "#2f1b12",
"nav_active": "#c2410c",
"card": "#2f1b12",
"card_muted": "#3a2419",
"table": "#2f1b12",
"table_header": "#3a2419",
"input": "#1c100a",
"overlay": "#09090b"
},
"text": {
"primary": "#ffe8d7",
"secondary": "#efb58f",
"muted": "#cb8f6b",
"inverse": "#24140d",
"link": "#fdba74"
},
"border": {
"default": "#4a2f20",
"strong": "#5d3b2a",
"subtle": "#3a2419",
"interactive": "#f97316"
},
"focus": {
"ring": "#fb923c",
"ring_offset": "#24140d"
},
"action": {
"primary": {
"bg": "#f97316",
"border": "#f97316",
"text": "#1c100a",
"hover_bg": "#fb923c",
"hover_border": "#fb923c",
"hover_text": "#1c100a"
},
"secondary": {
"bg": "#3a2419",
"border": "#5d3b2a",
"text": "#ffe8d7",
"hover_bg": "#4a2f20",
"hover_border": "#6f4834",
"hover_text": "#ffffff"
},
"ghost": {
"bg": "#24140d",
"border": "#4a2f20",
"text": "#efb58f",
"hover_bg": "#3a2419",
"hover_border": "#5d3b2a",
"hover_text": "#ffe8d7"
},
"danger": {
"bg": "#4c0519",
"border": "#9f1239",
"text": "#fecdd3",
"hover_bg": "#5d0b26",
"hover_border": "#be123c",
"hover_text": "#ffe4e6"
}
},
"state": {
"info": {
"bg": "#312e81",
"border": "#4338ca",
"text": "#c7d2fe"
},
"success": {
"bg": "#14532d",
"border": "#15803d",
"text": "#bbf7d0"
},
"warning": {
"bg": "#7c2d12",
"border": "#c2410c",
"text": "#fed7aa"
},
"danger": {
"bg": "#4c0519",
"border": "#be123c",
"text": "#fecdd3"
}
}
}
}
};

View file

@ -0,0 +1,126 @@
#!/usr/bin/env python3
from __future__ import annotations
import ast
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SETTINGS_PATH = ROOT / "app" / "settings.py"
ENV_EXAMPLE_PATH = ROOT / ".env.example"
SETTINGS_ENV_HELPERS = {"_env_bool", "_env_int", "_env_float", "_env_str"}
ALLOWED_ENV_EXAMPLE_EXTRAS = {
"POSTGRES_DB",
"POSTGRES_USER",
"POSTGRES_PASSWORD",
"TEST_DATABASE_URL",
"SCHOLARR_IMAGE",
"APP_HOST",
"APP_PORT",
"APP_HOST_PORT",
"APP_RELOAD",
"FRONTEND_HOST_PORT",
"CHOKIDAR_USEPOLLING",
"VITE_DEV_API_PROXY_TARGET",
"MIGRATE_ON_START",
"BOOTSTRAP_ADMIN_ON_START",
"BOOTSTRAP_ADMIN_EMAIL",
"BOOTSTRAP_ADMIN_PASSWORD",
"BOOTSTRAP_ADMIN_FORCE_PASSWORD",
"DB_WAIT_TIMEOUT_SECONDS",
"DB_WAIT_INTERVAL_SECONDS",
}
def _call_name(node: ast.AST) -> str | None:
if not isinstance(node, ast.Call):
return None
func = node.func
if isinstance(func, ast.Name):
return func.id
if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name):
return f"{func.value.id}.{func.attr}"
return None
def _first_str_arg(node: ast.Call) -> str | None:
if not node.args:
return None
first = node.args[0]
if isinstance(first, ast.Constant) and isinstance(first.value, str):
return first.value
return None
def _settings_env_names() -> set[str]:
tree = ast.parse(SETTINGS_PATH.read_text(encoding="utf-8"))
names: set[str] = set()
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
call_name = _call_name(node)
if call_name is None:
continue
if call_name != "os.getenv" and call_name not in SETTINGS_ENV_HELPERS:
continue
env_name = _first_str_arg(node)
if not env_name:
continue
names.add(env_name)
return names
def _env_example_names() -> tuple[set[str], set[str]]:
names: set[str] = set()
duplicates: set[str] = set()
for raw_line in ENV_EXAMPLE_PATH.read_text(encoding="utf-8").splitlines():
line = raw_line.strip()
if not line or line.startswith("#"):
continue
if "=" not in line:
continue
key = line.split("=", 1)[0].strip()
if not key:
continue
if key in names:
duplicates.add(key)
names.add(key)
return names, duplicates
def main() -> int:
settings_names = _settings_env_names()
env_names, duplicate_names = _env_example_names()
missing = sorted(settings_names - env_names)
unknown = sorted(env_names - settings_names - ALLOWED_ENV_EXAMPLE_EXTRAS)
duplicates = sorted(duplicate_names)
if missing or unknown or duplicates:
print("Environment contract check failed.")
if missing:
print("Missing from .env.example (referenced in app/settings.py):")
for name in missing:
print(f"- {name}")
if unknown:
print("Unknown keys in .env.example (not in app/settings.py or allowlist):")
for name in unknown:
print(f"- {name}")
if duplicates:
print("Duplicate keys in .env.example:")
for name in duplicates:
print(f"- {name}")
return 1
print(
"Environment contract check passed: "
f"{len(settings_names)} settings keys + {len(ALLOWED_ENV_EXAMPLE_EXTRAS)} allowed extras."
)
return 0
if __name__ == "__main__":
sys.exit(main())

63
scripts/db/backup_full.sh Executable file
View file

@ -0,0 +1,63 @@
#!/usr/bin/env bash
set -euo pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
project_root="$(cd "${script_dir}/../.." && pwd)"
usage() {
cat <<'USAGE'
Usage: scripts/db/backup_full.sh [--plain]
Creates a PostgreSQL logical backup from the running `db` compose service.
Options:
--plain Write a plain SQL dump instead of custom-format dump.
-h, --help Show this help.
Environment:
BACKUP_DIR Destination directory (default: <repo>/backups)
BACKUP_PREFIX File prefix (default: scholarr)
USE_DEV_COMPOSE Set to 1 to include docker-compose.dev.yml
USAGE
}
format="custom"
while (($#)); do
case "$1" in
--plain)
format="plain"
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "Unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
compose_cmd=(docker compose -f "${project_root}/docker-compose.yml")
if [[ "${USE_DEV_COMPOSE:-0}" == "1" && -f "${project_root}/docker-compose.dev.yml" ]]; then
compose_cmd+=(-f "${project_root}/docker-compose.dev.yml")
fi
backup_dir="${BACKUP_DIR:-${project_root}/backups}"
backup_prefix="${BACKUP_PREFIX:-scholarr}"
mkdir -p "${backup_dir}"
timestamp="$(date -u +"%Y%m%dT%H%M%SZ")"
if [[ "${format}" == "plain" ]]; then
backup_file="${backup_dir}/${backup_prefix}_${timestamp}.sql"
dump_cmd='pg_dump --no-owner --no-acl -U "$POSTGRES_USER" "$POSTGRES_DB"'
else
backup_file="${backup_dir}/${backup_prefix}_${timestamp}.dump"
dump_cmd='pg_dump --format=custom --no-owner --no-acl -U "$POSTGRES_USER" "$POSTGRES_DB"'
fi
"${compose_cmd[@]}" exec -T db sh -lc "${dump_cmd}" > "${backup_file}"
echo "Backup created: ${backup_file}"

50
scripts/db/check_integrity.py Executable file
View file

@ -0,0 +1,50 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import asyncio
import json
from app.db.session import get_session_factory
from app.services.domains.dbops import collect_integrity_report
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Run database integrity checks.")
parser.add_argument(
"--strict-warnings",
action="store_true",
help="Return non-zero exit code if any warning is present.",
)
return parser
async def _run() -> dict:
session_factory = get_session_factory()
async with session_factory() as db_session:
return await collect_integrity_report(db_session)
def _exit_code(report: dict, *, strict_warnings: bool) -> int:
if report.get("status") == "failed":
return 1
if strict_warnings and report.get("warnings"):
return 2
return 0
def main() -> int:
args = build_parser().parse_args()
try:
report = asyncio.run(_run())
except Exception as exc:
print(json.dumps({"status": "failed", "error": str(exc)}, indent=2))
return 1
print(json.dumps(report, indent=2))
return _exit_code(report, strict_warnings=args.strict_warnings)
if __name__ == "__main__":
raise SystemExit(main())

View file

@ -0,0 +1,66 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import asyncio
import json
from app.db.session import get_session_factory
from app.services.domains.dbops import run_publication_link_repair
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Repair scholar-publication links for a user with audit logging.",
)
parser.add_argument("--user-id", type=int, required=True, help="Target user ID.")
parser.add_argument(
"--scholar-profile-id",
type=int,
action="append",
default=[],
help="Optional scholar_profile_id filter. Repeat for multiple values.",
)
parser.add_argument("--apply", action="store_true", help="Apply changes. Default is dry-run.")
parser.add_argument(
"--gc-orphan-publications",
action="store_true",
help="Delete publications with zero links after cleanup.",
)
parser.add_argument(
"--requested-by",
default="",
help="Operator identifier for audit logs (email/name/ticket).",
)
return parser
async def _run(args: argparse.Namespace) -> dict:
session_factory = get_session_factory()
async with session_factory() as db_session:
return await run_publication_link_repair(
db_session,
user_id=args.user_id,
scholar_profile_ids=args.scholar_profile_id,
dry_run=not args.apply,
gc_orphan_publications=args.gc_orphan_publications,
requested_by=args.requested_by,
)
def main() -> int:
parser = build_parser()
args = parser.parse_args()
try:
result = asyncio.run(_run(args))
except Exception as exc:
print(json.dumps({"status": "failed", "error": str(exc)}, indent=2))
return 1
print(json.dumps(result, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())

79
scripts/db/restore_dump.sh Executable file
View file

@ -0,0 +1,79 @@
#!/usr/bin/env bash
set -euo pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
project_root="$(cd "${script_dir}/../.." && pwd)"
usage() {
cat <<'USAGE'
Usage: scripts/db/restore_dump.sh --file <path> [--wipe-public]
Restores a PostgreSQL dump into the running `db` compose service.
Options:
--file <path> Required. Backup file (.dump custom format or .sql plain).
--wipe-public Drop and recreate public schema before restore.
-h, --help Show this help.
Environment:
USE_DEV_COMPOSE Set to 1 to include docker-compose.dev.yml
USAGE
}
dump_file=""
wipe_public="0"
while (($#)); do
case "$1" in
--file)
dump_file="${2:-}"
shift 2
;;
--wipe-public)
wipe_public="1"
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "Unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
if [[ -z "${dump_file}" ]]; then
echo "--file is required" >&2
usage >&2
exit 2
fi
if [[ ! -f "${dump_file}" ]]; then
echo "Dump file not found: ${dump_file}" >&2
exit 2
fi
compose_cmd=(docker compose -f "${project_root}/docker-compose.yml")
if [[ "${USE_DEV_COMPOSE:-0}" == "1" && -f "${project_root}/docker-compose.dev.yml" ]]; then
compose_cmd+=(-f "${project_root}/docker-compose.dev.yml")
fi
if [[ "${wipe_public}" == "1" ]]; then
"${compose_cmd[@]}" exec -T db sh -lc 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" "$POSTGRES_DB" -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"'
fi
case "${dump_file}" in
*.dump)
cat "${dump_file}" | "${compose_cmd[@]}" exec -T db sh -lc 'pg_restore --no-owner --no-acl -U "$POSTGRES_USER" -d "$POSTGRES_DB" -'
;;
*.sql)
cat "${dump_file}" | "${compose_cmd[@]}" exec -T db sh -lc 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" "$POSTGRES_DB"'
;;
*)
echo "Unsupported file extension. Expected .dump or .sql" >&2
exit 2
;;
esac
echo "Restore completed: ${dump_file}"

View file

@ -21,6 +21,9 @@ RESET_SQL = text(
TRUNCATE TABLE TRUNCATE TABLE
author_search_cache_entries, author_search_cache_entries,
author_search_runtime_state, author_search_runtime_state,
data_repair_jobs,
publication_pdf_job_events,
publication_pdf_jobs,
ingestion_queue_items, ingestion_queue_items,
scholar_publications, scholar_publications,
crawl_runs, crawl_runs,

View file

@ -10,6 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.api.runtime_deps import get_scholar_source from app.api.runtime_deps import get_scholar_source
from app.main import app from app.main import app
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.settings import application as user_settings_service from app.services.domains.settings import application as user_settings_service
from app.services.domains.scholar.source import FetchResult from app.services.domains.scholar.source import FetchResult
from app.settings import settings from app.settings import settings
@ -65,6 +66,49 @@ def _assert_safety_state_contract(payload: dict[str, object]) -> None:
assert set(counters.keys()) == SAFETY_COUNTER_KEYS assert set(counters.keys()) == SAFETY_COUNTER_KEYS
async def _seed_publication_link_for_user(
db_session: AsyncSession,
*,
user_id: int,
scholar_id: str,
title: str,
fingerprint: str,
) -> tuple[int, int]:
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, :scholar_id, :display_name, true)
RETURNING id
"""
),
{"user_id": user_id, "scholar_id": scholar_id, "display_name": scholar_id},
)
scholar_profile_id = int(scholar_result.scalar_one())
publication_result = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 4)
RETURNING id
"""
),
{"fingerprint": fingerprint, "title_raw": title, "title_normalized": title.lower()},
)
publication_id = int(publication_result.scalar_one())
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
VALUES (:scholar_profile_id, :publication_id, false)
"""
),
{"scholar_profile_id": scholar_profile_id, "publication_id": publication_id},
)
await db_session.commit()
return scholar_profile_id, publication_id
@pytest.mark.integration @pytest.mark.integration
@pytest.mark.db @pytest.mark.db
@pytest.mark.asyncio @pytest.mark.asyncio
@ -321,6 +365,265 @@ async def test_api_admin_user_management_endpoints(db_session: AsyncSession) ->
assert created_exists.scalar_one() == 1 assert created_exists.scalar_one() == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_integrity_and_repair_flow(db_session: AsyncSession) -> None:
await insert_user(db_session, email="api-admin-dbops@example.com", password="admin-password", is_admin=True)
target_user_id = await insert_user(db_session, email="api-dbops-target@example.com", password="target-password")
_scholar_id, publication_id = await _seed_publication_link_for_user(
db_session,
user_id=target_user_id,
scholar_id="dbopsTarget01",
title="DB Ops Target Paper",
fingerprint=f"{(target_user_id + 31):064x}",
)
client = TestClient(app)
login_user(client, email="api-admin-dbops@example.com", password="admin-password")
headers = _api_csrf_headers(client)
integrity_response = client.get("/api/v1/admin/db/integrity")
assert integrity_response.status_code == 200
integrity_payload = integrity_response.json()["data"]
assert integrity_payload["status"] in {"ok", "warning", "failed"}
assert any(
check["name"] == "missing_pdf_url"
for check in integrity_payload["checks"]
)
repair_response = client.post(
"/api/v1/admin/db/repairs/publication-links",
json={"user_id": target_user_id, "dry_run": True},
headers=headers,
)
assert repair_response.status_code == 200
repair_data = repair_response.json()["data"]
assert repair_data["status"] == "completed"
assert bool(repair_data["summary"]["dry_run"]) is True
job_id = int(repair_data["job_id"])
jobs_response = client.get("/api/v1/admin/db/repair-jobs?limit=10")
assert jobs_response.status_code == 200
jobs = jobs_response.json()["data"]["jobs"]
assert any(int(job["id"]) == job_id for job in jobs)
pdf_queue_response = client.get("/api/v1/admin/db/pdf-queue?limit=20")
assert pdf_queue_response.status_code == 200
pdf_queue_payload = pdf_queue_response.json()["data"]
assert isinstance(pdf_queue_payload["items"], list)
assert int(pdf_queue_payload["page"]) == 1
assert int(pdf_queue_payload["page_size"]) == 20
assert int(pdf_queue_payload["total_count"]) >= 0
assert isinstance(pdf_queue_payload["has_next"], bool)
assert isinstance(pdf_queue_payload["has_prev"], bool)
page_two_response = client.get("/api/v1/admin/db/pdf-queue?page=2&page_size=1")
assert page_two_response.status_code == 200
page_two_payload = page_two_response.json()["data"]
assert int(page_two_payload["page"]) == 2
assert int(page_two_payload["page_size"]) == 1
assert page_two_payload["has_prev"] is True
untracked_response = client.get("/api/v1/admin/db/pdf-queue?limit=20&status=untracked")
assert untracked_response.status_code == 200
assert any(
item["status"] == "untracked"
for item in untracked_response.json()["data"]["items"]
)
link_count_result = await db_session.execute(
text("SELECT count(*) FROM scholar_publications WHERE publication_id = :publication_id"),
{"publication_id": publication_id},
)
assert int(link_count_result.scalar_one()) == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_pdf_queue_requeue_endpoint(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
await insert_user(db_session, email="api-admin-pdf@example.com", password="admin-password", is_admin=True)
target_user_id = await insert_user(db_session, email="api-pdf-target@example.com", password="target-password")
_scholar_id, publication_id = await _seed_publication_link_for_user(
db_session,
user_id=target_user_id,
scholar_id="dbopsPdf01",
title="PDF Queue Target",
fingerprint=f"{(target_user_id + 73):064x}",
)
monkeypatch.setattr(
"app.services.domains.publications.pdf_queue._schedule_rows",
lambda **_kwargs: None,
)
client = TestClient(app)
login_user(client, email="api-admin-pdf@example.com", password="admin-password")
headers = _api_csrf_headers(client)
first_response = client.post(
f"/api/v1/admin/db/pdf-queue/{publication_id}/requeue",
headers=headers,
)
assert first_response.status_code == 200
first_data = first_response.json()["data"]
assert first_data["publication_id"] == publication_id
assert first_data["queued"] is True
assert first_data["status"] == "queued"
second_response = client.post(
f"/api/v1/admin/db/pdf-queue/{publication_id}/requeue",
headers=headers,
)
assert second_response.status_code == 200
second_data = second_response.json()["data"]
assert second_data["queued"] is False
assert second_data["status"] == "blocked"
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_pdf_queue_requeue_all_endpoint(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch,
) -> None:
await insert_user(db_session, email="api-admin-pdf-all@example.com", password="admin-password", is_admin=True)
target_user_id = await insert_user(db_session, email="api-pdf-all-target@example.com", password="target-password")
_scholar_id, _publication_id = await _seed_publication_link_for_user(
db_session,
user_id=target_user_id,
scholar_id="dbopsPdfAll01",
title="PDF Queue All Target",
fingerprint=f"{(target_user_id + 83):064x}",
)
monkeypatch.setattr(
"app.services.domains.publications.pdf_queue._schedule_rows",
lambda **_kwargs: None,
)
client = TestClient(app)
login_user(client, email="api-admin-pdf-all@example.com", password="admin-password")
headers = _api_csrf_headers(client)
response = client.post(
"/api/v1/admin/db/pdf-queue/requeue-all?limit=500",
headers=headers,
)
assert response.status_code == 200
payload = response.json()["data"]
assert int(payload["requested_count"]) >= 1
assert int(payload["queued_count"]) >= 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_forbidden_for_non_admin_and_validates_scope(db_session: AsyncSession) -> None:
await insert_user(db_session, email="api-admin-dbops2@example.com", password="admin-password", is_admin=True)
member_user_id = await insert_user(db_session, email="api-dbops-member@example.com", password="member-password")
client = TestClient(app)
login_user(client, email="api-dbops-member@example.com", password="member-password")
headers = _api_csrf_headers(client)
forbidden_integrity = client.get("/api/v1/admin/db/integrity")
assert forbidden_integrity.status_code == 403
assert forbidden_integrity.json()["error"]["code"] == "forbidden"
forbidden_pdf_queue = client.get("/api/v1/admin/db/pdf-queue")
assert forbidden_pdf_queue.status_code == 403
assert forbidden_pdf_queue.json()["error"]["code"] == "forbidden"
forbidden_requeue = client.post(
"/api/v1/admin/db/pdf-queue/1/requeue",
headers=headers,
)
assert forbidden_requeue.status_code == 403
assert forbidden_requeue.json()["error"]["code"] == "forbidden"
forbidden_requeue_all = client.post(
"/api/v1/admin/db/pdf-queue/requeue-all?limit=100",
headers=headers,
)
assert forbidden_requeue_all.status_code == 403
assert forbidden_requeue_all.json()["error"]["code"] == "forbidden"
forbidden_repair = client.post(
"/api/v1/admin/db/repairs/publication-links",
json={"user_id": member_user_id, "dry_run": True},
headers=headers,
)
assert forbidden_repair.status_code == 403
assert forbidden_repair.json()["error"]["code"] == "forbidden"
admin_headers = _api_bootstrap_csrf_headers(client)
admin_login = client.post(
"/api/v1/auth/login",
json={"email": "api-admin-dbops2@example.com", "password": "admin-password"},
headers=admin_headers,
)
assert admin_login.status_code == 200
post_login_headers = {"X-CSRF-Token": admin_login.json()["data"]["csrf_token"]}
invalid_scope = client.post(
"/api/v1/admin/db/repairs/publication-links",
json={"user_id": member_user_id, "dry_run": True},
headers=post_login_headers,
)
assert invalid_scope.status_code == 400
assert invalid_scope.json()["error"]["code"] == "invalid_repair_scope"
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_admin_dbops_all_users_apply_requires_confirmation(db_session: AsyncSession) -> None:
await insert_user(db_session, email="api-admin-dbops3@example.com", password="admin-password", is_admin=True)
first_user_id = await insert_user(db_session, email="api-dbops-a@example.com", password="user-password")
second_user_id = await insert_user(db_session, email="api-dbops-b@example.com", password="user-password")
await _seed_publication_link_for_user(
db_session,
user_id=first_user_id,
scholar_id="dbopsAll01",
title="DB Ops All User Paper One",
fingerprint=f"{(first_user_id + 61):064x}",
)
await _seed_publication_link_for_user(
db_session,
user_id=second_user_id,
scholar_id="dbopsAll02",
title="DB Ops All User Paper Two",
fingerprint=f"{(second_user_id + 71):064x}",
)
client = TestClient(app)
login_user(client, email="api-admin-dbops3@example.com", password="admin-password")
headers = _api_csrf_headers(client)
missing_confirmation = client.post(
"/api/v1/admin/db/repairs/publication-links",
json={"scope_mode": "all_users", "dry_run": False},
headers=headers,
)
assert missing_confirmation.status_code == 422
assert "confirmation_text" in str(missing_confirmation.json())
apply_response = client.post(
"/api/v1/admin/db/repairs/publication-links",
json={
"scope_mode": "all_users",
"dry_run": False,
"confirmation_text": "REPAIR ALL USERS",
},
headers=headers,
)
assert apply_response.status_code == 200
apply_data = apply_response.json()["data"]
assert apply_data["status"] == "completed"
assert apply_data["scope"]["scope_mode"] == "all_users"
assert int(apply_data["summary"]["links_deleted"]) >= 2
remaining_links = await db_session.execute(text("SELECT count(*) FROM scholar_publications"))
assert int(remaining_links.scalar_one()) == 0
@pytest.mark.integration @pytest.mark.integration
@pytest.mark.db @pytest.mark.db
@pytest.mark.asyncio @pytest.mark.asyncio
@ -1400,12 +1703,19 @@ async def test_api_publications_list_and_mark_read(db_session: AsyncSession) ->
assert list_response.status_code == 200 assert list_response.status_code == 200
data = list_response.json()["data"] data = list_response.json()["data"]
assert data["mode"] == "all" assert data["mode"] == "all"
assert data["favorite_only"] is False
assert data["total_count"] == 2 assert data["total_count"] == 2
assert data["unread_count"] == 2 assert data["unread_count"] == 2
assert data["favorites_count"] == 0
assert data["latest_count"] == 0 assert data["latest_count"] == 0
assert data["new_count"] == data["latest_count"] assert data["new_count"] == data["latest_count"]
assert data["page"] == 1
assert data["page_size"] == 100
assert data["has_prev"] is False
assert data["has_next"] is False
assert isinstance(data["publications"], list) assert isinstance(data["publications"], list)
assert len(data["publications"]) == 2 assert len(data["publications"]) == 2
assert all(bool(item["is_favorite"]) is False for item in data["publications"])
pdf_urls = {item["title"]: item["pdf_url"] for item in data["publications"]} pdf_urls = {item["title"]: item["pdf_url"] for item in data["publications"]}
assert pdf_urls["Paper A"] == "https://example.org/paper-a.pdf" assert pdf_urls["Paper A"] == "https://example.org/paper-a.pdf"
assert pdf_urls["Paper B"] is None assert pdf_urls["Paper B"] is None
@ -1465,6 +1775,220 @@ async def test_api_publications_list_and_mark_read(db_session: AsyncSession) ->
assert mark_response.json()["data"]["updated_count"] == 1 assert mark_response.json()["data"]["updated_count"] == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_publications_list_supports_pagination(db_session: AsyncSession) -> None:
user_id = await insert_user(
db_session,
email="api-pubs-paging@example.com",
password="api-password",
)
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, :scholar_id, :display_name, true)
RETURNING id
"""
),
{
"user_id": user_id,
"scholar_id": "pagingScholar01",
"display_name": "Paging Scholar",
},
)
scholar_profile_id = int(scholar_result.scalar_one())
publication_ids: list[int] = []
for index in range(3):
created = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 1)
RETURNING id
"""
),
{
"fingerprint": f"{(user_id + 500 + index):064x}",
"title_raw": f"Paged Paper {index}",
"title_normalized": f"paged paper {index}",
},
)
publication_ids.append(int(created.scalar_one()))
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
VALUES
(:scholar_profile_id, :publication_1, false, false),
(:scholar_profile_id, :publication_2, false, false),
(:scholar_profile_id, :publication_3, false, false)
"""
),
{
"scholar_profile_id": scholar_profile_id,
"publication_1": publication_ids[0],
"publication_2": publication_ids[1],
"publication_3": publication_ids[2],
},
)
await db_session.commit()
client = TestClient(app)
login_user(client, email="api-pubs-paging@example.com", password="api-password")
first_page = client.get("/api/v1/publications?mode=all&page=1&page_size=2")
assert first_page.status_code == 200
first_data = first_page.json()["data"]
assert first_data["total_count"] == 3
assert first_data["page"] == 1
assert first_data["page_size"] == 2
assert first_data["has_prev"] is False
assert first_data["has_next"] is True
assert len(first_data["publications"]) == 2
second_page = client.get("/api/v1/publications?mode=all&page=2&page_size=2")
assert second_page.status_code == 200
second_data = second_page.json()["data"]
assert second_data["total_count"] == 3
assert second_data["page"] == 2
assert second_data["page_size"] == 2
assert second_data["has_prev"] is True
assert second_data["has_next"] is False
assert len(second_data["publications"]) == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_publications_favorite_toggle_and_filter(db_session: AsyncSession) -> None:
user_id = await insert_user(
db_session,
email="api-pubs-favorites@example.com",
password="api-password",
)
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, :scholar_id, :display_name, true)
RETURNING id
"""
),
{
"user_id": user_id,
"scholar_id": "favoriteScholar01",
"display_name": "Favorite Scholar",
},
)
scholar_profile_id = int(scholar_result.scalar_one())
publication_a_result = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 9)
RETURNING id
"""
),
{
"fingerprint": f"{(user_id + 101):064x}",
"title_raw": "Favorite Target",
"title_normalized": "favorite target",
},
)
publication_a_id = int(publication_a_result.scalar_one())
publication_b_result = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 2)
RETURNING id
"""
),
{
"fingerprint": f"{(user_id + 102):064x}",
"title_raw": "Not Favorite",
"title_normalized": "not favorite",
},
)
publication_b_id = int(publication_b_result.scalar_one())
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
VALUES
(:scholar_profile_id, :publication_a_id, false, false),
(:scholar_profile_id, :publication_b_id, false, false)
"""
),
{
"scholar_profile_id": scholar_profile_id,
"publication_a_id": publication_a_id,
"publication_b_id": publication_b_id,
},
)
await db_session.commit()
client = TestClient(app)
login_user(client, email="api-pubs-favorites@example.com", password="api-password")
headers = _api_csrf_headers(client)
set_response = client.post(
f"/api/v1/publications/{publication_a_id}/favorite",
json={"scholar_profile_id": scholar_profile_id, "is_favorite": True},
headers=headers,
)
assert set_response.status_code == 200
set_data = set_response.json()["data"]
assert set_data["publication"]["publication_id"] == publication_a_id
assert set_data["publication"]["is_favorite"] is True
favorite_only_response = client.get("/api/v1/publications?mode=all&favorite_only=true")
assert favorite_only_response.status_code == 200
favorite_only_data = favorite_only_response.json()["data"]
assert favorite_only_data["favorite_only"] is True
assert favorite_only_data["favorites_count"] == 1
assert favorite_only_data["total_count"] == 1
assert len(favorite_only_data["publications"]) == 1
assert int(favorite_only_data["publications"][0]["publication_id"]) == publication_a_id
clear_response = client.post(
f"/api/v1/publications/{publication_a_id}/favorite",
json={"scholar_profile_id": scholar_profile_id, "is_favorite": False},
headers=headers,
)
assert clear_response.status_code == 200
clear_data = clear_response.json()["data"]
assert clear_data["publication"]["publication_id"] == publication_a_id
assert clear_data["publication"]["is_favorite"] is False
favorite_only_after_clear_response = client.get("/api/v1/publications?mode=all&favorite_only=true")
assert favorite_only_after_clear_response.status_code == 200
favorite_only_after_clear_data = favorite_only_after_clear_response.json()["data"]
assert favorite_only_after_clear_data["favorites_count"] == 0
assert favorite_only_after_clear_data["total_count"] == 0
assert favorite_only_after_clear_data["publications"] == []
favorite_state_result = await db_session.execute(
text(
"""
SELECT is_favorite
FROM scholar_publications
WHERE scholar_profile_id = :scholar_profile_id
AND publication_id = :publication_id
"""
),
{
"scholar_profile_id": scholar_profile_id,
"publication_id": publication_a_id,
},
)
assert bool(favorite_state_result.scalar_one()) is False
@pytest.mark.integration @pytest.mark.integration
@pytest.mark.db @pytest.mark.db
@pytest.mark.asyncio @pytest.mark.asyncio
@ -1521,20 +2045,14 @@ async def test_api_publications_list_schedules_background_enrichment(
) )
await db_session.commit() await db_session.commit()
async def _fail_resolver(*args, **kwargs): async def _fake_schedule(db_session, *, user_id: int, request_email: str | None, items, max_items: int):
raise AssertionError("list endpoint should not resolve OA metadata inline") assert db_session is not None
async def _fake_schedule(*, user_id: int, request_email: str | None, items, max_items: int):
assert user_id > 0 assert user_id > 0
assert request_email == "api-pubs-bg@example.com" assert request_email == "api-pubs-bg@example.com"
assert int(max_items) > 0 assert int(max_items) > 0
assert any(int(item.publication_id) == publication_id for item in items) assert any(int(item.publication_id) == publication_id for item in items)
return 1 return 1
monkeypatch.setattr(
"app.services.domains.publications.listing.resolve_publication_oa_metadata",
_fail_resolver,
)
monkeypatch.setattr( monkeypatch.setattr(
"app.services.domains.publications.application.schedule_missing_pdf_enrichment_for_user", "app.services.domains.publications.application.schedule_missing_pdf_enrichment_for_user",
_fake_schedule, _fake_schedule,
@ -1549,12 +2067,13 @@ async def test_api_publications_list_schedules_background_enrichment(
assert len(data["publications"]) == 1 assert len(data["publications"]) == 1
assert int(data["publications"][0]["publication_id"]) == publication_id assert int(data["publications"][0]["publication_id"]) == publication_id
assert data["publications"][0]["pdf_url"] is None assert data["publications"][0]["pdf_url"] is None
assert data["publications"][0]["pdf_status"] in {"untracked", "queued", "running", "failed"}
@pytest.mark.integration @pytest.mark.integration
@pytest.mark.db @pytest.mark.db
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_api_publication_retry_pdf_updates_publication( async def test_api_publication_retry_pdf_queues_resolution_job(
db_session: AsyncSession, db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
) -> None: ) -> None:
@ -1607,13 +2126,47 @@ async def test_api_publication_retry_pdf_updates_publication(
) )
await db_session.commit() await db_session.commit()
async def _fake_resolver(items, *, request_email=None): async def _fake_retry_scheduler(db_session, *, user_id: int, request_email: str | None, item: PublicationListItem):
assert db_session is not None
assert user_id > 0
assert request_email == "api-pubs-retry@example.com" assert request_email == "api-pubs-retry@example.com"
return {int(items[0].publication_id): ("10.1000/retry-target", "https://oa.example/retry.pdf")} assert int(item.publication_id) == publication_id
return True
monkeypatch.setattr( monkeypatch.setattr(
"app.services.domains.publications.listing.resolve_publication_oa_metadata", "app.services.domains.publications.application.schedule_retry_pdf_enrichment_for_row",
_fake_resolver, _fake_retry_scheduler,
)
async def _fake_hydrate(db_session, *, items: list[PublicationListItem]):
assert db_session is not None
assert len(items) == 1
item = items[0]
return [
PublicationListItem(
publication_id=item.publication_id,
scholar_profile_id=item.scholar_profile_id,
scholar_label=item.scholar_label,
title=item.title,
year=item.year,
citation_count=item.citation_count,
venue_text=item.venue_text,
pub_url=item.pub_url,
doi=item.doi,
pdf_url=item.pdf_url,
is_read=item.is_read,
first_seen_at=item.first_seen_at,
is_new_in_latest_run=item.is_new_in_latest_run,
pdf_status="queued",
pdf_attempt_count=1,
pdf_failure_reason="no_pdf_found",
pdf_failure_detail="no_pdf_found",
)
]
monkeypatch.setattr(
"app.services.domains.publications.application.hydrate_pdf_enrichment_state",
_fake_hydrate,
) )
client = TestClient(app) client = TestClient(app)
@ -1627,18 +2180,21 @@ async def test_api_publication_retry_pdf_updates_publication(
) )
assert response.status_code == 200 assert response.status_code == 200
payload = response.json()["data"] payload = response.json()["data"]
assert payload["resolved_pdf"] is True assert payload["queued"] is True
assert payload["resolved_pdf"] is False
assert payload["publication"]["publication_id"] == publication_id assert payload["publication"]["publication_id"] == publication_id
assert payload["publication"]["pdf_url"] == "https://oa.example/retry.pdf" assert payload["publication"]["pdf_url"] is None
assert payload["publication"]["doi"] == "10.1000/retry-target" assert payload["publication"]["pdf_status"] == "queued"
assert payload["publication"]["pdf_attempt_count"] == 1
assert payload["publication"]["pdf_failure_reason"] == "no_pdf_found"
stored = await db_session.execute( stored = await db_session.execute(
text("SELECT doi, pdf_url FROM publications WHERE id = :publication_id"), text("SELECT doi, pdf_url FROM publications WHERE id = :publication_id"),
{"publication_id": publication_id}, {"publication_id": publication_id},
) )
stored_doi, stored_pdf_url = stored.one() stored_doi, stored_pdf_url = stored.one()
assert stored_doi == "10.1000/retry-target" assert stored_doi is None
assert stored_pdf_url == "https://oa.example/retry.pdf" assert stored_pdf_url is None
@pytest.mark.integration @pytest.mark.integration

View file

@ -0,0 +1,252 @@
from __future__ import annotations
import pytest
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.services.domains.dbops import run_publication_link_repair
from tests.integration.helpers import insert_user
async def _insert_scholar_profile(
db_session: AsyncSession,
*,
user_id: int,
scholar_id: str,
display_name: str,
baseline_completed: bool = False,
) -> int:
result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled, baseline_completed)
VALUES (:user_id, :scholar_id, :display_name, true, :baseline_completed)
RETURNING id
"""
),
{
"user_id": user_id,
"scholar_id": scholar_id,
"display_name": display_name,
"baseline_completed": baseline_completed,
},
)
return int(result.scalar_one())
async def _insert_publication(
db_session: AsyncSession,
*,
fingerprint: str,
title_raw: str,
title_normalized: str,
citation_count: int,
) -> int:
result = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, :citation_count)
RETURNING id
"""
),
{
"fingerprint": fingerprint,
"title_raw": title_raw,
"title_normalized": title_normalized,
"citation_count": citation_count,
},
)
return int(result.scalar_one())
async def _insert_scholar_publication_link(
db_session: AsyncSession,
*,
scholar_profile_id: int,
publication_id: int,
) -> None:
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
VALUES (:scholar_profile_id, :publication_id, false)
"""
),
{
"scholar_profile_id": scholar_profile_id,
"publication_id": publication_id,
},
)
async def _insert_queue_item(
db_session: AsyncSession,
*,
user_id: int,
scholar_profile_id: int,
reason: str,
) -> None:
await db_session.execute(
text(
"""
INSERT INTO ingestion_queue_items (user_id, scholar_profile_id, reason)
VALUES (:user_id, :scholar_profile_id, :reason)
"""
),
{
"user_id": user_id,
"scholar_profile_id": scholar_profile_id,
"reason": reason,
},
)
async def _count_rows(
db_session: AsyncSession,
*,
sql: str,
params: dict[str, int],
) -> int:
result = await db_session.execute(text(sql), params)
return int(result.scalar_one())
async def _seed_apply_case(db_session: AsyncSession, *, user_id: int) -> tuple[int, int]:
scholar_profile_id = await _insert_scholar_profile(
db_session,
user_id=user_id,
scholar_id="repairApply01",
display_name="Repair Apply",
baseline_completed=True,
)
publication_id = await _insert_publication(
db_session,
fingerprint=f"{(user_id + 1):064x}",
title_raw="Repair Apply Paper",
title_normalized="repair apply paper",
citation_count=9,
)
await _insert_scholar_publication_link(
db_session,
scholar_profile_id=scholar_profile_id,
publication_id=publication_id,
)
await _insert_queue_item(
db_session,
user_id=user_id,
scholar_profile_id=scholar_profile_id,
reason="manual_repair_test",
)
await db_session.commit()
return scholar_profile_id, publication_id
async def _assert_apply_effects(
db_session: AsyncSession,
*,
scholar_profile_id: int,
publication_id: int,
) -> None:
links_count = await _count_rows(
db_session,
sql="SELECT count(*) FROM scholar_publications WHERE scholar_profile_id = :scholar_profile_id",
params={"scholar_profile_id": scholar_profile_id},
)
queue_count = await _count_rows(
db_session,
sql="SELECT count(*) FROM ingestion_queue_items WHERE scholar_profile_id = :scholar_profile_id",
params={"scholar_profile_id": scholar_profile_id},
)
baseline_completed = await _count_rows(
db_session,
sql=(
"SELECT count(*) FROM scholar_profiles "
"WHERE id = :scholar_profile_id AND baseline_completed = false"
),
params={"scholar_profile_id": scholar_profile_id},
)
publication_count = await _count_rows(
db_session,
sql="SELECT count(*) FROM publications WHERE id = :publication_id",
params={"publication_id": publication_id},
)
assert links_count == 0
assert queue_count == 0
assert baseline_completed == 1
assert publication_count == 0
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_publication_link_repair_dry_run_records_job_without_mutation(
db_session: AsyncSession,
) -> None:
user_id = await insert_user(db_session, email="repair-dry@example.com", password="api-password")
scholar_profile_id = await _insert_scholar_profile(
db_session,
user_id=user_id,
scholar_id="repairDry001",
display_name="Repair Dry",
)
publication_id = await _insert_publication(
db_session,
fingerprint=f"{user_id:064x}",
title_raw="Repair Dry Paper",
title_normalized="repair dry paper",
citation_count=1,
)
await _insert_scholar_publication_link(
db_session,
scholar_profile_id=scholar_profile_id,
publication_id=publication_id,
)
await db_session.commit()
result = await run_publication_link_repair(
db_session,
user_id=user_id,
dry_run=True,
requested_by="test-suite",
)
assert result["status"] == "completed"
assert bool(result["summary"]["dry_run"]) is True
assert int(result["summary"]["links_deleted"]) == 0
links_count = await _count_rows(
db_session,
sql="SELECT count(*) FROM scholar_publications WHERE scholar_profile_id = :scholar_profile_id",
params={"scholar_profile_id": scholar_profile_id},
)
assert links_count == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_publication_link_repair_apply_clears_links_and_can_gc_orphans(
db_session: AsyncSession,
) -> None:
user_id = await insert_user(db_session, email="repair-apply@example.com", password="api-password")
scholar_profile_id, publication_id = await _seed_apply_case(db_session, user_id=user_id)
result = await run_publication_link_repair(
db_session,
user_id=user_id,
dry_run=False,
gc_orphan_publications=True,
requested_by="test-suite",
)
assert result["status"] == "completed"
assert bool(result["summary"]["dry_run"]) is False
assert int(result["summary"]["links_deleted"]) == 1
assert int(result["summary"]["queue_items_deleted"]) == 1
assert int(result["summary"]["scholars_reset"]) == 1
await _assert_apply_effects(
db_session,
scholar_profile_id=scholar_profile_id,
publication_id=publication_id,
)

View file

@ -0,0 +1,70 @@
from __future__ import annotations
import pytest
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.services.domains.dbops.integrity import collect_integrity_report
def _check_counts(report: dict) -> dict[str, int]:
return {row["name"]: int(row["count"]) for row in report["checks"]}
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_collect_integrity_report_ok_on_clean_database(db_session: AsyncSession) -> None:
report = await collect_integrity_report(db_session)
assert report["status"] == "ok"
assert report["failures"] == []
assert report["warnings"] == []
check_counts = _check_counts(report)
assert all(count == 0 for count in check_counts.values())
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_collect_integrity_report_detects_warning_and_failure(db_session: AsyncSession) -> None:
await db_session.execute(
text(
"""
INSERT INTO publications (
fingerprint_sha256,
cluster_id,
title_raw,
title_normalized,
citation_count
)
VALUES (
:fingerprint_sha256,
:cluster_id,
:title_raw,
:title_normalized,
:citation_count
)
"""
),
{
"fingerprint_sha256": f"{1:064x}",
"cluster_id": "legacy-cluster-123",
"title_raw": "Integrity test paper",
"title_normalized": "integrity test paper",
"citation_count": -5,
},
)
await db_session.commit()
report = await collect_integrity_report(db_session)
check_counts = _check_counts(report)
assert report["status"] == "failed"
assert "negative_citation_count" in report["failures"]
assert "legacy_cluster_id_format" in report["warnings"]
assert "orphan_publications" in report["warnings"]
assert check_counts["negative_citation_count"] == 1
assert check_counts["legacy_cluster_id_format"] == 1
assert check_counts["orphan_publications"] == 1

View file

@ -13,10 +13,13 @@ EXPECTED_TABLES = {
"ingestion_queue_items", "ingestion_queue_items",
"author_search_runtime_state", "author_search_runtime_state",
"author_search_cache_entries", "author_search_cache_entries",
"data_repair_jobs",
"publication_pdf_jobs",
"publication_pdf_job_events",
} }
EXPECTED_ENUMS = {"run_status", "run_trigger_type"} EXPECTED_ENUMS = {"run_status", "run_trigger_type"}
EXPECTED_REVISION = "20260220_0011" EXPECTED_REVISION = "20260221_0015"
@pytest.mark.integration @pytest.mark.integration
@ -220,3 +223,44 @@ async def test_user_settings_has_scrape_safety_columns(db_session: AsyncSession)
) )
columns = {row[0] for row in result} columns = {row[0] for row in result}
assert columns == {"scrape_safety_state", "scrape_cooldown_until", "scrape_cooldown_reason"} assert columns == {"scrape_safety_state", "scrape_cooldown_until", "scrape_cooldown_reason"}
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.migrations
@pytest.mark.asyncio
async def test_scholar_publications_has_is_favorite_column(db_session: AsyncSession) -> None:
result = await db_session.execute(
text(
"""
SELECT 1
FROM information_schema.columns
WHERE table_name = 'scholar_publications'
AND column_name = 'is_favorite'
"""
)
)
assert result.scalar_one() == 1
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.migrations
@pytest.mark.asyncio
async def test_user_settings_request_delay_constraint_enforces_two_second_floor(
db_session: AsyncSession,
) -> None:
result = await db_session.execute(
text(
"""
SELECT pg_get_constraintdef(c.oid)
FROM pg_constraint c
JOIN pg_class t ON t.oid = c.conrelid
WHERE t.relname = 'user_settings'
AND c.contype = 'c'
AND pg_get_constraintdef(c.oid) ILIKE '%request_delay_seconds >= 2%'
"""
)
)
definition = result.scalar_one()
assert "request_delay_seconds >= 2" in definition

View file

@ -0,0 +1,109 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
import pytest
from app.db.models import PublicationPdfJob
from app.services.domains.publications import pdf_queue
def _job(
*,
status: str,
attempt_count: int,
last_attempt_at: datetime | None,
) -> PublicationPdfJob:
return PublicationPdfJob(
publication_id=1,
status=status,
attempt_count=attempt_count,
last_attempt_at=last_attempt_at,
)
def test_pdf_queue_auto_enqueue_blocks_recent_attempt(monkeypatch: pytest.MonkeyPatch) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
job = _job(
status=pdf_queue.PDF_STATUS_FAILED,
attempt_count=1,
last_attempt_at=now - timedelta(hours=2),
)
assert pdf_queue._can_enqueue_job(job, force_retry=False) is True
def test_pdf_queue_auto_enqueue_blocks_recent_first_retry(monkeypatch: pytest.MonkeyPatch) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
job = _job(
status=pdf_queue.PDF_STATUS_FAILED,
attempt_count=1,
last_attempt_at=now - timedelta(minutes=20),
)
assert pdf_queue._can_enqueue_job(job, force_retry=False) is False
def test_pdf_queue_auto_enqueue_blocks_after_max_attempts(monkeypatch: pytest.MonkeyPatch) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
job = _job(
status=pdf_queue.PDF_STATUS_FAILED,
attempt_count=3,
last_attempt_at=now - timedelta(days=2),
)
assert pdf_queue._can_enqueue_job(job, force_retry=False) is False
def test_pdf_queue_auto_enqueue_blocks_second_retry_within_day(monkeypatch: pytest.MonkeyPatch) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
job = _job(
status=pdf_queue.PDF_STATUS_FAILED,
attempt_count=2,
last_attempt_at=now - timedelta(hours=2),
)
assert pdf_queue._can_enqueue_job(job, force_retry=False) is False
def test_pdf_queue_manual_requeue_bypasses_cooldown_and_max_attempts(
monkeypatch: pytest.MonkeyPatch,
) -> None:
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
job = _job(
status=pdf_queue.PDF_STATUS_FAILED,
attempt_count=5,
last_attempt_at=now - timedelta(minutes=10),
)
assert pdf_queue._can_enqueue_job(job, force_retry=True) is True
def test_pdf_queue_manual_requeue_still_blocks_when_inflight() -> None:
running = _job(
status=pdf_queue.PDF_STATUS_RUNNING,
attempt_count=1,
last_attempt_at=None,
)
queued = _job(
status=pdf_queue.PDF_STATUS_QUEUED,
attempt_count=1,
last_attempt_at=None,
)
assert pdf_queue._can_enqueue_job(running, force_retry=True) is False
assert pdf_queue._can_enqueue_job(queued, force_retry=True) is False

View file

@ -0,0 +1,45 @@
from __future__ import annotations
from datetime import datetime, timezone
from app.services.domains.ingestion.application import ScholarIngestionService
from app.services.domains.ingestion import scheduler as scheduler_module
from app.settings import settings
def _set_policy_minimum(value: int) -> int:
previous = int(settings.ingestion_min_request_delay_seconds)
object.__setattr__(settings, "ingestion_min_request_delay_seconds", int(value))
return previous
def test_ingestion_effective_request_delay_respects_policy_minimum() -> None:
previous = _set_policy_minimum(8)
try:
assert ScholarIngestionService._effective_request_delay_seconds(7) == 8
assert ScholarIngestionService._effective_request_delay_seconds(12) == 12
finally:
object.__setattr__(settings, "ingestion_min_request_delay_seconds", previous)
def test_scheduler_effective_request_delay_respects_policy_minimum() -> None:
previous = _set_policy_minimum(9)
try:
assert scheduler_module._effective_request_delay_seconds(None) == 9
assert scheduler_module._effective_request_delay_seconds(6) == 9
assert scheduler_module._effective_request_delay_seconds(11) == 11
finally:
object.__setattr__(settings, "ingestion_min_request_delay_seconds", previous)
def test_scheduler_candidate_row_clamps_request_delay() -> None:
previous = _set_policy_minimum(6)
try:
candidate = scheduler_module.SchedulerService._candidate_from_row(
(1, 15, 1, None, None),
now_utc=datetime(2026, 2, 21, tzinfo=timezone.utc),
)
assert candidate is not None
assert candidate.request_delay_seconds == 6
finally:
object.__setattr__(settings, "ingestion_min_request_delay_seconds", previous)

View file

@ -0,0 +1,94 @@
from __future__ import annotations
import pytest
from app.services.domains.unpaywall import pdf_discovery
def test_looks_like_pdf_url_detects_path_and_query_variants() -> None:
assert pdf_discovery.looks_like_pdf_url("https://example.org/file.pdf")
assert pdf_discovery.looks_like_pdf_url("https://example.org/download?target=paper.pdf")
assert not pdf_discovery.looks_like_pdf_url("https://example.org/landing")
assert not pdf_discovery.looks_like_pdf_url(None)
def test_normalized_candidate_urls_extracts_and_resolves_relative_links() -> None:
html = """
<html>
<head>
<base href="https://publisher.example.org/articles/42/" />
<meta name="citation_pdf_url" content="/pdfs/42-main.pdf" />
<link rel="alternate" type="application/pdf" href="https://cdn.example.org/42-supp.pdf" />
</head>
<body>
<a href="appendix.pdf">Appendix</a>
</body>
</html>
"""
candidates = pdf_discovery._normalized_candidate_urls(
page_url="https://publisher.example.org/articles/42",
html=html,
)
assert len(candidates) == 3
assert "https://cdn.example.org/42-supp.pdf" in candidates
assert "https://publisher.example.org/pdfs/42-main.pdf" in candidates
assert "https://publisher.example.org/articles/42/appendix.pdf" in candidates
def test_normalized_candidate_urls_extracts_plain_text_urls() -> None:
html = """
<html>
<body>
Footnote URL: http://www.mext.go.jp/component/english/file.pdf
</body>
</html>
"""
candidates = pdf_discovery._normalized_candidate_urls(
page_url="https://www.science.org/doi/10.1126/science.1239057",
html=html,
)
assert "http://www.mext.go.jp/component/english/file.pdf" in candidates
class _FakeResponse:
def __init__(self, *, status_code: int, content_type: str, text: str) -> None:
self.status_code = status_code
self.headers = {"content-type": content_type}
self.text = text
class _FakeClient:
def __init__(self, pages: dict[str, _FakeResponse]) -> None:
self._pages = pages
async def get(self, url: str, follow_redirects: bool = True): # noqa: ARG002
return self._pages[url]
@pytest.mark.asyncio
async def test_resolve_pdf_from_landing_page_follows_one_hop_html_candidate(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def _skip_wait(*args, **kwargs): # noqa: ARG001, ANN002
return None
monkeypatch.setattr(pdf_discovery, "wait_for_unpaywall_slot", _skip_wait)
landing_url = "https://example.org/article"
hop_url = "https://example.org/doi/full/abc"
pdf_url = "https://downloads.example.org/archive/paper.pdf"
client = _FakeClient(
{
landing_url: _FakeResponse(
status_code=200,
content_type="text/html",
text=f"<html><body><a href=\"{hop_url}\">View article</a></body></html>",
),
hop_url: _FakeResponse(
status_code=200,
content_type="text/html",
text=f"<html><body>Download here: {pdf_url}</body></html>",
),
}
)
resolved = await pdf_discovery.resolve_pdf_from_landing_page(client, page_url=landing_url)
assert resolved == pdf_url

View file

@ -0,0 +1,89 @@
from __future__ import annotations
from datetime import datetime, timezone
import pytest
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.unpaywall import application as unpaywall_app
class _DummyAsyncClient:
def __init__(self, *args, **kwargs) -> None:
self.args = args
self.kwargs = kwargs
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return None
def _item(publication_id: int) -> PublicationListItem:
return PublicationListItem(
publication_id=publication_id,
scholar_profile_id=1,
scholar_label="Shinya Yamanaka",
title="Induction of pluripotent stem cells",
year=2007,
citation_count=1000,
venue_text="Cell",
pub_url="https://doi.org/10.1016/j.cell.2007.11.019",
doi=None,
pdf_url=None,
is_read=False,
first_seen_at=datetime.now(timezone.utc),
is_new_in_latest_run=True,
)
@pytest.mark.asyncio
async def test_unpaywall_resolve_prefers_direct_pdf_without_landing_crawl(monkeypatch: pytest.MonkeyPatch) -> None:
payload = {
"doi": "10.1016/j.cell.2007.11.019",
"best_oa_location": {
"url_for_pdf": "https://oa.example.org/article.pdf",
"url": "https://oa.example.org/landing",
},
}
async def _fake_resolve_item_payload(**_kwargs):
return payload, False
async def _fail_crawl(_client, *, page_url: str):
raise AssertionError(f"unexpected landing crawl: {page_url}")
monkeypatch.setattr(unpaywall_app, "_resolve_item_payload", _fake_resolve_item_payload)
monkeypatch.setattr(unpaywall_app, "resolve_pdf_from_landing_page", _fail_crawl)
monkeypatch.setattr("httpx.AsyncClient", _DummyAsyncClient)
resolved = await unpaywall_app.resolve_publication_oa_metadata([_item(1)], request_email="user@example.com")
assert resolved == {1: ("10.1016/j.cell.2007.11.019", "https://oa.example.org/article.pdf")}
@pytest.mark.asyncio
async def test_unpaywall_resolve_crawls_landing_page_when_pdf_url_is_not_direct(
monkeypatch: pytest.MonkeyPatch,
) -> None:
payload = {
"doi": "10.1016/j.cell.2007.11.019",
"best_oa_location": {
"url_for_pdf": "https://oa.example.org/view?paper=42",
"url": "https://oa.example.org/landing/42",
},
}
crawled_pages: list[str] = []
async def _fake_resolve_item_payload(**_kwargs):
return payload, False
async def _fake_crawl(_client, *, page_url: str):
crawled_pages.append(page_url)
return "https://oa.example.org/files/paper-42.pdf"
monkeypatch.setattr(unpaywall_app, "_resolve_item_payload", _fake_resolve_item_payload)
monkeypatch.setattr(unpaywall_app, "resolve_pdf_from_landing_page", _fake_crawl)
monkeypatch.setattr("httpx.AsyncClient", _DummyAsyncClient)
resolved = await unpaywall_app.resolve_publication_oa_metadata([_item(2)], request_email="user@example.com")
assert resolved == {2: ("10.1016/j.cell.2007.11.019", "https://oa.example.org/files/paper-42.pdf")}
assert "https://oa.example.org/landing/42" in crawled_pages

View file

@ -0,0 +1,80 @@
// @ts-check
/** @type {import('@docusaurus/types').Config} */
const config = {
title: "scholarr",
tagline: "Self-hosted scholar tracking docs",
favicon: "img/favicon.ico",
url: "https://justinzeus.github.io",
baseUrl: "/scholar-scraper/",
organizationName: "JustinZeus",
projectName: "scholar-scraper",
onBrokenLinks: "throw",
onBrokenMarkdownLinks: "warn",
i18n: {
defaultLocale: "en",
locales: ["en"],
},
presets: [
[
"classic",
{
docs: {
path: "../docs",
routeBasePath: "/",
sidebarPath: require.resolve("./sidebars.js"),
editUrl: "https://github.com/JustinZeus/scholar-scraper/tree/main/",
},
blog: false,
theme: {
customCss: require.resolve("./src/css/custom.css"),
},
},
],
],
themeConfig: {
navbar: {
title: "scholarr",
items: [
{
type: "docSidebar",
sidebarId: "docsSidebar",
position: "left",
label: "Docs",
},
{
href: "https://github.com/JustinZeus/scholar-scraper",
label: "GitHub",
position: "right",
},
],
},
footer: {
style: "dark",
links: [
{
title: "Docs",
items: [{ label: "Index", to: "/" }],
},
{
title: "Community",
items: [
{
label: "GitHub",
href: "https://github.com/JustinZeus/scholar-scraper",
},
],
},
],
copyright: `Copyright © ${new Date().getFullYear()} scholarr.`,
},
},
};
module.exports = config;

18
website/package.json Normal file
View file

@ -0,0 +1,18 @@
{
"name": "scholarr-docs",
"version": "0.0.1",
"private": true,
"scripts": {
"start": "docusaurus start",
"build": "docusaurus build",
"serve": "docusaurus serve"
},
"dependencies": {
"@docusaurus/core": "3.9.2",
"@docusaurus/preset-classic": "3.9.2",
"clsx": "^2.1.1",
"prism-react-renderer": "^2.3.1",
"react": "^18.2.0",
"react-dom": "^18.2.0"
}
}

6
website/sidebars.js Normal file
View file

@ -0,0 +1,6 @@
/** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */
const sidebars = {
docsSidebar: [{ type: "autogenerated", dirName: "." }],
};
module.exports = sidebars;

View file

@ -0,0 +1,9 @@
:root {
--ifm-color-primary: #2a6b4e;
--ifm-color-primary-dark: #255f46;
--ifm-color-primary-darker: #225942;
--ifm-color-primary-darkest: #1c4936;
--ifm-color-primary-light: #2f7756;
--ifm-color-primary-lighter: #327d5a;
--ifm-color-primary-lightest: #3b946b;
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB