Merge pull request #16 from JustinZeus/feat/database-backup-repair
Big changes
61
.env.example
|
|
@ -1,26 +1,52 @@
|
||||||
|
# ------------------------------
|
||||||
|
# Compose + Database
|
||||||
|
# ------------------------------
|
||||||
POSTGRES_DB=scholar
|
POSTGRES_DB=scholar
|
||||||
POSTGRES_USER=scholar
|
POSTGRES_USER=scholar
|
||||||
POSTGRES_PASSWORD=change-me
|
POSTGRES_PASSWORD=change-me
|
||||||
|
|
||||||
DATABASE_URL=postgresql+asyncpg://scholar:scholar@db:5432/scholar
|
DATABASE_URL=postgresql+asyncpg://scholar:scholar@db:5432/scholar
|
||||||
# Optional override. If empty, tests derive "<DATABASE_URL db name>_test".
|
# Optional override. If empty, tests derive "<DATABASE_URL db name>_test".
|
||||||
TEST_DATABASE_URL=
|
TEST_DATABASE_URL=
|
||||||
SCHOLARR_IMAGE=justinzeus/scholarr:latest
|
SCHOLARR_IMAGE=justinzeus/scholarr:latest
|
||||||
|
|
||||||
|
# ------------------------------
|
||||||
|
# App Runtime + Networking
|
||||||
|
# ------------------------------
|
||||||
APP_NAME=scholarr
|
APP_NAME=scholarr
|
||||||
APP_HOST=0.0.0.0
|
APP_HOST=0.0.0.0
|
||||||
APP_PORT=8000
|
APP_PORT=8000
|
||||||
APP_HOST_PORT=8000
|
APP_HOST_PORT=8000
|
||||||
APP_RELOAD=0
|
APP_RELOAD=0
|
||||||
|
MIGRATE_ON_START=1
|
||||||
FRONTEND_ENABLED=1
|
FRONTEND_ENABLED=1
|
||||||
FRONTEND_DIST_DIR=/app/frontend/dist
|
FRONTEND_DIST_DIR=/app/frontend/dist
|
||||||
|
|
||||||
|
# ------------------------------
|
||||||
|
# Database Pool
|
||||||
|
# ------------------------------
|
||||||
|
DATABASE_POOL_MODE=auto
|
||||||
|
DATABASE_POOL_SIZE=5
|
||||||
|
DATABASE_POOL_MAX_OVERFLOW=10
|
||||||
|
DATABASE_POOL_TIMEOUT_SECONDS=30
|
||||||
|
|
||||||
|
# ------------------------------
|
||||||
|
# Frontend Dev Overrides
|
||||||
|
# ------------------------------
|
||||||
FRONTEND_HOST_PORT=5173
|
FRONTEND_HOST_PORT=5173
|
||||||
CHOKIDAR_USEPOLLING=1
|
CHOKIDAR_USEPOLLING=1
|
||||||
VITE_DEV_API_PROXY_TARGET=http://app:8000
|
VITE_DEV_API_PROXY_TARGET=http://app:8000
|
||||||
MIGRATE_ON_START=1
|
|
||||||
|
|
||||||
|
# ------------------------------
|
||||||
|
# Auth + Session
|
||||||
|
# ------------------------------
|
||||||
SESSION_SECRET_KEY=replace-with-a-long-random-secret-at-least-32-characters
|
SESSION_SECRET_KEY=replace-with-a-long-random-secret-at-least-32-characters
|
||||||
SESSION_COOKIE_SECURE=1
|
SESSION_COOKIE_SECURE=1
|
||||||
|
LOGIN_RATE_LIMIT_ATTEMPTS=5
|
||||||
|
LOGIN_RATE_LIMIT_WINDOW_SECONDS=60
|
||||||
|
|
||||||
|
# ------------------------------
|
||||||
|
# HTTP Security Headers + CSP
|
||||||
|
# ------------------------------
|
||||||
SECURITY_HEADERS_ENABLED=1
|
SECURITY_HEADERS_ENABLED=1
|
||||||
SECURITY_X_CONTENT_TYPE_OPTIONS=nosniff
|
SECURITY_X_CONTENT_TYPE_OPTIONS=nosniff
|
||||||
SECURITY_X_FRAME_OPTIONS=DENY
|
SECURITY_X_FRAME_OPTIONS=DENY
|
||||||
|
|
@ -36,16 +62,23 @@ SECURITY_STRICT_TRANSPORT_SECURITY_ENABLED=0
|
||||||
SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE=31536000
|
SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE=31536000
|
||||||
SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS=1
|
SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS=1
|
||||||
SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD=0
|
SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD=0
|
||||||
LOGIN_RATE_LIMIT_ATTEMPTS=5
|
|
||||||
LOGIN_RATE_LIMIT_WINDOW_SECONDS=60
|
# ------------------------------
|
||||||
|
# Logging
|
||||||
|
# ------------------------------
|
||||||
LOG_LEVEL=INFO
|
LOG_LEVEL=INFO
|
||||||
LOG_FORMAT=console
|
LOG_FORMAT=console
|
||||||
LOG_REQUESTS=1
|
LOG_REQUESTS=1
|
||||||
LOG_UVICORN_ACCESS=0
|
LOG_UVICORN_ACCESS=0
|
||||||
LOG_REQUEST_SKIP_PATHS=/healthz
|
LOG_REQUEST_SKIP_PATHS=/healthz
|
||||||
LOG_REDACT_FIELDS=
|
LOG_REDACT_FIELDS=
|
||||||
|
|
||||||
|
# ------------------------------
|
||||||
|
# Scheduler + Ingestion Safety
|
||||||
|
# ------------------------------
|
||||||
SCHEDULER_ENABLED=1
|
SCHEDULER_ENABLED=1
|
||||||
SCHEDULER_TICK_SECONDS=60
|
SCHEDULER_TICK_SECONDS=60
|
||||||
|
SCHEDULER_QUEUE_BATCH_SIZE=10
|
||||||
INGESTION_AUTOMATION_ALLOWED=1
|
INGESTION_AUTOMATION_ALLOWED=1
|
||||||
INGESTION_MANUAL_RUN_ALLOWED=1
|
INGESTION_MANUAL_RUN_ALLOWED=1
|
||||||
INGESTION_MIN_RUN_INTERVAL_MINUTES=15
|
INGESTION_MIN_RUN_INTERVAL_MINUTES=15
|
||||||
|
|
@ -63,7 +96,10 @@ INGESTION_CONTINUATION_QUEUE_ENABLED=1
|
||||||
INGESTION_CONTINUATION_BASE_DELAY_SECONDS=120
|
INGESTION_CONTINUATION_BASE_DELAY_SECONDS=120
|
||||||
INGESTION_CONTINUATION_MAX_DELAY_SECONDS=3600
|
INGESTION_CONTINUATION_MAX_DELAY_SECONDS=3600
|
||||||
INGESTION_CONTINUATION_MAX_ATTEMPTS=6
|
INGESTION_CONTINUATION_MAX_ATTEMPTS=6
|
||||||
SCHEDULER_QUEUE_BATCH_SIZE=10
|
|
||||||
|
# ------------------------------
|
||||||
|
# Scholar Images + Name Search Safety
|
||||||
|
# ------------------------------
|
||||||
SCHOLAR_IMAGE_UPLOAD_DIR=/var/lib/scholarr/uploads
|
SCHOLAR_IMAGE_UPLOAD_DIR=/var/lib/scholarr/uploads
|
||||||
SCHOLAR_IMAGE_UPLOAD_MAX_BYTES=2000000
|
SCHOLAR_IMAGE_UPLOAD_MAX_BYTES=2000000
|
||||||
SCHOLAR_NAME_SEARCH_ENABLED=1
|
SCHOLAR_NAME_SEARCH_ENABLED=1
|
||||||
|
|
@ -76,21 +112,34 @@ SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD=1
|
||||||
SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS=1800
|
SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS=1800
|
||||||
SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD=2
|
SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD=2
|
||||||
SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD=3
|
SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD=3
|
||||||
|
|
||||||
|
# ------------------------------
|
||||||
|
# OA Enrichment + PDF Resolution
|
||||||
|
# ------------------------------
|
||||||
UNPAYWALL_ENABLED=1
|
UNPAYWALL_ENABLED=1
|
||||||
UNPAYWALL_EMAIL=
|
UNPAYWALL_EMAIL=
|
||||||
UNPAYWALL_TIMEOUT_SECONDS=4.0
|
UNPAYWALL_TIMEOUT_SECONDS=4.0
|
||||||
|
UNPAYWALL_MIN_INTERVAL_SECONDS=0.6
|
||||||
UNPAYWALL_MAX_ITEMS_PER_REQUEST=20
|
UNPAYWALL_MAX_ITEMS_PER_REQUEST=20
|
||||||
UNPAYWALL_RETRY_COOLDOWN_SECONDS=1800
|
UNPAYWALL_RETRY_COOLDOWN_SECONDS=1800
|
||||||
|
UNPAYWALL_PDF_DISCOVERY_ENABLED=1
|
||||||
|
UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES=5
|
||||||
|
UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES=500000
|
||||||
|
PDF_AUTO_RETRY_INTERVAL_SECONDS=86400
|
||||||
|
PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS=3600
|
||||||
|
PDF_AUTO_RETRY_MAX_ATTEMPTS=3
|
||||||
CROSSREF_ENABLED=1
|
CROSSREF_ENABLED=1
|
||||||
CROSSREF_MAX_ROWS=10
|
CROSSREF_MAX_ROWS=10
|
||||||
CROSSREF_TIMEOUT_SECONDS=8.0
|
CROSSREF_TIMEOUT_SECONDS=8.0
|
||||||
CROSSREF_MIN_INTERVAL_SECONDS=0.6
|
CROSSREF_MIN_INTERVAL_SECONDS=0.6
|
||||||
CROSSREF_MAX_LOOKUPS_PER_REQUEST=8
|
CROSSREF_MAX_LOOKUPS_PER_REQUEST=8
|
||||||
|
|
||||||
|
# ------------------------------
|
||||||
|
# Startup Bootstrap + DB Wait
|
||||||
|
# ------------------------------
|
||||||
BOOTSTRAP_ADMIN_ON_START=0
|
BOOTSTRAP_ADMIN_ON_START=0
|
||||||
BOOTSTRAP_ADMIN_EMAIL=
|
BOOTSTRAP_ADMIN_EMAIL=
|
||||||
BOOTSTRAP_ADMIN_PASSWORD=
|
BOOTSTRAP_ADMIN_PASSWORD=
|
||||||
BOOTSTRAP_ADMIN_FORCE_PASSWORD=0
|
BOOTSTRAP_ADMIN_FORCE_PASSWORD=0
|
||||||
|
|
||||||
DB_WAIT_TIMEOUT_SECONDS=60
|
DB_WAIT_TIMEOUT_SECONDS=60
|
||||||
DB_WAIT_INTERVAL_SECONDS=2
|
DB_WAIT_INTERVAL_SECONDS=2
|
||||||
|
|
|
||||||
24
.github/workflows/ci.yml
vendored
|
|
@ -17,6 +17,9 @@ jobs:
|
||||||
- name: Enforce no generated artifacts
|
- name: Enforce no generated artifacts
|
||||||
run: ./scripts/check_no_generated_artifacts.sh
|
run: ./scripts/check_no_generated_artifacts.sh
|
||||||
|
|
||||||
|
- name: Enforce env contract parity
|
||||||
|
run: python3 scripts/check_env_contract.py
|
||||||
|
|
||||||
test:
|
test:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs:
|
needs:
|
||||||
|
|
@ -106,12 +109,33 @@ jobs:
|
||||||
working-directory: frontend
|
working-directory: frontend
|
||||||
run: npm run build
|
run: npm run build
|
||||||
|
|
||||||
|
docs-quality:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs:
|
||||||
|
- repo-hygiene
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Setup Node
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "20"
|
||||||
|
|
||||||
|
- name: Install docs dependencies
|
||||||
|
run: npm install --prefix website
|
||||||
|
|
||||||
|
- name: Docs build
|
||||||
|
run: npm --prefix website run build
|
||||||
|
|
||||||
docker-publish:
|
docker-publish:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs:
|
needs:
|
||||||
- repo-hygiene
|
- repo-hygiene
|
||||||
- test
|
- test
|
||||||
- frontend-quality
|
- frontend-quality
|
||||||
|
- docs-quality
|
||||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
|
|
|
||||||
58
.github/workflows/docs-pages.yml
vendored
Normal file
|
|
@ -0,0 +1,58 @@
|
||||||
|
name: Docs Pages
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths:
|
||||||
|
- docs/**
|
||||||
|
- website/**
|
||||||
|
- .github/workflows/docs-pages.yml
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: docs-pages
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Setup Pages
|
||||||
|
uses: actions/configure-pages@v5
|
||||||
|
|
||||||
|
- name: Setup Node
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "20"
|
||||||
|
|
||||||
|
- name: Install docs dependencies
|
||||||
|
run: npm install --prefix website
|
||||||
|
|
||||||
|
- name: Build docs site
|
||||||
|
run: npm --prefix website run build
|
||||||
|
|
||||||
|
- name: Upload pages artifact
|
||||||
|
uses: actions/upload-pages-artifact@v3
|
||||||
|
with:
|
||||||
|
path: website/build
|
||||||
|
|
||||||
|
deploy:
|
||||||
|
needs: build
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
pages: write
|
||||||
|
id-token: write
|
||||||
|
environment:
|
||||||
|
name: github-pages
|
||||||
|
url: ${{ steps.deployment.outputs.page_url }}
|
||||||
|
steps:
|
||||||
|
- name: Deploy to GitHub Pages
|
||||||
|
id: deployment
|
||||||
|
uses: actions/deploy-pages@v4
|
||||||
2
.gitignore
vendored
|
|
@ -15,4 +15,6 @@ planning/
|
||||||
frontend/node_modules/
|
frontend/node_modules/
|
||||||
frontend/dist/
|
frontend/dist/
|
||||||
frontend/.vite/
|
frontend/.vite/
|
||||||
|
website/node_modules/
|
||||||
|
website/build/
|
||||||
AGENTS.MD
|
AGENTS.MD
|
||||||
|
|
|
||||||
291
README.md
|
|
@ -22,11 +22,7 @@ cp .env.example .env
|
||||||
- `POSTGRES_PASSWORD`
|
- `POSTGRES_PASSWORD`
|
||||||
- `SESSION_SECRET_KEY`
|
- `SESSION_SECRET_KEY`
|
||||||
|
|
||||||
3. Choose a deploy method below.
|
3. Start stack:
|
||||||
|
|
||||||
## Deploy Method A: Prebuilt Image (Recommended)
|
|
||||||
|
|
||||||
Use this for normal self-hosted deployment.
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker compose pull
|
docker compose pull
|
||||||
|
|
@ -37,234 +33,36 @@ Open:
|
||||||
- App/API: `http://localhost:8000`
|
- App/API: `http://localhost:8000`
|
||||||
- Health: `http://localhost:8000/healthz`
|
- Health: `http://localhost:8000/healthz`
|
||||||
|
|
||||||
Upgrade:
|
## Documentation
|
||||||
|
|
||||||
|
Primary docs live under `docs/`.
|
||||||
|
|
||||||
|
- Index: `docs/README.md`
|
||||||
|
- Deploy and dev workflow: `docs/deploy/quickstart.md`
|
||||||
|
- Environment reference: `docs/reference/environment.md`
|
||||||
|
- API contract and payload conventions: `docs/reference/api_contract.md`
|
||||||
|
- Architecture and domain boundaries: `docs/architecture/domain_boundaries.md`
|
||||||
|
- Scrape safety runbook: `docs/ops/scrape_safety_runbook.md`
|
||||||
|
- DB backup/restore/integrity runbook: `docs/ops/db_runbook.md`
|
||||||
|
- Migration rollout checklist: `docs/ops/migration_checklist.md`
|
||||||
|
- Frontend theme inventory: `docs/frontend/theme_phase0_inventory.md`
|
||||||
|
- Contributing policy: `docs/contributing.md`
|
||||||
|
|
||||||
|
## Docs Site (Docusaurus)
|
||||||
|
|
||||||
|
Docusaurus source lives in `website/` and consumes markdown from `docs/`.
|
||||||
|
|
||||||
|
Local build:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker compose pull
|
cd website
|
||||||
docker compose up -d
|
npm install
|
||||||
|
npm run build
|
||||||
```
|
```
|
||||||
|
|
||||||
## Deploy Method B: Local Source Build + Dev Frontend
|
GitHub Pages deploy is automated via `.github/workflows/docs-pages.yml`.
|
||||||
|
|
||||||
Use this for development on this repository.
|
## Quality Gates
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
|
|
||||||
```
|
|
||||||
|
|
||||||
Open:
|
|
||||||
- API (FastAPI): `http://localhost:8000`
|
|
||||||
- Frontend dev server (Vite): `http://localhost:5173`
|
|
||||||
|
|
||||||
Stop:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose -f docker-compose.yml -f docker-compose.dev.yml down
|
|
||||||
```
|
|
||||||
|
|
||||||
## Essential Files
|
|
||||||
|
|
||||||
- `docker-compose.yml`: deployment compose (prebuilt image).
|
|
||||||
- `docker-compose.dev.yml`: dev override (source mounts + Vite dev server).
|
|
||||||
- `.env.example`: full env variable template.
|
|
||||||
- `Dockerfile`: multi-stage build (frontend + backend runtime).
|
|
||||||
- `README.md`: deployment and operations reference.
|
|
||||||
- `CONTRIBUTING.md`: contribution policy and merge checklist.
|
|
||||||
- `docs/ops/scrape_safety_runbook.md`: scrape cooldown and blocked-IP operations guide.
|
|
||||||
- `scripts/check_no_generated_artifacts.sh`: tracked-artifact guard used by CI.
|
|
||||||
|
|
||||||
## Data Model Notes
|
|
||||||
|
|
||||||
- Scholar tracking is user-scoped: each account can track the same Scholar ID independently.
|
|
||||||
- Publications are shared/global records deduplicated by Scholar cluster ID and normalized fingerprint.
|
|
||||||
- Per-account visibility and read state is stored on scholar-publication links, not on the global publication row.
|
|
||||||
- Publication records include canonical Scholar detail URLs (`pub_url`), normalized DOI (`doi`), and resolved OA PDF links (`pdf_url`) when available.
|
|
||||||
|
|
||||||
## Backend Architecture (Refactored)
|
|
||||||
|
|
||||||
- Canonical service logic lives in `app/services/domains/*`; root-level `app/services/*.py` business-logic modules are removed.
|
|
||||||
- `app/services/domains/ingestion/*`: run orchestration (`application.py`), continuation queue (`queue.py`), scrape safety cooldown policy (`safety.py`), scheduler/queue draining (`scheduler.py`), plus shared constants/types/fingerprint helpers.
|
|
||||||
- `app/services/domains/scholar/*`: fail-fast Google Scholar parsing and source access. Parser flow is modularized across row extractors, state detection, constants, and parser types.
|
|
||||||
- `app/services/domains/scholars/*`: scholar CRUD, name-search safety/caching, profile-image validation/upload handling, and shared scholar validation helpers.
|
|
||||||
- `app/services/domains/publications/*`: publication listing/read-state query modules, lazy/background OA enrichment scheduling (non-blocking for list responses), and per-publication PDF retry support.
|
|
||||||
- `app/services/domains/doi/*`: DOI normalization helpers used by ingestion and enrichment.
|
|
||||||
- `app/services/domains/crossref/*`: DOI discovery fallback using Crossref metadata queries with bounded/paced request behavior.
|
|
||||||
- `app/services/domains/unpaywall/*`: OA resolution by DOI (best OA location + PDF URL extraction), using per-user email for API calls.
|
|
||||||
- `app/services/domains/runs/*`: run history summaries plus continuation queue status/retry/drop/clear operations.
|
|
||||||
- `app/services/domains/portability/*`: import/export of tracked scholars and publication-link state while preserving global publication deduplication.
|
|
||||||
- `app/services/domains/settings/*` and `app/services/domains/users/*`: user settings and user management services.
|
|
||||||
|
|
||||||
## Frontend Navigation and Layout Notes
|
|
||||||
|
|
||||||
- On mobile (`< lg`), primary navigation is behind a header hamburger and opens as a left-side drawer with backdrop.
|
|
||||||
- Mobile nav closes on route change, nav link click, and logout.
|
|
||||||
- Long lists are constrained to internal scroll containers in fill-layout pages (including dashboard recent publications and tracked scholars table view).
|
|
||||||
|
|
||||||
## Name Search Status
|
|
||||||
|
|
||||||
- Scholar name search is intentionally WIP in the UI.
|
|
||||||
- Current Google Scholar behavior often redirects name-search traffic to login/challenge flows, so production onboarding should use direct Scholar ID/profile URL adds.
|
|
||||||
|
|
||||||
## Environment Variables (Complete Reference)
|
|
||||||
|
|
||||||
Notes:
|
|
||||||
- Boolean envs accept: `1/0`, `true/false`, `yes/no`, `on/off`.
|
|
||||||
- Values shown are deployment defaults from `.env.example` and `docker-compose.yml`.
|
|
||||||
- Some internal app fallbacks may differ for local test/dev safety when env vars are omitted.
|
|
||||||
- `deploy` means used in regular deployment.
|
|
||||||
- `dev` means used in local dev workflow.
|
|
||||||
|
|
||||||
### Core Compose and Database
|
|
||||||
|
|
||||||
| Variable | Default | Options | Scope | Description |
|
|
||||||
| --- | --- | --- | --- | --- |
|
|
||||||
| `POSTGRES_DB` | `scholar` | any valid DB name | deploy, dev | PostgreSQL database name. |
|
|
||||||
| `POSTGRES_USER` | `scholar` | any valid DB user | deploy, dev | PostgreSQL user. |
|
|
||||||
| `POSTGRES_PASSWORD` | `change-me` | strong password | deploy, dev | PostgreSQL password. Required. |
|
|
||||||
| `DATABASE_URL` | `postgresql+asyncpg://scholar:scholar@db:5432/scholar` | valid SQLAlchemy asyncpg URL | deploy, dev, test | App database connection URL. |
|
|
||||||
| `TEST_DATABASE_URL` | empty | valid SQLAlchemy asyncpg URL | test | Optional explicit integration test DB URL. |
|
|
||||||
| `SCHOLARR_IMAGE` | `justinzeus/scholarr:latest` | any image ref | deploy | App image tag used by deployment compose. |
|
|
||||||
|
|
||||||
### App Runtime and Networking
|
|
||||||
|
|
||||||
| Variable | Default | Options | Scope | Description |
|
|
||||||
| --- | --- | --- | --- | --- |
|
|
||||||
| `APP_NAME` | `scholarr` | any string | deploy, dev | FastAPI app name/title. |
|
|
||||||
| `APP_HOST` | `0.0.0.0` | bind host | deploy, dev | Uvicorn bind host inside container. |
|
|
||||||
| `APP_PORT` | `8000` | integer | deploy, dev | Uvicorn bind port inside container. |
|
|
||||||
| `APP_HOST_PORT` | `8000` | integer | deploy, dev | Host port mapped to app container port 8000. |
|
|
||||||
| `APP_RELOAD` | `0` | boolean | deploy, dev | Enable uvicorn reload mode. Recommended `0` in deploy, `1` in dev. |
|
|
||||||
| `MIGRATE_ON_START` | `1` | boolean | deploy, dev | Run Alembic migrations during app startup. |
|
|
||||||
| `FRONTEND_ENABLED` | `1` | boolean | deploy, dev | Serve bundled frontend assets from FastAPI. |
|
|
||||||
| `FRONTEND_DIST_DIR` | `/app/frontend/dist` | absolute path | deploy, dev | Path to built frontend assets inside app container. |
|
|
||||||
|
|
||||||
### Dev Frontend Overrides
|
|
||||||
|
|
||||||
| Variable | Default | Options | Scope | Description |
|
|
||||||
| --- | --- | --- | --- | --- |
|
|
||||||
| `FRONTEND_HOST_PORT` | `5173` | integer | dev | Host port mapped to Vite dev server. |
|
|
||||||
| `CHOKIDAR_USEPOLLING` | `1` | boolean | dev | File watching mode for containerized Vite. |
|
|
||||||
| `VITE_DEV_API_PROXY_TARGET` | `http://app:8000` | URL | dev | Vite proxy target for `/api` and `/healthz`. |
|
|
||||||
|
|
||||||
### Auth and Session Security
|
|
||||||
|
|
||||||
| Variable | Default | Options | Scope | Description |
|
|
||||||
| --- | --- | --- | --- | --- |
|
|
||||||
| `SESSION_SECRET_KEY` | `replace-with-a-long-random-secret-at-least-32-characters` | long random string | deploy, dev | Session signing secret. Required in deploy. |
|
|
||||||
| `SESSION_COOKIE_SECURE` | `1` | boolean | deploy, dev | Mark session cookie as HTTPS-only. Set `1` behind HTTPS. |
|
|
||||||
| `LOGIN_RATE_LIMIT_ATTEMPTS` | `5` | integer >= 1 | deploy, dev | Login attempts allowed per window. |
|
|
||||||
| `LOGIN_RATE_LIMIT_WINDOW_SECONDS` | `60` | integer >= 1 | deploy, dev | Login rate limit window in seconds. |
|
|
||||||
|
|
||||||
### HTTP Security Headers and CSP
|
|
||||||
|
|
||||||
| Variable | Default | Options | Scope | Description |
|
|
||||||
| --- | --- | --- | --- | --- |
|
|
||||||
| `SECURITY_HEADERS_ENABLED` | `1` | boolean | deploy, dev | Global switch for response security headers middleware. |
|
|
||||||
| `SECURITY_X_CONTENT_TYPE_OPTIONS` | `nosniff` | header value | deploy, dev | `X-Content-Type-Options` response header value. |
|
|
||||||
| `SECURITY_X_FRAME_OPTIONS` | `DENY` | header value | deploy, dev | `X-Frame-Options` response header value. |
|
|
||||||
| `SECURITY_REFERRER_POLICY` | `strict-origin-when-cross-origin` | header value | deploy, dev | `Referrer-Policy` response header value. |
|
|
||||||
| `SECURITY_PERMISSIONS_POLICY` | `accelerometer=(), autoplay=(), camera=(), display-capture=(), geolocation=(), gyroscope=(), microphone=(), payment=(), usb=()` | permissions policy string | deploy, dev | `Permissions-Policy` response header value. |
|
|
||||||
| `SECURITY_CROSS_ORIGIN_OPENER_POLICY` | `same-origin` | header value | deploy, dev | `Cross-Origin-Opener-Policy` response header value. |
|
|
||||||
| `SECURITY_CROSS_ORIGIN_RESOURCE_POLICY` | `same-origin` | header value | deploy, dev | `Cross-Origin-Resource-Policy` response header value. |
|
|
||||||
| `SECURITY_CSP_ENABLED` | `1` | boolean | deploy, dev | Enable Content Security Policy headers. |
|
|
||||||
| `SECURITY_CSP_POLICY` | strict SPA/API default | CSP policy string | deploy, dev | CSP applied to app/API routes (excluding docs paths). |
|
|
||||||
| `SECURITY_CSP_DOCS_POLICY` | relaxed docs default | CSP policy string | deploy, dev | CSP override for `/docs` and `/redoc` to keep Swagger/ReDoc usable. |
|
|
||||||
| `SECURITY_CSP_REPORT_ONLY` | `0` | boolean | deploy, dev | Emit CSP in report-only mode instead of enforcement mode. |
|
|
||||||
| `SECURITY_STRICT_TRANSPORT_SECURITY_ENABLED` | `0` | boolean | deploy, dev | Enable `Strict-Transport-Security` header. |
|
|
||||||
| `SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE` | `31536000` | integer >= 0 | deploy, dev | `max-age` for HSTS header. |
|
|
||||||
| `SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS` | `1` | boolean | deploy, dev | Add `includeSubDomains` directive to HSTS header. |
|
|
||||||
| `SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD` | `0` | boolean | deploy, dev | Add `preload` directive to HSTS header. |
|
|
||||||
|
|
||||||
### Logging
|
|
||||||
|
|
||||||
| Variable | Default | Options | Scope | Description |
|
|
||||||
| --- | --- | --- | --- | --- |
|
|
||||||
| `LOG_LEVEL` | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` | deploy, dev | Application log level. |
|
|
||||||
| `LOG_FORMAT` | `console` | `console`, `json` | deploy, dev | Log output format. |
|
|
||||||
| `LOG_REQUESTS` | `1` | boolean | deploy, dev | Enable request start/completion logs. |
|
|
||||||
| `LOG_UVICORN_ACCESS` | `0` | boolean | deploy, dev | Enable uvicorn access logs. |
|
|
||||||
| `LOG_REQUEST_SKIP_PATHS` | `/healthz` | comma-separated path prefixes | deploy, dev | Request log skip list. |
|
|
||||||
| `LOG_REDACT_FIELDS` | empty | comma-separated keys | deploy, dev | Extra fields to redact in structured logs. |
|
|
||||||
|
|
||||||
### Scheduler and Ingestion
|
|
||||||
|
|
||||||
| Variable | Default | Options | Scope | Description |
|
|
||||||
| --- | --- | --- | --- | --- |
|
|
||||||
| `SCHEDULER_ENABLED` | `1` | boolean | deploy, dev | Enable background scheduler loop. |
|
|
||||||
| `SCHEDULER_TICK_SECONDS` | `60` | integer >= 1 | deploy, dev | Scheduler interval in seconds. |
|
|
||||||
| `INGESTION_AUTOMATION_ALLOWED` | `1` | boolean | deploy, dev | Global safety switch for scheduled/automatic checks. When disabled, auto-run settings are forced off. |
|
|
||||||
| `INGESTION_MANUAL_RUN_ALLOWED` | `1` | boolean | deploy, dev | Global safety switch for manual checks from API/UI. |
|
|
||||||
| `INGESTION_MIN_RUN_INTERVAL_MINUTES` | `15` | integer >= 15 | deploy, dev | Server-enforced minimum for user-configured automatic check interval. |
|
|
||||||
| `INGESTION_MIN_REQUEST_DELAY_SECONDS` | `2` | integer >= 2 | deploy, dev | Server-enforced minimum delay between scholar requests. |
|
|
||||||
| `SCHEDULER_QUEUE_BATCH_SIZE` | `10` | integer >= 1 | deploy, dev | Queue items processed per scheduler tick. |
|
|
||||||
| `INGESTION_NETWORK_ERROR_RETRIES` | `1` | integer >= 0 | deploy, dev | Retries for transient ingestion network errors. |
|
|
||||||
| `INGESTION_RETRY_BACKOFF_SECONDS` | `1.0` | float >= 0 | deploy, dev | Backoff delay for retry attempts. |
|
|
||||||
| `INGESTION_MAX_PAGES_PER_SCHOLAR` | `30` | integer >= 1 | deploy, dev | Upper bound of pages fetched per scholar run. |
|
|
||||||
| `INGESTION_PAGE_SIZE` | `100` | integer >= 1 | deploy, dev | Requested Scholar page size. |
|
|
||||||
| `INGESTION_ALERT_BLOCKED_FAILURE_THRESHOLD` | `1` | integer >= 1 | deploy, dev | Trigger blocked/captcha scrape alert flag when this many blocked failures occur in a run. |
|
|
||||||
| `INGESTION_ALERT_NETWORK_FAILURE_THRESHOLD` | `2` | integer >= 1 | deploy, dev | Trigger network scrape alert flag when this many network failures occur in a run. |
|
|
||||||
| `INGESTION_ALERT_RETRY_SCHEDULED_THRESHOLD` | `3` | integer >= 1 | deploy, dev | Trigger retry alert flag when scheduled retry count reaches this threshold in a run. |
|
|
||||||
| `INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS` | `1800` | integer >= 60 | deploy, dev | Cooldown duration applied when blocked-failure threshold is exceeded. |
|
|
||||||
| `INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS` | `900` | integer >= 60 | deploy, dev | Cooldown duration applied when network-failure threshold is exceeded. |
|
|
||||||
| `INGESTION_CONTINUATION_QUEUE_ENABLED` | `1` | boolean | deploy, dev | Enable continuation queue for long runs. |
|
|
||||||
| `INGESTION_CONTINUATION_BASE_DELAY_SECONDS` | `120` | integer >= 0 | deploy, dev | Initial delay before retrying continuation queue items. |
|
|
||||||
| `INGESTION_CONTINUATION_MAX_DELAY_SECONDS` | `3600` | integer >= 0 | deploy, dev | Maximum continuation retry delay. |
|
|
||||||
| `INGESTION_CONTINUATION_MAX_ATTEMPTS` | `6` | integer >= 1 | deploy, dev | Max failed continuation attempts before dropping item. |
|
|
||||||
|
|
||||||
### Scholar Images and Name Search Safety
|
|
||||||
|
|
||||||
| Variable | Default | Options | Scope | Description |
|
|
||||||
| --- | --- | --- | --- | --- |
|
|
||||||
| `SCHOLAR_IMAGE_UPLOAD_DIR` | `/var/lib/scholarr/uploads` | writable absolute path | deploy, dev | Storage path for uploaded scholar images in compose deployments. App fallback without env is `/tmp/scholarr_uploads/scholar_images`. |
|
|
||||||
| `SCHOLAR_IMAGE_UPLOAD_MAX_BYTES` | `2000000` | integer >= 1 | deploy, dev | Max uploaded image size in bytes. |
|
|
||||||
| `SCHOLAR_NAME_SEARCH_ENABLED` | `1` | boolean | deploy, dev | Enable name-search helper endpoint. |
|
|
||||||
| `SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS` | `21600` | integer >= 1 | deploy, dev | Cache TTL for successful name-search responses. |
|
|
||||||
| `SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS` | `300` | integer >= 1 | deploy, dev | Cache TTL for blocked/captcha responses. |
|
|
||||||
| `SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES` | `512` | integer >= 1 | deploy, dev | Max cached search entries. |
|
|
||||||
| `SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS` | `8.0` | float >= 0 | deploy, dev | Minimum interval between live name-search requests. |
|
|
||||||
| `SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS` | `2.0` | float >= 0 | deploy, dev | Added jitter to reduce request burst patterns. |
|
|
||||||
| `SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD` | `1` | integer >= 1 | deploy, dev | Consecutive blocked responses before cooldown starts. |
|
|
||||||
| `SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS` | `1800` | integer >= 1 | deploy, dev | Cooldown duration after repeated blocked responses. |
|
|
||||||
| `SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD` | `2` | integer >= 1 | deploy, dev | Emit retry-threshold observability warning when name-search retry count reaches this value. |
|
|
||||||
| `SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD` | `3` | integer >= 1 | deploy, dev | Emit cooldown-threshold observability alert after this many requests are rejected during active cooldown. |
|
|
||||||
|
|
||||||
### Open-Access Enrichment (Crossref + Unpaywall)
|
|
||||||
|
|
||||||
| Variable | Default | Options | Scope | Description |
|
|
||||||
| --- | --- | --- | --- | --- |
|
|
||||||
| `UNPAYWALL_ENABLED` | `1` | boolean | deploy, dev | Enable Unpaywall OA resolution. |
|
|
||||||
| `UNPAYWALL_EMAIL` | empty | valid email | deploy, dev | Fallback email if per-user email is unavailable. |
|
|
||||||
| `UNPAYWALL_TIMEOUT_SECONDS` | `4.0` | float >= 0.5 | deploy, dev | Timeout for Unpaywall requests. |
|
|
||||||
| `UNPAYWALL_MAX_ITEMS_PER_REQUEST` | `20` | integer >= 0 | deploy, dev | Max publications queued per lazy enrichment pass. |
|
|
||||||
| `UNPAYWALL_RETRY_COOLDOWN_SECONDS` | `1800` | integer >= 1 | deploy, dev | Per-user/per-publication cooldown before re-scheduling unresolved OA lookups. |
|
|
||||||
| `CROSSREF_ENABLED` | `1` | boolean | deploy, dev | Enable Crossref DOI discovery fallback when DOI is missing/insufficient. |
|
|
||||||
| `CROSSREF_MAX_ROWS` | `10` | integer >= 1 | deploy, dev | Max Crossref rows evaluated per lookup. |
|
|
||||||
| `CROSSREF_TIMEOUT_SECONDS` | `8.0` | float >= 0.5 | deploy, dev | Timeout budget per Crossref lookup task. |
|
|
||||||
| `CROSSREF_MIN_INTERVAL_SECONDS` | `0.6` | float >= 0 | deploy, dev | Minimum pacing interval between Crossref requests. |
|
|
||||||
| `CROSSREF_MAX_LOOKUPS_PER_REQUEST` | `8` | integer >= 0 | deploy, dev | Max Crossref DOI lookups performed during one lazy pass or manual retry. |
|
|
||||||
|
|
||||||
### Scrape Safety Operations
|
|
||||||
|
|
||||||
- Structured safety events are emitted for:
|
|
||||||
- policy/manual run blocks,
|
|
||||||
- cooldown entered/cleared transitions,
|
|
||||||
- blocked/network/retry threshold trips,
|
|
||||||
- scheduler cooldown skips/deferments.
|
|
||||||
- Use `LOG_FORMAT=json` and ship logs to your preferred collector for alerting.
|
|
||||||
- Runbook: `docs/ops/scrape_safety_runbook.md`.
|
|
||||||
|
|
||||||
### Startup Bootstrap and DB Wait
|
|
||||||
|
|
||||||
| Variable | Default | Options | Scope | Description |
|
|
||||||
| --- | --- | --- | --- | --- |
|
|
||||||
| `BOOTSTRAP_ADMIN_ON_START` | `0` | boolean | deploy, dev | Auto-create admin at startup. |
|
|
||||||
| `BOOTSTRAP_ADMIN_EMAIL` | empty | valid email | deploy, dev | Bootstrap admin email. |
|
|
||||||
| `BOOTSTRAP_ADMIN_PASSWORD` | empty | strong password | deploy, dev | Bootstrap admin password. |
|
|
||||||
| `BOOTSTRAP_ADMIN_FORCE_PASSWORD` | `0` | boolean | deploy, dev | Force-reset bootstrap admin password if user already exists. |
|
|
||||||
| `DB_WAIT_TIMEOUT_SECONDS` | `60` | integer >= 1 | deploy, dev | Max seconds to wait for DB readiness at startup. |
|
|
||||||
| `DB_WAIT_INTERVAL_SECONDS` | `2` | integer >= 1 | deploy, dev | Interval between DB readiness checks. |
|
|
||||||
|
|
||||||
## Quality and Test Commands
|
|
||||||
|
|
||||||
Backend:
|
Backend:
|
||||||
|
|
||||||
|
|
@ -283,43 +81,10 @@ npm run test:run
|
||||||
npm run build
|
npm run build
|
||||||
```
|
```
|
||||||
|
|
||||||
Contract drift:
|
Contract and env checks:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python3 scripts/check_frontend_api_contract.py
|
python3 scripts/check_frontend_api_contract.py
|
||||||
```
|
python3 scripts/check_env_contract.py
|
||||||
|
|
||||||
Repository hygiene:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/check_no_generated_artifacts.sh
|
./scripts/check_no_generated_artifacts.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
Scheduled fixture probes run in GitHub Actions via `.github/workflows/scheduled-probes.yml`.
|
|
||||||
|
|
||||||
## API Contract
|
|
||||||
|
|
||||||
- Base path: `/api/v1`
|
|
||||||
- Success envelope: `{"data": ..., "meta": {"request_id": "..."}}`
|
|
||||||
- Error envelope: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}`
|
|
||||||
|
|
||||||
### Publications semantics
|
|
||||||
|
|
||||||
- `GET /api/v1/publications` supports `mode=all|unread|latest` (plus temporary alias `mode=new`).
|
|
||||||
- `unread` = actionable read-state (`is_read=false`).
|
|
||||||
- `latest` = discovery-state (`first seen in the latest completed check`).
|
|
||||||
- Publications include `pub_url`, `doi`, and `pdf_url` in API responses.
|
|
||||||
- OA enrichment flow is DOI-first:
|
|
||||||
- existing DOI (or explicit DOI in source metadata) -> Unpaywall lookup,
|
|
||||||
- Crossref DOI discovery fallback (paced + bounded),
|
|
||||||
- Unpaywall DOI lookup for OA status and `pdf_url`.
|
|
||||||
- `POST /api/v1/publications/{publication_id}/retry-pdf` retries enrichment for a single publication in current user scope.
|
|
||||||
- Response counters:
|
|
||||||
- `unread_count`: unread publications in current scope.
|
|
||||||
- `latest_count`: publications discovered in latest completed check.
|
|
||||||
- `new_count`: compatibility alias for `latest_count` (temporary).
|
|
||||||
|
|
||||||
### Scholar Data Portability
|
|
||||||
|
|
||||||
- `GET /api/v1/scholars/export`: exports tracked scholars plus scholar-publication link data as JSON.
|
|
||||||
- `POST /api/v1/scholars/import`: imports that JSON payload, upserting scholars, global publication records, and per-scholar read state.
|
|
||||||
|
|
|
||||||
|
|
@ -36,3 +36,8 @@ These limits prevent IP bans and are not to be optimized away.
|
||||||
* **`app/services/domains/unpaywall/*`:** OA resolver by DOI only (best OA location + PDF URL extraction). Do not use Google Scholar for PDF resolution to avoid N+1 scrape amplification.
|
* **`app/services/domains/unpaywall/*`:** OA resolver by DOI only (best OA location + PDF URL extraction). Do not use Google Scholar for PDF resolution to avoid N+1 scrape amplification.
|
||||||
* **`app/services/domains/portability/*`:** Handles JSON import/export for user-scoped scholars and scholar-publication link state while preserving global publication dedup rules.
|
* **`app/services/domains/portability/*`:** Handles JSON import/export for user-scoped scholars and scholar-publication link state while preserving global publication dedup rules.
|
||||||
* **`app/services/domains/ingestion/scheduler.py`:** Owns automatic runs and continuation queue retries/drops; do not bypass safety gate or cooldown logic.
|
* **`app/services/domains/ingestion/scheduler.py`:** Owns automatic runs and continuation queue retries/drops; do not bypass safety gate or cooldown logic.
|
||||||
|
|
||||||
|
|
||||||
|
## 6. UI rules
|
||||||
|
Make sure to properly integrate tailwind in combination with the preset theming
|
||||||
|
Clarity through both styling and language are a priority. all UI elements need to have a proper reason for existing.
|
||||||
106
alembic/versions/20260220_0012_add_data_repair_jobs.py
Normal file
|
|
@ -0,0 +1,106 @@
|
||||||
|
"""Add data repair job audit table.
|
||||||
|
|
||||||
|
Revision ID: 20260220_0012
|
||||||
|
Revises: 20260220_0011
|
||||||
|
Create Date: 2026-02-20 23:35:00.000000
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
|
||||||
|
revision: str = "20260220_0012"
|
||||||
|
down_revision: str | Sequence[str] | None = "20260220_0011"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _table_names() -> set[str]:
|
||||||
|
bind = op.get_bind()
|
||||||
|
inspector = sa.inspect(bind)
|
||||||
|
return set(inspector.get_table_names())
|
||||||
|
|
||||||
|
|
||||||
|
def _create_data_repair_jobs_table() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"data_repair_jobs",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("job_name", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("requested_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"scope",
|
||||||
|
postgresql.JSONB(astext_type=sa.Text()),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("'{}'::jsonb"),
|
||||||
|
),
|
||||||
|
sa.Column("dry_run", sa.Boolean(), nullable=False, server_default=sa.text("true")),
|
||||||
|
sa.Column(
|
||||||
|
"status",
|
||||||
|
sa.String(length=16),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("'planned'"),
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"summary",
|
||||||
|
postgresql.JSONB(astext_type=sa.Text()),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("'{}'::jsonb"),
|
||||||
|
),
|
||||||
|
sa.Column("error_text", sa.Text(), nullable=True),
|
||||||
|
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"status IN ('planned', 'running', 'completed', 'failed')",
|
||||||
|
name="data_repair_jobs_status_valid",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_data_repair_jobs")),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _create_data_repair_jobs_indexes() -> None:
|
||||||
|
op.create_index(
|
||||||
|
"ix_data_repair_jobs_created_at",
|
||||||
|
"data_repair_jobs",
|
||||||
|
["created_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_data_repair_jobs_job_name_created_at",
|
||||||
|
"data_repair_jobs",
|
||||||
|
["job_name", "created_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_data_repair_jobs_indexes() -> None:
|
||||||
|
op.drop_index("ix_data_repair_jobs_job_name_created_at", table_name="data_repair_jobs")
|
||||||
|
op.drop_index("ix_data_repair_jobs_created_at", table_name="data_repair_jobs")
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
if "data_repair_jobs" in _table_names():
|
||||||
|
return
|
||||||
|
_create_data_repair_jobs_table()
|
||||||
|
_create_data_repair_jobs_indexes()
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
if "data_repair_jobs" not in _table_names():
|
||||||
|
return
|
||||||
|
_drop_data_repair_jobs_indexes()
|
||||||
|
op.drop_table("data_repair_jobs")
|
||||||
179
alembic/versions/20260221_0013_add_publication_pdf_jobs.py
Normal file
|
|
@ -0,0 +1,179 @@
|
||||||
|
"""Add persistent publication PDF job tracking tables.
|
||||||
|
|
||||||
|
Revision ID: 20260221_0013
|
||||||
|
Revises: 20260220_0012
|
||||||
|
Create Date: 2026-02-21 12:25:00.000000
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision: str = "20260221_0013"
|
||||||
|
down_revision: str | Sequence[str] | None = "20260220_0012"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _table_names() -> set[str]:
|
||||||
|
bind = op.get_bind()
|
||||||
|
inspector = sa.inspect(bind)
|
||||||
|
return set(inspector.get_table_names())
|
||||||
|
|
||||||
|
|
||||||
|
def _create_publication_pdf_jobs_table() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"publication_pdf_jobs",
|
||||||
|
sa.Column("publication_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"status",
|
||||||
|
sa.String(length=16),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("'queued'"),
|
||||||
|
),
|
||||||
|
sa.Column("attempt_count", sa.Integer(), nullable=False, server_default=sa.text("0")),
|
||||||
|
sa.Column("queued_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("last_failure_reason", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column("last_failure_detail", sa.Text(), nullable=True),
|
||||||
|
sa.Column("last_source", sa.String(length=32), nullable=True),
|
||||||
|
sa.Column("last_requested_by_user_id", sa.Integer(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"status IN ('queued', 'running', 'resolved', 'failed')",
|
||||||
|
name="publication_pdf_jobs_status_valid",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["last_requested_by_user_id"],
|
||||||
|
["users.id"],
|
||||||
|
ondelete="SET NULL",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["publication_id"],
|
||||||
|
["publications.id"],
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("publication_id", name=op.f("pk_publication_pdf_jobs")),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _create_publication_pdf_jobs_indexes() -> None:
|
||||||
|
op.create_index(
|
||||||
|
"ix_publication_pdf_jobs_status",
|
||||||
|
"publication_pdf_jobs",
|
||||||
|
["status"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_publication_pdf_jobs_updated_at",
|
||||||
|
"publication_pdf_jobs",
|
||||||
|
["updated_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_publication_pdf_jobs_queued_at",
|
||||||
|
"publication_pdf_jobs",
|
||||||
|
["queued_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _create_publication_pdf_job_events_table() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"publication_pdf_job_events",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("publication_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("user_id", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("event_type", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=16), nullable=True),
|
||||||
|
sa.Column("source", sa.String(length=32), nullable=True),
|
||||||
|
sa.Column("failure_reason", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column("message", sa.Text(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["publication_id"],
|
||||||
|
["publications.id"],
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["user_id"],
|
||||||
|
["users.id"],
|
||||||
|
ondelete="SET NULL",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_publication_pdf_job_events")),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _create_publication_pdf_job_events_indexes() -> None:
|
||||||
|
op.create_index(
|
||||||
|
"ix_publication_pdf_job_events_publication_created",
|
||||||
|
"publication_pdf_job_events",
|
||||||
|
["publication_id", "created_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_publication_pdf_job_events_created_at",
|
||||||
|
"publication_pdf_job_events",
|
||||||
|
["created_at"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_publication_pdf_job_events_event_type",
|
||||||
|
"publication_pdf_job_events",
|
||||||
|
["event_type"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_publication_pdf_jobs_indexes() -> None:
|
||||||
|
op.drop_index("ix_publication_pdf_jobs_queued_at", table_name="publication_pdf_jobs")
|
||||||
|
op.drop_index("ix_publication_pdf_jobs_updated_at", table_name="publication_pdf_jobs")
|
||||||
|
op.drop_index("ix_publication_pdf_jobs_status", table_name="publication_pdf_jobs")
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_publication_pdf_job_events_indexes() -> None:
|
||||||
|
op.drop_index("ix_publication_pdf_job_events_event_type", table_name="publication_pdf_job_events")
|
||||||
|
op.drop_index("ix_publication_pdf_job_events_created_at", table_name="publication_pdf_job_events")
|
||||||
|
op.drop_index(
|
||||||
|
"ix_publication_pdf_job_events_publication_created",
|
||||||
|
table_name="publication_pdf_job_events",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
tables = _table_names()
|
||||||
|
if "publication_pdf_jobs" not in tables:
|
||||||
|
_create_publication_pdf_jobs_table()
|
||||||
|
_create_publication_pdf_jobs_indexes()
|
||||||
|
if "publication_pdf_job_events" not in tables:
|
||||||
|
_create_publication_pdf_job_events_table()
|
||||||
|
_create_publication_pdf_job_events_indexes()
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
tables = _table_names()
|
||||||
|
if "publication_pdf_job_events" in tables:
|
||||||
|
_drop_publication_pdf_job_events_indexes()
|
||||||
|
op.drop_table("publication_pdf_job_events")
|
||||||
|
if "publication_pdf_jobs" in tables:
|
||||||
|
_drop_publication_pdf_jobs_indexes()
|
||||||
|
op.drop_table("publication_pdf_jobs")
|
||||||
|
|
@ -0,0 +1,44 @@
|
||||||
|
"""add scholar publication favorite state
|
||||||
|
|
||||||
|
Revision ID: 20260221_0014
|
||||||
|
Revises: 20260221_0013
|
||||||
|
Create Date: 2026-02-21 14:35:00
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = "20260221_0014"
|
||||||
|
down_revision = "20260221_0013"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"scholar_publications",
|
||||||
|
sa.Column(
|
||||||
|
"is_favorite",
|
||||||
|
sa.Boolean(),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("false"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_scholar_publications_is_favorite",
|
||||||
|
"scholar_publications",
|
||||||
|
["is_favorite"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index(
|
||||||
|
"ix_scholar_publications_is_favorite",
|
||||||
|
table_name="scholar_publications",
|
||||||
|
)
|
||||||
|
op.drop_column("scholar_publications", "is_favorite")
|
||||||
|
|
@ -0,0 +1,80 @@
|
||||||
|
"""enforce request delay floor at two seconds
|
||||||
|
|
||||||
|
Revision ID: 20260221_0015
|
||||||
|
Revises: 20260221_0014
|
||||||
|
Create Date: 2026-02-21 18:10:00
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = "20260221_0015"
|
||||||
|
down_revision: str | Sequence[str] | None = "20260221_0014"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
TABLE_NAME = "user_settings"
|
||||||
|
CHECK_NAME_MIN_1 = "request_delay_seconds_min_1"
|
||||||
|
CHECK_NAME_MIN_2 = "request_delay_seconds_min_2"
|
||||||
|
LEGACY_CHECK_NAME_MIN_1 = "ck_user_settings_request_delay_seconds_min_1"
|
||||||
|
LEGACY_CHECK_NAME_MIN_2 = "ck_user_settings_request_delay_seconds_min_2"
|
||||||
|
|
||||||
|
|
||||||
|
def _check_constraints() -> set[str]:
|
||||||
|
bind = op.get_bind()
|
||||||
|
inspector = sa.inspect(bind)
|
||||||
|
return {
|
||||||
|
str(item.get("name"))
|
||||||
|
for item in inspector.get_check_constraints(TABLE_NAME)
|
||||||
|
if item.get("name")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_check_if_exists(name: str) -> None:
|
||||||
|
if name not in _check_constraints():
|
||||||
|
return
|
||||||
|
op.drop_constraint(
|
||||||
|
name,
|
||||||
|
TABLE_NAME,
|
||||||
|
type_="check",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.execute(
|
||||||
|
sa.text(
|
||||||
|
"UPDATE user_settings SET request_delay_seconds = 2 "
|
||||||
|
"WHERE request_delay_seconds < 2"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
_drop_check_if_exists(CHECK_NAME_MIN_1)
|
||||||
|
_drop_check_if_exists(LEGACY_CHECK_NAME_MIN_1)
|
||||||
|
|
||||||
|
existing = _check_constraints()
|
||||||
|
if CHECK_NAME_MIN_2 not in existing and LEGACY_CHECK_NAME_MIN_2 not in existing:
|
||||||
|
op.create_check_constraint(
|
||||||
|
CHECK_NAME_MIN_2,
|
||||||
|
TABLE_NAME,
|
||||||
|
"request_delay_seconds >= 2",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
_drop_check_if_exists(CHECK_NAME_MIN_2)
|
||||||
|
_drop_check_if_exists(LEGACY_CHECK_NAME_MIN_2)
|
||||||
|
|
||||||
|
existing = _check_constraints()
|
||||||
|
if CHECK_NAME_MIN_1 not in existing and LEGACY_CHECK_NAME_MIN_1 not in existing:
|
||||||
|
op.create_check_constraint(
|
||||||
|
CHECK_NAME_MIN_1,
|
||||||
|
TABLE_NAME,
|
||||||
|
"request_delay_seconds >= 1",
|
||||||
|
)
|
||||||
|
|
@ -2,11 +2,12 @@ from __future__ import annotations
|
||||||
|
|
||||||
from fastapi import APIRouter
|
from fastapi import APIRouter
|
||||||
|
|
||||||
from app.api.routers import admin, auth, publications, runs, scholars, settings
|
from app.api.routers import admin, admin_dbops, auth, publications, runs, scholars, settings
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1")
|
router = APIRouter(prefix="/api/v1")
|
||||||
router.include_router(auth.router)
|
router.include_router(auth.router)
|
||||||
router.include_router(admin.router)
|
router.include_router(admin.router)
|
||||||
|
router.include_router(admin_dbops.router)
|
||||||
router.include_router(scholars.router)
|
router.include_router(scholars.router)
|
||||||
router.include_router(settings.router)
|
router.include_router(settings.router)
|
||||||
router.include_router(runs.router)
|
router.include_router(runs.router)
|
||||||
|
|
|
||||||
337
app/api/routers/admin_dbops.py
Normal file
|
|
@ -0,0 +1,337 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Path, Query, Request
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.api.deps import get_api_admin_user
|
||||||
|
from app.api.errors import ApiException
|
||||||
|
from app.api.responses import success_payload
|
||||||
|
from app.api.schemas import (
|
||||||
|
AdminDbIntegrityEnvelope,
|
||||||
|
AdminPdfQueueBulkEnqueueEnvelope,
|
||||||
|
AdminPdfQueueRequeueEnvelope,
|
||||||
|
AdminPdfQueueEnvelope,
|
||||||
|
AdminDbRepairJobsEnvelope,
|
||||||
|
AdminRepairPublicationLinksEnvelope,
|
||||||
|
AdminRepairPublicationLinksRequest,
|
||||||
|
)
|
||||||
|
from app.db.models import DataRepairJob, User
|
||||||
|
from app.db.session import get_db_session
|
||||||
|
from app.services.domains.dbops import (
|
||||||
|
collect_integrity_report,
|
||||||
|
list_repair_jobs,
|
||||||
|
run_publication_link_repair,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications import application as publication_service
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/admin/db", tags=["api-admin-dbops"])
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize_repair_job(job: DataRepairJob) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"id": int(job.id),
|
||||||
|
"job_name": job.job_name,
|
||||||
|
"requested_by": job.requested_by,
|
||||||
|
"dry_run": bool(job.dry_run),
|
||||||
|
"status": job.status,
|
||||||
|
"scope": dict(job.scope or {}),
|
||||||
|
"summary": dict(job.summary or {}),
|
||||||
|
"error_text": job.error_text,
|
||||||
|
"started_at": job.started_at,
|
||||||
|
"finished_at": job.finished_at,
|
||||||
|
"created_at": job.created_at,
|
||||||
|
"updated_at": job.updated_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _requested_by_value(*, payload: AdminRepairPublicationLinksRequest, admin_user: User) -> str:
|
||||||
|
from_payload = (payload.requested_by or "").strip()
|
||||||
|
return from_payload or admin_user.email
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize_pdf_queue_item(item) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"publication_id": item.publication_id,
|
||||||
|
"title": item.title,
|
||||||
|
"doi": item.doi,
|
||||||
|
"pdf_url": item.pdf_url,
|
||||||
|
"status": item.status,
|
||||||
|
"attempt_count": item.attempt_count,
|
||||||
|
"last_failure_reason": item.last_failure_reason,
|
||||||
|
"last_failure_detail": item.last_failure_detail,
|
||||||
|
"last_source": item.last_source,
|
||||||
|
"requested_by_user_id": item.requested_by_user_id,
|
||||||
|
"requested_by_email": item.requested_by_email,
|
||||||
|
"queued_at": item.queued_at,
|
||||||
|
"last_attempt_at": item.last_attempt_at,
|
||||||
|
"resolved_at": item.resolved_at,
|
||||||
|
"updated_at": item.updated_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _requeue_response_state(*, queued: bool) -> tuple[str, str]:
|
||||||
|
if queued:
|
||||||
|
return "queued", "PDF lookup queued."
|
||||||
|
return "blocked", "PDF lookup is already queued or currently running."
|
||||||
|
|
||||||
|
|
||||||
|
def _bulk_enqueue_message(*, queued_count: int, requested_count: int) -> str:
|
||||||
|
if requested_count == 0:
|
||||||
|
return "No missing-PDF publications were found."
|
||||||
|
if queued_count == 0:
|
||||||
|
return "No publications were queued; all candidates were already in-flight."
|
||||||
|
return f"Queued {queued_count} publication(s) for PDF lookup."
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_pdf_queue_paging(
|
||||||
|
*,
|
||||||
|
page: int,
|
||||||
|
page_size: int,
|
||||||
|
limit: int | None,
|
||||||
|
offset: int | None,
|
||||||
|
) -> tuple[int, int, int]:
|
||||||
|
resolved_limit = int(limit) if limit is not None else int(page_size)
|
||||||
|
resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0)
|
||||||
|
resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1)
|
||||||
|
return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0)
|
||||||
|
|
||||||
|
|
||||||
|
def _pdf_queue_page_data(*, total_count: int, page: int, page_size: int, offset: int) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"total_count": int(total_count),
|
||||||
|
"page": int(page),
|
||||||
|
"page_size": int(page_size),
|
||||||
|
"has_prev": int(offset) > 0,
|
||||||
|
"has_next": int(offset) + int(page_size) < int(total_count),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/integrity",
|
||||||
|
response_model=AdminDbIntegrityEnvelope,
|
||||||
|
)
|
||||||
|
async def get_integrity_report(
|
||||||
|
request: Request,
|
||||||
|
db_session: AsyncSession = Depends(get_db_session),
|
||||||
|
admin_user: User = Depends(get_api_admin_user),
|
||||||
|
):
|
||||||
|
report = await collect_integrity_report(db_session)
|
||||||
|
logger.info(
|
||||||
|
"api.admin.db.integrity_checked",
|
||||||
|
extra={
|
||||||
|
"event": "api.admin.db.integrity_checked",
|
||||||
|
"admin_user_id": int(admin_user.id),
|
||||||
|
"status": report.get("status"),
|
||||||
|
"failure_count": len(report.get("failures", [])),
|
||||||
|
"warning_count": len(report.get("warnings", [])),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return success_payload(request, data=report)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/repair-jobs",
|
||||||
|
response_model=AdminDbRepairJobsEnvelope,
|
||||||
|
)
|
||||||
|
async def get_repair_jobs(
|
||||||
|
request: Request,
|
||||||
|
limit: int = Query(default=50, ge=1, le=200),
|
||||||
|
db_session: AsyncSession = Depends(get_db_session),
|
||||||
|
admin_user: User = Depends(get_api_admin_user),
|
||||||
|
):
|
||||||
|
jobs = await list_repair_jobs(db_session, limit=limit)
|
||||||
|
logger.info(
|
||||||
|
"api.admin.db.repair_jobs_listed",
|
||||||
|
extra={
|
||||||
|
"event": "api.admin.db.repair_jobs_listed",
|
||||||
|
"admin_user_id": int(admin_user.id),
|
||||||
|
"limit": int(limit),
|
||||||
|
"job_count": len(jobs),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return success_payload(
|
||||||
|
request,
|
||||||
|
data={"jobs": [_serialize_repair_job(job) for job in jobs]},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/pdf-queue",
|
||||||
|
response_model=AdminPdfQueueEnvelope,
|
||||||
|
)
|
||||||
|
async def get_pdf_queue(
|
||||||
|
request: Request,
|
||||||
|
page: int = Query(default=1, ge=1),
|
||||||
|
page_size: int = Query(default=100, ge=1, le=500),
|
||||||
|
limit: int | None = Query(default=None, ge=1, le=500),
|
||||||
|
offset: int | None = Query(default=None, ge=0),
|
||||||
|
status: str | None = Query(default=None),
|
||||||
|
db_session: AsyncSession = Depends(get_db_session),
|
||||||
|
admin_user: User = Depends(get_api_admin_user),
|
||||||
|
):
|
||||||
|
normalized_status = (status or "").strip().lower() or None
|
||||||
|
resolved_page, resolved_limit, resolved_offset = _resolve_pdf_queue_paging(
|
||||||
|
page=page,
|
||||||
|
page_size=page_size,
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
|
)
|
||||||
|
queue_page = await publication_service.list_pdf_queue_page(
|
||||||
|
db_session,
|
||||||
|
limit=resolved_limit,
|
||||||
|
offset=resolved_offset,
|
||||||
|
status=normalized_status,
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"api.admin.db.pdf_queue_listed",
|
||||||
|
extra={
|
||||||
|
"event": "api.admin.db.pdf_queue_listed",
|
||||||
|
"admin_user_id": int(admin_user.id),
|
||||||
|
"page": int(resolved_page),
|
||||||
|
"page_size": int(resolved_limit),
|
||||||
|
"offset": int(resolved_offset),
|
||||||
|
"status": normalized_status,
|
||||||
|
"item_count": len(queue_page.items),
|
||||||
|
"total_count": int(queue_page.total_count),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return success_payload(
|
||||||
|
request,
|
||||||
|
data={
|
||||||
|
"items": [_serialize_pdf_queue_item(item) for item in queue_page.items],
|
||||||
|
**_pdf_queue_page_data(
|
||||||
|
total_count=queue_page.total_count,
|
||||||
|
page=resolved_page,
|
||||||
|
page_size=resolved_limit,
|
||||||
|
offset=resolved_offset,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/pdf-queue/{publication_id}/requeue",
|
||||||
|
response_model=AdminPdfQueueRequeueEnvelope,
|
||||||
|
)
|
||||||
|
async def requeue_pdf_lookup(
|
||||||
|
request: Request,
|
||||||
|
publication_id: int = Path(ge=1),
|
||||||
|
db_session: AsyncSession = Depends(get_db_session),
|
||||||
|
admin_user: User = Depends(get_api_admin_user),
|
||||||
|
):
|
||||||
|
result = await publication_service.enqueue_retry_pdf_job_for_publication_id(
|
||||||
|
db_session,
|
||||||
|
user_id=int(admin_user.id),
|
||||||
|
request_email=admin_user.email,
|
||||||
|
publication_id=publication_id,
|
||||||
|
)
|
||||||
|
if not result.publication_exists:
|
||||||
|
raise ApiException(
|
||||||
|
status_code=404,
|
||||||
|
code="publication_not_found",
|
||||||
|
message="Publication not found.",
|
||||||
|
)
|
||||||
|
status, message = _requeue_response_state(queued=result.queued)
|
||||||
|
logger.info(
|
||||||
|
"api.admin.db.pdf_queue_requeue_requested",
|
||||||
|
extra={
|
||||||
|
"event": "api.admin.db.pdf_queue_requeue_requested",
|
||||||
|
"admin_user_id": int(admin_user.id),
|
||||||
|
"publication_id": int(publication_id),
|
||||||
|
"queued": bool(result.queued),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return success_payload(
|
||||||
|
request,
|
||||||
|
data={
|
||||||
|
"publication_id": int(publication_id),
|
||||||
|
"queued": bool(result.queued),
|
||||||
|
"status": status,
|
||||||
|
"message": message,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/pdf-queue/requeue-all",
|
||||||
|
response_model=AdminPdfQueueBulkEnqueueEnvelope,
|
||||||
|
)
|
||||||
|
async def requeue_all_missing_pdfs(
|
||||||
|
request: Request,
|
||||||
|
limit: int = Query(default=1000, ge=1, le=5000),
|
||||||
|
db_session: AsyncSession = Depends(get_db_session),
|
||||||
|
admin_user: User = Depends(get_api_admin_user),
|
||||||
|
):
|
||||||
|
result = await publication_service.enqueue_all_missing_pdf_jobs(
|
||||||
|
db_session,
|
||||||
|
user_id=int(admin_user.id),
|
||||||
|
request_email=admin_user.email,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"api.admin.db.pdf_queue_requeue_all",
|
||||||
|
extra={
|
||||||
|
"event": "api.admin.db.pdf_queue_requeue_all",
|
||||||
|
"admin_user_id": int(admin_user.id),
|
||||||
|
"limit": int(limit),
|
||||||
|
"requested_count": int(result.requested_count),
|
||||||
|
"queued_count": int(result.queued_count),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return success_payload(
|
||||||
|
request,
|
||||||
|
data={
|
||||||
|
"requested_count": int(result.requested_count),
|
||||||
|
"queued_count": int(result.queued_count),
|
||||||
|
"message": _bulk_enqueue_message(
|
||||||
|
queued_count=int(result.queued_count),
|
||||||
|
requested_count=int(result.requested_count),
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/repairs/publication-links",
|
||||||
|
response_model=AdminRepairPublicationLinksEnvelope,
|
||||||
|
)
|
||||||
|
async def trigger_publication_link_repair(
|
||||||
|
payload: AdminRepairPublicationLinksRequest,
|
||||||
|
request: Request,
|
||||||
|
db_session: AsyncSession = Depends(get_db_session),
|
||||||
|
admin_user: User = Depends(get_api_admin_user),
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
result = await run_publication_link_repair(
|
||||||
|
db_session,
|
||||||
|
scope_mode=payload.scope_mode,
|
||||||
|
user_id=payload.user_id,
|
||||||
|
scholar_profile_ids=payload.scholar_profile_ids,
|
||||||
|
dry_run=bool(payload.dry_run),
|
||||||
|
gc_orphan_publications=bool(payload.gc_orphan_publications),
|
||||||
|
requested_by=_requested_by_value(payload=payload, admin_user=admin_user),
|
||||||
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise ApiException(
|
||||||
|
status_code=400,
|
||||||
|
code="invalid_repair_scope",
|
||||||
|
message=str(exc),
|
||||||
|
) from exc
|
||||||
|
logger.info(
|
||||||
|
"api.admin.db.publication_link_repair_triggered",
|
||||||
|
extra={
|
||||||
|
"event": "api.admin.db.publication_link_repair_triggered",
|
||||||
|
"admin_user_id": int(admin_user.id),
|
||||||
|
"scope_mode": payload.scope_mode,
|
||||||
|
"target_user_id": int(payload.user_id) if payload.user_id is not None else None,
|
||||||
|
"dry_run": bool(payload.dry_run),
|
||||||
|
"gc_orphan_publications": bool(payload.gc_orphan_publications),
|
||||||
|
"job_id": int(result["job_id"]),
|
||||||
|
"status": result["status"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return success_payload(request, data=result)
|
||||||
|
|
@ -16,6 +16,8 @@ from app.api.schemas import (
|
||||||
PublicationsListEnvelope,
|
PublicationsListEnvelope,
|
||||||
RetryPublicationPdfEnvelope,
|
RetryPublicationPdfEnvelope,
|
||||||
RetryPublicationPdfRequest,
|
RetryPublicationPdfRequest,
|
||||||
|
TogglePublicationFavoriteEnvelope,
|
||||||
|
TogglePublicationFavoriteRequest,
|
||||||
)
|
)
|
||||||
from app.db.models import User
|
from app.db.models import User
|
||||||
from app.db.session import get_db_session
|
from app.db.session import get_db_session
|
||||||
|
|
@ -61,7 +63,12 @@ def _serialize_publication_item(item) -> dict[str, object]:
|
||||||
"pub_url": item.pub_url,
|
"pub_url": item.pub_url,
|
||||||
"doi": item.doi,
|
"doi": item.doi,
|
||||||
"pdf_url": item.pdf_url,
|
"pdf_url": item.pdf_url,
|
||||||
|
"pdf_status": item.pdf_status,
|
||||||
|
"pdf_attempt_count": item.pdf_attempt_count,
|
||||||
|
"pdf_failure_reason": item.pdf_failure_reason,
|
||||||
|
"pdf_failure_detail": item.pdf_failure_detail,
|
||||||
"is_read": item.is_read,
|
"is_read": item.is_read,
|
||||||
|
"is_favorite": item.is_favorite,
|
||||||
"first_seen_at": item.first_seen_at,
|
"first_seen_at": item.first_seen_at,
|
||||||
"is_new_in_latest_run": item.is_new_in_latest_run,
|
"is_new_in_latest_run": item.is_new_in_latest_run,
|
||||||
}
|
}
|
||||||
|
|
@ -72,46 +79,235 @@ async def _publication_counts(
|
||||||
*,
|
*,
|
||||||
user_id: int,
|
user_id: int,
|
||||||
selected_scholar_id: int | None,
|
selected_scholar_id: int | None,
|
||||||
) -> tuple[int, int, int]:
|
favorite_only: bool,
|
||||||
|
) -> tuple[int, int, int, int]:
|
||||||
unread_count = await publication_service.count_unread_for_user(
|
unread_count = await publication_service.count_unread_for_user(
|
||||||
db_session,
|
db_session,
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
scholar_profile_id=selected_scholar_id,
|
scholar_profile_id=selected_scholar_id,
|
||||||
|
favorite_only=favorite_only,
|
||||||
|
)
|
||||||
|
favorites_count = await publication_service.count_favorite_for_user(
|
||||||
|
db_session,
|
||||||
|
user_id=user_id,
|
||||||
|
scholar_profile_id=selected_scholar_id,
|
||||||
)
|
)
|
||||||
latest_count = await publication_service.count_latest_for_user(
|
latest_count = await publication_service.count_latest_for_user(
|
||||||
db_session,
|
db_session,
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
scholar_profile_id=selected_scholar_id,
|
scholar_profile_id=selected_scholar_id,
|
||||||
|
favorite_only=favorite_only,
|
||||||
)
|
)
|
||||||
total_count = await publication_service.count_for_user(
|
total_count = await publication_service.count_for_user(
|
||||||
db_session,
|
db_session,
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
mode=publication_service.MODE_ALL,
|
mode=publication_service.MODE_ALL,
|
||||||
scholar_profile_id=selected_scholar_id,
|
scholar_profile_id=selected_scholar_id,
|
||||||
|
favorite_only=favorite_only,
|
||||||
)
|
)
|
||||||
return unread_count, latest_count, total_count
|
return unread_count, favorites_count, latest_count, total_count
|
||||||
|
|
||||||
|
|
||||||
|
async def _list_publications_for_request(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
current_user: User,
|
||||||
|
mode: Literal["all", "unread", "latest", "new"] | None,
|
||||||
|
favorite_only: bool,
|
||||||
|
scholar_profile_id: int | None,
|
||||||
|
limit: int,
|
||||||
|
offset: int,
|
||||||
|
) -> tuple[str, int | None, list]:
|
||||||
|
resolved_mode = publication_service.resolve_publication_view_mode(mode)
|
||||||
|
selected_scholar_id = scholar_profile_id
|
||||||
|
await _require_selected_profile(
|
||||||
|
db_session,
|
||||||
|
user_id=current_user.id,
|
||||||
|
selected_scholar_id=selected_scholar_id,
|
||||||
|
)
|
||||||
|
publications = await publication_service.list_for_user(
|
||||||
|
db_session,
|
||||||
|
user_id=current_user.id,
|
||||||
|
mode=resolved_mode,
|
||||||
|
scholar_profile_id=selected_scholar_id,
|
||||||
|
favorite_only=favorite_only,
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
|
)
|
||||||
|
await publication_service.schedule_missing_pdf_enrichment_for_user(
|
||||||
|
db_session,
|
||||||
|
user_id=current_user.id,
|
||||||
|
request_email=current_user.email,
|
||||||
|
items=publications,
|
||||||
|
max_items=settings.unpaywall_max_items_per_request,
|
||||||
|
)
|
||||||
|
hydrated = await publication_service.hydrate_pdf_enrichment_state(
|
||||||
|
db_session,
|
||||||
|
items=publications,
|
||||||
|
)
|
||||||
|
return resolved_mode, selected_scholar_id, hydrated
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_publications_paging(
|
||||||
|
*,
|
||||||
|
page: int,
|
||||||
|
page_size: int,
|
||||||
|
limit: int | None,
|
||||||
|
offset: int | None,
|
||||||
|
) -> tuple[int, int, int]:
|
||||||
|
resolved_limit = int(limit) if limit is not None else int(page_size)
|
||||||
|
resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0)
|
||||||
|
resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1)
|
||||||
|
return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0)
|
||||||
|
|
||||||
|
|
||||||
def _publications_list_data(
|
def _publications_list_data(
|
||||||
*,
|
*,
|
||||||
mode: str,
|
mode: str,
|
||||||
|
favorite_only: bool,
|
||||||
selected_scholar_id: int | None,
|
selected_scholar_id: int | None,
|
||||||
unread_count: int,
|
unread_count: int,
|
||||||
|
favorites_count: int,
|
||||||
latest_count: int,
|
latest_count: int,
|
||||||
total_count: int,
|
total_count: int,
|
||||||
publications: list,
|
publications: list,
|
||||||
|
page: int,
|
||||||
|
page_size: int,
|
||||||
|
offset: int,
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
return {
|
return {
|
||||||
"mode": mode,
|
"mode": mode,
|
||||||
|
"favorite_only": favorite_only,
|
||||||
"selected_scholar_profile_id": selected_scholar_id,
|
"selected_scholar_profile_id": selected_scholar_id,
|
||||||
"unread_count": unread_count,
|
"unread_count": unread_count,
|
||||||
|
"favorites_count": favorites_count,
|
||||||
"latest_count": latest_count,
|
"latest_count": latest_count,
|
||||||
"new_count": latest_count,
|
"new_count": latest_count,
|
||||||
"total_count": total_count,
|
"total_count": total_count,
|
||||||
|
"page": int(page),
|
||||||
|
"page_size": int(page_size),
|
||||||
|
"has_prev": int(offset) > 0,
|
||||||
|
"has_next": int(offset) + int(page_size) < int(total_count),
|
||||||
"publications": [_serialize_publication_item(item) for item in publications],
|
"publications": [_serialize_publication_item(item) for item in publications],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _retry_pdf_message(*, queued: bool, resolved_pdf: bool, pdf_status: str) -> str:
|
||||||
|
if resolved_pdf:
|
||||||
|
return "Open-access PDF link already resolved."
|
||||||
|
if queued:
|
||||||
|
return "PDF lookup queued."
|
||||||
|
if pdf_status in {"queued", "running"}:
|
||||||
|
return "PDF lookup is already queued."
|
||||||
|
return "No open-access PDF link found."
|
||||||
|
|
||||||
|
|
||||||
|
def _favorite_message(*, is_favorite: bool) -> str:
|
||||||
|
if is_favorite:
|
||||||
|
return "Publication marked as favorite."
|
||||||
|
return "Publication removed from favorites."
|
||||||
|
|
||||||
|
|
||||||
|
async def _retry_publication_state(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
current_user: User,
|
||||||
|
scholar_profile_id: int,
|
||||||
|
publication_id: int,
|
||||||
|
):
|
||||||
|
publication = await publication_service.retry_pdf_for_user(
|
||||||
|
db_session,
|
||||||
|
user_id=current_user.id,
|
||||||
|
scholar_profile_id=scholar_profile_id,
|
||||||
|
publication_id=publication_id,
|
||||||
|
)
|
||||||
|
if publication is None:
|
||||||
|
raise ApiException(
|
||||||
|
status_code=404,
|
||||||
|
code="publication_not_found",
|
||||||
|
message="Publication not found.",
|
||||||
|
)
|
||||||
|
queued = False
|
||||||
|
if not publication.pdf_url:
|
||||||
|
queued = await publication_service.schedule_retry_pdf_enrichment_for_row(
|
||||||
|
db_session,
|
||||||
|
user_id=current_user.id,
|
||||||
|
request_email=current_user.email,
|
||||||
|
item=publication,
|
||||||
|
)
|
||||||
|
hydrated = await publication_service.hydrate_pdf_enrichment_state(
|
||||||
|
db_session,
|
||||||
|
items=[publication],
|
||||||
|
)
|
||||||
|
current = hydrated[0] if hydrated else publication
|
||||||
|
return current, queued
|
||||||
|
|
||||||
|
|
||||||
|
async def _favorite_publication_state(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
current_user: User,
|
||||||
|
scholar_profile_id: int,
|
||||||
|
publication_id: int,
|
||||||
|
is_favorite: bool,
|
||||||
|
):
|
||||||
|
updated_count = await publication_service.set_publication_favorite_for_user(
|
||||||
|
db_session,
|
||||||
|
user_id=current_user.id,
|
||||||
|
scholar_profile_id=scholar_profile_id,
|
||||||
|
publication_id=publication_id,
|
||||||
|
is_favorite=is_favorite,
|
||||||
|
)
|
||||||
|
if updated_count <= 0:
|
||||||
|
raise ApiException(
|
||||||
|
status_code=404,
|
||||||
|
code="publication_not_found",
|
||||||
|
message="Publication not found.",
|
||||||
|
)
|
||||||
|
publication = await publication_service.get_publication_item_for_user(
|
||||||
|
db_session,
|
||||||
|
user_id=current_user.id,
|
||||||
|
scholar_profile_id=scholar_profile_id,
|
||||||
|
publication_id=publication_id,
|
||||||
|
)
|
||||||
|
if publication is None:
|
||||||
|
raise ApiException(
|
||||||
|
status_code=404,
|
||||||
|
code="publication_not_found",
|
||||||
|
message="Publication not found.",
|
||||||
|
)
|
||||||
|
hydrated = await publication_service.hydrate_pdf_enrichment_state(
|
||||||
|
db_session,
|
||||||
|
items=[publication],
|
||||||
|
)
|
||||||
|
return hydrated[0] if hydrated else publication
|
||||||
|
|
||||||
|
|
||||||
|
def _log_retry_pdf_result(
|
||||||
|
*,
|
||||||
|
current_user: User,
|
||||||
|
scholar_profile_id: int,
|
||||||
|
publication_id: int,
|
||||||
|
queued: bool,
|
||||||
|
resolved_pdf: bool,
|
||||||
|
pdf_status: str,
|
||||||
|
doi: str | None,
|
||||||
|
) -> None:
|
||||||
|
logger.info(
|
||||||
|
"api.publications.retry_pdf",
|
||||||
|
extra={
|
||||||
|
"event": "api.publications.retry_pdf",
|
||||||
|
"user_id": current_user.id,
|
||||||
|
"scholar_profile_id": scholar_profile_id,
|
||||||
|
"publication_id": publication_id,
|
||||||
|
"queued": queued,
|
||||||
|
"resolved_pdf": resolved_pdf,
|
||||||
|
"pdf_status": pdf_status,
|
||||||
|
"has_doi": bool(doi),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get(
|
@router.get(
|
||||||
"",
|
"",
|
||||||
response_model=PublicationsListEnvelope,
|
response_model=PublicationsListEnvelope,
|
||||||
|
|
@ -119,44 +315,48 @@ def _publications_list_data(
|
||||||
async def list_publications(
|
async def list_publications(
|
||||||
request: Request,
|
request: Request,
|
||||||
mode: Literal["all", "unread", "latest", "new"] | None = Query(default=None),
|
mode: Literal["all", "unread", "latest", "new"] | None = Query(default=None),
|
||||||
|
favorite_only: bool = Query(default=False),
|
||||||
scholar_profile_id: int | None = Query(default=None, ge=1),
|
scholar_profile_id: int | None = Query(default=None, ge=1),
|
||||||
limit: int = Query(default=300, ge=1, le=1000),
|
page: int = Query(default=1, ge=1),
|
||||||
|
page_size: int = Query(default=100, ge=1, le=500),
|
||||||
|
limit: int | None = Query(default=None, ge=1, le=1000),
|
||||||
|
offset: int | None = Query(default=None, ge=0),
|
||||||
db_session: AsyncSession = Depends(get_db_session),
|
db_session: AsyncSession = Depends(get_db_session),
|
||||||
current_user: User = Depends(get_api_current_user),
|
current_user: User = Depends(get_api_current_user),
|
||||||
):
|
):
|
||||||
resolved_mode = publication_service.resolve_publication_view_mode(mode)
|
resolved_page, resolved_limit, resolved_offset = _resolve_publications_paging(
|
||||||
selected_scholar_id = scholar_profile_id
|
page=page,
|
||||||
await _require_selected_profile(
|
page_size=page_size,
|
||||||
db_session,
|
|
||||||
user_id=current_user.id,
|
|
||||||
selected_scholar_id=selected_scholar_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
publications = await publication_service.list_for_user(
|
|
||||||
db_session,
|
|
||||||
user_id=current_user.id,
|
|
||||||
mode=resolved_mode,
|
|
||||||
scholar_profile_id=selected_scholar_id,
|
|
||||||
limit=limit,
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
)
|
)
|
||||||
await publication_service.schedule_missing_pdf_enrichment_for_user(
|
resolved_mode, selected_scholar_id, publications = await _list_publications_for_request(
|
||||||
user_id=current_user.id,
|
db_session,
|
||||||
request_email=current_user.email,
|
current_user=current_user,
|
||||||
items=publications,
|
mode=mode,
|
||||||
max_items=settings.unpaywall_max_items_per_request,
|
favorite_only=favorite_only,
|
||||||
|
scholar_profile_id=scholar_profile_id,
|
||||||
|
limit=resolved_limit,
|
||||||
|
offset=resolved_offset,
|
||||||
)
|
)
|
||||||
unread_count, latest_count, total_count = await _publication_counts(
|
unread_count, favorites_count, latest_count, total_count = await _publication_counts(
|
||||||
db_session,
|
db_session,
|
||||||
user_id=current_user.id,
|
user_id=current_user.id,
|
||||||
selected_scholar_id=selected_scholar_id,
|
selected_scholar_id=selected_scholar_id,
|
||||||
|
favorite_only=favorite_only,
|
||||||
)
|
)
|
||||||
data = _publications_list_data(
|
data = _publications_list_data(
|
||||||
mode=resolved_mode,
|
mode=resolved_mode,
|
||||||
|
favorite_only=favorite_only,
|
||||||
selected_scholar_id=selected_scholar_id,
|
selected_scholar_id=selected_scholar_id,
|
||||||
unread_count=unread_count,
|
unread_count=unread_count,
|
||||||
|
favorites_count=favorites_count,
|
||||||
latest_count=latest_count,
|
latest_count=latest_count,
|
||||||
total_count=total_count,
|
total_count=total_count,
|
||||||
publications=publications,
|
publications=publications,
|
||||||
|
page=resolved_page,
|
||||||
|
page_size=resolved_limit,
|
||||||
|
offset=resolved_offset,
|
||||||
)
|
)
|
||||||
return success_payload(request, data=data)
|
return success_payload(request, data=data)
|
||||||
|
|
||||||
|
|
@ -242,37 +442,70 @@ async def retry_publication_pdf(
|
||||||
db_session: AsyncSession = Depends(get_db_session),
|
db_session: AsyncSession = Depends(get_db_session),
|
||||||
current_user: User = Depends(get_api_current_user),
|
current_user: User = Depends(get_api_current_user),
|
||||||
):
|
):
|
||||||
publication = await publication_service.retry_pdf_for_user(
|
current, queued = await _retry_publication_state(
|
||||||
db_session,
|
db_session,
|
||||||
user_id=current_user.id,
|
current_user=current_user,
|
||||||
scholar_profile_id=payload.scholar_profile_id,
|
scholar_profile_id=payload.scholar_profile_id,
|
||||||
publication_id=publication_id,
|
publication_id=publication_id,
|
||||||
unpaywall_email=current_user.email,
|
|
||||||
)
|
)
|
||||||
if publication is None:
|
resolved_pdf = bool(current.pdf_url)
|
||||||
raise ApiException(
|
message = _retry_pdf_message(
|
||||||
status_code=404,
|
queued=queued,
|
||||||
code="publication_not_found",
|
resolved_pdf=resolved_pdf,
|
||||||
message="Publication not found.",
|
pdf_status=current.pdf_status,
|
||||||
)
|
)
|
||||||
resolved_pdf = bool(publication.pdf_url)
|
_log_retry_pdf_result(
|
||||||
logger.info(
|
current_user=current_user,
|
||||||
"api.publications.retry_pdf",
|
scholar_profile_id=payload.scholar_profile_id,
|
||||||
extra={
|
publication_id=publication_id,
|
||||||
"event": "api.publications.retry_pdf",
|
queued=queued,
|
||||||
"user_id": current_user.id,
|
resolved_pdf=resolved_pdf,
|
||||||
"scholar_profile_id": payload.scholar_profile_id,
|
pdf_status=current.pdf_status,
|
||||||
"publication_id": publication_id,
|
doi=current.doi,
|
||||||
"resolved_pdf": resolved_pdf,
|
|
||||||
"has_doi": bool(publication.doi),
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
message = "Open-access PDF link resolved." if resolved_pdf else "No open-access PDF link found."
|
|
||||||
return success_payload(
|
return success_payload(
|
||||||
request,
|
request,
|
||||||
data={
|
data={
|
||||||
"message": message,
|
"message": message,
|
||||||
|
"queued": queued,
|
||||||
"resolved_pdf": resolved_pdf,
|
"resolved_pdf": resolved_pdf,
|
||||||
"publication": _serialize_publication_item(publication),
|
"publication": _serialize_publication_item(current),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/{publication_id}/favorite",
|
||||||
|
response_model=TogglePublicationFavoriteEnvelope,
|
||||||
|
)
|
||||||
|
async def toggle_publication_favorite(
|
||||||
|
payload: TogglePublicationFavoriteRequest,
|
||||||
|
request: Request,
|
||||||
|
publication_id: int = Path(ge=1),
|
||||||
|
db_session: AsyncSession = Depends(get_db_session),
|
||||||
|
current_user: User = Depends(get_api_current_user),
|
||||||
|
):
|
||||||
|
current = await _favorite_publication_state(
|
||||||
|
db_session,
|
||||||
|
current_user=current_user,
|
||||||
|
scholar_profile_id=payload.scholar_profile_id,
|
||||||
|
publication_id=publication_id,
|
||||||
|
is_favorite=payload.is_favorite,
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"api.publications.favorite",
|
||||||
|
extra={
|
||||||
|
"event": "api.publications.favorite",
|
||||||
|
"user_id": current_user.id,
|
||||||
|
"scholar_profile_id": payload.scholar_profile_id,
|
||||||
|
"publication_id": publication_id,
|
||||||
|
"is_favorite": bool(payload.is_favorite),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return success_payload(
|
||||||
|
request,
|
||||||
|
data={
|
||||||
|
"message": _favorite_message(is_favorite=bool(payload.is_favorite)),
|
||||||
|
"publication": _serialize_publication_item(current),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,9 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any
|
from typing import Any, Literal
|
||||||
|
|
||||||
from pydantic import BaseModel, ConfigDict, Field
|
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||||
|
|
||||||
|
|
||||||
class ApiMeta(BaseModel):
|
class ApiMeta(BaseModel):
|
||||||
|
|
@ -525,6 +525,175 @@ class AdminResetPasswordRequest(BaseModel):
|
||||||
model_config = ConfigDict(extra="forbid")
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminDbIntegrityCheckData(BaseModel):
|
||||||
|
name: str
|
||||||
|
count: int
|
||||||
|
severity: str
|
||||||
|
message: str
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminDbIntegrityData(BaseModel):
|
||||||
|
status: str
|
||||||
|
checked_at: datetime
|
||||||
|
failures: list[str] = Field(default_factory=list)
|
||||||
|
warnings: list[str] = Field(default_factory=list)
|
||||||
|
checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminDbIntegrityEnvelope(BaseModel):
|
||||||
|
data: AdminDbIntegrityData
|
||||||
|
meta: ApiMeta
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminDbRepairJobData(BaseModel):
|
||||||
|
id: int
|
||||||
|
job_name: str
|
||||||
|
requested_by: str | None
|
||||||
|
dry_run: bool
|
||||||
|
status: str
|
||||||
|
scope: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
summary: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
error_text: str | None
|
||||||
|
started_at: datetime | None
|
||||||
|
finished_at: datetime | None
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminDbRepairJobsData(BaseModel):
|
||||||
|
jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminDbRepairJobsEnvelope(BaseModel):
|
||||||
|
data: AdminDbRepairJobsData
|
||||||
|
meta: ApiMeta
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminPdfQueueItemData(BaseModel):
|
||||||
|
publication_id: int
|
||||||
|
title: str
|
||||||
|
doi: str | None
|
||||||
|
pdf_url: str | None
|
||||||
|
status: str
|
||||||
|
attempt_count: int
|
||||||
|
last_failure_reason: str | None
|
||||||
|
last_failure_detail: str | None
|
||||||
|
last_source: str | None
|
||||||
|
requested_by_user_id: int | None
|
||||||
|
requested_by_email: str | None
|
||||||
|
queued_at: datetime | None
|
||||||
|
last_attempt_at: datetime | None
|
||||||
|
resolved_at: datetime | None
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminPdfQueueData(BaseModel):
|
||||||
|
items: list[AdminPdfQueueItemData] = Field(default_factory=list)
|
||||||
|
total_count: int = 0
|
||||||
|
page: int = 1
|
||||||
|
page_size: int = 100
|
||||||
|
has_next: bool = False
|
||||||
|
has_prev: bool = False
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminPdfQueueEnvelope(BaseModel):
|
||||||
|
data: AdminPdfQueueData
|
||||||
|
meta: ApiMeta
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminPdfQueueRequeueData(BaseModel):
|
||||||
|
publication_id: int
|
||||||
|
queued: bool
|
||||||
|
status: str
|
||||||
|
message: str
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminPdfQueueRequeueEnvelope(BaseModel):
|
||||||
|
data: AdminPdfQueueRequeueData
|
||||||
|
meta: ApiMeta
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminPdfQueueBulkEnqueueData(BaseModel):
|
||||||
|
requested_count: int
|
||||||
|
queued_count: int
|
||||||
|
message: str
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
|
||||||
|
data: AdminPdfQueueBulkEnqueueData
|
||||||
|
meta: ApiMeta
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminRepairPublicationLinksRequest(BaseModel):
|
||||||
|
scope_mode: Literal["single_user", "all_users"] = "single_user"
|
||||||
|
user_id: int | None = Field(default=None, ge=1)
|
||||||
|
scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
|
||||||
|
dry_run: bool = True
|
||||||
|
gc_orphan_publications: bool = False
|
||||||
|
requested_by: str | None = None
|
||||||
|
confirmation_text: str | None = None
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_scope(self) -> "AdminRepairPublicationLinksRequest":
|
||||||
|
if self.scope_mode == "single_user" and self.user_id is None:
|
||||||
|
raise ValueError("user_id is required when scope_mode=single_user.")
|
||||||
|
if self.scope_mode == "all_users" and self.user_id is not None:
|
||||||
|
raise ValueError("user_id must be omitted when scope_mode=all_users.")
|
||||||
|
if not self.dry_run and self.scope_mode == "all_users":
|
||||||
|
expected = "REPAIR ALL USERS"
|
||||||
|
provided = (self.confirmation_text or "").strip()
|
||||||
|
if provided != expected:
|
||||||
|
raise ValueError(
|
||||||
|
"confirmation_text must equal 'REPAIR ALL USERS' "
|
||||||
|
"when applying a repair to all users."
|
||||||
|
)
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class AdminRepairPublicationLinksResultData(BaseModel):
|
||||||
|
job_id: int
|
||||||
|
status: str
|
||||||
|
scope: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
summary: dict[str, Any] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class AdminRepairPublicationLinksEnvelope(BaseModel):
|
||||||
|
data: AdminRepairPublicationLinksResultData
|
||||||
|
meta: ApiMeta
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
class SettingsPolicyData(BaseModel):
|
class SettingsPolicyData(BaseModel):
|
||||||
min_run_interval_minutes: int
|
min_run_interval_minutes: int
|
||||||
min_request_delay_seconds: int
|
min_request_delay_seconds: int
|
||||||
|
|
@ -576,7 +745,12 @@ class PublicationItemData(BaseModel):
|
||||||
pub_url: str | None
|
pub_url: str | None
|
||||||
doi: str | None
|
doi: str | None
|
||||||
pdf_url: str | None
|
pdf_url: str | None
|
||||||
|
pdf_status: str = "untracked"
|
||||||
|
pdf_attempt_count: int = 0
|
||||||
|
pdf_failure_reason: str | None = None
|
||||||
|
pdf_failure_detail: str | None = None
|
||||||
is_read: bool
|
is_read: bool
|
||||||
|
is_favorite: bool = False
|
||||||
first_seen_at: datetime
|
first_seen_at: datetime
|
||||||
is_new_in_latest_run: bool
|
is_new_in_latest_run: bool
|
||||||
|
|
||||||
|
|
@ -585,11 +759,17 @@ class PublicationItemData(BaseModel):
|
||||||
|
|
||||||
class PublicationsListData(BaseModel):
|
class PublicationsListData(BaseModel):
|
||||||
mode: str
|
mode: str
|
||||||
|
favorite_only: bool = False
|
||||||
selected_scholar_profile_id: int | None
|
selected_scholar_profile_id: int | None
|
||||||
unread_count: int
|
unread_count: int
|
||||||
|
favorites_count: int
|
||||||
latest_count: int
|
latest_count: int
|
||||||
new_count: int
|
new_count: int
|
||||||
total_count: int
|
total_count: int
|
||||||
|
page: int
|
||||||
|
page_size: int
|
||||||
|
has_next: bool = False
|
||||||
|
has_prev: bool = False
|
||||||
publications: list[PublicationItemData]
|
publications: list[PublicationItemData]
|
||||||
|
|
||||||
model_config = ConfigDict(extra="forbid")
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
@ -652,6 +832,7 @@ class RetryPublicationPdfRequest(BaseModel):
|
||||||
|
|
||||||
class RetryPublicationPdfData(BaseModel):
|
class RetryPublicationPdfData(BaseModel):
|
||||||
message: str
|
message: str
|
||||||
|
queued: bool
|
||||||
resolved_pdf: bool
|
resolved_pdf: bool
|
||||||
publication: PublicationItemData
|
publication: PublicationItemData
|
||||||
|
|
||||||
|
|
@ -663,3 +844,24 @@ class RetryPublicationPdfEnvelope(BaseModel):
|
||||||
meta: ApiMeta
|
meta: ApiMeta
|
||||||
|
|
||||||
model_config = ConfigDict(extra="forbid")
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class TogglePublicationFavoriteRequest(BaseModel):
|
||||||
|
scholar_profile_id: int = Field(ge=1)
|
||||||
|
is_favorite: bool
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class TogglePublicationFavoriteData(BaseModel):
|
||||||
|
message: str
|
||||||
|
publication: PublicationItemData
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
|
||||||
|
class TogglePublicationFavoriteEnvelope(BaseModel):
|
||||||
|
data: TogglePublicationFavoriteData
|
||||||
|
meta: ApiMeta
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
|
||||||
122
app/db/models.py
|
|
@ -81,8 +81,8 @@ class UserSetting(Base):
|
||||||
name="run_interval_minutes_min_15",
|
name="run_interval_minutes_min_15",
|
||||||
),
|
),
|
||||||
CheckConstraint(
|
CheckConstraint(
|
||||||
"request_delay_seconds >= 1",
|
"request_delay_seconds >= 2",
|
||||||
name="request_delay_seconds_min_1",
|
name="request_delay_seconds_min_2",
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -237,10 +237,80 @@ class Publication(Base):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PublicationPdfJob(Base):
|
||||||
|
__tablename__ = "publication_pdf_jobs"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint(
|
||||||
|
"status IN ('queued', 'running', 'resolved', 'failed')",
|
||||||
|
name="publication_pdf_jobs_status_valid",
|
||||||
|
),
|
||||||
|
Index("ix_publication_pdf_jobs_status", "status"),
|
||||||
|
Index("ix_publication_pdf_jobs_updated_at", "updated_at"),
|
||||||
|
Index("ix_publication_pdf_jobs_queued_at", "queued_at"),
|
||||||
|
)
|
||||||
|
|
||||||
|
publication_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("publications.id", ondelete="CASCADE"),
|
||||||
|
primary_key=True,
|
||||||
|
)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(16),
|
||||||
|
nullable=False,
|
||||||
|
server_default=text("'queued'"),
|
||||||
|
)
|
||||||
|
attempt_count: Mapped[int] = mapped_column(
|
||||||
|
Integer,
|
||||||
|
nullable=False,
|
||||||
|
server_default=text("0"),
|
||||||
|
)
|
||||||
|
queued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
last_failure_reason: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
last_failure_detail: Mapped[str | None] = mapped_column(Text)
|
||||||
|
last_source: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
last_requested_by_user_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("users.id", ondelete="SET NULL"),
|
||||||
|
)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PublicationPdfJobEvent(Base):
|
||||||
|
__tablename__ = "publication_pdf_job_events"
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_publication_pdf_job_events_publication_created", "publication_id", "created_at"),
|
||||||
|
Index("ix_publication_pdf_job_events_created_at", "created_at"),
|
||||||
|
Index("ix_publication_pdf_job_events_event_type", "event_type"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
publication_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("publications.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
user_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("users.id", ondelete="SET NULL"),
|
||||||
|
)
|
||||||
|
event_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
status: Mapped[str | None] = mapped_column(String(16))
|
||||||
|
source: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
failure_reason: Mapped[str | None] = mapped_column(String(64))
|
||||||
|
message: Mapped[str | None] = mapped_column(Text)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ScholarPublication(Base):
|
class ScholarPublication(Base):
|
||||||
__tablename__ = "scholar_publications"
|
__tablename__ = "scholar_publications"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
Index("ix_scholar_publications_is_read", "is_read"),
|
Index("ix_scholar_publications_is_read", "is_read"),
|
||||||
|
Index("ix_scholar_publications_is_favorite", "is_favorite"),
|
||||||
)
|
)
|
||||||
|
|
||||||
scholar_profile_id: Mapped[int] = mapped_column(
|
scholar_profile_id: Mapped[int] = mapped_column(
|
||||||
|
|
@ -254,6 +324,9 @@ class ScholarPublication(Base):
|
||||||
is_read: Mapped[bool] = mapped_column(
|
is_read: Mapped[bool] = mapped_column(
|
||||||
Boolean, nullable=False, server_default=text("false")
|
Boolean, nullable=False, server_default=text("false")
|
||||||
)
|
)
|
||||||
|
is_favorite: Mapped[bool] = mapped_column(
|
||||||
|
Boolean, nullable=False, server_default=text("false")
|
||||||
|
)
|
||||||
first_seen_run_id: Mapped[int | None] = mapped_column(
|
first_seen_run_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("crawl_runs.id", ondelete="SET NULL")
|
ForeignKey("crawl_runs.id", ondelete="SET NULL")
|
||||||
)
|
)
|
||||||
|
|
@ -322,6 +395,51 @@ class IngestionQueueItem(Base):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class DataRepairJob(Base):
|
||||||
|
__tablename__ = "data_repair_jobs"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint(
|
||||||
|
"status IN ('planned', 'running', 'completed', 'failed')",
|
||||||
|
name="data_repair_jobs_status_valid",
|
||||||
|
),
|
||||||
|
Index("ix_data_repair_jobs_created_at", "created_at"),
|
||||||
|
Index("ix_data_repair_jobs_job_name_created_at", "job_name", "created_at"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
job_name: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
requested_by: Mapped[str | None] = mapped_column(String(255))
|
||||||
|
scope: Mapped[dict] = mapped_column(
|
||||||
|
JSONB,
|
||||||
|
nullable=False,
|
||||||
|
server_default=text("'{}'::jsonb"),
|
||||||
|
)
|
||||||
|
dry_run: Mapped[bool] = mapped_column(
|
||||||
|
Boolean,
|
||||||
|
nullable=False,
|
||||||
|
server_default=text("true"),
|
||||||
|
)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(16),
|
||||||
|
nullable=False,
|
||||||
|
server_default=text("'planned'"),
|
||||||
|
)
|
||||||
|
summary: Mapped[dict] = mapped_column(
|
||||||
|
JSONB,
|
||||||
|
nullable=False,
|
||||||
|
server_default=text("'{}'::jsonb"),
|
||||||
|
)
|
||||||
|
error_text: Mapped[str | None] = mapped_column(Text)
|
||||||
|
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class AuthorSearchRuntimeState(Base):
|
class AuthorSearchRuntimeState(Base):
|
||||||
__tablename__ = "author_search_runtime_state"
|
__tablename__ = "author_search_runtime_state"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -61,7 +61,7 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
|
||||||
start = time.perf_counter()
|
start = time.perf_counter()
|
||||||
should_log = self._log_requests and not self._is_skipped_path(request.url.path)
|
should_log = self._log_requests and not self._is_skipped_path(request.url.path)
|
||||||
if should_log:
|
if should_log:
|
||||||
logger.info(
|
logger.debug(
|
||||||
"request.started",
|
"request.started",
|
||||||
extra={
|
extra={
|
||||||
"event": "request.started",
|
"event": "request.started",
|
||||||
|
|
@ -88,7 +88,7 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
|
||||||
duration_ms = int((time.perf_counter() - start) * 1000)
|
duration_ms = int((time.perf_counter() - start) * 1000)
|
||||||
response.headers[REQUEST_ID_HEADER] = request_id
|
response.headers[REQUEST_ID_HEADER] = request_id
|
||||||
if should_log:
|
if should_log:
|
||||||
logger.info(
|
logger.debug(
|
||||||
"request.completed",
|
"request.completed",
|
||||||
extra={
|
extra={
|
||||||
"event": "request.completed",
|
"event": "request.completed",
|
||||||
|
|
|
||||||
5
app/services/domains/dbops/__init__.py
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
from app.services.domains.dbops.application import run_publication_link_repair
|
||||||
|
from app.services.domains.dbops.integrity import collect_integrity_report
|
||||||
|
from app.services.domains.dbops.query import list_repair_jobs
|
||||||
|
|
||||||
|
__all__ = ["collect_integrity_report", "list_repair_jobs", "run_publication_link_repair"]
|
||||||
364
app/services/domains/dbops/application.py
Normal file
|
|
@ -0,0 +1,364 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import delete, exists, func, select, update
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.db.models import DataRepairJob, IngestionQueueItem, Publication, ScholarProfile, ScholarPublication
|
||||||
|
|
||||||
|
|
||||||
|
REPAIR_STATUS_PLANNED = "planned"
|
||||||
|
REPAIR_STATUS_RUNNING = "running"
|
||||||
|
REPAIR_STATUS_COMPLETED = "completed"
|
||||||
|
REPAIR_STATUS_FAILED = "failed"
|
||||||
|
SCOPE_MODE_SINGLE_USER = "single_user"
|
||||||
|
SCOPE_MODE_ALL_USERS = "all_users"
|
||||||
|
|
||||||
|
|
||||||
|
def _utcnow() -> datetime:
|
||||||
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_scope_mode(scope_mode: str) -> str:
|
||||||
|
normalized = scope_mode.strip().lower()
|
||||||
|
if normalized in {SCOPE_MODE_SINGLE_USER, SCOPE_MODE_ALL_USERS}:
|
||||||
|
return normalized
|
||||||
|
raise ValueError("Unknown scope mode.")
|
||||||
|
|
||||||
|
|
||||||
|
def _scope_user_id(*, scope_mode: str, user_id: int | None) -> int | None:
|
||||||
|
if scope_mode == SCOPE_MODE_SINGLE_USER:
|
||||||
|
if user_id is None:
|
||||||
|
raise ValueError("user_id is required when scope_mode=single_user.")
|
||||||
|
return int(user_id)
|
||||||
|
if user_id is not None:
|
||||||
|
raise ValueError("user_id must be omitted when scope_mode=all_users.")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _scope_payload(
|
||||||
|
*,
|
||||||
|
scope_mode: str,
|
||||||
|
user_id: int | None,
|
||||||
|
target_scholar_profile_ids: list[int],
|
||||||
|
orphan_gc: bool,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
payload: dict[str, Any] = {
|
||||||
|
"scope_mode": scope_mode,
|
||||||
|
"scholar_profile_ids": [int(value) for value in target_scholar_profile_ids],
|
||||||
|
"gc_orphan_publications": bool(orphan_gc),
|
||||||
|
}
|
||||||
|
if user_id is not None:
|
||||||
|
payload["user_id"] = int(user_id)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
async def _target_scholar_profile_ids(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
scope_mode: str,
|
||||||
|
user_id: int | None,
|
||||||
|
scholar_profile_ids: list[int] | None,
|
||||||
|
) -> list[int]:
|
||||||
|
stmt = select(ScholarProfile.id)
|
||||||
|
if scope_mode == SCOPE_MODE_SINGLE_USER:
|
||||||
|
stmt = stmt.where(ScholarProfile.user_id == user_id)
|
||||||
|
if scholar_profile_ids:
|
||||||
|
normalized_ids = [int(value) for value in scholar_profile_ids]
|
||||||
|
stmt = stmt.where(ScholarProfile.id.in_(normalized_ids))
|
||||||
|
result = await db_session.execute(stmt.order_by(ScholarProfile.id.asc()))
|
||||||
|
ids = [int(row[0]) for row in result.all()]
|
||||||
|
if not ids:
|
||||||
|
raise ValueError("No target scholar profiles found for the requested scope.")
|
||||||
|
return ids
|
||||||
|
|
||||||
|
|
||||||
|
async def _count_scope(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int | None,
|
||||||
|
target_scholar_profile_ids: list[int],
|
||||||
|
) -> dict[str, int]:
|
||||||
|
links_result = await db_session.execute(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(ScholarPublication)
|
||||||
|
.where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids))
|
||||||
|
)
|
||||||
|
queue_stmt = (
|
||||||
|
select(func.count())
|
||||||
|
.select_from(IngestionQueueItem)
|
||||||
|
.where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids))
|
||||||
|
)
|
||||||
|
if user_id is not None:
|
||||||
|
queue_stmt = queue_stmt.where(IngestionQueueItem.user_id == user_id)
|
||||||
|
queue_result = await db_session.execute(queue_stmt)
|
||||||
|
return {
|
||||||
|
"target_scholar_count": len(target_scholar_profile_ids),
|
||||||
|
"links_in_scope": int(links_result.scalar_one() or 0),
|
||||||
|
"queue_items_in_scope": int(queue_result.scalar_one() or 0),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _count_orphan_publications(db_session: AsyncSession) -> int:
|
||||||
|
stmt = (
|
||||||
|
select(func.count())
|
||||||
|
.select_from(Publication)
|
||||||
|
.where(
|
||||||
|
~exists(
|
||||||
|
select(1).where(ScholarPublication.publication_id == Publication.id)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
result = await db_session.execute(stmt)
|
||||||
|
return int(result.scalar_one() or 0)
|
||||||
|
|
||||||
|
|
||||||
|
async def _create_job(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
requested_by: str | None,
|
||||||
|
scope: dict[str, Any],
|
||||||
|
dry_run: bool,
|
||||||
|
) -> DataRepairJob:
|
||||||
|
job = DataRepairJob(
|
||||||
|
job_name="repair_publication_links",
|
||||||
|
requested_by=(requested_by or "").strip() or None,
|
||||||
|
scope=scope,
|
||||||
|
dry_run=dry_run,
|
||||||
|
status=REPAIR_STATUS_PLANNED,
|
||||||
|
summary={},
|
||||||
|
)
|
||||||
|
db_session.add(job)
|
||||||
|
await db_session.flush()
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def _job_summary(
|
||||||
|
*,
|
||||||
|
counts: dict[str, int],
|
||||||
|
dry_run: bool,
|
||||||
|
links_deleted: int,
|
||||||
|
queue_items_deleted: int,
|
||||||
|
scholars_reset: int,
|
||||||
|
orphan_publications_before: int,
|
||||||
|
orphan_publications_deleted: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
**counts,
|
||||||
|
"dry_run": bool(dry_run),
|
||||||
|
"links_deleted": int(links_deleted),
|
||||||
|
"queue_items_deleted": int(queue_items_deleted),
|
||||||
|
"scholars_reset": int(scholars_reset),
|
||||||
|
"orphan_publications_before": int(orphan_publications_before),
|
||||||
|
"orphan_publications_deleted": int(orphan_publications_deleted),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _delete_links_for_targets(db_session: AsyncSession, *, target_scholar_profile_ids: list[int]) -> int:
|
||||||
|
result = await db_session.execute(
|
||||||
|
delete(ScholarPublication).where(
|
||||||
|
ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return int(result.rowcount or 0)
|
||||||
|
|
||||||
|
|
||||||
|
async def _delete_queue_for_targets(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int | None,
|
||||||
|
target_scholar_profile_ids: list[int],
|
||||||
|
) -> int:
|
||||||
|
stmt = delete(IngestionQueueItem).where(
|
||||||
|
IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids)
|
||||||
|
)
|
||||||
|
if user_id is not None:
|
||||||
|
stmt = stmt.where(IngestionQueueItem.user_id == user_id)
|
||||||
|
result = await db_session.execute(stmt)
|
||||||
|
return int(result.rowcount or 0)
|
||||||
|
|
||||||
|
|
||||||
|
async def _reset_scholar_tracking_state(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int | None,
|
||||||
|
target_scholar_profile_ids: list[int],
|
||||||
|
) -> int:
|
||||||
|
stmt = update(ScholarProfile).where(ScholarProfile.id.in_(target_scholar_profile_ids))
|
||||||
|
if user_id is not None:
|
||||||
|
stmt = stmt.where(ScholarProfile.user_id == user_id)
|
||||||
|
result = await db_session.execute(
|
||||||
|
stmt.values(
|
||||||
|
baseline_completed=False,
|
||||||
|
last_initial_page_fingerprint_sha256=None,
|
||||||
|
last_initial_page_checked_at=None,
|
||||||
|
last_run_dt=None,
|
||||||
|
last_run_status=None,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return int(result.rowcount or 0)
|
||||||
|
|
||||||
|
|
||||||
|
async def _delete_orphan_publications(db_session: AsyncSession) -> int:
|
||||||
|
result = await db_session.execute(
|
||||||
|
delete(Publication).where(
|
||||||
|
~exists(select(1).where(ScholarPublication.publication_id == Publication.id))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return int(result.rowcount or 0)
|
||||||
|
|
||||||
|
|
||||||
|
async def _mutation_counts(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int | None,
|
||||||
|
target_ids: list[int],
|
||||||
|
dry_run: bool,
|
||||||
|
gc_orphan_publications: bool,
|
||||||
|
) -> tuple[int, int, int, int]:
|
||||||
|
if dry_run:
|
||||||
|
return 0, 0, 0, 0
|
||||||
|
links_deleted = await _delete_links_for_targets(
|
||||||
|
db_session,
|
||||||
|
target_scholar_profile_ids=target_ids,
|
||||||
|
)
|
||||||
|
queue_deleted = await _delete_queue_for_targets(
|
||||||
|
db_session,
|
||||||
|
user_id=user_id,
|
||||||
|
target_scholar_profile_ids=target_ids,
|
||||||
|
)
|
||||||
|
scholars_reset = await _reset_scholar_tracking_state(
|
||||||
|
db_session,
|
||||||
|
user_id=user_id,
|
||||||
|
target_scholar_profile_ids=target_ids,
|
||||||
|
)
|
||||||
|
orphan_deleted = 0
|
||||||
|
if gc_orphan_publications:
|
||||||
|
orphan_deleted = await _delete_orphan_publications(db_session)
|
||||||
|
return links_deleted, queue_deleted, scholars_reset, orphan_deleted
|
||||||
|
|
||||||
|
|
||||||
|
def _result_payload(*, job: DataRepairJob, scope: dict[str, Any], summary: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"job_id": int(job.id),
|
||||||
|
"status": job.status,
|
||||||
|
"scope": scope,
|
||||||
|
"summary": summary,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _complete_job(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
job: DataRepairJob,
|
||||||
|
summary: dict[str, Any],
|
||||||
|
scope: dict[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
job.summary = summary
|
||||||
|
job.status = REPAIR_STATUS_COMPLETED
|
||||||
|
job.finished_at = _utcnow()
|
||||||
|
await db_session.commit()
|
||||||
|
return _result_payload(job=job, scope=scope, summary=summary)
|
||||||
|
|
||||||
|
|
||||||
|
async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None:
|
||||||
|
await db_session.rollback()
|
||||||
|
job.status = REPAIR_STATUS_FAILED
|
||||||
|
job.error_text = str(error)
|
||||||
|
job.finished_at = _utcnow()
|
||||||
|
db_session.add(job)
|
||||||
|
await db_session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
async def _prepare_repair_job(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
scope_mode: str,
|
||||||
|
user_id: int | None,
|
||||||
|
scholar_profile_ids: list[int] | None,
|
||||||
|
dry_run: bool,
|
||||||
|
gc_orphan_publications: bool,
|
||||||
|
requested_by: str | None,
|
||||||
|
) -> tuple[int | None, list[int], dict[str, Any], DataRepairJob]:
|
||||||
|
normalized_scope = _normalize_scope_mode(scope_mode)
|
||||||
|
scope_user_id = _scope_user_id(scope_mode=normalized_scope, user_id=user_id)
|
||||||
|
target_ids = await _target_scholar_profile_ids(
|
||||||
|
db_session,
|
||||||
|
scope_mode=normalized_scope,
|
||||||
|
user_id=scope_user_id,
|
||||||
|
scholar_profile_ids=scholar_profile_ids,
|
||||||
|
)
|
||||||
|
scope = _scope_payload(
|
||||||
|
scope_mode=normalized_scope,
|
||||||
|
user_id=scope_user_id,
|
||||||
|
target_scholar_profile_ids=target_ids,
|
||||||
|
orphan_gc=gc_orphan_publications,
|
||||||
|
)
|
||||||
|
job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run)
|
||||||
|
job.status = REPAIR_STATUS_RUNNING
|
||||||
|
job.started_at = _utcnow()
|
||||||
|
return scope_user_id, target_ids, scope, job
|
||||||
|
|
||||||
|
|
||||||
|
async def _build_repair_summary(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
scope_user_id: int | None,
|
||||||
|
target_ids: list[int],
|
||||||
|
dry_run: bool,
|
||||||
|
gc_orphan_publications: bool,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
counts = await _count_scope(db_session, user_id=scope_user_id, target_scholar_profile_ids=target_ids)
|
||||||
|
orphan_before = await _count_orphan_publications(db_session)
|
||||||
|
links_deleted, queue_deleted, scholars_reset, orphan_deleted = await _mutation_counts(
|
||||||
|
db_session,
|
||||||
|
user_id=scope_user_id,
|
||||||
|
target_ids=target_ids,
|
||||||
|
dry_run=dry_run,
|
||||||
|
gc_orphan_publications=gc_orphan_publications,
|
||||||
|
)
|
||||||
|
return _job_summary(
|
||||||
|
counts=counts,
|
||||||
|
dry_run=dry_run,
|
||||||
|
links_deleted=links_deleted,
|
||||||
|
queue_items_deleted=queue_deleted,
|
||||||
|
scholars_reset=scholars_reset,
|
||||||
|
orphan_publications_before=orphan_before,
|
||||||
|
orphan_publications_deleted=orphan_deleted,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def run_publication_link_repair(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
scope_mode: str = SCOPE_MODE_SINGLE_USER,
|
||||||
|
user_id: int | None = None,
|
||||||
|
scholar_profile_ids: list[int] | None = None,
|
||||||
|
dry_run: bool = True,
|
||||||
|
gc_orphan_publications: bool = False,
|
||||||
|
requested_by: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
scope_user_id, target_ids, scope, job = await _prepare_repair_job(
|
||||||
|
db_session,
|
||||||
|
scope_mode=scope_mode,
|
||||||
|
user_id=user_id,
|
||||||
|
scholar_profile_ids=scholar_profile_ids,
|
||||||
|
dry_run=dry_run,
|
||||||
|
gc_orphan_publications=gc_orphan_publications,
|
||||||
|
requested_by=requested_by,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
summary = await _build_repair_summary(
|
||||||
|
db_session,
|
||||||
|
scope_user_id=scope_user_id,
|
||||||
|
target_ids=target_ids,
|
||||||
|
dry_run=dry_run,
|
||||||
|
gc_orphan_publications=gc_orphan_publications,
|
||||||
|
)
|
||||||
|
return await _complete_job(db_session, job=job, summary=summary, scope=scope)
|
||||||
|
except Exception as exc:
|
||||||
|
await _fail_job(db_session, job=job, error=exc)
|
||||||
|
raise
|
||||||
175
app/services/domains/dbops/integrity.py
Normal file
|
|
@ -0,0 +1,175 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import func, select, text
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.db.models import Publication
|
||||||
|
|
||||||
|
INTEGRITY_CHECK_DEFS = (
|
||||||
|
(
|
||||||
|
"legacy_cluster_id_format",
|
||||||
|
"warning",
|
||||||
|
"Publications with non-namespaced cluster IDs.",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"negative_citation_count",
|
||||||
|
"failure",
|
||||||
|
"Publications with negative citation counts.",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"orphan_publications",
|
||||||
|
"warning",
|
||||||
|
"Publications with no scholar links.",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"orphan_scholar_publication_links",
|
||||||
|
"failure",
|
||||||
|
"Link rows missing parent scholar/publication.",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"duplicate_fingerprint_keys",
|
||||||
|
"failure",
|
||||||
|
"Duplicate publication fingerprint keys.",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"duplicate_cluster_ids",
|
||||||
|
"failure",
|
||||||
|
"Duplicate non-null publication cluster IDs.",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"missing_pdf_url",
|
||||||
|
"metric",
|
||||||
|
"Publications without a resolved PDF URL.",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _legacy_cluster_id_count(db_session: AsyncSession) -> int:
|
||||||
|
stmt = (
|
||||||
|
select(func.count())
|
||||||
|
.select_from(Publication)
|
||||||
|
.where(Publication.cluster_id.is_not(None))
|
||||||
|
.where(~Publication.cluster_id.like("cfv:%"))
|
||||||
|
.where(~Publication.cluster_id.like("cluster:%"))
|
||||||
|
)
|
||||||
|
result = await db_session.execute(stmt)
|
||||||
|
return int(result.scalar_one() or 0)
|
||||||
|
|
||||||
|
|
||||||
|
async def _negative_citation_count(db_session: AsyncSession) -> int:
|
||||||
|
result = await db_session.execute(
|
||||||
|
select(func.count()).select_from(Publication).where(Publication.citation_count < 0)
|
||||||
|
)
|
||||||
|
return int(result.scalar_one() or 0)
|
||||||
|
|
||||||
|
|
||||||
|
async def _missing_pdf_url_count(db_session: AsyncSession) -> int:
|
||||||
|
result = await db_session.execute(
|
||||||
|
select(func.count()).select_from(Publication).where(Publication.pdf_url.is_(None))
|
||||||
|
)
|
||||||
|
return int(result.scalar_one() or 0)
|
||||||
|
|
||||||
|
|
||||||
|
async def _count_from_sql(db_session: AsyncSession, *, sql: str) -> int:
|
||||||
|
result = await db_session.execute(text(sql))
|
||||||
|
return int(result.scalar_one() or 0)
|
||||||
|
|
||||||
|
|
||||||
|
def _issues_for_severity(*, checks: list[dict[str, Any]], severity: str) -> list[str]:
|
||||||
|
return [row["name"] for row in checks if row["severity"] == severity and row["count"] > 0]
|
||||||
|
|
||||||
|
|
||||||
|
def _check_row(*, name: str, count: int, severity: str, message: str) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"name": name,
|
||||||
|
"count": int(count),
|
||||||
|
"severity": severity,
|
||||||
|
"message": message,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _collect_counts(db_session: AsyncSession) -> dict[str, int]:
|
||||||
|
orphan_publications = await _count_from_sql(
|
||||||
|
db_session,
|
||||||
|
sql=(
|
||||||
|
"SELECT count(*) FROM publications p "
|
||||||
|
"LEFT JOIN scholar_publications sp ON sp.publication_id = p.id "
|
||||||
|
"WHERE sp.publication_id IS NULL"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
orphan_links = await _count_from_sql(
|
||||||
|
db_session,
|
||||||
|
sql=(
|
||||||
|
"SELECT count(*) FROM scholar_publications sp "
|
||||||
|
"LEFT JOIN publications p ON p.id = sp.publication_id "
|
||||||
|
"LEFT JOIN scholar_profiles s ON s.id = sp.scholar_profile_id "
|
||||||
|
"WHERE p.id IS NULL OR s.id IS NULL"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
duplicate_fingerprints = await _count_from_sql(
|
||||||
|
db_session,
|
||||||
|
sql=(
|
||||||
|
"SELECT count(*) FROM ("
|
||||||
|
"SELECT fingerprint_sha256 FROM publications "
|
||||||
|
"GROUP BY fingerprint_sha256 HAVING count(*) > 1"
|
||||||
|
") dup"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
duplicate_cluster_ids = await _count_from_sql(
|
||||||
|
db_session,
|
||||||
|
sql=(
|
||||||
|
"SELECT count(*) FROM ("
|
||||||
|
"SELECT cluster_id FROM publications "
|
||||||
|
"WHERE cluster_id IS NOT NULL "
|
||||||
|
"GROUP BY cluster_id HAVING count(*) > 1"
|
||||||
|
") dup"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"legacy_cluster_id_format": await _legacy_cluster_id_count(db_session),
|
||||||
|
"negative_citation_count": await _negative_citation_count(db_session),
|
||||||
|
"orphan_publications": orphan_publications,
|
||||||
|
"orphan_scholar_publication_links": orphan_links,
|
||||||
|
"duplicate_fingerprint_keys": duplicate_fingerprints,
|
||||||
|
"duplicate_cluster_ids": duplicate_cluster_ids,
|
||||||
|
"missing_pdf_url": await _missing_pdf_url_count(db_session),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _build_checks(*, counts: dict[str, int]) -> list[dict[str, Any]]:
|
||||||
|
return [
|
||||||
|
_check_row(
|
||||||
|
name=name,
|
||||||
|
count=counts[name],
|
||||||
|
severity=severity,
|
||||||
|
message=message,
|
||||||
|
)
|
||||||
|
for name, severity, message in INTEGRITY_CHECK_DEFS
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _status_from_issues(*, failures: list[str], warnings: list[str]) -> str:
|
||||||
|
status = "ok"
|
||||||
|
if failures:
|
||||||
|
status = "failed"
|
||||||
|
elif warnings:
|
||||||
|
status = "warning"
|
||||||
|
return status
|
||||||
|
|
||||||
|
|
||||||
|
async def collect_integrity_report(db_session: AsyncSession) -> dict[str, Any]:
|
||||||
|
counts = await _collect_counts(db_session)
|
||||||
|
checks = _build_checks(counts=counts)
|
||||||
|
failures = _issues_for_severity(checks=checks, severity="failure")
|
||||||
|
warnings = _issues_for_severity(checks=checks, severity="warning")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"status": _status_from_issues(failures=failures, warnings=warnings),
|
||||||
|
"checked_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"failures": failures,
|
||||||
|
"warnings": warnings,
|
||||||
|
"checks": checks,
|
||||||
|
}
|
||||||
22
app/services/domains/dbops/query.py
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.db.models import DataRepairJob
|
||||||
|
|
||||||
|
|
||||||
|
def _bounded_limit(limit: int) -> int:
|
||||||
|
return max(1, min(int(limit), 200))
|
||||||
|
|
||||||
|
|
||||||
|
async def list_repair_jobs(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
limit: int = 50,
|
||||||
|
) -> list[DataRepairJob]:
|
||||||
|
bounded = _bounded_limit(limit)
|
||||||
|
result = await db_session.execute(
|
||||||
|
select(DataRepairJob).order_by(DataRepairJob.created_at.desc()).limit(bounded)
|
||||||
|
)
|
||||||
|
return list(result.scalars())
|
||||||
|
|
@ -91,6 +91,35 @@ class ScholarIngestionService:
|
||||||
def __init__(self, *, source: ScholarSource) -> None:
|
def __init__(self, *, source: ScholarSource) -> None:
|
||||||
self._source = source
|
self._source = source
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _effective_request_delay_seconds(value: int) -> int:
|
||||||
|
policy_minimum = user_settings_service.resolve_request_delay_minimum(
|
||||||
|
settings.ingestion_min_request_delay_seconds
|
||||||
|
)
|
||||||
|
return max(policy_minimum, _int_or_default(value, policy_minimum))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _log_request_delay_coercion(
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
requested_request_delay_seconds: int,
|
||||||
|
effective_request_delay_seconds: int,
|
||||||
|
) -> None:
|
||||||
|
logger.warning(
|
||||||
|
"ingestion.request_delay_coerced_to_policy_floor",
|
||||||
|
extra={
|
||||||
|
"event": "ingestion.request_delay_coerced_to_policy_floor",
|
||||||
|
"user_id": user_id,
|
||||||
|
"requested_request_delay_seconds": requested_request_delay_seconds,
|
||||||
|
"effective_request_delay_seconds": effective_request_delay_seconds,
|
||||||
|
"policy_minimum_request_delay_seconds": user_settings_service.resolve_request_delay_minimum(
|
||||||
|
settings.ingestion_min_request_delay_seconds
|
||||||
|
),
|
||||||
|
"metric_name": "ingestion_request_delay_coerced_total",
|
||||||
|
"metric_value": 1,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
async def _load_user_settings_for_run(
|
async def _load_user_settings_for_run(
|
||||||
self,
|
self,
|
||||||
db_session: AsyncSession,
|
db_session: AsyncSession,
|
||||||
|
|
@ -1278,7 +1307,14 @@ class ScholarIngestionService:
|
||||||
return failure_summary, alert_summary
|
return failure_summary, alert_summary
|
||||||
|
|
||||||
async def run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, request_delay_seconds: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, max_pages_per_scholar: int = 30, page_size: int = 100, scholar_profile_ids: set[int] | None = None, start_cstart_by_scholar_id: dict[int, int] | None = None, auto_queue_continuations: bool = True, queue_delay_seconds: int = 60, idempotency_key: str | None = None, alert_blocked_failure_threshold: int = 1, alert_network_failure_threshold: int = 2, alert_retry_scheduled_threshold: int = 3) -> RunExecutionSummary:
|
async def run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, request_delay_seconds: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, max_pages_per_scholar: int = 30, page_size: int = 100, scholar_profile_ids: set[int] | None = None, start_cstart_by_scholar_id: dict[int, int] | None = None, auto_queue_continuations: bool = True, queue_delay_seconds: int = 60, idempotency_key: str | None = None, alert_blocked_failure_threshold: int = 1, alert_network_failure_threshold: int = 2, alert_retry_scheduled_threshold: int = 3) -> RunExecutionSummary:
|
||||||
paging_kwargs = self._paging_kwargs(request_delay_seconds=request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size)
|
effective_request_delay_seconds = self._effective_request_delay_seconds(request_delay_seconds)
|
||||||
|
if effective_request_delay_seconds != _int_or_default(request_delay_seconds, effective_request_delay_seconds):
|
||||||
|
self._log_request_delay_coercion(
|
||||||
|
user_id=user_id,
|
||||||
|
requested_request_delay_seconds=_int_or_default(request_delay_seconds, 0),
|
||||||
|
effective_request_delay_seconds=effective_request_delay_seconds,
|
||||||
|
)
|
||||||
|
paging_kwargs = self._paging_kwargs(request_delay_seconds=effective_request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size)
|
||||||
threshold_kwargs = self._threshold_kwargs(alert_blocked_failure_threshold=alert_blocked_failure_threshold, alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold)
|
threshold_kwargs = self._threshold_kwargs(alert_blocked_failure_threshold=alert_blocked_failure_threshold, alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold)
|
||||||
user_settings, run, scholars, start_cstart_map = await self._initialize_run_for_user(
|
user_settings, run, scholars, start_cstart_map = await self._initialize_run_for_user(
|
||||||
db_session,
|
db_session,
|
||||||
|
|
|
||||||
|
|
@ -23,12 +23,28 @@ from app.services.domains.ingestion.application import (
|
||||||
RunBlockedBySafetyPolicyError,
|
RunBlockedBySafetyPolicyError,
|
||||||
ScholarIngestionService,
|
ScholarIngestionService,
|
||||||
)
|
)
|
||||||
|
from app.services.domains.settings import application as user_settings_service
|
||||||
from app.services.domains.scholar.source import LiveScholarSource
|
from app.services.domains.scholar.source import LiveScholarSource
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _request_delay_floor_seconds() -> int:
|
||||||
|
return user_settings_service.resolve_request_delay_minimum(
|
||||||
|
settings.ingestion_min_request_delay_seconds
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _effective_request_delay_seconds(value: int | None) -> int:
|
||||||
|
floor = _request_delay_floor_seconds()
|
||||||
|
try:
|
||||||
|
parsed = int(value) if value is not None else floor
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
parsed = floor
|
||||||
|
return max(floor, parsed)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class _AutoRunCandidate:
|
class _AutoRunCandidate:
|
||||||
user_id: int
|
user_id: int
|
||||||
|
|
@ -178,7 +194,7 @@ class SchedulerService:
|
||||||
return _AutoRunCandidate(
|
return _AutoRunCandidate(
|
||||||
user_id=int(user_id),
|
user_id=int(user_id),
|
||||||
run_interval_minutes=int(run_interval_minutes),
|
run_interval_minutes=int(run_interval_minutes),
|
||||||
request_delay_seconds=int(request_delay_seconds),
|
request_delay_seconds=_effective_request_delay_seconds(request_delay_seconds),
|
||||||
cooldown_until=cooldown_until,
|
cooldown_until=cooldown_until,
|
||||||
cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
|
cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
|
||||||
)
|
)
|
||||||
|
|
@ -577,6 +593,4 @@ class SchedulerService:
|
||||||
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
|
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
|
||||||
)
|
)
|
||||||
delay = result.scalar_one_or_none()
|
delay = result.scalar_one_or_none()
|
||||||
if delay is None:
|
return _effective_request_delay_seconds(delay)
|
||||||
return 10
|
|
||||||
return max(1, int(delay))
|
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
from app.services.domains.publications.counts import (
|
from app.services.domains.publications.counts import (
|
||||||
count_for_user,
|
count_for_user,
|
||||||
|
count_favorite_for_user,
|
||||||
count_latest_for_user,
|
count_latest_for_user,
|
||||||
count_unread_for_user,
|
count_unread_for_user,
|
||||||
)
|
)
|
||||||
|
|
@ -12,7 +13,9 @@ from app.services.domains.publications.listing import (
|
||||||
list_unread_for_user,
|
list_unread_for_user,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.enrichment import (
|
from app.services.domains.publications.enrichment import (
|
||||||
|
hydrate_pdf_enrichment_state,
|
||||||
schedule_missing_pdf_enrichment_for_user,
|
schedule_missing_pdf_enrichment_for_user,
|
||||||
|
schedule_retry_pdf_enrichment_for_row,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.modes import (
|
from app.services.domains.publications.modes import (
|
||||||
MODE_ALL,
|
MODE_ALL,
|
||||||
|
|
@ -30,6 +33,14 @@ from app.services.domains.publications.queries import (
|
||||||
from app.services.domains.publications.read_state import (
|
from app.services.domains.publications.read_state import (
|
||||||
mark_all_unread_as_read_for_user,
|
mark_all_unread_as_read_for_user,
|
||||||
mark_selected_as_read_for_user,
|
mark_selected_as_read_for_user,
|
||||||
|
set_publication_favorite_for_user,
|
||||||
|
)
|
||||||
|
from app.services.domains.publications.pdf_queue import (
|
||||||
|
count_pdf_queue_items,
|
||||||
|
enqueue_all_missing_pdf_jobs,
|
||||||
|
enqueue_retry_pdf_job_for_publication_id,
|
||||||
|
list_pdf_queue_page,
|
||||||
|
list_pdf_queue_items,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||||
|
|
||||||
|
|
@ -49,10 +60,19 @@ __all__ = [
|
||||||
"list_unread_for_user",
|
"list_unread_for_user",
|
||||||
"list_new_for_latest_run_for_user",
|
"list_new_for_latest_run_for_user",
|
||||||
"retry_pdf_for_user",
|
"retry_pdf_for_user",
|
||||||
|
"hydrate_pdf_enrichment_state",
|
||||||
|
"schedule_retry_pdf_enrichment_for_row",
|
||||||
|
"list_pdf_queue_items",
|
||||||
|
"list_pdf_queue_page",
|
||||||
|
"count_pdf_queue_items",
|
||||||
|
"enqueue_all_missing_pdf_jobs",
|
||||||
|
"enqueue_retry_pdf_job_for_publication_id",
|
||||||
"schedule_missing_pdf_enrichment_for_user",
|
"schedule_missing_pdf_enrichment_for_user",
|
||||||
"count_for_user",
|
"count_for_user",
|
||||||
|
"count_favorite_for_user",
|
||||||
"count_unread_for_user",
|
"count_unread_for_user",
|
||||||
"count_latest_for_user",
|
"count_latest_for_user",
|
||||||
"mark_all_unread_as_read_for_user",
|
"mark_all_unread_as_read_for_user",
|
||||||
"mark_selected_as_read_for_user",
|
"mark_selected_as_read_for_user",
|
||||||
|
"set_publication_favorite_for_user",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,7 @@ async def count_for_user(
|
||||||
user_id: int,
|
user_id: int,
|
||||||
mode: str = MODE_ALL,
|
mode: str = MODE_ALL,
|
||||||
scholar_profile_id: int | None = None,
|
scholar_profile_id: int | None = None,
|
||||||
|
favorite_only: bool = False,
|
||||||
) -> int:
|
) -> int:
|
||||||
resolved_mode = resolve_publication_view_mode(mode)
|
resolved_mode = resolve_publication_view_mode(mode)
|
||||||
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
|
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
|
||||||
|
|
@ -30,6 +31,8 @@ async def count_for_user(
|
||||||
)
|
)
|
||||||
if scholar_profile_id is not None:
|
if scholar_profile_id is not None:
|
||||||
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
|
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
|
||||||
|
if favorite_only:
|
||||||
|
stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
|
||||||
if resolved_mode == MODE_UNREAD:
|
if resolved_mode == MODE_UNREAD:
|
||||||
stmt = stmt.where(ScholarPublication.is_read.is_(False))
|
stmt = stmt.where(ScholarPublication.is_read.is_(False))
|
||||||
if resolved_mode == MODE_LATEST:
|
if resolved_mode == MODE_LATEST:
|
||||||
|
|
@ -45,12 +48,14 @@ async def count_unread_for_user(
|
||||||
*,
|
*,
|
||||||
user_id: int,
|
user_id: int,
|
||||||
scholar_profile_id: int | None = None,
|
scholar_profile_id: int | None = None,
|
||||||
|
favorite_only: bool = False,
|
||||||
) -> int:
|
) -> int:
|
||||||
return await count_for_user(
|
return await count_for_user(
|
||||||
db_session,
|
db_session,
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
mode=MODE_UNREAD,
|
mode=MODE_UNREAD,
|
||||||
scholar_profile_id=scholar_profile_id,
|
scholar_profile_id=scholar_profile_id,
|
||||||
|
favorite_only=favorite_only,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -59,10 +64,27 @@ async def count_latest_for_user(
|
||||||
*,
|
*,
|
||||||
user_id: int,
|
user_id: int,
|
||||||
scholar_profile_id: int | None = None,
|
scholar_profile_id: int | None = None,
|
||||||
|
favorite_only: bool = False,
|
||||||
) -> int:
|
) -> int:
|
||||||
return await count_for_user(
|
return await count_for_user(
|
||||||
db_session,
|
db_session,
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
mode=MODE_LATEST,
|
mode=MODE_LATEST,
|
||||||
scholar_profile_id=scholar_profile_id,
|
scholar_profile_id=scholar_profile_id,
|
||||||
|
favorite_only=favorite_only,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def count_favorite_for_user(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
scholar_profile_id: int | None = None,
|
||||||
|
) -> int:
|
||||||
|
return await count_for_user(
|
||||||
|
db_session,
|
||||||
|
user_id=user_id,
|
||||||
|
mode=MODE_ALL,
|
||||||
|
scholar_profile_id=scholar_profile_id,
|
||||||
|
favorite_only=True,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,135 +1,73 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import logging
|
import logging
|
||||||
import time
|
|
||||||
|
|
||||||
from app.db.session import get_session_factory
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from app.services.domains.publications.listing import (
|
|
||||||
missing_pdf_items,
|
from app.services.domains.publications.pdf_queue import (
|
||||||
resolve_and_persist_oa_metadata,
|
enqueue_missing_pdf_jobs,
|
||||||
|
enqueue_retry_pdf_job,
|
||||||
|
overlay_pdf_job_state,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.types import PublicationListItem
|
from app.services.domains.publications.types import PublicationListItem
|
||||||
from app.settings import settings
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
_enrichment_lock = asyncio.Lock()
|
|
||||||
_inflight_publications: set[tuple[int, int]] = set()
|
|
||||||
_recent_attempt_seconds: dict[tuple[int, int], float] = {}
|
|
||||||
_scheduled_tasks: set[asyncio.Task[None]] = set()
|
|
||||||
|
|
||||||
|
|
||||||
def _cooldown_seconds() -> float:
|
|
||||||
return max(float(settings.unpaywall_retry_cooldown_seconds), 1.0)
|
|
||||||
|
|
||||||
|
|
||||||
def _prune_recent_attempts(now_seconds: float, *, cooldown_seconds: float) -> None:
|
|
||||||
expiry = cooldown_seconds * 3
|
|
||||||
stale_keys = [
|
|
||||||
key for key, attempted_seconds in _recent_attempt_seconds.items()
|
|
||||||
if now_seconds - attempted_seconds >= expiry
|
|
||||||
]
|
|
||||||
for key in stale_keys:
|
|
||||||
_recent_attempt_seconds.pop(key, None)
|
|
||||||
|
|
||||||
|
|
||||||
async def _claim_items(
|
|
||||||
*,
|
|
||||||
user_id: int,
|
|
||||||
items: list[PublicationListItem],
|
|
||||||
max_items: int,
|
|
||||||
) -> list[PublicationListItem]:
|
|
||||||
candidates = missing_pdf_items(items, limit=max_items)
|
|
||||||
if not candidates:
|
|
||||||
return []
|
|
||||||
now_seconds = time.monotonic()
|
|
||||||
cooldown_seconds = _cooldown_seconds()
|
|
||||||
claimed: list[PublicationListItem] = []
|
|
||||||
async with _enrichment_lock:
|
|
||||||
_prune_recent_attempts(now_seconds, cooldown_seconds=cooldown_seconds)
|
|
||||||
for item in candidates:
|
|
||||||
key = (user_id, item.publication_id)
|
|
||||||
attempted_seconds = _recent_attempt_seconds.get(key)
|
|
||||||
if key in _inflight_publications:
|
|
||||||
continue
|
|
||||||
if attempted_seconds is not None and now_seconds - attempted_seconds < cooldown_seconds:
|
|
||||||
continue
|
|
||||||
_inflight_publications.add(key)
|
|
||||||
_recent_attempt_seconds[key] = now_seconds
|
|
||||||
claimed.append(item)
|
|
||||||
return claimed
|
|
||||||
|
|
||||||
|
|
||||||
async def _release_claims(*, user_id: int, publication_ids: list[int]) -> None:
|
|
||||||
async with _enrichment_lock:
|
|
||||||
for publication_id in publication_ids:
|
|
||||||
_inflight_publications.discard((user_id, publication_id))
|
|
||||||
|
|
||||||
|
|
||||||
def _on_task_done(task: asyncio.Task[None]) -> None:
|
|
||||||
_scheduled_tasks.discard(task)
|
|
||||||
try:
|
|
||||||
task.result()
|
|
||||||
except Exception:
|
|
||||||
logger.exception(
|
|
||||||
"publications.enrichment.task_failed",
|
|
||||||
extra={"event": "publications.enrichment.task_failed"},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def _run_enrichment(
|
|
||||||
*,
|
|
||||||
user_id: int,
|
|
||||||
request_email: str | None,
|
|
||||||
items: list[PublicationListItem],
|
|
||||||
) -> None:
|
|
||||||
publication_ids = [item.publication_id for item in items]
|
|
||||||
try:
|
|
||||||
session_factory = get_session_factory()
|
|
||||||
async with session_factory() as db_session:
|
|
||||||
await resolve_and_persist_oa_metadata(
|
|
||||||
db_session,
|
|
||||||
rows=items,
|
|
||||||
unpaywall_email=request_email,
|
|
||||||
)
|
|
||||||
finally:
|
|
||||||
await _release_claims(user_id=user_id, publication_ids=publication_ids)
|
|
||||||
logger.info(
|
|
||||||
"publications.enrichment.completed",
|
|
||||||
extra={
|
|
||||||
"event": "publications.enrichment.completed",
|
|
||||||
"user_id": user_id,
|
|
||||||
"publication_count": len(items),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def schedule_missing_pdf_enrichment_for_user(
|
async def schedule_missing_pdf_enrichment_for_user(
|
||||||
|
db_session: AsyncSession,
|
||||||
*,
|
*,
|
||||||
user_id: int,
|
user_id: int,
|
||||||
request_email: str | None,
|
request_email: str | None,
|
||||||
items: list[PublicationListItem],
|
items: list[PublicationListItem],
|
||||||
max_items: int,
|
max_items: int,
|
||||||
) -> int:
|
) -> int:
|
||||||
claimed_items = await _claim_items(user_id=user_id, items=items, max_items=max_items)
|
queued_ids = await enqueue_missing_pdf_jobs(
|
||||||
if not claimed_items:
|
db_session,
|
||||||
return 0
|
user_id=user_id,
|
||||||
task = asyncio.create_task(
|
request_email=request_email,
|
||||||
_run_enrichment(
|
rows=items,
|
||||||
user_id=user_id,
|
max_items=max_items,
|
||||||
request_email=request_email,
|
|
||||||
items=claimed_items,
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
_scheduled_tasks.add(task)
|
|
||||||
task.add_done_callback(_on_task_done)
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"publications.enrichment.scheduled",
|
"publications.enrichment.scheduled",
|
||||||
extra={
|
extra={
|
||||||
"event": "publications.enrichment.scheduled",
|
"event": "publications.enrichment.scheduled",
|
||||||
"user_id": user_id,
|
"user_id": user_id,
|
||||||
"publication_count": len(claimed_items),
|
"publication_count": len(queued_ids),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
return len(claimed_items)
|
return len(queued_ids)
|
||||||
|
|
||||||
|
|
||||||
|
async def schedule_retry_pdf_enrichment_for_row(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
request_email: str | None,
|
||||||
|
item: PublicationListItem,
|
||||||
|
) -> bool:
|
||||||
|
queued = await enqueue_retry_pdf_job(
|
||||||
|
db_session,
|
||||||
|
user_id=user_id,
|
||||||
|
request_email=request_email,
|
||||||
|
row=item,
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"publications.enrichment.retry_scheduled",
|
||||||
|
extra={
|
||||||
|
"event": "publications.enrichment.retry_scheduled",
|
||||||
|
"user_id": user_id,
|
||||||
|
"publication_id": item.publication_id,
|
||||||
|
"queued": queued,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return queued
|
||||||
|
|
||||||
|
|
||||||
|
async def hydrate_pdf_enrichment_state(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
items: list[PublicationListItem],
|
||||||
|
) -> list[PublicationListItem]:
|
||||||
|
return await overlay_pdf_job_state(db_session, rows=items)
|
||||||
|
|
|
||||||
|
|
@ -16,94 +16,6 @@ from app.services.domains.publications.queries import (
|
||||||
unread_item_from_row,
|
unread_item_from_row,
|
||||||
)
|
)
|
||||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||||
from app.services.domains.unpaywall.application import resolve_publication_oa_metadata
|
|
||||||
from sqlalchemy import update
|
|
||||||
from app.db.models import Publication
|
|
||||||
|
|
||||||
|
|
||||||
def _with_oa_overrides(
|
|
||||||
rows: list[PublicationListItem],
|
|
||||||
oa_data: dict[int, tuple[str | None, str | None]],
|
|
||||||
) -> list[PublicationListItem]:
|
|
||||||
return [
|
|
||||||
PublicationListItem(
|
|
||||||
publication_id=row.publication_id,
|
|
||||||
scholar_profile_id=row.scholar_profile_id,
|
|
||||||
scholar_label=row.scholar_label,
|
|
||||||
title=row.title,
|
|
||||||
year=row.year,
|
|
||||||
citation_count=row.citation_count,
|
|
||||||
venue_text=row.venue_text,
|
|
||||||
pub_url=row.pub_url,
|
|
||||||
doi=(oa_data.get(row.publication_id) or (None, None))[0] or row.doi,
|
|
||||||
pdf_url=(oa_data.get(row.publication_id) or (None, None))[1] or row.pdf_url,
|
|
||||||
is_read=row.is_read,
|
|
||||||
first_seen_at=row.first_seen_at,
|
|
||||||
is_new_in_latest_run=row.is_new_in_latest_run,
|
|
||||||
)
|
|
||||||
for row in rows
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _resolved_fields(
|
|
||||||
*,
|
|
||||||
row: PublicationListItem,
|
|
||||||
resolved: tuple[str | None, str | None] | None,
|
|
||||||
) -> tuple[str | None, str | None]:
|
|
||||||
if resolved is None:
|
|
||||||
return row.doi, row.pdf_url
|
|
||||||
resolved_doi, resolved_pdf = resolved
|
|
||||||
return resolved_doi or row.doi, resolved_pdf or row.pdf_url
|
|
||||||
|
|
||||||
|
|
||||||
async def _persist_resolved_metadata(
|
|
||||||
db_session: AsyncSession,
|
|
||||||
*,
|
|
||||||
rows: list[PublicationListItem],
|
|
||||||
oa_data: dict[int, tuple[str | None, str | None]],
|
|
||||||
) -> None:
|
|
||||||
by_id = {row.publication_id: row for row in rows}
|
|
||||||
updates: list[tuple[int, str | None, str | None]] = []
|
|
||||||
for publication_id, resolved in oa_data.items():
|
|
||||||
existing = by_id.get(publication_id)
|
|
||||||
if existing is None:
|
|
||||||
continue
|
|
||||||
next_doi, next_pdf = _resolved_fields(row=existing, resolved=resolved)
|
|
||||||
if next_doi == existing.doi and next_pdf == existing.pdf_url:
|
|
||||||
continue
|
|
||||||
updates.append((publication_id, next_doi, next_pdf))
|
|
||||||
for publication_id, doi, pdf_url in updates:
|
|
||||||
await db_session.execute(
|
|
||||||
update(Publication)
|
|
||||||
.where(Publication.id == publication_id)
|
|
||||||
.values(doi=doi, pdf_url=pdf_url)
|
|
||||||
)
|
|
||||||
if updates:
|
|
||||||
await db_session.commit()
|
|
||||||
|
|
||||||
|
|
||||||
def missing_pdf_items(
|
|
||||||
rows: list[PublicationListItem],
|
|
||||||
*,
|
|
||||||
limit: int,
|
|
||||||
) -> list[PublicationListItem]:
|
|
||||||
bounded_limit = max(0, int(limit))
|
|
||||||
if bounded_limit == 0:
|
|
||||||
return []
|
|
||||||
return [row for row in rows if not row.pdf_url][:bounded_limit]
|
|
||||||
|
|
||||||
|
|
||||||
async def resolve_and_persist_oa_metadata(
|
|
||||||
db_session: AsyncSession,
|
|
||||||
*,
|
|
||||||
rows: list[PublicationListItem],
|
|
||||||
unpaywall_email: str | None = None,
|
|
||||||
) -> dict[int, tuple[str | None, str | None]]:
|
|
||||||
if not rows:
|
|
||||||
return {}
|
|
||||||
oa_data = await resolve_publication_oa_metadata(rows, request_email=unpaywall_email)
|
|
||||||
await _persist_resolved_metadata(db_session, rows=rows, oa_data=oa_data)
|
|
||||||
return oa_data
|
|
||||||
|
|
||||||
|
|
||||||
async def list_for_user(
|
async def list_for_user(
|
||||||
|
|
@ -112,7 +24,9 @@ async def list_for_user(
|
||||||
user_id: int,
|
user_id: int,
|
||||||
mode: str = MODE_ALL,
|
mode: str = MODE_ALL,
|
||||||
scholar_profile_id: int | None = None,
|
scholar_profile_id: int | None = None,
|
||||||
|
favorite_only: bool = False,
|
||||||
limit: int = 300,
|
limit: int = 300,
|
||||||
|
offset: int = 0,
|
||||||
) -> list[PublicationListItem]:
|
) -> list[PublicationListItem]:
|
||||||
resolved_mode = resolve_publication_view_mode(mode)
|
resolved_mode = resolve_publication_view_mode(mode)
|
||||||
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
|
latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
|
||||||
|
|
@ -122,7 +36,9 @@ async def list_for_user(
|
||||||
mode=resolved_mode,
|
mode=resolved_mode,
|
||||||
latest_run_id=latest_run_id,
|
latest_run_id=latest_run_id,
|
||||||
scholar_profile_id=scholar_profile_id,
|
scholar_profile_id=scholar_profile_id,
|
||||||
|
favorite_only=favorite_only,
|
||||||
limit=limit,
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
rows = [
|
rows = [
|
||||||
|
|
@ -138,22 +54,13 @@ async def retry_pdf_for_user(
|
||||||
user_id: int,
|
user_id: int,
|
||||||
scholar_profile_id: int,
|
scholar_profile_id: int,
|
||||||
publication_id: int,
|
publication_id: int,
|
||||||
unpaywall_email: str | None = None,
|
|
||||||
) -> PublicationListItem | None:
|
) -> PublicationListItem | None:
|
||||||
item = await get_publication_item_for_user(
|
return await get_publication_item_for_user(
|
||||||
db_session,
|
db_session,
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
scholar_profile_id=scholar_profile_id,
|
scholar_profile_id=scholar_profile_id,
|
||||||
publication_id=publication_id,
|
publication_id=publication_id,
|
||||||
)
|
)
|
||||||
if item is None:
|
|
||||||
return None
|
|
||||||
oa_data = await resolve_and_persist_oa_metadata(
|
|
||||||
db_session,
|
|
||||||
rows=[item],
|
|
||||||
unpaywall_email=unpaywall_email,
|
|
||||||
)
|
|
||||||
return _with_oa_overrides([item], oa_data)[0]
|
|
||||||
|
|
||||||
|
|
||||||
async def list_unread_for_user(
|
async def list_unread_for_user(
|
||||||
|
|
@ -168,7 +75,9 @@ async def list_unread_for_user(
|
||||||
mode=MODE_UNREAD,
|
mode=MODE_UNREAD,
|
||||||
latest_run_id=None,
|
latest_run_id=None,
|
||||||
scholar_profile_id=None,
|
scholar_profile_id=None,
|
||||||
|
favorite_only=False,
|
||||||
limit=limit,
|
limit=limit,
|
||||||
|
offset=0,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
return [unread_item_from_row(row) for row in result.all()]
|
return [unread_item_from_row(row) for row in result.all()]
|
||||||
|
|
|
||||||
877
app/services/domains/publications/pdf_queue.py
Normal file
|
|
@ -0,0 +1,877 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from sqlalchemy import Select, func, literal, or_, select, union_all
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.db.models import (
|
||||||
|
Publication,
|
||||||
|
PublicationPdfJob,
|
||||||
|
PublicationPdfJobEvent,
|
||||||
|
ScholarProfile,
|
||||||
|
ScholarPublication,
|
||||||
|
User,
|
||||||
|
)
|
||||||
|
from app.db.session import get_session_factory
|
||||||
|
from app.services.domains.publications.types import PublicationListItem
|
||||||
|
from app.services.domains.unpaywall.application import (
|
||||||
|
FAILURE_RESOLUTION_EXCEPTION,
|
||||||
|
OaResolutionOutcome,
|
||||||
|
resolve_publication_oa_outcomes,
|
||||||
|
)
|
||||||
|
from app.settings import settings
|
||||||
|
|
||||||
|
PDF_STATUS_UNTRACKED = "untracked"
|
||||||
|
PDF_STATUS_QUEUED = "queued"
|
||||||
|
PDF_STATUS_RUNNING = "running"
|
||||||
|
PDF_STATUS_RESOLVED = "resolved"
|
||||||
|
PDF_STATUS_FAILED = "failed"
|
||||||
|
|
||||||
|
PDF_EVENT_QUEUED = "queued"
|
||||||
|
PDF_EVENT_ATTEMPT_STARTED = "attempt_started"
|
||||||
|
PDF_EVENT_RESOLVED = "resolved"
|
||||||
|
PDF_EVENT_FAILED = "failed"
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
_scheduled_tasks: set[asyncio.Task[None]] = set()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PdfQueueListItem:
|
||||||
|
publication_id: int
|
||||||
|
title: str
|
||||||
|
doi: str | None
|
||||||
|
pdf_url: str | None
|
||||||
|
status: str
|
||||||
|
attempt_count: int
|
||||||
|
last_failure_reason: str | None
|
||||||
|
last_failure_detail: str | None
|
||||||
|
last_source: str | None
|
||||||
|
requested_by_user_id: int | None
|
||||||
|
requested_by_email: str | None
|
||||||
|
queued_at: datetime | None
|
||||||
|
last_attempt_at: datetime | None
|
||||||
|
resolved_at: datetime | None
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PdfRequeueResult:
|
||||||
|
publication_exists: bool
|
||||||
|
queued: bool
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PdfBulkQueueResult:
|
||||||
|
requested_count: int
|
||||||
|
queued_count: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PdfQueuePage:
|
||||||
|
items: list[PdfQueueListItem]
|
||||||
|
total_count: int
|
||||||
|
limit: int
|
||||||
|
offset: int
|
||||||
|
|
||||||
|
|
||||||
|
def _utcnow() -> datetime:
|
||||||
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _publication_ids(rows: list[PublicationListItem]) -> list[int]:
|
||||||
|
return sorted({row.publication_id for row in rows})
|
||||||
|
|
||||||
|
|
||||||
|
def _status_from_job(row: PublicationListItem, job: PublicationPdfJob | None) -> str:
|
||||||
|
if row.pdf_url:
|
||||||
|
return PDF_STATUS_RESOLVED
|
||||||
|
if job is None:
|
||||||
|
return PDF_STATUS_UNTRACKED
|
||||||
|
return job.status
|
||||||
|
|
||||||
|
|
||||||
|
def _item_from_row_and_job(
|
||||||
|
row: PublicationListItem,
|
||||||
|
job: PublicationPdfJob | None,
|
||||||
|
) -> PublicationListItem:
|
||||||
|
return PublicationListItem(
|
||||||
|
publication_id=row.publication_id,
|
||||||
|
scholar_profile_id=row.scholar_profile_id,
|
||||||
|
scholar_label=row.scholar_label,
|
||||||
|
title=row.title,
|
||||||
|
year=row.year,
|
||||||
|
citation_count=row.citation_count,
|
||||||
|
venue_text=row.venue_text,
|
||||||
|
pub_url=row.pub_url,
|
||||||
|
doi=row.doi,
|
||||||
|
pdf_url=row.pdf_url,
|
||||||
|
is_read=row.is_read,
|
||||||
|
is_favorite=row.is_favorite,
|
||||||
|
first_seen_at=row.first_seen_at,
|
||||||
|
is_new_in_latest_run=row.is_new_in_latest_run,
|
||||||
|
pdf_status=_status_from_job(row, job),
|
||||||
|
pdf_attempt_count=int(job.attempt_count) if job is not None else 0,
|
||||||
|
pdf_failure_reason=job.last_failure_reason if job is not None else None,
|
||||||
|
pdf_failure_detail=job.last_failure_detail if job is not None else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _queueable_rows(
|
||||||
|
rows: list[PublicationListItem],
|
||||||
|
*,
|
||||||
|
max_items: int,
|
||||||
|
) -> list[PublicationListItem]:
|
||||||
|
bounded = max(0, int(max_items))
|
||||||
|
if bounded == 0:
|
||||||
|
return []
|
||||||
|
candidates = [row for row in rows if not row.pdf_url]
|
||||||
|
return candidates[:bounded]
|
||||||
|
|
||||||
|
|
||||||
|
def _bounded_limit(limit: int, *, max_value: int = 500) -> int:
|
||||||
|
return max(1, min(int(limit), max_value))
|
||||||
|
|
||||||
|
|
||||||
|
def _bounded_offset(offset: int) -> int:
|
||||||
|
return max(int(offset), 0)
|
||||||
|
|
||||||
|
|
||||||
|
def _auto_retry_interval_seconds() -> int:
|
||||||
|
return max(int(settings.pdf_auto_retry_interval_seconds), 1)
|
||||||
|
|
||||||
|
|
||||||
|
def _auto_retry_first_interval_seconds() -> int:
|
||||||
|
return max(int(settings.pdf_auto_retry_first_interval_seconds), 1)
|
||||||
|
|
||||||
|
|
||||||
|
def _auto_retry_max_attempts() -> int:
|
||||||
|
return max(int(settings.pdf_auto_retry_max_attempts), 1)
|
||||||
|
|
||||||
|
|
||||||
|
def _retry_interval_seconds_for_attempt_count(attempt_count: int) -> int:
|
||||||
|
if int(attempt_count) <= 1:
|
||||||
|
return _auto_retry_first_interval_seconds()
|
||||||
|
return _auto_retry_interval_seconds()
|
||||||
|
|
||||||
|
|
||||||
|
def _cooldown_active(
|
||||||
|
*,
|
||||||
|
last_attempt_at: datetime | None,
|
||||||
|
attempt_count: int,
|
||||||
|
) -> bool:
|
||||||
|
if last_attempt_at is None:
|
||||||
|
return False
|
||||||
|
elapsed = (_utcnow() - last_attempt_at).total_seconds()
|
||||||
|
return elapsed < _retry_interval_seconds_for_attempt_count(int(attempt_count))
|
||||||
|
|
||||||
|
|
||||||
|
def _can_enqueue_job(
|
||||||
|
job: PublicationPdfJob | None,
|
||||||
|
*,
|
||||||
|
force_retry: bool,
|
||||||
|
) -> bool:
|
||||||
|
if job is None:
|
||||||
|
return True
|
||||||
|
if job.status in {PDF_STATUS_QUEUED, PDF_STATUS_RUNNING}:
|
||||||
|
return False
|
||||||
|
if force_retry:
|
||||||
|
return job.status in {PDF_STATUS_FAILED, PDF_STATUS_RESOLVED, PDF_STATUS_UNTRACKED}
|
||||||
|
if job.status == PDF_STATUS_RESOLVED:
|
||||||
|
return False
|
||||||
|
if int(job.attempt_count) >= _auto_retry_max_attempts():
|
||||||
|
return False
|
||||||
|
if _cooldown_active(
|
||||||
|
last_attempt_at=job.last_attempt_at,
|
||||||
|
attempt_count=int(job.attempt_count),
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _event_row(
|
||||||
|
*,
|
||||||
|
publication_id: int,
|
||||||
|
user_id: int | None,
|
||||||
|
event_type: str,
|
||||||
|
status: str | None,
|
||||||
|
source: str | None = None,
|
||||||
|
failure_reason: str | None = None,
|
||||||
|
message: str | None = None,
|
||||||
|
) -> PublicationPdfJobEvent:
|
||||||
|
return PublicationPdfJobEvent(
|
||||||
|
publication_id=publication_id,
|
||||||
|
user_id=user_id,
|
||||||
|
event_type=event_type,
|
||||||
|
status=status,
|
||||||
|
source=source,
|
||||||
|
failure_reason=failure_reason,
|
||||||
|
message=message,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _queued_job(
|
||||||
|
*,
|
||||||
|
publication_id: int,
|
||||||
|
user_id: int,
|
||||||
|
) -> PublicationPdfJob:
|
||||||
|
now = _utcnow()
|
||||||
|
return PublicationPdfJob(
|
||||||
|
publication_id=publication_id,
|
||||||
|
status=PDF_STATUS_QUEUED,
|
||||||
|
queued_at=now,
|
||||||
|
last_requested_by_user_id=user_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _mark_job_queued(job: PublicationPdfJob, *, user_id: int) -> None:
|
||||||
|
now = _utcnow()
|
||||||
|
job.status = PDF_STATUS_QUEUED
|
||||||
|
job.queued_at = now
|
||||||
|
job.last_requested_by_user_id = user_id
|
||||||
|
job.last_failure_reason = None
|
||||||
|
job.last_failure_detail = None
|
||||||
|
job.last_source = None
|
||||||
|
|
||||||
|
|
||||||
|
def _state_map(jobs: list[PublicationPdfJob]) -> dict[int, PublicationPdfJob]:
|
||||||
|
return {int(job.publication_id): job for job in jobs}
|
||||||
|
|
||||||
|
|
||||||
|
async def _jobs_for_publication_ids(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
publication_ids: list[int],
|
||||||
|
) -> dict[int, PublicationPdfJob]:
|
||||||
|
if not publication_ids:
|
||||||
|
return {}
|
||||||
|
result = await db_session.execute(
|
||||||
|
select(PublicationPdfJob).where(PublicationPdfJob.publication_id.in_(publication_ids))
|
||||||
|
)
|
||||||
|
return _state_map(list(result.scalars()))
|
||||||
|
|
||||||
|
|
||||||
|
async def overlay_pdf_job_state(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
rows: list[PublicationListItem],
|
||||||
|
) -> list[PublicationListItem]:
|
||||||
|
if not rows:
|
||||||
|
return []
|
||||||
|
jobs = await _jobs_for_publication_ids(
|
||||||
|
db_session,
|
||||||
|
publication_ids=_publication_ids(rows),
|
||||||
|
)
|
||||||
|
return [_item_from_row_and_job(row, jobs.get(row.publication_id)) for row in rows]
|
||||||
|
|
||||||
|
|
||||||
|
async def _enqueue_rows(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
rows: list[PublicationListItem],
|
||||||
|
force_retry: bool,
|
||||||
|
) -> list[PublicationListItem]:
|
||||||
|
if not rows:
|
||||||
|
return []
|
||||||
|
queued: list[PublicationListItem] = []
|
||||||
|
jobs = await _jobs_for_publication_ids(
|
||||||
|
db_session,
|
||||||
|
publication_ids=_publication_ids(rows),
|
||||||
|
)
|
||||||
|
for row in rows:
|
||||||
|
job = jobs.get(row.publication_id)
|
||||||
|
if not _can_enqueue_job(job, force_retry=force_retry):
|
||||||
|
continue
|
||||||
|
if job is None:
|
||||||
|
job = _queued_job(publication_id=row.publication_id, user_id=user_id)
|
||||||
|
jobs[row.publication_id] = job
|
||||||
|
db_session.add(job)
|
||||||
|
else:
|
||||||
|
_mark_job_queued(job, user_id=user_id)
|
||||||
|
db_session.add(
|
||||||
|
_event_row(
|
||||||
|
publication_id=row.publication_id,
|
||||||
|
user_id=user_id,
|
||||||
|
event_type=PDF_EVENT_QUEUED,
|
||||||
|
status=PDF_STATUS_QUEUED,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
queued.append(row)
|
||||||
|
if queued:
|
||||||
|
await db_session.commit()
|
||||||
|
return queued
|
||||||
|
|
||||||
|
|
||||||
|
def _register_task(task: asyncio.Task[None]) -> None:
|
||||||
|
_scheduled_tasks.add(task)
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_finished_task(task: asyncio.Task[None]) -> None:
|
||||||
|
_scheduled_tasks.discard(task)
|
||||||
|
try:
|
||||||
|
task.result()
|
||||||
|
except Exception:
|
||||||
|
logger.exception(
|
||||||
|
"publications.pdf_queue.task_failed",
|
||||||
|
extra={"event": "publications.pdf_queue.task_failed"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _mark_attempt_started(
|
||||||
|
*,
|
||||||
|
publication_id: int,
|
||||||
|
user_id: int,
|
||||||
|
) -> None:
|
||||||
|
session_factory = get_session_factory()
|
||||||
|
async with session_factory() as db_session:
|
||||||
|
job = await db_session.get(PublicationPdfJob, publication_id)
|
||||||
|
if job is None:
|
||||||
|
job = _queued_job(publication_id=publication_id, user_id=user_id)
|
||||||
|
db_session.add(job)
|
||||||
|
job.status = PDF_STATUS_RUNNING
|
||||||
|
job.last_attempt_at = _utcnow()
|
||||||
|
job.attempt_count = int(job.attempt_count) + 1
|
||||||
|
db_session.add(
|
||||||
|
_event_row(
|
||||||
|
publication_id=publication_id,
|
||||||
|
user_id=user_id,
|
||||||
|
event_type=PDF_EVENT_ATTEMPT_STARTED,
|
||||||
|
status=PDF_STATUS_RUNNING,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await db_session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _failed_outcome(
|
||||||
|
*,
|
||||||
|
row: PublicationListItem,
|
||||||
|
) -> OaResolutionOutcome:
|
||||||
|
return OaResolutionOutcome(
|
||||||
|
publication_id=row.publication_id,
|
||||||
|
doi=row.doi,
|
||||||
|
pdf_url=None,
|
||||||
|
failure_reason=FAILURE_RESOLUTION_EXCEPTION,
|
||||||
|
source=None,
|
||||||
|
used_crossref=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _fetch_outcome_for_row(
|
||||||
|
*,
|
||||||
|
row: PublicationListItem,
|
||||||
|
request_email: str | None,
|
||||||
|
) -> OaResolutionOutcome:
|
||||||
|
outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email)
|
||||||
|
outcome = outcomes.get(row.publication_id)
|
||||||
|
if outcome is not None:
|
||||||
|
return outcome
|
||||||
|
return _failed_outcome(row=row)
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_publication_update(
|
||||||
|
publication: Publication,
|
||||||
|
*,
|
||||||
|
doi: str | None,
|
||||||
|
pdf_url: str | None,
|
||||||
|
) -> None:
|
||||||
|
if doi and publication.doi != doi:
|
||||||
|
publication.doi = doi
|
||||||
|
if pdf_url and publication.pdf_url != pdf_url:
|
||||||
|
publication.pdf_url = pdf_url
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_job_outcome(job: PublicationPdfJob, *, outcome: OaResolutionOutcome) -> None:
|
||||||
|
job.last_source = outcome.source
|
||||||
|
if outcome.pdf_url:
|
||||||
|
job.status = PDF_STATUS_RESOLVED
|
||||||
|
job.resolved_at = _utcnow()
|
||||||
|
job.last_failure_reason = None
|
||||||
|
job.last_failure_detail = None
|
||||||
|
return
|
||||||
|
job.status = PDF_STATUS_FAILED
|
||||||
|
job.last_failure_reason = outcome.failure_reason
|
||||||
|
job.last_failure_detail = outcome.failure_reason
|
||||||
|
|
||||||
|
|
||||||
|
def _result_event(outcome: OaResolutionOutcome) -> tuple[str, str]:
|
||||||
|
if outcome.pdf_url:
|
||||||
|
return PDF_EVENT_RESOLVED, PDF_STATUS_RESOLVED
|
||||||
|
return PDF_EVENT_FAILED, PDF_STATUS_FAILED
|
||||||
|
|
||||||
|
|
||||||
|
async def _persist_outcome(
|
||||||
|
*,
|
||||||
|
publication_id: int,
|
||||||
|
user_id: int,
|
||||||
|
outcome: OaResolutionOutcome,
|
||||||
|
) -> None:
|
||||||
|
session_factory = get_session_factory()
|
||||||
|
async with session_factory() as db_session:
|
||||||
|
publication = await db_session.get(Publication, publication_id)
|
||||||
|
job = await db_session.get(PublicationPdfJob, publication_id)
|
||||||
|
if publication is None or job is None:
|
||||||
|
return
|
||||||
|
_apply_publication_update(publication, doi=outcome.doi, pdf_url=outcome.pdf_url)
|
||||||
|
_apply_job_outcome(job, outcome=outcome)
|
||||||
|
event_type, status = _result_event(outcome)
|
||||||
|
db_session.add(
|
||||||
|
_event_row(
|
||||||
|
publication_id=publication_id,
|
||||||
|
user_id=user_id,
|
||||||
|
event_type=event_type,
|
||||||
|
status=status,
|
||||||
|
source=outcome.source,
|
||||||
|
failure_reason=outcome.failure_reason,
|
||||||
|
message=outcome.failure_reason,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await db_session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
async def _resolve_publication_row(
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
request_email: str | None,
|
||||||
|
row: PublicationListItem,
|
||||||
|
) -> None:
|
||||||
|
await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
|
||||||
|
try:
|
||||||
|
outcome = await _fetch_outcome_for_row(row=row, request_email=request_email)
|
||||||
|
except Exception as exc: # pragma: no cover - defensive network boundary
|
||||||
|
logger.warning(
|
||||||
|
"publications.pdf_queue.resolve_failed",
|
||||||
|
extra={
|
||||||
|
"event": "publications.pdf_queue.resolve_failed",
|
||||||
|
"publication_id": row.publication_id,
|
||||||
|
"error": str(exc),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
outcome = _failed_outcome(row=row)
|
||||||
|
await _persist_outcome(
|
||||||
|
publication_id=row.publication_id,
|
||||||
|
user_id=user_id,
|
||||||
|
outcome=outcome,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_resolution_task(
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
request_email: str | None,
|
||||||
|
rows: list[PublicationListItem],
|
||||||
|
) -> None:
|
||||||
|
for row in rows:
|
||||||
|
await _resolve_publication_row(
|
||||||
|
user_id=user_id,
|
||||||
|
request_email=request_email,
|
||||||
|
row=row,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _schedule_rows(
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
request_email: str | None,
|
||||||
|
rows: list[PublicationListItem],
|
||||||
|
) -> None:
|
||||||
|
if not rows:
|
||||||
|
return
|
||||||
|
task = asyncio.create_task(
|
||||||
|
_run_resolution_task(
|
||||||
|
user_id=user_id,
|
||||||
|
request_email=request_email,
|
||||||
|
rows=rows,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
_register_task(task)
|
||||||
|
task.add_done_callback(_drop_finished_task)
|
||||||
|
|
||||||
|
|
||||||
|
async def enqueue_missing_pdf_jobs(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
request_email: str | None,
|
||||||
|
rows: list[PublicationListItem],
|
||||||
|
max_items: int,
|
||||||
|
) -> list[int]:
|
||||||
|
queueable = _queueable_rows(rows, max_items=max_items)
|
||||||
|
queued_rows = await _enqueue_rows(
|
||||||
|
db_session,
|
||||||
|
user_id=user_id,
|
||||||
|
rows=queueable,
|
||||||
|
force_retry=False,
|
||||||
|
)
|
||||||
|
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
|
||||||
|
return [row.publication_id for row in queued_rows]
|
||||||
|
|
||||||
|
|
||||||
|
async def enqueue_retry_pdf_job(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
request_email: str | None,
|
||||||
|
row: PublicationListItem,
|
||||||
|
) -> bool:
|
||||||
|
queued_rows = await _enqueue_rows(
|
||||||
|
db_session,
|
||||||
|
user_id=user_id,
|
||||||
|
rows=[row],
|
||||||
|
force_retry=True,
|
||||||
|
)
|
||||||
|
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
|
||||||
|
return bool(queued_rows)
|
||||||
|
|
||||||
|
|
||||||
|
def _retry_item_label(display_name: str | None, scholar_id: str | None) -> str:
|
||||||
|
return str(display_name or scholar_id or "unknown")
|
||||||
|
|
||||||
|
|
||||||
|
def _retry_item_from_publication(
|
||||||
|
publication: Publication,
|
||||||
|
*,
|
||||||
|
link_row: tuple | None,
|
||||||
|
) -> PublicationListItem:
|
||||||
|
if link_row is None:
|
||||||
|
scholar_profile_id = 0
|
||||||
|
scholar_label = "unknown"
|
||||||
|
is_read = True
|
||||||
|
first_seen_at = publication.created_at or _utcnow()
|
||||||
|
else:
|
||||||
|
scholar_profile_id = int(link_row[0])
|
||||||
|
scholar_label = _retry_item_label(link_row[1], link_row[2])
|
||||||
|
is_read = bool(link_row[3])
|
||||||
|
first_seen_at = link_row[4] or publication.created_at or _utcnow()
|
||||||
|
return PublicationListItem(
|
||||||
|
publication_id=int(publication.id),
|
||||||
|
scholar_profile_id=scholar_profile_id,
|
||||||
|
scholar_label=scholar_label,
|
||||||
|
title=publication.title_raw,
|
||||||
|
year=publication.year,
|
||||||
|
citation_count=int(publication.citation_count or 0),
|
||||||
|
venue_text=publication.venue_text,
|
||||||
|
pub_url=publication.pub_url,
|
||||||
|
doi=publication.doi,
|
||||||
|
pdf_url=publication.pdf_url,
|
||||||
|
is_read=is_read,
|
||||||
|
first_seen_at=first_seen_at,
|
||||||
|
is_new_in_latest_run=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _retry_item_for_publication_id(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
publication_id: int,
|
||||||
|
) -> PublicationListItem | None:
|
||||||
|
publication = await db_session.get(Publication, publication_id)
|
||||||
|
if publication is None:
|
||||||
|
return None
|
||||||
|
result = await db_session.execute(
|
||||||
|
select(
|
||||||
|
ScholarProfile.id,
|
||||||
|
ScholarProfile.display_name,
|
||||||
|
ScholarProfile.scholar_id,
|
||||||
|
ScholarPublication.is_read,
|
||||||
|
ScholarPublication.created_at,
|
||||||
|
)
|
||||||
|
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
||||||
|
.where(ScholarPublication.publication_id == publication_id)
|
||||||
|
.order_by(ScholarPublication.created_at.asc())
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
return _retry_item_from_publication(publication, link_row=result.one_or_none())
|
||||||
|
|
||||||
|
|
||||||
|
async def enqueue_retry_pdf_job_for_publication_id(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
request_email: str | None,
|
||||||
|
publication_id: int,
|
||||||
|
) -> PdfRequeueResult:
|
||||||
|
row = await _retry_item_for_publication_id(
|
||||||
|
db_session,
|
||||||
|
publication_id=publication_id,
|
||||||
|
)
|
||||||
|
if row is None:
|
||||||
|
return PdfRequeueResult(publication_exists=False, queued=False)
|
||||||
|
queued = await enqueue_retry_pdf_job(
|
||||||
|
db_session,
|
||||||
|
user_id=user_id,
|
||||||
|
request_email=request_email,
|
||||||
|
row=row,
|
||||||
|
)
|
||||||
|
return PdfRequeueResult(publication_exists=True, queued=queued)
|
||||||
|
|
||||||
|
|
||||||
|
def _queue_candidate_from_publication(publication: Publication) -> PublicationListItem:
|
||||||
|
return PublicationListItem(
|
||||||
|
publication_id=int(publication.id),
|
||||||
|
scholar_profile_id=0,
|
||||||
|
scholar_label="admin",
|
||||||
|
title=publication.title_raw,
|
||||||
|
year=publication.year,
|
||||||
|
citation_count=int(publication.citation_count or 0),
|
||||||
|
venue_text=publication.venue_text,
|
||||||
|
pub_url=publication.pub_url,
|
||||||
|
doi=publication.doi,
|
||||||
|
pdf_url=publication.pdf_url,
|
||||||
|
is_read=True,
|
||||||
|
first_seen_at=publication.created_at or _utcnow(),
|
||||||
|
is_new_in_latest_run=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _missing_pdf_candidates(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
limit: int,
|
||||||
|
) -> list[PublicationListItem]:
|
||||||
|
bounded_limit = max(1, min(int(limit), 5000))
|
||||||
|
result = await db_session.execute(
|
||||||
|
select(Publication)
|
||||||
|
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
|
||||||
|
.where(Publication.pdf_url.is_(None))
|
||||||
|
.where(
|
||||||
|
or_(
|
||||||
|
PublicationPdfJob.publication_id.is_(None),
|
||||||
|
PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.order_by(Publication.updated_at.desc(), Publication.id.desc())
|
||||||
|
.limit(bounded_limit)
|
||||||
|
)
|
||||||
|
return [
|
||||||
|
_queue_candidate_from_publication(publication)
|
||||||
|
for publication in result.scalars()
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
async def enqueue_all_missing_pdf_jobs(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
request_email: str | None,
|
||||||
|
limit: int = 1000,
|
||||||
|
) -> PdfBulkQueueResult:
|
||||||
|
candidates = await _missing_pdf_candidates(db_session, limit=limit)
|
||||||
|
queued_rows = await _enqueue_rows(
|
||||||
|
db_session,
|
||||||
|
user_id=user_id,
|
||||||
|
rows=candidates,
|
||||||
|
force_retry=True,
|
||||||
|
)
|
||||||
|
_schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
|
||||||
|
return PdfBulkQueueResult(
|
||||||
|
requested_count=len(candidates),
|
||||||
|
queued_count=len(queued_rows),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]:
|
||||||
|
stmt = (
|
||||||
|
select(
|
||||||
|
PublicationPdfJob.publication_id,
|
||||||
|
Publication.title_raw,
|
||||||
|
Publication.doi,
|
||||||
|
Publication.pdf_url,
|
||||||
|
PublicationPdfJob.status,
|
||||||
|
PublicationPdfJob.attempt_count,
|
||||||
|
PublicationPdfJob.last_failure_reason,
|
||||||
|
PublicationPdfJob.last_failure_detail,
|
||||||
|
PublicationPdfJob.last_source,
|
||||||
|
PublicationPdfJob.last_requested_by_user_id,
|
||||||
|
User.email,
|
||||||
|
PublicationPdfJob.queued_at,
|
||||||
|
PublicationPdfJob.last_attempt_at,
|
||||||
|
PublicationPdfJob.resolved_at,
|
||||||
|
PublicationPdfJob.updated_at,
|
||||||
|
)
|
||||||
|
.join(Publication, Publication.id == PublicationPdfJob.publication_id)
|
||||||
|
.outerjoin(User, User.id == PublicationPdfJob.last_requested_by_user_id)
|
||||||
|
)
|
||||||
|
if status:
|
||||||
|
stmt = stmt.where(PublicationPdfJob.status == status)
|
||||||
|
return stmt
|
||||||
|
|
||||||
|
|
||||||
|
def _tracked_queue_select(*, limit: int, offset: int, status: str | None) -> Select[tuple]:
|
||||||
|
return (
|
||||||
|
_tracked_queue_select_base(status=status)
|
||||||
|
.order_by(PublicationPdfJob.updated_at.desc())
|
||||||
|
.limit(_bounded_limit(limit))
|
||||||
|
.offset(_bounded_offset(offset))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _untracked_queue_select_base() -> Select[tuple]:
|
||||||
|
return (
|
||||||
|
select(
|
||||||
|
Publication.id,
|
||||||
|
Publication.title_raw,
|
||||||
|
Publication.doi,
|
||||||
|
Publication.pdf_url,
|
||||||
|
literal(PDF_STATUS_UNTRACKED),
|
||||||
|
literal(0),
|
||||||
|
literal(None),
|
||||||
|
literal(None),
|
||||||
|
literal(None),
|
||||||
|
literal(None),
|
||||||
|
literal(None),
|
||||||
|
literal(None),
|
||||||
|
literal(None),
|
||||||
|
literal(None),
|
||||||
|
Publication.updated_at,
|
||||||
|
)
|
||||||
|
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
|
||||||
|
.where(Publication.pdf_url.is_(None))
|
||||||
|
.where(PublicationPdfJob.publication_id.is_(None))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _untracked_queue_select(*, limit: int, offset: int) -> Select[tuple]:
|
||||||
|
return (
|
||||||
|
_untracked_queue_select_base()
|
||||||
|
.order_by(Publication.updated_at.desc(), Publication.id.desc())
|
||||||
|
.limit(_bounded_limit(limit))
|
||||||
|
.offset(_bounded_offset(offset))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _all_queue_select(*, limit: int, offset: int) -> Select[tuple]:
|
||||||
|
union_stmt = union_all(
|
||||||
|
_tracked_queue_select_base(status=None),
|
||||||
|
_untracked_queue_select_base(),
|
||||||
|
).subquery()
|
||||||
|
return (
|
||||||
|
select(union_stmt)
|
||||||
|
.order_by(union_stmt.c.updated_at.desc())
|
||||||
|
.limit(_bounded_limit(limit))
|
||||||
|
.offset(_bounded_offset(offset))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _tracked_queue_count_select(*, status: str | None) -> Select[tuple]:
|
||||||
|
stmt = select(func.count()).select_from(PublicationPdfJob)
|
||||||
|
if status:
|
||||||
|
stmt = stmt.where(PublicationPdfJob.status == status)
|
||||||
|
return stmt
|
||||||
|
|
||||||
|
|
||||||
|
def _untracked_queue_count_select() -> Select[tuple]:
|
||||||
|
return (
|
||||||
|
select(func.count())
|
||||||
|
.select_from(Publication)
|
||||||
|
.outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
|
||||||
|
.where(Publication.pdf_url.is_(None))
|
||||||
|
.where(PublicationPdfJob.publication_id.is_(None))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _queue_item_from_row(row: tuple) -> PdfQueueListItem:
|
||||||
|
return PdfQueueListItem(
|
||||||
|
publication_id=int(row[0]),
|
||||||
|
title=str(row[1] or ""),
|
||||||
|
doi=row[2],
|
||||||
|
pdf_url=row[3],
|
||||||
|
status=str(row[4] or PDF_STATUS_UNTRACKED),
|
||||||
|
attempt_count=int(row[5] or 0),
|
||||||
|
last_failure_reason=row[6],
|
||||||
|
last_failure_detail=row[7],
|
||||||
|
last_source=row[8],
|
||||||
|
requested_by_user_id=int(row[9]) if row[9] is not None else None,
|
||||||
|
requested_by_email=row[10],
|
||||||
|
queued_at=row[11],
|
||||||
|
last_attempt_at=row[12],
|
||||||
|
resolved_at=row[13],
|
||||||
|
updated_at=row[14],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def list_pdf_queue_items(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
limit: int = 100,
|
||||||
|
offset: int = 0,
|
||||||
|
status: str | None = None,
|
||||||
|
) -> list[PdfQueueListItem]:
|
||||||
|
bounded_limit = _bounded_limit(limit)
|
||||||
|
bounded_offset = _bounded_offset(offset)
|
||||||
|
normalized_status = (status or "").strip().lower() or None
|
||||||
|
if normalized_status == PDF_STATUS_UNTRACKED:
|
||||||
|
result = await db_session.execute(
|
||||||
|
_untracked_queue_select(
|
||||||
|
limit=bounded_limit,
|
||||||
|
offset=bounded_offset,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return [_queue_item_from_row(row) for row in result.all()]
|
||||||
|
if normalized_status is None:
|
||||||
|
result = await db_session.execute(
|
||||||
|
_all_queue_select(
|
||||||
|
limit=bounded_limit,
|
||||||
|
offset=bounded_offset,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return [_queue_item_from_row(row) for row in result.all()]
|
||||||
|
result = await db_session.execute(
|
||||||
|
_tracked_queue_select(
|
||||||
|
limit=bounded_limit,
|
||||||
|
offset=bounded_offset,
|
||||||
|
status=normalized_status,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return [_queue_item_from_row(row) for row in result.all()]
|
||||||
|
|
||||||
|
|
||||||
|
async def count_pdf_queue_items(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
status: str | None = None,
|
||||||
|
) -> int:
|
||||||
|
normalized_status = (status or "").strip().lower() or None
|
||||||
|
if normalized_status == PDF_STATUS_UNTRACKED:
|
||||||
|
result = await db_session.execute(_untracked_queue_count_select())
|
||||||
|
return int(result.scalar_one() or 0)
|
||||||
|
tracked_result = await db_session.execute(
|
||||||
|
_tracked_queue_count_select(status=normalized_status)
|
||||||
|
)
|
||||||
|
tracked_count = int(tracked_result.scalar_one() or 0)
|
||||||
|
if normalized_status is not None:
|
||||||
|
return tracked_count
|
||||||
|
untracked_result = await db_session.execute(_untracked_queue_count_select())
|
||||||
|
untracked_count = int(untracked_result.scalar_one() or 0)
|
||||||
|
return tracked_count + untracked_count
|
||||||
|
|
||||||
|
|
||||||
|
async def list_pdf_queue_page(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
limit: int = 100,
|
||||||
|
offset: int = 0,
|
||||||
|
status: str | None = None,
|
||||||
|
) -> PdfQueuePage:
|
||||||
|
bounded_limit = _bounded_limit(limit)
|
||||||
|
bounded_offset = _bounded_offset(offset)
|
||||||
|
items = await list_pdf_queue_items(
|
||||||
|
db_session,
|
||||||
|
limit=bounded_limit,
|
||||||
|
offset=bounded_offset,
|
||||||
|
status=status,
|
||||||
|
)
|
||||||
|
total_count = await count_pdf_queue_items(
|
||||||
|
db_session,
|
||||||
|
status=status,
|
||||||
|
)
|
||||||
|
return PdfQueuePage(
|
||||||
|
items=items,
|
||||||
|
total_count=total_count,
|
||||||
|
limit=bounded_limit,
|
||||||
|
offset=bounded_offset,
|
||||||
|
)
|
||||||
|
|
@ -36,7 +36,9 @@ def publications_query(
|
||||||
mode: str,
|
mode: str,
|
||||||
latest_run_id: int | None,
|
latest_run_id: int | None,
|
||||||
scholar_profile_id: int | None,
|
scholar_profile_id: int | None,
|
||||||
|
favorite_only: bool,
|
||||||
limit: int,
|
limit: int,
|
||||||
|
offset: int = 0,
|
||||||
) -> Select[tuple]:
|
) -> Select[tuple]:
|
||||||
scholar_label = ScholarProfile.display_name
|
scholar_label = ScholarProfile.display_name
|
||||||
stmt = (
|
stmt = (
|
||||||
|
|
@ -53,6 +55,7 @@ def publications_query(
|
||||||
Publication.doi,
|
Publication.doi,
|
||||||
Publication.pdf_url,
|
Publication.pdf_url,
|
||||||
ScholarPublication.is_read,
|
ScholarPublication.is_read,
|
||||||
|
ScholarPublication.is_favorite,
|
||||||
ScholarPublication.first_seen_run_id,
|
ScholarPublication.first_seen_run_id,
|
||||||
ScholarPublication.created_at,
|
ScholarPublication.created_at,
|
||||||
)
|
)
|
||||||
|
|
@ -60,10 +63,13 @@ def publications_query(
|
||||||
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
.join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
|
||||||
.where(ScholarProfile.user_id == user_id)
|
.where(ScholarProfile.user_id == user_id)
|
||||||
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
|
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
|
||||||
|
.offset(max(int(offset), 0))
|
||||||
.limit(limit)
|
.limit(limit)
|
||||||
)
|
)
|
||||||
if scholar_profile_id is not None:
|
if scholar_profile_id is not None:
|
||||||
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
|
stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
|
||||||
|
if favorite_only:
|
||||||
|
stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
|
||||||
if mode == MODE_UNREAD:
|
if mode == MODE_UNREAD:
|
||||||
stmt = stmt.where(ScholarPublication.is_read.is_(False))
|
stmt = stmt.where(ScholarPublication.is_read.is_(False))
|
||||||
if mode == MODE_LATEST:
|
if mode == MODE_LATEST:
|
||||||
|
|
@ -93,6 +99,7 @@ def publication_query_for_user(
|
||||||
Publication.doi,
|
Publication.doi,
|
||||||
Publication.pdf_url,
|
Publication.pdf_url,
|
||||||
ScholarPublication.is_read,
|
ScholarPublication.is_read,
|
||||||
|
ScholarPublication.is_favorite,
|
||||||
ScholarPublication.first_seen_run_id,
|
ScholarPublication.first_seen_run_id,
|
||||||
ScholarPublication.created_at,
|
ScholarPublication.created_at,
|
||||||
)
|
)
|
||||||
|
|
@ -146,6 +153,7 @@ def publication_list_item_from_row(
|
||||||
doi,
|
doi,
|
||||||
pdf_url,
|
pdf_url,
|
||||||
is_read,
|
is_read,
|
||||||
|
is_favorite,
|
||||||
first_seen_run_id,
|
first_seen_run_id,
|
||||||
created_at,
|
created_at,
|
||||||
) = row
|
) = row
|
||||||
|
|
@ -161,6 +169,7 @@ def publication_list_item_from_row(
|
||||||
doi=doi,
|
doi=doi,
|
||||||
pdf_url=pdf_url,
|
pdf_url=pdf_url,
|
||||||
is_read=bool(is_read),
|
is_read=bool(is_read),
|
||||||
|
is_favorite=bool(is_favorite),
|
||||||
first_seen_at=created_at,
|
first_seen_at=created_at,
|
||||||
is_new_in_latest_run=(
|
is_new_in_latest_run=(
|
||||||
latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
|
latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
|
||||||
|
|
@ -182,6 +191,7 @@ def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
|
||||||
doi,
|
doi,
|
||||||
pdf_url,
|
pdf_url,
|
||||||
_is_read,
|
_is_read,
|
||||||
|
_is_favorite,
|
||||||
_first_seen_run_id,
|
_first_seen_run_id,
|
||||||
_created_at,
|
_created_at,
|
||||||
) = row
|
) = row
|
||||||
|
|
|
||||||
|
|
@ -16,16 +16,20 @@ def _normalized_selection_pairs(selections: list[tuple[int, int]]) -> set[tuple[
|
||||||
return pairs
|
return pairs
|
||||||
|
|
||||||
|
|
||||||
|
def _scoped_scholar_ids_query(*, user_id: int):
|
||||||
|
return (
|
||||||
|
select(ScholarProfile.id)
|
||||||
|
.where(ScholarProfile.user_id == user_id)
|
||||||
|
.scalar_subquery()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def mark_all_unread_as_read_for_user(
|
async def mark_all_unread_as_read_for_user(
|
||||||
db_session: AsyncSession,
|
db_session: AsyncSession,
|
||||||
*,
|
*,
|
||||||
user_id: int,
|
user_id: int,
|
||||||
) -> int:
|
) -> int:
|
||||||
scholar_ids = (
|
scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
|
||||||
select(ScholarProfile.id)
|
|
||||||
.where(ScholarProfile.user_id == user_id)
|
|
||||||
.scalar_subquery()
|
|
||||||
)
|
|
||||||
stmt = (
|
stmt = (
|
||||||
update(ScholarPublication)
|
update(ScholarPublication)
|
||||||
.where(
|
.where(
|
||||||
|
|
@ -49,11 +53,7 @@ async def mark_selected_as_read_for_user(
|
||||||
if not normalized_pairs:
|
if not normalized_pairs:
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
scholar_ids = (
|
scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
|
||||||
select(ScholarProfile.id)
|
|
||||||
.where(ScholarProfile.user_id == user_id)
|
|
||||||
.scalar_subquery()
|
|
||||||
)
|
|
||||||
stmt = (
|
stmt = (
|
||||||
update(ScholarPublication)
|
update(ScholarPublication)
|
||||||
.where(
|
.where(
|
||||||
|
|
@ -69,3 +69,26 @@ async def mark_selected_as_read_for_user(
|
||||||
result = await db_session.execute(stmt)
|
result = await db_session.execute(stmt)
|
||||||
await db_session.commit()
|
await db_session.commit()
|
||||||
return int(result.rowcount or 0)
|
return int(result.rowcount or 0)
|
||||||
|
|
||||||
|
|
||||||
|
async def set_publication_favorite_for_user(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
scholar_profile_id: int,
|
||||||
|
publication_id: int,
|
||||||
|
is_favorite: bool,
|
||||||
|
) -> int:
|
||||||
|
scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
|
||||||
|
stmt = (
|
||||||
|
update(ScholarPublication)
|
||||||
|
.where(
|
||||||
|
ScholarPublication.scholar_profile_id.in_(scholar_ids),
|
||||||
|
ScholarPublication.scholar_profile_id == int(scholar_profile_id),
|
||||||
|
ScholarPublication.publication_id == int(publication_id),
|
||||||
|
)
|
||||||
|
.values(is_favorite=bool(is_favorite))
|
||||||
|
)
|
||||||
|
result = await db_session.execute(stmt)
|
||||||
|
await db_session.commit()
|
||||||
|
return int(result.rowcount or 0)
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,11 @@ class PublicationListItem:
|
||||||
is_read: bool
|
is_read: bool
|
||||||
first_seen_at: datetime
|
first_seen_at: datetime
|
||||||
is_new_in_latest_run: bool
|
is_new_in_latest_run: bool
|
||||||
|
is_favorite: bool = False
|
||||||
|
pdf_status: str = "untracked"
|
||||||
|
pdf_attempt_count: int = 0
|
||||||
|
pdf_failure_reason: str | None = None
|
||||||
|
pdf_failure_detail: str | None = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,10 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from app.services.domains.unpaywall.application import resolve_publication_pdf_urls
|
|
||||||
|
async def resolve_publication_pdf_urls(*args, **kwargs):
|
||||||
|
from app.services.domains.unpaywall.application import resolve_publication_pdf_urls as _impl
|
||||||
|
|
||||||
|
return await _impl(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["resolve_publication_pdf_urls"]
|
__all__ = ["resolve_publication_pdf_urls"]
|
||||||
|
|
|
||||||
|
|
@ -1,21 +1,44 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
from urllib.parse import unquote
|
from urllib.parse import unquote
|
||||||
|
|
||||||
from app.services.domains.crossref.application import discover_doi_for_publication
|
from app.services.domains.crossref.application import discover_doi_for_publication
|
||||||
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
|
||||||
from app.services.domains.doi.normalize import normalize_doi
|
from app.services.domains.doi.normalize import normalize_doi
|
||||||
|
from app.services.domains.unpaywall.pdf_discovery import (
|
||||||
|
looks_like_pdf_url,
|
||||||
|
resolve_pdf_from_landing_page,
|
||||||
|
)
|
||||||
|
from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
|
||||||
|
|
||||||
DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
|
DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
|
||||||
DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
|
DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
|
||||||
DOI_URL_RE = re.compile(r"(?:https?://)?(?:dx\.)?doi\.org/(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
|
DOI_URL_RE = re.compile(r"(?:https?://)?(?:dx\.)?doi\.org/(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I)
|
||||||
UNPAYWALL_URL_TEMPLATE = "https://api.unpaywall.org/v2/{doi}"
|
UNPAYWALL_URL_TEMPLATE = "https://api.unpaywall.org/v2/{doi}"
|
||||||
|
FAILURE_MISSING_DOI = "missing_doi"
|
||||||
|
FAILURE_NO_RECORD = "no_unpaywall_record"
|
||||||
|
FAILURE_NO_PDF = "no_pdf_found"
|
||||||
|
FAILURE_RESOLUTION_EXCEPTION = "resolution_exception"
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class OaResolutionOutcome:
|
||||||
|
publication_id: int
|
||||||
|
doi: str | None
|
||||||
|
pdf_url: str | None
|
||||||
|
failure_reason: str | None
|
||||||
|
source: str | None
|
||||||
|
used_crossref: bool
|
||||||
|
|
||||||
|
|
||||||
def _extract_doi_candidate(text: str | None) -> str | None:
|
def _extract_doi_candidate(text: str | None) -> str | None:
|
||||||
if not text:
|
if not text:
|
||||||
return None
|
return None
|
||||||
|
|
@ -57,21 +80,89 @@ def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str |
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _payload_pdf_url(payload: dict) -> str | None:
|
def _payload_locations(payload: dict) -> list[dict]:
|
||||||
|
locations: list[dict] = []
|
||||||
best = payload.get("best_oa_location")
|
best = payload.get("best_oa_location")
|
||||||
if isinstance(best, dict):
|
if isinstance(best, dict):
|
||||||
pdf_url = best.get("url_for_pdf")
|
locations.append(best)
|
||||||
if isinstance(pdf_url, str) and pdf_url.strip():
|
oa_locations = payload.get("oa_locations")
|
||||||
return pdf_url.strip()
|
if isinstance(oa_locations, list):
|
||||||
locations = payload.get("oa_locations")
|
locations.extend(location for location in oa_locations if isinstance(location, dict))
|
||||||
if not isinstance(locations, list):
|
return locations
|
||||||
|
|
||||||
|
|
||||||
|
def _location_value(location: dict, key: str) -> str | None:
|
||||||
|
value = location.get(key)
|
||||||
|
if not isinstance(value, str):
|
||||||
return None
|
return None
|
||||||
for location in locations:
|
normalized = value.strip()
|
||||||
if not isinstance(location, dict):
|
return normalized or None
|
||||||
|
|
||||||
|
|
||||||
|
def _payload_pdf_candidates(payload: dict) -> list[str]:
|
||||||
|
candidates: list[str] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for location in _payload_locations(payload):
|
||||||
|
candidate = _location_value(location, "url_for_pdf")
|
||||||
|
if candidate is None or candidate in seen:
|
||||||
continue
|
continue
|
||||||
pdf_url = location.get("url_for_pdf")
|
seen.add(candidate)
|
||||||
if isinstance(pdf_url, str) and pdf_url.strip():
|
candidates.append(candidate)
|
||||||
return pdf_url.strip()
|
return candidates
|
||||||
|
|
||||||
|
|
||||||
|
def _payload_landing_candidates(payload: dict) -> list[str]:
|
||||||
|
candidates: list[str] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for location in _payload_locations(payload):
|
||||||
|
candidate = _location_value(location, "url")
|
||||||
|
if candidate is None or candidate in seen:
|
||||||
|
continue
|
||||||
|
seen.add(candidate)
|
||||||
|
candidates.append(candidate)
|
||||||
|
return candidates
|
||||||
|
|
||||||
|
|
||||||
|
def _crawl_targets(
|
||||||
|
*,
|
||||||
|
payload: dict,
|
||||||
|
pdf_candidates: list[str],
|
||||||
|
) -> list[str]:
|
||||||
|
targets = _payload_landing_candidates(payload)
|
||||||
|
seen = set(targets)
|
||||||
|
for candidate in pdf_candidates:
|
||||||
|
if candidate in seen:
|
||||||
|
continue
|
||||||
|
targets.append(candidate)
|
||||||
|
seen.add(candidate)
|
||||||
|
doi = normalize_doi(str(payload.get("doi") or ""))
|
||||||
|
doi_landing_url = f"https://doi.org/{doi}" if doi else None
|
||||||
|
if doi_landing_url and doi_landing_url not in seen:
|
||||||
|
targets.append(doi_landing_url)
|
||||||
|
return targets
|
||||||
|
|
||||||
|
|
||||||
|
def _has_direct_payload_pdf(payload: dict) -> bool:
|
||||||
|
return any(looks_like_pdf_url(candidate) for candidate in _payload_pdf_candidates(payload))
|
||||||
|
|
||||||
|
|
||||||
|
async def _resolved_pdf_url_from_payload(
|
||||||
|
payload: dict,
|
||||||
|
*,
|
||||||
|
client,
|
||||||
|
) -> str | None:
|
||||||
|
pdf_candidates = _payload_pdf_candidates(payload)
|
||||||
|
for candidate in pdf_candidates:
|
||||||
|
if looks_like_pdf_url(candidate):
|
||||||
|
return candidate
|
||||||
|
for page_url in _crawl_targets(payload=payload, pdf_candidates=pdf_candidates)[:3]:
|
||||||
|
discovered = await resolve_pdf_from_landing_page(client, page_url=page_url)
|
||||||
|
if discovered:
|
||||||
|
logger.info(
|
||||||
|
"unpaywall.pdf_discovered_from_landing",
|
||||||
|
extra={"event": "unpaywall.pdf_discovered_from_landing", "landing_url": page_url},
|
||||||
|
)
|
||||||
|
return discovered
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -81,6 +172,7 @@ async def _fetch_unpaywall_payload_by_doi(
|
||||||
doi: str,
|
doi: str,
|
||||||
email: str,
|
email: str,
|
||||||
) -> dict | None:
|
) -> dict | None:
|
||||||
|
await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
|
||||||
response = await client.get(
|
response = await client.get(
|
||||||
UNPAYWALL_URL_TEMPLATE.format(doi=doi),
|
UNPAYWALL_URL_TEMPLATE.format(doi=doi),
|
||||||
params={"email": email},
|
params={"email": email},
|
||||||
|
|
@ -130,7 +222,7 @@ async def _resolve_item_payload(
|
||||||
payload: dict | None = None
|
payload: dict | None = None
|
||||||
if doi:
|
if doi:
|
||||||
payload = await _fetch_unpaywall_payload_by_doi(client=client, doi=doi, email=email)
|
payload = await _fetch_unpaywall_payload_by_doi(client=client, doi=doi, email=email)
|
||||||
if payload is not None and _payload_pdf_url(payload):
|
if payload is not None and _has_direct_payload_pdf(payload):
|
||||||
return payload, False
|
return payload, False
|
||||||
if not allow_crossref or not settings.crossref_enabled:
|
if not allow_crossref or not settings.crossref_enabled:
|
||||||
return payload, False
|
return payload, False
|
||||||
|
|
@ -151,9 +243,13 @@ async def _resolve_item_payload(
|
||||||
return payload, True
|
return payload, True
|
||||||
|
|
||||||
|
|
||||||
def _doi_and_pdf_from_payload(payload: dict) -> tuple[str | None, str | None]:
|
async def _doi_and_pdf_from_payload(
|
||||||
|
payload: dict,
|
||||||
|
*,
|
||||||
|
client,
|
||||||
|
) -> tuple[str | None, str | None]:
|
||||||
doi = normalize_doi(str(payload.get("doi") or ""))
|
doi = normalize_doi(str(payload.get("doi") or ""))
|
||||||
return doi, _payload_pdf_url(payload)
|
return doi, await _resolved_pdf_url_from_payload(payload, client=client)
|
||||||
|
|
||||||
|
|
||||||
def _resolution_targets(items: list[PublicationListItem]) -> list[PublicationListItem]:
|
def _resolution_targets(items: list[PublicationListItem]) -> list[PublicationListItem]:
|
||||||
|
|
@ -164,11 +260,165 @@ def _crossref_budget_value() -> int:
|
||||||
return max(int(settings.crossref_max_lookups_per_request), 0)
|
return max(int(settings.crossref_max_lookups_per_request), 0)
|
||||||
|
|
||||||
|
|
||||||
|
def _outcome_with_failure(
|
||||||
|
*,
|
||||||
|
item: PublicationListItem,
|
||||||
|
failure_reason: str,
|
||||||
|
used_crossref: bool,
|
||||||
|
) -> OaResolutionOutcome:
|
||||||
|
return OaResolutionOutcome(
|
||||||
|
publication_id=item.publication_id,
|
||||||
|
doi=_publication_doi(item),
|
||||||
|
pdf_url=None,
|
||||||
|
failure_reason=failure_reason,
|
||||||
|
source=None,
|
||||||
|
used_crossref=used_crossref,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _missing_payload_failure_reason(item: PublicationListItem, *, used_crossref: bool) -> str:
|
||||||
|
if _publication_doi(item):
|
||||||
|
return FAILURE_NO_RECORD
|
||||||
|
if used_crossref:
|
||||||
|
return FAILURE_NO_RECORD
|
||||||
|
return FAILURE_MISSING_DOI
|
||||||
|
|
||||||
|
|
||||||
|
def _source_name(*, used_crossref: bool) -> str:
|
||||||
|
return "crossref+unpaywall" if used_crossref else "unpaywall"
|
||||||
|
|
||||||
|
|
||||||
|
def _outcome_from_payload(
|
||||||
|
*,
|
||||||
|
item: PublicationListItem,
|
||||||
|
doi: str | None,
|
||||||
|
pdf_url: str | None,
|
||||||
|
used_crossref: bool,
|
||||||
|
) -> OaResolutionOutcome:
|
||||||
|
return OaResolutionOutcome(
|
||||||
|
publication_id=item.publication_id,
|
||||||
|
doi=doi,
|
||||||
|
pdf_url=pdf_url,
|
||||||
|
failure_reason=None if pdf_url else FAILURE_NO_PDF,
|
||||||
|
source=_source_name(used_crossref=used_crossref),
|
||||||
|
used_crossref=used_crossref,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolved_pdf_count(outcomes: dict[int, OaResolutionOutcome]) -> int:
|
||||||
|
return sum(1 for outcome in outcomes.values() if outcome.pdf_url)
|
||||||
|
|
||||||
|
|
||||||
|
def _outcome_metadata(outcomes: dict[int, OaResolutionOutcome]) -> dict[int, tuple[str | None, str | None]]:
|
||||||
|
return {
|
||||||
|
publication_id: (outcome.doi, outcome.pdf_url)
|
||||||
|
for publication_id, outcome in outcomes.items()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _resolve_outcome_for_item(
|
||||||
|
*,
|
||||||
|
client,
|
||||||
|
item: PublicationListItem,
|
||||||
|
email: str,
|
||||||
|
allow_crossref: bool,
|
||||||
|
) -> OaResolutionOutcome:
|
||||||
|
payload, used_crossref = await _resolve_item_payload(
|
||||||
|
client=client,
|
||||||
|
item=item,
|
||||||
|
email=email,
|
||||||
|
allow_crossref=allow_crossref,
|
||||||
|
)
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
return _outcome_with_failure(
|
||||||
|
item=item,
|
||||||
|
failure_reason=_missing_payload_failure_reason(item, used_crossref=used_crossref),
|
||||||
|
used_crossref=used_crossref,
|
||||||
|
)
|
||||||
|
doi, pdf_url = await _doi_and_pdf_from_payload(payload, client=client)
|
||||||
|
return _outcome_from_payload(
|
||||||
|
item=item,
|
||||||
|
doi=doi,
|
||||||
|
pdf_url=pdf_url,
|
||||||
|
used_crossref=used_crossref,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _doi_input_count(items: list[PublicationListItem]) -> int:
|
||||||
|
return len([item for item in items if _publication_doi(item)])
|
||||||
|
|
||||||
|
|
||||||
|
def _search_attempt_count(*, targets: list[PublicationListItem]) -> int:
|
||||||
|
return len([item for item in targets if not _publication_doi(item)])
|
||||||
|
|
||||||
|
|
||||||
|
async def _safe_outcome_for_item(
|
||||||
|
*,
|
||||||
|
client,
|
||||||
|
item: PublicationListItem,
|
||||||
|
email: str,
|
||||||
|
allow_crossref: bool,
|
||||||
|
) -> OaResolutionOutcome:
|
||||||
|
try:
|
||||||
|
return await _resolve_outcome_for_item(
|
||||||
|
client=client,
|
||||||
|
item=item,
|
||||||
|
email=email,
|
||||||
|
allow_crossref=allow_crossref,
|
||||||
|
)
|
||||||
|
except Exception as exc: # pragma: no cover - defensive network boundary
|
||||||
|
logger.warning(
|
||||||
|
"unpaywall.resolve_item_failed",
|
||||||
|
extra={
|
||||||
|
"event": "unpaywall.resolve_item_failed",
|
||||||
|
"publication_id": item.publication_id,
|
||||||
|
"error": str(exc),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return _outcome_with_failure(
|
||||||
|
item=item,
|
||||||
|
failure_reason=FAILURE_RESOLUTION_EXCEPTION,
|
||||||
|
used_crossref=allow_crossref and settings.crossref_enabled,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _resolve_outcomes_with_client(
|
||||||
|
*,
|
||||||
|
client,
|
||||||
|
targets: list[PublicationListItem],
|
||||||
|
email: str,
|
||||||
|
) -> dict[int, OaResolutionOutcome]:
|
||||||
|
outcomes: dict[int, OaResolutionOutcome] = {}
|
||||||
|
crossref_budget = _crossref_budget_value()
|
||||||
|
crossref_lookups = 0
|
||||||
|
for item in targets:
|
||||||
|
allow_crossref = crossref_budget > 0 and crossref_lookups < crossref_budget
|
||||||
|
outcome = await _safe_outcome_for_item(
|
||||||
|
client=client,
|
||||||
|
item=item,
|
||||||
|
email=email,
|
||||||
|
allow_crossref=allow_crossref,
|
||||||
|
)
|
||||||
|
if outcome.used_crossref:
|
||||||
|
crossref_lookups += 1
|
||||||
|
outcomes[item.publication_id] = outcome
|
||||||
|
return outcomes
|
||||||
|
|
||||||
|
|
||||||
async def resolve_publication_oa_metadata(
|
async def resolve_publication_oa_metadata(
|
||||||
items: list[PublicationListItem],
|
items: list[PublicationListItem],
|
||||||
*,
|
*,
|
||||||
request_email: str | None = None,
|
request_email: str | None = None,
|
||||||
) -> dict[int, tuple[str | None, str | None]]:
|
) -> dict[int, tuple[str | None, str | None]]:
|
||||||
|
outcomes = await resolve_publication_oa_outcomes(items, request_email=request_email)
|
||||||
|
return _outcome_metadata(outcomes)
|
||||||
|
|
||||||
|
|
||||||
|
async def resolve_publication_oa_outcomes(
|
||||||
|
items: list[PublicationListItem],
|
||||||
|
*,
|
||||||
|
request_email: str | None = None,
|
||||||
|
) -> dict[int, OaResolutionOutcome]:
|
||||||
if not settings.unpaywall_enabled:
|
if not settings.unpaywall_enabled:
|
||||||
return {}
|
return {}
|
||||||
email = _email_for_request(request_email)
|
email = _email_for_request(request_email)
|
||||||
|
|
@ -178,35 +428,21 @@ async def resolve_publication_oa_metadata(
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
timeout_seconds = max(float(settings.unpaywall_timeout_seconds), 0.5)
|
timeout_seconds = max(float(settings.unpaywall_timeout_seconds), 0.5)
|
||||||
resolved: dict[int, tuple[str | None, str | None]] = {}
|
|
||||||
crossref_budget = _crossref_budget_value()
|
|
||||||
crossref_lookups = 0
|
|
||||||
targets = _resolution_targets(items)[: max(int(settings.unpaywall_max_items_per_request), 0)]
|
targets = _resolution_targets(items)[: max(int(settings.unpaywall_max_items_per_request), 0)]
|
||||||
async with httpx.AsyncClient(timeout=timeout_seconds) as client:
|
async with httpx.AsyncClient(timeout=timeout_seconds) as client:
|
||||||
for item in targets:
|
outcomes = await _resolve_outcomes_with_client(
|
||||||
allow_crossref = crossref_budget > 0 and crossref_lookups < crossref_budget
|
client=client,
|
||||||
payload, used_crossref = await _resolve_item_payload(
|
targets=targets,
|
||||||
client=client,
|
email=email,
|
||||||
item=item,
|
)
|
||||||
email=email,
|
|
||||||
allow_crossref=allow_crossref,
|
|
||||||
)
|
|
||||||
if used_crossref:
|
|
||||||
crossref_lookups += 1
|
|
||||||
if not isinstance(payload, dict):
|
|
||||||
continue
|
|
||||||
resolved[item.publication_id] = _doi_and_pdf_from_payload(payload)
|
|
||||||
resolved_count = sum(1 for _doi, pdf in resolved.values() if pdf)
|
|
||||||
doi_input_count = len([item for item in items if _publication_doi(item)])
|
|
||||||
target_doi_count = len([item for item in targets if _publication_doi(item)])
|
|
||||||
_log_resolution_summary(
|
_log_resolution_summary(
|
||||||
publication_count=len(items),
|
publication_count=len(items),
|
||||||
doi_input_count=doi_input_count,
|
doi_input_count=_doi_input_count(items),
|
||||||
search_attempt_count=max(0, len(targets) - target_doi_count),
|
search_attempt_count=_search_attempt_count(targets=targets),
|
||||||
resolved_pdf_count=resolved_count,
|
resolved_pdf_count=_resolved_pdf_count(outcomes),
|
||||||
email=email,
|
email=email,
|
||||||
)
|
)
|
||||||
return resolved
|
return outcomes
|
||||||
|
|
||||||
|
|
||||||
async def resolve_publication_pdf_urls(
|
async def resolve_publication_pdf_urls(
|
||||||
|
|
|
||||||
156
app/services/domains/unpaywall/pdf_discovery.py
Normal file
|
|
@ -0,0 +1,156 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
import re
|
||||||
|
from urllib.parse import urljoin, urlparse
|
||||||
|
|
||||||
|
from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
|
||||||
|
from app.settings import settings
|
||||||
|
|
||||||
|
PDF_MIME = "application/pdf"
|
||||||
|
URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.I)
|
||||||
|
|
||||||
|
|
||||||
|
class _LandingPdfParser(HTMLParser):
|
||||||
|
def __init__(self) -> None:
|
||||||
|
super().__init__(convert_charrefs=True)
|
||||||
|
self.base_href: str | None = None
|
||||||
|
self.candidates: list[str] = []
|
||||||
|
|
||||||
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||||
|
attrs_map = {key.lower(): (value or "").strip() for key, value in attrs}
|
||||||
|
if tag == "base":
|
||||||
|
self.base_href = attrs_map.get("href") or self.base_href
|
||||||
|
return
|
||||||
|
if tag == "meta":
|
||||||
|
self._append_meta_candidate(attrs_map)
|
||||||
|
return
|
||||||
|
if tag == "link":
|
||||||
|
self._append_link_candidate(attrs_map)
|
||||||
|
return
|
||||||
|
if tag == "a":
|
||||||
|
self._append_anchor_candidate(attrs_map)
|
||||||
|
|
||||||
|
def _append_meta_candidate(self, attrs_map: dict[str, str]) -> None:
|
||||||
|
meta_name = (attrs_map.get("name") or attrs_map.get("property") or "").lower()
|
||||||
|
if meta_name != "citation_pdf_url":
|
||||||
|
return
|
||||||
|
content = attrs_map.get("content")
|
||||||
|
if content:
|
||||||
|
self.candidates.append(content)
|
||||||
|
|
||||||
|
def _append_link_candidate(self, attrs_map: dict[str, str]) -> None:
|
||||||
|
href = attrs_map.get("href")
|
||||||
|
link_type = (attrs_map.get("type") or "").lower()
|
||||||
|
if href and "pdf" in link_type:
|
||||||
|
self.candidates.append(href)
|
||||||
|
|
||||||
|
def _append_anchor_candidate(self, attrs_map: dict[str, str]) -> None:
|
||||||
|
href = attrs_map.get("href")
|
||||||
|
if href:
|
||||||
|
self.candidates.append(href)
|
||||||
|
|
||||||
|
|
||||||
|
def looks_like_pdf_url(url: str | None) -> bool:
|
||||||
|
if not isinstance(url, str):
|
||||||
|
return False
|
||||||
|
value = url.strip()
|
||||||
|
if not value:
|
||||||
|
return False
|
||||||
|
parsed = urlparse(value)
|
||||||
|
path = (parsed.path or "").lower()
|
||||||
|
query = (parsed.query or "").lower()
|
||||||
|
return path.endswith(".pdf") or ".pdf" in query
|
||||||
|
|
||||||
|
|
||||||
|
def _normalized_candidate_urls(*, page_url: str, html: str) -> list[str]:
|
||||||
|
parser = _LandingPdfParser()
|
||||||
|
parser.feed(html)
|
||||||
|
parser.close()
|
||||||
|
base_url = urljoin(page_url, parser.base_href) if parser.base_href else page_url
|
||||||
|
raw_candidates = [*parser.candidates, *_text_url_candidates(html)]
|
||||||
|
deduped: list[str] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for raw in raw_candidates:
|
||||||
|
absolute = urljoin(base_url, raw.strip())
|
||||||
|
parsed = urlparse(absolute)
|
||||||
|
if parsed.scheme not in {"http", "https"}:
|
||||||
|
continue
|
||||||
|
if absolute in seen:
|
||||||
|
continue
|
||||||
|
seen.add(absolute)
|
||||||
|
deduped.append(absolute)
|
||||||
|
return sorted(deduped, key=_candidate_sort_key)
|
||||||
|
|
||||||
|
|
||||||
|
def _text_url_candidates(html: str) -> list[str]:
|
||||||
|
candidates: list[str] = []
|
||||||
|
for match in URL_RE.findall(html):
|
||||||
|
cleaned = match.rstrip(".,);]}>")
|
||||||
|
if "pdf" not in cleaned.lower():
|
||||||
|
continue
|
||||||
|
candidates.append(cleaned)
|
||||||
|
return candidates
|
||||||
|
|
||||||
|
|
||||||
|
def _candidate_sort_key(candidate: str) -> tuple[int, str]:
|
||||||
|
lowered = candidate.lower()
|
||||||
|
if looks_like_pdf_url(candidate):
|
||||||
|
return (0, lowered)
|
||||||
|
if any(token in lowered for token in ("doi", "full", "article", "download")):
|
||||||
|
return (1, lowered)
|
||||||
|
return (2, lowered)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_html_response(response) -> bool:
|
||||||
|
content_type = str(response.headers.get("content-type") or "").lower()
|
||||||
|
return "text/html" in content_type or "application/xhtml+xml" in content_type
|
||||||
|
|
||||||
|
|
||||||
|
async def _fetch_page_html(client, *, page_url: str) -> str | None:
|
||||||
|
await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
|
||||||
|
response = await client.get(page_url, follow_redirects=True)
|
||||||
|
if response.status_code != 200 or not _is_html_response(response):
|
||||||
|
return None
|
||||||
|
text = response.text or ""
|
||||||
|
return text[: max(int(settings.unpaywall_pdf_discovery_max_html_bytes), 0)]
|
||||||
|
|
||||||
|
|
||||||
|
async def _candidate_is_pdf(client, *, candidate_url: str) -> bool:
|
||||||
|
if looks_like_pdf_url(candidate_url):
|
||||||
|
return True
|
||||||
|
await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds)
|
||||||
|
response = await client.get(candidate_url, follow_redirects=True)
|
||||||
|
content_type = str(response.headers.get("content-type") or "").lower()
|
||||||
|
return response.status_code == 200 and PDF_MIME in content_type
|
||||||
|
|
||||||
|
|
||||||
|
def _candidate_limit() -> int:
|
||||||
|
return max(int(settings.unpaywall_pdf_discovery_max_candidates), 1)
|
||||||
|
|
||||||
|
|
||||||
|
async def _resolve_pdf_from_candidate_page(client, *, candidate_url: str) -> str | None:
|
||||||
|
html = await _fetch_page_html(client, page_url=candidate_url)
|
||||||
|
if not html:
|
||||||
|
return None
|
||||||
|
nested_candidates = _normalized_candidate_urls(page_url=candidate_url, html=html)
|
||||||
|
for nested in nested_candidates[: _candidate_limit()]:
|
||||||
|
if await _candidate_is_pdf(client, candidate_url=nested):
|
||||||
|
return nested
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def resolve_pdf_from_landing_page(client, *, page_url: str) -> str | None:
|
||||||
|
if not settings.unpaywall_pdf_discovery_enabled:
|
||||||
|
return None
|
||||||
|
html = await _fetch_page_html(client, page_url=page_url)
|
||||||
|
if not html:
|
||||||
|
return None
|
||||||
|
candidates = _normalized_candidate_urls(page_url=page_url, html=html)
|
||||||
|
for candidate in candidates[: _candidate_limit()]:
|
||||||
|
if await _candidate_is_pdf(client, candidate_url=candidate):
|
||||||
|
return candidate
|
||||||
|
nested_pdf = await _resolve_pdf_from_candidate_page(client, candidate_url=candidate)
|
||||||
|
if nested_pdf:
|
||||||
|
return nested_pdf
|
||||||
|
return None
|
||||||
18
app/services/domains/unpaywall/rate_limit.py
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
|
||||||
|
_REQUEST_LOCK = asyncio.Lock()
|
||||||
|
_LAST_REQUEST_AT = 0.0
|
||||||
|
|
||||||
|
|
||||||
|
async def wait_for_unpaywall_slot(*, min_interval_seconds: float) -> None:
|
||||||
|
global _LAST_REQUEST_AT
|
||||||
|
interval = max(float(min_interval_seconds), 0.0)
|
||||||
|
async with _REQUEST_LOCK:
|
||||||
|
elapsed = time.monotonic() - _LAST_REQUEST_AT
|
||||||
|
remaining = interval - elapsed
|
||||||
|
if remaining > 0:
|
||||||
|
await asyncio.sleep(remaining)
|
||||||
|
_LAST_REQUEST_AT = time.monotonic()
|
||||||
|
|
@ -232,8 +232,21 @@ class Settings:
|
||||||
unpaywall_enabled: bool = _env_bool("UNPAYWALL_ENABLED", True)
|
unpaywall_enabled: bool = _env_bool("UNPAYWALL_ENABLED", True)
|
||||||
unpaywall_email: str = _env_str("UNPAYWALL_EMAIL", "")
|
unpaywall_email: str = _env_str("UNPAYWALL_EMAIL", "")
|
||||||
unpaywall_timeout_seconds: float = _env_float("UNPAYWALL_TIMEOUT_SECONDS", 4.0)
|
unpaywall_timeout_seconds: float = _env_float("UNPAYWALL_TIMEOUT_SECONDS", 4.0)
|
||||||
|
unpaywall_min_interval_seconds: float = _env_float("UNPAYWALL_MIN_INTERVAL_SECONDS", 0.6)
|
||||||
unpaywall_max_items_per_request: int = _env_int("UNPAYWALL_MAX_ITEMS_PER_REQUEST", 20)
|
unpaywall_max_items_per_request: int = _env_int("UNPAYWALL_MAX_ITEMS_PER_REQUEST", 20)
|
||||||
unpaywall_retry_cooldown_seconds: int = _env_int("UNPAYWALL_RETRY_COOLDOWN_SECONDS", 1800)
|
unpaywall_retry_cooldown_seconds: int = _env_int("UNPAYWALL_RETRY_COOLDOWN_SECONDS", 1800)
|
||||||
|
pdf_auto_retry_interval_seconds: int = _env_int(
|
||||||
|
"PDF_AUTO_RETRY_INTERVAL_SECONDS",
|
||||||
|
86_400,
|
||||||
|
)
|
||||||
|
pdf_auto_retry_first_interval_seconds: int = _env_int(
|
||||||
|
"PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS",
|
||||||
|
3_600,
|
||||||
|
)
|
||||||
|
pdf_auto_retry_max_attempts: int = _env_int("PDF_AUTO_RETRY_MAX_ATTEMPTS", 3)
|
||||||
|
unpaywall_pdf_discovery_enabled: bool = _env_bool("UNPAYWALL_PDF_DISCOVERY_ENABLED", True)
|
||||||
|
unpaywall_pdf_discovery_max_candidates: int = _env_int("UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES", 5)
|
||||||
|
unpaywall_pdf_discovery_max_html_bytes: int = _env_int("UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES", 500_000)
|
||||||
crossref_enabled: bool = _env_bool("CROSSREF_ENABLED", True)
|
crossref_enabled: bool = _env_bool("CROSSREF_ENABLED", True)
|
||||||
crossref_max_rows: int = _env_int("CROSSREF_MAX_ROWS", 10)
|
crossref_max_rows: int = _env_int("CROSSREF_MAX_ROWS", 10)
|
||||||
crossref_timeout_seconds: float = _env_float("CROSSREF_TIMEOUT_SECONDS", 8.0)
|
crossref_timeout_seconds: float = _env_float("CROSSREF_TIMEOUT_SECONDS", 8.0)
|
||||||
|
|
|
||||||
30
docs/README.md
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
# Documentation Index
|
||||||
|
|
||||||
|
Use this directory for project documentation outside `README.md` and `agents.md`.
|
||||||
|
|
||||||
|
## Deploy
|
||||||
|
|
||||||
|
- `docs/deploy/quickstart.md`: deployment modes, local dev startup, and quality command reference.
|
||||||
|
|
||||||
|
## Reference
|
||||||
|
|
||||||
|
- `docs/reference/environment.md`: complete environment-variable catalog and contract/audit notes.
|
||||||
|
- `docs/reference/api_contract.md`: API envelope contract and endpoint semantics.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
- `docs/architecture/domain_boundaries.md`: domain boundaries, user-scoped data rules, and frontend behavior notes.
|
||||||
|
|
||||||
|
## Operations
|
||||||
|
|
||||||
|
- `docs/ops/scrape_safety_runbook.md`: scrape cooldown and blocked-IP operations.
|
||||||
|
- `docs/ops/db_runbook.md`: backup, restore, and verification workflows.
|
||||||
|
- `docs/ops/migration_checklist.md`: migration planning and rollout checklist.
|
||||||
|
|
||||||
|
## Frontend
|
||||||
|
|
||||||
|
- `docs/frontend/theme_phase0_inventory.md`: theme-token inventory and adoption notes.
|
||||||
|
|
||||||
|
## Contribution
|
||||||
|
|
||||||
|
- `docs/contributing.md`: contribution policy and merge checklist.
|
||||||
25
docs/architecture/domain_boundaries.md
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
# Domain Boundaries
|
||||||
|
|
||||||
|
## Data Model Rules
|
||||||
|
|
||||||
|
- Scholar tracking is user-scoped.
|
||||||
|
- Publications are global/deduplicated records.
|
||||||
|
- Read/favorite/visibility state stays on scholar-publication link rows.
|
||||||
|
|
||||||
|
## Service Boundaries
|
||||||
|
|
||||||
|
Canonical business logic belongs in `app/services/domains/*`.
|
||||||
|
|
||||||
|
- `app/services/domains/ingestion/*`: run orchestration, continuation queue, scheduler, and scrape safety.
|
||||||
|
- `app/services/domains/scholar/*`: fail-fast scholar parsing and source fetch adapters.
|
||||||
|
- `app/services/domains/scholars/*`: scholar CRUD, profile image, and name-search controls.
|
||||||
|
- `app/services/domains/publications/*`: listing/read-state, favorite toggles, enrichment scheduling, and retry paths.
|
||||||
|
- `app/services/domains/crossref/*` + `app/services/domains/unpaywall/*`: DOI/OA enrichment with bounded pacing.
|
||||||
|
- `app/services/domains/runs/*`: run history and continuation queue operations.
|
||||||
|
- `app/services/domains/portability/*`: import/export workflows.
|
||||||
|
|
||||||
|
## Frontend Behavior Notes
|
||||||
|
|
||||||
|
- Mobile primary nav is in a left drawer and closes on route change or logout.
|
||||||
|
- Long list views use internal scroll containers.
|
||||||
|
- Name search remains intentionally constrained due upstream anti-bot behavior; production onboarding should prefer scholar ID/profile URL.
|
||||||
68
docs/deploy/quickstart.md
Normal file
|
|
@ -0,0 +1,68 @@
|
||||||
|
# Deploy and Dev Quickstart
|
||||||
|
|
||||||
|
## Required Setup
|
||||||
|
|
||||||
|
1. Copy `.env.example` to `.env`.
|
||||||
|
2. Set `POSTGRES_PASSWORD`.
|
||||||
|
3. Set `SESSION_SECRET_KEY`.
|
||||||
|
|
||||||
|
## Deploy Method A: Prebuilt Image
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose pull
|
||||||
|
docker compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
Open:
|
||||||
|
- App/API: `http://localhost:8000`
|
||||||
|
- Health endpoint: `http://localhost:8000/healthz`
|
||||||
|
|
||||||
|
Upgrade:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose pull
|
||||||
|
docker compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
## Deploy Method B: Local Source + Dev Frontend
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
|
||||||
|
```
|
||||||
|
|
||||||
|
Open:
|
||||||
|
- API: `http://localhost:8000`
|
||||||
|
- Frontend dev server: `http://localhost:5173`
|
||||||
|
|
||||||
|
Stop:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose -f docker-compose.yml -f docker-compose.dev.yml down
|
||||||
|
```
|
||||||
|
|
||||||
|
## Quality Commands
|
||||||
|
|
||||||
|
Backend:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest tests/unit
|
||||||
|
docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest -m integration
|
||||||
|
```
|
||||||
|
|
||||||
|
Frontend:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd frontend
|
||||||
|
npm install
|
||||||
|
npm run typecheck
|
||||||
|
npm run test:run
|
||||||
|
npm run build
|
||||||
|
```
|
||||||
|
|
||||||
|
Repository checks:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 scripts/check_frontend_api_contract.py
|
||||||
|
python3 scripts/check_env_contract.py
|
||||||
|
./scripts/check_no_generated_artifacts.sh
|
||||||
|
```
|
||||||
12
docs/index.md
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
---
|
||||||
|
slug: /
|
||||||
|
---
|
||||||
|
|
||||||
|
# scholarr Documentation
|
||||||
|
|
||||||
|
- Start here: `docs/deploy/quickstart.md`
|
||||||
|
- Runtime config: `docs/reference/environment.md`
|
||||||
|
- API contract: `docs/reference/api_contract.md`
|
||||||
|
- Domain boundaries: `docs/architecture/domain_boundaries.md`
|
||||||
|
- Ops runbooks: `docs/ops/`
|
||||||
|
- Frontend theme inventory: `docs/frontend/theme_phase0_inventory.md`
|
||||||
100
docs/ops/db_runbook.md
Normal file
|
|
@ -0,0 +1,100 @@
|
||||||
|
# Database Operations Runbook
|
||||||
|
|
||||||
|
This runbook defines backup, restore, and verification procedures for the
|
||||||
|
PostgreSQL database used by `scholarr`.
|
||||||
|
|
||||||
|
## Objectives
|
||||||
|
|
||||||
|
- Keep data loss bounded via scheduled backups.
|
||||||
|
- Provide repeatable restore procedures.
|
||||||
|
- Verify backups by running regular restore drills.
|
||||||
|
|
||||||
|
Recommended starting targets:
|
||||||
|
|
||||||
|
- `RPO`: 24 hours (maximum acceptable data loss).
|
||||||
|
- `RTO`: 60 minutes (maximum acceptable restore time).
|
||||||
|
|
||||||
|
## Backup Strategy
|
||||||
|
|
||||||
|
Use logical backups from the running `db` compose service.
|
||||||
|
|
||||||
|
- Custom-format backup (recommended for restore flexibility):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/db/backup_full.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
- Plain SQL backup:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/db/backup_full.sh --plain
|
||||||
|
```
|
||||||
|
|
||||||
|
Optional environment variables:
|
||||||
|
|
||||||
|
- `BACKUP_DIR`: destination directory (default: `<repo>/backups`)
|
||||||
|
- `BACKUP_PREFIX`: backup filename prefix (default: `scholarr`)
|
||||||
|
- `USE_DEV_COMPOSE=1`: include `docker-compose.dev.yml`
|
||||||
|
|
||||||
|
## Restore Strategy
|
||||||
|
|
||||||
|
Restore from a `.dump` (custom format) or `.sql` file into the running `db`
|
||||||
|
compose service.
|
||||||
|
|
||||||
|
- Restore without schema wipe:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/db/restore_dump.sh --file backups/scholarr_20260220T120000Z.dump
|
||||||
|
```
|
||||||
|
|
||||||
|
- Restore with full `public` schema reset:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
scripts/db/restore_dump.sh --file backups/scholarr_20260220T120000Z.dump --wipe-public
|
||||||
|
```
|
||||||
|
|
||||||
|
## Safety Checklist Before Restore
|
||||||
|
|
||||||
|
1. Pause writes (stop app/scheduler or set maintenance mode).
|
||||||
|
2. Create a fresh pre-restore backup.
|
||||||
|
3. Validate target dump checksum/size.
|
||||||
|
4. Confirm operator, scope, and rollback plan.
|
||||||
|
|
||||||
|
## Post-Restore Verification
|
||||||
|
|
||||||
|
1. Run migrations head check.
|
||||||
|
2. Run health endpoint checks.
|
||||||
|
3. Verify table row counts for core tables:
|
||||||
|
- `users`
|
||||||
|
- `scholar_profiles`
|
||||||
|
- `publications`
|
||||||
|
- `scholar_publications`
|
||||||
|
- `crawl_runs`
|
||||||
|
4. Run integrity report and require zero failures:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 scripts/db/check_integrity.py
|
||||||
|
```
|
||||||
|
|
||||||
|
5. Run API smoke checks for scholars/publications/runs pages.
|
||||||
|
|
||||||
|
## Data Cleanup and Repair
|
||||||
|
|
||||||
|
Use the audited repair job for scholar-publication relinking cleanup:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 scripts/db/repair_publication_links.py --user-id <id> --requested-by "<operator>" --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
Dry-run first, then re-run with `--apply` once scope and summary counts match expectation.
|
||||||
|
|
||||||
|
If cleanup changes schema assumptions, follow `docs/ops/migration_checklist.md` before any migration rollout.
|
||||||
|
|
||||||
|
## Restore Drill Cadence
|
||||||
|
|
||||||
|
- Run at least one full restore drill per month.
|
||||||
|
- Record:
|
||||||
|
- backup artifact used,
|
||||||
|
- restore start/end timestamps,
|
||||||
|
- issues encountered,
|
||||||
|
- achieved `RTO`.
|
||||||
48
docs/ops/migration_checklist.md
Normal file
|
|
@ -0,0 +1,48 @@
|
||||||
|
# Migration Checklist
|
||||||
|
|
||||||
|
Use this checklist for every schema/data migration.
|
||||||
|
|
||||||
|
## 1. Design Review
|
||||||
|
|
||||||
|
- Define change type: `expand`, `backfill`, `contract`.
|
||||||
|
- Document expected lock behavior and index impact.
|
||||||
|
- Confirm backward compatibility with currently deployed app version.
|
||||||
|
- Define rollback strategy before implementation.
|
||||||
|
|
||||||
|
## 2. Pre-Migration Controls
|
||||||
|
|
||||||
|
- Capture a fresh backup before applying migration.
|
||||||
|
- Confirm migration head revision and working tree cleanliness.
|
||||||
|
- Prepare validation queries for new/changed tables and indexes.
|
||||||
|
- Identify high-risk tables (large row count, hot write paths).
|
||||||
|
|
||||||
|
## 3. Implementation Standards
|
||||||
|
|
||||||
|
- Keep migrations idempotent when feasible.
|
||||||
|
- Prefer additive steps first (`nullable`, new index, new table).
|
||||||
|
- For destructive changes, separate into later contract migration.
|
||||||
|
- Avoid large blocking rewrites in a single deployment step.
|
||||||
|
|
||||||
|
## 4. Verification
|
||||||
|
|
||||||
|
- Apply migration in staging against production-like snapshot.
|
||||||
|
- Verify:
|
||||||
|
- expected tables/columns/indexes,
|
||||||
|
- app startup and health endpoint,
|
||||||
|
- affected API flows,
|
||||||
|
- data consistency queries.
|
||||||
|
|
||||||
|
## 5. Rollout and Recovery
|
||||||
|
|
||||||
|
- Apply migration during planned window.
|
||||||
|
- Monitor logs/errors and DB metrics during rollout.
|
||||||
|
- If rollback needed:
|
||||||
|
- follow downgrade/recovery runbook,
|
||||||
|
- restore from backup if downgrade is unsafe,
|
||||||
|
- document incident timeline.
|
||||||
|
|
||||||
|
## 6. Post-Migration Tasks
|
||||||
|
|
||||||
|
- Update `README.md` / `.env.example` / ops docs.
|
||||||
|
- Add/update integration tests for new schema assumptions.
|
||||||
|
- Record migration notes in changelog.
|
||||||
29
docs/reference/api_contract.md
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
# API Contract
|
||||||
|
|
||||||
|
## Envelope Invariant
|
||||||
|
|
||||||
|
All API responses under `/api/v1` use one of these envelopes:
|
||||||
|
|
||||||
|
- Success: `{"data": ..., "meta": {"request_id": "..."}}`
|
||||||
|
- Error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}`
|
||||||
|
|
||||||
|
`meta.request_id` must be present for success and error responses.
|
||||||
|
|
||||||
|
## Publications Semantics
|
||||||
|
|
||||||
|
- `GET /api/v1/publications` supports `mode=all|unread|latest`.
|
||||||
|
- `mode=new` is currently accepted as a compatibility alias for `latest`.
|
||||||
|
- Pagination controls: `page`, `page_size` (with backward-compatible `limit`/`offset` support).
|
||||||
|
- Pagination fields in response: `page`, `page_size`, `has_prev`, `has_next`, `total_count`.
|
||||||
|
- `unread` represents read-state (`is_read=false`).
|
||||||
|
- `latest` represents discovery-state (`first seen in the latest completed run`).
|
||||||
|
|
||||||
|
Publication payloads expose:
|
||||||
|
- `pub_url` (canonical scholar detail URL)
|
||||||
|
- `doi` (normalized DOI)
|
||||||
|
- `pdf_url` (resolved OA PDF when available)
|
||||||
|
|
||||||
|
## Scholar Portability
|
||||||
|
|
||||||
|
- `GET /api/v1/scholars/export` exports tracked scholars and scholar-publication link state.
|
||||||
|
- `POST /api/v1/scholars/import` imports that payload while preserving global publication deduplication.
|
||||||
68
docs/reference/environment.md
Normal file
|
|
@ -0,0 +1,68 @@
|
||||||
|
# Environment Reference and Audit
|
||||||
|
|
||||||
|
This document is the source-of-truth audit for what remains configurable versus internal defaults.
|
||||||
|
|
||||||
|
## Decision Summary
|
||||||
|
|
||||||
|
- Keep in `.env.example`: all runtime and compose controls currently exposed by `app/settings.py` plus compose/bootstrap extras.
|
||||||
|
- Move to internal defaults only: none at this time.
|
||||||
|
- Internal fallback behavior still exists in code to keep local/test startup resilient when values are omitted.
|
||||||
|
|
||||||
|
## Configurable Variables (By Section)
|
||||||
|
|
||||||
|
### Compose + Database
|
||||||
|
|
||||||
|
`POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `DATABASE_URL`, `TEST_DATABASE_URL`, `SCHOLARR_IMAGE`
|
||||||
|
|
||||||
|
### App Runtime + Networking
|
||||||
|
|
||||||
|
`APP_NAME`, `APP_HOST`, `APP_PORT`, `APP_HOST_PORT`, `APP_RELOAD`, `MIGRATE_ON_START`, `FRONTEND_ENABLED`, `FRONTEND_DIST_DIR`
|
||||||
|
|
||||||
|
### Database Pool
|
||||||
|
|
||||||
|
`DATABASE_POOL_MODE`, `DATABASE_POOL_SIZE`, `DATABASE_POOL_MAX_OVERFLOW`, `DATABASE_POOL_TIMEOUT_SECONDS`
|
||||||
|
|
||||||
|
### Frontend Dev Overrides
|
||||||
|
|
||||||
|
`FRONTEND_HOST_PORT`, `CHOKIDAR_USEPOLLING`, `VITE_DEV_API_PROXY_TARGET`
|
||||||
|
|
||||||
|
### Auth + Session
|
||||||
|
|
||||||
|
`SESSION_SECRET_KEY`, `SESSION_COOKIE_SECURE`, `LOGIN_RATE_LIMIT_ATTEMPTS`, `LOGIN_RATE_LIMIT_WINDOW_SECONDS`
|
||||||
|
|
||||||
|
### HTTP Security Headers + CSP
|
||||||
|
|
||||||
|
`SECURITY_HEADERS_ENABLED`, `SECURITY_X_CONTENT_TYPE_OPTIONS`, `SECURITY_X_FRAME_OPTIONS`, `SECURITY_REFERRER_POLICY`, `SECURITY_PERMISSIONS_POLICY`, `SECURITY_CROSS_ORIGIN_OPENER_POLICY`, `SECURITY_CROSS_ORIGIN_RESOURCE_POLICY`, `SECURITY_CSP_ENABLED`, `SECURITY_CSP_POLICY`, `SECURITY_CSP_DOCS_POLICY`, `SECURITY_CSP_REPORT_ONLY`, `SECURITY_STRICT_TRANSPORT_SECURITY_ENABLED`, `SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE`, `SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS`, `SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD`
|
||||||
|
|
||||||
|
### Logging
|
||||||
|
|
||||||
|
`LOG_LEVEL`, `LOG_FORMAT`, `LOG_REQUESTS`, `LOG_UVICORN_ACCESS`, `LOG_REQUEST_SKIP_PATHS`, `LOG_REDACT_FIELDS`
|
||||||
|
|
||||||
|
### Scheduler + Ingestion Safety
|
||||||
|
|
||||||
|
`SCHEDULER_ENABLED`, `SCHEDULER_TICK_SECONDS`, `SCHEDULER_QUEUE_BATCH_SIZE`, `INGESTION_AUTOMATION_ALLOWED`, `INGESTION_MANUAL_RUN_ALLOWED`, `INGESTION_MIN_RUN_INTERVAL_MINUTES`, `INGESTION_MIN_REQUEST_DELAY_SECONDS`, `INGESTION_NETWORK_ERROR_RETRIES`, `INGESTION_RETRY_BACKOFF_SECONDS`, `INGESTION_MAX_PAGES_PER_SCHOLAR`, `INGESTION_PAGE_SIZE`, `INGESTION_ALERT_BLOCKED_FAILURE_THRESHOLD`, `INGESTION_ALERT_NETWORK_FAILURE_THRESHOLD`, `INGESTION_ALERT_RETRY_SCHEDULED_THRESHOLD`, `INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS`, `INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS`, `INGESTION_CONTINUATION_QUEUE_ENABLED`, `INGESTION_CONTINUATION_BASE_DELAY_SECONDS`, `INGESTION_CONTINUATION_MAX_DELAY_SECONDS`, `INGESTION_CONTINUATION_MAX_ATTEMPTS`
|
||||||
|
|
||||||
|
### Scholar Images + Name Search Safety
|
||||||
|
|
||||||
|
`SCHOLAR_IMAGE_UPLOAD_DIR`, `SCHOLAR_IMAGE_UPLOAD_MAX_BYTES`, `SCHOLAR_NAME_SEARCH_ENABLED`, `SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS`, `SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS`, `SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES`, `SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS`, `SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS`, `SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD`, `SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS`, `SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD`, `SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD`
|
||||||
|
|
||||||
|
### OA Enrichment + PDF Resolution
|
||||||
|
|
||||||
|
`UNPAYWALL_ENABLED`, `UNPAYWALL_EMAIL`, `UNPAYWALL_TIMEOUT_SECONDS`, `UNPAYWALL_MIN_INTERVAL_SECONDS`, `UNPAYWALL_MAX_ITEMS_PER_REQUEST`, `UNPAYWALL_RETRY_COOLDOWN_SECONDS`, `UNPAYWALL_PDF_DISCOVERY_ENABLED`, `UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES`, `UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES`, `PDF_AUTO_RETRY_INTERVAL_SECONDS`, `PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS`, `PDF_AUTO_RETRY_MAX_ATTEMPTS`, `CROSSREF_ENABLED`, `CROSSREF_MAX_ROWS`, `CROSSREF_TIMEOUT_SECONDS`, `CROSSREF_MIN_INTERVAL_SECONDS`, `CROSSREF_MAX_LOOKUPS_PER_REQUEST`
|
||||||
|
|
||||||
|
### Startup Bootstrap + DB Wait
|
||||||
|
|
||||||
|
`BOOTSTRAP_ADMIN_ON_START`, `BOOTSTRAP_ADMIN_EMAIL`, `BOOTSTRAP_ADMIN_PASSWORD`, `BOOTSTRAP_ADMIN_FORCE_PASSWORD`, `DB_WAIT_TIMEOUT_SECONDS`, `DB_WAIT_INTERVAL_SECONDS`
|
||||||
|
|
||||||
|
## Drift Guard
|
||||||
|
|
||||||
|
CI enforces env parity with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 scripts/check_env_contract.py
|
||||||
|
```
|
||||||
|
|
||||||
|
The check fails when:
|
||||||
|
- `app/settings.py` references a variable missing from `.env.example`.
|
||||||
|
- `.env.example` contains an unknown key (outside the approved compose/bootstrap extras).
|
||||||
|
- `.env.example` contains duplicate keys.
|
||||||
|
|
@ -3,6 +3,12 @@
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8" />
|
<meta charset="UTF-8" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="theme-color" content="#495e6f" />
|
||||||
|
<link rel="icon" type="image/x-icon" href="/favicon.ico" />
|
||||||
|
<link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png" />
|
||||||
|
<link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png" />
|
||||||
|
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png" />
|
||||||
|
<link rel="manifest" href="/site.webmanifest" />
|
||||||
<title>scholarr</title>
|
<title>scholarr</title>
|
||||||
<script>
|
<script>
|
||||||
(function () {
|
(function () {
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@
|
||||||
"build": "vue-tsc --noEmit && vite build",
|
"build": "vue-tsc --noEmit && vite build",
|
||||||
"preview": "vite preview --host 0.0.0.0 --port 4173",
|
"preview": "vite preview --host 0.0.0.0 --port 4173",
|
||||||
"typecheck": "vue-tsc --noEmit",
|
"typecheck": "vue-tsc --noEmit",
|
||||||
|
"generate:brand-assets": "./scripts/generate_brand_assets.sh",
|
||||||
"check:theme-tokens": "node ./scripts/check_theme_tokens.mjs",
|
"check:theme-tokens": "node ./scripts/check_theme_tokens.mjs",
|
||||||
"test": "vitest",
|
"test": "vitest",
|
||||||
"test:run": "vitest run"
|
"test:run": "vitest run"
|
||||||
|
|
|
||||||
BIN
frontend/public/android-chrome-192x192.png
Normal file
|
After Width: | Height: | Size: 9.4 KiB |
BIN
frontend/public/android-chrome-512x512.png
Normal file
|
After Width: | Height: | Size: 40 KiB |
BIN
frontend/public/apple-touch-icon.png
Normal file
|
After Width: | Height: | Size: 8.7 KiB |
BIN
frontend/public/favicon-16x16.png
Normal file
|
After Width: | Height: | Size: 580 B |
BIN
frontend/public/favicon-32x32.png
Normal file
|
After Width: | Height: | Size: 990 B |
BIN
frontend/public/favicon.ico
Normal file
|
After Width: | Height: | Size: 15 KiB |
21
frontend/public/site.webmanifest
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
{
|
||||||
|
"name": "scholarr",
|
||||||
|
"short_name": "scholarr",
|
||||||
|
"description": "Track scholar publications and open-access PDF availability.",
|
||||||
|
"icons": [
|
||||||
|
{
|
||||||
|
"src": "/android-chrome-192x192.png",
|
||||||
|
"sizes": "192x192",
|
||||||
|
"type": "image/png"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"src": "/android-chrome-512x512.png",
|
||||||
|
"sizes": "512x512",
|
||||||
|
"type": "image/png",
|
||||||
|
"purpose": "any maskable"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"theme_color": "#495e6f",
|
||||||
|
"background_color": "#f2eee9",
|
||||||
|
"display": "standalone"
|
||||||
|
}
|
||||||
56
frontend/scripts/generate_brand_assets.sh
Executable file
|
|
@ -0,0 +1,56 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
FRONTEND_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||||
|
SRC_LOGO="${FRONTEND_DIR}/src/logo.png"
|
||||||
|
PUBLIC_DIR="${FRONTEND_DIR}/public"
|
||||||
|
TMP_DIR="$(mktemp -d)"
|
||||||
|
TMP_ALPHA="${TMP_DIR}/logo-alpha-mask.png"
|
||||||
|
TMP_GLYPH="${TMP_DIR}/logo-glyph-white.png"
|
||||||
|
TMP_CANVAS="${TMP_DIR}/logo-canvas.png"
|
||||||
|
TMP_ICON="${TMP_DIR}/logo-icon.png"
|
||||||
|
|
||||||
|
BRAND_COLOR="${1:-#495e6f}"
|
||||||
|
|
||||||
|
if ! command -v magick >/dev/null 2>&1; then
|
||||||
|
echo "ImageMagick is required (missing 'magick' command)." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! -f "${SRC_LOGO}" ]]; then
|
||||||
|
echo "Missing source logo at ${SRC_LOGO}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p "${PUBLIC_DIR}"
|
||||||
|
|
||||||
|
# Build a high-contrast icon: solid brand background + white logo.
|
||||||
|
magick "${SRC_LOGO}" -alpha extract "${TMP_ALPHA}"
|
||||||
|
magick -size 1024x1024 xc:white "${TMP_ALPHA}" \
|
||||||
|
-compose CopyOpacity \
|
||||||
|
-composite \
|
||||||
|
"${TMP_GLYPH}"
|
||||||
|
|
||||||
|
magick -size 1024x1024 "xc:${BRAND_COLOR}" "${TMP_CANVAS}"
|
||||||
|
magick "${TMP_GLYPH}" -resize 760x760 "${TMP_GLYPH}"
|
||||||
|
magick "${TMP_CANVAS}" "${TMP_GLYPH}" \
|
||||||
|
-gravity center \
|
||||||
|
-compose over \
|
||||||
|
-composite \
|
||||||
|
"${TMP_ICON}"
|
||||||
|
|
||||||
|
magick "${TMP_ICON}" -resize 16x16 "${PUBLIC_DIR}/favicon-16x16.png"
|
||||||
|
magick "${TMP_ICON}" -resize 32x32 "${PUBLIC_DIR}/favicon-32x32.png"
|
||||||
|
magick "${TMP_ICON}" -resize 180x180 "${PUBLIC_DIR}/apple-touch-icon.png"
|
||||||
|
magick "${TMP_ICON}" -resize 192x192 "${PUBLIC_DIR}/android-chrome-192x192.png"
|
||||||
|
magick "${TMP_ICON}" -resize 512x512 "${PUBLIC_DIR}/android-chrome-512x512.png"
|
||||||
|
magick \
|
||||||
|
\( "${TMP_ICON}" -resize 16x16 \) \
|
||||||
|
\( "${TMP_ICON}" -resize 32x32 \) \
|
||||||
|
\( "${TMP_ICON}" -resize 48x48 \) \
|
||||||
|
"${PUBLIC_DIR}/favicon.ico"
|
||||||
|
|
||||||
|
rm -rf "${TMP_DIR}"
|
||||||
|
|
||||||
|
echo "Generated favicon + app icon assets in ${PUBLIC_DIR}"
|
||||||
|
|
@ -8,7 +8,6 @@ import PublicationsPage from "@/pages/PublicationsPage.vue";
|
||||||
import RunsPage from "@/pages/RunsPage.vue";
|
import RunsPage from "@/pages/RunsPage.vue";
|
||||||
import RunDetailPage from "@/pages/RunDetailPage.vue";
|
import RunDetailPage from "@/pages/RunDetailPage.vue";
|
||||||
import SettingsPage from "@/pages/SettingsPage.vue";
|
import SettingsPage from "@/pages/SettingsPage.vue";
|
||||||
import AdminUsersPage from "@/pages/AdminUsersPage.vue";
|
|
||||||
import StyleGuidePage from "@/pages/StyleGuidePage.vue";
|
import StyleGuidePage from "@/pages/StyleGuidePage.vue";
|
||||||
|
|
||||||
export const router = createRouter({
|
export const router = createRouter({
|
||||||
|
|
@ -66,12 +65,6 @@ export const router = createRouter({
|
||||||
component: RunDetailPage,
|
component: RunDetailPage,
|
||||||
meta: { requiresAuth: true, requiresAdmin: true },
|
meta: { requiresAuth: true, requiresAdmin: true },
|
||||||
},
|
},
|
||||||
{
|
|
||||||
path: "/admin/users",
|
|
||||||
name: "admin-users",
|
|
||||||
component: AdminUsersPage,
|
|
||||||
meta: { requiresAuth: true, requiresAdmin: true },
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
path: "/:pathMatch(.*)*",
|
path: "/:pathMatch(.*)*",
|
||||||
redirect: "/dashboard",
|
redirect: "/dashboard",
|
||||||
|
|
|
||||||
|
|
@ -2,9 +2,8 @@
|
||||||
import { computed } from "vue";
|
import { computed } from "vue";
|
||||||
|
|
||||||
import AppButton from "@/components/ui/AppButton.vue";
|
import AppButton from "@/components/ui/AppButton.vue";
|
||||||
import AppSelect from "@/components/ui/AppSelect.vue";
|
import AppBrandMark from "@/components/ui/AppBrandMark.vue";
|
||||||
import { useThemeStore } from "@/stores/theme";
|
import AppThemePicker from "@/components/ui/AppThemePicker.vue";
|
||||||
import type { ThemePresetId } from "@/theme/presets";
|
|
||||||
|
|
||||||
const props = withDefaults(
|
const props = withDefaults(
|
||||||
defineProps<{
|
defineProps<{
|
||||||
|
|
@ -20,25 +19,9 @@ const emit = defineEmits<{
|
||||||
(e: "toggle-menu"): void;
|
(e: "toggle-menu"): void;
|
||||||
}>();
|
}>();
|
||||||
|
|
||||||
const theme = useThemeStore();
|
|
||||||
|
|
||||||
const isDarkTheme = computed(() => theme.active === "dark");
|
|
||||||
const toggleThemeLabel = computed(() =>
|
|
||||||
isDarkTheme.value ? "Switch to light theme" : "Switch to dark theme",
|
|
||||||
);
|
|
||||||
const menuButtonLabel = computed(() =>
|
const menuButtonLabel = computed(() =>
|
||||||
props.menuOpen ? "Close navigation menu" : "Open navigation menu",
|
props.menuOpen ? "Close navigation menu" : "Open navigation menu",
|
||||||
);
|
);
|
||||||
const selectedPreset = computed<ThemePresetId>({
|
|
||||||
get: () => theme.preset,
|
|
||||||
set: (value) => theme.setPreset(value),
|
|
||||||
});
|
|
||||||
const presetOptions = computed(() => theme.availablePresets);
|
|
||||||
const themePresetLabel = computed(() => theme.presetLabel);
|
|
||||||
|
|
||||||
function onToggleTheme(): void {
|
|
||||||
theme.setPreference(isDarkTheme.value ? "light" : "dark");
|
|
||||||
}
|
|
||||||
|
|
||||||
function onToggleMenu(): void {
|
function onToggleMenu(): void {
|
||||||
emit("toggle-menu");
|
emit("toggle-menu");
|
||||||
|
|
@ -75,64 +58,14 @@ function onToggleMenu(): void {
|
||||||
</AppButton>
|
</AppButton>
|
||||||
<RouterLink
|
<RouterLink
|
||||||
to="/dashboard"
|
to="/dashboard"
|
||||||
class="rounded-sm font-display text-lg tracking-tight text-ink-primary transition hover:text-ink-secondary focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset"
|
class="inline-flex items-center gap-2.5 rounded-sm font-display text-xl tracking-tight text-ink-primary transition hover:text-ink-secondary focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset"
|
||||||
>
|
>
|
||||||
scholarr
|
<AppBrandMark size="lg" />
|
||||||
|
<span>scholarr</span>
|
||||||
</RouterLink>
|
</RouterLink>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="flex items-center justify-end gap-2">
|
<AppThemePicker id-prefix="header-theme" />
|
||||||
<div class="w-36 sm:w-44">
|
|
||||||
<label for="theme-preset-select" class="sr-only">Theme preset</label>
|
|
||||||
<AppSelect
|
|
||||||
id="theme-preset-select"
|
|
||||||
v-model="selectedPreset"
|
|
||||||
:disabled="presetOptions.length <= 1"
|
|
||||||
:title="`Theme preset: ${themePresetLabel}`"
|
|
||||||
class="py-1.5"
|
|
||||||
>
|
|
||||||
<option v-for="preset in presetOptions" :key="preset.id" :value="preset.id">
|
|
||||||
{{ preset.label }}
|
|
||||||
</option>
|
|
||||||
</AppSelect>
|
|
||||||
</div>
|
|
||||||
<AppButton
|
|
||||||
variant="ghost"
|
|
||||||
class="h-10 w-10 rounded-full p-0"
|
|
||||||
:aria-label="toggleThemeLabel"
|
|
||||||
:title="toggleThemeLabel"
|
|
||||||
@click="onToggleTheme"
|
|
||||||
>
|
|
||||||
<span class="sr-only">{{ toggleThemeLabel }}</span>
|
|
||||||
<svg
|
|
||||||
v-if="isDarkTheme"
|
|
||||||
xmlns="http://www.w3.org/2000/svg"
|
|
||||||
viewBox="0 0 24 24"
|
|
||||||
fill="none"
|
|
||||||
stroke="currentColor"
|
|
||||||
stroke-width="1.8"
|
|
||||||
class="h-5 w-5"
|
|
||||||
aria-hidden="true"
|
|
||||||
>
|
|
||||||
<circle cx="12" cy="12" r="4" />
|
|
||||||
<path
|
|
||||||
d="M12 2v2.5M12 19.5V22M4.9 4.9l1.8 1.8M17.3 17.3l1.8 1.8M2 12h2.5M19.5 12H22M4.9 19.1l1.8-1.8M17.3 6.7l1.8-1.8"
|
|
||||||
/>
|
|
||||||
</svg>
|
|
||||||
<svg
|
|
||||||
v-else
|
|
||||||
xmlns="http://www.w3.org/2000/svg"
|
|
||||||
viewBox="0 0 24 24"
|
|
||||||
fill="none"
|
|
||||||
stroke="currentColor"
|
|
||||||
stroke-width="1.8"
|
|
||||||
class="h-5 w-5"
|
|
||||||
aria-hidden="true"
|
|
||||||
>
|
|
||||||
<path d="M21 14.5A8.5 8.5 0 1 1 9.5 3 7 7 0 0 0 21 14.5z" />
|
|
||||||
</svg>
|
|
||||||
</AppButton>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
</header>
|
</header>
|
||||||
</template>
|
</template>
|
||||||
|
|
|
||||||
|
|
@ -19,22 +19,12 @@ const emit = defineEmits<{
|
||||||
}>();
|
}>();
|
||||||
|
|
||||||
const links = computed(() => {
|
const links = computed(() => {
|
||||||
const base = [
|
return [
|
||||||
{ id: "dashboard", to: "/dashboard", label: "Dashboard", adminOnly: false },
|
{ id: "dashboard", to: "/dashboard", label: "Dashboard" },
|
||||||
{ id: "scholars", to: "/scholars", label: "Scholars", adminOnly: false },
|
{ id: "scholars", to: "/scholars", label: "Scholars" },
|
||||||
{ id: "publications", to: "/publications", label: "Publications", adminOnly: false },
|
{ id: "publications", to: "/publications", label: "Publications" },
|
||||||
{ id: "settings", to: "/settings", label: "Settings", adminOnly: false },
|
{ id: "settings", to: "/settings", label: "Settings" },
|
||||||
{ id: "style-guide", to: "/admin/style-guide", label: "Style Guide", adminOnly: true },
|
|
||||||
{ id: "runs", to: "/admin/runs", label: "Runs", adminOnly: true },
|
|
||||||
{ id: "users", to: "/admin/users", label: "Users", adminOnly: true },
|
|
||||||
];
|
];
|
||||||
|
|
||||||
return base.filter((item) => {
|
|
||||||
if (item.adminOnly && !auth.isAdmin) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return userSettings.isPageVisible(item.id);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
const navSafetyText = computed(() => {
|
const navSafetyText = computed(() => {
|
||||||
if (!userSettings.manualRunAllowed) {
|
if (!userSettings.manualRunAllowed) {
|
||||||
|
|
|
||||||
55
frontend/src/components/ui/AppBrandMark.vue
Normal file
|
|
@ -0,0 +1,55 @@
|
||||||
|
<script setup lang="ts">
|
||||||
|
import { computed } from "vue";
|
||||||
|
|
||||||
|
import brandLogo from "@/logo.png";
|
||||||
|
|
||||||
|
const props = withDefaults(
|
||||||
|
defineProps<{
|
||||||
|
size?: "sm" | "md" | "lg" | "xl";
|
||||||
|
}>(),
|
||||||
|
{
|
||||||
|
size: "md",
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const sizeClass = computed(() => {
|
||||||
|
if (props.size === "sm") {
|
||||||
|
return "h-5 w-5";
|
||||||
|
}
|
||||||
|
if (props.size === "lg") {
|
||||||
|
return "h-8 w-8";
|
||||||
|
}
|
||||||
|
if (props.size === "xl") {
|
||||||
|
return "h-12 w-12";
|
||||||
|
}
|
||||||
|
return "h-6 w-6";
|
||||||
|
});
|
||||||
|
|
||||||
|
const logoMaskStyle: Record<string, string> = {
|
||||||
|
"--brand-logo-mask": `url(${brandLogo})`,
|
||||||
|
};
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<span
|
||||||
|
:class="['brand-logo-mark', sizeClass]"
|
||||||
|
:style="logoMaskStyle"
|
||||||
|
aria-hidden="true"
|
||||||
|
/>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.brand-logo-mark {
|
||||||
|
display: inline-block;
|
||||||
|
flex-shrink: 0;
|
||||||
|
background-color: rgb(var(--color-brand-700));
|
||||||
|
-webkit-mask-image: var(--brand-logo-mask);
|
||||||
|
mask-image: var(--brand-logo-mask);
|
||||||
|
-webkit-mask-position: center;
|
||||||
|
mask-position: center;
|
||||||
|
-webkit-mask-repeat: no-repeat;
|
||||||
|
mask-repeat: no-repeat;
|
||||||
|
-webkit-mask-size: contain;
|
||||||
|
mask-size: contain;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
61
frontend/src/components/ui/AppRefreshButton.vue
Normal file
|
|
@ -0,0 +1,61 @@
|
||||||
|
<script setup lang="ts">
|
||||||
|
import { computed } from "vue";
|
||||||
|
|
||||||
|
import AppButton from "@/components/ui/AppButton.vue";
|
||||||
|
|
||||||
|
const props = withDefaults(
|
||||||
|
defineProps<{
|
||||||
|
disabled?: boolean;
|
||||||
|
loading?: boolean;
|
||||||
|
title?: string;
|
||||||
|
loadingTitle?: string;
|
||||||
|
variant?: "primary" | "secondary" | "ghost" | "danger";
|
||||||
|
size?: "md" | "sm";
|
||||||
|
}>(),
|
||||||
|
{
|
||||||
|
disabled: false,
|
||||||
|
loading: false,
|
||||||
|
title: "Refresh",
|
||||||
|
loadingTitle: "Refreshing",
|
||||||
|
variant: "secondary",
|
||||||
|
size: "md",
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const resolvedTitle = computed(() => (props.loading ? props.loadingTitle : props.title));
|
||||||
|
const isDisabled = computed(() => props.disabled || props.loading);
|
||||||
|
const buttonSizeClass = computed(() => {
|
||||||
|
if (props.size === "sm") {
|
||||||
|
return "min-h-8 h-8 w-8";
|
||||||
|
}
|
||||||
|
return "min-h-10 h-10 w-10";
|
||||||
|
});
|
||||||
|
const iconSizeClass = computed(() => (props.size === "sm" ? "h-3.5 w-3.5" : "h-4 w-4"));
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<AppButton
|
||||||
|
:variant="props.variant"
|
||||||
|
:disabled="isDisabled"
|
||||||
|
class="rounded-full p-0"
|
||||||
|
:class="buttonSizeClass"
|
||||||
|
:title="resolvedTitle"
|
||||||
|
:aria-label="resolvedTitle"
|
||||||
|
>
|
||||||
|
<span class="sr-only">{{ resolvedTitle }}</span>
|
||||||
|
<svg
|
||||||
|
xmlns="http://www.w3.org/2000/svg"
|
||||||
|
viewBox="0 0 24 24"
|
||||||
|
fill="none"
|
||||||
|
stroke="currentColor"
|
||||||
|
stroke-width="1.8"
|
||||||
|
:class="[iconSizeClass, props.loading ? 'animate-spin' : '']"
|
||||||
|
aria-hidden="true"
|
||||||
|
>
|
||||||
|
<path d="M20 4v6h-6" />
|
||||||
|
<path d="M4 20v-6h6" />
|
||||||
|
<path d="M6.5 9A7.5 7.5 0 0 1 19 7.5L20 10" />
|
||||||
|
<path d="M17.5 15A7.5 7.5 0 0 1 5 16.5L4 14" />
|
||||||
|
</svg>
|
||||||
|
</AppButton>
|
||||||
|
</template>
|
||||||
51
frontend/src/components/ui/AppTabs.vue
Normal file
|
|
@ -0,0 +1,51 @@
|
||||||
|
<script setup lang="ts">
|
||||||
|
export interface AppTabItem {
|
||||||
|
id: string;
|
||||||
|
label: string;
|
||||||
|
disabled?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
const props = defineProps<{
|
||||||
|
modelValue: string;
|
||||||
|
items: AppTabItem[];
|
||||||
|
ariaLabel?: string;
|
||||||
|
}>();
|
||||||
|
|
||||||
|
const emit = defineEmits<{
|
||||||
|
(e: "update:modelValue", value: string): void;
|
||||||
|
}>();
|
||||||
|
|
||||||
|
function onSelect(tabId: string, disabled: boolean | undefined): void {
|
||||||
|
if (disabled || tabId === props.modelValue) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
emit("update:modelValue", tabId);
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div
|
||||||
|
class="flex flex-wrap gap-2 rounded-lg border border-stroke-default bg-surface-card-muted p-2"
|
||||||
|
role="tablist"
|
||||||
|
:aria-label="props.ariaLabel || 'Tabs'"
|
||||||
|
>
|
||||||
|
<button
|
||||||
|
v-for="item in props.items"
|
||||||
|
:key="item.id"
|
||||||
|
type="button"
|
||||||
|
role="tab"
|
||||||
|
:aria-selected="item.id === props.modelValue"
|
||||||
|
:tabindex="item.id === props.modelValue ? 0 : -1"
|
||||||
|
:disabled="item.disabled"
|
||||||
|
class="inline-flex min-h-9 items-center rounded-md border px-3 py-1.5 text-sm font-medium transition focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset disabled:cursor-not-allowed disabled:opacity-60"
|
||||||
|
:class="
|
||||||
|
item.id === props.modelValue
|
||||||
|
? 'border-stroke-interactive bg-surface-nav-active text-ink-primary'
|
||||||
|
: 'border-stroke-default bg-surface-card text-ink-secondary hover:border-stroke-interactive hover:text-ink-primary'
|
||||||
|
"
|
||||||
|
@click="onSelect(item.id, item.disabled)"
|
||||||
|
>
|
||||||
|
{{ item.label }}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
95
frontend/src/components/ui/AppThemePicker.vue
Normal file
|
|
@ -0,0 +1,95 @@
|
||||||
|
<script setup lang="ts">
|
||||||
|
import { computed } from "vue";
|
||||||
|
|
||||||
|
import AppButton from "@/components/ui/AppButton.vue";
|
||||||
|
import AppSelect from "@/components/ui/AppSelect.vue";
|
||||||
|
import { useThemeStore } from "@/stores/theme";
|
||||||
|
import type { ThemePresetId } from "@/theme/presets";
|
||||||
|
|
||||||
|
const props = withDefaults(
|
||||||
|
defineProps<{
|
||||||
|
compact?: boolean;
|
||||||
|
idPrefix?: string;
|
||||||
|
}>(),
|
||||||
|
{
|
||||||
|
compact: false,
|
||||||
|
idPrefix: "theme",
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const theme = useThemeStore();
|
||||||
|
const isDarkTheme = computed(() => theme.active === "dark");
|
||||||
|
const toggleThemeLabel = computed(() =>
|
||||||
|
isDarkTheme.value ? "Switch to light theme" : "Switch to dark theme",
|
||||||
|
);
|
||||||
|
const selectedPreset = computed<ThemePresetId>({
|
||||||
|
get: () => theme.preset,
|
||||||
|
set: (value) => theme.setPreset(value),
|
||||||
|
});
|
||||||
|
const presetOptions = computed(() => theme.availablePresets);
|
||||||
|
const themePresetLabel = computed(() => theme.presetLabel);
|
||||||
|
const selectId = computed(() => `${props.idPrefix}-preset-select`);
|
||||||
|
const selectWidthClass = computed(() => (props.compact ? "w-32 sm:w-36" : "w-36 sm:w-44"));
|
||||||
|
const toggleButtonClass = computed(() =>
|
||||||
|
props.compact ? "h-9 w-9 rounded-full p-0" : "h-10 w-10 rounded-full p-0",
|
||||||
|
);
|
||||||
|
|
||||||
|
function onToggleTheme(): void {
|
||||||
|
theme.setPreference(isDarkTheme.value ? "light" : "dark");
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div class="flex items-center justify-end gap-2">
|
||||||
|
<div :class="selectWidthClass">
|
||||||
|
<label :for="selectId" class="sr-only">Theme preset</label>
|
||||||
|
<AppSelect
|
||||||
|
:id="selectId"
|
||||||
|
v-model="selectedPreset"
|
||||||
|
:disabled="presetOptions.length <= 1"
|
||||||
|
:title="`Theme preset: ${themePresetLabel}`"
|
||||||
|
class="py-1.5"
|
||||||
|
>
|
||||||
|
<option v-for="preset in presetOptions" :key="preset.id" :value="preset.id">
|
||||||
|
{{ preset.label }}
|
||||||
|
</option>
|
||||||
|
</AppSelect>
|
||||||
|
</div>
|
||||||
|
<AppButton
|
||||||
|
variant="ghost"
|
||||||
|
:class="toggleButtonClass"
|
||||||
|
:aria-label="toggleThemeLabel"
|
||||||
|
:title="toggleThemeLabel"
|
||||||
|
@click="onToggleTheme"
|
||||||
|
>
|
||||||
|
<span class="sr-only">{{ toggleThemeLabel }}</span>
|
||||||
|
<svg
|
||||||
|
v-if="isDarkTheme"
|
||||||
|
xmlns="http://www.w3.org/2000/svg"
|
||||||
|
viewBox="0 0 24 24"
|
||||||
|
fill="none"
|
||||||
|
stroke="currentColor"
|
||||||
|
stroke-width="1.8"
|
||||||
|
class="h-5 w-5"
|
||||||
|
aria-hidden="true"
|
||||||
|
>
|
||||||
|
<circle cx="12" cy="12" r="4" />
|
||||||
|
<path
|
||||||
|
d="M12 2v2.5M12 19.5V22M4.9 4.9l1.8 1.8M17.3 17.3l1.8 1.8M2 12h2.5M19.5 12H22M4.9 19.1l1.8-1.8M17.3 6.7l1.8-1.8"
|
||||||
|
/>
|
||||||
|
</svg>
|
||||||
|
<svg
|
||||||
|
v-else
|
||||||
|
xmlns="http://www.w3.org/2000/svg"
|
||||||
|
viewBox="0 0 24 24"
|
||||||
|
fill="none"
|
||||||
|
stroke="currentColor"
|
||||||
|
stroke-width="1.8"
|
||||||
|
class="h-5 w-5"
|
||||||
|
aria-hidden="true"
|
||||||
|
>
|
||||||
|
<path d="M21 14.5A8.5 8.5 0 1 1 9.5 3 7 7 0 0 0 21 14.5z" />
|
||||||
|
</svg>
|
||||||
|
</AppButton>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
153
frontend/src/features/admin_dbops/index.ts
Normal file
|
|
@ -0,0 +1,153 @@
|
||||||
|
import { apiRequest } from "@/lib/api/client";
|
||||||
|
|
||||||
|
export interface AdminDbIntegrityCheck {
|
||||||
|
name: string;
|
||||||
|
count: number;
|
||||||
|
severity: "warning" | "failure" | "metric";
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AdminDbIntegrityReport {
|
||||||
|
status: "ok" | "warning" | "failed";
|
||||||
|
checked_at: string;
|
||||||
|
failures: string[];
|
||||||
|
warnings: string[];
|
||||||
|
checks: AdminDbIntegrityCheck[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AdminDbRepairJob {
|
||||||
|
id: number;
|
||||||
|
job_name: string;
|
||||||
|
requested_by: string | null;
|
||||||
|
dry_run: boolean;
|
||||||
|
status: string;
|
||||||
|
scope: Record<string, unknown>;
|
||||||
|
summary: Record<string, unknown>;
|
||||||
|
error_text: string | null;
|
||||||
|
started_at: string | null;
|
||||||
|
finished_at: string | null;
|
||||||
|
created_at: string;
|
||||||
|
updated_at: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AdminPdfQueueItem {
|
||||||
|
publication_id: number;
|
||||||
|
title: string;
|
||||||
|
doi: string | null;
|
||||||
|
pdf_url: string | null;
|
||||||
|
status: string;
|
||||||
|
attempt_count: number;
|
||||||
|
last_failure_reason: string | null;
|
||||||
|
last_failure_detail: string | null;
|
||||||
|
last_source: string | null;
|
||||||
|
requested_by_user_id: number | null;
|
||||||
|
requested_by_email: string | null;
|
||||||
|
queued_at: string | null;
|
||||||
|
last_attempt_at: string | null;
|
||||||
|
resolved_at: string | null;
|
||||||
|
updated_at: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AdminPdfQueuePage {
|
||||||
|
items: AdminPdfQueueItem[];
|
||||||
|
total_count: number;
|
||||||
|
page: number;
|
||||||
|
page_size: number;
|
||||||
|
has_next: boolean;
|
||||||
|
has_prev: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AdminPdfQueueRequeueResult {
|
||||||
|
publication_id: number;
|
||||||
|
queued: boolean;
|
||||||
|
status: string;
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AdminPdfQueueBulkEnqueueResult {
|
||||||
|
requested_count: number;
|
||||||
|
queued_count: number;
|
||||||
|
message: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TriggerPublicationLinkRepairPayload {
|
||||||
|
scope_mode?: "single_user" | "all_users";
|
||||||
|
user_id?: number;
|
||||||
|
scholar_profile_ids?: number[];
|
||||||
|
dry_run?: boolean;
|
||||||
|
gc_orphan_publications?: boolean;
|
||||||
|
requested_by?: string;
|
||||||
|
confirmation_text?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TriggerPublicationLinkRepairResult {
|
||||||
|
job_id: number;
|
||||||
|
status: string;
|
||||||
|
scope: Record<string, unknown>;
|
||||||
|
summary: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getAdminDbIntegrityReport(): Promise<AdminDbIntegrityReport> {
|
||||||
|
const response = await apiRequest<AdminDbIntegrityReport>("/admin/db/integrity", { method: "GET" });
|
||||||
|
return response.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listAdminDbRepairJobs(limit = 30): Promise<AdminDbRepairJob[]> {
|
||||||
|
const parsedLimit = Number.isFinite(limit) ? Math.max(1, Math.min(200, Math.trunc(limit))) : 30;
|
||||||
|
const response = await apiRequest<{ jobs: AdminDbRepairJob[] }>(
|
||||||
|
`/admin/db/repair-jobs?limit=${parsedLimit}`,
|
||||||
|
{ method: "GET" },
|
||||||
|
);
|
||||||
|
return response.data.jobs;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function triggerPublicationLinkRepair(
|
||||||
|
payload: TriggerPublicationLinkRepairPayload,
|
||||||
|
): Promise<TriggerPublicationLinkRepairResult> {
|
||||||
|
const response = await apiRequest<TriggerPublicationLinkRepairResult>(
|
||||||
|
"/admin/db/repairs/publication-links",
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
body: payload,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
return response.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listAdminPdfQueue(
|
||||||
|
page = 1,
|
||||||
|
pageSize = 100,
|
||||||
|
status: string | null = null,
|
||||||
|
): Promise<AdminPdfQueuePage> {
|
||||||
|
const parsedPage = Number.isFinite(page) ? Math.max(1, Math.trunc(page)) : 1;
|
||||||
|
const parsedPageSize = Number.isFinite(pageSize) ? Math.max(1, Math.min(500, Math.trunc(pageSize))) : 100;
|
||||||
|
const params = new URLSearchParams();
|
||||||
|
params.set("page", String(parsedPage));
|
||||||
|
params.set("page_size", String(parsedPageSize));
|
||||||
|
if (status && status.trim().length > 0) {
|
||||||
|
params.set("status", status.trim().toLowerCase());
|
||||||
|
}
|
||||||
|
const response = await apiRequest<AdminPdfQueuePage>(
|
||||||
|
`/admin/db/pdf-queue?${params.toString()}`,
|
||||||
|
{ method: "GET" },
|
||||||
|
);
|
||||||
|
return response.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function requeueAdminPdfLookup(publicationId: number): Promise<AdminPdfQueueRequeueResult> {
|
||||||
|
const id = Number.isFinite(publicationId) ? Math.trunc(publicationId) : 0;
|
||||||
|
const response = await apiRequest<AdminPdfQueueRequeueResult>(
|
||||||
|
`/admin/db/pdf-queue/${Math.max(1, id)}/requeue`,
|
||||||
|
{ method: "POST" },
|
||||||
|
);
|
||||||
|
return response.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function requeueAllAdminPdfLookups(limit = 1000): Promise<AdminPdfQueueBulkEnqueueResult> {
|
||||||
|
const parsedLimit = Number.isFinite(limit) ? Math.max(1, Math.min(5000, Math.trunc(limit))) : 1000;
|
||||||
|
const response = await apiRequest<AdminPdfQueueBulkEnqueueResult>(
|
||||||
|
`/admin/db/pdf-queue/requeue-all?limit=${parsedLimit}`,
|
||||||
|
{ method: "POST" },
|
||||||
|
);
|
||||||
|
return response.data;
|
||||||
|
}
|
||||||
|
|
@ -41,7 +41,7 @@ function countQueueStatuses(statuses: string[]): QueueHealth {
|
||||||
|
|
||||||
export async function fetchDashboardSnapshot(): Promise<DashboardSnapshot> {
|
export async function fetchDashboardSnapshot(): Promise<DashboardSnapshot> {
|
||||||
const [publications, runsPayload, queueItems] = await Promise.all([
|
const [publications, runsPayload, queueItems] = await Promise.all([
|
||||||
listPublications({ mode: "latest", limit: 20 }),
|
listPublications({ mode: "latest", page: 1, pageSize: 20 }),
|
||||||
listRuns({ limit: 20 }),
|
listRuns({ limit: 20 }),
|
||||||
listQueueItems(200),
|
listQueueItems(200),
|
||||||
]);
|
]);
|
||||||
|
|
|
||||||
|
|
@ -13,26 +13,39 @@ export interface PublicationItem {
|
||||||
pub_url: string | null;
|
pub_url: string | null;
|
||||||
doi: string | null;
|
doi: string | null;
|
||||||
pdf_url: string | null;
|
pdf_url: string | null;
|
||||||
|
pdf_status: "untracked" | "queued" | "running" | "resolved" | "failed";
|
||||||
|
pdf_attempt_count: number;
|
||||||
|
pdf_failure_reason: string | null;
|
||||||
|
pdf_failure_detail: string | null;
|
||||||
is_read: boolean;
|
is_read: boolean;
|
||||||
|
is_favorite: boolean;
|
||||||
first_seen_at: string;
|
first_seen_at: string;
|
||||||
is_new_in_latest_run: boolean;
|
is_new_in_latest_run: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PublicationsResult {
|
export interface PublicationsResult {
|
||||||
mode: PublicationMode;
|
mode: PublicationMode;
|
||||||
|
favorite_only: boolean;
|
||||||
selected_scholar_profile_id: number | null;
|
selected_scholar_profile_id: number | null;
|
||||||
unread_count: number;
|
unread_count: number;
|
||||||
|
favorites_count: number;
|
||||||
latest_count: number;
|
latest_count: number;
|
||||||
// Compatibility alias for latest_count; retained while API clients migrate.
|
// Compatibility alias for latest_count; retained while API clients migrate.
|
||||||
new_count: number;
|
new_count: number;
|
||||||
total_count: number;
|
total_count: number;
|
||||||
|
page: number;
|
||||||
|
page_size: number;
|
||||||
|
has_next: boolean;
|
||||||
|
has_prev: boolean;
|
||||||
publications: PublicationItem[];
|
publications: PublicationItem[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PublicationsQuery {
|
export interface PublicationsQuery {
|
||||||
mode?: PublicationMode;
|
mode?: PublicationMode;
|
||||||
|
favoriteOnly?: boolean;
|
||||||
scholarProfileId?: number;
|
scholarProfileId?: number;
|
||||||
limit?: number;
|
page?: number;
|
||||||
|
pageSize?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PublicationSelection {
|
export interface PublicationSelection {
|
||||||
|
|
@ -46,12 +59,18 @@ export async function listPublications(query: PublicationsQuery = {}): Promise<P
|
||||||
if (query.mode) {
|
if (query.mode) {
|
||||||
params.set("mode", query.mode);
|
params.set("mode", query.mode);
|
||||||
}
|
}
|
||||||
|
if (query.favoriteOnly) {
|
||||||
|
params.set("favorite_only", "true");
|
||||||
|
}
|
||||||
if (query.scholarProfileId) {
|
if (query.scholarProfileId) {
|
||||||
params.set("scholar_profile_id", String(query.scholarProfileId));
|
params.set("scholar_profile_id", String(query.scholarProfileId));
|
||||||
}
|
}
|
||||||
if (query.limit) {
|
const parsedPage = Number.isFinite(query.page) ? Math.max(1, Math.trunc(Number(query.page))) : 1;
|
||||||
params.set("limit", String(query.limit));
|
const parsedPageSize = Number.isFinite(query.pageSize)
|
||||||
}
|
? Math.max(1, Math.min(500, Math.trunc(Number(query.pageSize))))
|
||||||
|
: 100;
|
||||||
|
params.set("page", String(parsedPage));
|
||||||
|
params.set("page_size", String(parsedPageSize));
|
||||||
|
|
||||||
const suffix = params.toString();
|
const suffix = params.toString();
|
||||||
const response = await apiRequest<PublicationsResult>(
|
const response = await apiRequest<PublicationsResult>(
|
||||||
|
|
@ -87,10 +106,16 @@ export async function markSelectedRead(selections: PublicationSelection[]): Prom
|
||||||
|
|
||||||
export interface RetryPublicationPdfResult {
|
export interface RetryPublicationPdfResult {
|
||||||
message: string;
|
message: string;
|
||||||
|
queued: boolean;
|
||||||
resolved_pdf: boolean;
|
resolved_pdf: boolean;
|
||||||
publication: PublicationItem;
|
publication: PublicationItem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface TogglePublicationFavoriteResult {
|
||||||
|
message: string;
|
||||||
|
publication: PublicationItem;
|
||||||
|
}
|
||||||
|
|
||||||
export async function retryPublicationPdf(
|
export async function retryPublicationPdf(
|
||||||
publicationId: number,
|
publicationId: number,
|
||||||
scholarProfileId: number,
|
scholarProfileId: number,
|
||||||
|
|
@ -104,3 +129,21 @@ export async function retryPublicationPdf(
|
||||||
);
|
);
|
||||||
return response.data;
|
return response.data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function togglePublicationFavorite(
|
||||||
|
publicationId: number,
|
||||||
|
scholarProfileId: number,
|
||||||
|
isFavorite: boolean,
|
||||||
|
): Promise<TogglePublicationFavoriteResult> {
|
||||||
|
const response = await apiRequest<TogglePublicationFavoriteResult>(
|
||||||
|
`/publications/${publicationId}/favorite`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
body: {
|
||||||
|
scholar_profile_id: scholarProfileId,
|
||||||
|
is_favorite: isFavorite,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
return response.data;
|
||||||
|
}
|
||||||
|
|
|
||||||
826
frontend/src/features/settings/SettingsAdminPanel.vue
Normal file
|
|
@ -0,0 +1,826 @@
|
||||||
|
<script setup lang="ts">
|
||||||
|
import { computed, onMounted, ref, watch } from "vue";
|
||||||
|
|
||||||
|
import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
|
||||||
|
import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue";
|
||||||
|
import AppBadge from "@/components/ui/AppBadge.vue";
|
||||||
|
import AppButton from "@/components/ui/AppButton.vue";
|
||||||
|
import AppCard from "@/components/ui/AppCard.vue";
|
||||||
|
import AppCheckbox from "@/components/ui/AppCheckbox.vue";
|
||||||
|
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
||||||
|
import AppInput from "@/components/ui/AppInput.vue";
|
||||||
|
import AppModal from "@/components/ui/AppModal.vue";
|
||||||
|
import AppRefreshButton from "@/components/ui/AppRefreshButton.vue";
|
||||||
|
import AppSelect from "@/components/ui/AppSelect.vue";
|
||||||
|
import AppTable from "@/components/ui/AppTable.vue";
|
||||||
|
import {
|
||||||
|
getAdminDbIntegrityReport,
|
||||||
|
listAdminDbRepairJobs,
|
||||||
|
listAdminPdfQueue,
|
||||||
|
requeueAdminPdfLookup,
|
||||||
|
requeueAllAdminPdfLookups,
|
||||||
|
triggerPublicationLinkRepair,
|
||||||
|
type AdminDbIntegrityCheck,
|
||||||
|
type AdminDbIntegrityReport,
|
||||||
|
type AdminDbRepairJob,
|
||||||
|
type AdminPdfQueueItem,
|
||||||
|
type TriggerPublicationLinkRepairResult,
|
||||||
|
} from "@/features/admin_dbops";
|
||||||
|
import {
|
||||||
|
createAdminUser,
|
||||||
|
listAdminUsers,
|
||||||
|
resetAdminUserPassword,
|
||||||
|
setAdminUserActive,
|
||||||
|
type AdminUser,
|
||||||
|
} from "@/features/admin_users";
|
||||||
|
import { ApiRequestError } from "@/lib/api/errors";
|
||||||
|
|
||||||
|
const SECTION_USERS = "users";
|
||||||
|
const SECTION_INTEGRITY = "integrity";
|
||||||
|
const SECTION_REPAIRS = "repairs";
|
||||||
|
const SECTION_PDF = "pdf";
|
||||||
|
const SCOPE_SINGLE_USER = "single_user";
|
||||||
|
const SCOPE_ALL_USERS = "all_users";
|
||||||
|
const APPLY_ALL_USERS_CONFIRM_TEXT = "REPAIR ALL USERS";
|
||||||
|
|
||||||
|
type RepairScopeMode = typeof SCOPE_SINGLE_USER | typeof SCOPE_ALL_USERS;
|
||||||
|
|
||||||
|
const props = defineProps<{
|
||||||
|
section: "users" | "integrity" | "repairs" | "pdf";
|
||||||
|
}>();
|
||||||
|
|
||||||
|
const loading = ref(true);
|
||||||
|
const errorMessage = ref<string | null>(null);
|
||||||
|
const errorRequestId = ref<string | null>(null);
|
||||||
|
const successMessage = ref<string | null>(null);
|
||||||
|
|
||||||
|
const refreshingUsers = ref(false);
|
||||||
|
const creating = ref(false);
|
||||||
|
const togglingUserId = ref<number | null>(null);
|
||||||
|
const resettingPassword = ref(false);
|
||||||
|
|
||||||
|
const users = ref<AdminUser[]>([]);
|
||||||
|
const email = ref("");
|
||||||
|
const password = ref("");
|
||||||
|
const createIsAdmin = ref(false);
|
||||||
|
const activeUserId = ref<number | null>(null);
|
||||||
|
const resetPassword = ref("");
|
||||||
|
|
||||||
|
const refreshingIntegrity = ref(false);
|
||||||
|
const integrityReport = ref<AdminDbIntegrityReport | null>(null);
|
||||||
|
|
||||||
|
const refreshingJobs = ref(false);
|
||||||
|
const runningRepair = ref(false);
|
||||||
|
const repairJobs = ref<AdminDbRepairJob[]>([]);
|
||||||
|
const lastRepairResult = ref<TriggerPublicationLinkRepairResult | null>(null);
|
||||||
|
const repairScopeMode = ref<RepairScopeMode>(SCOPE_SINGLE_USER);
|
||||||
|
const repairUserId = ref("");
|
||||||
|
const repairScholarIds = ref("");
|
||||||
|
const repairRequestedBy = ref("");
|
||||||
|
const repairDryRun = ref(true);
|
||||||
|
const repairGcOrphans = ref(false);
|
||||||
|
const repairConfirmationText = ref("");
|
||||||
|
|
||||||
|
const refreshingPdfQueue = ref(false);
|
||||||
|
const requeueingPublicationId = ref<number | null>(null);
|
||||||
|
const requeueingAllPdfs = ref(false);
|
||||||
|
const pdfQueueItems = ref<AdminPdfQueueItem[]>([]);
|
||||||
|
const pdfQueueStatusFilter = ref("");
|
||||||
|
const pdfQueuePage = ref(1);
|
||||||
|
const pdfQueuePageSize = ref("50");
|
||||||
|
const pdfQueueTotalCount = ref(0);
|
||||||
|
const pdfQueueHasNext = ref(false);
|
||||||
|
const pdfQueueHasPrev = ref(false);
|
||||||
|
|
||||||
|
const activeUser = computed(() => users.value.find((user) => user.id === activeUserId.value) ?? null);
|
||||||
|
const typedConfirmationRequired = computed(
|
||||||
|
() => repairScopeMode.value === SCOPE_ALL_USERS && !repairDryRun.value,
|
||||||
|
);
|
||||||
|
const pdfQueuePageSizeValue = computed(() => {
|
||||||
|
const parsed = Number(pdfQueuePageSize.value);
|
||||||
|
if (!Number.isFinite(parsed)) {
|
||||||
|
return 50;
|
||||||
|
}
|
||||||
|
return Math.max(1, Math.min(500, Math.trunc(parsed)));
|
||||||
|
});
|
||||||
|
const pdfQueueTotalPages = computed(() =>
|
||||||
|
Math.max(1, Math.ceil(pdfQueueTotalCount.value / Math.max(pdfQueuePageSizeValue.value, 1))),
|
||||||
|
);
|
||||||
|
const pdfQueueSummary = computed(() =>
|
||||||
|
`${pdfQueueTotalCount.value} item${pdfQueueTotalCount.value === 1 ? "" : "s"} total`,
|
||||||
|
);
|
||||||
|
|
||||||
|
function clearAlerts(): void {
|
||||||
|
errorMessage.value = null;
|
||||||
|
errorRequestId.value = null;
|
||||||
|
successMessage.value = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatTimestamp(value: string | null): string {
|
||||||
|
if (!value) {
|
||||||
|
return "n/a";
|
||||||
|
}
|
||||||
|
const parsed = new Date(value);
|
||||||
|
return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function statusTone(status: string): "success" | "warning" | "danger" | "info" | "neutral" {
|
||||||
|
if (status === "ok" || status === "completed" || status === "resolved") {
|
||||||
|
return "success";
|
||||||
|
}
|
||||||
|
if (status === "warning" || status === "running" || status === "queued") {
|
||||||
|
return "warning";
|
||||||
|
}
|
||||||
|
if (status === "failed") {
|
||||||
|
return "danger";
|
||||||
|
}
|
||||||
|
return "info";
|
||||||
|
}
|
||||||
|
|
||||||
|
function checkTone(check: AdminDbIntegrityCheck): "warning" | "danger" | "neutral" | "info" {
|
||||||
|
if (check.severity === "metric") {
|
||||||
|
return "info";
|
||||||
|
}
|
||||||
|
if (check.count <= 0) {
|
||||||
|
return "neutral";
|
||||||
|
}
|
||||||
|
return check.severity === "failure" ? "danger" : "warning";
|
||||||
|
}
|
||||||
|
|
||||||
|
function userRoleLabel(user: AdminUser): string {
|
||||||
|
return user.is_admin ? "Admin" : "User";
|
||||||
|
}
|
||||||
|
|
||||||
|
function canRequeuePdf(item: AdminPdfQueueItem): boolean {
|
||||||
|
return item.status !== "queued" && item.status !== "running";
|
||||||
|
}
|
||||||
|
|
||||||
|
function statusDotClass(user: AdminUser): string {
|
||||||
|
return user.is_active ? "bg-success-500 ring-success-200" : "bg-ink-muted/70 ring-stroke-default";
|
||||||
|
}
|
||||||
|
|
||||||
|
function assignError(error: unknown, fallback: string): void {
|
||||||
|
if (error instanceof ApiRequestError) {
|
||||||
|
errorMessage.value = error.message;
|
||||||
|
errorRequestId.value = error.requestId;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (error instanceof Error && error.message) {
|
||||||
|
errorMessage.value = error.message;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
errorMessage.value = fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseScholarIds(raw: string): number[] {
|
||||||
|
const tokens = raw
|
||||||
|
.split(/[,\s]+/)
|
||||||
|
.map((token) => token.trim())
|
||||||
|
.filter((token) => token.length > 0);
|
||||||
|
const deduped = new Set<number>();
|
||||||
|
for (const token of tokens) {
|
||||||
|
const parsed = Number(token);
|
||||||
|
if (!Number.isInteger(parsed) || parsed < 1) {
|
||||||
|
throw new Error("Scholar profile IDs must be positive integers.");
|
||||||
|
}
|
||||||
|
deduped.add(parsed);
|
||||||
|
}
|
||||||
|
return [...deduped];
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseRepairUserIdOrThrow(raw: string): number {
|
||||||
|
const parsed = Number((raw || "").trim());
|
||||||
|
if (!Number.isInteger(parsed) || parsed < 1) {
|
||||||
|
throw new Error("Select a valid target user.");
|
||||||
|
}
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateTypedConfirmation(): string {
|
||||||
|
const normalized = repairConfirmationText.value.trim();
|
||||||
|
if (typedConfirmationRequired.value && normalized !== APPLY_ALL_USERS_CONFIRM_TEXT) {
|
||||||
|
throw new Error(`Type '${APPLY_ALL_USERS_CONFIRM_TEXT}' to apply repair for all users.`);
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
function summaryCount(job: AdminDbRepairJob, key: string): string {
|
||||||
|
const value = job.summary[key];
|
||||||
|
return typeof value === "number" ? String(value) : "n/a";
|
||||||
|
}
|
||||||
|
|
||||||
|
function openUserModal(user: AdminUser): void {
|
||||||
|
activeUserId.value = user.id;
|
||||||
|
resetPassword.value = "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function closeUserModal(): void {
|
||||||
|
activeUserId.value = null;
|
||||||
|
resetPassword.value = "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function ensureRepairUserSelected(): void {
|
||||||
|
if (repairScopeMode.value !== SCOPE_SINGLE_USER || repairUserId.value || users.value.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
repairUserId.value = String(users.value[0].id);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshUsers(): Promise<void> {
|
||||||
|
refreshingUsers.value = true;
|
||||||
|
try {
|
||||||
|
users.value = await listAdminUsers();
|
||||||
|
ensureRepairUserSelected();
|
||||||
|
if (activeUserId.value !== null && !users.value.some((item) => item.id === activeUserId.value)) {
|
||||||
|
closeUserModal();
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
refreshingUsers.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshIntegrity(): Promise<void> {
|
||||||
|
refreshingIntegrity.value = true;
|
||||||
|
try {
|
||||||
|
integrityReport.value = await getAdminDbIntegrityReport();
|
||||||
|
} finally {
|
||||||
|
refreshingIntegrity.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshRepairJobs(): Promise<void> {
|
||||||
|
refreshingJobs.value = true;
|
||||||
|
try {
|
||||||
|
repairJobs.value = await listAdminDbRepairJobs(30);
|
||||||
|
} finally {
|
||||||
|
refreshingJobs.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshPdfQueue(): Promise<void> {
|
||||||
|
refreshingPdfQueue.value = true;
|
||||||
|
try {
|
||||||
|
const status = pdfQueueStatusFilter.value.trim() || null;
|
||||||
|
const page = await listAdminPdfQueue(pdfQueuePage.value, pdfQueuePageSizeValue.value, status);
|
||||||
|
pdfQueueItems.value = page.items;
|
||||||
|
pdfQueueTotalCount.value = page.total_count;
|
||||||
|
pdfQueueHasNext.value = page.has_next;
|
||||||
|
pdfQueueHasPrev.value = page.has_prev;
|
||||||
|
pdfQueuePage.value = page.page;
|
||||||
|
pdfQueuePageSize.value = String(page.page_size);
|
||||||
|
} finally {
|
||||||
|
refreshingPdfQueue.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshForSection(): Promise<void> {
|
||||||
|
if (props.section === SECTION_USERS) {
|
||||||
|
await refreshUsers();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (props.section === SECTION_INTEGRITY) {
|
||||||
|
await refreshIntegrity();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (props.section === SECTION_REPAIRS) {
|
||||||
|
await Promise.all([refreshUsers(), refreshRepairJobs()]);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await refreshPdfQueue();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onCreateUser(): Promise<void> {
|
||||||
|
creating.value = true;
|
||||||
|
clearAlerts();
|
||||||
|
try {
|
||||||
|
if (!email.value.trim() || !password.value) {
|
||||||
|
throw new Error("Email and password are required.");
|
||||||
|
}
|
||||||
|
const created = await createAdminUser({
|
||||||
|
email: email.value.trim(),
|
||||||
|
password: password.value,
|
||||||
|
is_admin: createIsAdmin.value,
|
||||||
|
});
|
||||||
|
email.value = "";
|
||||||
|
password.value = "";
|
||||||
|
createIsAdmin.value = false;
|
||||||
|
successMessage.value = `User created: ${created.email}`;
|
||||||
|
await refreshUsers();
|
||||||
|
} catch (error) {
|
||||||
|
assignError(error, "Unable to create user.");
|
||||||
|
} finally {
|
||||||
|
creating.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onToggleUser(user: AdminUser): Promise<void> {
|
||||||
|
togglingUserId.value = user.id;
|
||||||
|
clearAlerts();
|
||||||
|
try {
|
||||||
|
const updated = await setAdminUserActive(user.id, !user.is_active);
|
||||||
|
successMessage.value = `${updated.email} is now ${updated.is_active ? "active" : "inactive"}.`;
|
||||||
|
await refreshUsers();
|
||||||
|
} catch (error) {
|
||||||
|
assignError(error, "Unable to update user.");
|
||||||
|
} finally {
|
||||||
|
togglingUserId.value = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onResetPassword(): Promise<void> {
|
||||||
|
const user = activeUser.value;
|
||||||
|
if (!user) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
resettingPassword.value = true;
|
||||||
|
clearAlerts();
|
||||||
|
try {
|
||||||
|
const candidate = resetPassword.value.trim();
|
||||||
|
if (candidate.length < 12) {
|
||||||
|
throw new Error("New password must be at least 12 characters.");
|
||||||
|
}
|
||||||
|
const result = await resetAdminUserPassword(user.id, candidate);
|
||||||
|
resetPassword.value = "";
|
||||||
|
successMessage.value = result.message || `Password reset for ${user.email}.`;
|
||||||
|
} catch (error) {
|
||||||
|
assignError(error, "Unable to reset password.");
|
||||||
|
} finally {
|
||||||
|
resettingPassword.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onRunRepair(): Promise<void> {
|
||||||
|
runningRepair.value = true;
|
||||||
|
clearAlerts();
|
||||||
|
try {
|
||||||
|
const scopeMode = repairScopeMode.value;
|
||||||
|
const payload = {
|
||||||
|
scope_mode: scopeMode,
|
||||||
|
scholar_profile_ids: parseScholarIds(repairScholarIds.value),
|
||||||
|
dry_run: repairDryRun.value,
|
||||||
|
gc_orphan_publications: repairGcOrphans.value,
|
||||||
|
requested_by: repairRequestedBy.value.trim() || undefined,
|
||||||
|
confirmation_text: validateTypedConfirmation() || undefined,
|
||||||
|
user_id: scopeMode === SCOPE_SINGLE_USER ? parseRepairUserIdOrThrow(repairUserId.value) : undefined,
|
||||||
|
};
|
||||||
|
const result = await triggerPublicationLinkRepair(payload);
|
||||||
|
lastRepairResult.value = result;
|
||||||
|
successMessage.value = `Repair job #${result.job_id} completed (${result.status}).`;
|
||||||
|
repairConfirmationText.value = "";
|
||||||
|
await Promise.all([refreshIntegrity(), refreshRepairJobs(), refreshPdfQueue()]);
|
||||||
|
} catch (error) {
|
||||||
|
assignError(error, "Unable to run publication link repair.");
|
||||||
|
} finally {
|
||||||
|
runningRepair.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function onScopeModeChange(): void {
|
||||||
|
ensureRepairUserSelected();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onPdfQueueFilterChanged(): Promise<void> {
|
||||||
|
pdfQueuePage.value = 1;
|
||||||
|
await refreshPdfQueue();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onPdfQueuePageSizeChanged(): Promise<void> {
|
||||||
|
pdfQueuePage.value = 1;
|
||||||
|
await refreshPdfQueue();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onPdfQueuePrevPage(): Promise<void> {
|
||||||
|
if (!pdfQueueHasPrev.value || pdfQueuePage.value <= 1) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pdfQueuePage.value = Math.max(pdfQueuePage.value - 1, 1);
|
||||||
|
await refreshPdfQueue();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onPdfQueueNextPage(): Promise<void> {
|
||||||
|
if (!pdfQueueHasNext.value) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pdfQueuePage.value += 1;
|
||||||
|
await refreshPdfQueue();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onRequeuePdf(item: AdminPdfQueueItem): Promise<void> {
|
||||||
|
requeueingPublicationId.value = item.publication_id;
|
||||||
|
clearAlerts();
|
||||||
|
try {
|
||||||
|
const result = await requeueAdminPdfLookup(item.publication_id);
|
||||||
|
successMessage.value = result.message;
|
||||||
|
await refreshPdfQueue();
|
||||||
|
} catch (error) {
|
||||||
|
assignError(error, "Unable to requeue PDF lookup.");
|
||||||
|
} finally {
|
||||||
|
requeueingPublicationId.value = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onRequeueAllPdfs(): Promise<void> {
|
||||||
|
requeueingAllPdfs.value = true;
|
||||||
|
clearAlerts();
|
||||||
|
try {
|
||||||
|
const result = await requeueAllAdminPdfLookups(5000);
|
||||||
|
successMessage.value = result.message;
|
||||||
|
await refreshPdfQueue();
|
||||||
|
} catch (error) {
|
||||||
|
assignError(error, "Unable to queue missing PDF lookups.");
|
||||||
|
} finally {
|
||||||
|
requeueingAllPdfs.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
onMounted(async () => {
|
||||||
|
loading.value = true;
|
||||||
|
clearAlerts();
|
||||||
|
try {
|
||||||
|
await refreshForSection();
|
||||||
|
} catch (error) {
|
||||||
|
assignError(error, "Unable to load admin data.");
|
||||||
|
} finally {
|
||||||
|
loading.value = false;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
watch(
|
||||||
|
() => props.section,
|
||||||
|
async () => {
|
||||||
|
loading.value = true;
|
||||||
|
clearAlerts();
|
||||||
|
try {
|
||||||
|
await refreshForSection();
|
||||||
|
} catch (error) {
|
||||||
|
assignError(error, "Unable to load admin data.");
|
||||||
|
} finally {
|
||||||
|
loading.value = false;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<section class="grid gap-4">
|
||||||
|
<RequestStateAlerts
|
||||||
|
:success-message="successMessage"
|
||||||
|
:error-message="errorMessage"
|
||||||
|
:error-request-id="errorRequestId"
|
||||||
|
success-title="Operation complete"
|
||||||
|
error-title="Operation failed"
|
||||||
|
@dismiss-success="successMessage = null"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<AsyncStateGate :loading="loading" :loading-lines="10" :show-empty="false">
|
||||||
|
<AppCard v-if="props.section === SECTION_USERS" class="space-y-4">
|
||||||
|
<div class="flex flex-wrap items-center justify-between gap-2">
|
||||||
|
<div class="flex items-center gap-1">
|
||||||
|
<h2 class="text-lg font-semibold text-ink-primary">Users</h2>
|
||||||
|
<AppHelpHint text="Create accounts, toggle active status, and reset passwords." />
|
||||||
|
</div>
|
||||||
|
<AppRefreshButton
|
||||||
|
variant="secondary"
|
||||||
|
:loading="refreshingUsers"
|
||||||
|
title="Refresh users"
|
||||||
|
loading-title="Refreshing users"
|
||||||
|
@click="refreshUsers"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<form class="grid gap-3 md:grid-cols-3" @submit.prevent="onCreateUser">
|
||||||
|
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
|
||||||
|
<span>Email</span>
|
||||||
|
<AppInput v-model="email" type="email" autocomplete="off" />
|
||||||
|
</label>
|
||||||
|
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
|
||||||
|
<span>Password</span>
|
||||||
|
<AppInput v-model="password" type="password" autocomplete="new-password" />
|
||||||
|
</label>
|
||||||
|
<div class="grid gap-2 text-sm font-medium text-ink-secondary">
|
||||||
|
<span>Role</span>
|
||||||
|
<div class="flex items-center gap-3">
|
||||||
|
<AppCheckbox id="admin-create-user-is-admin" v-model="createIsAdmin" label="Grant admin" />
|
||||||
|
<AppButton type="submit" :disabled="creating">
|
||||||
|
{{ creating ? "Creating..." : "Create user" }}
|
||||||
|
</AppButton>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<AppTable label="Users table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th scope="col">Email</th>
|
||||||
|
<th scope="col">Role</th>
|
||||||
|
<th scope="col">Status</th>
|
||||||
|
<th scope="col">Actions</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="user in users" :key="user.id">
|
||||||
|
<td class="align-middle">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="group inline-flex items-center gap-2 rounded-md px-1 py-0.5 text-left text-ink-primary transition hover:bg-surface-card-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset"
|
||||||
|
@click="openUserModal(user)"
|
||||||
|
>
|
||||||
|
<span
|
||||||
|
:class="statusDotClass(user)"
|
||||||
|
class="h-2.5 w-2.5 rounded-full ring-2"
|
||||||
|
:aria-label="user.is_active ? 'Active user' : 'Inactive user'"
|
||||||
|
/>
|
||||||
|
<span class="underline-offset-2 group-hover:underline">{{ user.email }}</span>
|
||||||
|
</button>
|
||||||
|
</td>
|
||||||
|
<td>{{ userRoleLabel(user) }}</td>
|
||||||
|
<td>{{ user.is_active ? "active" : "inactive" }}</td>
|
||||||
|
<td class="flex flex-wrap items-center gap-2">
|
||||||
|
<AppButton variant="secondary" :disabled="togglingUserId === user.id" @click="onToggleUser(user)">
|
||||||
|
{{ togglingUserId === user.id ? "Updating..." : user.is_active ? "Deactivate" : "Activate" }}
|
||||||
|
</AppButton>
|
||||||
|
<AppButton variant="ghost" @click="openUserModal(user)">Manage</AppButton>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</AppTable>
|
||||||
|
</AppCard>
|
||||||
|
|
||||||
|
<AppCard v-if="props.section === SECTION_INTEGRITY" class="space-y-3">
|
||||||
|
<div class="flex flex-wrap items-center justify-between gap-2">
|
||||||
|
<div class="flex items-center gap-1">
|
||||||
|
<h2 class="text-lg font-semibold text-ink-primary">Integrity Report</h2>
|
||||||
|
<AppHelpHint text="Read-only checks for known corruption patterns and data drift." />
|
||||||
|
</div>
|
||||||
|
<AppRefreshButton
|
||||||
|
variant="secondary"
|
||||||
|
:loading="refreshingIntegrity"
|
||||||
|
title="Refresh integrity report"
|
||||||
|
loading-title="Refreshing integrity report"
|
||||||
|
@click="refreshIntegrity"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div v-if="integrityReport" class="flex flex-wrap items-center gap-2 text-xs">
|
||||||
|
<AppBadge :tone="statusTone(integrityReport.status)">Status: {{ integrityReport.status }}</AppBadge>
|
||||||
|
<AppBadge tone="warning">Warnings: {{ integrityReport.warnings.length }}</AppBadge>
|
||||||
|
<AppBadge tone="danger">Failures: {{ integrityReport.failures.length }}</AppBadge>
|
||||||
|
<span class="text-secondary">Checked: {{ formatTimestamp(integrityReport.checked_at) }}</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<AppTable v-if="integrityReport" label="Integrity checks">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th scope="col">Check</th>
|
||||||
|
<th scope="col">Count</th>
|
||||||
|
<th scope="col">Severity</th>
|
||||||
|
<th scope="col">Message</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="check in integrityReport.checks" :key="check.name">
|
||||||
|
<td>{{ check.name }}</td>
|
||||||
|
<td>{{ check.count }}</td>
|
||||||
|
<td><AppBadge :tone="checkTone(check)">{{ check.severity }}</AppBadge></td>
|
||||||
|
<td>{{ check.message }}</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</AppTable>
|
||||||
|
</AppCard>
|
||||||
|
|
||||||
|
<section v-if="props.section === SECTION_REPAIRS" class="grid gap-4">
|
||||||
|
<AppCard class="space-y-3">
|
||||||
|
<div class="flex items-center gap-1">
|
||||||
|
<h2 class="text-lg font-semibold text-ink-primary">Publication Link Repair</h2>
|
||||||
|
<AppHelpHint text="Dry-run first. For all-users apply mode, typed confirmation is required." />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<form class="grid gap-3 md:grid-cols-2" @submit.prevent="onRunRepair">
|
||||||
|
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
|
||||||
|
<span>Scope</span>
|
||||||
|
<AppSelect v-model="repairScopeMode" @change="onScopeModeChange">
|
||||||
|
<option :value="SCOPE_SINGLE_USER">Single user</option>
|
||||||
|
<option :value="SCOPE_ALL_USERS">All users</option>
|
||||||
|
</AppSelect>
|
||||||
|
</label>
|
||||||
|
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
|
||||||
|
<span>Target user</span>
|
||||||
|
<AppSelect v-model="repairUserId" :disabled="repairScopeMode === SCOPE_ALL_USERS || users.length === 0">
|
||||||
|
<option value="" disabled>Select user</option>
|
||||||
|
<option v-for="user in users" :key="user.id" :value="String(user.id)">
|
||||||
|
{{ user.email }} (ID {{ user.id }})
|
||||||
|
</option>
|
||||||
|
</AppSelect>
|
||||||
|
</label>
|
||||||
|
<label class="grid gap-1 text-sm font-medium text-ink-secondary md:col-span-2">
|
||||||
|
<span>Scholar profile IDs (optional)</span>
|
||||||
|
<AppInput v-model="repairScholarIds" placeholder="e.g. 12,13,14" />
|
||||||
|
</label>
|
||||||
|
<label class="grid gap-1 text-sm font-medium text-ink-secondary md:col-span-2">
|
||||||
|
<span>Requested by (optional)</span>
|
||||||
|
<AppInput v-model="repairRequestedBy" placeholder="email/name/ticket id" />
|
||||||
|
</label>
|
||||||
|
<div class="flex flex-wrap items-center gap-4 md:col-span-2">
|
||||||
|
<AppCheckbox id="repair-dry-run" v-model="repairDryRun" label="Dry-run (no writes)" />
|
||||||
|
<AppCheckbox id="repair-gc-orphans" v-model="repairGcOrphans" label="Delete orphan publications" />
|
||||||
|
</div>
|
||||||
|
<label v-if="typedConfirmationRequired" class="grid gap-1 text-sm font-medium text-ink-secondary md:col-span-2">
|
||||||
|
<span>Type '{{ APPLY_ALL_USERS_CONFIRM_TEXT }}' to confirm</span>
|
||||||
|
<AppInput v-model="repairConfirmationText" :placeholder="APPLY_ALL_USERS_CONFIRM_TEXT" />
|
||||||
|
</label>
|
||||||
|
<p v-if="repairScopeMode === SCOPE_ALL_USERS" class="text-xs text-secondary md:col-span-2">
|
||||||
|
All-users scope includes every scholar profile across all accounts.
|
||||||
|
</p>
|
||||||
|
<div class="md:col-span-2">
|
||||||
|
<AppButton type="submit" :disabled="runningRepair">
|
||||||
|
{{ runningRepair ? "Running..." : "Run repair job" }}
|
||||||
|
</AppButton>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<div v-if="lastRepairResult" class="rounded-lg border border-stroke-default bg-surface-card-muted p-3 text-xs">
|
||||||
|
<div class="mb-2 flex flex-wrap items-center gap-2">
|
||||||
|
<AppBadge :tone="statusTone(lastRepairResult.status)">Job #{{ lastRepairResult.job_id }}</AppBadge>
|
||||||
|
<span class="text-secondary">Status: {{ lastRepairResult.status }}</span>
|
||||||
|
</div>
|
||||||
|
<pre class="overflow-x-auto text-secondary">{{ JSON.stringify(lastRepairResult.summary, null, 2) }}</pre>
|
||||||
|
</div>
|
||||||
|
</AppCard>
|
||||||
|
|
||||||
|
<AppCard class="space-y-3">
|
||||||
|
<div class="flex flex-wrap items-center justify-between gap-2">
|
||||||
|
<div class="flex items-center gap-1">
|
||||||
|
<h2 class="text-lg font-semibold text-ink-primary">Recent Repair Jobs</h2>
|
||||||
|
<AppHelpHint text="Audit history and summary counters for each repair job." />
|
||||||
|
</div>
|
||||||
|
<AppRefreshButton
|
||||||
|
variant="secondary"
|
||||||
|
:loading="refreshingJobs"
|
||||||
|
title="Refresh repair jobs"
|
||||||
|
loading-title="Refreshing repair jobs"
|
||||||
|
@click="refreshRepairJobs"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<AppTable label="Repair jobs table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th scope="col">Job</th>
|
||||||
|
<th scope="col">Status</th>
|
||||||
|
<th scope="col">Mode</th>
|
||||||
|
<th scope="col">Requested by</th>
|
||||||
|
<th scope="col">Created</th>
|
||||||
|
<th scope="col">Links in scope</th>
|
||||||
|
<th scope="col">Links deleted</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="job in repairJobs" :key="job.id">
|
||||||
|
<td>#{{ job.id }} · {{ job.job_name }}</td>
|
||||||
|
<td><AppBadge :tone="statusTone(job.status)">{{ job.status }}</AppBadge></td>
|
||||||
|
<td>{{ job.dry_run ? "dry-run" : "apply" }}</td>
|
||||||
|
<td>{{ job.requested_by || "n/a" }}</td>
|
||||||
|
<td>{{ formatTimestamp(job.created_at) }}</td>
|
||||||
|
<td>{{ summaryCount(job, "links_in_scope") }}</td>
|
||||||
|
<td>{{ summaryCount(job, "links_deleted") }}</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</AppTable>
|
||||||
|
</AppCard>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<AppCard v-if="props.section === SECTION_PDF" class="space-y-3">
|
||||||
|
<div class="flex flex-wrap items-center justify-between gap-2">
|
||||||
|
<div class="flex items-center gap-1">
|
||||||
|
<h2 class="text-lg font-semibold text-ink-primary">PDF Gathering Queue</h2>
|
||||||
|
<AppHelpHint text="Live queue and outcome history for PDF acquisition across all publications." />
|
||||||
|
</div>
|
||||||
|
<div class="flex items-center gap-2">
|
||||||
|
<AppSelect v-model="pdfQueueStatusFilter" class="min-w-[12rem] !py-1.5 !text-sm" @change="onPdfQueueFilterChanged">
|
||||||
|
<option value="">All statuses</option>
|
||||||
|
<option value="untracked">untracked</option>
|
||||||
|
<option value="queued">queued</option>
|
||||||
|
<option value="running">running</option>
|
||||||
|
<option value="failed">failed</option>
|
||||||
|
<option value="resolved">resolved</option>
|
||||||
|
</AppSelect>
|
||||||
|
<AppSelect v-model="pdfQueuePageSize" class="min-w-[10rem] !py-1.5 !text-sm" @change="onPdfQueuePageSizeChanged">
|
||||||
|
<option value="25">25 / page</option>
|
||||||
|
<option value="50">50 / page</option>
|
||||||
|
<option value="100">100 / page</option>
|
||||||
|
</AppSelect>
|
||||||
|
<AppButton
|
||||||
|
variant="secondary"
|
||||||
|
class="!min-h-8 whitespace-nowrap !px-2.5 !py-1 !text-xs"
|
||||||
|
:disabled="requeueingAllPdfs"
|
||||||
|
title="Queue all missing PDFs"
|
||||||
|
@click="onRequeueAllPdfs"
|
||||||
|
>
|
||||||
|
{{ requeueingAllPdfs ? "Queueing..." : "Queue all" }}
|
||||||
|
</AppButton>
|
||||||
|
<AppRefreshButton
|
||||||
|
variant="secondary"
|
||||||
|
size="sm"
|
||||||
|
:loading="refreshingPdfQueue"
|
||||||
|
title="Refresh PDF queue"
|
||||||
|
loading-title="Refreshing PDF queue"
|
||||||
|
@click="refreshPdfQueue"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<AppTable label="PDF queue table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th scope="col">Publication</th>
|
||||||
|
<th scope="col">Status</th>
|
||||||
|
<th scope="col">Attempts</th>
|
||||||
|
<th scope="col">Failure reason</th>
|
||||||
|
<th scope="col">Source</th>
|
||||||
|
<th scope="col">Requested by</th>
|
||||||
|
<th scope="col">Queued</th>
|
||||||
|
<th scope="col">Last attempt</th>
|
||||||
|
<th scope="col">Resolved</th>
|
||||||
|
<th scope="col">Actions</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="item in pdfQueueItems" :key="item.publication_id">
|
||||||
|
<td>
|
||||||
|
<div class="grid gap-1">
|
||||||
|
<span class="font-medium text-ink-primary">{{ item.title }}</span>
|
||||||
|
<a v-if="item.doi" :href="`https://doi.org/${item.doi}`" target="_blank" rel="noreferrer" class="link-inline text-xs">
|
||||||
|
DOI: {{ item.doi }}
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
<td><AppBadge :tone="statusTone(item.status)">{{ item.status }}</AppBadge></td>
|
||||||
|
<td>{{ item.attempt_count }}</td>
|
||||||
|
<td>{{ item.last_failure_reason || "n/a" }}</td>
|
||||||
|
<td>{{ item.last_source || "n/a" }}</td>
|
||||||
|
<td>{{ item.requested_by_email || "n/a" }}</td>
|
||||||
|
<td>{{ formatTimestamp(item.queued_at) }}</td>
|
||||||
|
<td>{{ formatTimestamp(item.last_attempt_at) }}</td>
|
||||||
|
<td>{{ formatTimestamp(item.resolved_at) }}</td>
|
||||||
|
<td>
|
||||||
|
<AppButton
|
||||||
|
variant="ghost"
|
||||||
|
:disabled="requeueingPublicationId === item.publication_id || !canRequeuePdf(item)"
|
||||||
|
@click="onRequeuePdf(item)"
|
||||||
|
>
|
||||||
|
{{ requeueingPublicationId === item.publication_id ? "Requeueing..." : "Requeue" }}
|
||||||
|
</AppButton>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</AppTable>
|
||||||
|
|
||||||
|
<div class="flex flex-wrap items-center justify-between gap-2 text-xs text-secondary">
|
||||||
|
<span>{{ pdfQueueSummary }}</span>
|
||||||
|
<div class="flex items-center gap-2">
|
||||||
|
<span>Page {{ pdfQueuePage }} / {{ pdfQueueTotalPages }}</span>
|
||||||
|
<AppButton variant="ghost" :disabled="!pdfQueueHasPrev" @click="onPdfQueuePrevPage">Prev</AppButton>
|
||||||
|
<AppButton variant="ghost" :disabled="!pdfQueueHasNext" @click="onPdfQueueNextPage">Next</AppButton>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</AppCard>
|
||||||
|
</AsyncStateGate>
|
||||||
|
|
||||||
|
<AppModal :open="activeUser !== null" title="User settings" @close="closeUserModal">
|
||||||
|
<div v-if="activeUser" class="grid gap-4">
|
||||||
|
<div class="space-y-1">
|
||||||
|
<div class="flex items-center gap-2">
|
||||||
|
<span
|
||||||
|
:class="statusDotClass(activeUser)"
|
||||||
|
class="h-2.5 w-2.5 rounded-full ring-2"
|
||||||
|
:aria-label="activeUser.is_active ? 'Active user' : 'Inactive user'"
|
||||||
|
/>
|
||||||
|
<p class="truncate text-sm font-semibold text-ink-primary">{{ activeUser.email }}</p>
|
||||||
|
</div>
|
||||||
|
<p class="text-sm text-secondary">Role: {{ userRoleLabel(activeUser) }}</p>
|
||||||
|
<p class="text-xs text-secondary">Last updated: {{ formatTimestamp(activeUser.updated_at) }}</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="grid gap-2">
|
||||||
|
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
|
||||||
|
<span>New password</span>
|
||||||
|
<AppInput
|
||||||
|
id="admin-reset-password"
|
||||||
|
v-model="resetPassword"
|
||||||
|
type="password"
|
||||||
|
autocomplete="new-password"
|
||||||
|
placeholder="At least 12 characters"
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
<div class="flex flex-wrap gap-2">
|
||||||
|
<AppButton :disabled="resettingPassword" @click="onResetPassword">
|
||||||
|
{{ resettingPassword ? "Resetting..." : "Reset password" }}
|
||||||
|
</AppButton>
|
||||||
|
<AppButton variant="secondary" :disabled="togglingUserId === activeUser.id" @click="onToggleUser(activeUser)">
|
||||||
|
{{ togglingUserId === activeUser.id ? "Updating..." : activeUser.is_active ? "Deactivate user" : "Activate user" }}
|
||||||
|
</AppButton>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</AppModal>
|
||||||
|
</section>
|
||||||
|
</template>
|
||||||
BIN
frontend/src/logo.png
Normal file
|
After Width: | Height: | Size: 104 KiB |
|
|
@ -1,316 +0,0 @@
|
||||||
<script setup lang="ts">
|
|
||||||
import { computed, onMounted, ref } from "vue";
|
|
||||||
|
|
||||||
import AppPage from "@/components/layout/AppPage.vue";
|
|
||||||
import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
|
|
||||||
import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue";
|
|
||||||
import AppButton from "@/components/ui/AppButton.vue";
|
|
||||||
import AppCard from "@/components/ui/AppCard.vue";
|
|
||||||
import AppCheckbox from "@/components/ui/AppCheckbox.vue";
|
|
||||||
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
|
||||||
import AppInput from "@/components/ui/AppInput.vue";
|
|
||||||
import AppModal from "@/components/ui/AppModal.vue";
|
|
||||||
import AppTable from "@/components/ui/AppTable.vue";
|
|
||||||
import {
|
|
||||||
createAdminUser,
|
|
||||||
listAdminUsers,
|
|
||||||
resetAdminUserPassword,
|
|
||||||
setAdminUserActive,
|
|
||||||
type AdminUser,
|
|
||||||
} from "@/features/admin_users";
|
|
||||||
import { ApiRequestError } from "@/lib/api/errors";
|
|
||||||
|
|
||||||
const loading = ref(true);
|
|
||||||
const creating = ref(false);
|
|
||||||
const togglingUserId = ref<number | null>(null);
|
|
||||||
const resettingPassword = ref(false);
|
|
||||||
const activeUserId = ref<number | null>(null);
|
|
||||||
const users = ref<AdminUser[]>([]);
|
|
||||||
|
|
||||||
const email = ref("");
|
|
||||||
const password = ref("");
|
|
||||||
const createIsAdmin = ref(false);
|
|
||||||
const resetPassword = ref("");
|
|
||||||
|
|
||||||
const errorMessage = ref<string | null>(null);
|
|
||||||
const errorRequestId = ref<string | null>(null);
|
|
||||||
const successMessage = ref<string | null>(null);
|
|
||||||
|
|
||||||
const activeUser = computed(() => users.value.find((item) => item.id === activeUserId.value) ?? null);
|
|
||||||
|
|
||||||
function formatDate(value: string): string {
|
|
||||||
const asDate = new Date(value);
|
|
||||||
if (Number.isNaN(asDate.getTime())) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
return asDate.toLocaleString();
|
|
||||||
}
|
|
||||||
|
|
||||||
function userRoleLabel(user: AdminUser): string {
|
|
||||||
return user.is_admin ? "Admin" : "User";
|
|
||||||
}
|
|
||||||
|
|
||||||
function statusDotClass(user: AdminUser): string {
|
|
||||||
return user.is_active
|
|
||||||
? "bg-success-500 ring-success-200"
|
|
||||||
: "bg-ink-muted/70 ring-stroke-default";
|
|
||||||
}
|
|
||||||
|
|
||||||
function openUserModal(user: AdminUser): void {
|
|
||||||
activeUserId.value = user.id;
|
|
||||||
resetPassword.value = "";
|
|
||||||
}
|
|
||||||
|
|
||||||
function closeUserModal(): void {
|
|
||||||
activeUserId.value = null;
|
|
||||||
resetPassword.value = "";
|
|
||||||
}
|
|
||||||
|
|
||||||
async function loadUsers(): Promise<void> {
|
|
||||||
loading.value = true;
|
|
||||||
errorMessage.value = null;
|
|
||||||
errorRequestId.value = null;
|
|
||||||
|
|
||||||
try {
|
|
||||||
users.value = await listAdminUsers();
|
|
||||||
if (activeUserId.value !== null && !users.value.some((item) => item.id === activeUserId.value)) {
|
|
||||||
closeUserModal();
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
users.value = [];
|
|
||||||
if (error instanceof ApiRequestError) {
|
|
||||||
errorMessage.value = error.message;
|
|
||||||
errorRequestId.value = error.requestId;
|
|
||||||
} else {
|
|
||||||
errorMessage.value = "Unable to load users.";
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
loading.value = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onCreateUser(): Promise<void> {
|
|
||||||
creating.value = true;
|
|
||||||
successMessage.value = null;
|
|
||||||
errorMessage.value = null;
|
|
||||||
errorRequestId.value = null;
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (!email.value.trim() || !password.value) {
|
|
||||||
throw new Error("Email and password are required.");
|
|
||||||
}
|
|
||||||
|
|
||||||
const created = await createAdminUser({
|
|
||||||
email: email.value.trim(),
|
|
||||||
password: password.value,
|
|
||||||
is_admin: createIsAdmin.value,
|
|
||||||
});
|
|
||||||
|
|
||||||
email.value = "";
|
|
||||||
password.value = "";
|
|
||||||
createIsAdmin.value = false;
|
|
||||||
successMessage.value = `User created: ${created.email}`;
|
|
||||||
await loadUsers();
|
|
||||||
} catch (error) {
|
|
||||||
if (error instanceof ApiRequestError) {
|
|
||||||
errorMessage.value = error.message;
|
|
||||||
errorRequestId.value = error.requestId;
|
|
||||||
} else if (error instanceof Error) {
|
|
||||||
errorMessage.value = error.message;
|
|
||||||
} else {
|
|
||||||
errorMessage.value = "Unable to create user.";
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
creating.value = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onToggleUser(user: AdminUser): Promise<void> {
|
|
||||||
togglingUserId.value = user.id;
|
|
||||||
successMessage.value = null;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const updated = await setAdminUserActive(user.id, !user.is_active);
|
|
||||||
successMessage.value = `${updated.email} is now ${updated.is_active ? "active" : "inactive"}.`;
|
|
||||||
await loadUsers();
|
|
||||||
} catch (error) {
|
|
||||||
if (error instanceof ApiRequestError) {
|
|
||||||
errorMessage.value = error.message;
|
|
||||||
errorRequestId.value = error.requestId;
|
|
||||||
} else {
|
|
||||||
errorMessage.value = "Unable to update user.";
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
togglingUserId.value = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onResetPassword(): Promise<void> {
|
|
||||||
const user = activeUser.value;
|
|
||||||
if (!user) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
resettingPassword.value = true;
|
|
||||||
successMessage.value = null;
|
|
||||||
errorMessage.value = null;
|
|
||||||
errorRequestId.value = null;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const candidate = resetPassword.value.trim();
|
|
||||||
if (candidate.length < 12) {
|
|
||||||
throw new Error("New password must be at least 12 characters.");
|
|
||||||
}
|
|
||||||
|
|
||||||
const result = await resetAdminUserPassword(user.id, candidate);
|
|
||||||
successMessage.value = result.message || `Password reset for ${user.email}.`;
|
|
||||||
resetPassword.value = "";
|
|
||||||
} catch (error) {
|
|
||||||
if (error instanceof ApiRequestError) {
|
|
||||||
errorMessage.value = error.message;
|
|
||||||
errorRequestId.value = error.requestId;
|
|
||||||
} else if (error instanceof Error) {
|
|
||||||
errorMessage.value = error.message;
|
|
||||||
} else {
|
|
||||||
errorMessage.value = "Unable to reset password.";
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
resettingPassword.value = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
onMounted(() => {
|
|
||||||
void loadUsers();
|
|
||||||
});
|
|
||||||
</script>
|
|
||||||
|
|
||||||
<template>
|
|
||||||
<AppPage title="Admin Users" subtitle="Create and manage user access for this instance.">
|
|
||||||
<RequestStateAlerts
|
|
||||||
:success-message="successMessage"
|
|
||||||
:error-message="errorMessage"
|
|
||||||
:error-request-id="errorRequestId"
|
|
||||||
error-title="User management request failed"
|
|
||||||
@dismiss-success="successMessage = null"
|
|
||||||
/>
|
|
||||||
|
|
||||||
<section class="grid gap-4 lg:grid-cols-2">
|
|
||||||
<AppCard class="space-y-4">
|
|
||||||
<div class="flex items-center gap-1">
|
|
||||||
<h2 class="text-lg font-semibold text-ink-primary">Create User</h2>
|
|
||||||
<AppHelpHint text="Create local accounts for this Scholarr instance and optionally grant admin rights." />
|
|
||||||
</div>
|
|
||||||
<form class="grid gap-3" @submit.prevent="onCreateUser">
|
|
||||||
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
|
||||||
<span>Email</span>
|
|
||||||
<AppInput id="admin-user-email" v-model="email" type="email" autocomplete="off" />
|
|
||||||
</label>
|
|
||||||
|
|
||||||
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
|
||||||
<span>Password</span>
|
|
||||||
<AppInput id="admin-user-password" v-model="password" type="password" autocomplete="new-password" />
|
|
||||||
</label>
|
|
||||||
|
|
||||||
<AppCheckbox id="admin-user-is-admin" v-model="createIsAdmin" label="Grant admin privileges" />
|
|
||||||
|
|
||||||
<AppButton type="submit" :disabled="creating">
|
|
||||||
{{ creating ? "Creating..." : "Create user" }}
|
|
||||||
</AppButton>
|
|
||||||
</form>
|
|
||||||
</AppCard>
|
|
||||||
|
|
||||||
<AppCard class="space-y-4">
|
|
||||||
<div class="flex items-center gap-1">
|
|
||||||
<h2 class="text-lg font-semibold text-ink-primary">Users</h2>
|
|
||||||
<AppHelpHint text="Review account roles and active status. Inactive users cannot sign in." />
|
|
||||||
</div>
|
|
||||||
<AsyncStateGate
|
|
||||||
:loading="loading"
|
|
||||||
:loading-lines="5"
|
|
||||||
:empty="users.length === 0"
|
|
||||||
:show-empty="!errorMessage"
|
|
||||||
empty-title="No users available"
|
|
||||||
empty-body="Create an account to begin assigning access."
|
|
||||||
>
|
|
||||||
<AppTable label="Admin users table">
|
|
||||||
<thead>
|
|
||||||
<tr>
|
|
||||||
<th scope="col">Email</th>
|
|
||||||
<th scope="col">Role</th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody>
|
|
||||||
<tr v-for="user in users" :key="user.id">
|
|
||||||
<td>
|
|
||||||
<button
|
|
||||||
type="button"
|
|
||||||
class="group inline-flex items-center gap-2 rounded-md px-1 py-0.5 text-left text-ink-primary transition hover:bg-surface-card-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset"
|
|
||||||
@click="openUserModal(user)"
|
|
||||||
>
|
|
||||||
<span
|
|
||||||
:class="statusDotClass(user)"
|
|
||||||
class="h-2.5 w-2.5 rounded-full ring-2"
|
|
||||||
:aria-label="user.is_active ? 'Active user' : 'Inactive user'"
|
|
||||||
/>
|
|
||||||
<span class="underline-offset-2 group-hover:underline">{{ user.email }}</span>
|
|
||||||
</button>
|
|
||||||
</td>
|
|
||||||
<td>{{ userRoleLabel(user) }}</td>
|
|
||||||
</tr>
|
|
||||||
</tbody>
|
|
||||||
</AppTable>
|
|
||||||
<p class="text-xs text-secondary">Status dot: green is active, gray is inactive. Click an email to manage.</p>
|
|
||||||
</AsyncStateGate>
|
|
||||||
</AppCard>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<AppModal :open="activeUser !== null" title="User settings" @close="closeUserModal">
|
|
||||||
<div v-if="activeUser" class="grid gap-4">
|
|
||||||
<div class="space-y-1">
|
|
||||||
<div class="flex items-center gap-2">
|
|
||||||
<span
|
|
||||||
:class="statusDotClass(activeUser)"
|
|
||||||
class="h-2.5 w-2.5 rounded-full ring-2"
|
|
||||||
:aria-label="activeUser.is_active ? 'Active user' : 'Inactive user'"
|
|
||||||
/>
|
|
||||||
<p class="truncate text-sm font-semibold text-ink-primary">{{ activeUser.email }}</p>
|
|
||||||
</div>
|
|
||||||
<p class="text-sm text-secondary">Role: {{ userRoleLabel(activeUser) }}</p>
|
|
||||||
<p class="text-xs text-secondary">Last updated: {{ formatDate(activeUser.updated_at) }}</p>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="grid gap-2 border-t border-stroke-default pt-3">
|
|
||||||
<label class="grid gap-2 text-sm font-medium text-ink-secondary" for="admin-user-reset-password">
|
|
||||||
<span>Reset password</span>
|
|
||||||
<AppInput
|
|
||||||
id="admin-user-reset-password"
|
|
||||||
v-model="resetPassword"
|
|
||||||
type="password"
|
|
||||||
autocomplete="new-password"
|
|
||||||
placeholder="Minimum 12 characters"
|
|
||||||
:disabled="resettingPassword"
|
|
||||||
/>
|
|
||||||
</label>
|
|
||||||
<div class="flex flex-wrap justify-end gap-2">
|
|
||||||
<AppButton variant="secondary" :disabled="resettingPassword" @click="onResetPassword">
|
|
||||||
{{ resettingPassword ? "Resetting..." : "Reset password" }}
|
|
||||||
</AppButton>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="flex flex-wrap items-center justify-between gap-2 border-t border-stroke-default pt-3">
|
|
||||||
<AppButton
|
|
||||||
variant="secondary"
|
|
||||||
:disabled="togglingUserId === activeUser.id"
|
|
||||||
@click="onToggleUser(activeUser)"
|
|
||||||
>
|
|
||||||
{{ activeUser.is_active ? "Deactivate user" : "Activate user" }}
|
|
||||||
</AppButton>
|
|
||||||
<AppButton variant="ghost" :disabled="togglingUserId === activeUser.id || resettingPassword" @click="closeUserModal">
|
|
||||||
Close
|
|
||||||
</AppButton>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</AppModal>
|
|
||||||
</AppPage>
|
|
||||||
</template>
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed, onMounted, onUnmounted, ref, watch } from "vue";
|
import { computed, onMounted, ref, watch } from "vue";
|
||||||
|
|
||||||
import { fetchDashboardSnapshot, type DashboardSnapshot } from "@/features/dashboard";
|
import { fetchDashboardSnapshot, type DashboardSnapshot } from "@/features/dashboard";
|
||||||
import { ApiRequestError } from "@/lib/api/errors";
|
import { ApiRequestError } from "@/lib/api/errors";
|
||||||
|
|
@ -14,7 +14,7 @@ import AppCard from "@/components/ui/AppCard.vue";
|
||||||
import AppEmptyState from "@/components/ui/AppEmptyState.vue";
|
import AppEmptyState from "@/components/ui/AppEmptyState.vue";
|
||||||
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
||||||
import { useAuthStore } from "@/stores/auth";
|
import { useAuthStore } from "@/stores/auth";
|
||||||
import { RUN_STATUS_POLL_INTERVAL_MS, useRunStatusStore } from "@/stores/run_status";
|
import { useRunStatusStore } from "@/stores/run_status";
|
||||||
import { useUserSettingsStore } from "@/stores/user_settings";
|
import { useUserSettingsStore } from "@/stores/user_settings";
|
||||||
|
|
||||||
const loading = ref(true);
|
const loading = ref(true);
|
||||||
|
|
@ -26,7 +26,6 @@ const refreshingAfterCompletion = ref(false);
|
||||||
const auth = useAuthStore();
|
const auth = useAuthStore();
|
||||||
const runStatus = useRunStatusStore();
|
const runStatus = useRunStatusStore();
|
||||||
const userSettings = useUserSettingsStore();
|
const userSettings = useUserSettingsStore();
|
||||||
let latestRunSyncTimer: ReturnType<typeof setInterval> | null = null;
|
|
||||||
|
|
||||||
const isStartBlocked = computed(
|
const isStartBlocked = computed(
|
||||||
() =>
|
() =>
|
||||||
|
|
@ -135,23 +134,6 @@ function shouldRefreshAfterRunChange(
|
||||||
return previousRun.status === "running";
|
return previousRun.status === "running";
|
||||||
}
|
}
|
||||||
|
|
||||||
function startLatestRunSyncLoop(): void {
|
|
||||||
if (latestRunSyncTimer !== null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
latestRunSyncTimer = setInterval(() => {
|
|
||||||
void runStatus.syncLatest();
|
|
||||||
}, RUN_STATUS_POLL_INTERVAL_MS);
|
|
||||||
}
|
|
||||||
|
|
||||||
function stopLatestRunSyncLoop(): void {
|
|
||||||
if (latestRunSyncTimer === null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
clearInterval(latestRunSyncTimer);
|
|
||||||
latestRunSyncTimer = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function loadSnapshot(): Promise<void> {
|
async function loadSnapshot(): Promise<void> {
|
||||||
loading.value = true;
|
loading.value = true;
|
||||||
errorMessage.value = null;
|
errorMessage.value = null;
|
||||||
|
|
@ -207,11 +189,6 @@ async function onTriggerRun(): Promise<void> {
|
||||||
onMounted(() => {
|
onMounted(() => {
|
||||||
void loadSnapshot();
|
void loadSnapshot();
|
||||||
void runStatus.syncLatest();
|
void runStatus.syncLatest();
|
||||||
startLatestRunSyncLoop();
|
|
||||||
});
|
|
||||||
|
|
||||||
onUnmounted(() => {
|
|
||||||
stopLatestRunSyncLoop();
|
|
||||||
});
|
});
|
||||||
|
|
||||||
watch(
|
watch(
|
||||||
|
|
@ -266,7 +243,12 @@ watch(
|
||||||
</div>
|
</div>
|
||||||
<p class="text-sm text-secondary">Newest papers found while checking your tracked scholar profiles.</p>
|
<p class="text-sm text-secondary">Newest papers found while checking your tracked scholar profiles.</p>
|
||||||
</div>
|
</div>
|
||||||
<RouterLink to="/publications" class="link-inline text-sm">Open full publications view</RouterLink>
|
<RouterLink
|
||||||
|
to="/publications"
|
||||||
|
class="inline-flex min-h-10 items-center justify-center rounded-lg border border-action-secondary-border bg-action-secondary-bg px-3 py-2 text-sm font-semibold text-action-secondary-text shadow-sm transition hover:border-action-secondary-hover-border hover:bg-action-secondary-hover-bg hover:text-action-secondary-hover-text focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset"
|
||||||
|
>
|
||||||
|
view all
|
||||||
|
</RouterLink>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<AppEmptyState
|
<AppEmptyState
|
||||||
|
|
|
||||||
|
|
@ -3,9 +3,11 @@ import { computed, ref } from "vue";
|
||||||
import { useRouter } from "vue-router";
|
import { useRouter } from "vue-router";
|
||||||
|
|
||||||
import AppAlert from "@/components/ui/AppAlert.vue";
|
import AppAlert from "@/components/ui/AppAlert.vue";
|
||||||
|
import AppBrandMark from "@/components/ui/AppBrandMark.vue";
|
||||||
import AppButton from "@/components/ui/AppButton.vue";
|
import AppButton from "@/components/ui/AppButton.vue";
|
||||||
import AppCard from "@/components/ui/AppCard.vue";
|
import AppCard from "@/components/ui/AppCard.vue";
|
||||||
import AppInput from "@/components/ui/AppInput.vue";
|
import AppInput from "@/components/ui/AppInput.vue";
|
||||||
|
import AppThemePicker from "@/components/ui/AppThemePicker.vue";
|
||||||
import { ApiRequestError } from "@/lib/api/errors";
|
import { ApiRequestError } from "@/lib/api/errors";
|
||||||
import { useAuthStore } from "@/stores/auth";
|
import { useAuthStore } from "@/stores/auth";
|
||||||
|
|
||||||
|
|
@ -54,46 +56,22 @@ async function onSubmit(): Promise<void> {
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<template>
|
<template>
|
||||||
<div class="relative h-[100dvh] max-h-[100dvh] overflow-y-auto overflow-x-hidden bg-surface-app">
|
<div class="relative h-[100dvh] max-h-[100dvh] overflow-hidden bg-surface-app">
|
||||||
<div class="pointer-events-none absolute inset-0">
|
<div class="absolute right-4 top-4 z-10 sm:right-6 sm:top-6">
|
||||||
<div class="absolute -top-20 left-[-8rem] h-72 w-72 rounded-full bg-brand-300/30 blur-3xl" />
|
<AppThemePicker compact id-prefix="login-theme" />
|
||||||
<div class="absolute bottom-[-7rem] right-[-6rem] h-80 w-80 rounded-full bg-success-300/25 blur-3xl" />
|
|
||||||
<div class="absolute left-1/3 top-1/3 h-52 w-52 rounded-full bg-warning-300/20 blur-3xl" />
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div
|
<div class="pointer-events-none absolute inset-0">
|
||||||
class="relative mx-auto grid min-h-full w-full max-w-6xl items-center gap-8 px-4 py-6 sm:px-6 lg:grid-cols-[minmax(0,1fr)_26rem] lg:px-8"
|
<div class="absolute -top-28 right-[-8rem] h-72 w-72 rounded-full bg-brand-300/25 blur-3xl" />
|
||||||
>
|
<div class="absolute bottom-[-8rem] left-[-7rem] h-80 w-80 rounded-full bg-info-300/20 blur-3xl" />
|
||||||
<section class="hidden space-y-5 lg:block">
|
</div>
|
||||||
<p class="inline-flex items-center rounded-full border border-stroke-strong bg-surface-card/70 px-3 py-1 text-xs font-medium uppercase tracking-[0.12em] text-ink-muted">
|
|
||||||
Scholarr Control Center
|
|
||||||
</p>
|
|
||||||
<h1 class="max-w-xl font-display text-4xl font-semibold tracking-tight text-ink-primary">
|
|
||||||
Scholar tracking with reliable operational controls.
|
|
||||||
</h1>
|
|
||||||
<p class="max-w-xl text-base leading-relaxed text-ink-muted">
|
|
||||||
Use your account to review ingestion runs, publication changes, and continuation queue diagnostics from a
|
|
||||||
single workspace.
|
|
||||||
</p>
|
|
||||||
<ul class="grid max-w-xl gap-2 text-sm text-ink-muted">
|
|
||||||
<li class="rounded-lg border border-stroke-default bg-surface-card/60 px-3 py-2">
|
|
||||||
Session-based authentication with CSRF enforcement.
|
|
||||||
</li>
|
|
||||||
<li class="rounded-lg border border-stroke-default bg-surface-card/60 px-3 py-2">
|
|
||||||
Run and queue diagnostics are available for support workflows.
|
|
||||||
</li>
|
|
||||||
<li class="rounded-lg border border-stroke-default bg-surface-card/60 px-3 py-2">
|
|
||||||
Theme-aware interface with light and dark mode support.
|
|
||||||
</li>
|
|
||||||
</ul>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<AppCard class="w-full max-w-md justify-self-end space-y-6 border-stroke-default/80 bg-surface-card/90 backdrop-blur">
|
<div class="relative mx-auto grid h-full w-full max-w-md items-center px-4 py-8 sm:px-6">
|
||||||
<div class="space-y-2">
|
<AppCard class="space-y-6 border-stroke-default/80 bg-surface-card/90 p-6 backdrop-blur sm:p-7">
|
||||||
<h1 class="font-display text-3xl font-semibold tracking-tight text-ink-primary">Sign In</h1>
|
<div class="grid justify-items-center gap-2 text-center">
|
||||||
<p class="text-sm text-secondary">
|
<AppBrandMark size="xl" />
|
||||||
Sign in to access scholar tracking, publication updates, and run diagnostics.
|
<p class="font-display text-2xl font-semibold tracking-tight text-ink-primary">scholarr</p>
|
||||||
</p>
|
<h1 class="text-sm font-medium uppercase tracking-[0.12em] text-ink-secondary">Sign in</h1>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<AppAlert v-if="errorMessage" tone="danger">
|
<AppAlert v-if="errorMessage" tone="danger">
|
||||||
|
|
@ -106,7 +84,7 @@ async function onSubmit(): Promise<void> {
|
||||||
<form class="grid gap-4" @submit.prevent="onSubmit">
|
<form class="grid gap-4" @submit.prevent="onSubmit">
|
||||||
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
||||||
<span>Email</span>
|
<span>Email</span>
|
||||||
<AppInput id="login-email" v-model="email" type="email" autocomplete="email" />
|
<AppInput id="login-email" v-model="email" type="email" autocomplete="email" autofocus />
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
||||||
|
|
@ -123,6 +101,10 @@ async function onSubmit(): Promise<void> {
|
||||||
{{ pending ? "Signing in..." : "Sign in" }}
|
{{ pending ? "Signing in..." : "Sign in" }}
|
||||||
</AppButton>
|
</AppButton>
|
||||||
</form>
|
</form>
|
||||||
|
|
||||||
|
<p class="text-center text-xs text-secondary">
|
||||||
|
Use your assigned account credentials.
|
||||||
|
</p>
|
||||||
</AppCard>
|
</AppCard>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -6,10 +6,10 @@ import AppPage from "@/components/layout/AppPage.vue";
|
||||||
import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
|
import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
|
||||||
import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue";
|
import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue";
|
||||||
import RunStatusBadge from "@/components/patterns/RunStatusBadge.vue";
|
import RunStatusBadge from "@/components/patterns/RunStatusBadge.vue";
|
||||||
import AppButton from "@/components/ui/AppButton.vue";
|
|
||||||
import AppCard from "@/components/ui/AppCard.vue";
|
import AppCard from "@/components/ui/AppCard.vue";
|
||||||
import AppEmptyState from "@/components/ui/AppEmptyState.vue";
|
import AppEmptyState from "@/components/ui/AppEmptyState.vue";
|
||||||
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
||||||
|
import AppRefreshButton from "@/components/ui/AppRefreshButton.vue";
|
||||||
import AppTable from "@/components/ui/AppTable.vue";
|
import AppTable from "@/components/ui/AppTable.vue";
|
||||||
import { getRunDetail, type RunDetail } from "@/features/runs";
|
import { getRunDetail, type RunDetail } from "@/features/runs";
|
||||||
import { ApiRequestError } from "@/lib/api/errors";
|
import { ApiRequestError } from "@/lib/api/errors";
|
||||||
|
|
@ -108,9 +108,13 @@ onMounted(() => {
|
||||||
</div>
|
</div>
|
||||||
<p class="text-sm text-secondary">Refresh this run detail view after queue/run updates.</p>
|
<p class="text-sm text-secondary">Refresh this run detail view after queue/run updates.</p>
|
||||||
</div>
|
</div>
|
||||||
<AppButton variant="secondary" @click="loadDetail" :disabled="loading">
|
<AppRefreshButton
|
||||||
{{ loading ? "Refreshing..." : "Refresh" }}
|
variant="secondary"
|
||||||
</AppButton>
|
:loading="loading"
|
||||||
|
title="Refresh run detail"
|
||||||
|
loading-title="Refreshing run detail"
|
||||||
|
@click="loadDetail"
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
</AppCard>
|
</AppCard>
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,7 @@ import AppButton from "@/components/ui/AppButton.vue";
|
||||||
import AppCard from "@/components/ui/AppCard.vue";
|
import AppCard from "@/components/ui/AppCard.vue";
|
||||||
import AppEmptyState from "@/components/ui/AppEmptyState.vue";
|
import AppEmptyState from "@/components/ui/AppEmptyState.vue";
|
||||||
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
||||||
|
import AppRefreshButton from "@/components/ui/AppRefreshButton.vue";
|
||||||
import AppTable from "@/components/ui/AppTable.vue";
|
import AppTable from "@/components/ui/AppTable.vue";
|
||||||
import {
|
import {
|
||||||
clearQueueItem,
|
clearQueueItem,
|
||||||
|
|
@ -210,9 +211,13 @@ onMounted(() => {
|
||||||
<AppButton :disabled="isStartBlocked" :title="startCheckDisabledReason || undefined" @click="onTriggerManualRun">
|
<AppButton :disabled="isStartBlocked" :title="startCheckDisabledReason || undefined" @click="onTriggerManualRun">
|
||||||
{{ runButtonLabel }}
|
{{ runButtonLabel }}
|
||||||
</AppButton>
|
</AppButton>
|
||||||
<AppButton variant="secondary" :disabled="loading" @click="loadData">
|
<AppRefreshButton
|
||||||
{{ loading ? "Refreshing..." : "Refresh" }}
|
variant="secondary"
|
||||||
</AppButton>
|
:loading="loading"
|
||||||
|
title="Refresh runs and queue"
|
||||||
|
loading-title="Refreshing runs and queue"
|
||||||
|
@click="loadData"
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
<div
|
<div
|
||||||
v-if="runStatus.isLikelyRunning"
|
v-if="runStatus.isLikelyRunning"
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ import AppEmptyState from "@/components/ui/AppEmptyState.vue";
|
||||||
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
||||||
import AppInput from "@/components/ui/AppInput.vue";
|
import AppInput from "@/components/ui/AppInput.vue";
|
||||||
import AppModal from "@/components/ui/AppModal.vue";
|
import AppModal from "@/components/ui/AppModal.vue";
|
||||||
|
import AppRefreshButton from "@/components/ui/AppRefreshButton.vue";
|
||||||
import AppSelect from "@/components/ui/AppSelect.vue";
|
import AppSelect from "@/components/ui/AppSelect.vue";
|
||||||
import AppTable from "@/components/ui/AppTable.vue";
|
import AppTable from "@/components/ui/AppTable.vue";
|
||||||
import {
|
import {
|
||||||
|
|
@ -812,9 +813,14 @@ onMounted(() => {
|
||||||
<AppButton variant="secondary" :disabled="loading || importingData" @click="onOpenImportPicker">
|
<AppButton variant="secondary" :disabled="loading || importingData" @click="onOpenImportPicker">
|
||||||
{{ importingData ? "Importing..." : "Import" }}
|
{{ importingData ? "Importing..." : "Import" }}
|
||||||
</AppButton>
|
</AppButton>
|
||||||
<AppButton variant="secondary" :disabled="loading || saving" @click="loadScholars">
|
<AppRefreshButton
|
||||||
{{ loading ? "Refreshing..." : "Refresh" }}
|
variant="secondary"
|
||||||
</AppButton>
|
:disabled="saving"
|
||||||
|
:loading="loading"
|
||||||
|
title="Refresh scholars"
|
||||||
|
loading-title="Refreshing scholars"
|
||||||
|
@click="loadScholars"
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<input
|
<input
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed, onMounted, ref } from "vue";
|
import { computed, onMounted, ref, watch } from "vue";
|
||||||
|
import { useRoute, useRouter } from "vue-router";
|
||||||
|
|
||||||
import AppPage from "@/components/layout/AppPage.vue";
|
import AppPage from "@/components/layout/AppPage.vue";
|
||||||
import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
|
import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
|
||||||
|
|
@ -9,7 +10,8 @@ import AppCard from "@/components/ui/AppCard.vue";
|
||||||
import AppCheckbox from "@/components/ui/AppCheckbox.vue";
|
import AppCheckbox from "@/components/ui/AppCheckbox.vue";
|
||||||
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
||||||
import AppInput from "@/components/ui/AppInput.vue";
|
import AppInput from "@/components/ui/AppInput.vue";
|
||||||
import AppModal from "@/components/ui/AppModal.vue";
|
import AppTabs, { type AppTabItem } from "@/components/ui/AppTabs.vue";
|
||||||
|
import SettingsAdminPanel from "@/features/settings/SettingsAdminPanel.vue";
|
||||||
import {
|
import {
|
||||||
changePassword,
|
changePassword,
|
||||||
fetchSettings,
|
fetchSettings,
|
||||||
|
|
@ -22,65 +24,18 @@ import { useAuthStore } from "@/stores/auth";
|
||||||
import { useRunStatusStore } from "@/stores/run_status";
|
import { useRunStatusStore } from "@/stores/run_status";
|
||||||
import { normalizeUserNavVisiblePages, useUserSettingsStore } from "@/stores/user_settings";
|
import { normalizeUserNavVisiblePages, useUserSettingsStore } from "@/stores/user_settings";
|
||||||
|
|
||||||
interface NavPageOption {
|
const TAB_CHECKING = "checking";
|
||||||
id: string;
|
const TAB_ACCOUNT = "account";
|
||||||
label: string;
|
const TAB_ADMIN_USERS = "admin-users";
|
||||||
description: string;
|
const TAB_ADMIN_INTEGRITY = "admin-integrity";
|
||||||
required: boolean;
|
const TAB_ADMIN_REPAIRS = "admin-repairs";
|
||||||
adminOnly?: boolean;
|
const TAB_ADMIN_PDF = "admin-pdf";
|
||||||
}
|
|
||||||
|
|
||||||
const NAV_PAGE_OPTIONS: NavPageOption[] = [
|
|
||||||
{
|
|
||||||
id: "dashboard",
|
|
||||||
label: "Dashboard",
|
|
||||||
description: "Overview and latest publication updates.",
|
|
||||||
required: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "scholars",
|
|
||||||
label: "Scholars",
|
|
||||||
description: "Tracked scholar profiles and profile management.",
|
|
||||||
required: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "publications",
|
|
||||||
label: "Publications",
|
|
||||||
description: "Review and search discovered publication records.",
|
|
||||||
required: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "settings",
|
|
||||||
label: "Settings",
|
|
||||||
description: "Configuration and account controls.",
|
|
||||||
required: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "style-guide",
|
|
||||||
label: "Style Guide",
|
|
||||||
description: "Admin-only visual reference for theme and component tokens.",
|
|
||||||
required: false,
|
|
||||||
adminOnly: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "runs",
|
|
||||||
label: "Runs",
|
|
||||||
description: "Admin-only diagnostics and queue operations.",
|
|
||||||
required: false,
|
|
||||||
adminOnly: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "users",
|
|
||||||
label: "Users",
|
|
||||||
description: "Admin-only user management.",
|
|
||||||
required: false,
|
|
||||||
adminOnly: true,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const auth = useAuthStore();
|
const auth = useAuthStore();
|
||||||
const userSettings = useUserSettingsStore();
|
const userSettings = useUserSettingsStore();
|
||||||
const runStatus = useRunStatusStore();
|
const runStatus = useRunStatusStore();
|
||||||
|
const route = useRoute();
|
||||||
|
const router = useRouter();
|
||||||
|
|
||||||
const loading = ref(true);
|
const loading = ref(true);
|
||||||
const saving = ref(false);
|
const saving = ref(false);
|
||||||
|
|
@ -98,55 +53,82 @@ const confirmPassword = ref("");
|
||||||
const errorMessage = ref<string | null>(null);
|
const errorMessage = ref<string | null>(null);
|
||||||
const errorRequestId = ref<string | null>(null);
|
const errorRequestId = ref<string | null>(null);
|
||||||
const successMessage = ref<string | null>(null);
|
const successMessage = ref<string | null>(null);
|
||||||
const showIngestionModal = ref(false);
|
|
||||||
const showPasswordModal = ref(false);
|
|
||||||
const showNavigationModal = ref(false);
|
|
||||||
const minCheckIntervalMinutes = ref(15);
|
const minCheckIntervalMinutes = ref(15);
|
||||||
const minRequestDelaySeconds = ref(2);
|
const minRequestDelaySeconds = ref(2);
|
||||||
const automationAllowed = ref(true);
|
const automationAllowed = ref(true);
|
||||||
const manualRunAllowed = ref(true);
|
const manualRunAllowed = ref(true);
|
||||||
const blockedFailureThreshold = ref(1);
|
|
||||||
const networkFailureThreshold = ref(2);
|
|
||||||
const cooldownBlockedSeconds = ref(1800);
|
|
||||||
const cooldownNetworkSeconds = ref(900);
|
|
||||||
|
|
||||||
const visibleNavOptions = computed(() =>
|
const activeTab = ref(TAB_CHECKING);
|
||||||
NAV_PAGE_OPTIONS.filter((option) => !option.adminOnly || auth.isAdmin),
|
|
||||||
);
|
const tabItems = computed<AppTabItem[]>(() => {
|
||||||
const visibleNavLabels = computed(() =>
|
const tabs: AppTabItem[] = [
|
||||||
visibleNavOptions.value
|
{ id: TAB_CHECKING, label: "Checking" },
|
||||||
.filter((option) => navVisiblePages.value.includes(option.id))
|
{ id: TAB_ACCOUNT, label: "Account" },
|
||||||
.map((option) => option.label),
|
];
|
||||||
);
|
if (auth.isAdmin) {
|
||||||
|
tabs.push(
|
||||||
|
{ id: TAB_ADMIN_USERS, label: "Users" },
|
||||||
|
{ id: TAB_ADMIN_INTEGRITY, label: "Integrity" },
|
||||||
|
{ id: TAB_ADMIN_REPAIRS, label: "Repairs" },
|
||||||
|
{ id: TAB_ADMIN_PDF, label: "PDF Queue" },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return tabs;
|
||||||
|
});
|
||||||
|
|
||||||
|
const activeAdminSection = computed<"users" | "integrity" | "repairs" | "pdf" | null>(() => {
|
||||||
|
if (activeTab.value === TAB_ADMIN_USERS) {
|
||||||
|
return "users";
|
||||||
|
}
|
||||||
|
if (activeTab.value === TAB_ADMIN_INTEGRITY) {
|
||||||
|
return "integrity";
|
||||||
|
}
|
||||||
|
if (activeTab.value === TAB_ADMIN_REPAIRS) {
|
||||||
|
return "repairs";
|
||||||
|
}
|
||||||
|
if (activeTab.value === TAB_ADMIN_PDF) {
|
||||||
|
return "pdf";
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
function isKnownTab(tabId: string): boolean {
|
||||||
|
return tabItems.value.some((item) => item.id === tabId);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function syncRouteTab(candidate: string): Promise<void> {
|
||||||
|
const fallback = tabItems.value[0]?.id || TAB_CHECKING;
|
||||||
|
const nextTab = isKnownTab(candidate) ? candidate : fallback;
|
||||||
|
activeTab.value = nextTab;
|
||||||
|
|
||||||
|
const currentQueryTab = typeof route.query.tab === "string" ? route.query.tab : "";
|
||||||
|
if (currentQueryTab === nextTab) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await router.replace({ query: { ...route.query, tab: nextTab } });
|
||||||
|
}
|
||||||
|
|
||||||
function hydrateSettings(settings: UserSettings): void {
|
function hydrateSettings(settings: UserSettings): void {
|
||||||
const parsedMinRunInterval = Number(settings.policy?.min_run_interval_minutes);
|
const parsedMinRunInterval = Number(settings.policy?.min_run_interval_minutes);
|
||||||
minCheckIntervalMinutes.value = Number.isFinite(parsedMinRunInterval)
|
minCheckIntervalMinutes.value = Number.isFinite(parsedMinRunInterval)
|
||||||
? Math.max(15, parsedMinRunInterval)
|
? Math.max(15, parsedMinRunInterval)
|
||||||
: 15;
|
: 15;
|
||||||
|
|
||||||
const parsedMinRequestDelay = Number(settings.policy?.min_request_delay_seconds);
|
const parsedMinRequestDelay = Number(settings.policy?.min_request_delay_seconds);
|
||||||
minRequestDelaySeconds.value = Number.isFinite(parsedMinRequestDelay)
|
minRequestDelaySeconds.value = Number.isFinite(parsedMinRequestDelay)
|
||||||
? Math.max(2, parsedMinRequestDelay)
|
? Math.max(2, parsedMinRequestDelay)
|
||||||
: 2;
|
: 2;
|
||||||
|
|
||||||
automationAllowed.value = Boolean(settings.policy?.automation_allowed ?? true);
|
automationAllowed.value = Boolean(settings.policy?.automation_allowed ?? true);
|
||||||
manualRunAllowed.value = Boolean(settings.policy?.manual_run_allowed ?? true);
|
manualRunAllowed.value = Boolean(settings.policy?.manual_run_allowed ?? true);
|
||||||
blockedFailureThreshold.value = Number.isFinite(settings.policy?.blocked_failure_threshold)
|
|
||||||
? Math.max(1, settings.policy.blocked_failure_threshold)
|
|
||||||
: 1;
|
|
||||||
networkFailureThreshold.value = Number.isFinite(settings.policy?.network_failure_threshold)
|
|
||||||
? Math.max(1, settings.policy.network_failure_threshold)
|
|
||||||
: 2;
|
|
||||||
cooldownBlockedSeconds.value = Number.isFinite(settings.policy?.cooldown_blocked_seconds)
|
|
||||||
? Math.max(60, settings.policy.cooldown_blocked_seconds)
|
|
||||||
: 1800;
|
|
||||||
cooldownNetworkSeconds.value = Number.isFinite(settings.policy?.cooldown_network_seconds)
|
|
||||||
? Math.max(60, settings.policy.cooldown_network_seconds)
|
|
||||||
: 900;
|
|
||||||
|
|
||||||
autoRunEnabled.value = Boolean(settings.auto_run_enabled) && automationAllowed.value;
|
autoRunEnabled.value = Boolean(settings.auto_run_enabled) && automationAllowed.value;
|
||||||
runIntervalMinutes.value = String(settings.run_interval_minutes);
|
runIntervalMinutes.value = String(settings.run_interval_minutes);
|
||||||
requestDelaySeconds.value = String(settings.request_delay_seconds);
|
requestDelaySeconds.value = String(settings.request_delay_seconds);
|
||||||
navVisiblePages.value = normalizeUserNavVisiblePages(settings.nav_visible_pages);
|
navVisiblePages.value = normalizeUserNavVisiblePages(settings.nav_visible_pages);
|
||||||
|
|
||||||
userSettings.applySettings(settings);
|
userSettings.applySettings(settings);
|
||||||
runStatus.setSafetyState(settings.safety_state);
|
runStatus.setSafetyState(settings.safety_state);
|
||||||
}
|
}
|
||||||
|
|
@ -162,31 +144,10 @@ function parseBoundedInteger(value: string, label: string, minimum: number): num
|
||||||
return parsed;
|
return parsed;
|
||||||
}
|
}
|
||||||
|
|
||||||
function isNavPageVisible(pageId: string): boolean {
|
|
||||||
return navVisiblePages.value.includes(pageId);
|
|
||||||
}
|
|
||||||
|
|
||||||
function onToggleNavPage(page: NavPageOption, event: Event): void {
|
|
||||||
const input = event.target as HTMLInputElement;
|
|
||||||
if (page.required && !input.checked) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (input.checked) {
|
|
||||||
navVisiblePages.value = normalizeUserNavVisiblePages([...navVisiblePages.value, page.id]);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
navVisiblePages.value = normalizeUserNavVisiblePages(
|
|
||||||
navVisiblePages.value.filter((pageId) => pageId !== page.id),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function loadSettings(): Promise<void> {
|
async function loadSettings(): Promise<void> {
|
||||||
loading.value = true;
|
loading.value = true;
|
||||||
errorMessage.value = null;
|
errorMessage.value = null;
|
||||||
errorRequestId.value = null;
|
errorRequestId.value = null;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const settings = await fetchSettings();
|
const settings = await fetchSettings();
|
||||||
hydrateSettings(settings);
|
hydrateSettings(settings);
|
||||||
|
|
@ -227,8 +188,6 @@ async function onSaveSettings(): Promise<void> {
|
||||||
const saved = await updateSettings(payload);
|
const saved = await updateSettings(payload);
|
||||||
hydrateSettings(saved);
|
hydrateSettings(saved);
|
||||||
successMessage.value = "Settings updated.";
|
successMessage.value = "Settings updated.";
|
||||||
showIngestionModal.value = false;
|
|
||||||
showNavigationModal.value = false;
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof ApiRequestError) {
|
if (error instanceof ApiRequestError) {
|
||||||
errorMessage.value = error.message;
|
errorMessage.value = error.message;
|
||||||
|
|
@ -264,7 +223,6 @@ async function onChangePassword(): Promise<void> {
|
||||||
newPassword.value = "";
|
newPassword.value = "";
|
||||||
confirmPassword.value = "";
|
confirmPassword.value = "";
|
||||||
successMessage.value = response.message;
|
successMessage.value = response.message;
|
||||||
showPasswordModal.value = false;
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof ApiRequestError) {
|
if (error instanceof ApiRequestError) {
|
||||||
errorMessage.value = error.message;
|
errorMessage.value = error.message;
|
||||||
|
|
@ -279,16 +237,35 @@ async function onChangePassword(): Promise<void> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
onMounted(() => {
|
async function onSelectTab(tabId: string): Promise<void> {
|
||||||
void loadSettings();
|
await syncRouteTab(tabId);
|
||||||
|
}
|
||||||
|
|
||||||
|
watch(
|
||||||
|
() => route.query.tab,
|
||||||
|
async (value) => {
|
||||||
|
const requestedTab = typeof value === "string" ? value : TAB_CHECKING;
|
||||||
|
await syncRouteTab(requestedTab);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
watch(
|
||||||
|
() => auth.isAdmin,
|
||||||
|
async () => {
|
||||||
|
const requestedTab = typeof route.query.tab === "string" ? route.query.tab : TAB_CHECKING;
|
||||||
|
await syncRouteTab(requestedTab);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
onMounted(async () => {
|
||||||
|
await loadSettings();
|
||||||
|
const requestedTab = typeof route.query.tab === "string" ? route.query.tab : TAB_CHECKING;
|
||||||
|
await syncRouteTab(requestedTab);
|
||||||
});
|
});
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<template>
|
<template>
|
||||||
<AppPage
|
<AppPage title="Settings" subtitle="Configuration and account controls in one place.">
|
||||||
title="Settings"
|
|
||||||
subtitle="Control how often Scholarr checks profiles and how cautiously it sends requests."
|
|
||||||
>
|
|
||||||
<RequestStateAlerts
|
<RequestStateAlerts
|
||||||
:success-message="successMessage"
|
:success-message="successMessage"
|
||||||
success-title="Saved"
|
success-title="Saved"
|
||||||
|
|
@ -298,189 +275,91 @@ onMounted(() => {
|
||||||
@dismiss-success="successMessage = null"
|
@dismiss-success="successMessage = null"
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<AsyncStateGate :loading="loading" :loading-lines="7" :show-empty="false">
|
<AsyncStateGate :loading="loading" :loading-lines="8" :show-empty="false">
|
||||||
<section class="grid gap-4 xl:grid-cols-3">
|
<AppCard class="grid gap-4">
|
||||||
<AppCard class="flex h-full flex-col gap-4">
|
<AppTabs :model-value="activeTab" :items="tabItems" aria-label="Settings sections" @update:model-value="onSelectTab" />
|
||||||
<div class="flex items-center gap-1">
|
|
||||||
<h2 class="text-lg font-semibold text-ink-primary">Automatic Checking</h2>
|
|
||||||
<AppHelpHint text="Controls when Scholarr runs automatic profile checks and how cautiously it scrapes." />
|
|
||||||
</div>
|
|
||||||
<p class="text-sm text-secondary">
|
|
||||||
Configure the background checker that looks for new publications on your tracked profiles.
|
|
||||||
</p>
|
|
||||||
<AppButton variant="secondary" class="mt-auto self-start" @click="showIngestionModal = true">
|
|
||||||
Edit checking rules
|
|
||||||
</AppButton>
|
|
||||||
</AppCard>
|
|
||||||
|
|
||||||
<AppCard class="flex h-full flex-col gap-4">
|
<section v-if="activeTab === TAB_CHECKING" class="grid gap-4">
|
||||||
|
<div class="flex items-center gap-1">
|
||||||
|
<h2 class="text-lg font-semibold text-ink-primary">Automatic Checking</h2>
|
||||||
|
<AppHelpHint text="Controls when Scholarr runs automatic profile checks and how cautiously it scrapes." />
|
||||||
|
</div>
|
||||||
|
<p class="text-sm text-secondary">Configure background checks and request pacing for ingestion.</p>
|
||||||
|
|
||||||
|
<div class="grid gap-3 md:grid-cols-2 md:items-start">
|
||||||
|
<div class="grid gap-2 rounded-lg border border-stroke-default bg-surface-card-muted p-3 md:col-span-2">
|
||||||
|
<AppCheckbox
|
||||||
|
id="auto-run-enabled"
|
||||||
|
v-model="autoRunEnabled"
|
||||||
|
:disabled="!automationAllowed"
|
||||||
|
label="Enable automatic background checks"
|
||||||
|
/>
|
||||||
|
<p class="text-xs text-secondary">
|
||||||
|
Automatic checks are {{ automationAllowed ? "enabled by policy." : "disabled by server safety policy." }}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
||||||
|
<span class="inline-flex items-center gap-1">
|
||||||
|
Check interval (minutes)
|
||||||
|
<AppHelpHint text="Minimum is controlled by server policy." />
|
||||||
|
</span>
|
||||||
|
<AppInput v-model="runIntervalMinutes" inputmode="numeric" />
|
||||||
|
<span class="text-xs text-secondary">Minimum: {{ minCheckIntervalMinutes }}</span>
|
||||||
|
</label>
|
||||||
|
|
||||||
|
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
||||||
|
<span class="inline-flex items-center gap-1">
|
||||||
|
Delay between requests (seconds)
|
||||||
|
<AppHelpHint text="Minimum is controlled by safety policy to reduce blocks." />
|
||||||
|
</span>
|
||||||
|
<AppInput v-model="requestDelaySeconds" inputmode="numeric" />
|
||||||
|
<span class="text-xs text-secondary">Minimum: {{ minRequestDelaySeconds }}</span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex flex-wrap items-center gap-3 text-xs text-secondary">
|
||||||
|
<span>Automation allowed: {{ automationAllowed ? "yes" : "no" }}</span>
|
||||||
|
<span>Manual runs allowed: {{ manualRunAllowed ? "yes" : "no" }}</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<AppButton :disabled="saving" @click="onSaveSettings">
|
||||||
|
{{ saving ? "Saving..." : "Save checking settings" }}
|
||||||
|
</AppButton>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section v-if="activeTab === TAB_ACCOUNT" class="grid gap-4">
|
||||||
<div class="flex items-center gap-1">
|
<div class="flex items-center gap-1">
|
||||||
<h2 class="text-lg font-semibold text-ink-primary">Account Access</h2>
|
<h2 class="text-lg font-semibold text-ink-primary">Account Access</h2>
|
||||||
<AppHelpHint text="Manage credentials for your current signed-in account on this instance." />
|
<AppHelpHint text="Manage credentials for your current signed-in account on this instance." />
|
||||||
</div>
|
</div>
|
||||||
<p class="text-sm text-secondary">
|
<p class="text-sm text-secondary">Change your sign-in password directly from this tab.</p>
|
||||||
Change your sign-in password from a focused view. This does not affect other users.
|
<form class="grid gap-3 rounded-lg border border-stroke-default bg-surface-card-muted p-3" @submit.prevent="onChangePassword">
|
||||||
</p>
|
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
|
||||||
<AppButton variant="secondary" class="mt-auto self-start" @click="showPasswordModal = true">
|
<span>Current password</span>
|
||||||
Change password
|
<AppInput v-model="currentPassword" type="password" autocomplete="current-password" />
|
||||||
</AppButton>
|
</label>
|
||||||
</AppCard>
|
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
|
||||||
|
<span>New password</span>
|
||||||
|
<AppInput v-model="newPassword" type="password" autocomplete="new-password" />
|
||||||
|
</label>
|
||||||
|
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
|
||||||
|
<span>Confirm new password</span>
|
||||||
|
<AppInput v-model="confirmPassword" type="password" autocomplete="new-password" />
|
||||||
|
</label>
|
||||||
|
<div class="flex gap-2">
|
||||||
|
<AppButton type="submit" :disabled="updatingPassword">
|
||||||
|
{{ updatingPassword ? "Updating..." : "Update password" }}
|
||||||
|
</AppButton>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</section>
|
||||||
|
|
||||||
<AppCard class="flex h-full flex-col gap-4">
|
<SettingsAdminPanel v-if="activeAdminSection" :section="activeAdminSection" />
|
||||||
<div class="flex items-center gap-1">
|
</AppCard>
|
||||||
<h2 class="text-lg font-semibold text-ink-primary">Navigation</h2>
|
|
||||||
<AppHelpHint text="Choose which pages appear in the left sidebar. Dashboard, Scholars, and Settings are always visible." />
|
|
||||||
</div>
|
|
||||||
<p class="text-sm text-secondary">
|
|
||||||
Visible now: {{ visibleNavLabels.length > 0 ? visibleNavLabels.join(", ") : "none" }}.
|
|
||||||
</p>
|
|
||||||
<AppButton variant="secondary" class="mt-auto self-start" @click="showNavigationModal = true">
|
|
||||||
Customize sidebar pages
|
|
||||||
</AppButton>
|
|
||||||
</AppCard>
|
|
||||||
</section>
|
|
||||||
</AsyncStateGate>
|
</AsyncStateGate>
|
||||||
|
|
||||||
<AppModal
|
|
||||||
:open="showIngestionModal"
|
|
||||||
title="Automatic Checking Settings"
|
|
||||||
@close="showIngestionModal = false"
|
|
||||||
>
|
|
||||||
<form class="grid gap-3" @submit.prevent="onSaveSettings">
|
|
||||||
<AppCheckbox
|
|
||||||
id="auto-run-enabled"
|
|
||||||
v-model="autoRunEnabled"
|
|
||||||
:disabled="!automationAllowed"
|
|
||||||
label="Enable automatic background checks"
|
|
||||||
/>
|
|
||||||
<p v-if="!automationAllowed" class="text-xs text-secondary">
|
|
||||||
Automatic checks are disabled by server safety policy.
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
|
||||||
<span class="inline-flex items-center gap-1">
|
|
||||||
Check interval (minutes)
|
|
||||||
<AppHelpHint text="How often Scholarr starts a background update check." />
|
|
||||||
</span>
|
|
||||||
<AppInput id="run-interval" v-model="runIntervalMinutes" type="number" :min="minCheckIntervalMinutes" />
|
|
||||||
<span class="text-xs text-secondary">Minimum: {{ minCheckIntervalMinutes }} minutes.</span>
|
|
||||||
</label>
|
|
||||||
|
|
||||||
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
|
||||||
<span class="inline-flex items-center gap-1">
|
|
||||||
Delay between requests (seconds)
|
|
||||||
<AppHelpHint text="Pause between profile requests during a check. Higher values are slower but safer." />
|
|
||||||
</span>
|
|
||||||
<AppInput
|
|
||||||
id="request-delay"
|
|
||||||
v-model="requestDelaySeconds"
|
|
||||||
type="number"
|
|
||||||
:min="minRequestDelaySeconds"
|
|
||||||
/>
|
|
||||||
<span class="text-xs text-secondary">Minimum: {{ minRequestDelaySeconds }} seconds.</span>
|
|
||||||
</label>
|
|
||||||
|
|
||||||
<div class="grid gap-1 rounded-lg border border-stroke-default bg-surface-card-muted px-3 py-2 text-xs text-secondary">
|
|
||||||
<p class="font-medium text-ink-primary">Server-enforced scrape safety policy</p>
|
|
||||||
<p>Blocked failures trigger cooldown at {{ blockedFailureThreshold }} failures.</p>
|
|
||||||
<p>Network failures trigger cooldown at {{ networkFailureThreshold }} failures.</p>
|
|
||||||
<p>Blocked cooldown: {{ cooldownBlockedSeconds }}s. Network cooldown: {{ cooldownNetworkSeconds }}s.</p>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="mt-2 flex flex-wrap justify-end gap-2">
|
|
||||||
<AppButton
|
|
||||||
variant="ghost"
|
|
||||||
type="button"
|
|
||||||
:disabled="saving"
|
|
||||||
@click="showIngestionModal = false"
|
|
||||||
>
|
|
||||||
Cancel
|
|
||||||
</AppButton>
|
|
||||||
<AppButton type="submit" :disabled="saving">
|
|
||||||
{{ saving ? "Saving..." : "Save settings" }}
|
|
||||||
</AppButton>
|
|
||||||
</div>
|
|
||||||
</form>
|
|
||||||
</AppModal>
|
|
||||||
|
|
||||||
<AppModal :open="showPasswordModal" title="Change Sign-in Password" @close="showPasswordModal = false">
|
|
||||||
<form class="grid gap-3" @submit.prevent="onChangePassword">
|
|
||||||
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
|
||||||
<span>Current password</span>
|
|
||||||
<AppInput v-model="currentPassword" type="password" autocomplete="current-password" />
|
|
||||||
</label>
|
|
||||||
|
|
||||||
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
|
||||||
<span>New password</span>
|
|
||||||
<AppInput v-model="newPassword" type="password" autocomplete="new-password" />
|
|
||||||
</label>
|
|
||||||
|
|
||||||
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
|
||||||
<span>Confirm new password</span>
|
|
||||||
<AppInput v-model="confirmPassword" type="password" autocomplete="new-password" />
|
|
||||||
</label>
|
|
||||||
|
|
||||||
<div class="mt-2 flex flex-wrap justify-end gap-2">
|
|
||||||
<AppButton
|
|
||||||
variant="ghost"
|
|
||||||
type="button"
|
|
||||||
:disabled="updatingPassword"
|
|
||||||
@click="showPasswordModal = false"
|
|
||||||
>
|
|
||||||
Cancel
|
|
||||||
</AppButton>
|
|
||||||
<AppButton type="submit" :disabled="updatingPassword">
|
|
||||||
{{ updatingPassword ? "Updating..." : "Change password" }}
|
|
||||||
</AppButton>
|
|
||||||
</div>
|
|
||||||
</form>
|
|
||||||
</AppModal>
|
|
||||||
|
|
||||||
<AppModal :open="showNavigationModal" title="Sidebar Page Visibility" @close="showNavigationModal = false">
|
|
||||||
<form class="grid gap-3" @submit.prevent="onSaveSettings">
|
|
||||||
<p class="text-sm text-secondary">
|
|
||||||
Turn optional pages on or off in the sidebar for your account.
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<ul class="grid gap-2">
|
|
||||||
<li
|
|
||||||
v-for="option in visibleNavOptions"
|
|
||||||
:key="option.id"
|
|
||||||
class="rounded-lg border border-stroke-default bg-surface-card-muted/70 px-3 py-2"
|
|
||||||
>
|
|
||||||
<label class="flex items-start gap-3">
|
|
||||||
<input
|
|
||||||
type="checkbox"
|
|
||||||
class="mt-0.5 h-4 w-4 rounded border-stroke-interactive bg-surface-input text-brand-600 focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset"
|
|
||||||
:checked="isNavPageVisible(option.id)"
|
|
||||||
:disabled="saving || option.required"
|
|
||||||
@change="onToggleNavPage(option, $event)"
|
|
||||||
/>
|
|
||||||
<span class="grid gap-0.5">
|
|
||||||
<span class="text-sm font-medium text-ink-primary">
|
|
||||||
{{ option.label }}
|
|
||||||
<span v-if="option.required" class="ml-1 text-xs text-ink-muted">(required)</span>
|
|
||||||
</span>
|
|
||||||
<span class="text-xs text-secondary">{{ option.description }}</span>
|
|
||||||
</span>
|
|
||||||
</label>
|
|
||||||
</li>
|
|
||||||
</ul>
|
|
||||||
|
|
||||||
<div class="mt-2 flex flex-wrap justify-end gap-2">
|
|
||||||
<AppButton
|
|
||||||
variant="ghost"
|
|
||||||
type="button"
|
|
||||||
:disabled="saving"
|
|
||||||
@click="showNavigationModal = false"
|
|
||||||
>
|
|
||||||
Cancel
|
|
||||||
</AppButton>
|
|
||||||
<AppButton type="submit" :disabled="saving">
|
|
||||||
{{ saving ? "Saving..." : "Save sidebar settings" }}
|
|
||||||
</AppButton>
|
|
||||||
</div>
|
|
||||||
</form>
|
|
||||||
</AppModal>
|
|
||||||
</AppPage>
|
</AppPage>
|
||||||
</template>
|
</template>
|
||||||
|
|
|
||||||
313
frontend/src/theme/presets/canopy.js
Normal file
|
|
@ -0,0 +1,313 @@
|
||||||
|
export default {
|
||||||
|
"id": "canopy",
|
||||||
|
"label": "Canopy",
|
||||||
|
"description": "Vibrant forest green palette with crisp mint contrast.",
|
||||||
|
"modes": {
|
||||||
|
"light": {
|
||||||
|
"scale": {
|
||||||
|
"brand": {
|
||||||
|
"50": "#ecfdf5",
|
||||||
|
"100": "#d1fae5",
|
||||||
|
"200": "#a7f3d0",
|
||||||
|
"300": "#6ee7b7",
|
||||||
|
"400": "#34d399",
|
||||||
|
"500": "#10b981",
|
||||||
|
"600": "#059669",
|
||||||
|
"700": "#047857",
|
||||||
|
"800": "#065f46",
|
||||||
|
"900": "#064e3b",
|
||||||
|
"950": "#022c22"
|
||||||
|
},
|
||||||
|
"info": {
|
||||||
|
"50": "#ecfeff",
|
||||||
|
"100": "#cffafe",
|
||||||
|
"200": "#a5f3fc",
|
||||||
|
"300": "#67e8f9",
|
||||||
|
"400": "#22d3ee",
|
||||||
|
"500": "#06b6d4",
|
||||||
|
"600": "#0891b2",
|
||||||
|
"700": "#0e7490",
|
||||||
|
"800": "#155e75",
|
||||||
|
"900": "#164e63",
|
||||||
|
"950": "#083344"
|
||||||
|
},
|
||||||
|
"success": {
|
||||||
|
"50": "#f0fdf4",
|
||||||
|
"100": "#dcfce7",
|
||||||
|
"200": "#bbf7d0",
|
||||||
|
"300": "#86efac",
|
||||||
|
"400": "#4ade80",
|
||||||
|
"500": "#22c55e",
|
||||||
|
"600": "#16a34a",
|
||||||
|
"700": "#15803d",
|
||||||
|
"800": "#166534",
|
||||||
|
"900": "#14532d",
|
||||||
|
"950": "#052e16"
|
||||||
|
},
|
||||||
|
"warning": {
|
||||||
|
"50": "#fffbeb",
|
||||||
|
"100": "#fef3c7",
|
||||||
|
"200": "#fde68a",
|
||||||
|
"300": "#fcd34d",
|
||||||
|
"400": "#fbbf24",
|
||||||
|
"500": "#f59e0b",
|
||||||
|
"600": "#d97706",
|
||||||
|
"700": "#b45309",
|
||||||
|
"800": "#92400e",
|
||||||
|
"900": "#78350f",
|
||||||
|
"950": "#451a03"
|
||||||
|
},
|
||||||
|
"danger": {
|
||||||
|
"50": "#fff1f2",
|
||||||
|
"100": "#ffe4e6",
|
||||||
|
"200": "#fecdd3",
|
||||||
|
"300": "#fda4af",
|
||||||
|
"400": "#fb7185",
|
||||||
|
"500": "#f43f5e",
|
||||||
|
"600": "#e11d48",
|
||||||
|
"700": "#be123c",
|
||||||
|
"800": "#9f1239",
|
||||||
|
"900": "#881337",
|
||||||
|
"950": "#4c0519"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"surface": {
|
||||||
|
"app": "#eefbf4",
|
||||||
|
"nav": "#ffffff",
|
||||||
|
"nav_active": "#d1fae5",
|
||||||
|
"card": "#ffffff",
|
||||||
|
"card_muted": "#e8f7ef",
|
||||||
|
"table": "#ffffff",
|
||||||
|
"table_header": "#eefbf4",
|
||||||
|
"input": "#ffffff",
|
||||||
|
"overlay": "#0f172a"
|
||||||
|
},
|
||||||
|
"text": {
|
||||||
|
"primary": "#123524",
|
||||||
|
"secondary": "#166534",
|
||||||
|
"muted": "#4b7c67",
|
||||||
|
"inverse": "#ffffff",
|
||||||
|
"link": "#047857"
|
||||||
|
},
|
||||||
|
"border": {
|
||||||
|
"default": "#ccefe0",
|
||||||
|
"strong": "#9dd8bf",
|
||||||
|
"subtle": "#eaf8f1",
|
||||||
|
"interactive": "#34d399"
|
||||||
|
},
|
||||||
|
"focus": {
|
||||||
|
"ring": "#10b981",
|
||||||
|
"ring_offset": "#eefbf4"
|
||||||
|
},
|
||||||
|
"action": {
|
||||||
|
"primary": {
|
||||||
|
"bg": "#10b981",
|
||||||
|
"border": "#10b981",
|
||||||
|
"text": "#ffffff",
|
||||||
|
"hover_bg": "#059669",
|
||||||
|
"hover_border": "#059669",
|
||||||
|
"hover_text": "#ffffff"
|
||||||
|
},
|
||||||
|
"secondary": {
|
||||||
|
"bg": "#e8f7ef",
|
||||||
|
"border": "#ccefe0",
|
||||||
|
"text": "#123524",
|
||||||
|
"hover_bg": "#d1fae5",
|
||||||
|
"hover_border": "#9dd8bf",
|
||||||
|
"hover_text": "#052e16"
|
||||||
|
},
|
||||||
|
"ghost": {
|
||||||
|
"bg": "#eefbf4",
|
||||||
|
"border": "#ccefe0",
|
||||||
|
"text": "#047857",
|
||||||
|
"hover_bg": "#e2f4ea",
|
||||||
|
"hover_border": "#9dd8bf",
|
||||||
|
"hover_text": "#0b3f2b"
|
||||||
|
},
|
||||||
|
"danger": {
|
||||||
|
"bg": "#ffe4e6",
|
||||||
|
"border": "#fecdd3",
|
||||||
|
"text": "#be123c",
|
||||||
|
"hover_bg": "#fecdd3",
|
||||||
|
"hover_border": "#fda4af",
|
||||||
|
"hover_text": "#9f1239"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"info": {
|
||||||
|
"bg": "#cffafe",
|
||||||
|
"border": "#a5f3fc",
|
||||||
|
"text": "#155e75"
|
||||||
|
},
|
||||||
|
"success": {
|
||||||
|
"bg": "#dcfce7",
|
||||||
|
"border": "#bbf7d0",
|
||||||
|
"text": "#166534"
|
||||||
|
},
|
||||||
|
"warning": {
|
||||||
|
"bg": "#fef3c7",
|
||||||
|
"border": "#fde68a",
|
||||||
|
"text": "#92400e"
|
||||||
|
},
|
||||||
|
"danger": {
|
||||||
|
"bg": "#ffe4e6",
|
||||||
|
"border": "#fecdd3",
|
||||||
|
"text": "#be123c"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"dark": {
|
||||||
|
"scale": {
|
||||||
|
"brand": {
|
||||||
|
"50": "#ecfdf5",
|
||||||
|
"100": "#d1fae5",
|
||||||
|
"200": "#a7f3d0",
|
||||||
|
"300": "#6ee7b7",
|
||||||
|
"400": "#34d399",
|
||||||
|
"500": "#10b981",
|
||||||
|
"600": "#059669",
|
||||||
|
"700": "#047857",
|
||||||
|
"800": "#065f46",
|
||||||
|
"900": "#064e3b",
|
||||||
|
"950": "#022c22"
|
||||||
|
},
|
||||||
|
"info": {
|
||||||
|
"50": "#ecfeff",
|
||||||
|
"100": "#cffafe",
|
||||||
|
"200": "#a5f3fc",
|
||||||
|
"300": "#67e8f9",
|
||||||
|
"400": "#22d3ee",
|
||||||
|
"500": "#06b6d4",
|
||||||
|
"600": "#0891b2",
|
||||||
|
"700": "#0e7490",
|
||||||
|
"800": "#155e75",
|
||||||
|
"900": "#164e63",
|
||||||
|
"950": "#083344"
|
||||||
|
},
|
||||||
|
"success": {
|
||||||
|
"50": "#f0fdf4",
|
||||||
|
"100": "#dcfce7",
|
||||||
|
"200": "#bbf7d0",
|
||||||
|
"300": "#86efac",
|
||||||
|
"400": "#4ade80",
|
||||||
|
"500": "#22c55e",
|
||||||
|
"600": "#16a34a",
|
||||||
|
"700": "#15803d",
|
||||||
|
"800": "#166534",
|
||||||
|
"900": "#14532d",
|
||||||
|
"950": "#052e16"
|
||||||
|
},
|
||||||
|
"warning": {
|
||||||
|
"50": "#fffbeb",
|
||||||
|
"100": "#fef3c7",
|
||||||
|
"200": "#fde68a",
|
||||||
|
"300": "#fcd34d",
|
||||||
|
"400": "#fbbf24",
|
||||||
|
"500": "#f59e0b",
|
||||||
|
"600": "#d97706",
|
||||||
|
"700": "#b45309",
|
||||||
|
"800": "#92400e",
|
||||||
|
"900": "#78350f",
|
||||||
|
"950": "#451a03"
|
||||||
|
},
|
||||||
|
"danger": {
|
||||||
|
"50": "#fff1f2",
|
||||||
|
"100": "#ffe4e6",
|
||||||
|
"200": "#fecdd3",
|
||||||
|
"300": "#fda4af",
|
||||||
|
"400": "#fb7185",
|
||||||
|
"500": "#f43f5e",
|
||||||
|
"600": "#e11d48",
|
||||||
|
"700": "#be123c",
|
||||||
|
"800": "#9f1239",
|
||||||
|
"900": "#881337",
|
||||||
|
"950": "#4c0519"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"surface": {
|
||||||
|
"app": "#0b1f17",
|
||||||
|
"nav": "#10271d",
|
||||||
|
"nav_active": "#047857",
|
||||||
|
"card": "#10271d",
|
||||||
|
"card_muted": "#163427",
|
||||||
|
"table": "#10271d",
|
||||||
|
"table_header": "#163427",
|
||||||
|
"input": "#0a1a14",
|
||||||
|
"overlay": "#030712"
|
||||||
|
},
|
||||||
|
"text": {
|
||||||
|
"primary": "#d9f9e8",
|
||||||
|
"secondary": "#9ad8b8",
|
||||||
|
"muted": "#6ea58a",
|
||||||
|
"inverse": "#0b1f17",
|
||||||
|
"link": "#6ee7b7"
|
||||||
|
},
|
||||||
|
"border": {
|
||||||
|
"default": "#1f4635",
|
||||||
|
"strong": "#2a5b45",
|
||||||
|
"subtle": "#163427",
|
||||||
|
"interactive": "#10b981"
|
||||||
|
},
|
||||||
|
"focus": {
|
||||||
|
"ring": "#34d399",
|
||||||
|
"ring_offset": "#0b1f17"
|
||||||
|
},
|
||||||
|
"action": {
|
||||||
|
"primary": {
|
||||||
|
"bg": "#10b981",
|
||||||
|
"border": "#10b981",
|
||||||
|
"text": "#052e16",
|
||||||
|
"hover_bg": "#34d399",
|
||||||
|
"hover_border": "#34d399",
|
||||||
|
"hover_text": "#052e16"
|
||||||
|
},
|
||||||
|
"secondary": {
|
||||||
|
"bg": "#163427",
|
||||||
|
"border": "#2a5b45",
|
||||||
|
"text": "#d9f9e8",
|
||||||
|
"hover_bg": "#214c38",
|
||||||
|
"hover_border": "#2f654d",
|
||||||
|
"hover_text": "#ffffff"
|
||||||
|
},
|
||||||
|
"ghost": {
|
||||||
|
"bg": "#0b1f17",
|
||||||
|
"border": "#1f4635",
|
||||||
|
"text": "#9ad8b8",
|
||||||
|
"hover_bg": "#163427",
|
||||||
|
"hover_border": "#2a5b45",
|
||||||
|
"hover_text": "#d9f9e8"
|
||||||
|
},
|
||||||
|
"danger": {
|
||||||
|
"bg": "#4c0519",
|
||||||
|
"border": "#9f1239",
|
||||||
|
"text": "#fecdd3",
|
||||||
|
"hover_bg": "#5b0b24",
|
||||||
|
"hover_border": "#be123c",
|
||||||
|
"hover_text": "#ffe4e6"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"info": {
|
||||||
|
"bg": "#164e63",
|
||||||
|
"border": "#0e7490",
|
||||||
|
"text": "#cffafe"
|
||||||
|
},
|
||||||
|
"success": {
|
||||||
|
"bg": "#14532d",
|
||||||
|
"border": "#15803d",
|
||||||
|
"text": "#bbf7d0"
|
||||||
|
},
|
||||||
|
"warning": {
|
||||||
|
"bg": "#78350f",
|
||||||
|
"border": "#b45309",
|
||||||
|
"text": "#fde68a"
|
||||||
|
},
|
||||||
|
"danger": {
|
||||||
|
"bg": "#4c0519",
|
||||||
|
"border": "#be123c",
|
||||||
|
"text": "#fecdd3"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
@ -1,22 +1,22 @@
|
||||||
export default {
|
export default {
|
||||||
"id": "dune",
|
"id": "dune",
|
||||||
"label": "Dune",
|
"label": "Dune",
|
||||||
"description": "Sandy beige environments featuring soft mauve highlights.",
|
"description": "Sandy clay surfaces with warm terracotta accents.",
|
||||||
"modes": {
|
"modes": {
|
||||||
"light": {
|
"light": {
|
||||||
"scale": {
|
"scale": {
|
||||||
"brand": {
|
"brand": {
|
||||||
"50": "#f7f5f6",
|
"50": "#fff5ef",
|
||||||
"100": "#ece7e9",
|
"100": "#fee8da",
|
||||||
"200": "#d7cccd",
|
"200": "#fbcfb2",
|
||||||
"300": "#baabb2",
|
"300": "#f6ae81",
|
||||||
"400": "#9e8893",
|
"400": "#ef8753",
|
||||||
"500": "#856b77",
|
"500": "#e2672f",
|
||||||
"600": "#6c5460",
|
"600": "#bf4f25",
|
||||||
"700": "#58424d",
|
"700": "#9a3f21",
|
||||||
"800": "#4a3841",
|
"800": "#7d3420",
|
||||||
"900": "#3d3036",
|
"900": "#662d1d",
|
||||||
"950": "#21181d"
|
"950": "#38140e"
|
||||||
},
|
},
|
||||||
"info": {
|
"info": {
|
||||||
"50": "#f4f5f8",
|
"50": "#f4f5f8",
|
||||||
|
|
@ -74,7 +74,7 @@ export default {
|
||||||
"surface": {
|
"surface": {
|
||||||
"app": "#efece5",
|
"app": "#efece5",
|
||||||
"nav": "#f7f5f0",
|
"nav": "#f7f5f0",
|
||||||
"nav_active": "#d7cccd",
|
"nav_active": "#fbcfb2",
|
||||||
"card": "#f7f5f0",
|
"card": "#f7f5f0",
|
||||||
"card_muted": "#e6e3dd",
|
"card_muted": "#e6e3dd",
|
||||||
"table": "#f7f5f0",
|
"table": "#f7f5f0",
|
||||||
|
|
@ -87,25 +87,25 @@ export default {
|
||||||
"secondary": "#757072",
|
"secondary": "#757072",
|
||||||
"muted": "#9c9699",
|
"muted": "#9c9699",
|
||||||
"inverse": "#f7f5f0",
|
"inverse": "#f7f5f0",
|
||||||
"link": "#6c5460"
|
"link": "#bf4f25"
|
||||||
},
|
},
|
||||||
"border": {
|
"border": {
|
||||||
"default": "#e6e3dd",
|
"default": "#e6e3dd",
|
||||||
"strong": "#d1cdc6",
|
"strong": "#d1cdc6",
|
||||||
"subtle": "#f2f0eb",
|
"subtle": "#f2f0eb",
|
||||||
"interactive": "#baabb2"
|
"interactive": "#f6ae81"
|
||||||
},
|
},
|
||||||
"focus": {
|
"focus": {
|
||||||
"ring": "#9e8893",
|
"ring": "#ef8753",
|
||||||
"ring_offset": "#efece5"
|
"ring_offset": "#efece5"
|
||||||
},
|
},
|
||||||
"action": {
|
"action": {
|
||||||
"primary": {
|
"primary": {
|
||||||
"bg": "#9e8893",
|
"bg": "#ef8753",
|
||||||
"border": "#9e8893",
|
"border": "#ef8753",
|
||||||
"text": "#f7f5f0",
|
"text": "#f7f5f0",
|
||||||
"hover_bg": "#856b77",
|
"hover_bg": "#e2672f",
|
||||||
"hover_border": "#856b77",
|
"hover_border": "#e2672f",
|
||||||
"hover_text": "#ffffff"
|
"hover_text": "#ffffff"
|
||||||
},
|
},
|
||||||
"secondary": {
|
"secondary": {
|
||||||
|
|
@ -119,7 +119,7 @@ export default {
|
||||||
"ghost": {
|
"ghost": {
|
||||||
"bg": "#efece5",
|
"bg": "#efece5",
|
||||||
"border": "#e6e3dd",
|
"border": "#e6e3dd",
|
||||||
"text": "#6c5460",
|
"text": "#bf4f25",
|
||||||
"hover_bg": "#e6e3dd",
|
"hover_bg": "#e6e3dd",
|
||||||
"hover_border": "#d1cdc6",
|
"hover_border": "#d1cdc6",
|
||||||
"hover_text": "#3d3a3b"
|
"hover_text": "#3d3a3b"
|
||||||
|
|
@ -159,17 +159,17 @@ export default {
|
||||||
"dark": {
|
"dark": {
|
||||||
"scale": {
|
"scale": {
|
||||||
"brand": {
|
"brand": {
|
||||||
"50": "#f7f5f6",
|
"50": "#fff5ef",
|
||||||
"100": "#ece7e9",
|
"100": "#fee8da",
|
||||||
"200": "#d7cccd",
|
"200": "#fbcfb2",
|
||||||
"300": "#baabb2",
|
"300": "#f6ae81",
|
||||||
"400": "#9e8893",
|
"400": "#ef8753",
|
||||||
"500": "#856b77",
|
"500": "#e2672f",
|
||||||
"600": "#6c5460",
|
"600": "#bf4f25",
|
||||||
"700": "#58424d",
|
"700": "#9a3f21",
|
||||||
"800": "#4a3841",
|
"800": "#7d3420",
|
||||||
"900": "#3d3036",
|
"900": "#662d1d",
|
||||||
"950": "#21181d"
|
"950": "#38140e"
|
||||||
},
|
},
|
||||||
"info": {
|
"info": {
|
||||||
"50": "#f4f5f8",
|
"50": "#f4f5f8",
|
||||||
|
|
@ -227,7 +227,7 @@ export default {
|
||||||
"surface": {
|
"surface": {
|
||||||
"app": "#3d3a3b",
|
"app": "#3d3a3b",
|
||||||
"nav": "#4a4748",
|
"nav": "#4a4748",
|
||||||
"nav_active": "#6c5460",
|
"nav_active": "#bf4f25",
|
||||||
"card": "#4a4748",
|
"card": "#4a4748",
|
||||||
"card_muted": "#575354",
|
"card_muted": "#575354",
|
||||||
"table": "#4a4748",
|
"table": "#4a4748",
|
||||||
|
|
@ -240,25 +240,25 @@ export default {
|
||||||
"secondary": "#b8b4b6",
|
"secondary": "#b8b4b6",
|
||||||
"muted": "#918e8f",
|
"muted": "#918e8f",
|
||||||
"inverse": "#3d3a3b",
|
"inverse": "#3d3a3b",
|
||||||
"link": "#baabb2"
|
"link": "#f6ae81"
|
||||||
},
|
},
|
||||||
"border": {
|
"border": {
|
||||||
"default": "#575354",
|
"default": "#575354",
|
||||||
"strong": "#696466",
|
"strong": "#696466",
|
||||||
"subtle": "#474345",
|
"subtle": "#474345",
|
||||||
"interactive": "#9e8893"
|
"interactive": "#ef8753"
|
||||||
},
|
},
|
||||||
"focus": {
|
"focus": {
|
||||||
"ring": "#9e8893",
|
"ring": "#ef8753",
|
||||||
"ring_offset": "#3d3a3b"
|
"ring_offset": "#3d3a3b"
|
||||||
},
|
},
|
||||||
"action": {
|
"action": {
|
||||||
"primary": {
|
"primary": {
|
||||||
"bg": "#9e8893",
|
"bg": "#ef8753",
|
||||||
"border": "#9e8893",
|
"border": "#ef8753",
|
||||||
"text": "#1f1d1e",
|
"text": "#1f1d1e",
|
||||||
"hover_bg": "#baabb2",
|
"hover_bg": "#f6ae81",
|
||||||
"hover_border": "#baabb2",
|
"hover_border": "#f6ae81",
|
||||||
"hover_text": "#1f1d1e"
|
"hover_text": "#1f1d1e"
|
||||||
},
|
},
|
||||||
"secondary": {
|
"secondary": {
|
||||||
|
|
|
||||||
|
|
@ -1,22 +1,22 @@
|
||||||
export default {
|
export default {
|
||||||
"id": "oatmeal",
|
"id": "oatmeal",
|
||||||
"label": "Oatmeal",
|
"label": "Oatmeal",
|
||||||
"description": "Warm beige foundation with muted sage green interactions.",
|
"description": "Cream backdrop with vivid olive-green accents.",
|
||||||
"modes": {
|
"modes": {
|
||||||
"light": {
|
"light": {
|
||||||
"scale": {
|
"scale": {
|
||||||
"brand": {
|
"brand": {
|
||||||
"50": "#f4f7f5",
|
"50": "#effbf4",
|
||||||
"100": "#e5ece7",
|
"100": "#d8f4e5",
|
||||||
"200": "#ccd9cf",
|
"200": "#b4e8cd",
|
||||||
"300": "#a8bdae",
|
"300": "#7fd8ab",
|
||||||
"400": "#8ba390",
|
"400": "#4ec788",
|
||||||
"500": "#6c8471",
|
"500": "#2faa6e",
|
||||||
"600": "#546958",
|
"600": "#248a58",
|
||||||
"700": "#435446",
|
"700": "#1f6f48",
|
||||||
"800": "#364338",
|
"800": "#1b593c",
|
||||||
"900": "#2d372f",
|
"900": "#174a32",
|
||||||
"950": "#161d18"
|
"950": "#0c2a1d"
|
||||||
},
|
},
|
||||||
"info": {
|
"info": {
|
||||||
"50": "#f0f5f6",
|
"50": "#f0f5f6",
|
||||||
|
|
@ -74,7 +74,7 @@ export default {
|
||||||
"surface": {
|
"surface": {
|
||||||
"app": "#f5f2eb",
|
"app": "#f5f2eb",
|
||||||
"nav": "#fcfaf5",
|
"nav": "#fcfaf5",
|
||||||
"nav_active": "#e5e1d8",
|
"nav_active": "#d8f4e5",
|
||||||
"card": "#fcfaf5",
|
"card": "#fcfaf5",
|
||||||
"card_muted": "#ece9e1",
|
"card_muted": "#ece9e1",
|
||||||
"table": "#fcfaf5",
|
"table": "#fcfaf5",
|
||||||
|
|
@ -87,25 +87,25 @@ export default {
|
||||||
"secondary": "#716d68",
|
"secondary": "#716d68",
|
||||||
"muted": "#96918a",
|
"muted": "#96918a",
|
||||||
"inverse": "#fcfaf5",
|
"inverse": "#fcfaf5",
|
||||||
"link": "#546958"
|
"link": "#248a58"
|
||||||
},
|
},
|
||||||
"border": {
|
"border": {
|
||||||
"default": "#e5e1d8",
|
"default": "#e5e1d8",
|
||||||
"strong": "#cecabf",
|
"strong": "#cecabf",
|
||||||
"subtle": "#f0ede6",
|
"subtle": "#f0ede6",
|
||||||
"interactive": "#a8bdae"
|
"interactive": "#7fd8ab"
|
||||||
},
|
},
|
||||||
"focus": {
|
"focus": {
|
||||||
"ring": "#8ba390",
|
"ring": "#4ec788",
|
||||||
"ring_offset": "#f5f2eb"
|
"ring_offset": "#f5f2eb"
|
||||||
},
|
},
|
||||||
"action": {
|
"action": {
|
||||||
"primary": {
|
"primary": {
|
||||||
"bg": "#8ba390",
|
"bg": "#4ec788",
|
||||||
"border": "#8ba390",
|
"border": "#4ec788",
|
||||||
"text": "#fcfaf5",
|
"text": "#fcfaf5",
|
||||||
"hover_bg": "#6c8471",
|
"hover_bg": "#2faa6e",
|
||||||
"hover_border": "#6c8471",
|
"hover_border": "#2faa6e",
|
||||||
"hover_text": "#ffffff"
|
"hover_text": "#ffffff"
|
||||||
},
|
},
|
||||||
"secondary": {
|
"secondary": {
|
||||||
|
|
@ -119,7 +119,7 @@ export default {
|
||||||
"ghost": {
|
"ghost": {
|
||||||
"bg": "#f5f2eb",
|
"bg": "#f5f2eb",
|
||||||
"border": "#e5e1d8",
|
"border": "#e5e1d8",
|
||||||
"text": "#546958",
|
"text": "#248a58",
|
||||||
"hover_bg": "#ece9e1",
|
"hover_bg": "#ece9e1",
|
||||||
"hover_border": "#cecabf",
|
"hover_border": "#cecabf",
|
||||||
"hover_text": "#3d3b38"
|
"hover_text": "#3d3b38"
|
||||||
|
|
@ -159,17 +159,17 @@ export default {
|
||||||
"dark": {
|
"dark": {
|
||||||
"scale": {
|
"scale": {
|
||||||
"brand": {
|
"brand": {
|
||||||
"50": "#f4f7f5",
|
"50": "#effbf4",
|
||||||
"100": "#e5ece7",
|
"100": "#d8f4e5",
|
||||||
"200": "#ccd9cf",
|
"200": "#b4e8cd",
|
||||||
"300": "#a8bdae",
|
"300": "#7fd8ab",
|
||||||
"400": "#8ba390",
|
"400": "#4ec788",
|
||||||
"500": "#6c8471",
|
"500": "#2faa6e",
|
||||||
"600": "#546958",
|
"600": "#248a58",
|
||||||
"700": "#435446",
|
"700": "#1f6f48",
|
||||||
"800": "#364338",
|
"800": "#1b593c",
|
||||||
"900": "#2d372f",
|
"900": "#174a32",
|
||||||
"950": "#161d18"
|
"950": "#0c2a1d"
|
||||||
},
|
},
|
||||||
"info": {
|
"info": {
|
||||||
"50": "#f0f5f6",
|
"50": "#f0f5f6",
|
||||||
|
|
@ -227,7 +227,7 @@ export default {
|
||||||
"surface": {
|
"surface": {
|
||||||
"app": "#3a3937",
|
"app": "#3a3937",
|
||||||
"nav": "#454442",
|
"nav": "#454442",
|
||||||
"nav_active": "#546958",
|
"nav_active": "#248a58",
|
||||||
"card": "#454442",
|
"card": "#454442",
|
||||||
"card_muted": "#504e4c",
|
"card_muted": "#504e4c",
|
||||||
"table": "#454442",
|
"table": "#454442",
|
||||||
|
|
@ -240,25 +240,25 @@ export default {
|
||||||
"secondary": "#bebcb8",
|
"secondary": "#bebcb8",
|
||||||
"muted": "#969490",
|
"muted": "#969490",
|
||||||
"inverse": "#3a3937",
|
"inverse": "#3a3937",
|
||||||
"link": "#a8bdae"
|
"link": "#7fd8ab"
|
||||||
},
|
},
|
||||||
"border": {
|
"border": {
|
||||||
"default": "#504e4c",
|
"default": "#504e4c",
|
||||||
"strong": "#62605d",
|
"strong": "#62605d",
|
||||||
"subtle": "#403f3d",
|
"subtle": "#403f3d",
|
||||||
"interactive": "#8ba390"
|
"interactive": "#4ec788"
|
||||||
},
|
},
|
||||||
"focus": {
|
"focus": {
|
||||||
"ring": "#8ba390",
|
"ring": "#4ec788",
|
||||||
"ring_offset": "#3a3937"
|
"ring_offset": "#3a3937"
|
||||||
},
|
},
|
||||||
"action": {
|
"action": {
|
||||||
"primary": {
|
"primary": {
|
||||||
"bg": "#8ba390",
|
"bg": "#4ec788",
|
||||||
"border": "#8ba390",
|
"border": "#4ec788",
|
||||||
"text": "#1a1918",
|
"text": "#1a1918",
|
||||||
"hover_bg": "#a8bdae",
|
"hover_bg": "#7fd8ab",
|
||||||
"hover_border": "#a8bdae",
|
"hover_border": "#7fd8ab",
|
||||||
"hover_text": "#1a1918"
|
"hover_text": "#1a1918"
|
||||||
},
|
},
|
||||||
"secondary": {
|
"secondary": {
|
||||||
|
|
|
||||||
|
|
@ -1,22 +1,22 @@
|
||||||
export default {
|
export default {
|
||||||
"id": "parchment",
|
"id": "parchment",
|
||||||
"label": "Parchment",
|
"label": "Parchment",
|
||||||
"description": "Dusty beige canvas paired with slate blue accents.",
|
"description": "Cool parchment base with bold indigo-blue accents.",
|
||||||
"modes": {
|
"modes": {
|
||||||
"light": {
|
"light": {
|
||||||
"scale": {
|
"scale": {
|
||||||
"brand": {
|
"brand": {
|
||||||
"50": "#f5f7f9",
|
"50": "#f2f6ff",
|
||||||
"100": "#eaf0f4",
|
"100": "#e3edff",
|
||||||
"200": "#d5e0e9",
|
"200": "#c6d8ff",
|
||||||
"300": "#b3c9d7",
|
"300": "#9db9ff",
|
||||||
"400": "#8daabc",
|
"400": "#7596f4",
|
||||||
"500": "#7d93a4",
|
"500": "#5878da",
|
||||||
"600": "#5c7487",
|
"600": "#4362bc",
|
||||||
"700": "#495e6f",
|
"700": "#384f95",
|
||||||
"800": "#3f4d5b",
|
"800": "#31427b",
|
||||||
"900": "#36414e",
|
"900": "#2b3965",
|
||||||
"950": "#232b35"
|
"950": "#1b223f"
|
||||||
},
|
},
|
||||||
"info": {
|
"info": {
|
||||||
"50": "#f2f6fa",
|
"50": "#f2f6fa",
|
||||||
|
|
@ -74,7 +74,7 @@ export default {
|
||||||
"surface": {
|
"surface": {
|
||||||
"app": "#f2eee9",
|
"app": "#f2eee9",
|
||||||
"nav": "#faf8f5",
|
"nav": "#faf8f5",
|
||||||
"nav_active": "#d5e0e9",
|
"nav_active": "#c6d8ff",
|
||||||
"card": "#faf8f5",
|
"card": "#faf8f5",
|
||||||
"card_muted": "#eae5df",
|
"card_muted": "#eae5df",
|
||||||
"table": "#faf8f5",
|
"table": "#faf8f5",
|
||||||
|
|
@ -87,25 +87,25 @@ export default {
|
||||||
"secondary": "#6b737c",
|
"secondary": "#6b737c",
|
||||||
"muted": "#929ba5",
|
"muted": "#929ba5",
|
||||||
"inverse": "#faf8f5",
|
"inverse": "#faf8f5",
|
||||||
"link": "#5c7487"
|
"link": "#4362bc"
|
||||||
},
|
},
|
||||||
"border": {
|
"border": {
|
||||||
"default": "#eae5df",
|
"default": "#eae5df",
|
||||||
"strong": "#d4cec6",
|
"strong": "#d4cec6",
|
||||||
"subtle": "#f6f3ef",
|
"subtle": "#f6f3ef",
|
||||||
"interactive": "#b3c9d7"
|
"interactive": "#9db9ff"
|
||||||
},
|
},
|
||||||
"focus": {
|
"focus": {
|
||||||
"ring": "#7d93a4",
|
"ring": "#5878da",
|
||||||
"ring_offset": "#f2eee9"
|
"ring_offset": "#f2eee9"
|
||||||
},
|
},
|
||||||
"action": {
|
"action": {
|
||||||
"primary": {
|
"primary": {
|
||||||
"bg": "#7d93a4",
|
"bg": "#5878da",
|
||||||
"border": "#7d93a4",
|
"border": "#5878da",
|
||||||
"text": "#faf8f5",
|
"text": "#faf8f5",
|
||||||
"hover_bg": "#5c7487",
|
"hover_bg": "#4362bc",
|
||||||
"hover_border": "#5c7487",
|
"hover_border": "#4362bc",
|
||||||
"hover_text": "#ffffff"
|
"hover_text": "#ffffff"
|
||||||
},
|
},
|
||||||
"secondary": {
|
"secondary": {
|
||||||
|
|
@ -119,7 +119,7 @@ export default {
|
||||||
"ghost": {
|
"ghost": {
|
||||||
"bg": "#f2eee9",
|
"bg": "#f2eee9",
|
||||||
"border": "#eae5df",
|
"border": "#eae5df",
|
||||||
"text": "#5c7487",
|
"text": "#4362bc",
|
||||||
"hover_bg": "#eae5df",
|
"hover_bg": "#eae5df",
|
||||||
"hover_border": "#d4cec6",
|
"hover_border": "#d4cec6",
|
||||||
"hover_text": "#333638"
|
"hover_text": "#333638"
|
||||||
|
|
@ -159,17 +159,17 @@ export default {
|
||||||
"dark": {
|
"dark": {
|
||||||
"scale": {
|
"scale": {
|
||||||
"brand": {
|
"brand": {
|
||||||
"50": "#f5f7f9",
|
"50": "#f2f6ff",
|
||||||
"100": "#eaf0f4",
|
"100": "#e3edff",
|
||||||
"200": "#d5e0e9",
|
"200": "#c6d8ff",
|
||||||
"300": "#b3c9d7",
|
"300": "#9db9ff",
|
||||||
"400": "#8daabc",
|
"400": "#7596f4",
|
||||||
"500": "#7d93a4",
|
"500": "#5878da",
|
||||||
"600": "#5c7487",
|
"600": "#4362bc",
|
||||||
"700": "#495e6f",
|
"700": "#384f95",
|
||||||
"800": "#3f4d5b",
|
"800": "#31427b",
|
||||||
"900": "#36414e",
|
"900": "#2b3965",
|
||||||
"950": "#232b35"
|
"950": "#1b223f"
|
||||||
},
|
},
|
||||||
"info": {
|
"info": {
|
||||||
"50": "#f2f6fa",
|
"50": "#f2f6fa",
|
||||||
|
|
@ -227,7 +227,7 @@ export default {
|
||||||
"surface": {
|
"surface": {
|
||||||
"app": "#35373a",
|
"app": "#35373a",
|
||||||
"nav": "#424549",
|
"nav": "#424549",
|
||||||
"nav_active": "#5c7487",
|
"nav_active": "#4362bc",
|
||||||
"card": "#424549",
|
"card": "#424549",
|
||||||
"card_muted": "#4d5156",
|
"card_muted": "#4d5156",
|
||||||
"table": "#424549",
|
"table": "#424549",
|
||||||
|
|
@ -240,25 +240,25 @@ export default {
|
||||||
"secondary": "#aeb5bc",
|
"secondary": "#aeb5bc",
|
||||||
"muted": "#889098",
|
"muted": "#889098",
|
||||||
"inverse": "#35373a",
|
"inverse": "#35373a",
|
||||||
"link": "#b3c9d7"
|
"link": "#9db9ff"
|
||||||
},
|
},
|
||||||
"border": {
|
"border": {
|
||||||
"default": "#4d5156",
|
"default": "#4d5156",
|
||||||
"strong": "#5d6268",
|
"strong": "#5d6268",
|
||||||
"subtle": "#3e4145",
|
"subtle": "#3e4145",
|
||||||
"interactive": "#7d93a4"
|
"interactive": "#5878da"
|
||||||
},
|
},
|
||||||
"focus": {
|
"focus": {
|
||||||
"ring": "#7d93a4",
|
"ring": "#5878da",
|
||||||
"ring_offset": "#35373a"
|
"ring_offset": "#35373a"
|
||||||
},
|
},
|
||||||
"action": {
|
"action": {
|
||||||
"primary": {
|
"primary": {
|
||||||
"bg": "#7d93a4",
|
"bg": "#5878da",
|
||||||
"border": "#7d93a4",
|
"border": "#5878da",
|
||||||
"text": "#1a1b1d",
|
"text": "#1a1b1d",
|
||||||
"hover_bg": "#b3c9d7",
|
"hover_bg": "#9db9ff",
|
||||||
"hover_border": "#b3c9d7",
|
"hover_border": "#9db9ff",
|
||||||
"hover_text": "#1a1b1d"
|
"hover_text": "#1a1b1d"
|
||||||
},
|
},
|
||||||
"secondary": {
|
"secondary": {
|
||||||
|
|
|
||||||
313
frontend/src/theme/presets/sunset.js
Normal file
|
|
@ -0,0 +1,313 @@
|
||||||
|
export default {
|
||||||
|
"id": "sunset",
|
||||||
|
"label": "Sunset",
|
||||||
|
"description": "High-contrast amber and ember tones for a warmer visual rhythm.",
|
||||||
|
"modes": {
|
||||||
|
"light": {
|
||||||
|
"scale": {
|
||||||
|
"brand": {
|
||||||
|
"50": "#fff7ed",
|
||||||
|
"100": "#ffedd5",
|
||||||
|
"200": "#fed7aa",
|
||||||
|
"300": "#fdba74",
|
||||||
|
"400": "#fb923c",
|
||||||
|
"500": "#f97316",
|
||||||
|
"600": "#ea580c",
|
||||||
|
"700": "#c2410c",
|
||||||
|
"800": "#9a3412",
|
||||||
|
"900": "#7c2d12",
|
||||||
|
"950": "#431407"
|
||||||
|
},
|
||||||
|
"info": {
|
||||||
|
"50": "#eef2ff",
|
||||||
|
"100": "#e0e7ff",
|
||||||
|
"200": "#c7d2fe",
|
||||||
|
"300": "#a5b4fc",
|
||||||
|
"400": "#818cf8",
|
||||||
|
"500": "#6366f1",
|
||||||
|
"600": "#4f46e5",
|
||||||
|
"700": "#4338ca",
|
||||||
|
"800": "#3730a3",
|
||||||
|
"900": "#312e81",
|
||||||
|
"950": "#1e1b4b"
|
||||||
|
},
|
||||||
|
"success": {
|
||||||
|
"50": "#f0fdf4",
|
||||||
|
"100": "#dcfce7",
|
||||||
|
"200": "#bbf7d0",
|
||||||
|
"300": "#86efac",
|
||||||
|
"400": "#4ade80",
|
||||||
|
"500": "#22c55e",
|
||||||
|
"600": "#16a34a",
|
||||||
|
"700": "#15803d",
|
||||||
|
"800": "#166534",
|
||||||
|
"900": "#14532d",
|
||||||
|
"950": "#052e16"
|
||||||
|
},
|
||||||
|
"warning": {
|
||||||
|
"50": "#fffbeb",
|
||||||
|
"100": "#fef3c7",
|
||||||
|
"200": "#fde68a",
|
||||||
|
"300": "#fcd34d",
|
||||||
|
"400": "#fbbf24",
|
||||||
|
"500": "#f59e0b",
|
||||||
|
"600": "#d97706",
|
||||||
|
"700": "#b45309",
|
||||||
|
"800": "#92400e",
|
||||||
|
"900": "#78350f",
|
||||||
|
"950": "#451a03"
|
||||||
|
},
|
||||||
|
"danger": {
|
||||||
|
"50": "#fff1f2",
|
||||||
|
"100": "#ffe4e6",
|
||||||
|
"200": "#fecdd3",
|
||||||
|
"300": "#fda4af",
|
||||||
|
"400": "#fb7185",
|
||||||
|
"500": "#f43f5e",
|
||||||
|
"600": "#e11d48",
|
||||||
|
"700": "#be123c",
|
||||||
|
"800": "#9f1239",
|
||||||
|
"900": "#881337",
|
||||||
|
"950": "#4c0519"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"surface": {
|
||||||
|
"app": "#fff5ef",
|
||||||
|
"nav": "#ffffff",
|
||||||
|
"nav_active": "#fed7aa",
|
||||||
|
"card": "#ffffff",
|
||||||
|
"card_muted": "#fff0e5",
|
||||||
|
"table": "#ffffff",
|
||||||
|
"table_header": "#fff5ef",
|
||||||
|
"input": "#ffffff",
|
||||||
|
"overlay": "#2b1a12"
|
||||||
|
},
|
||||||
|
"text": {
|
||||||
|
"primary": "#3b2217",
|
||||||
|
"secondary": "#7c3f1d",
|
||||||
|
"muted": "#a05d37",
|
||||||
|
"inverse": "#ffffff",
|
||||||
|
"link": "#c2410c"
|
||||||
|
},
|
||||||
|
"border": {
|
||||||
|
"default": "#f4d8c6",
|
||||||
|
"strong": "#e9bc9f",
|
||||||
|
"subtle": "#fff0e5",
|
||||||
|
"interactive": "#fb923c"
|
||||||
|
},
|
||||||
|
"focus": {
|
||||||
|
"ring": "#f97316",
|
||||||
|
"ring_offset": "#fff5ef"
|
||||||
|
},
|
||||||
|
"action": {
|
||||||
|
"primary": {
|
||||||
|
"bg": "#f97316",
|
||||||
|
"border": "#f97316",
|
||||||
|
"text": "#ffffff",
|
||||||
|
"hover_bg": "#ea580c",
|
||||||
|
"hover_border": "#ea580c",
|
||||||
|
"hover_text": "#ffffff"
|
||||||
|
},
|
||||||
|
"secondary": {
|
||||||
|
"bg": "#fff0e5",
|
||||||
|
"border": "#f4d8c6",
|
||||||
|
"text": "#3b2217",
|
||||||
|
"hover_bg": "#fee2cf",
|
||||||
|
"hover_border": "#e9bc9f",
|
||||||
|
"hover_text": "#2a170f"
|
||||||
|
},
|
||||||
|
"ghost": {
|
||||||
|
"bg": "#fff5ef",
|
||||||
|
"border": "#f4d8c6",
|
||||||
|
"text": "#c2410c",
|
||||||
|
"hover_bg": "#fff0e5",
|
||||||
|
"hover_border": "#e9bc9f",
|
||||||
|
"hover_text": "#7c2d12"
|
||||||
|
},
|
||||||
|
"danger": {
|
||||||
|
"bg": "#ffe4e6",
|
||||||
|
"border": "#fecdd3",
|
||||||
|
"text": "#be123c",
|
||||||
|
"hover_bg": "#fecdd3",
|
||||||
|
"hover_border": "#fda4af",
|
||||||
|
"hover_text": "#9f1239"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"info": {
|
||||||
|
"bg": "#e0e7ff",
|
||||||
|
"border": "#c7d2fe",
|
||||||
|
"text": "#4338ca"
|
||||||
|
},
|
||||||
|
"success": {
|
||||||
|
"bg": "#dcfce7",
|
||||||
|
"border": "#bbf7d0",
|
||||||
|
"text": "#166534"
|
||||||
|
},
|
||||||
|
"warning": {
|
||||||
|
"bg": "#ffedd5",
|
||||||
|
"border": "#fed7aa",
|
||||||
|
"text": "#9a3412"
|
||||||
|
},
|
||||||
|
"danger": {
|
||||||
|
"bg": "#ffe4e6",
|
||||||
|
"border": "#fecdd3",
|
||||||
|
"text": "#be123c"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"dark": {
|
||||||
|
"scale": {
|
||||||
|
"brand": {
|
||||||
|
"50": "#fff7ed",
|
||||||
|
"100": "#ffedd5",
|
||||||
|
"200": "#fed7aa",
|
||||||
|
"300": "#fdba74",
|
||||||
|
"400": "#fb923c",
|
||||||
|
"500": "#f97316",
|
||||||
|
"600": "#ea580c",
|
||||||
|
"700": "#c2410c",
|
||||||
|
"800": "#9a3412",
|
||||||
|
"900": "#7c2d12",
|
||||||
|
"950": "#431407"
|
||||||
|
},
|
||||||
|
"info": {
|
||||||
|
"50": "#eef2ff",
|
||||||
|
"100": "#e0e7ff",
|
||||||
|
"200": "#c7d2fe",
|
||||||
|
"300": "#a5b4fc",
|
||||||
|
"400": "#818cf8",
|
||||||
|
"500": "#6366f1",
|
||||||
|
"600": "#4f46e5",
|
||||||
|
"700": "#4338ca",
|
||||||
|
"800": "#3730a3",
|
||||||
|
"900": "#312e81",
|
||||||
|
"950": "#1e1b4b"
|
||||||
|
},
|
||||||
|
"success": {
|
||||||
|
"50": "#f0fdf4",
|
||||||
|
"100": "#dcfce7",
|
||||||
|
"200": "#bbf7d0",
|
||||||
|
"300": "#86efac",
|
||||||
|
"400": "#4ade80",
|
||||||
|
"500": "#22c55e",
|
||||||
|
"600": "#16a34a",
|
||||||
|
"700": "#15803d",
|
||||||
|
"800": "#166534",
|
||||||
|
"900": "#14532d",
|
||||||
|
"950": "#052e16"
|
||||||
|
},
|
||||||
|
"warning": {
|
||||||
|
"50": "#fffbeb",
|
||||||
|
"100": "#fef3c7",
|
||||||
|
"200": "#fde68a",
|
||||||
|
"300": "#fcd34d",
|
||||||
|
"400": "#fbbf24",
|
||||||
|
"500": "#f59e0b",
|
||||||
|
"600": "#d97706",
|
||||||
|
"700": "#b45309",
|
||||||
|
"800": "#92400e",
|
||||||
|
"900": "#78350f",
|
||||||
|
"950": "#451a03"
|
||||||
|
},
|
||||||
|
"danger": {
|
||||||
|
"50": "#fff1f2",
|
||||||
|
"100": "#ffe4e6",
|
||||||
|
"200": "#fecdd3",
|
||||||
|
"300": "#fda4af",
|
||||||
|
"400": "#fb7185",
|
||||||
|
"500": "#f43f5e",
|
||||||
|
"600": "#e11d48",
|
||||||
|
"700": "#be123c",
|
||||||
|
"800": "#9f1239",
|
||||||
|
"900": "#881337",
|
||||||
|
"950": "#4c0519"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"surface": {
|
||||||
|
"app": "#24140d",
|
||||||
|
"nav": "#2f1b12",
|
||||||
|
"nav_active": "#c2410c",
|
||||||
|
"card": "#2f1b12",
|
||||||
|
"card_muted": "#3a2419",
|
||||||
|
"table": "#2f1b12",
|
||||||
|
"table_header": "#3a2419",
|
||||||
|
"input": "#1c100a",
|
||||||
|
"overlay": "#09090b"
|
||||||
|
},
|
||||||
|
"text": {
|
||||||
|
"primary": "#ffe8d7",
|
||||||
|
"secondary": "#efb58f",
|
||||||
|
"muted": "#cb8f6b",
|
||||||
|
"inverse": "#24140d",
|
||||||
|
"link": "#fdba74"
|
||||||
|
},
|
||||||
|
"border": {
|
||||||
|
"default": "#4a2f20",
|
||||||
|
"strong": "#5d3b2a",
|
||||||
|
"subtle": "#3a2419",
|
||||||
|
"interactive": "#f97316"
|
||||||
|
},
|
||||||
|
"focus": {
|
||||||
|
"ring": "#fb923c",
|
||||||
|
"ring_offset": "#24140d"
|
||||||
|
},
|
||||||
|
"action": {
|
||||||
|
"primary": {
|
||||||
|
"bg": "#f97316",
|
||||||
|
"border": "#f97316",
|
||||||
|
"text": "#1c100a",
|
||||||
|
"hover_bg": "#fb923c",
|
||||||
|
"hover_border": "#fb923c",
|
||||||
|
"hover_text": "#1c100a"
|
||||||
|
},
|
||||||
|
"secondary": {
|
||||||
|
"bg": "#3a2419",
|
||||||
|
"border": "#5d3b2a",
|
||||||
|
"text": "#ffe8d7",
|
||||||
|
"hover_bg": "#4a2f20",
|
||||||
|
"hover_border": "#6f4834",
|
||||||
|
"hover_text": "#ffffff"
|
||||||
|
},
|
||||||
|
"ghost": {
|
||||||
|
"bg": "#24140d",
|
||||||
|
"border": "#4a2f20",
|
||||||
|
"text": "#efb58f",
|
||||||
|
"hover_bg": "#3a2419",
|
||||||
|
"hover_border": "#5d3b2a",
|
||||||
|
"hover_text": "#ffe8d7"
|
||||||
|
},
|
||||||
|
"danger": {
|
||||||
|
"bg": "#4c0519",
|
||||||
|
"border": "#9f1239",
|
||||||
|
"text": "#fecdd3",
|
||||||
|
"hover_bg": "#5d0b26",
|
||||||
|
"hover_border": "#be123c",
|
||||||
|
"hover_text": "#ffe4e6"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"info": {
|
||||||
|
"bg": "#312e81",
|
||||||
|
"border": "#4338ca",
|
||||||
|
"text": "#c7d2fe"
|
||||||
|
},
|
||||||
|
"success": {
|
||||||
|
"bg": "#14532d",
|
||||||
|
"border": "#15803d",
|
||||||
|
"text": "#bbf7d0"
|
||||||
|
},
|
||||||
|
"warning": {
|
||||||
|
"bg": "#7c2d12",
|
||||||
|
"border": "#c2410c",
|
||||||
|
"text": "#fed7aa"
|
||||||
|
},
|
||||||
|
"danger": {
|
||||||
|
"bg": "#4c0519",
|
||||||
|
"border": "#be123c",
|
||||||
|
"text": "#fecdd3"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
126
scripts/check_env_contract.py
Normal file
|
|
@ -0,0 +1,126 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
SETTINGS_PATH = ROOT / "app" / "settings.py"
|
||||||
|
ENV_EXAMPLE_PATH = ROOT / ".env.example"
|
||||||
|
|
||||||
|
SETTINGS_ENV_HELPERS = {"_env_bool", "_env_int", "_env_float", "_env_str"}
|
||||||
|
ALLOWED_ENV_EXAMPLE_EXTRAS = {
|
||||||
|
"POSTGRES_DB",
|
||||||
|
"POSTGRES_USER",
|
||||||
|
"POSTGRES_PASSWORD",
|
||||||
|
"TEST_DATABASE_URL",
|
||||||
|
"SCHOLARR_IMAGE",
|
||||||
|
"APP_HOST",
|
||||||
|
"APP_PORT",
|
||||||
|
"APP_HOST_PORT",
|
||||||
|
"APP_RELOAD",
|
||||||
|
"FRONTEND_HOST_PORT",
|
||||||
|
"CHOKIDAR_USEPOLLING",
|
||||||
|
"VITE_DEV_API_PROXY_TARGET",
|
||||||
|
"MIGRATE_ON_START",
|
||||||
|
"BOOTSTRAP_ADMIN_ON_START",
|
||||||
|
"BOOTSTRAP_ADMIN_EMAIL",
|
||||||
|
"BOOTSTRAP_ADMIN_PASSWORD",
|
||||||
|
"BOOTSTRAP_ADMIN_FORCE_PASSWORD",
|
||||||
|
"DB_WAIT_TIMEOUT_SECONDS",
|
||||||
|
"DB_WAIT_INTERVAL_SECONDS",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _call_name(node: ast.AST) -> str | None:
|
||||||
|
if not isinstance(node, ast.Call):
|
||||||
|
return None
|
||||||
|
func = node.func
|
||||||
|
if isinstance(func, ast.Name):
|
||||||
|
return func.id
|
||||||
|
if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name):
|
||||||
|
return f"{func.value.id}.{func.attr}"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _first_str_arg(node: ast.Call) -> str | None:
|
||||||
|
if not node.args:
|
||||||
|
return None
|
||||||
|
first = node.args[0]
|
||||||
|
if isinstance(first, ast.Constant) and isinstance(first.value, str):
|
||||||
|
return first.value
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _settings_env_names() -> set[str]:
|
||||||
|
tree = ast.parse(SETTINGS_PATH.read_text(encoding="utf-8"))
|
||||||
|
names: set[str] = set()
|
||||||
|
|
||||||
|
for node in ast.walk(tree):
|
||||||
|
if not isinstance(node, ast.Call):
|
||||||
|
continue
|
||||||
|
call_name = _call_name(node)
|
||||||
|
if call_name is None:
|
||||||
|
continue
|
||||||
|
if call_name != "os.getenv" and call_name not in SETTINGS_ENV_HELPERS:
|
||||||
|
continue
|
||||||
|
env_name = _first_str_arg(node)
|
||||||
|
if not env_name:
|
||||||
|
continue
|
||||||
|
names.add(env_name)
|
||||||
|
return names
|
||||||
|
|
||||||
|
|
||||||
|
def _env_example_names() -> tuple[set[str], set[str]]:
|
||||||
|
names: set[str] = set()
|
||||||
|
duplicates: set[str] = set()
|
||||||
|
|
||||||
|
for raw_line in ENV_EXAMPLE_PATH.read_text(encoding="utf-8").splitlines():
|
||||||
|
line = raw_line.strip()
|
||||||
|
if not line or line.startswith("#"):
|
||||||
|
continue
|
||||||
|
if "=" not in line:
|
||||||
|
continue
|
||||||
|
key = line.split("=", 1)[0].strip()
|
||||||
|
if not key:
|
||||||
|
continue
|
||||||
|
if key in names:
|
||||||
|
duplicates.add(key)
|
||||||
|
names.add(key)
|
||||||
|
return names, duplicates
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
settings_names = _settings_env_names()
|
||||||
|
env_names, duplicate_names = _env_example_names()
|
||||||
|
|
||||||
|
missing = sorted(settings_names - env_names)
|
||||||
|
unknown = sorted(env_names - settings_names - ALLOWED_ENV_EXAMPLE_EXTRAS)
|
||||||
|
duplicates = sorted(duplicate_names)
|
||||||
|
|
||||||
|
if missing or unknown or duplicates:
|
||||||
|
print("Environment contract check failed.")
|
||||||
|
if missing:
|
||||||
|
print("Missing from .env.example (referenced in app/settings.py):")
|
||||||
|
for name in missing:
|
||||||
|
print(f"- {name}")
|
||||||
|
if unknown:
|
||||||
|
print("Unknown keys in .env.example (not in app/settings.py or allowlist):")
|
||||||
|
for name in unknown:
|
||||||
|
print(f"- {name}")
|
||||||
|
if duplicates:
|
||||||
|
print("Duplicate keys in .env.example:")
|
||||||
|
for name in duplicates:
|
||||||
|
print(f"- {name}")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print(
|
||||||
|
"Environment contract check passed: "
|
||||||
|
f"{len(settings_names)} settings keys + {len(ALLOWED_ENV_EXAMPLE_EXTRAS)} allowed extras."
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
63
scripts/db/backup_full.sh
Executable file
|
|
@ -0,0 +1,63 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
project_root="$(cd "${script_dir}/../.." && pwd)"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Usage: scripts/db/backup_full.sh [--plain]
|
||||||
|
|
||||||
|
Creates a PostgreSQL logical backup from the running `db` compose service.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--plain Write a plain SQL dump instead of custom-format dump.
|
||||||
|
-h, --help Show this help.
|
||||||
|
|
||||||
|
Environment:
|
||||||
|
BACKUP_DIR Destination directory (default: <repo>/backups)
|
||||||
|
BACKUP_PREFIX File prefix (default: scholarr)
|
||||||
|
USE_DEV_COMPOSE Set to 1 to include docker-compose.dev.yml
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
format="custom"
|
||||||
|
while (($#)); do
|
||||||
|
case "$1" in
|
||||||
|
--plain)
|
||||||
|
format="plain"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
-h|--help)
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown argument: $1" >&2
|
||||||
|
usage >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
compose_cmd=(docker compose -f "${project_root}/docker-compose.yml")
|
||||||
|
if [[ "${USE_DEV_COMPOSE:-0}" == "1" && -f "${project_root}/docker-compose.dev.yml" ]]; then
|
||||||
|
compose_cmd+=(-f "${project_root}/docker-compose.dev.yml")
|
||||||
|
fi
|
||||||
|
|
||||||
|
backup_dir="${BACKUP_DIR:-${project_root}/backups}"
|
||||||
|
backup_prefix="${BACKUP_PREFIX:-scholarr}"
|
||||||
|
mkdir -p "${backup_dir}"
|
||||||
|
|
||||||
|
timestamp="$(date -u +"%Y%m%dT%H%M%SZ")"
|
||||||
|
if [[ "${format}" == "plain" ]]; then
|
||||||
|
backup_file="${backup_dir}/${backup_prefix}_${timestamp}.sql"
|
||||||
|
dump_cmd='pg_dump --no-owner --no-acl -U "$POSTGRES_USER" "$POSTGRES_DB"'
|
||||||
|
else
|
||||||
|
backup_file="${backup_dir}/${backup_prefix}_${timestamp}.dump"
|
||||||
|
dump_cmd='pg_dump --format=custom --no-owner --no-acl -U "$POSTGRES_USER" "$POSTGRES_DB"'
|
||||||
|
fi
|
||||||
|
|
||||||
|
"${compose_cmd[@]}" exec -T db sh -lc "${dump_cmd}" > "${backup_file}"
|
||||||
|
|
||||||
|
echo "Backup created: ${backup_file}"
|
||||||
50
scripts/db/check_integrity.py
Executable file
|
|
@ -0,0 +1,50 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
|
||||||
|
from app.db.session import get_session_factory
|
||||||
|
from app.services.domains.dbops import collect_integrity_report
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(description="Run database integrity checks.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--strict-warnings",
|
||||||
|
action="store_true",
|
||||||
|
help="Return non-zero exit code if any warning is present.",
|
||||||
|
)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
async def _run() -> dict:
|
||||||
|
session_factory = get_session_factory()
|
||||||
|
async with session_factory() as db_session:
|
||||||
|
return await collect_integrity_report(db_session)
|
||||||
|
|
||||||
|
|
||||||
|
def _exit_code(report: dict, *, strict_warnings: bool) -> int:
|
||||||
|
if report.get("status") == "failed":
|
||||||
|
return 1
|
||||||
|
if strict_warnings and report.get("warnings"):
|
||||||
|
return 2
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = build_parser().parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
report = asyncio.run(_run())
|
||||||
|
except Exception as exc:
|
||||||
|
print(json.dumps({"status": "failed", "error": str(exc)}, indent=2))
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print(json.dumps(report, indent=2))
|
||||||
|
return _exit_code(report, strict_warnings=args.strict_warnings)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
66
scripts/db/repair_publication_links.py
Executable file
|
|
@ -0,0 +1,66 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
|
||||||
|
from app.db.session import get_session_factory
|
||||||
|
from app.services.domains.dbops import run_publication_link_repair
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Repair scholar-publication links for a user with audit logging.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--user-id", type=int, required=True, help="Target user ID.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--scholar-profile-id",
|
||||||
|
type=int,
|
||||||
|
action="append",
|
||||||
|
default=[],
|
||||||
|
help="Optional scholar_profile_id filter. Repeat for multiple values.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--apply", action="store_true", help="Apply changes. Default is dry-run.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--gc-orphan-publications",
|
||||||
|
action="store_true",
|
||||||
|
help="Delete publications with zero links after cleanup.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--requested-by",
|
||||||
|
default="",
|
||||||
|
help="Operator identifier for audit logs (email/name/ticket).",
|
||||||
|
)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
async def _run(args: argparse.Namespace) -> dict:
|
||||||
|
session_factory = get_session_factory()
|
||||||
|
async with session_factory() as db_session:
|
||||||
|
return await run_publication_link_repair(
|
||||||
|
db_session,
|
||||||
|
user_id=args.user_id,
|
||||||
|
scholar_profile_ids=args.scholar_profile_id,
|
||||||
|
dry_run=not args.apply,
|
||||||
|
gc_orphan_publications=args.gc_orphan_publications,
|
||||||
|
requested_by=args.requested_by,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = build_parser()
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = asyncio.run(_run(args))
|
||||||
|
except Exception as exc:
|
||||||
|
print(json.dumps({"status": "failed", "error": str(exc)}, indent=2))
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print(json.dumps(result, indent=2))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
79
scripts/db/restore_dump.sh
Executable file
|
|
@ -0,0 +1,79 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
project_root="$(cd "${script_dir}/../.." && pwd)"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Usage: scripts/db/restore_dump.sh --file <path> [--wipe-public]
|
||||||
|
|
||||||
|
Restores a PostgreSQL dump into the running `db` compose service.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--file <path> Required. Backup file (.dump custom format or .sql plain).
|
||||||
|
--wipe-public Drop and recreate public schema before restore.
|
||||||
|
-h, --help Show this help.
|
||||||
|
|
||||||
|
Environment:
|
||||||
|
USE_DEV_COMPOSE Set to 1 to include docker-compose.dev.yml
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
dump_file=""
|
||||||
|
wipe_public="0"
|
||||||
|
while (($#)); do
|
||||||
|
case "$1" in
|
||||||
|
--file)
|
||||||
|
dump_file="${2:-}"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--wipe-public)
|
||||||
|
wipe_public="1"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
-h|--help)
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown argument: $1" >&2
|
||||||
|
usage >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -z "${dump_file}" ]]; then
|
||||||
|
echo "--file is required" >&2
|
||||||
|
usage >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
if [[ ! -f "${dump_file}" ]]; then
|
||||||
|
echo "Dump file not found: ${dump_file}" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
compose_cmd=(docker compose -f "${project_root}/docker-compose.yml")
|
||||||
|
if [[ "${USE_DEV_COMPOSE:-0}" == "1" && -f "${project_root}/docker-compose.dev.yml" ]]; then
|
||||||
|
compose_cmd+=(-f "${project_root}/docker-compose.dev.yml")
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${wipe_public}" == "1" ]]; then
|
||||||
|
"${compose_cmd[@]}" exec -T db sh -lc 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" "$POSTGRES_DB" -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"'
|
||||||
|
fi
|
||||||
|
|
||||||
|
case "${dump_file}" in
|
||||||
|
*.dump)
|
||||||
|
cat "${dump_file}" | "${compose_cmd[@]}" exec -T db sh -lc 'pg_restore --no-owner --no-acl -U "$POSTGRES_USER" -d "$POSTGRES_DB" -'
|
||||||
|
;;
|
||||||
|
*.sql)
|
||||||
|
cat "${dump_file}" | "${compose_cmd[@]}" exec -T db sh -lc 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" "$POSTGRES_DB"'
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unsupported file extension. Expected .dump or .sql" >&2
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
echo "Restore completed: ${dump_file}"
|
||||||
|
|
@ -21,6 +21,9 @@ RESET_SQL = text(
|
||||||
TRUNCATE TABLE
|
TRUNCATE TABLE
|
||||||
author_search_cache_entries,
|
author_search_cache_entries,
|
||||||
author_search_runtime_state,
|
author_search_runtime_state,
|
||||||
|
data_repair_jobs,
|
||||||
|
publication_pdf_job_events,
|
||||||
|
publication_pdf_jobs,
|
||||||
ingestion_queue_items,
|
ingestion_queue_items,
|
||||||
scholar_publications,
|
scholar_publications,
|
||||||
crawl_runs,
|
crawl_runs,
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.api.runtime_deps import get_scholar_source
|
from app.api.runtime_deps import get_scholar_source
|
||||||
from app.main import app
|
from app.main import app
|
||||||
|
from app.services.domains.publications.types import PublicationListItem
|
||||||
from app.services.domains.settings import application as user_settings_service
|
from app.services.domains.settings import application as user_settings_service
|
||||||
from app.services.domains.scholar.source import FetchResult
|
from app.services.domains.scholar.source import FetchResult
|
||||||
from app.settings import settings
|
from app.settings import settings
|
||||||
|
|
@ -65,6 +66,49 @@ def _assert_safety_state_contract(payload: dict[str, object]) -> None:
|
||||||
assert set(counters.keys()) == SAFETY_COUNTER_KEYS
|
assert set(counters.keys()) == SAFETY_COUNTER_KEYS
|
||||||
|
|
||||||
|
|
||||||
|
async def _seed_publication_link_for_user(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
scholar_id: str,
|
||||||
|
title: str,
|
||||||
|
fingerprint: str,
|
||||||
|
) -> tuple[int, int]:
|
||||||
|
scholar_result = await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
|
||||||
|
VALUES (:user_id, :scholar_id, :display_name, true)
|
||||||
|
RETURNING id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"user_id": user_id, "scholar_id": scholar_id, "display_name": scholar_id},
|
||||||
|
)
|
||||||
|
scholar_profile_id = int(scholar_result.scalar_one())
|
||||||
|
publication_result = await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
|
||||||
|
VALUES (:fingerprint, :title_raw, :title_normalized, 4)
|
||||||
|
RETURNING id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"fingerprint": fingerprint, "title_raw": title, "title_normalized": title.lower()},
|
||||||
|
)
|
||||||
|
publication_id = int(publication_result.scalar_one())
|
||||||
|
await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
|
||||||
|
VALUES (:scholar_profile_id, :publication_id, false)
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"scholar_profile_id": scholar_profile_id, "publication_id": publication_id},
|
||||||
|
)
|
||||||
|
await db_session.commit()
|
||||||
|
return scholar_profile_id, publication_id
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.integration
|
@pytest.mark.integration
|
||||||
@pytest.mark.db
|
@pytest.mark.db
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|
@ -321,6 +365,265 @@ async def test_api_admin_user_management_endpoints(db_session: AsyncSession) ->
|
||||||
assert created_exists.scalar_one() == 1
|
assert created_exists.scalar_one() == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.db
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_api_admin_dbops_integrity_and_repair_flow(db_session: AsyncSession) -> None:
|
||||||
|
await insert_user(db_session, email="api-admin-dbops@example.com", password="admin-password", is_admin=True)
|
||||||
|
target_user_id = await insert_user(db_session, email="api-dbops-target@example.com", password="target-password")
|
||||||
|
_scholar_id, publication_id = await _seed_publication_link_for_user(
|
||||||
|
db_session,
|
||||||
|
user_id=target_user_id,
|
||||||
|
scholar_id="dbopsTarget01",
|
||||||
|
title="DB Ops Target Paper",
|
||||||
|
fingerprint=f"{(target_user_id + 31):064x}",
|
||||||
|
)
|
||||||
|
client = TestClient(app)
|
||||||
|
login_user(client, email="api-admin-dbops@example.com", password="admin-password")
|
||||||
|
headers = _api_csrf_headers(client)
|
||||||
|
|
||||||
|
integrity_response = client.get("/api/v1/admin/db/integrity")
|
||||||
|
assert integrity_response.status_code == 200
|
||||||
|
integrity_payload = integrity_response.json()["data"]
|
||||||
|
assert integrity_payload["status"] in {"ok", "warning", "failed"}
|
||||||
|
assert any(
|
||||||
|
check["name"] == "missing_pdf_url"
|
||||||
|
for check in integrity_payload["checks"]
|
||||||
|
)
|
||||||
|
|
||||||
|
repair_response = client.post(
|
||||||
|
"/api/v1/admin/db/repairs/publication-links",
|
||||||
|
json={"user_id": target_user_id, "dry_run": True},
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
assert repair_response.status_code == 200
|
||||||
|
repair_data = repair_response.json()["data"]
|
||||||
|
assert repair_data["status"] == "completed"
|
||||||
|
assert bool(repair_data["summary"]["dry_run"]) is True
|
||||||
|
job_id = int(repair_data["job_id"])
|
||||||
|
|
||||||
|
jobs_response = client.get("/api/v1/admin/db/repair-jobs?limit=10")
|
||||||
|
assert jobs_response.status_code == 200
|
||||||
|
jobs = jobs_response.json()["data"]["jobs"]
|
||||||
|
assert any(int(job["id"]) == job_id for job in jobs)
|
||||||
|
|
||||||
|
pdf_queue_response = client.get("/api/v1/admin/db/pdf-queue?limit=20")
|
||||||
|
assert pdf_queue_response.status_code == 200
|
||||||
|
pdf_queue_payload = pdf_queue_response.json()["data"]
|
||||||
|
assert isinstance(pdf_queue_payload["items"], list)
|
||||||
|
assert int(pdf_queue_payload["page"]) == 1
|
||||||
|
assert int(pdf_queue_payload["page_size"]) == 20
|
||||||
|
assert int(pdf_queue_payload["total_count"]) >= 0
|
||||||
|
assert isinstance(pdf_queue_payload["has_next"], bool)
|
||||||
|
assert isinstance(pdf_queue_payload["has_prev"], bool)
|
||||||
|
page_two_response = client.get("/api/v1/admin/db/pdf-queue?page=2&page_size=1")
|
||||||
|
assert page_two_response.status_code == 200
|
||||||
|
page_two_payload = page_two_response.json()["data"]
|
||||||
|
assert int(page_two_payload["page"]) == 2
|
||||||
|
assert int(page_two_payload["page_size"]) == 1
|
||||||
|
assert page_two_payload["has_prev"] is True
|
||||||
|
untracked_response = client.get("/api/v1/admin/db/pdf-queue?limit=20&status=untracked")
|
||||||
|
assert untracked_response.status_code == 200
|
||||||
|
assert any(
|
||||||
|
item["status"] == "untracked"
|
||||||
|
for item in untracked_response.json()["data"]["items"]
|
||||||
|
)
|
||||||
|
|
||||||
|
link_count_result = await db_session.execute(
|
||||||
|
text("SELECT count(*) FROM scholar_publications WHERE publication_id = :publication_id"),
|
||||||
|
{"publication_id": publication_id},
|
||||||
|
)
|
||||||
|
assert int(link_count_result.scalar_one()) == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.db
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_api_admin_dbops_pdf_queue_requeue_endpoint(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
await insert_user(db_session, email="api-admin-pdf@example.com", password="admin-password", is_admin=True)
|
||||||
|
target_user_id = await insert_user(db_session, email="api-pdf-target@example.com", password="target-password")
|
||||||
|
_scholar_id, publication_id = await _seed_publication_link_for_user(
|
||||||
|
db_session,
|
||||||
|
user_id=target_user_id,
|
||||||
|
scholar_id="dbopsPdf01",
|
||||||
|
title="PDF Queue Target",
|
||||||
|
fingerprint=f"{(target_user_id + 73):064x}",
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.domains.publications.pdf_queue._schedule_rows",
|
||||||
|
lambda **_kwargs: None,
|
||||||
|
)
|
||||||
|
client = TestClient(app)
|
||||||
|
login_user(client, email="api-admin-pdf@example.com", password="admin-password")
|
||||||
|
headers = _api_csrf_headers(client)
|
||||||
|
|
||||||
|
first_response = client.post(
|
||||||
|
f"/api/v1/admin/db/pdf-queue/{publication_id}/requeue",
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
assert first_response.status_code == 200
|
||||||
|
first_data = first_response.json()["data"]
|
||||||
|
assert first_data["publication_id"] == publication_id
|
||||||
|
assert first_data["queued"] is True
|
||||||
|
assert first_data["status"] == "queued"
|
||||||
|
|
||||||
|
second_response = client.post(
|
||||||
|
f"/api/v1/admin/db/pdf-queue/{publication_id}/requeue",
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
assert second_response.status_code == 200
|
||||||
|
second_data = second_response.json()["data"]
|
||||||
|
assert second_data["queued"] is False
|
||||||
|
assert second_data["status"] == "blocked"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.db
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_api_admin_dbops_pdf_queue_requeue_all_endpoint(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
await insert_user(db_session, email="api-admin-pdf-all@example.com", password="admin-password", is_admin=True)
|
||||||
|
target_user_id = await insert_user(db_session, email="api-pdf-all-target@example.com", password="target-password")
|
||||||
|
_scholar_id, _publication_id = await _seed_publication_link_for_user(
|
||||||
|
db_session,
|
||||||
|
user_id=target_user_id,
|
||||||
|
scholar_id="dbopsPdfAll01",
|
||||||
|
title="PDF Queue All Target",
|
||||||
|
fingerprint=f"{(target_user_id + 83):064x}",
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.domains.publications.pdf_queue._schedule_rows",
|
||||||
|
lambda **_kwargs: None,
|
||||||
|
)
|
||||||
|
client = TestClient(app)
|
||||||
|
login_user(client, email="api-admin-pdf-all@example.com", password="admin-password")
|
||||||
|
headers = _api_csrf_headers(client)
|
||||||
|
|
||||||
|
response = client.post(
|
||||||
|
"/api/v1/admin/db/pdf-queue/requeue-all?limit=500",
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
assert response.status_code == 200
|
||||||
|
payload = response.json()["data"]
|
||||||
|
assert int(payload["requested_count"]) >= 1
|
||||||
|
assert int(payload["queued_count"]) >= 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.db
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_api_admin_dbops_forbidden_for_non_admin_and_validates_scope(db_session: AsyncSession) -> None:
|
||||||
|
await insert_user(db_session, email="api-admin-dbops2@example.com", password="admin-password", is_admin=True)
|
||||||
|
member_user_id = await insert_user(db_session, email="api-dbops-member@example.com", password="member-password")
|
||||||
|
client = TestClient(app)
|
||||||
|
login_user(client, email="api-dbops-member@example.com", password="member-password")
|
||||||
|
headers = _api_csrf_headers(client)
|
||||||
|
|
||||||
|
forbidden_integrity = client.get("/api/v1/admin/db/integrity")
|
||||||
|
assert forbidden_integrity.status_code == 403
|
||||||
|
assert forbidden_integrity.json()["error"]["code"] == "forbidden"
|
||||||
|
|
||||||
|
forbidden_pdf_queue = client.get("/api/v1/admin/db/pdf-queue")
|
||||||
|
assert forbidden_pdf_queue.status_code == 403
|
||||||
|
assert forbidden_pdf_queue.json()["error"]["code"] == "forbidden"
|
||||||
|
|
||||||
|
forbidden_requeue = client.post(
|
||||||
|
"/api/v1/admin/db/pdf-queue/1/requeue",
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
assert forbidden_requeue.status_code == 403
|
||||||
|
assert forbidden_requeue.json()["error"]["code"] == "forbidden"
|
||||||
|
|
||||||
|
forbidden_requeue_all = client.post(
|
||||||
|
"/api/v1/admin/db/pdf-queue/requeue-all?limit=100",
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
assert forbidden_requeue_all.status_code == 403
|
||||||
|
assert forbidden_requeue_all.json()["error"]["code"] == "forbidden"
|
||||||
|
|
||||||
|
forbidden_repair = client.post(
|
||||||
|
"/api/v1/admin/db/repairs/publication-links",
|
||||||
|
json={"user_id": member_user_id, "dry_run": True},
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
assert forbidden_repair.status_code == 403
|
||||||
|
assert forbidden_repair.json()["error"]["code"] == "forbidden"
|
||||||
|
|
||||||
|
admin_headers = _api_bootstrap_csrf_headers(client)
|
||||||
|
admin_login = client.post(
|
||||||
|
"/api/v1/auth/login",
|
||||||
|
json={"email": "api-admin-dbops2@example.com", "password": "admin-password"},
|
||||||
|
headers=admin_headers,
|
||||||
|
)
|
||||||
|
assert admin_login.status_code == 200
|
||||||
|
post_login_headers = {"X-CSRF-Token": admin_login.json()["data"]["csrf_token"]}
|
||||||
|
invalid_scope = client.post(
|
||||||
|
"/api/v1/admin/db/repairs/publication-links",
|
||||||
|
json={"user_id": member_user_id, "dry_run": True},
|
||||||
|
headers=post_login_headers,
|
||||||
|
)
|
||||||
|
assert invalid_scope.status_code == 400
|
||||||
|
assert invalid_scope.json()["error"]["code"] == "invalid_repair_scope"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.db
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_api_admin_dbops_all_users_apply_requires_confirmation(db_session: AsyncSession) -> None:
|
||||||
|
await insert_user(db_session, email="api-admin-dbops3@example.com", password="admin-password", is_admin=True)
|
||||||
|
first_user_id = await insert_user(db_session, email="api-dbops-a@example.com", password="user-password")
|
||||||
|
second_user_id = await insert_user(db_session, email="api-dbops-b@example.com", password="user-password")
|
||||||
|
await _seed_publication_link_for_user(
|
||||||
|
db_session,
|
||||||
|
user_id=first_user_id,
|
||||||
|
scholar_id="dbopsAll01",
|
||||||
|
title="DB Ops All User Paper One",
|
||||||
|
fingerprint=f"{(first_user_id + 61):064x}",
|
||||||
|
)
|
||||||
|
await _seed_publication_link_for_user(
|
||||||
|
db_session,
|
||||||
|
user_id=second_user_id,
|
||||||
|
scholar_id="dbopsAll02",
|
||||||
|
title="DB Ops All User Paper Two",
|
||||||
|
fingerprint=f"{(second_user_id + 71):064x}",
|
||||||
|
)
|
||||||
|
|
||||||
|
client = TestClient(app)
|
||||||
|
login_user(client, email="api-admin-dbops3@example.com", password="admin-password")
|
||||||
|
headers = _api_csrf_headers(client)
|
||||||
|
|
||||||
|
missing_confirmation = client.post(
|
||||||
|
"/api/v1/admin/db/repairs/publication-links",
|
||||||
|
json={"scope_mode": "all_users", "dry_run": False},
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
assert missing_confirmation.status_code == 422
|
||||||
|
assert "confirmation_text" in str(missing_confirmation.json())
|
||||||
|
|
||||||
|
apply_response = client.post(
|
||||||
|
"/api/v1/admin/db/repairs/publication-links",
|
||||||
|
json={
|
||||||
|
"scope_mode": "all_users",
|
||||||
|
"dry_run": False,
|
||||||
|
"confirmation_text": "REPAIR ALL USERS",
|
||||||
|
},
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
assert apply_response.status_code == 200
|
||||||
|
apply_data = apply_response.json()["data"]
|
||||||
|
assert apply_data["status"] == "completed"
|
||||||
|
assert apply_data["scope"]["scope_mode"] == "all_users"
|
||||||
|
assert int(apply_data["summary"]["links_deleted"]) >= 2
|
||||||
|
|
||||||
|
remaining_links = await db_session.execute(text("SELECT count(*) FROM scholar_publications"))
|
||||||
|
assert int(remaining_links.scalar_one()) == 0
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.integration
|
@pytest.mark.integration
|
||||||
@pytest.mark.db
|
@pytest.mark.db
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|
@ -1400,12 +1703,19 @@ async def test_api_publications_list_and_mark_read(db_session: AsyncSession) ->
|
||||||
assert list_response.status_code == 200
|
assert list_response.status_code == 200
|
||||||
data = list_response.json()["data"]
|
data = list_response.json()["data"]
|
||||||
assert data["mode"] == "all"
|
assert data["mode"] == "all"
|
||||||
|
assert data["favorite_only"] is False
|
||||||
assert data["total_count"] == 2
|
assert data["total_count"] == 2
|
||||||
assert data["unread_count"] == 2
|
assert data["unread_count"] == 2
|
||||||
|
assert data["favorites_count"] == 0
|
||||||
assert data["latest_count"] == 0
|
assert data["latest_count"] == 0
|
||||||
assert data["new_count"] == data["latest_count"]
|
assert data["new_count"] == data["latest_count"]
|
||||||
|
assert data["page"] == 1
|
||||||
|
assert data["page_size"] == 100
|
||||||
|
assert data["has_prev"] is False
|
||||||
|
assert data["has_next"] is False
|
||||||
assert isinstance(data["publications"], list)
|
assert isinstance(data["publications"], list)
|
||||||
assert len(data["publications"]) == 2
|
assert len(data["publications"]) == 2
|
||||||
|
assert all(bool(item["is_favorite"]) is False for item in data["publications"])
|
||||||
pdf_urls = {item["title"]: item["pdf_url"] for item in data["publications"]}
|
pdf_urls = {item["title"]: item["pdf_url"] for item in data["publications"]}
|
||||||
assert pdf_urls["Paper A"] == "https://example.org/paper-a.pdf"
|
assert pdf_urls["Paper A"] == "https://example.org/paper-a.pdf"
|
||||||
assert pdf_urls["Paper B"] is None
|
assert pdf_urls["Paper B"] is None
|
||||||
|
|
@ -1465,6 +1775,220 @@ async def test_api_publications_list_and_mark_read(db_session: AsyncSession) ->
|
||||||
assert mark_response.json()["data"]["updated_count"] == 1
|
assert mark_response.json()["data"]["updated_count"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.db
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_api_publications_list_supports_pagination(db_session: AsyncSession) -> None:
|
||||||
|
user_id = await insert_user(
|
||||||
|
db_session,
|
||||||
|
email="api-pubs-paging@example.com",
|
||||||
|
password="api-password",
|
||||||
|
)
|
||||||
|
scholar_result = await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
|
||||||
|
VALUES (:user_id, :scholar_id, :display_name, true)
|
||||||
|
RETURNING id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"user_id": user_id,
|
||||||
|
"scholar_id": "pagingScholar01",
|
||||||
|
"display_name": "Paging Scholar",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
scholar_profile_id = int(scholar_result.scalar_one())
|
||||||
|
|
||||||
|
publication_ids: list[int] = []
|
||||||
|
for index in range(3):
|
||||||
|
created = await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
|
||||||
|
VALUES (:fingerprint, :title_raw, :title_normalized, 1)
|
||||||
|
RETURNING id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"fingerprint": f"{(user_id + 500 + index):064x}",
|
||||||
|
"title_raw": f"Paged Paper {index}",
|
||||||
|
"title_normalized": f"paged paper {index}",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
publication_ids.append(int(created.scalar_one()))
|
||||||
|
|
||||||
|
await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
|
||||||
|
VALUES
|
||||||
|
(:scholar_profile_id, :publication_1, false, false),
|
||||||
|
(:scholar_profile_id, :publication_2, false, false),
|
||||||
|
(:scholar_profile_id, :publication_3, false, false)
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"scholar_profile_id": scholar_profile_id,
|
||||||
|
"publication_1": publication_ids[0],
|
||||||
|
"publication_2": publication_ids[1],
|
||||||
|
"publication_3": publication_ids[2],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
await db_session.commit()
|
||||||
|
|
||||||
|
client = TestClient(app)
|
||||||
|
login_user(client, email="api-pubs-paging@example.com", password="api-password")
|
||||||
|
|
||||||
|
first_page = client.get("/api/v1/publications?mode=all&page=1&page_size=2")
|
||||||
|
assert first_page.status_code == 200
|
||||||
|
first_data = first_page.json()["data"]
|
||||||
|
assert first_data["total_count"] == 3
|
||||||
|
assert first_data["page"] == 1
|
||||||
|
assert first_data["page_size"] == 2
|
||||||
|
assert first_data["has_prev"] is False
|
||||||
|
assert first_data["has_next"] is True
|
||||||
|
assert len(first_data["publications"]) == 2
|
||||||
|
|
||||||
|
second_page = client.get("/api/v1/publications?mode=all&page=2&page_size=2")
|
||||||
|
assert second_page.status_code == 200
|
||||||
|
second_data = second_page.json()["data"]
|
||||||
|
assert second_data["total_count"] == 3
|
||||||
|
assert second_data["page"] == 2
|
||||||
|
assert second_data["page_size"] == 2
|
||||||
|
assert second_data["has_prev"] is True
|
||||||
|
assert second_data["has_next"] is False
|
||||||
|
assert len(second_data["publications"]) == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.db
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_api_publications_favorite_toggle_and_filter(db_session: AsyncSession) -> None:
|
||||||
|
user_id = await insert_user(
|
||||||
|
db_session,
|
||||||
|
email="api-pubs-favorites@example.com",
|
||||||
|
password="api-password",
|
||||||
|
)
|
||||||
|
scholar_result = await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
|
||||||
|
VALUES (:user_id, :scholar_id, :display_name, true)
|
||||||
|
RETURNING id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"user_id": user_id,
|
||||||
|
"scholar_id": "favoriteScholar01",
|
||||||
|
"display_name": "Favorite Scholar",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
scholar_profile_id = int(scholar_result.scalar_one())
|
||||||
|
publication_a_result = await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
|
||||||
|
VALUES (:fingerprint, :title_raw, :title_normalized, 9)
|
||||||
|
RETURNING id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"fingerprint": f"{(user_id + 101):064x}",
|
||||||
|
"title_raw": "Favorite Target",
|
||||||
|
"title_normalized": "favorite target",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
publication_a_id = int(publication_a_result.scalar_one())
|
||||||
|
publication_b_result = await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
|
||||||
|
VALUES (:fingerprint, :title_raw, :title_normalized, 2)
|
||||||
|
RETURNING id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"fingerprint": f"{(user_id + 102):064x}",
|
||||||
|
"title_raw": "Not Favorite",
|
||||||
|
"title_normalized": "not favorite",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
publication_b_id = int(publication_b_result.scalar_one())
|
||||||
|
await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read, is_favorite)
|
||||||
|
VALUES
|
||||||
|
(:scholar_profile_id, :publication_a_id, false, false),
|
||||||
|
(:scholar_profile_id, :publication_b_id, false, false)
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"scholar_profile_id": scholar_profile_id,
|
||||||
|
"publication_a_id": publication_a_id,
|
||||||
|
"publication_b_id": publication_b_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
await db_session.commit()
|
||||||
|
|
||||||
|
client = TestClient(app)
|
||||||
|
login_user(client, email="api-pubs-favorites@example.com", password="api-password")
|
||||||
|
headers = _api_csrf_headers(client)
|
||||||
|
|
||||||
|
set_response = client.post(
|
||||||
|
f"/api/v1/publications/{publication_a_id}/favorite",
|
||||||
|
json={"scholar_profile_id": scholar_profile_id, "is_favorite": True},
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
assert set_response.status_code == 200
|
||||||
|
set_data = set_response.json()["data"]
|
||||||
|
assert set_data["publication"]["publication_id"] == publication_a_id
|
||||||
|
assert set_data["publication"]["is_favorite"] is True
|
||||||
|
|
||||||
|
favorite_only_response = client.get("/api/v1/publications?mode=all&favorite_only=true")
|
||||||
|
assert favorite_only_response.status_code == 200
|
||||||
|
favorite_only_data = favorite_only_response.json()["data"]
|
||||||
|
assert favorite_only_data["favorite_only"] is True
|
||||||
|
assert favorite_only_data["favorites_count"] == 1
|
||||||
|
assert favorite_only_data["total_count"] == 1
|
||||||
|
assert len(favorite_only_data["publications"]) == 1
|
||||||
|
assert int(favorite_only_data["publications"][0]["publication_id"]) == publication_a_id
|
||||||
|
|
||||||
|
clear_response = client.post(
|
||||||
|
f"/api/v1/publications/{publication_a_id}/favorite",
|
||||||
|
json={"scholar_profile_id": scholar_profile_id, "is_favorite": False},
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
assert clear_response.status_code == 200
|
||||||
|
clear_data = clear_response.json()["data"]
|
||||||
|
assert clear_data["publication"]["publication_id"] == publication_a_id
|
||||||
|
assert clear_data["publication"]["is_favorite"] is False
|
||||||
|
|
||||||
|
favorite_only_after_clear_response = client.get("/api/v1/publications?mode=all&favorite_only=true")
|
||||||
|
assert favorite_only_after_clear_response.status_code == 200
|
||||||
|
favorite_only_after_clear_data = favorite_only_after_clear_response.json()["data"]
|
||||||
|
assert favorite_only_after_clear_data["favorites_count"] == 0
|
||||||
|
assert favorite_only_after_clear_data["total_count"] == 0
|
||||||
|
assert favorite_only_after_clear_data["publications"] == []
|
||||||
|
|
||||||
|
favorite_state_result = await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT is_favorite
|
||||||
|
FROM scholar_publications
|
||||||
|
WHERE scholar_profile_id = :scholar_profile_id
|
||||||
|
AND publication_id = :publication_id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"scholar_profile_id": scholar_profile_id,
|
||||||
|
"publication_id": publication_a_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert bool(favorite_state_result.scalar_one()) is False
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.integration
|
@pytest.mark.integration
|
||||||
@pytest.mark.db
|
@pytest.mark.db
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|
@ -1521,20 +2045,14 @@ async def test_api_publications_list_schedules_background_enrichment(
|
||||||
)
|
)
|
||||||
await db_session.commit()
|
await db_session.commit()
|
||||||
|
|
||||||
async def _fail_resolver(*args, **kwargs):
|
async def _fake_schedule(db_session, *, user_id: int, request_email: str | None, items, max_items: int):
|
||||||
raise AssertionError("list endpoint should not resolve OA metadata inline")
|
assert db_session is not None
|
||||||
|
|
||||||
async def _fake_schedule(*, user_id: int, request_email: str | None, items, max_items: int):
|
|
||||||
assert user_id > 0
|
assert user_id > 0
|
||||||
assert request_email == "api-pubs-bg@example.com"
|
assert request_email == "api-pubs-bg@example.com"
|
||||||
assert int(max_items) > 0
|
assert int(max_items) > 0
|
||||||
assert any(int(item.publication_id) == publication_id for item in items)
|
assert any(int(item.publication_id) == publication_id for item in items)
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"app.services.domains.publications.listing.resolve_publication_oa_metadata",
|
|
||||||
_fail_resolver,
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.publications.application.schedule_missing_pdf_enrichment_for_user",
|
"app.services.domains.publications.application.schedule_missing_pdf_enrichment_for_user",
|
||||||
_fake_schedule,
|
_fake_schedule,
|
||||||
|
|
@ -1549,12 +2067,13 @@ async def test_api_publications_list_schedules_background_enrichment(
|
||||||
assert len(data["publications"]) == 1
|
assert len(data["publications"]) == 1
|
||||||
assert int(data["publications"][0]["publication_id"]) == publication_id
|
assert int(data["publications"][0]["publication_id"]) == publication_id
|
||||||
assert data["publications"][0]["pdf_url"] is None
|
assert data["publications"][0]["pdf_url"] is None
|
||||||
|
assert data["publications"][0]["pdf_status"] in {"untracked", "queued", "running", "failed"}
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.integration
|
@pytest.mark.integration
|
||||||
@pytest.mark.db
|
@pytest.mark.db
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_api_publication_retry_pdf_updates_publication(
|
async def test_api_publication_retry_pdf_queues_resolution_job(
|
||||||
db_session: AsyncSession,
|
db_session: AsyncSession,
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|
@ -1607,13 +2126,47 @@ async def test_api_publication_retry_pdf_updates_publication(
|
||||||
)
|
)
|
||||||
await db_session.commit()
|
await db_session.commit()
|
||||||
|
|
||||||
async def _fake_resolver(items, *, request_email=None):
|
async def _fake_retry_scheduler(db_session, *, user_id: int, request_email: str | None, item: PublicationListItem):
|
||||||
|
assert db_session is not None
|
||||||
|
assert user_id > 0
|
||||||
assert request_email == "api-pubs-retry@example.com"
|
assert request_email == "api-pubs-retry@example.com"
|
||||||
return {int(items[0].publication_id): ("10.1000/retry-target", "https://oa.example/retry.pdf")}
|
assert int(item.publication_id) == publication_id
|
||||||
|
return True
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.domains.publications.listing.resolve_publication_oa_metadata",
|
"app.services.domains.publications.application.schedule_retry_pdf_enrichment_for_row",
|
||||||
_fake_resolver,
|
_fake_retry_scheduler,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _fake_hydrate(db_session, *, items: list[PublicationListItem]):
|
||||||
|
assert db_session is not None
|
||||||
|
assert len(items) == 1
|
||||||
|
item = items[0]
|
||||||
|
return [
|
||||||
|
PublicationListItem(
|
||||||
|
publication_id=item.publication_id,
|
||||||
|
scholar_profile_id=item.scholar_profile_id,
|
||||||
|
scholar_label=item.scholar_label,
|
||||||
|
title=item.title,
|
||||||
|
year=item.year,
|
||||||
|
citation_count=item.citation_count,
|
||||||
|
venue_text=item.venue_text,
|
||||||
|
pub_url=item.pub_url,
|
||||||
|
doi=item.doi,
|
||||||
|
pdf_url=item.pdf_url,
|
||||||
|
is_read=item.is_read,
|
||||||
|
first_seen_at=item.first_seen_at,
|
||||||
|
is_new_in_latest_run=item.is_new_in_latest_run,
|
||||||
|
pdf_status="queued",
|
||||||
|
pdf_attempt_count=1,
|
||||||
|
pdf_failure_reason="no_pdf_found",
|
||||||
|
pdf_failure_detail="no_pdf_found",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.domains.publications.application.hydrate_pdf_enrichment_state",
|
||||||
|
_fake_hydrate,
|
||||||
)
|
)
|
||||||
|
|
||||||
client = TestClient(app)
|
client = TestClient(app)
|
||||||
|
|
@ -1627,18 +2180,21 @@ async def test_api_publication_retry_pdf_updates_publication(
|
||||||
)
|
)
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
payload = response.json()["data"]
|
payload = response.json()["data"]
|
||||||
assert payload["resolved_pdf"] is True
|
assert payload["queued"] is True
|
||||||
|
assert payload["resolved_pdf"] is False
|
||||||
assert payload["publication"]["publication_id"] == publication_id
|
assert payload["publication"]["publication_id"] == publication_id
|
||||||
assert payload["publication"]["pdf_url"] == "https://oa.example/retry.pdf"
|
assert payload["publication"]["pdf_url"] is None
|
||||||
assert payload["publication"]["doi"] == "10.1000/retry-target"
|
assert payload["publication"]["pdf_status"] == "queued"
|
||||||
|
assert payload["publication"]["pdf_attempt_count"] == 1
|
||||||
|
assert payload["publication"]["pdf_failure_reason"] == "no_pdf_found"
|
||||||
|
|
||||||
stored = await db_session.execute(
|
stored = await db_session.execute(
|
||||||
text("SELECT doi, pdf_url FROM publications WHERE id = :publication_id"),
|
text("SELECT doi, pdf_url FROM publications WHERE id = :publication_id"),
|
||||||
{"publication_id": publication_id},
|
{"publication_id": publication_id},
|
||||||
)
|
)
|
||||||
stored_doi, stored_pdf_url = stored.one()
|
stored_doi, stored_pdf_url = stored.one()
|
||||||
assert stored_doi == "10.1000/retry-target"
|
assert stored_doi is None
|
||||||
assert stored_pdf_url == "https://oa.example/retry.pdf"
|
assert stored_pdf_url is None
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.integration
|
@pytest.mark.integration
|
||||||
|
|
|
||||||
252
tests/integration/test_data_repair_jobs.py
Normal file
|
|
@ -0,0 +1,252 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.services.domains.dbops import run_publication_link_repair
|
||||||
|
from tests.integration.helpers import insert_user
|
||||||
|
|
||||||
|
|
||||||
|
async def _insert_scholar_profile(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
scholar_id: str,
|
||||||
|
display_name: str,
|
||||||
|
baseline_completed: bool = False,
|
||||||
|
) -> int:
|
||||||
|
result = await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled, baseline_completed)
|
||||||
|
VALUES (:user_id, :scholar_id, :display_name, true, :baseline_completed)
|
||||||
|
RETURNING id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"user_id": user_id,
|
||||||
|
"scholar_id": scholar_id,
|
||||||
|
"display_name": display_name,
|
||||||
|
"baseline_completed": baseline_completed,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return int(result.scalar_one())
|
||||||
|
|
||||||
|
|
||||||
|
async def _insert_publication(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
fingerprint: str,
|
||||||
|
title_raw: str,
|
||||||
|
title_normalized: str,
|
||||||
|
citation_count: int,
|
||||||
|
) -> int:
|
||||||
|
result = await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
|
||||||
|
VALUES (:fingerprint, :title_raw, :title_normalized, :citation_count)
|
||||||
|
RETURNING id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"fingerprint": fingerprint,
|
||||||
|
"title_raw": title_raw,
|
||||||
|
"title_normalized": title_normalized,
|
||||||
|
"citation_count": citation_count,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return int(result.scalar_one())
|
||||||
|
|
||||||
|
|
||||||
|
async def _insert_scholar_publication_link(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
scholar_profile_id: int,
|
||||||
|
publication_id: int,
|
||||||
|
) -> None:
|
||||||
|
await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
|
||||||
|
VALUES (:scholar_profile_id, :publication_id, false)
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"scholar_profile_id": scholar_profile_id,
|
||||||
|
"publication_id": publication_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _insert_queue_item(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
user_id: int,
|
||||||
|
scholar_profile_id: int,
|
||||||
|
reason: str,
|
||||||
|
) -> None:
|
||||||
|
await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO ingestion_queue_items (user_id, scholar_profile_id, reason)
|
||||||
|
VALUES (:user_id, :scholar_profile_id, :reason)
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"user_id": user_id,
|
||||||
|
"scholar_profile_id": scholar_profile_id,
|
||||||
|
"reason": reason,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _count_rows(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
sql: str,
|
||||||
|
params: dict[str, int],
|
||||||
|
) -> int:
|
||||||
|
result = await db_session.execute(text(sql), params)
|
||||||
|
return int(result.scalar_one())
|
||||||
|
|
||||||
|
|
||||||
|
async def _seed_apply_case(db_session: AsyncSession, *, user_id: int) -> tuple[int, int]:
|
||||||
|
scholar_profile_id = await _insert_scholar_profile(
|
||||||
|
db_session,
|
||||||
|
user_id=user_id,
|
||||||
|
scholar_id="repairApply01",
|
||||||
|
display_name="Repair Apply",
|
||||||
|
baseline_completed=True,
|
||||||
|
)
|
||||||
|
publication_id = await _insert_publication(
|
||||||
|
db_session,
|
||||||
|
fingerprint=f"{(user_id + 1):064x}",
|
||||||
|
title_raw="Repair Apply Paper",
|
||||||
|
title_normalized="repair apply paper",
|
||||||
|
citation_count=9,
|
||||||
|
)
|
||||||
|
await _insert_scholar_publication_link(
|
||||||
|
db_session,
|
||||||
|
scholar_profile_id=scholar_profile_id,
|
||||||
|
publication_id=publication_id,
|
||||||
|
)
|
||||||
|
await _insert_queue_item(
|
||||||
|
db_session,
|
||||||
|
user_id=user_id,
|
||||||
|
scholar_profile_id=scholar_profile_id,
|
||||||
|
reason="manual_repair_test",
|
||||||
|
)
|
||||||
|
await db_session.commit()
|
||||||
|
return scholar_profile_id, publication_id
|
||||||
|
|
||||||
|
|
||||||
|
async def _assert_apply_effects(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
*,
|
||||||
|
scholar_profile_id: int,
|
||||||
|
publication_id: int,
|
||||||
|
) -> None:
|
||||||
|
links_count = await _count_rows(
|
||||||
|
db_session,
|
||||||
|
sql="SELECT count(*) FROM scholar_publications WHERE scholar_profile_id = :scholar_profile_id",
|
||||||
|
params={"scholar_profile_id": scholar_profile_id},
|
||||||
|
)
|
||||||
|
queue_count = await _count_rows(
|
||||||
|
db_session,
|
||||||
|
sql="SELECT count(*) FROM ingestion_queue_items WHERE scholar_profile_id = :scholar_profile_id",
|
||||||
|
params={"scholar_profile_id": scholar_profile_id},
|
||||||
|
)
|
||||||
|
baseline_completed = await _count_rows(
|
||||||
|
db_session,
|
||||||
|
sql=(
|
||||||
|
"SELECT count(*) FROM scholar_profiles "
|
||||||
|
"WHERE id = :scholar_profile_id AND baseline_completed = false"
|
||||||
|
),
|
||||||
|
params={"scholar_profile_id": scholar_profile_id},
|
||||||
|
)
|
||||||
|
publication_count = await _count_rows(
|
||||||
|
db_session,
|
||||||
|
sql="SELECT count(*) FROM publications WHERE id = :publication_id",
|
||||||
|
params={"publication_id": publication_id},
|
||||||
|
)
|
||||||
|
assert links_count == 0
|
||||||
|
assert queue_count == 0
|
||||||
|
assert baseline_completed == 1
|
||||||
|
assert publication_count == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.db
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_publication_link_repair_dry_run_records_job_without_mutation(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
) -> None:
|
||||||
|
user_id = await insert_user(db_session, email="repair-dry@example.com", password="api-password")
|
||||||
|
scholar_profile_id = await _insert_scholar_profile(
|
||||||
|
db_session,
|
||||||
|
user_id=user_id,
|
||||||
|
scholar_id="repairDry001",
|
||||||
|
display_name="Repair Dry",
|
||||||
|
)
|
||||||
|
publication_id = await _insert_publication(
|
||||||
|
db_session,
|
||||||
|
fingerprint=f"{user_id:064x}",
|
||||||
|
title_raw="Repair Dry Paper",
|
||||||
|
title_normalized="repair dry paper",
|
||||||
|
citation_count=1,
|
||||||
|
)
|
||||||
|
await _insert_scholar_publication_link(
|
||||||
|
db_session,
|
||||||
|
scholar_profile_id=scholar_profile_id,
|
||||||
|
publication_id=publication_id,
|
||||||
|
)
|
||||||
|
await db_session.commit()
|
||||||
|
|
||||||
|
result = await run_publication_link_repair(
|
||||||
|
db_session,
|
||||||
|
user_id=user_id,
|
||||||
|
dry_run=True,
|
||||||
|
requested_by="test-suite",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["status"] == "completed"
|
||||||
|
assert bool(result["summary"]["dry_run"]) is True
|
||||||
|
assert int(result["summary"]["links_deleted"]) == 0
|
||||||
|
|
||||||
|
links_count = await _count_rows(
|
||||||
|
db_session,
|
||||||
|
sql="SELECT count(*) FROM scholar_publications WHERE scholar_profile_id = :scholar_profile_id",
|
||||||
|
params={"scholar_profile_id": scholar_profile_id},
|
||||||
|
)
|
||||||
|
assert links_count == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.db
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_publication_link_repair_apply_clears_links_and_can_gc_orphans(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
) -> None:
|
||||||
|
user_id = await insert_user(db_session, email="repair-apply@example.com", password="api-password")
|
||||||
|
scholar_profile_id, publication_id = await _seed_apply_case(db_session, user_id=user_id)
|
||||||
|
|
||||||
|
result = await run_publication_link_repair(
|
||||||
|
db_session,
|
||||||
|
user_id=user_id,
|
||||||
|
dry_run=False,
|
||||||
|
gc_orphan_publications=True,
|
||||||
|
requested_by="test-suite",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["status"] == "completed"
|
||||||
|
assert bool(result["summary"]["dry_run"]) is False
|
||||||
|
assert int(result["summary"]["links_deleted"]) == 1
|
||||||
|
assert int(result["summary"]["queue_items_deleted"]) == 1
|
||||||
|
assert int(result["summary"]["scholars_reset"]) == 1
|
||||||
|
await _assert_apply_effects(
|
||||||
|
db_session,
|
||||||
|
scholar_profile_id=scholar_profile_id,
|
||||||
|
publication_id=publication_id,
|
||||||
|
)
|
||||||
70
tests/integration/test_db_integrity.py
Normal file
|
|
@ -0,0 +1,70 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.services.domains.dbops.integrity import collect_integrity_report
|
||||||
|
|
||||||
|
|
||||||
|
def _check_counts(report: dict) -> dict[str, int]:
|
||||||
|
return {row["name"]: int(row["count"]) for row in report["checks"]}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.db
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_collect_integrity_report_ok_on_clean_database(db_session: AsyncSession) -> None:
|
||||||
|
report = await collect_integrity_report(db_session)
|
||||||
|
|
||||||
|
assert report["status"] == "ok"
|
||||||
|
assert report["failures"] == []
|
||||||
|
assert report["warnings"] == []
|
||||||
|
|
||||||
|
check_counts = _check_counts(report)
|
||||||
|
assert all(count == 0 for count in check_counts.values())
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.db
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_collect_integrity_report_detects_warning_and_failure(db_session: AsyncSession) -> None:
|
||||||
|
await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO publications (
|
||||||
|
fingerprint_sha256,
|
||||||
|
cluster_id,
|
||||||
|
title_raw,
|
||||||
|
title_normalized,
|
||||||
|
citation_count
|
||||||
|
)
|
||||||
|
VALUES (
|
||||||
|
:fingerprint_sha256,
|
||||||
|
:cluster_id,
|
||||||
|
:title_raw,
|
||||||
|
:title_normalized,
|
||||||
|
:citation_count
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"fingerprint_sha256": f"{1:064x}",
|
||||||
|
"cluster_id": "legacy-cluster-123",
|
||||||
|
"title_raw": "Integrity test paper",
|
||||||
|
"title_normalized": "integrity test paper",
|
||||||
|
"citation_count": -5,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
await db_session.commit()
|
||||||
|
|
||||||
|
report = await collect_integrity_report(db_session)
|
||||||
|
check_counts = _check_counts(report)
|
||||||
|
|
||||||
|
assert report["status"] == "failed"
|
||||||
|
assert "negative_citation_count" in report["failures"]
|
||||||
|
assert "legacy_cluster_id_format" in report["warnings"]
|
||||||
|
assert "orphan_publications" in report["warnings"]
|
||||||
|
assert check_counts["negative_citation_count"] == 1
|
||||||
|
assert check_counts["legacy_cluster_id_format"] == 1
|
||||||
|
assert check_counts["orphan_publications"] == 1
|
||||||
|
|
@ -13,10 +13,13 @@ EXPECTED_TABLES = {
|
||||||
"ingestion_queue_items",
|
"ingestion_queue_items",
|
||||||
"author_search_runtime_state",
|
"author_search_runtime_state",
|
||||||
"author_search_cache_entries",
|
"author_search_cache_entries",
|
||||||
|
"data_repair_jobs",
|
||||||
|
"publication_pdf_jobs",
|
||||||
|
"publication_pdf_job_events",
|
||||||
}
|
}
|
||||||
|
|
||||||
EXPECTED_ENUMS = {"run_status", "run_trigger_type"}
|
EXPECTED_ENUMS = {"run_status", "run_trigger_type"}
|
||||||
EXPECTED_REVISION = "20260220_0011"
|
EXPECTED_REVISION = "20260221_0015"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.integration
|
@pytest.mark.integration
|
||||||
|
|
@ -220,3 +223,44 @@ async def test_user_settings_has_scrape_safety_columns(db_session: AsyncSession)
|
||||||
)
|
)
|
||||||
columns = {row[0] for row in result}
|
columns = {row[0] for row in result}
|
||||||
assert columns == {"scrape_safety_state", "scrape_cooldown_until", "scrape_cooldown_reason"}
|
assert columns == {"scrape_safety_state", "scrape_cooldown_until", "scrape_cooldown_reason"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.db
|
||||||
|
@pytest.mark.migrations
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_scholar_publications_has_is_favorite_column(db_session: AsyncSession) -> None:
|
||||||
|
result = await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT 1
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_name = 'scholar_publications'
|
||||||
|
AND column_name = 'is_favorite'
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert result.scalar_one() == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.db
|
||||||
|
@pytest.mark.migrations
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_user_settings_request_delay_constraint_enforces_two_second_floor(
|
||||||
|
db_session: AsyncSession,
|
||||||
|
) -> None:
|
||||||
|
result = await db_session.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT pg_get_constraintdef(c.oid)
|
||||||
|
FROM pg_constraint c
|
||||||
|
JOIN pg_class t ON t.oid = c.conrelid
|
||||||
|
WHERE t.relname = 'user_settings'
|
||||||
|
AND c.contype = 'c'
|
||||||
|
AND pg_get_constraintdef(c.oid) ILIKE '%request_delay_seconds >= 2%'
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
)
|
||||||
|
definition = result.scalar_one()
|
||||||
|
assert "request_delay_seconds >= 2" in definition
|
||||||
|
|
|
||||||
109
tests/unit/test_publication_pdf_queue_policy.py
Normal file
|
|
@ -0,0 +1,109 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.db.models import PublicationPdfJob
|
||||||
|
from app.services.domains.publications import pdf_queue
|
||||||
|
|
||||||
|
|
||||||
|
def _job(
|
||||||
|
*,
|
||||||
|
status: str,
|
||||||
|
attempt_count: int,
|
||||||
|
last_attempt_at: datetime | None,
|
||||||
|
) -> PublicationPdfJob:
|
||||||
|
return PublicationPdfJob(
|
||||||
|
publication_id=1,
|
||||||
|
status=status,
|
||||||
|
attempt_count=attempt_count,
|
||||||
|
last_attempt_at=last_attempt_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pdf_queue_auto_enqueue_blocks_recent_attempt(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
|
||||||
|
job = _job(
|
||||||
|
status=pdf_queue.PDF_STATUS_FAILED,
|
||||||
|
attempt_count=1,
|
||||||
|
last_attempt_at=now - timedelta(hours=2),
|
||||||
|
)
|
||||||
|
assert pdf_queue._can_enqueue_job(job, force_retry=False) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_pdf_queue_auto_enqueue_blocks_recent_first_retry(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
|
||||||
|
job = _job(
|
||||||
|
status=pdf_queue.PDF_STATUS_FAILED,
|
||||||
|
attempt_count=1,
|
||||||
|
last_attempt_at=now - timedelta(minutes=20),
|
||||||
|
)
|
||||||
|
assert pdf_queue._can_enqueue_job(job, force_retry=False) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_pdf_queue_auto_enqueue_blocks_after_max_attempts(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
|
||||||
|
job = _job(
|
||||||
|
status=pdf_queue.PDF_STATUS_FAILED,
|
||||||
|
attempt_count=3,
|
||||||
|
last_attempt_at=now - timedelta(days=2),
|
||||||
|
)
|
||||||
|
assert pdf_queue._can_enqueue_job(job, force_retry=False) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_pdf_queue_auto_enqueue_blocks_second_retry_within_day(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
|
||||||
|
job = _job(
|
||||||
|
status=pdf_queue.PDF_STATUS_FAILED,
|
||||||
|
attempt_count=2,
|
||||||
|
last_attempt_at=now - timedelta(hours=2),
|
||||||
|
)
|
||||||
|
assert pdf_queue._can_enqueue_job(job, force_retry=False) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_pdf_queue_manual_requeue_bypasses_cooldown_and_max_attempts(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
now = datetime(2026, 2, 21, 12, 0, tzinfo=timezone.utc)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_utcnow", lambda: now)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_first_interval_seconds", lambda: 3_600)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_interval_seconds", lambda: 86_400)
|
||||||
|
monkeypatch.setattr(pdf_queue, "_auto_retry_max_attempts", lambda: 3)
|
||||||
|
job = _job(
|
||||||
|
status=pdf_queue.PDF_STATUS_FAILED,
|
||||||
|
attempt_count=5,
|
||||||
|
last_attempt_at=now - timedelta(minutes=10),
|
||||||
|
)
|
||||||
|
assert pdf_queue._can_enqueue_job(job, force_retry=True) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_pdf_queue_manual_requeue_still_blocks_when_inflight() -> None:
|
||||||
|
running = _job(
|
||||||
|
status=pdf_queue.PDF_STATUS_RUNNING,
|
||||||
|
attempt_count=1,
|
||||||
|
last_attempt_at=None,
|
||||||
|
)
|
||||||
|
queued = _job(
|
||||||
|
status=pdf_queue.PDF_STATUS_QUEUED,
|
||||||
|
attempt_count=1,
|
||||||
|
last_attempt_at=None,
|
||||||
|
)
|
||||||
|
assert pdf_queue._can_enqueue_job(running, force_retry=True) is False
|
||||||
|
assert pdf_queue._can_enqueue_job(queued, force_retry=True) is False
|
||||||
45
tests/unit/test_request_delay_policy.py
Normal file
|
|
@ -0,0 +1,45 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from app.services.domains.ingestion.application import ScholarIngestionService
|
||||||
|
from app.services.domains.ingestion import scheduler as scheduler_module
|
||||||
|
from app.settings import settings
|
||||||
|
|
||||||
|
|
||||||
|
def _set_policy_minimum(value: int) -> int:
|
||||||
|
previous = int(settings.ingestion_min_request_delay_seconds)
|
||||||
|
object.__setattr__(settings, "ingestion_min_request_delay_seconds", int(value))
|
||||||
|
return previous
|
||||||
|
|
||||||
|
|
||||||
|
def test_ingestion_effective_request_delay_respects_policy_minimum() -> None:
|
||||||
|
previous = _set_policy_minimum(8)
|
||||||
|
try:
|
||||||
|
assert ScholarIngestionService._effective_request_delay_seconds(7) == 8
|
||||||
|
assert ScholarIngestionService._effective_request_delay_seconds(12) == 12
|
||||||
|
finally:
|
||||||
|
object.__setattr__(settings, "ingestion_min_request_delay_seconds", previous)
|
||||||
|
|
||||||
|
|
||||||
|
def test_scheduler_effective_request_delay_respects_policy_minimum() -> None:
|
||||||
|
previous = _set_policy_minimum(9)
|
||||||
|
try:
|
||||||
|
assert scheduler_module._effective_request_delay_seconds(None) == 9
|
||||||
|
assert scheduler_module._effective_request_delay_seconds(6) == 9
|
||||||
|
assert scheduler_module._effective_request_delay_seconds(11) == 11
|
||||||
|
finally:
|
||||||
|
object.__setattr__(settings, "ingestion_min_request_delay_seconds", previous)
|
||||||
|
|
||||||
|
|
||||||
|
def test_scheduler_candidate_row_clamps_request_delay() -> None:
|
||||||
|
previous = _set_policy_minimum(6)
|
||||||
|
try:
|
||||||
|
candidate = scheduler_module.SchedulerService._candidate_from_row(
|
||||||
|
(1, 15, 1, None, None),
|
||||||
|
now_utc=datetime(2026, 2, 21, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
assert candidate is not None
|
||||||
|
assert candidate.request_delay_seconds == 6
|
||||||
|
finally:
|
||||||
|
object.__setattr__(settings, "ingestion_min_request_delay_seconds", previous)
|
||||||
94
tests/unit/test_unpaywall_pdf_discovery.py
Normal file
|
|
@ -0,0 +1,94 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.services.domains.unpaywall import pdf_discovery
|
||||||
|
|
||||||
|
|
||||||
|
def test_looks_like_pdf_url_detects_path_and_query_variants() -> None:
|
||||||
|
assert pdf_discovery.looks_like_pdf_url("https://example.org/file.pdf")
|
||||||
|
assert pdf_discovery.looks_like_pdf_url("https://example.org/download?target=paper.pdf")
|
||||||
|
assert not pdf_discovery.looks_like_pdf_url("https://example.org/landing")
|
||||||
|
assert not pdf_discovery.looks_like_pdf_url(None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalized_candidate_urls_extracts_and_resolves_relative_links() -> None:
|
||||||
|
html = """
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<base href="https://publisher.example.org/articles/42/" />
|
||||||
|
<meta name="citation_pdf_url" content="/pdfs/42-main.pdf" />
|
||||||
|
<link rel="alternate" type="application/pdf" href="https://cdn.example.org/42-supp.pdf" />
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<a href="appendix.pdf">Appendix</a>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
candidates = pdf_discovery._normalized_candidate_urls(
|
||||||
|
page_url="https://publisher.example.org/articles/42",
|
||||||
|
html=html,
|
||||||
|
)
|
||||||
|
assert len(candidates) == 3
|
||||||
|
assert "https://cdn.example.org/42-supp.pdf" in candidates
|
||||||
|
assert "https://publisher.example.org/pdfs/42-main.pdf" in candidates
|
||||||
|
assert "https://publisher.example.org/articles/42/appendix.pdf" in candidates
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalized_candidate_urls_extracts_plain_text_urls() -> None:
|
||||||
|
html = """
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
Footnote URL: http://www.mext.go.jp/component/english/file.pdf
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
candidates = pdf_discovery._normalized_candidate_urls(
|
||||||
|
page_url="https://www.science.org/doi/10.1126/science.1239057",
|
||||||
|
html=html,
|
||||||
|
)
|
||||||
|
assert "http://www.mext.go.jp/component/english/file.pdf" in candidates
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeResponse:
|
||||||
|
def __init__(self, *, status_code: int, content_type: str, text: str) -> None:
|
||||||
|
self.status_code = status_code
|
||||||
|
self.headers = {"content-type": content_type}
|
||||||
|
self.text = text
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeClient:
|
||||||
|
def __init__(self, pages: dict[str, _FakeResponse]) -> None:
|
||||||
|
self._pages = pages
|
||||||
|
|
||||||
|
async def get(self, url: str, follow_redirects: bool = True): # noqa: ARG002
|
||||||
|
return self._pages[url]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_resolve_pdf_from_landing_page_follows_one_hop_html_candidate(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
async def _skip_wait(*args, **kwargs): # noqa: ARG001, ANN002
|
||||||
|
return None
|
||||||
|
|
||||||
|
monkeypatch.setattr(pdf_discovery, "wait_for_unpaywall_slot", _skip_wait)
|
||||||
|
landing_url = "https://example.org/article"
|
||||||
|
hop_url = "https://example.org/doi/full/abc"
|
||||||
|
pdf_url = "https://downloads.example.org/archive/paper.pdf"
|
||||||
|
client = _FakeClient(
|
||||||
|
{
|
||||||
|
landing_url: _FakeResponse(
|
||||||
|
status_code=200,
|
||||||
|
content_type="text/html",
|
||||||
|
text=f"<html><body><a href=\"{hop_url}\">View article</a></body></html>",
|
||||||
|
),
|
||||||
|
hop_url: _FakeResponse(
|
||||||
|
status_code=200,
|
||||||
|
content_type="text/html",
|
||||||
|
text=f"<html><body>Download here: {pdf_url}</body></html>",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
resolved = await pdf_discovery.resolve_pdf_from_landing_page(client, page_url=landing_url)
|
||||||
|
assert resolved == pdf_url
|
||||||
89
tests/unit/test_unpaywall_resolution.py
Normal file
|
|
@ -0,0 +1,89 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.services.domains.publications.types import PublicationListItem
|
||||||
|
from app.services.domains.unpaywall import application as unpaywall_app
|
||||||
|
|
||||||
|
|
||||||
|
class _DummyAsyncClient:
|
||||||
|
def __init__(self, *args, **kwargs) -> None:
|
||||||
|
self.args = args
|
||||||
|
self.kwargs = kwargs
|
||||||
|
|
||||||
|
async def __aenter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, exc_type, exc, tb):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _item(publication_id: int) -> PublicationListItem:
|
||||||
|
return PublicationListItem(
|
||||||
|
publication_id=publication_id,
|
||||||
|
scholar_profile_id=1,
|
||||||
|
scholar_label="Shinya Yamanaka",
|
||||||
|
title="Induction of pluripotent stem cells",
|
||||||
|
year=2007,
|
||||||
|
citation_count=1000,
|
||||||
|
venue_text="Cell",
|
||||||
|
pub_url="https://doi.org/10.1016/j.cell.2007.11.019",
|
||||||
|
doi=None,
|
||||||
|
pdf_url=None,
|
||||||
|
is_read=False,
|
||||||
|
first_seen_at=datetime.now(timezone.utc),
|
||||||
|
is_new_in_latest_run=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_unpaywall_resolve_prefers_direct_pdf_without_landing_crawl(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
payload = {
|
||||||
|
"doi": "10.1016/j.cell.2007.11.019",
|
||||||
|
"best_oa_location": {
|
||||||
|
"url_for_pdf": "https://oa.example.org/article.pdf",
|
||||||
|
"url": "https://oa.example.org/landing",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _fake_resolve_item_payload(**_kwargs):
|
||||||
|
return payload, False
|
||||||
|
|
||||||
|
async def _fail_crawl(_client, *, page_url: str):
|
||||||
|
raise AssertionError(f"unexpected landing crawl: {page_url}")
|
||||||
|
|
||||||
|
monkeypatch.setattr(unpaywall_app, "_resolve_item_payload", _fake_resolve_item_payload)
|
||||||
|
monkeypatch.setattr(unpaywall_app, "resolve_pdf_from_landing_page", _fail_crawl)
|
||||||
|
monkeypatch.setattr("httpx.AsyncClient", _DummyAsyncClient)
|
||||||
|
resolved = await unpaywall_app.resolve_publication_oa_metadata([_item(1)], request_email="user@example.com")
|
||||||
|
assert resolved == {1: ("10.1016/j.cell.2007.11.019", "https://oa.example.org/article.pdf")}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_unpaywall_resolve_crawls_landing_page_when_pdf_url_is_not_direct(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
payload = {
|
||||||
|
"doi": "10.1016/j.cell.2007.11.019",
|
||||||
|
"best_oa_location": {
|
||||||
|
"url_for_pdf": "https://oa.example.org/view?paper=42",
|
||||||
|
"url": "https://oa.example.org/landing/42",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
crawled_pages: list[str] = []
|
||||||
|
|
||||||
|
async def _fake_resolve_item_payload(**_kwargs):
|
||||||
|
return payload, False
|
||||||
|
|
||||||
|
async def _fake_crawl(_client, *, page_url: str):
|
||||||
|
crawled_pages.append(page_url)
|
||||||
|
return "https://oa.example.org/files/paper-42.pdf"
|
||||||
|
|
||||||
|
monkeypatch.setattr(unpaywall_app, "_resolve_item_payload", _fake_resolve_item_payload)
|
||||||
|
monkeypatch.setattr(unpaywall_app, "resolve_pdf_from_landing_page", _fake_crawl)
|
||||||
|
monkeypatch.setattr("httpx.AsyncClient", _DummyAsyncClient)
|
||||||
|
resolved = await unpaywall_app.resolve_publication_oa_metadata([_item(2)], request_email="user@example.com")
|
||||||
|
assert resolved == {2: ("10.1016/j.cell.2007.11.019", "https://oa.example.org/files/paper-42.pdf")}
|
||||||
|
assert "https://oa.example.org/landing/42" in crawled_pages
|
||||||
80
website/docusaurus.config.js
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
// @ts-check
|
||||||
|
|
||||||
|
/** @type {import('@docusaurus/types').Config} */
|
||||||
|
const config = {
|
||||||
|
title: "scholarr",
|
||||||
|
tagline: "Self-hosted scholar tracking docs",
|
||||||
|
favicon: "img/favicon.ico",
|
||||||
|
|
||||||
|
url: "https://justinzeus.github.io",
|
||||||
|
baseUrl: "/scholar-scraper/",
|
||||||
|
|
||||||
|
organizationName: "JustinZeus",
|
||||||
|
projectName: "scholar-scraper",
|
||||||
|
|
||||||
|
onBrokenLinks: "throw",
|
||||||
|
onBrokenMarkdownLinks: "warn",
|
||||||
|
|
||||||
|
i18n: {
|
||||||
|
defaultLocale: "en",
|
||||||
|
locales: ["en"],
|
||||||
|
},
|
||||||
|
|
||||||
|
presets: [
|
||||||
|
[
|
||||||
|
"classic",
|
||||||
|
{
|
||||||
|
docs: {
|
||||||
|
path: "../docs",
|
||||||
|
routeBasePath: "/",
|
||||||
|
sidebarPath: require.resolve("./sidebars.js"),
|
||||||
|
editUrl: "https://github.com/JustinZeus/scholar-scraper/tree/main/",
|
||||||
|
},
|
||||||
|
blog: false,
|
||||||
|
theme: {
|
||||||
|
customCss: require.resolve("./src/css/custom.css"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
|
||||||
|
themeConfig: {
|
||||||
|
navbar: {
|
||||||
|
title: "scholarr",
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
type: "docSidebar",
|
||||||
|
sidebarId: "docsSidebar",
|
||||||
|
position: "left",
|
||||||
|
label: "Docs",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
href: "https://github.com/JustinZeus/scholar-scraper",
|
||||||
|
label: "GitHub",
|
||||||
|
position: "right",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
footer: {
|
||||||
|
style: "dark",
|
||||||
|
links: [
|
||||||
|
{
|
||||||
|
title: "Docs",
|
||||||
|
items: [{ label: "Index", to: "/" }],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: "Community",
|
||||||
|
items: [
|
||||||
|
{
|
||||||
|
label: "GitHub",
|
||||||
|
href: "https://github.com/JustinZeus/scholar-scraper",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
copyright: `Copyright © ${new Date().getFullYear()} scholarr.`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
module.exports = config;
|
||||||
18
website/package.json
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
{
|
||||||
|
"name": "scholarr-docs",
|
||||||
|
"version": "0.0.1",
|
||||||
|
"private": true,
|
||||||
|
"scripts": {
|
||||||
|
"start": "docusaurus start",
|
||||||
|
"build": "docusaurus build",
|
||||||
|
"serve": "docusaurus serve"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@docusaurus/core": "3.9.2",
|
||||||
|
"@docusaurus/preset-classic": "3.9.2",
|
||||||
|
"clsx": "^2.1.1",
|
||||||
|
"prism-react-renderer": "^2.3.1",
|
||||||
|
"react": "^18.2.0",
|
||||||
|
"react-dom": "^18.2.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
6
website/sidebars.js
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
/** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */
|
||||||
|
const sidebars = {
|
||||||
|
docsSidebar: [{ type: "autogenerated", dirName: "." }],
|
||||||
|
};
|
||||||
|
|
||||||
|
module.exports = sidebars;
|
||||||
9
website/src/css/custom.css
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
:root {
|
||||||
|
--ifm-color-primary: #2a6b4e;
|
||||||
|
--ifm-color-primary-dark: #255f46;
|
||||||
|
--ifm-color-primary-darker: #225942;
|
||||||
|
--ifm-color-primary-darkest: #1c4936;
|
||||||
|
--ifm-color-primary-light: #2f7756;
|
||||||
|
--ifm-color-primary-lighter: #327d5a;
|
||||||
|
--ifm-color-primary-lightest: #3b946b;
|
||||||
|
}
|
||||||
BIN
website/static/img/favicon.ico
Normal file
|
After Width: | Height: | Size: 15 KiB |