diff --git a/.env.example b/.env.example index dc38b6b..326e7d3 100644 --- a/.env.example +++ b/.env.example @@ -1,26 +1,52 @@ +# ------------------------------ +# Compose + Database +# ------------------------------ POSTGRES_DB=scholar POSTGRES_USER=scholar POSTGRES_PASSWORD=change-me - DATABASE_URL=postgresql+asyncpg://scholar:scholar@db:5432/scholar # Optional override. If empty, tests derive "_test". TEST_DATABASE_URL= SCHOLARR_IMAGE=justinzeus/scholarr:latest +# ------------------------------ +# App Runtime + Networking +# ------------------------------ APP_NAME=scholarr APP_HOST=0.0.0.0 APP_PORT=8000 APP_HOST_PORT=8000 APP_RELOAD=0 +MIGRATE_ON_START=1 FRONTEND_ENABLED=1 FRONTEND_DIST_DIR=/app/frontend/dist + +# ------------------------------ +# Database Pool +# ------------------------------ +DATABASE_POOL_MODE=auto +DATABASE_POOL_SIZE=5 +DATABASE_POOL_MAX_OVERFLOW=10 +DATABASE_POOL_TIMEOUT_SECONDS=30 + +# ------------------------------ +# Frontend Dev Overrides +# ------------------------------ FRONTEND_HOST_PORT=5173 CHOKIDAR_USEPOLLING=1 VITE_DEV_API_PROXY_TARGET=http://app:8000 -MIGRATE_ON_START=1 +# ------------------------------ +# Auth + Session +# ------------------------------ SESSION_SECRET_KEY=replace-with-a-long-random-secret-at-least-32-characters SESSION_COOKIE_SECURE=1 +LOGIN_RATE_LIMIT_ATTEMPTS=5 +LOGIN_RATE_LIMIT_WINDOW_SECONDS=60 + +# ------------------------------ +# HTTP Security Headers + CSP +# ------------------------------ SECURITY_HEADERS_ENABLED=1 SECURITY_X_CONTENT_TYPE_OPTIONS=nosniff SECURITY_X_FRAME_OPTIONS=DENY @@ -36,16 +62,23 @@ SECURITY_STRICT_TRANSPORT_SECURITY_ENABLED=0 SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE=31536000 SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS=1 SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD=0 -LOGIN_RATE_LIMIT_ATTEMPTS=5 -LOGIN_RATE_LIMIT_WINDOW_SECONDS=60 + +# ------------------------------ +# Logging +# ------------------------------ LOG_LEVEL=INFO LOG_FORMAT=console LOG_REQUESTS=1 LOG_UVICORN_ACCESS=0 LOG_REQUEST_SKIP_PATHS=/healthz LOG_REDACT_FIELDS= + +# ------------------------------ +# Scheduler + Ingestion Safety +# ------------------------------ SCHEDULER_ENABLED=1 SCHEDULER_TICK_SECONDS=60 +SCHEDULER_QUEUE_BATCH_SIZE=10 INGESTION_AUTOMATION_ALLOWED=1 INGESTION_MANUAL_RUN_ALLOWED=1 INGESTION_MIN_RUN_INTERVAL_MINUTES=15 @@ -63,7 +96,10 @@ INGESTION_CONTINUATION_QUEUE_ENABLED=1 INGESTION_CONTINUATION_BASE_DELAY_SECONDS=120 INGESTION_CONTINUATION_MAX_DELAY_SECONDS=3600 INGESTION_CONTINUATION_MAX_ATTEMPTS=6 -SCHEDULER_QUEUE_BATCH_SIZE=10 + +# ------------------------------ +# Scholar Images + Name Search Safety +# ------------------------------ SCHOLAR_IMAGE_UPLOAD_DIR=/var/lib/scholarr/uploads SCHOLAR_IMAGE_UPLOAD_MAX_BYTES=2000000 SCHOLAR_NAME_SEARCH_ENABLED=1 @@ -76,21 +112,34 @@ SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD=1 SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS=1800 SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD=2 SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD=3 + +# ------------------------------ +# OA Enrichment + PDF Resolution +# ------------------------------ UNPAYWALL_ENABLED=1 UNPAYWALL_EMAIL= UNPAYWALL_TIMEOUT_SECONDS=4.0 +UNPAYWALL_MIN_INTERVAL_SECONDS=0.6 UNPAYWALL_MAX_ITEMS_PER_REQUEST=20 UNPAYWALL_RETRY_COOLDOWN_SECONDS=1800 +UNPAYWALL_PDF_DISCOVERY_ENABLED=1 +UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES=5 +UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES=500000 +PDF_AUTO_RETRY_INTERVAL_SECONDS=86400 +PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS=3600 +PDF_AUTO_RETRY_MAX_ATTEMPTS=3 CROSSREF_ENABLED=1 CROSSREF_MAX_ROWS=10 CROSSREF_TIMEOUT_SECONDS=8.0 CROSSREF_MIN_INTERVAL_SECONDS=0.6 CROSSREF_MAX_LOOKUPS_PER_REQUEST=8 +# ------------------------------ +# Startup Bootstrap + DB Wait +# ------------------------------ BOOTSTRAP_ADMIN_ON_START=0 BOOTSTRAP_ADMIN_EMAIL= BOOTSTRAP_ADMIN_PASSWORD= BOOTSTRAP_ADMIN_FORCE_PASSWORD=0 - DB_WAIT_TIMEOUT_SECONDS=60 DB_WAIT_INTERVAL_SECONDS=2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ff6965..1459666 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,9 @@ jobs: - name: Enforce no generated artifacts run: ./scripts/check_no_generated_artifacts.sh + - name: Enforce env contract parity + run: python3 scripts/check_env_contract.py + test: runs-on: ubuntu-latest needs: @@ -106,12 +109,33 @@ jobs: working-directory: frontend run: npm run build + docs-quality: + runs-on: ubuntu-latest + needs: + - repo-hygiene + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install docs dependencies + run: npm install --prefix website + + - name: Docs build + run: npm --prefix website run build + docker-publish: runs-on: ubuntu-latest needs: - repo-hygiene - test - frontend-quality + - docs-quality if: github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: contents: read diff --git a/.github/workflows/docs-pages.yml b/.github/workflows/docs-pages.yml new file mode 100644 index 0000000..8bdb167 --- /dev/null +++ b/.github/workflows/docs-pages.yml @@ -0,0 +1,58 @@ +name: Docs Pages + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - docs/** + - website/** + - .github/workflows/docs-pages.yml + +permissions: + contents: read + +concurrency: + group: docs-pages + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Pages + uses: actions/configure-pages@v5 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install docs dependencies + run: npm install --prefix website + + - name: Build docs site + run: npm --prefix website run build + + - name: Upload pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: website/build + + deploy: + needs: build + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index f539821..218252b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,6 @@ planning/ frontend/node_modules/ frontend/dist/ frontend/.vite/ +website/node_modules/ +website/build/ AGENTS.MD diff --git a/README.md b/README.md index 3ec9573..b31ab8f 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,7 @@ cp .env.example .env - `POSTGRES_PASSWORD` - `SESSION_SECRET_KEY` -3. Choose a deploy method below. - -## Deploy Method A: Prebuilt Image (Recommended) - -Use this for normal self-hosted deployment. +3. Start stack: ```bash docker compose pull @@ -37,234 +33,36 @@ Open: - App/API: `http://localhost:8000` - Health: `http://localhost:8000/healthz` -Upgrade: +## Documentation + +Primary docs live under `docs/`. + +- Index: `docs/README.md` +- Deploy and dev workflow: `docs/deploy/quickstart.md` +- Environment reference: `docs/reference/environment.md` +- API contract and payload conventions: `docs/reference/api_contract.md` +- Architecture and domain boundaries: `docs/architecture/domain_boundaries.md` +- Scrape safety runbook: `docs/ops/scrape_safety_runbook.md` +- DB backup/restore/integrity runbook: `docs/ops/db_runbook.md` +- Migration rollout checklist: `docs/ops/migration_checklist.md` +- Frontend theme inventory: `docs/frontend/theme_phase0_inventory.md` +- Contributing policy: `docs/contributing.md` + +## Docs Site (Docusaurus) + +Docusaurus source lives in `website/` and consumes markdown from `docs/`. + +Local build: ```bash -docker compose pull -docker compose up -d +cd website +npm install +npm run build ``` -## Deploy Method B: Local Source Build + Dev Frontend +GitHub Pages deploy is automated via `.github/workflows/docs-pages.yml`. -Use this for development on this repository. - -```bash -docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build -``` - -Open: -- API (FastAPI): `http://localhost:8000` -- Frontend dev server (Vite): `http://localhost:5173` - -Stop: - -```bash -docker compose -f docker-compose.yml -f docker-compose.dev.yml down -``` - -## Essential Files - -- `docker-compose.yml`: deployment compose (prebuilt image). -- `docker-compose.dev.yml`: dev override (source mounts + Vite dev server). -- `.env.example`: full env variable template. -- `Dockerfile`: multi-stage build (frontend + backend runtime). -- `README.md`: deployment and operations reference. -- `CONTRIBUTING.md`: contribution policy and merge checklist. -- `docs/ops/scrape_safety_runbook.md`: scrape cooldown and blocked-IP operations guide. -- `scripts/check_no_generated_artifacts.sh`: tracked-artifact guard used by CI. - -## Data Model Notes - -- Scholar tracking is user-scoped: each account can track the same Scholar ID independently. -- Publications are shared/global records deduplicated by Scholar cluster ID and normalized fingerprint. -- Per-account visibility and read state is stored on scholar-publication links, not on the global publication row. -- Publication records include canonical Scholar detail URLs (`pub_url`), normalized DOI (`doi`), and resolved OA PDF links (`pdf_url`) when available. - -## Backend Architecture (Refactored) - -- Canonical service logic lives in `app/services/domains/*`; root-level `app/services/*.py` business-logic modules are removed. -- `app/services/domains/ingestion/*`: run orchestration (`application.py`), continuation queue (`queue.py`), scrape safety cooldown policy (`safety.py`), scheduler/queue draining (`scheduler.py`), plus shared constants/types/fingerprint helpers. -- `app/services/domains/scholar/*`: fail-fast Google Scholar parsing and source access. Parser flow is modularized across row extractors, state detection, constants, and parser types. -- `app/services/domains/scholars/*`: scholar CRUD, name-search safety/caching, profile-image validation/upload handling, and shared scholar validation helpers. -- `app/services/domains/publications/*`: publication listing/read-state query modules, lazy/background OA enrichment scheduling (non-blocking for list responses), and per-publication PDF retry support. -- `app/services/domains/doi/*`: DOI normalization helpers used by ingestion and enrichment. -- `app/services/domains/crossref/*`: DOI discovery fallback using Crossref metadata queries with bounded/paced request behavior. -- `app/services/domains/unpaywall/*`: OA resolution by DOI (best OA location + PDF URL extraction), using per-user email for API calls. -- `app/services/domains/runs/*`: run history summaries plus continuation queue status/retry/drop/clear operations. -- `app/services/domains/portability/*`: import/export of tracked scholars and publication-link state while preserving global publication deduplication. -- `app/services/domains/settings/*` and `app/services/domains/users/*`: user settings and user management services. - -## Frontend Navigation and Layout Notes - -- On mobile (`< lg`), primary navigation is behind a header hamburger and opens as a left-side drawer with backdrop. -- Mobile nav closes on route change, nav link click, and logout. -- Long lists are constrained to internal scroll containers in fill-layout pages (including dashboard recent publications and tracked scholars table view). - -## Name Search Status - -- Scholar name search is intentionally WIP in the UI. -- Current Google Scholar behavior often redirects name-search traffic to login/challenge flows, so production onboarding should use direct Scholar ID/profile URL adds. - -## Environment Variables (Complete Reference) - -Notes: -- Boolean envs accept: `1/0`, `true/false`, `yes/no`, `on/off`. -- Values shown are deployment defaults from `.env.example` and `docker-compose.yml`. -- Some internal app fallbacks may differ for local test/dev safety when env vars are omitted. -- `deploy` means used in regular deployment. -- `dev` means used in local dev workflow. - -### Core Compose and Database - -| Variable | Default | Options | Scope | Description | -| --- | --- | --- | --- | --- | -| `POSTGRES_DB` | `scholar` | any valid DB name | deploy, dev | PostgreSQL database name. | -| `POSTGRES_USER` | `scholar` | any valid DB user | deploy, dev | PostgreSQL user. | -| `POSTGRES_PASSWORD` | `change-me` | strong password | deploy, dev | PostgreSQL password. Required. | -| `DATABASE_URL` | `postgresql+asyncpg://scholar:scholar@db:5432/scholar` | valid SQLAlchemy asyncpg URL | deploy, dev, test | App database connection URL. | -| `TEST_DATABASE_URL` | empty | valid SQLAlchemy asyncpg URL | test | Optional explicit integration test DB URL. | -| `SCHOLARR_IMAGE` | `justinzeus/scholarr:latest` | any image ref | deploy | App image tag used by deployment compose. | - -### App Runtime and Networking - -| Variable | Default | Options | Scope | Description | -| --- | --- | --- | --- | --- | -| `APP_NAME` | `scholarr` | any string | deploy, dev | FastAPI app name/title. | -| `APP_HOST` | `0.0.0.0` | bind host | deploy, dev | Uvicorn bind host inside container. | -| `APP_PORT` | `8000` | integer | deploy, dev | Uvicorn bind port inside container. | -| `APP_HOST_PORT` | `8000` | integer | deploy, dev | Host port mapped to app container port 8000. | -| `APP_RELOAD` | `0` | boolean | deploy, dev | Enable uvicorn reload mode. Recommended `0` in deploy, `1` in dev. | -| `MIGRATE_ON_START` | `1` | boolean | deploy, dev | Run Alembic migrations during app startup. | -| `FRONTEND_ENABLED` | `1` | boolean | deploy, dev | Serve bundled frontend assets from FastAPI. | -| `FRONTEND_DIST_DIR` | `/app/frontend/dist` | absolute path | deploy, dev | Path to built frontend assets inside app container. | - -### Dev Frontend Overrides - -| Variable | Default | Options | Scope | Description | -| --- | --- | --- | --- | --- | -| `FRONTEND_HOST_PORT` | `5173` | integer | dev | Host port mapped to Vite dev server. | -| `CHOKIDAR_USEPOLLING` | `1` | boolean | dev | File watching mode for containerized Vite. | -| `VITE_DEV_API_PROXY_TARGET` | `http://app:8000` | URL | dev | Vite proxy target for `/api` and `/healthz`. | - -### Auth and Session Security - -| Variable | Default | Options | Scope | Description | -| --- | --- | --- | --- | --- | -| `SESSION_SECRET_KEY` | `replace-with-a-long-random-secret-at-least-32-characters` | long random string | deploy, dev | Session signing secret. Required in deploy. | -| `SESSION_COOKIE_SECURE` | `1` | boolean | deploy, dev | Mark session cookie as HTTPS-only. Set `1` behind HTTPS. | -| `LOGIN_RATE_LIMIT_ATTEMPTS` | `5` | integer >= 1 | deploy, dev | Login attempts allowed per window. | -| `LOGIN_RATE_LIMIT_WINDOW_SECONDS` | `60` | integer >= 1 | deploy, dev | Login rate limit window in seconds. | - -### HTTP Security Headers and CSP - -| Variable | Default | Options | Scope | Description | -| --- | --- | --- | --- | --- | -| `SECURITY_HEADERS_ENABLED` | `1` | boolean | deploy, dev | Global switch for response security headers middleware. | -| `SECURITY_X_CONTENT_TYPE_OPTIONS` | `nosniff` | header value | deploy, dev | `X-Content-Type-Options` response header value. | -| `SECURITY_X_FRAME_OPTIONS` | `DENY` | header value | deploy, dev | `X-Frame-Options` response header value. | -| `SECURITY_REFERRER_POLICY` | `strict-origin-when-cross-origin` | header value | deploy, dev | `Referrer-Policy` response header value. | -| `SECURITY_PERMISSIONS_POLICY` | `accelerometer=(), autoplay=(), camera=(), display-capture=(), geolocation=(), gyroscope=(), microphone=(), payment=(), usb=()` | permissions policy string | deploy, dev | `Permissions-Policy` response header value. | -| `SECURITY_CROSS_ORIGIN_OPENER_POLICY` | `same-origin` | header value | deploy, dev | `Cross-Origin-Opener-Policy` response header value. | -| `SECURITY_CROSS_ORIGIN_RESOURCE_POLICY` | `same-origin` | header value | deploy, dev | `Cross-Origin-Resource-Policy` response header value. | -| `SECURITY_CSP_ENABLED` | `1` | boolean | deploy, dev | Enable Content Security Policy headers. | -| `SECURITY_CSP_POLICY` | strict SPA/API default | CSP policy string | deploy, dev | CSP applied to app/API routes (excluding docs paths). | -| `SECURITY_CSP_DOCS_POLICY` | relaxed docs default | CSP policy string | deploy, dev | CSP override for `/docs` and `/redoc` to keep Swagger/ReDoc usable. | -| `SECURITY_CSP_REPORT_ONLY` | `0` | boolean | deploy, dev | Emit CSP in report-only mode instead of enforcement mode. | -| `SECURITY_STRICT_TRANSPORT_SECURITY_ENABLED` | `0` | boolean | deploy, dev | Enable `Strict-Transport-Security` header. | -| `SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE` | `31536000` | integer >= 0 | deploy, dev | `max-age` for HSTS header. | -| `SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS` | `1` | boolean | deploy, dev | Add `includeSubDomains` directive to HSTS header. | -| `SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD` | `0` | boolean | deploy, dev | Add `preload` directive to HSTS header. | - -### Logging - -| Variable | Default | Options | Scope | Description | -| --- | --- | --- | --- | --- | -| `LOG_LEVEL` | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` | deploy, dev | Application log level. | -| `LOG_FORMAT` | `console` | `console`, `json` | deploy, dev | Log output format. | -| `LOG_REQUESTS` | `1` | boolean | deploy, dev | Enable request start/completion logs. | -| `LOG_UVICORN_ACCESS` | `0` | boolean | deploy, dev | Enable uvicorn access logs. | -| `LOG_REQUEST_SKIP_PATHS` | `/healthz` | comma-separated path prefixes | deploy, dev | Request log skip list. | -| `LOG_REDACT_FIELDS` | empty | comma-separated keys | deploy, dev | Extra fields to redact in structured logs. | - -### Scheduler and Ingestion - -| Variable | Default | Options | Scope | Description | -| --- | --- | --- | --- | --- | -| `SCHEDULER_ENABLED` | `1` | boolean | deploy, dev | Enable background scheduler loop. | -| `SCHEDULER_TICK_SECONDS` | `60` | integer >= 1 | deploy, dev | Scheduler interval in seconds. | -| `INGESTION_AUTOMATION_ALLOWED` | `1` | boolean | deploy, dev | Global safety switch for scheduled/automatic checks. When disabled, auto-run settings are forced off. | -| `INGESTION_MANUAL_RUN_ALLOWED` | `1` | boolean | deploy, dev | Global safety switch for manual checks from API/UI. | -| `INGESTION_MIN_RUN_INTERVAL_MINUTES` | `15` | integer >= 15 | deploy, dev | Server-enforced minimum for user-configured automatic check interval. | -| `INGESTION_MIN_REQUEST_DELAY_SECONDS` | `2` | integer >= 2 | deploy, dev | Server-enforced minimum delay between scholar requests. | -| `SCHEDULER_QUEUE_BATCH_SIZE` | `10` | integer >= 1 | deploy, dev | Queue items processed per scheduler tick. | -| `INGESTION_NETWORK_ERROR_RETRIES` | `1` | integer >= 0 | deploy, dev | Retries for transient ingestion network errors. | -| `INGESTION_RETRY_BACKOFF_SECONDS` | `1.0` | float >= 0 | deploy, dev | Backoff delay for retry attempts. | -| `INGESTION_MAX_PAGES_PER_SCHOLAR` | `30` | integer >= 1 | deploy, dev | Upper bound of pages fetched per scholar run. | -| `INGESTION_PAGE_SIZE` | `100` | integer >= 1 | deploy, dev | Requested Scholar page size. | -| `INGESTION_ALERT_BLOCKED_FAILURE_THRESHOLD` | `1` | integer >= 1 | deploy, dev | Trigger blocked/captcha scrape alert flag when this many blocked failures occur in a run. | -| `INGESTION_ALERT_NETWORK_FAILURE_THRESHOLD` | `2` | integer >= 1 | deploy, dev | Trigger network scrape alert flag when this many network failures occur in a run. | -| `INGESTION_ALERT_RETRY_SCHEDULED_THRESHOLD` | `3` | integer >= 1 | deploy, dev | Trigger retry alert flag when scheduled retry count reaches this threshold in a run. | -| `INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS` | `1800` | integer >= 60 | deploy, dev | Cooldown duration applied when blocked-failure threshold is exceeded. | -| `INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS` | `900` | integer >= 60 | deploy, dev | Cooldown duration applied when network-failure threshold is exceeded. | -| `INGESTION_CONTINUATION_QUEUE_ENABLED` | `1` | boolean | deploy, dev | Enable continuation queue for long runs. | -| `INGESTION_CONTINUATION_BASE_DELAY_SECONDS` | `120` | integer >= 0 | deploy, dev | Initial delay before retrying continuation queue items. | -| `INGESTION_CONTINUATION_MAX_DELAY_SECONDS` | `3600` | integer >= 0 | deploy, dev | Maximum continuation retry delay. | -| `INGESTION_CONTINUATION_MAX_ATTEMPTS` | `6` | integer >= 1 | deploy, dev | Max failed continuation attempts before dropping item. | - -### Scholar Images and Name Search Safety - -| Variable | Default | Options | Scope | Description | -| --- | --- | --- | --- | --- | -| `SCHOLAR_IMAGE_UPLOAD_DIR` | `/var/lib/scholarr/uploads` | writable absolute path | deploy, dev | Storage path for uploaded scholar images in compose deployments. App fallback without env is `/tmp/scholarr_uploads/scholar_images`. | -| `SCHOLAR_IMAGE_UPLOAD_MAX_BYTES` | `2000000` | integer >= 1 | deploy, dev | Max uploaded image size in bytes. | -| `SCHOLAR_NAME_SEARCH_ENABLED` | `1` | boolean | deploy, dev | Enable name-search helper endpoint. | -| `SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS` | `21600` | integer >= 1 | deploy, dev | Cache TTL for successful name-search responses. | -| `SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS` | `300` | integer >= 1 | deploy, dev | Cache TTL for blocked/captcha responses. | -| `SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES` | `512` | integer >= 1 | deploy, dev | Max cached search entries. | -| `SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS` | `8.0` | float >= 0 | deploy, dev | Minimum interval between live name-search requests. | -| `SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS` | `2.0` | float >= 0 | deploy, dev | Added jitter to reduce request burst patterns. | -| `SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD` | `1` | integer >= 1 | deploy, dev | Consecutive blocked responses before cooldown starts. | -| `SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS` | `1800` | integer >= 1 | deploy, dev | Cooldown duration after repeated blocked responses. | -| `SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD` | `2` | integer >= 1 | deploy, dev | Emit retry-threshold observability warning when name-search retry count reaches this value. | -| `SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD` | `3` | integer >= 1 | deploy, dev | Emit cooldown-threshold observability alert after this many requests are rejected during active cooldown. | - -### Open-Access Enrichment (Crossref + Unpaywall) - -| Variable | Default | Options | Scope | Description | -| --- | --- | --- | --- | --- | -| `UNPAYWALL_ENABLED` | `1` | boolean | deploy, dev | Enable Unpaywall OA resolution. | -| `UNPAYWALL_EMAIL` | empty | valid email | deploy, dev | Fallback email if per-user email is unavailable. | -| `UNPAYWALL_TIMEOUT_SECONDS` | `4.0` | float >= 0.5 | deploy, dev | Timeout for Unpaywall requests. | -| `UNPAYWALL_MAX_ITEMS_PER_REQUEST` | `20` | integer >= 0 | deploy, dev | Max publications queued per lazy enrichment pass. | -| `UNPAYWALL_RETRY_COOLDOWN_SECONDS` | `1800` | integer >= 1 | deploy, dev | Per-user/per-publication cooldown before re-scheduling unresolved OA lookups. | -| `CROSSREF_ENABLED` | `1` | boolean | deploy, dev | Enable Crossref DOI discovery fallback when DOI is missing/insufficient. | -| `CROSSREF_MAX_ROWS` | `10` | integer >= 1 | deploy, dev | Max Crossref rows evaluated per lookup. | -| `CROSSREF_TIMEOUT_SECONDS` | `8.0` | float >= 0.5 | deploy, dev | Timeout budget per Crossref lookup task. | -| `CROSSREF_MIN_INTERVAL_SECONDS` | `0.6` | float >= 0 | deploy, dev | Minimum pacing interval between Crossref requests. | -| `CROSSREF_MAX_LOOKUPS_PER_REQUEST` | `8` | integer >= 0 | deploy, dev | Max Crossref DOI lookups performed during one lazy pass or manual retry. | - -### Scrape Safety Operations - -- Structured safety events are emitted for: - - policy/manual run blocks, - - cooldown entered/cleared transitions, - - blocked/network/retry threshold trips, - - scheduler cooldown skips/deferments. -- Use `LOG_FORMAT=json` and ship logs to your preferred collector for alerting. -- Runbook: `docs/ops/scrape_safety_runbook.md`. - -### Startup Bootstrap and DB Wait - -| Variable | Default | Options | Scope | Description | -| --- | --- | --- | --- | --- | -| `BOOTSTRAP_ADMIN_ON_START` | `0` | boolean | deploy, dev | Auto-create admin at startup. | -| `BOOTSTRAP_ADMIN_EMAIL` | empty | valid email | deploy, dev | Bootstrap admin email. | -| `BOOTSTRAP_ADMIN_PASSWORD` | empty | strong password | deploy, dev | Bootstrap admin password. | -| `BOOTSTRAP_ADMIN_FORCE_PASSWORD` | `0` | boolean | deploy, dev | Force-reset bootstrap admin password if user already exists. | -| `DB_WAIT_TIMEOUT_SECONDS` | `60` | integer >= 1 | deploy, dev | Max seconds to wait for DB readiness at startup. | -| `DB_WAIT_INTERVAL_SECONDS` | `2` | integer >= 1 | deploy, dev | Interval between DB readiness checks. | - -## Quality and Test Commands +## Quality Gates Backend: @@ -283,43 +81,10 @@ npm run test:run npm run build ``` -Contract drift: +Contract and env checks: ```bash python3 scripts/check_frontend_api_contract.py -``` - -Repository hygiene: - -```bash +python3 scripts/check_env_contract.py ./scripts/check_no_generated_artifacts.sh ``` - -Scheduled fixture probes run in GitHub Actions via `.github/workflows/scheduled-probes.yml`. - -## API Contract - -- Base path: `/api/v1` -- Success envelope: `{"data": ..., "meta": {"request_id": "..."}}` -- Error envelope: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}` - -### Publications semantics - -- `GET /api/v1/publications` supports `mode=all|unread|latest` (plus temporary alias `mode=new`). -- `unread` = actionable read-state (`is_read=false`). -- `latest` = discovery-state (`first seen in the latest completed check`). -- Publications include `pub_url`, `doi`, and `pdf_url` in API responses. -- OA enrichment flow is DOI-first: - - existing DOI (or explicit DOI in source metadata) -> Unpaywall lookup, - - Crossref DOI discovery fallback (paced + bounded), - - Unpaywall DOI lookup for OA status and `pdf_url`. -- `POST /api/v1/publications/{publication_id}/retry-pdf` retries enrichment for a single publication in current user scope. -- Response counters: - - `unread_count`: unread publications in current scope. - - `latest_count`: publications discovered in latest completed check. - - `new_count`: compatibility alias for `latest_count` (temporary). - -### Scholar Data Portability - -- `GET /api/v1/scholars/export`: exports tracked scholars plus scholar-publication link data as JSON. -- `POST /api/v1/scholars/import`: imports that JSON payload, upserting scholars, global publication records, and per-scholar read state. diff --git a/agents.md b/agents.md index 18abc7c..07b62d2 100644 --- a/agents.md +++ b/agents.md @@ -36,3 +36,8 @@ These limits prevent IP bans and are not to be optimized away. * **`app/services/domains/unpaywall/*`:** OA resolver by DOI only (best OA location + PDF URL extraction). Do not use Google Scholar for PDF resolution to avoid N+1 scrape amplification. * **`app/services/domains/portability/*`:** Handles JSON import/export for user-scoped scholars and scholar-publication link state while preserving global publication dedup rules. * **`app/services/domains/ingestion/scheduler.py`:** Owns automatic runs and continuation queue retries/drops; do not bypass safety gate or cooldown logic. + + +## 6. UI rules +Make sure to properly integrate tailwind in combination with the preset theming +Clarity through both styling and language are a priority. all UI elements need to have a proper reason for existing. \ No newline at end of file diff --git a/alembic/versions/20260220_0012_add_data_repair_jobs.py b/alembic/versions/20260220_0012_add_data_repair_jobs.py new file mode 100644 index 0000000..242876d --- /dev/null +++ b/alembic/versions/20260220_0012_add_data_repair_jobs.py @@ -0,0 +1,106 @@ +"""Add data repair job audit table. + +Revision ID: 20260220_0012 +Revises: 20260220_0011 +Create Date: 2026-02-20 23:35:00.000000 +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +revision: str = "20260220_0012" +down_revision: str | Sequence[str] | None = "20260220_0011" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _table_names() -> set[str]: + bind = op.get_bind() + inspector = sa.inspect(bind) + return set(inspector.get_table_names()) + + +def _create_data_repair_jobs_table() -> None: + op.create_table( + "data_repair_jobs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("job_name", sa.String(length=64), nullable=False), + sa.Column("requested_by", sa.String(length=255), nullable=True), + sa.Column( + "scope", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + server_default=sa.text("'{}'::jsonb"), + ), + sa.Column("dry_run", sa.Boolean(), nullable=False, server_default=sa.text("true")), + sa.Column( + "status", + sa.String(length=16), + nullable=False, + server_default=sa.text("'planned'"), + ), + sa.Column( + "summary", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + server_default=sa.text("'{}'::jsonb"), + ), + sa.Column("error_text", sa.Text(), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.CheckConstraint( + "status IN ('planned', 'running', 'completed', 'failed')", + name="data_repair_jobs_status_valid", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_data_repair_jobs")), + ) + + +def _create_data_repair_jobs_indexes() -> None: + op.create_index( + "ix_data_repair_jobs_created_at", + "data_repair_jobs", + ["created_at"], + unique=False, + ) + op.create_index( + "ix_data_repair_jobs_job_name_created_at", + "data_repair_jobs", + ["job_name", "created_at"], + unique=False, + ) + + +def _drop_data_repair_jobs_indexes() -> None: + op.drop_index("ix_data_repair_jobs_job_name_created_at", table_name="data_repair_jobs") + op.drop_index("ix_data_repair_jobs_created_at", table_name="data_repair_jobs") + + +def upgrade() -> None: + if "data_repair_jobs" in _table_names(): + return + _create_data_repair_jobs_table() + _create_data_repair_jobs_indexes() + + +def downgrade() -> None: + if "data_repair_jobs" not in _table_names(): + return + _drop_data_repair_jobs_indexes() + op.drop_table("data_repair_jobs") diff --git a/alembic/versions/20260221_0013_add_publication_pdf_jobs.py b/alembic/versions/20260221_0013_add_publication_pdf_jobs.py new file mode 100644 index 0000000..63db6b2 --- /dev/null +++ b/alembic/versions/20260221_0013_add_publication_pdf_jobs.py @@ -0,0 +1,179 @@ +"""Add persistent publication PDF job tracking tables. + +Revision ID: 20260221_0013 +Revises: 20260220_0012 +Create Date: 2026-02-21 12:25:00.000000 +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +revision: str = "20260221_0013" +down_revision: str | Sequence[str] | None = "20260220_0012" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _table_names() -> set[str]: + bind = op.get_bind() + inspector = sa.inspect(bind) + return set(inspector.get_table_names()) + + +def _create_publication_pdf_jobs_table() -> None: + op.create_table( + "publication_pdf_jobs", + sa.Column("publication_id", sa.Integer(), nullable=False), + sa.Column( + "status", + sa.String(length=16), + nullable=False, + server_default=sa.text("'queued'"), + ), + sa.Column("attempt_count", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column("queued_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_failure_reason", sa.String(length=64), nullable=True), + sa.Column("last_failure_detail", sa.Text(), nullable=True), + sa.Column("last_source", sa.String(length=32), nullable=True), + sa.Column("last_requested_by_user_id", sa.Integer(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.CheckConstraint( + "status IN ('queued', 'running', 'resolved', 'failed')", + name="publication_pdf_jobs_status_valid", + ), + sa.ForeignKeyConstraint( + ["last_requested_by_user_id"], + ["users.id"], + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["publication_id"], + ["publications.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("publication_id", name=op.f("pk_publication_pdf_jobs")), + ) + + +def _create_publication_pdf_jobs_indexes() -> None: + op.create_index( + "ix_publication_pdf_jobs_status", + "publication_pdf_jobs", + ["status"], + unique=False, + ) + op.create_index( + "ix_publication_pdf_jobs_updated_at", + "publication_pdf_jobs", + ["updated_at"], + unique=False, + ) + op.create_index( + "ix_publication_pdf_jobs_queued_at", + "publication_pdf_jobs", + ["queued_at"], + unique=False, + ) + + +def _create_publication_pdf_job_events_table() -> None: + op.create_table( + "publication_pdf_job_events", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("publication_id", sa.Integer(), nullable=False), + sa.Column("user_id", sa.Integer(), nullable=True), + sa.Column("event_type", sa.String(length=32), nullable=False), + sa.Column("status", sa.String(length=16), nullable=True), + sa.Column("source", sa.String(length=32), nullable=True), + sa.Column("failure_reason", sa.String(length=64), nullable=True), + sa.Column("message", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["publication_id"], + ["publications.id"], + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_publication_pdf_job_events")), + ) + + +def _create_publication_pdf_job_events_indexes() -> None: + op.create_index( + "ix_publication_pdf_job_events_publication_created", + "publication_pdf_job_events", + ["publication_id", "created_at"], + unique=False, + ) + op.create_index( + "ix_publication_pdf_job_events_created_at", + "publication_pdf_job_events", + ["created_at"], + unique=False, + ) + op.create_index( + "ix_publication_pdf_job_events_event_type", + "publication_pdf_job_events", + ["event_type"], + unique=False, + ) + + +def _drop_publication_pdf_jobs_indexes() -> None: + op.drop_index("ix_publication_pdf_jobs_queued_at", table_name="publication_pdf_jobs") + op.drop_index("ix_publication_pdf_jobs_updated_at", table_name="publication_pdf_jobs") + op.drop_index("ix_publication_pdf_jobs_status", table_name="publication_pdf_jobs") + + +def _drop_publication_pdf_job_events_indexes() -> None: + op.drop_index("ix_publication_pdf_job_events_event_type", table_name="publication_pdf_job_events") + op.drop_index("ix_publication_pdf_job_events_created_at", table_name="publication_pdf_job_events") + op.drop_index( + "ix_publication_pdf_job_events_publication_created", + table_name="publication_pdf_job_events", + ) + + +def upgrade() -> None: + tables = _table_names() + if "publication_pdf_jobs" not in tables: + _create_publication_pdf_jobs_table() + _create_publication_pdf_jobs_indexes() + if "publication_pdf_job_events" not in tables: + _create_publication_pdf_job_events_table() + _create_publication_pdf_job_events_indexes() + + +def downgrade() -> None: + tables = _table_names() + if "publication_pdf_job_events" in tables: + _drop_publication_pdf_job_events_indexes() + op.drop_table("publication_pdf_job_events") + if "publication_pdf_jobs" in tables: + _drop_publication_pdf_jobs_indexes() + op.drop_table("publication_pdf_jobs") diff --git a/alembic/versions/20260221_0014_add_scholar_publication_favorite_state.py b/alembic/versions/20260221_0014_add_scholar_publication_favorite_state.py new file mode 100644 index 0000000..1fa8f29 --- /dev/null +++ b/alembic/versions/20260221_0014_add_scholar_publication_favorite_state.py @@ -0,0 +1,44 @@ +"""add scholar publication favorite state + +Revision ID: 20260221_0014 +Revises: 20260221_0013 +Create Date: 2026-02-21 14:35:00 +""" + +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "20260221_0014" +down_revision = "20260221_0013" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "scholar_publications", + sa.Column( + "is_favorite", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + ) + op.create_index( + "ix_scholar_publications_is_favorite", + "scholar_publications", + ["is_favorite"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index( + "ix_scholar_publications_is_favorite", + table_name="scholar_publications", + ) + op.drop_column("scholar_publications", "is_favorite") diff --git a/alembic/versions/20260221_0015_enforce_request_delay_floor.py b/alembic/versions/20260221_0015_enforce_request_delay_floor.py new file mode 100644 index 0000000..688ce0a --- /dev/null +++ b/alembic/versions/20260221_0015_enforce_request_delay_floor.py @@ -0,0 +1,80 @@ +"""enforce request delay floor at two seconds + +Revision ID: 20260221_0015 +Revises: 20260221_0014 +Create Date: 2026-02-21 18:10:00 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "20260221_0015" +down_revision: str | Sequence[str] | None = "20260221_0014" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +TABLE_NAME = "user_settings" +CHECK_NAME_MIN_1 = "request_delay_seconds_min_1" +CHECK_NAME_MIN_2 = "request_delay_seconds_min_2" +LEGACY_CHECK_NAME_MIN_1 = "ck_user_settings_request_delay_seconds_min_1" +LEGACY_CHECK_NAME_MIN_2 = "ck_user_settings_request_delay_seconds_min_2" + + +def _check_constraints() -> set[str]: + bind = op.get_bind() + inspector = sa.inspect(bind) + return { + str(item.get("name")) + for item in inspector.get_check_constraints(TABLE_NAME) + if item.get("name") + } + + +def _drop_check_if_exists(name: str) -> None: + if name not in _check_constraints(): + return + op.drop_constraint( + name, + TABLE_NAME, + type_="check", + ) + + +def upgrade() -> None: + op.execute( + sa.text( + "UPDATE user_settings SET request_delay_seconds = 2 " + "WHERE request_delay_seconds < 2" + ) + ) + + _drop_check_if_exists(CHECK_NAME_MIN_1) + _drop_check_if_exists(LEGACY_CHECK_NAME_MIN_1) + + existing = _check_constraints() + if CHECK_NAME_MIN_2 not in existing and LEGACY_CHECK_NAME_MIN_2 not in existing: + op.create_check_constraint( + CHECK_NAME_MIN_2, + TABLE_NAME, + "request_delay_seconds >= 2", + ) + + +def downgrade() -> None: + _drop_check_if_exists(CHECK_NAME_MIN_2) + _drop_check_if_exists(LEGACY_CHECK_NAME_MIN_2) + + existing = _check_constraints() + if CHECK_NAME_MIN_1 not in existing and LEGACY_CHECK_NAME_MIN_1 not in existing: + op.create_check_constraint( + CHECK_NAME_MIN_1, + TABLE_NAME, + "request_delay_seconds >= 1", + ) diff --git a/app/api/router.py b/app/api/router.py index 09d49b2..a11d98a 100644 --- a/app/api/router.py +++ b/app/api/router.py @@ -2,11 +2,12 @@ from __future__ import annotations from fastapi import APIRouter -from app.api.routers import admin, auth, publications, runs, scholars, settings +from app.api.routers import admin, admin_dbops, auth, publications, runs, scholars, settings router = APIRouter(prefix="/api/v1") router.include_router(auth.router) router.include_router(admin.router) +router.include_router(admin_dbops.router) router.include_router(scholars.router) router.include_router(settings.router) router.include_router(runs.router) diff --git a/app/api/routers/admin_dbops.py b/app/api/routers/admin_dbops.py new file mode 100644 index 0000000..16271a0 --- /dev/null +++ b/app/api/routers/admin_dbops.py @@ -0,0 +1,337 @@ +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, Path, Query, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.deps import get_api_admin_user +from app.api.errors import ApiException +from app.api.responses import success_payload +from app.api.schemas import ( + AdminDbIntegrityEnvelope, + AdminPdfQueueBulkEnqueueEnvelope, + AdminPdfQueueRequeueEnvelope, + AdminPdfQueueEnvelope, + AdminDbRepairJobsEnvelope, + AdminRepairPublicationLinksEnvelope, + AdminRepairPublicationLinksRequest, +) +from app.db.models import DataRepairJob, User +from app.db.session import get_db_session +from app.services.domains.dbops import ( + collect_integrity_report, + list_repair_jobs, + run_publication_link_repair, +) +from app.services.domains.publications import application as publication_service + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/admin/db", tags=["api-admin-dbops"]) + + +def _serialize_repair_job(job: DataRepairJob) -> dict[str, object]: + return { + "id": int(job.id), + "job_name": job.job_name, + "requested_by": job.requested_by, + "dry_run": bool(job.dry_run), + "status": job.status, + "scope": dict(job.scope or {}), + "summary": dict(job.summary or {}), + "error_text": job.error_text, + "started_at": job.started_at, + "finished_at": job.finished_at, + "created_at": job.created_at, + "updated_at": job.updated_at, + } + + +def _requested_by_value(*, payload: AdminRepairPublicationLinksRequest, admin_user: User) -> str: + from_payload = (payload.requested_by or "").strip() + return from_payload or admin_user.email + + +def _serialize_pdf_queue_item(item) -> dict[str, object]: + return { + "publication_id": item.publication_id, + "title": item.title, + "doi": item.doi, + "pdf_url": item.pdf_url, + "status": item.status, + "attempt_count": item.attempt_count, + "last_failure_reason": item.last_failure_reason, + "last_failure_detail": item.last_failure_detail, + "last_source": item.last_source, + "requested_by_user_id": item.requested_by_user_id, + "requested_by_email": item.requested_by_email, + "queued_at": item.queued_at, + "last_attempt_at": item.last_attempt_at, + "resolved_at": item.resolved_at, + "updated_at": item.updated_at, + } + + +def _requeue_response_state(*, queued: bool) -> tuple[str, str]: + if queued: + return "queued", "PDF lookup queued." + return "blocked", "PDF lookup is already queued or currently running." + + +def _bulk_enqueue_message(*, queued_count: int, requested_count: int) -> str: + if requested_count == 0: + return "No missing-PDF publications were found." + if queued_count == 0: + return "No publications were queued; all candidates were already in-flight." + return f"Queued {queued_count} publication(s) for PDF lookup." + + +def _resolve_pdf_queue_paging( + *, + page: int, + page_size: int, + limit: int | None, + offset: int | None, +) -> tuple[int, int, int]: + resolved_limit = int(limit) if limit is not None else int(page_size) + resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0) + resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1) + return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0) + + +def _pdf_queue_page_data(*, total_count: int, page: int, page_size: int, offset: int) -> dict[str, object]: + return { + "total_count": int(total_count), + "page": int(page), + "page_size": int(page_size), + "has_prev": int(offset) > 0, + "has_next": int(offset) + int(page_size) < int(total_count), + } + + +@router.get( + "/integrity", + response_model=AdminDbIntegrityEnvelope, +) +async def get_integrity_report( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + report = await collect_integrity_report(db_session) + logger.info( + "api.admin.db.integrity_checked", + extra={ + "event": "api.admin.db.integrity_checked", + "admin_user_id": int(admin_user.id), + "status": report.get("status"), + "failure_count": len(report.get("failures", [])), + "warning_count": len(report.get("warnings", [])), + }, + ) + return success_payload(request, data=report) + + +@router.get( + "/repair-jobs", + response_model=AdminDbRepairJobsEnvelope, +) +async def get_repair_jobs( + request: Request, + limit: int = Query(default=50, ge=1, le=200), + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + jobs = await list_repair_jobs(db_session, limit=limit) + logger.info( + "api.admin.db.repair_jobs_listed", + extra={ + "event": "api.admin.db.repair_jobs_listed", + "admin_user_id": int(admin_user.id), + "limit": int(limit), + "job_count": len(jobs), + }, + ) + return success_payload( + request, + data={"jobs": [_serialize_repair_job(job) for job in jobs]}, + ) + + +@router.get( + "/pdf-queue", + response_model=AdminPdfQueueEnvelope, +) +async def get_pdf_queue( + request: Request, + page: int = Query(default=1, ge=1), + page_size: int = Query(default=100, ge=1, le=500), + limit: int | None = Query(default=None, ge=1, le=500), + offset: int | None = Query(default=None, ge=0), + status: str | None = Query(default=None), + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + normalized_status = (status or "").strip().lower() or None + resolved_page, resolved_limit, resolved_offset = _resolve_pdf_queue_paging( + page=page, + page_size=page_size, + limit=limit, + offset=offset, + ) + queue_page = await publication_service.list_pdf_queue_page( + db_session, + limit=resolved_limit, + offset=resolved_offset, + status=normalized_status, + ) + logger.info( + "api.admin.db.pdf_queue_listed", + extra={ + "event": "api.admin.db.pdf_queue_listed", + "admin_user_id": int(admin_user.id), + "page": int(resolved_page), + "page_size": int(resolved_limit), + "offset": int(resolved_offset), + "status": normalized_status, + "item_count": len(queue_page.items), + "total_count": int(queue_page.total_count), + }, + ) + return success_payload( + request, + data={ + "items": [_serialize_pdf_queue_item(item) for item in queue_page.items], + **_pdf_queue_page_data( + total_count=queue_page.total_count, + page=resolved_page, + page_size=resolved_limit, + offset=resolved_offset, + ), + }, + ) + + +@router.post( + "/pdf-queue/{publication_id}/requeue", + response_model=AdminPdfQueueRequeueEnvelope, +) +async def requeue_pdf_lookup( + request: Request, + publication_id: int = Path(ge=1), + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + result = await publication_service.enqueue_retry_pdf_job_for_publication_id( + db_session, + user_id=int(admin_user.id), + request_email=admin_user.email, + publication_id=publication_id, + ) + if not result.publication_exists: + raise ApiException( + status_code=404, + code="publication_not_found", + message="Publication not found.", + ) + status, message = _requeue_response_state(queued=result.queued) + logger.info( + "api.admin.db.pdf_queue_requeue_requested", + extra={ + "event": "api.admin.db.pdf_queue_requeue_requested", + "admin_user_id": int(admin_user.id), + "publication_id": int(publication_id), + "queued": bool(result.queued), + }, + ) + return success_payload( + request, + data={ + "publication_id": int(publication_id), + "queued": bool(result.queued), + "status": status, + "message": message, + }, + ) + + +@router.post( + "/pdf-queue/requeue-all", + response_model=AdminPdfQueueBulkEnqueueEnvelope, +) +async def requeue_all_missing_pdfs( + request: Request, + limit: int = Query(default=1000, ge=1, le=5000), + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + result = await publication_service.enqueue_all_missing_pdf_jobs( + db_session, + user_id=int(admin_user.id), + request_email=admin_user.email, + limit=limit, + ) + logger.info( + "api.admin.db.pdf_queue_requeue_all", + extra={ + "event": "api.admin.db.pdf_queue_requeue_all", + "admin_user_id": int(admin_user.id), + "limit": int(limit), + "requested_count": int(result.requested_count), + "queued_count": int(result.queued_count), + }, + ) + return success_payload( + request, + data={ + "requested_count": int(result.requested_count), + "queued_count": int(result.queued_count), + "message": _bulk_enqueue_message( + queued_count=int(result.queued_count), + requested_count=int(result.requested_count), + ), + }, + ) + + +@router.post( + "/repairs/publication-links", + response_model=AdminRepairPublicationLinksEnvelope, +) +async def trigger_publication_link_repair( + payload: AdminRepairPublicationLinksRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + admin_user: User = Depends(get_api_admin_user), +): + try: + result = await run_publication_link_repair( + db_session, + scope_mode=payload.scope_mode, + user_id=payload.user_id, + scholar_profile_ids=payload.scholar_profile_ids, + dry_run=bool(payload.dry_run), + gc_orphan_publications=bool(payload.gc_orphan_publications), + requested_by=_requested_by_value(payload=payload, admin_user=admin_user), + ) + except ValueError as exc: + raise ApiException( + status_code=400, + code="invalid_repair_scope", + message=str(exc), + ) from exc + logger.info( + "api.admin.db.publication_link_repair_triggered", + extra={ + "event": "api.admin.db.publication_link_repair_triggered", + "admin_user_id": int(admin_user.id), + "scope_mode": payload.scope_mode, + "target_user_id": int(payload.user_id) if payload.user_id is not None else None, + "dry_run": bool(payload.dry_run), + "gc_orphan_publications": bool(payload.gc_orphan_publications), + "job_id": int(result["job_id"]), + "status": result["status"], + }, + ) + return success_payload(request, data=result) diff --git a/app/api/routers/publications.py b/app/api/routers/publications.py index 998cdea..5055c6c 100644 --- a/app/api/routers/publications.py +++ b/app/api/routers/publications.py @@ -16,6 +16,8 @@ from app.api.schemas import ( PublicationsListEnvelope, RetryPublicationPdfEnvelope, RetryPublicationPdfRequest, + TogglePublicationFavoriteEnvelope, + TogglePublicationFavoriteRequest, ) from app.db.models import User from app.db.session import get_db_session @@ -61,7 +63,12 @@ def _serialize_publication_item(item) -> dict[str, object]: "pub_url": item.pub_url, "doi": item.doi, "pdf_url": item.pdf_url, + "pdf_status": item.pdf_status, + "pdf_attempt_count": item.pdf_attempt_count, + "pdf_failure_reason": item.pdf_failure_reason, + "pdf_failure_detail": item.pdf_failure_detail, "is_read": item.is_read, + "is_favorite": item.is_favorite, "first_seen_at": item.first_seen_at, "is_new_in_latest_run": item.is_new_in_latest_run, } @@ -72,46 +79,235 @@ async def _publication_counts( *, user_id: int, selected_scholar_id: int | None, -) -> tuple[int, int, int]: + favorite_only: bool, +) -> tuple[int, int, int, int]: unread_count = await publication_service.count_unread_for_user( db_session, user_id=user_id, scholar_profile_id=selected_scholar_id, + favorite_only=favorite_only, + ) + favorites_count = await publication_service.count_favorite_for_user( + db_session, + user_id=user_id, + scholar_profile_id=selected_scholar_id, ) latest_count = await publication_service.count_latest_for_user( db_session, user_id=user_id, scholar_profile_id=selected_scholar_id, + favorite_only=favorite_only, ) total_count = await publication_service.count_for_user( db_session, user_id=user_id, mode=publication_service.MODE_ALL, scholar_profile_id=selected_scholar_id, + favorite_only=favorite_only, ) - return unread_count, latest_count, total_count + return unread_count, favorites_count, latest_count, total_count + + +async def _list_publications_for_request( + db_session: AsyncSession, + *, + current_user: User, + mode: Literal["all", "unread", "latest", "new"] | None, + favorite_only: bool, + scholar_profile_id: int | None, + limit: int, + offset: int, +) -> tuple[str, int | None, list]: + resolved_mode = publication_service.resolve_publication_view_mode(mode) + selected_scholar_id = scholar_profile_id + await _require_selected_profile( + db_session, + user_id=current_user.id, + selected_scholar_id=selected_scholar_id, + ) + publications = await publication_service.list_for_user( + db_session, + user_id=current_user.id, + mode=resolved_mode, + scholar_profile_id=selected_scholar_id, + favorite_only=favorite_only, + limit=limit, + offset=offset, + ) + await publication_service.schedule_missing_pdf_enrichment_for_user( + db_session, + user_id=current_user.id, + request_email=current_user.email, + items=publications, + max_items=settings.unpaywall_max_items_per_request, + ) + hydrated = await publication_service.hydrate_pdf_enrichment_state( + db_session, + items=publications, + ) + return resolved_mode, selected_scholar_id, hydrated + + +def _resolve_publications_paging( + *, + page: int, + page_size: int, + limit: int | None, + offset: int | None, +) -> tuple[int, int, int]: + resolved_limit = int(limit) if limit is not None else int(page_size) + resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0) + resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1) + return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0) def _publications_list_data( *, mode: str, + favorite_only: bool, selected_scholar_id: int | None, unread_count: int, + favorites_count: int, latest_count: int, total_count: int, publications: list, + page: int, + page_size: int, + offset: int, ) -> dict[str, object]: return { "mode": mode, + "favorite_only": favorite_only, "selected_scholar_profile_id": selected_scholar_id, "unread_count": unread_count, + "favorites_count": favorites_count, "latest_count": latest_count, "new_count": latest_count, "total_count": total_count, + "page": int(page), + "page_size": int(page_size), + "has_prev": int(offset) > 0, + "has_next": int(offset) + int(page_size) < int(total_count), "publications": [_serialize_publication_item(item) for item in publications], } +def _retry_pdf_message(*, queued: bool, resolved_pdf: bool, pdf_status: str) -> str: + if resolved_pdf: + return "Open-access PDF link already resolved." + if queued: + return "PDF lookup queued." + if pdf_status in {"queued", "running"}: + return "PDF lookup is already queued." + return "No open-access PDF link found." + + +def _favorite_message(*, is_favorite: bool) -> str: + if is_favorite: + return "Publication marked as favorite." + return "Publication removed from favorites." + + +async def _retry_publication_state( + db_session: AsyncSession, + *, + current_user: User, + scholar_profile_id: int, + publication_id: int, +): + publication = await publication_service.retry_pdf_for_user( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + publication_id=publication_id, + ) + if publication is None: + raise ApiException( + status_code=404, + code="publication_not_found", + message="Publication not found.", + ) + queued = False + if not publication.pdf_url: + queued = await publication_service.schedule_retry_pdf_enrichment_for_row( + db_session, + user_id=current_user.id, + request_email=current_user.email, + item=publication, + ) + hydrated = await publication_service.hydrate_pdf_enrichment_state( + db_session, + items=[publication], + ) + current = hydrated[0] if hydrated else publication + return current, queued + + +async def _favorite_publication_state( + db_session: AsyncSession, + *, + current_user: User, + scholar_profile_id: int, + publication_id: int, + is_favorite: bool, +): + updated_count = await publication_service.set_publication_favorite_for_user( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + publication_id=publication_id, + is_favorite=is_favorite, + ) + if updated_count <= 0: + raise ApiException( + status_code=404, + code="publication_not_found", + message="Publication not found.", + ) + publication = await publication_service.get_publication_item_for_user( + db_session, + user_id=current_user.id, + scholar_profile_id=scholar_profile_id, + publication_id=publication_id, + ) + if publication is None: + raise ApiException( + status_code=404, + code="publication_not_found", + message="Publication not found.", + ) + hydrated = await publication_service.hydrate_pdf_enrichment_state( + db_session, + items=[publication], + ) + return hydrated[0] if hydrated else publication + + +def _log_retry_pdf_result( + *, + current_user: User, + scholar_profile_id: int, + publication_id: int, + queued: bool, + resolved_pdf: bool, + pdf_status: str, + doi: str | None, +) -> None: + logger.info( + "api.publications.retry_pdf", + extra={ + "event": "api.publications.retry_pdf", + "user_id": current_user.id, + "scholar_profile_id": scholar_profile_id, + "publication_id": publication_id, + "queued": queued, + "resolved_pdf": resolved_pdf, + "pdf_status": pdf_status, + "has_doi": bool(doi), + }, + ) + + @router.get( "", response_model=PublicationsListEnvelope, @@ -119,44 +315,48 @@ def _publications_list_data( async def list_publications( request: Request, mode: Literal["all", "unread", "latest", "new"] | None = Query(default=None), + favorite_only: bool = Query(default=False), scholar_profile_id: int | None = Query(default=None, ge=1), - limit: int = Query(default=300, ge=1, le=1000), + page: int = Query(default=1, ge=1), + page_size: int = Query(default=100, ge=1, le=500), + limit: int | None = Query(default=None, ge=1, le=1000), + offset: int | None = Query(default=None, ge=0), db_session: AsyncSession = Depends(get_db_session), current_user: User = Depends(get_api_current_user), ): - resolved_mode = publication_service.resolve_publication_view_mode(mode) - selected_scholar_id = scholar_profile_id - await _require_selected_profile( - db_session, - user_id=current_user.id, - selected_scholar_id=selected_scholar_id, - ) - - publications = await publication_service.list_for_user( - db_session, - user_id=current_user.id, - mode=resolved_mode, - scholar_profile_id=selected_scholar_id, + resolved_page, resolved_limit, resolved_offset = _resolve_publications_paging( + page=page, + page_size=page_size, limit=limit, + offset=offset, ) - await publication_service.schedule_missing_pdf_enrichment_for_user( - user_id=current_user.id, - request_email=current_user.email, - items=publications, - max_items=settings.unpaywall_max_items_per_request, + resolved_mode, selected_scholar_id, publications = await _list_publications_for_request( + db_session, + current_user=current_user, + mode=mode, + favorite_only=favorite_only, + scholar_profile_id=scholar_profile_id, + limit=resolved_limit, + offset=resolved_offset, ) - unread_count, latest_count, total_count = await _publication_counts( + unread_count, favorites_count, latest_count, total_count = await _publication_counts( db_session, user_id=current_user.id, selected_scholar_id=selected_scholar_id, + favorite_only=favorite_only, ) data = _publications_list_data( mode=resolved_mode, + favorite_only=favorite_only, selected_scholar_id=selected_scholar_id, unread_count=unread_count, + favorites_count=favorites_count, latest_count=latest_count, total_count=total_count, publications=publications, + page=resolved_page, + page_size=resolved_limit, + offset=resolved_offset, ) return success_payload(request, data=data) @@ -242,37 +442,70 @@ async def retry_publication_pdf( db_session: AsyncSession = Depends(get_db_session), current_user: User = Depends(get_api_current_user), ): - publication = await publication_service.retry_pdf_for_user( + current, queued = await _retry_publication_state( db_session, - user_id=current_user.id, + current_user=current_user, scholar_profile_id=payload.scholar_profile_id, publication_id=publication_id, - unpaywall_email=current_user.email, ) - if publication is None: - raise ApiException( - status_code=404, - code="publication_not_found", - message="Publication not found.", - ) - resolved_pdf = bool(publication.pdf_url) - logger.info( - "api.publications.retry_pdf", - extra={ - "event": "api.publications.retry_pdf", - "user_id": current_user.id, - "scholar_profile_id": payload.scholar_profile_id, - "publication_id": publication_id, - "resolved_pdf": resolved_pdf, - "has_doi": bool(publication.doi), - }, + resolved_pdf = bool(current.pdf_url) + message = _retry_pdf_message( + queued=queued, + resolved_pdf=resolved_pdf, + pdf_status=current.pdf_status, + ) + _log_retry_pdf_result( + current_user=current_user, + scholar_profile_id=payload.scholar_profile_id, + publication_id=publication_id, + queued=queued, + resolved_pdf=resolved_pdf, + pdf_status=current.pdf_status, + doi=current.doi, ) - message = "Open-access PDF link resolved." if resolved_pdf else "No open-access PDF link found." return success_payload( request, data={ "message": message, + "queued": queued, "resolved_pdf": resolved_pdf, - "publication": _serialize_publication_item(publication), + "publication": _serialize_publication_item(current), + }, + ) + + +@router.post( + "/{publication_id}/favorite", + response_model=TogglePublicationFavoriteEnvelope, +) +async def toggle_publication_favorite( + payload: TogglePublicationFavoriteRequest, + request: Request, + publication_id: int = Path(ge=1), + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + current = await _favorite_publication_state( + db_session, + current_user=current_user, + scholar_profile_id=payload.scholar_profile_id, + publication_id=publication_id, + is_favorite=payload.is_favorite, + ) + logger.info( + "api.publications.favorite", + extra={ + "event": "api.publications.favorite", + "user_id": current_user.id, + "scholar_profile_id": payload.scholar_profile_id, + "publication_id": publication_id, + "is_favorite": bool(payload.is_favorite), + }, + ) + return success_payload( + request, + data={ + "message": _favorite_message(is_favorite=bool(payload.is_favorite)), + "publication": _serialize_publication_item(current), }, ) diff --git a/app/api/schemas.py b/app/api/schemas.py index 9deb89a..3ae8876 100644 --- a/app/api/schemas.py +++ b/app/api/schemas.py @@ -1,9 +1,9 @@ from __future__ import annotations from datetime import datetime -from typing import Any +from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator class ApiMeta(BaseModel): @@ -525,6 +525,175 @@ class AdminResetPasswordRequest(BaseModel): model_config = ConfigDict(extra="forbid") +class AdminDbIntegrityCheckData(BaseModel): + name: str + count: int + severity: str + message: str + + model_config = ConfigDict(extra="forbid") + + +class AdminDbIntegrityData(BaseModel): + status: str + checked_at: datetime + failures: list[str] = Field(default_factory=list) + warnings: list[str] = Field(default_factory=list) + checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class AdminDbIntegrityEnvelope(BaseModel): + data: AdminDbIntegrityData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminDbRepairJobData(BaseModel): + id: int + job_name: str + requested_by: str | None + dry_run: bool + status: str + scope: dict[str, Any] = Field(default_factory=dict) + summary: dict[str, Any] = Field(default_factory=dict) + error_text: str | None + started_at: datetime | None + finished_at: datetime | None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(extra="forbid") + + +class AdminDbRepairJobsData(BaseModel): + jobs: list[AdminDbRepairJobData] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class AdminDbRepairJobsEnvelope(BaseModel): + data: AdminDbRepairJobsData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueItemData(BaseModel): + publication_id: int + title: str + doi: str | None + pdf_url: str | None + status: str + attempt_count: int + last_failure_reason: str | None + last_failure_detail: str | None + last_source: str | None + requested_by_user_id: int | None + requested_by_email: str | None + queued_at: datetime | None + last_attempt_at: datetime | None + resolved_at: datetime | None + updated_at: datetime + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueData(BaseModel): + items: list[AdminPdfQueueItemData] = Field(default_factory=list) + total_count: int = 0 + page: int = 1 + page_size: int = 100 + has_next: bool = False + has_prev: bool = False + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueEnvelope(BaseModel): + data: AdminPdfQueueData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueRequeueData(BaseModel): + publication_id: int + queued: bool + status: str + message: str + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueRequeueEnvelope(BaseModel): + data: AdminPdfQueueRequeueData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueBulkEnqueueData(BaseModel): + requested_count: int + queued_count: int + message: str + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueBulkEnqueueEnvelope(BaseModel): + data: AdminPdfQueueBulkEnqueueData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminRepairPublicationLinksRequest(BaseModel): + scope_mode: Literal["single_user", "all_users"] = "single_user" + user_id: int | None = Field(default=None, ge=1) + scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200) + dry_run: bool = True + gc_orphan_publications: bool = False + requested_by: str | None = None + confirmation_text: str | None = None + + model_config = ConfigDict(extra="forbid") + + @model_validator(mode="after") + def validate_scope(self) -> "AdminRepairPublicationLinksRequest": + if self.scope_mode == "single_user" and self.user_id is None: + raise ValueError("user_id is required when scope_mode=single_user.") + if self.scope_mode == "all_users" and self.user_id is not None: + raise ValueError("user_id must be omitted when scope_mode=all_users.") + if not self.dry_run and self.scope_mode == "all_users": + expected = "REPAIR ALL USERS" + provided = (self.confirmation_text or "").strip() + if provided != expected: + raise ValueError( + "confirmation_text must equal 'REPAIR ALL USERS' " + "when applying a repair to all users." + ) + return self + + +class AdminRepairPublicationLinksResultData(BaseModel): + job_id: int + status: str + scope: dict[str, Any] = Field(default_factory=dict) + summary: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") + + +class AdminRepairPublicationLinksEnvelope(BaseModel): + data: AdminRepairPublicationLinksResultData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + class SettingsPolicyData(BaseModel): min_run_interval_minutes: int min_request_delay_seconds: int @@ -576,7 +745,12 @@ class PublicationItemData(BaseModel): pub_url: str | None doi: str | None pdf_url: str | None + pdf_status: str = "untracked" + pdf_attempt_count: int = 0 + pdf_failure_reason: str | None = None + pdf_failure_detail: str | None = None is_read: bool + is_favorite: bool = False first_seen_at: datetime is_new_in_latest_run: bool @@ -585,11 +759,17 @@ class PublicationItemData(BaseModel): class PublicationsListData(BaseModel): mode: str + favorite_only: bool = False selected_scholar_profile_id: int | None unread_count: int + favorites_count: int latest_count: int new_count: int total_count: int + page: int + page_size: int + has_next: bool = False + has_prev: bool = False publications: list[PublicationItemData] model_config = ConfigDict(extra="forbid") @@ -652,6 +832,7 @@ class RetryPublicationPdfRequest(BaseModel): class RetryPublicationPdfData(BaseModel): message: str + queued: bool resolved_pdf: bool publication: PublicationItemData @@ -663,3 +844,24 @@ class RetryPublicationPdfEnvelope(BaseModel): meta: ApiMeta model_config = ConfigDict(extra="forbid") + + +class TogglePublicationFavoriteRequest(BaseModel): + scholar_profile_id: int = Field(ge=1) + is_favorite: bool + + model_config = ConfigDict(extra="forbid") + + +class TogglePublicationFavoriteData(BaseModel): + message: str + publication: PublicationItemData + + model_config = ConfigDict(extra="forbid") + + +class TogglePublicationFavoriteEnvelope(BaseModel): + data: TogglePublicationFavoriteData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") diff --git a/app/db/models.py b/app/db/models.py index e9e4303..43b44c2 100644 --- a/app/db/models.py +++ b/app/db/models.py @@ -81,8 +81,8 @@ class UserSetting(Base): name="run_interval_minutes_min_15", ), CheckConstraint( - "request_delay_seconds >= 1", - name="request_delay_seconds_min_1", + "request_delay_seconds >= 2", + name="request_delay_seconds_min_2", ), ) @@ -237,10 +237,80 @@ class Publication(Base): ) +class PublicationPdfJob(Base): + __tablename__ = "publication_pdf_jobs" + __table_args__ = ( + CheckConstraint( + "status IN ('queued', 'running', 'resolved', 'failed')", + name="publication_pdf_jobs_status_valid", + ), + Index("ix_publication_pdf_jobs_status", "status"), + Index("ix_publication_pdf_jobs_updated_at", "updated_at"), + Index("ix_publication_pdf_jobs_queued_at", "queued_at"), + ) + + publication_id: Mapped[int] = mapped_column( + ForeignKey("publications.id", ondelete="CASCADE"), + primary_key=True, + ) + status: Mapped[str] = mapped_column( + String(16), + nullable=False, + server_default=text("'queued'"), + ) + attempt_count: Mapped[int] = mapped_column( + Integer, + nullable=False, + server_default=text("0"), + ) + queued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + last_failure_reason: Mapped[str | None] = mapped_column(String(64)) + last_failure_detail: Mapped[str | None] = mapped_column(Text) + last_source: Mapped[str | None] = mapped_column(String(32)) + last_requested_by_user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + +class PublicationPdfJobEvent(Base): + __tablename__ = "publication_pdf_job_events" + __table_args__ = ( + Index("ix_publication_pdf_job_events_publication_created", "publication_id", "created_at"), + Index("ix_publication_pdf_job_events_created_at", "created_at"), + Index("ix_publication_pdf_job_events_event_type", "event_type"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + publication_id: Mapped[int] = mapped_column( + ForeignKey("publications.id", ondelete="CASCADE"), + nullable=False, + ) + user_id: Mapped[int | None] = mapped_column( + ForeignKey("users.id", ondelete="SET NULL"), + ) + event_type: Mapped[str] = mapped_column(String(32), nullable=False) + status: Mapped[str | None] = mapped_column(String(16)) + source: Mapped[str | None] = mapped_column(String(32)) + failure_reason: Mapped[str | None] = mapped_column(String(64)) + message: Mapped[str | None] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + class ScholarPublication(Base): __tablename__ = "scholar_publications" __table_args__ = ( Index("ix_scholar_publications_is_read", "is_read"), + Index("ix_scholar_publications_is_favorite", "is_favorite"), ) scholar_profile_id: Mapped[int] = mapped_column( @@ -254,6 +324,9 @@ class ScholarPublication(Base): is_read: Mapped[bool] = mapped_column( Boolean, nullable=False, server_default=text("false") ) + is_favorite: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) first_seen_run_id: Mapped[int | None] = mapped_column( ForeignKey("crawl_runs.id", ondelete="SET NULL") ) @@ -322,6 +395,51 @@ class IngestionQueueItem(Base): ) +class DataRepairJob(Base): + __tablename__ = "data_repair_jobs" + __table_args__ = ( + CheckConstraint( + "status IN ('planned', 'running', 'completed', 'failed')", + name="data_repair_jobs_status_valid", + ), + Index("ix_data_repair_jobs_created_at", "created_at"), + Index("ix_data_repair_jobs_job_name_created_at", "job_name", "created_at"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + job_name: Mapped[str] = mapped_column(String(64), nullable=False) + requested_by: Mapped[str | None] = mapped_column(String(255)) + scope: Mapped[dict] = mapped_column( + JSONB, + nullable=False, + server_default=text("'{}'::jsonb"), + ) + dry_run: Mapped[bool] = mapped_column( + Boolean, + nullable=False, + server_default=text("true"), + ) + status: Mapped[str] = mapped_column( + String(16), + nullable=False, + server_default=text("'planned'"), + ) + summary: Mapped[dict] = mapped_column( + JSONB, + nullable=False, + server_default=text("'{}'::jsonb"), + ) + error_text: Mapped[str | None] = mapped_column(Text) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + class AuthorSearchRuntimeState(Base): __tablename__ = "author_search_runtime_state" diff --git a/app/http/middleware.py b/app/http/middleware.py index 49a40a4..ebe72f8 100644 --- a/app/http/middleware.py +++ b/app/http/middleware.py @@ -61,7 +61,7 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware): start = time.perf_counter() should_log = self._log_requests and not self._is_skipped_path(request.url.path) if should_log: - logger.info( + logger.debug( "request.started", extra={ "event": "request.started", @@ -88,7 +88,7 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware): duration_ms = int((time.perf_counter() - start) * 1000) response.headers[REQUEST_ID_HEADER] = request_id if should_log: - logger.info( + logger.debug( "request.completed", extra={ "event": "request.completed", diff --git a/app/services/domains/dbops/__init__.py b/app/services/domains/dbops/__init__.py new file mode 100644 index 0000000..f5b02e4 --- /dev/null +++ b/app/services/domains/dbops/__init__.py @@ -0,0 +1,5 @@ +from app.services.domains.dbops.application import run_publication_link_repair +from app.services.domains.dbops.integrity import collect_integrity_report +from app.services.domains.dbops.query import list_repair_jobs + +__all__ = ["collect_integrity_report", "list_repair_jobs", "run_publication_link_repair"] diff --git a/app/services/domains/dbops/application.py b/app/services/domains/dbops/application.py new file mode 100644 index 0000000..e7ace77 --- /dev/null +++ b/app/services/domains/dbops/application.py @@ -0,0 +1,364 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +from sqlalchemy import delete, exists, func, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import DataRepairJob, IngestionQueueItem, Publication, ScholarProfile, ScholarPublication + + +REPAIR_STATUS_PLANNED = "planned" +REPAIR_STATUS_RUNNING = "running" +REPAIR_STATUS_COMPLETED = "completed" +REPAIR_STATUS_FAILED = "failed" +SCOPE_MODE_SINGLE_USER = "single_user" +SCOPE_MODE_ALL_USERS = "all_users" + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def _normalize_scope_mode(scope_mode: str) -> str: + normalized = scope_mode.strip().lower() + if normalized in {SCOPE_MODE_SINGLE_USER, SCOPE_MODE_ALL_USERS}: + return normalized + raise ValueError("Unknown scope mode.") + + +def _scope_user_id(*, scope_mode: str, user_id: int | None) -> int | None: + if scope_mode == SCOPE_MODE_SINGLE_USER: + if user_id is None: + raise ValueError("user_id is required when scope_mode=single_user.") + return int(user_id) + if user_id is not None: + raise ValueError("user_id must be omitted when scope_mode=all_users.") + return None + + +def _scope_payload( + *, + scope_mode: str, + user_id: int | None, + target_scholar_profile_ids: list[int], + orphan_gc: bool, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "scope_mode": scope_mode, + "scholar_profile_ids": [int(value) for value in target_scholar_profile_ids], + "gc_orphan_publications": bool(orphan_gc), + } + if user_id is not None: + payload["user_id"] = int(user_id) + return payload + + +async def _target_scholar_profile_ids( + db_session: AsyncSession, + *, + scope_mode: str, + user_id: int | None, + scholar_profile_ids: list[int] | None, +) -> list[int]: + stmt = select(ScholarProfile.id) + if scope_mode == SCOPE_MODE_SINGLE_USER: + stmt = stmt.where(ScholarProfile.user_id == user_id) + if scholar_profile_ids: + normalized_ids = [int(value) for value in scholar_profile_ids] + stmt = stmt.where(ScholarProfile.id.in_(normalized_ids)) + result = await db_session.execute(stmt.order_by(ScholarProfile.id.asc())) + ids = [int(row[0]) for row in result.all()] + if not ids: + raise ValueError("No target scholar profiles found for the requested scope.") + return ids + + +async def _count_scope( + db_session: AsyncSession, + *, + user_id: int | None, + target_scholar_profile_ids: list[int], +) -> dict[str, int]: + links_result = await db_session.execute( + select(func.count()) + .select_from(ScholarPublication) + .where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids)) + ) + queue_stmt = ( + select(func.count()) + .select_from(IngestionQueueItem) + .where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids)) + ) + if user_id is not None: + queue_stmt = queue_stmt.where(IngestionQueueItem.user_id == user_id) + queue_result = await db_session.execute(queue_stmt) + return { + "target_scholar_count": len(target_scholar_profile_ids), + "links_in_scope": int(links_result.scalar_one() or 0), + "queue_items_in_scope": int(queue_result.scalar_one() or 0), + } + + +async def _count_orphan_publications(db_session: AsyncSession) -> int: + stmt = ( + select(func.count()) + .select_from(Publication) + .where( + ~exists( + select(1).where(ScholarPublication.publication_id == Publication.id) + ) + ) + ) + result = await db_session.execute(stmt) + return int(result.scalar_one() or 0) + + +async def _create_job( + db_session: AsyncSession, + *, + requested_by: str | None, + scope: dict[str, Any], + dry_run: bool, +) -> DataRepairJob: + job = DataRepairJob( + job_name="repair_publication_links", + requested_by=(requested_by or "").strip() or None, + scope=scope, + dry_run=dry_run, + status=REPAIR_STATUS_PLANNED, + summary={}, + ) + db_session.add(job) + await db_session.flush() + return job + + +def _job_summary( + *, + counts: dict[str, int], + dry_run: bool, + links_deleted: int, + queue_items_deleted: int, + scholars_reset: int, + orphan_publications_before: int, + orphan_publications_deleted: int, +) -> dict[str, Any]: + return { + **counts, + "dry_run": bool(dry_run), + "links_deleted": int(links_deleted), + "queue_items_deleted": int(queue_items_deleted), + "scholars_reset": int(scholars_reset), + "orphan_publications_before": int(orphan_publications_before), + "orphan_publications_deleted": int(orphan_publications_deleted), + } + + +async def _delete_links_for_targets(db_session: AsyncSession, *, target_scholar_profile_ids: list[int]) -> int: + result = await db_session.execute( + delete(ScholarPublication).where( + ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids) + ) + ) + return int(result.rowcount or 0) + + +async def _delete_queue_for_targets( + db_session: AsyncSession, + *, + user_id: int | None, + target_scholar_profile_ids: list[int], +) -> int: + stmt = delete(IngestionQueueItem).where( + IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids) + ) + if user_id is not None: + stmt = stmt.where(IngestionQueueItem.user_id == user_id) + result = await db_session.execute(stmt) + return int(result.rowcount or 0) + + +async def _reset_scholar_tracking_state( + db_session: AsyncSession, + *, + user_id: int | None, + target_scholar_profile_ids: list[int], +) -> int: + stmt = update(ScholarProfile).where(ScholarProfile.id.in_(target_scholar_profile_ids)) + if user_id is not None: + stmt = stmt.where(ScholarProfile.user_id == user_id) + result = await db_session.execute( + stmt.values( + baseline_completed=False, + last_initial_page_fingerprint_sha256=None, + last_initial_page_checked_at=None, + last_run_dt=None, + last_run_status=None, + ) + ) + return int(result.rowcount or 0) + + +async def _delete_orphan_publications(db_session: AsyncSession) -> int: + result = await db_session.execute( + delete(Publication).where( + ~exists(select(1).where(ScholarPublication.publication_id == Publication.id)) + ) + ) + return int(result.rowcount or 0) + + +async def _mutation_counts( + db_session: AsyncSession, + *, + user_id: int | None, + target_ids: list[int], + dry_run: bool, + gc_orphan_publications: bool, +) -> tuple[int, int, int, int]: + if dry_run: + return 0, 0, 0, 0 + links_deleted = await _delete_links_for_targets( + db_session, + target_scholar_profile_ids=target_ids, + ) + queue_deleted = await _delete_queue_for_targets( + db_session, + user_id=user_id, + target_scholar_profile_ids=target_ids, + ) + scholars_reset = await _reset_scholar_tracking_state( + db_session, + user_id=user_id, + target_scholar_profile_ids=target_ids, + ) + orphan_deleted = 0 + if gc_orphan_publications: + orphan_deleted = await _delete_orphan_publications(db_session) + return links_deleted, queue_deleted, scholars_reset, orphan_deleted + + +def _result_payload(*, job: DataRepairJob, scope: dict[str, Any], summary: dict[str, Any]) -> dict[str, Any]: + return { + "job_id": int(job.id), + "status": job.status, + "scope": scope, + "summary": summary, + } + + +async def _complete_job( + db_session: AsyncSession, + *, + job: DataRepairJob, + summary: dict[str, Any], + scope: dict[str, Any], +) -> dict[str, Any]: + job.summary = summary + job.status = REPAIR_STATUS_COMPLETED + job.finished_at = _utcnow() + await db_session.commit() + return _result_payload(job=job, scope=scope, summary=summary) + + +async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None: + await db_session.rollback() + job.status = REPAIR_STATUS_FAILED + job.error_text = str(error) + job.finished_at = _utcnow() + db_session.add(job) + await db_session.commit() + + +async def _prepare_repair_job( + db_session: AsyncSession, + *, + scope_mode: str, + user_id: int | None, + scholar_profile_ids: list[int] | None, + dry_run: bool, + gc_orphan_publications: bool, + requested_by: str | None, +) -> tuple[int | None, list[int], dict[str, Any], DataRepairJob]: + normalized_scope = _normalize_scope_mode(scope_mode) + scope_user_id = _scope_user_id(scope_mode=normalized_scope, user_id=user_id) + target_ids = await _target_scholar_profile_ids( + db_session, + scope_mode=normalized_scope, + user_id=scope_user_id, + scholar_profile_ids=scholar_profile_ids, + ) + scope = _scope_payload( + scope_mode=normalized_scope, + user_id=scope_user_id, + target_scholar_profile_ids=target_ids, + orphan_gc=gc_orphan_publications, + ) + job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run) + job.status = REPAIR_STATUS_RUNNING + job.started_at = _utcnow() + return scope_user_id, target_ids, scope, job + + +async def _build_repair_summary( + db_session: AsyncSession, + *, + scope_user_id: int | None, + target_ids: list[int], + dry_run: bool, + gc_orphan_publications: bool, +) -> dict[str, Any]: + counts = await _count_scope(db_session, user_id=scope_user_id, target_scholar_profile_ids=target_ids) + orphan_before = await _count_orphan_publications(db_session) + links_deleted, queue_deleted, scholars_reset, orphan_deleted = await _mutation_counts( + db_session, + user_id=scope_user_id, + target_ids=target_ids, + dry_run=dry_run, + gc_orphan_publications=gc_orphan_publications, + ) + return _job_summary( + counts=counts, + dry_run=dry_run, + links_deleted=links_deleted, + queue_items_deleted=queue_deleted, + scholars_reset=scholars_reset, + orphan_publications_before=orphan_before, + orphan_publications_deleted=orphan_deleted, + ) + + +async def run_publication_link_repair( + db_session: AsyncSession, + *, + scope_mode: str = SCOPE_MODE_SINGLE_USER, + user_id: int | None = None, + scholar_profile_ids: list[int] | None = None, + dry_run: bool = True, + gc_orphan_publications: bool = False, + requested_by: str | None = None, +) -> dict[str, Any]: + scope_user_id, target_ids, scope, job = await _prepare_repair_job( + db_session, + scope_mode=scope_mode, + user_id=user_id, + scholar_profile_ids=scholar_profile_ids, + dry_run=dry_run, + gc_orphan_publications=gc_orphan_publications, + requested_by=requested_by, + ) + + try: + summary = await _build_repair_summary( + db_session, + scope_user_id=scope_user_id, + target_ids=target_ids, + dry_run=dry_run, + gc_orphan_publications=gc_orphan_publications, + ) + return await _complete_job(db_session, job=job, summary=summary, scope=scope) + except Exception as exc: + await _fail_job(db_session, job=job, error=exc) + raise diff --git a/app/services/domains/dbops/integrity.py b/app/services/domains/dbops/integrity.py new file mode 100644 index 0000000..4d9c43c --- /dev/null +++ b/app/services/domains/dbops/integrity.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +from sqlalchemy import func, select, text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import Publication + +INTEGRITY_CHECK_DEFS = ( + ( + "legacy_cluster_id_format", + "warning", + "Publications with non-namespaced cluster IDs.", + ), + ( + "negative_citation_count", + "failure", + "Publications with negative citation counts.", + ), + ( + "orphan_publications", + "warning", + "Publications with no scholar links.", + ), + ( + "orphan_scholar_publication_links", + "failure", + "Link rows missing parent scholar/publication.", + ), + ( + "duplicate_fingerprint_keys", + "failure", + "Duplicate publication fingerprint keys.", + ), + ( + "duplicate_cluster_ids", + "failure", + "Duplicate non-null publication cluster IDs.", + ), + ( + "missing_pdf_url", + "metric", + "Publications without a resolved PDF URL.", + ), +) + + +async def _legacy_cluster_id_count(db_session: AsyncSession) -> int: + stmt = ( + select(func.count()) + .select_from(Publication) + .where(Publication.cluster_id.is_not(None)) + .where(~Publication.cluster_id.like("cfv:%")) + .where(~Publication.cluster_id.like("cluster:%")) + ) + result = await db_session.execute(stmt) + return int(result.scalar_one() or 0) + + +async def _negative_citation_count(db_session: AsyncSession) -> int: + result = await db_session.execute( + select(func.count()).select_from(Publication).where(Publication.citation_count < 0) + ) + return int(result.scalar_one() or 0) + + +async def _missing_pdf_url_count(db_session: AsyncSession) -> int: + result = await db_session.execute( + select(func.count()).select_from(Publication).where(Publication.pdf_url.is_(None)) + ) + return int(result.scalar_one() or 0) + + +async def _count_from_sql(db_session: AsyncSession, *, sql: str) -> int: + result = await db_session.execute(text(sql)) + return int(result.scalar_one() or 0) + + +def _issues_for_severity(*, checks: list[dict[str, Any]], severity: str) -> list[str]: + return [row["name"] for row in checks if row["severity"] == severity and row["count"] > 0] + + +def _check_row(*, name: str, count: int, severity: str, message: str) -> dict[str, Any]: + return { + "name": name, + "count": int(count), + "severity": severity, + "message": message, + } + + +async def _collect_counts(db_session: AsyncSession) -> dict[str, int]: + orphan_publications = await _count_from_sql( + db_session, + sql=( + "SELECT count(*) FROM publications p " + "LEFT JOIN scholar_publications sp ON sp.publication_id = p.id " + "WHERE sp.publication_id IS NULL" + ), + ) + orphan_links = await _count_from_sql( + db_session, + sql=( + "SELECT count(*) FROM scholar_publications sp " + "LEFT JOIN publications p ON p.id = sp.publication_id " + "LEFT JOIN scholar_profiles s ON s.id = sp.scholar_profile_id " + "WHERE p.id IS NULL OR s.id IS NULL" + ), + ) + duplicate_fingerprints = await _count_from_sql( + db_session, + sql=( + "SELECT count(*) FROM (" + "SELECT fingerprint_sha256 FROM publications " + "GROUP BY fingerprint_sha256 HAVING count(*) > 1" + ") dup" + ), + ) + duplicate_cluster_ids = await _count_from_sql( + db_session, + sql=( + "SELECT count(*) FROM (" + "SELECT cluster_id FROM publications " + "WHERE cluster_id IS NOT NULL " + "GROUP BY cluster_id HAVING count(*) > 1" + ") dup" + ), + ) + return { + "legacy_cluster_id_format": await _legacy_cluster_id_count(db_session), + "negative_citation_count": await _negative_citation_count(db_session), + "orphan_publications": orphan_publications, + "orphan_scholar_publication_links": orphan_links, + "duplicate_fingerprint_keys": duplicate_fingerprints, + "duplicate_cluster_ids": duplicate_cluster_ids, + "missing_pdf_url": await _missing_pdf_url_count(db_session), + } + + +def _build_checks(*, counts: dict[str, int]) -> list[dict[str, Any]]: + return [ + _check_row( + name=name, + count=counts[name], + severity=severity, + message=message, + ) + for name, severity, message in INTEGRITY_CHECK_DEFS + ] + + +def _status_from_issues(*, failures: list[str], warnings: list[str]) -> str: + status = "ok" + if failures: + status = "failed" + elif warnings: + status = "warning" + return status + + +async def collect_integrity_report(db_session: AsyncSession) -> dict[str, Any]: + counts = await _collect_counts(db_session) + checks = _build_checks(counts=counts) + failures = _issues_for_severity(checks=checks, severity="failure") + warnings = _issues_for_severity(checks=checks, severity="warning") + + return { + "status": _status_from_issues(failures=failures, warnings=warnings), + "checked_at": datetime.now(timezone.utc).isoformat(), + "failures": failures, + "warnings": warnings, + "checks": checks, + } diff --git a/app/services/domains/dbops/query.py b/app/services/domains/dbops/query.py new file mode 100644 index 0000000..2e95e31 --- /dev/null +++ b/app/services/domains/dbops/query.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import DataRepairJob + + +def _bounded_limit(limit: int) -> int: + return max(1, min(int(limit), 200)) + + +async def list_repair_jobs( + db_session: AsyncSession, + *, + limit: int = 50, +) -> list[DataRepairJob]: + bounded = _bounded_limit(limit) + result = await db_session.execute( + select(DataRepairJob).order_by(DataRepairJob.created_at.desc()).limit(bounded) + ) + return list(result.scalars()) diff --git a/app/services/domains/ingestion/application.py b/app/services/domains/ingestion/application.py index aa2844d..54a7e25 100644 --- a/app/services/domains/ingestion/application.py +++ b/app/services/domains/ingestion/application.py @@ -91,6 +91,35 @@ class ScholarIngestionService: def __init__(self, *, source: ScholarSource) -> None: self._source = source + @staticmethod + def _effective_request_delay_seconds(value: int) -> int: + policy_minimum = user_settings_service.resolve_request_delay_minimum( + settings.ingestion_min_request_delay_seconds + ) + return max(policy_minimum, _int_or_default(value, policy_minimum)) + + @staticmethod + def _log_request_delay_coercion( + *, + user_id: int, + requested_request_delay_seconds: int, + effective_request_delay_seconds: int, + ) -> None: + logger.warning( + "ingestion.request_delay_coerced_to_policy_floor", + extra={ + "event": "ingestion.request_delay_coerced_to_policy_floor", + "user_id": user_id, + "requested_request_delay_seconds": requested_request_delay_seconds, + "effective_request_delay_seconds": effective_request_delay_seconds, + "policy_minimum_request_delay_seconds": user_settings_service.resolve_request_delay_minimum( + settings.ingestion_min_request_delay_seconds + ), + "metric_name": "ingestion_request_delay_coerced_total", + "metric_value": 1, + }, + ) + async def _load_user_settings_for_run( self, db_session: AsyncSession, @@ -1278,7 +1307,14 @@ class ScholarIngestionService: return failure_summary, alert_summary async def run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, request_delay_seconds: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, max_pages_per_scholar: int = 30, page_size: int = 100, scholar_profile_ids: set[int] | None = None, start_cstart_by_scholar_id: dict[int, int] | None = None, auto_queue_continuations: bool = True, queue_delay_seconds: int = 60, idempotency_key: str | None = None, alert_blocked_failure_threshold: int = 1, alert_network_failure_threshold: int = 2, alert_retry_scheduled_threshold: int = 3) -> RunExecutionSummary: - paging_kwargs = self._paging_kwargs(request_delay_seconds=request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size) + effective_request_delay_seconds = self._effective_request_delay_seconds(request_delay_seconds) + if effective_request_delay_seconds != _int_or_default(request_delay_seconds, effective_request_delay_seconds): + self._log_request_delay_coercion( + user_id=user_id, + requested_request_delay_seconds=_int_or_default(request_delay_seconds, 0), + effective_request_delay_seconds=effective_request_delay_seconds, + ) + paging_kwargs = self._paging_kwargs(request_delay_seconds=effective_request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size) threshold_kwargs = self._threshold_kwargs(alert_blocked_failure_threshold=alert_blocked_failure_threshold, alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold) user_settings, run, scholars, start_cstart_map = await self._initialize_run_for_user( db_session, diff --git a/app/services/domains/ingestion/scheduler.py b/app/services/domains/ingestion/scheduler.py index 2959b63..9db0204 100644 --- a/app/services/domains/ingestion/scheduler.py +++ b/app/services/domains/ingestion/scheduler.py @@ -23,12 +23,28 @@ from app.services.domains.ingestion.application import ( RunBlockedBySafetyPolicyError, ScholarIngestionService, ) +from app.services.domains.settings import application as user_settings_service from app.services.domains.scholar.source import LiveScholarSource from app.settings import settings logger = logging.getLogger(__name__) +def _request_delay_floor_seconds() -> int: + return user_settings_service.resolve_request_delay_minimum( + settings.ingestion_min_request_delay_seconds + ) + + +def _effective_request_delay_seconds(value: int | None) -> int: + floor = _request_delay_floor_seconds() + try: + parsed = int(value) if value is not None else floor + except (TypeError, ValueError): + parsed = floor + return max(floor, parsed) + + @dataclass(frozen=True) class _AutoRunCandidate: user_id: int @@ -178,7 +194,7 @@ class SchedulerService: return _AutoRunCandidate( user_id=int(user_id), run_interval_minutes=int(run_interval_minutes), - request_delay_seconds=int(request_delay_seconds), + request_delay_seconds=_effective_request_delay_seconds(request_delay_seconds), cooldown_until=cooldown_until, cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None), ) @@ -577,6 +593,4 @@ class SchedulerService: select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id) ) delay = result.scalar_one_or_none() - if delay is None: - return 10 - return max(1, int(delay)) + return _effective_request_delay_seconds(delay) diff --git a/app/services/domains/publications/application.py b/app/services/domains/publications/application.py index 8e9e32a..cad6828 100644 --- a/app/services/domains/publications/application.py +++ b/app/services/domains/publications/application.py @@ -2,6 +2,7 @@ from __future__ import annotations from app.services.domains.publications.counts import ( count_for_user, + count_favorite_for_user, count_latest_for_user, count_unread_for_user, ) @@ -12,7 +13,9 @@ from app.services.domains.publications.listing import ( list_unread_for_user, ) from app.services.domains.publications.enrichment import ( + hydrate_pdf_enrichment_state, schedule_missing_pdf_enrichment_for_user, + schedule_retry_pdf_enrichment_for_row, ) from app.services.domains.publications.modes import ( MODE_ALL, @@ -30,6 +33,14 @@ from app.services.domains.publications.queries import ( from app.services.domains.publications.read_state import ( mark_all_unread_as_read_for_user, mark_selected_as_read_for_user, + set_publication_favorite_for_user, +) +from app.services.domains.publications.pdf_queue import ( + count_pdf_queue_items, + enqueue_all_missing_pdf_jobs, + enqueue_retry_pdf_job_for_publication_id, + list_pdf_queue_page, + list_pdf_queue_items, ) from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem @@ -49,10 +60,19 @@ __all__ = [ "list_unread_for_user", "list_new_for_latest_run_for_user", "retry_pdf_for_user", + "hydrate_pdf_enrichment_state", + "schedule_retry_pdf_enrichment_for_row", + "list_pdf_queue_items", + "list_pdf_queue_page", + "count_pdf_queue_items", + "enqueue_all_missing_pdf_jobs", + "enqueue_retry_pdf_job_for_publication_id", "schedule_missing_pdf_enrichment_for_user", "count_for_user", + "count_favorite_for_user", "count_unread_for_user", "count_latest_for_user", "mark_all_unread_as_read_for_user", "mark_selected_as_read_for_user", + "set_publication_favorite_for_user", ] diff --git a/app/services/domains/publications/counts.py b/app/services/domains/publications/counts.py index 98aa57d..8f65172 100644 --- a/app/services/domains/publications/counts.py +++ b/app/services/domains/publications/counts.py @@ -19,6 +19,7 @@ async def count_for_user( user_id: int, mode: str = MODE_ALL, scholar_profile_id: int | None = None, + favorite_only: bool = False, ) -> int: resolved_mode = resolve_publication_view_mode(mode) latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) @@ -30,6 +31,8 @@ async def count_for_user( ) if scholar_profile_id is not None: stmt = stmt.where(ScholarProfile.id == scholar_profile_id) + if favorite_only: + stmt = stmt.where(ScholarPublication.is_favorite.is_(True)) if resolved_mode == MODE_UNREAD: stmt = stmt.where(ScholarPublication.is_read.is_(False)) if resolved_mode == MODE_LATEST: @@ -45,12 +48,14 @@ async def count_unread_for_user( *, user_id: int, scholar_profile_id: int | None = None, + favorite_only: bool = False, ) -> int: return await count_for_user( db_session, user_id=user_id, mode=MODE_UNREAD, scholar_profile_id=scholar_profile_id, + favorite_only=favorite_only, ) @@ -59,10 +64,27 @@ async def count_latest_for_user( *, user_id: int, scholar_profile_id: int | None = None, + favorite_only: bool = False, ) -> int: return await count_for_user( db_session, user_id=user_id, mode=MODE_LATEST, scholar_profile_id=scholar_profile_id, + favorite_only=favorite_only, + ) + + +async def count_favorite_for_user( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int | None = None, +) -> int: + return await count_for_user( + db_session, + user_id=user_id, + mode=MODE_ALL, + scholar_profile_id=scholar_profile_id, + favorite_only=True, ) diff --git a/app/services/domains/publications/enrichment.py b/app/services/domains/publications/enrichment.py index 5c09357..381107a 100644 --- a/app/services/domains/publications/enrichment.py +++ b/app/services/domains/publications/enrichment.py @@ -1,135 +1,73 @@ from __future__ import annotations -import asyncio import logging -import time -from app.db.session import get_session_factory -from app.services.domains.publications.listing import ( - missing_pdf_items, - resolve_and_persist_oa_metadata, +from sqlalchemy.ext.asyncio import AsyncSession + +from app.services.domains.publications.pdf_queue import ( + enqueue_missing_pdf_jobs, + enqueue_retry_pdf_job, + overlay_pdf_job_state, ) from app.services.domains.publications.types import PublicationListItem -from app.settings import settings logger = logging.getLogger(__name__) -_enrichment_lock = asyncio.Lock() -_inflight_publications: set[tuple[int, int]] = set() -_recent_attempt_seconds: dict[tuple[int, int], float] = {} -_scheduled_tasks: set[asyncio.Task[None]] = set() - - -def _cooldown_seconds() -> float: - return max(float(settings.unpaywall_retry_cooldown_seconds), 1.0) - - -def _prune_recent_attempts(now_seconds: float, *, cooldown_seconds: float) -> None: - expiry = cooldown_seconds * 3 - stale_keys = [ - key for key, attempted_seconds in _recent_attempt_seconds.items() - if now_seconds - attempted_seconds >= expiry - ] - for key in stale_keys: - _recent_attempt_seconds.pop(key, None) - - -async def _claim_items( - *, - user_id: int, - items: list[PublicationListItem], - max_items: int, -) -> list[PublicationListItem]: - candidates = missing_pdf_items(items, limit=max_items) - if not candidates: - return [] - now_seconds = time.monotonic() - cooldown_seconds = _cooldown_seconds() - claimed: list[PublicationListItem] = [] - async with _enrichment_lock: - _prune_recent_attempts(now_seconds, cooldown_seconds=cooldown_seconds) - for item in candidates: - key = (user_id, item.publication_id) - attempted_seconds = _recent_attempt_seconds.get(key) - if key in _inflight_publications: - continue - if attempted_seconds is not None and now_seconds - attempted_seconds < cooldown_seconds: - continue - _inflight_publications.add(key) - _recent_attempt_seconds[key] = now_seconds - claimed.append(item) - return claimed - - -async def _release_claims(*, user_id: int, publication_ids: list[int]) -> None: - async with _enrichment_lock: - for publication_id in publication_ids: - _inflight_publications.discard((user_id, publication_id)) - - -def _on_task_done(task: asyncio.Task[None]) -> None: - _scheduled_tasks.discard(task) - try: - task.result() - except Exception: - logger.exception( - "publications.enrichment.task_failed", - extra={"event": "publications.enrichment.task_failed"}, - ) - - -async def _run_enrichment( - *, - user_id: int, - request_email: str | None, - items: list[PublicationListItem], -) -> None: - publication_ids = [item.publication_id for item in items] - try: - session_factory = get_session_factory() - async with session_factory() as db_session: - await resolve_and_persist_oa_metadata( - db_session, - rows=items, - unpaywall_email=request_email, - ) - finally: - await _release_claims(user_id=user_id, publication_ids=publication_ids) - logger.info( - "publications.enrichment.completed", - extra={ - "event": "publications.enrichment.completed", - "user_id": user_id, - "publication_count": len(items), - }, - ) - async def schedule_missing_pdf_enrichment_for_user( + db_session: AsyncSession, *, user_id: int, request_email: str | None, items: list[PublicationListItem], max_items: int, ) -> int: - claimed_items = await _claim_items(user_id=user_id, items=items, max_items=max_items) - if not claimed_items: - return 0 - task = asyncio.create_task( - _run_enrichment( - user_id=user_id, - request_email=request_email, - items=claimed_items, - ) + queued_ids = await enqueue_missing_pdf_jobs( + db_session, + user_id=user_id, + request_email=request_email, + rows=items, + max_items=max_items, ) - _scheduled_tasks.add(task) - task.add_done_callback(_on_task_done) logger.info( "publications.enrichment.scheduled", extra={ "event": "publications.enrichment.scheduled", "user_id": user_id, - "publication_count": len(claimed_items), + "publication_count": len(queued_ids), }, ) - return len(claimed_items) + return len(queued_ids) + + +async def schedule_retry_pdf_enrichment_for_row( + db_session: AsyncSession, + *, + user_id: int, + request_email: str | None, + item: PublicationListItem, +) -> bool: + queued = await enqueue_retry_pdf_job( + db_session, + user_id=user_id, + request_email=request_email, + row=item, + ) + logger.info( + "publications.enrichment.retry_scheduled", + extra={ + "event": "publications.enrichment.retry_scheduled", + "user_id": user_id, + "publication_id": item.publication_id, + "queued": queued, + }, + ) + return queued + + +async def hydrate_pdf_enrichment_state( + db_session: AsyncSession, + *, + items: list[PublicationListItem], +) -> list[PublicationListItem]: + return await overlay_pdf_job_state(db_session, rows=items) diff --git a/app/services/domains/publications/listing.py b/app/services/domains/publications/listing.py index 69893a9..376168a 100644 --- a/app/services/domains/publications/listing.py +++ b/app/services/domains/publications/listing.py @@ -16,94 +16,6 @@ from app.services.domains.publications.queries import ( unread_item_from_row, ) from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem -from app.services.domains.unpaywall.application import resolve_publication_oa_metadata -from sqlalchemy import update -from app.db.models import Publication - - -def _with_oa_overrides( - rows: list[PublicationListItem], - oa_data: dict[int, tuple[str | None, str | None]], -) -> list[PublicationListItem]: - return [ - PublicationListItem( - publication_id=row.publication_id, - scholar_profile_id=row.scholar_profile_id, - scholar_label=row.scholar_label, - title=row.title, - year=row.year, - citation_count=row.citation_count, - venue_text=row.venue_text, - pub_url=row.pub_url, - doi=(oa_data.get(row.publication_id) or (None, None))[0] or row.doi, - pdf_url=(oa_data.get(row.publication_id) or (None, None))[1] or row.pdf_url, - is_read=row.is_read, - first_seen_at=row.first_seen_at, - is_new_in_latest_run=row.is_new_in_latest_run, - ) - for row in rows - ] - - -def _resolved_fields( - *, - row: PublicationListItem, - resolved: tuple[str | None, str | None] | None, -) -> tuple[str | None, str | None]: - if resolved is None: - return row.doi, row.pdf_url - resolved_doi, resolved_pdf = resolved - return resolved_doi or row.doi, resolved_pdf or row.pdf_url - - -async def _persist_resolved_metadata( - db_session: AsyncSession, - *, - rows: list[PublicationListItem], - oa_data: dict[int, tuple[str | None, str | None]], -) -> None: - by_id = {row.publication_id: row for row in rows} - updates: list[tuple[int, str | None, str | None]] = [] - for publication_id, resolved in oa_data.items(): - existing = by_id.get(publication_id) - if existing is None: - continue - next_doi, next_pdf = _resolved_fields(row=existing, resolved=resolved) - if next_doi == existing.doi and next_pdf == existing.pdf_url: - continue - updates.append((publication_id, next_doi, next_pdf)) - for publication_id, doi, pdf_url in updates: - await db_session.execute( - update(Publication) - .where(Publication.id == publication_id) - .values(doi=doi, pdf_url=pdf_url) - ) - if updates: - await db_session.commit() - - -def missing_pdf_items( - rows: list[PublicationListItem], - *, - limit: int, -) -> list[PublicationListItem]: - bounded_limit = max(0, int(limit)) - if bounded_limit == 0: - return [] - return [row for row in rows if not row.pdf_url][:bounded_limit] - - -async def resolve_and_persist_oa_metadata( - db_session: AsyncSession, - *, - rows: list[PublicationListItem], - unpaywall_email: str | None = None, -) -> dict[int, tuple[str | None, str | None]]: - if not rows: - return {} - oa_data = await resolve_publication_oa_metadata(rows, request_email=unpaywall_email) - await _persist_resolved_metadata(db_session, rows=rows, oa_data=oa_data) - return oa_data async def list_for_user( @@ -112,7 +24,9 @@ async def list_for_user( user_id: int, mode: str = MODE_ALL, scholar_profile_id: int | None = None, + favorite_only: bool = False, limit: int = 300, + offset: int = 0, ) -> list[PublicationListItem]: resolved_mode = resolve_publication_view_mode(mode) latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) @@ -122,7 +36,9 @@ async def list_for_user( mode=resolved_mode, latest_run_id=latest_run_id, scholar_profile_id=scholar_profile_id, + favorite_only=favorite_only, limit=limit, + offset=offset, ) ) rows = [ @@ -138,22 +54,13 @@ async def retry_pdf_for_user( user_id: int, scholar_profile_id: int, publication_id: int, - unpaywall_email: str | None = None, ) -> PublicationListItem | None: - item = await get_publication_item_for_user( + return await get_publication_item_for_user( db_session, user_id=user_id, scholar_profile_id=scholar_profile_id, publication_id=publication_id, ) - if item is None: - return None - oa_data = await resolve_and_persist_oa_metadata( - db_session, - rows=[item], - unpaywall_email=unpaywall_email, - ) - return _with_oa_overrides([item], oa_data)[0] async def list_unread_for_user( @@ -168,7 +75,9 @@ async def list_unread_for_user( mode=MODE_UNREAD, latest_run_id=None, scholar_profile_id=None, + favorite_only=False, limit=limit, + offset=0, ) ) return [unread_item_from_row(row) for row in result.all()] diff --git a/app/services/domains/publications/pdf_queue.py b/app/services/domains/publications/pdf_queue.py new file mode 100644 index 0000000..b420d46 --- /dev/null +++ b/app/services/domains/publications/pdf_queue.py @@ -0,0 +1,877 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from datetime import datetime, timezone +import logging + +from sqlalchemy import Select, func, literal, or_, select, union_all +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ( + Publication, + PublicationPdfJob, + PublicationPdfJobEvent, + ScholarProfile, + ScholarPublication, + User, +) +from app.db.session import get_session_factory +from app.services.domains.publications.types import PublicationListItem +from app.services.domains.unpaywall.application import ( + FAILURE_RESOLUTION_EXCEPTION, + OaResolutionOutcome, + resolve_publication_oa_outcomes, +) +from app.settings import settings + +PDF_STATUS_UNTRACKED = "untracked" +PDF_STATUS_QUEUED = "queued" +PDF_STATUS_RUNNING = "running" +PDF_STATUS_RESOLVED = "resolved" +PDF_STATUS_FAILED = "failed" + +PDF_EVENT_QUEUED = "queued" +PDF_EVENT_ATTEMPT_STARTED = "attempt_started" +PDF_EVENT_RESOLVED = "resolved" +PDF_EVENT_FAILED = "failed" + +logger = logging.getLogger(__name__) +_scheduled_tasks: set[asyncio.Task[None]] = set() + + +@dataclass(frozen=True) +class PdfQueueListItem: + publication_id: int + title: str + doi: str | None + pdf_url: str | None + status: str + attempt_count: int + last_failure_reason: str | None + last_failure_detail: str | None + last_source: str | None + requested_by_user_id: int | None + requested_by_email: str | None + queued_at: datetime | None + last_attempt_at: datetime | None + resolved_at: datetime | None + updated_at: datetime + + +@dataclass(frozen=True) +class PdfRequeueResult: + publication_exists: bool + queued: bool + + +@dataclass(frozen=True) +class PdfBulkQueueResult: + requested_count: int + queued_count: int + + +@dataclass(frozen=True) +class PdfQueuePage: + items: list[PdfQueueListItem] + total_count: int + limit: int + offset: int + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def _publication_ids(rows: list[PublicationListItem]) -> list[int]: + return sorted({row.publication_id for row in rows}) + + +def _status_from_job(row: PublicationListItem, job: PublicationPdfJob | None) -> str: + if row.pdf_url: + return PDF_STATUS_RESOLVED + if job is None: + return PDF_STATUS_UNTRACKED + return job.status + + +def _item_from_row_and_job( + row: PublicationListItem, + job: PublicationPdfJob | None, +) -> PublicationListItem: + return PublicationListItem( + publication_id=row.publication_id, + scholar_profile_id=row.scholar_profile_id, + scholar_label=row.scholar_label, + title=row.title, + year=row.year, + citation_count=row.citation_count, + venue_text=row.venue_text, + pub_url=row.pub_url, + doi=row.doi, + pdf_url=row.pdf_url, + is_read=row.is_read, + is_favorite=row.is_favorite, + first_seen_at=row.first_seen_at, + is_new_in_latest_run=row.is_new_in_latest_run, + pdf_status=_status_from_job(row, job), + pdf_attempt_count=int(job.attempt_count) if job is not None else 0, + pdf_failure_reason=job.last_failure_reason if job is not None else None, + pdf_failure_detail=job.last_failure_detail if job is not None else None, + ) + + +def _queueable_rows( + rows: list[PublicationListItem], + *, + max_items: int, +) -> list[PublicationListItem]: + bounded = max(0, int(max_items)) + if bounded == 0: + return [] + candidates = [row for row in rows if not row.pdf_url] + return candidates[:bounded] + + +def _bounded_limit(limit: int, *, max_value: int = 500) -> int: + return max(1, min(int(limit), max_value)) + + +def _bounded_offset(offset: int) -> int: + return max(int(offset), 0) + + +def _auto_retry_interval_seconds() -> int: + return max(int(settings.pdf_auto_retry_interval_seconds), 1) + + +def _auto_retry_first_interval_seconds() -> int: + return max(int(settings.pdf_auto_retry_first_interval_seconds), 1) + + +def _auto_retry_max_attempts() -> int: + return max(int(settings.pdf_auto_retry_max_attempts), 1) + + +def _retry_interval_seconds_for_attempt_count(attempt_count: int) -> int: + if int(attempt_count) <= 1: + return _auto_retry_first_interval_seconds() + return _auto_retry_interval_seconds() + + +def _cooldown_active( + *, + last_attempt_at: datetime | None, + attempt_count: int, +) -> bool: + if last_attempt_at is None: + return False + elapsed = (_utcnow() - last_attempt_at).total_seconds() + return elapsed < _retry_interval_seconds_for_attempt_count(int(attempt_count)) + + +def _can_enqueue_job( + job: PublicationPdfJob | None, + *, + force_retry: bool, +) -> bool: + if job is None: + return True + if job.status in {PDF_STATUS_QUEUED, PDF_STATUS_RUNNING}: + return False + if force_retry: + return job.status in {PDF_STATUS_FAILED, PDF_STATUS_RESOLVED, PDF_STATUS_UNTRACKED} + if job.status == PDF_STATUS_RESOLVED: + return False + if int(job.attempt_count) >= _auto_retry_max_attempts(): + return False + if _cooldown_active( + last_attempt_at=job.last_attempt_at, + attempt_count=int(job.attempt_count), + ): + return False + return True + + +def _event_row( + *, + publication_id: int, + user_id: int | None, + event_type: str, + status: str | None, + source: str | None = None, + failure_reason: str | None = None, + message: str | None = None, +) -> PublicationPdfJobEvent: + return PublicationPdfJobEvent( + publication_id=publication_id, + user_id=user_id, + event_type=event_type, + status=status, + source=source, + failure_reason=failure_reason, + message=message, + ) + + +def _queued_job( + *, + publication_id: int, + user_id: int, +) -> PublicationPdfJob: + now = _utcnow() + return PublicationPdfJob( + publication_id=publication_id, + status=PDF_STATUS_QUEUED, + queued_at=now, + last_requested_by_user_id=user_id, + ) + + +def _mark_job_queued(job: PublicationPdfJob, *, user_id: int) -> None: + now = _utcnow() + job.status = PDF_STATUS_QUEUED + job.queued_at = now + job.last_requested_by_user_id = user_id + job.last_failure_reason = None + job.last_failure_detail = None + job.last_source = None + + +def _state_map(jobs: list[PublicationPdfJob]) -> dict[int, PublicationPdfJob]: + return {int(job.publication_id): job for job in jobs} + + +async def _jobs_for_publication_ids( + db_session: AsyncSession, + *, + publication_ids: list[int], +) -> dict[int, PublicationPdfJob]: + if not publication_ids: + return {} + result = await db_session.execute( + select(PublicationPdfJob).where(PublicationPdfJob.publication_id.in_(publication_ids)) + ) + return _state_map(list(result.scalars())) + + +async def overlay_pdf_job_state( + db_session: AsyncSession, + *, + rows: list[PublicationListItem], +) -> list[PublicationListItem]: + if not rows: + return [] + jobs = await _jobs_for_publication_ids( + db_session, + publication_ids=_publication_ids(rows), + ) + return [_item_from_row_and_job(row, jobs.get(row.publication_id)) for row in rows] + + +async def _enqueue_rows( + db_session: AsyncSession, + *, + user_id: int, + rows: list[PublicationListItem], + force_retry: bool, +) -> list[PublicationListItem]: + if not rows: + return [] + queued: list[PublicationListItem] = [] + jobs = await _jobs_for_publication_ids( + db_session, + publication_ids=_publication_ids(rows), + ) + for row in rows: + job = jobs.get(row.publication_id) + if not _can_enqueue_job(job, force_retry=force_retry): + continue + if job is None: + job = _queued_job(publication_id=row.publication_id, user_id=user_id) + jobs[row.publication_id] = job + db_session.add(job) + else: + _mark_job_queued(job, user_id=user_id) + db_session.add( + _event_row( + publication_id=row.publication_id, + user_id=user_id, + event_type=PDF_EVENT_QUEUED, + status=PDF_STATUS_QUEUED, + ) + ) + queued.append(row) + if queued: + await db_session.commit() + return queued + + +def _register_task(task: asyncio.Task[None]) -> None: + _scheduled_tasks.add(task) + + +def _drop_finished_task(task: asyncio.Task[None]) -> None: + _scheduled_tasks.discard(task) + try: + task.result() + except Exception: + logger.exception( + "publications.pdf_queue.task_failed", + extra={"event": "publications.pdf_queue.task_failed"}, + ) + + +async def _mark_attempt_started( + *, + publication_id: int, + user_id: int, +) -> None: + session_factory = get_session_factory() + async with session_factory() as db_session: + job = await db_session.get(PublicationPdfJob, publication_id) + if job is None: + job = _queued_job(publication_id=publication_id, user_id=user_id) + db_session.add(job) + job.status = PDF_STATUS_RUNNING + job.last_attempt_at = _utcnow() + job.attempt_count = int(job.attempt_count) + 1 + db_session.add( + _event_row( + publication_id=publication_id, + user_id=user_id, + event_type=PDF_EVENT_ATTEMPT_STARTED, + status=PDF_STATUS_RUNNING, + ) + ) + await db_session.commit() + + +def _failed_outcome( + *, + row: PublicationListItem, +) -> OaResolutionOutcome: + return OaResolutionOutcome( + publication_id=row.publication_id, + doi=row.doi, + pdf_url=None, + failure_reason=FAILURE_RESOLUTION_EXCEPTION, + source=None, + used_crossref=False, + ) + + +async def _fetch_outcome_for_row( + *, + row: PublicationListItem, + request_email: str | None, +) -> OaResolutionOutcome: + outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email) + outcome = outcomes.get(row.publication_id) + if outcome is not None: + return outcome + return _failed_outcome(row=row) + + +def _apply_publication_update( + publication: Publication, + *, + doi: str | None, + pdf_url: str | None, +) -> None: + if doi and publication.doi != doi: + publication.doi = doi + if pdf_url and publication.pdf_url != pdf_url: + publication.pdf_url = pdf_url + + +def _apply_job_outcome(job: PublicationPdfJob, *, outcome: OaResolutionOutcome) -> None: + job.last_source = outcome.source + if outcome.pdf_url: + job.status = PDF_STATUS_RESOLVED + job.resolved_at = _utcnow() + job.last_failure_reason = None + job.last_failure_detail = None + return + job.status = PDF_STATUS_FAILED + job.last_failure_reason = outcome.failure_reason + job.last_failure_detail = outcome.failure_reason + + +def _result_event(outcome: OaResolutionOutcome) -> tuple[str, str]: + if outcome.pdf_url: + return PDF_EVENT_RESOLVED, PDF_STATUS_RESOLVED + return PDF_EVENT_FAILED, PDF_STATUS_FAILED + + +async def _persist_outcome( + *, + publication_id: int, + user_id: int, + outcome: OaResolutionOutcome, +) -> None: + session_factory = get_session_factory() + async with session_factory() as db_session: + publication = await db_session.get(Publication, publication_id) + job = await db_session.get(PublicationPdfJob, publication_id) + if publication is None or job is None: + return + _apply_publication_update(publication, doi=outcome.doi, pdf_url=outcome.pdf_url) + _apply_job_outcome(job, outcome=outcome) + event_type, status = _result_event(outcome) + db_session.add( + _event_row( + publication_id=publication_id, + user_id=user_id, + event_type=event_type, + status=status, + source=outcome.source, + failure_reason=outcome.failure_reason, + message=outcome.failure_reason, + ) + ) + await db_session.commit() + + +async def _resolve_publication_row( + *, + user_id: int, + request_email: str | None, + row: PublicationListItem, +) -> None: + await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id) + try: + outcome = await _fetch_outcome_for_row(row=row, request_email=request_email) + except Exception as exc: # pragma: no cover - defensive network boundary + logger.warning( + "publications.pdf_queue.resolve_failed", + extra={ + "event": "publications.pdf_queue.resolve_failed", + "publication_id": row.publication_id, + "error": str(exc), + }, + ) + outcome = _failed_outcome(row=row) + await _persist_outcome( + publication_id=row.publication_id, + user_id=user_id, + outcome=outcome, + ) + + +async def _run_resolution_task( + *, + user_id: int, + request_email: str | None, + rows: list[PublicationListItem], +) -> None: + for row in rows: + await _resolve_publication_row( + user_id=user_id, + request_email=request_email, + row=row, + ) + + +def _schedule_rows( + *, + user_id: int, + request_email: str | None, + rows: list[PublicationListItem], +) -> None: + if not rows: + return + task = asyncio.create_task( + _run_resolution_task( + user_id=user_id, + request_email=request_email, + rows=rows, + ) + ) + _register_task(task) + task.add_done_callback(_drop_finished_task) + + +async def enqueue_missing_pdf_jobs( + db_session: AsyncSession, + *, + user_id: int, + request_email: str | None, + rows: list[PublicationListItem], + max_items: int, +) -> list[int]: + queueable = _queueable_rows(rows, max_items=max_items) + queued_rows = await _enqueue_rows( + db_session, + user_id=user_id, + rows=queueable, + force_retry=False, + ) + _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows) + return [row.publication_id for row in queued_rows] + + +async def enqueue_retry_pdf_job( + db_session: AsyncSession, + *, + user_id: int, + request_email: str | None, + row: PublicationListItem, +) -> bool: + queued_rows = await _enqueue_rows( + db_session, + user_id=user_id, + rows=[row], + force_retry=True, + ) + _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows) + return bool(queued_rows) + + +def _retry_item_label(display_name: str | None, scholar_id: str | None) -> str: + return str(display_name or scholar_id or "unknown") + + +def _retry_item_from_publication( + publication: Publication, + *, + link_row: tuple | None, +) -> PublicationListItem: + if link_row is None: + scholar_profile_id = 0 + scholar_label = "unknown" + is_read = True + first_seen_at = publication.created_at or _utcnow() + else: + scholar_profile_id = int(link_row[0]) + scholar_label = _retry_item_label(link_row[1], link_row[2]) + is_read = bool(link_row[3]) + first_seen_at = link_row[4] or publication.created_at or _utcnow() + return PublicationListItem( + publication_id=int(publication.id), + scholar_profile_id=scholar_profile_id, + scholar_label=scholar_label, + title=publication.title_raw, + year=publication.year, + citation_count=int(publication.citation_count or 0), + venue_text=publication.venue_text, + pub_url=publication.pub_url, + doi=publication.doi, + pdf_url=publication.pdf_url, + is_read=is_read, + first_seen_at=first_seen_at, + is_new_in_latest_run=False, + ) + + +async def _retry_item_for_publication_id( + db_session: AsyncSession, + *, + publication_id: int, +) -> PublicationListItem | None: + publication = await db_session.get(Publication, publication_id) + if publication is None: + return None + result = await db_session.execute( + select( + ScholarProfile.id, + ScholarProfile.display_name, + ScholarProfile.scholar_id, + ScholarPublication.is_read, + ScholarPublication.created_at, + ) + .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) + .where(ScholarPublication.publication_id == publication_id) + .order_by(ScholarPublication.created_at.asc()) + .limit(1) + ) + return _retry_item_from_publication(publication, link_row=result.one_or_none()) + + +async def enqueue_retry_pdf_job_for_publication_id( + db_session: AsyncSession, + *, + user_id: int, + request_email: str | None, + publication_id: int, +) -> PdfRequeueResult: + row = await _retry_item_for_publication_id( + db_session, + publication_id=publication_id, + ) + if row is None: + return PdfRequeueResult(publication_exists=False, queued=False) + queued = await enqueue_retry_pdf_job( + db_session, + user_id=user_id, + request_email=request_email, + row=row, + ) + return PdfRequeueResult(publication_exists=True, queued=queued) + + +def _queue_candidate_from_publication(publication: Publication) -> PublicationListItem: + return PublicationListItem( + publication_id=int(publication.id), + scholar_profile_id=0, + scholar_label="admin", + title=publication.title_raw, + year=publication.year, + citation_count=int(publication.citation_count or 0), + venue_text=publication.venue_text, + pub_url=publication.pub_url, + doi=publication.doi, + pdf_url=publication.pdf_url, + is_read=True, + first_seen_at=publication.created_at or _utcnow(), + is_new_in_latest_run=False, + ) + + +async def _missing_pdf_candidates( + db_session: AsyncSession, + *, + limit: int, +) -> list[PublicationListItem]: + bounded_limit = max(1, min(int(limit), 5000)) + result = await db_session.execute( + select(Publication) + .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id) + .where(Publication.pdf_url.is_(None)) + .where( + or_( + PublicationPdfJob.publication_id.is_(None), + PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]), + ) + ) + .order_by(Publication.updated_at.desc(), Publication.id.desc()) + .limit(bounded_limit) + ) + return [ + _queue_candidate_from_publication(publication) + for publication in result.scalars() + ] + + +async def enqueue_all_missing_pdf_jobs( + db_session: AsyncSession, + *, + user_id: int, + request_email: str | None, + limit: int = 1000, +) -> PdfBulkQueueResult: + candidates = await _missing_pdf_candidates(db_session, limit=limit) + queued_rows = await _enqueue_rows( + db_session, + user_id=user_id, + rows=candidates, + force_retry=True, + ) + _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows) + return PdfBulkQueueResult( + requested_count=len(candidates), + queued_count=len(queued_rows), + ) + + +def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]: + stmt = ( + select( + PublicationPdfJob.publication_id, + Publication.title_raw, + Publication.doi, + Publication.pdf_url, + PublicationPdfJob.status, + PublicationPdfJob.attempt_count, + PublicationPdfJob.last_failure_reason, + PublicationPdfJob.last_failure_detail, + PublicationPdfJob.last_source, + PublicationPdfJob.last_requested_by_user_id, + User.email, + PublicationPdfJob.queued_at, + PublicationPdfJob.last_attempt_at, + PublicationPdfJob.resolved_at, + PublicationPdfJob.updated_at, + ) + .join(Publication, Publication.id == PublicationPdfJob.publication_id) + .outerjoin(User, User.id == PublicationPdfJob.last_requested_by_user_id) + ) + if status: + stmt = stmt.where(PublicationPdfJob.status == status) + return stmt + + +def _tracked_queue_select(*, limit: int, offset: int, status: str | None) -> Select[tuple]: + return ( + _tracked_queue_select_base(status=status) + .order_by(PublicationPdfJob.updated_at.desc()) + .limit(_bounded_limit(limit)) + .offset(_bounded_offset(offset)) + ) + + +def _untracked_queue_select_base() -> Select[tuple]: + return ( + select( + Publication.id, + Publication.title_raw, + Publication.doi, + Publication.pdf_url, + literal(PDF_STATUS_UNTRACKED), + literal(0), + literal(None), + literal(None), + literal(None), + literal(None), + literal(None), + literal(None), + literal(None), + literal(None), + Publication.updated_at, + ) + .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id) + .where(Publication.pdf_url.is_(None)) + .where(PublicationPdfJob.publication_id.is_(None)) + ) + + +def _untracked_queue_select(*, limit: int, offset: int) -> Select[tuple]: + return ( + _untracked_queue_select_base() + .order_by(Publication.updated_at.desc(), Publication.id.desc()) + .limit(_bounded_limit(limit)) + .offset(_bounded_offset(offset)) + ) + + +def _all_queue_select(*, limit: int, offset: int) -> Select[tuple]: + union_stmt = union_all( + _tracked_queue_select_base(status=None), + _untracked_queue_select_base(), + ).subquery() + return ( + select(union_stmt) + .order_by(union_stmt.c.updated_at.desc()) + .limit(_bounded_limit(limit)) + .offset(_bounded_offset(offset)) + ) + + +def _tracked_queue_count_select(*, status: str | None) -> Select[tuple]: + stmt = select(func.count()).select_from(PublicationPdfJob) + if status: + stmt = stmt.where(PublicationPdfJob.status == status) + return stmt + + +def _untracked_queue_count_select() -> Select[tuple]: + return ( + select(func.count()) + .select_from(Publication) + .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id) + .where(Publication.pdf_url.is_(None)) + .where(PublicationPdfJob.publication_id.is_(None)) + ) + + +def _queue_item_from_row(row: tuple) -> PdfQueueListItem: + return PdfQueueListItem( + publication_id=int(row[0]), + title=str(row[1] or ""), + doi=row[2], + pdf_url=row[3], + status=str(row[4] or PDF_STATUS_UNTRACKED), + attempt_count=int(row[5] or 0), + last_failure_reason=row[6], + last_failure_detail=row[7], + last_source=row[8], + requested_by_user_id=int(row[9]) if row[9] is not None else None, + requested_by_email=row[10], + queued_at=row[11], + last_attempt_at=row[12], + resolved_at=row[13], + updated_at=row[14], + ) + + +async def list_pdf_queue_items( + db_session: AsyncSession, + *, + limit: int = 100, + offset: int = 0, + status: str | None = None, +) -> list[PdfQueueListItem]: + bounded_limit = _bounded_limit(limit) + bounded_offset = _bounded_offset(offset) + normalized_status = (status or "").strip().lower() or None + if normalized_status == PDF_STATUS_UNTRACKED: + result = await db_session.execute( + _untracked_queue_select( + limit=bounded_limit, + offset=bounded_offset, + ) + ) + return [_queue_item_from_row(row) for row in result.all()] + if normalized_status is None: + result = await db_session.execute( + _all_queue_select( + limit=bounded_limit, + offset=bounded_offset, + ) + ) + return [_queue_item_from_row(row) for row in result.all()] + result = await db_session.execute( + _tracked_queue_select( + limit=bounded_limit, + offset=bounded_offset, + status=normalized_status, + ) + ) + return [_queue_item_from_row(row) for row in result.all()] + + +async def count_pdf_queue_items( + db_session: AsyncSession, + *, + status: str | None = None, +) -> int: + normalized_status = (status or "").strip().lower() or None + if normalized_status == PDF_STATUS_UNTRACKED: + result = await db_session.execute(_untracked_queue_count_select()) + return int(result.scalar_one() or 0) + tracked_result = await db_session.execute( + _tracked_queue_count_select(status=normalized_status) + ) + tracked_count = int(tracked_result.scalar_one() or 0) + if normalized_status is not None: + return tracked_count + untracked_result = await db_session.execute(_untracked_queue_count_select()) + untracked_count = int(untracked_result.scalar_one() or 0) + return tracked_count + untracked_count + + +async def list_pdf_queue_page( + db_session: AsyncSession, + *, + limit: int = 100, + offset: int = 0, + status: str | None = None, +) -> PdfQueuePage: + bounded_limit = _bounded_limit(limit) + bounded_offset = _bounded_offset(offset) + items = await list_pdf_queue_items( + db_session, + limit=bounded_limit, + offset=bounded_offset, + status=status, + ) + total_count = await count_pdf_queue_items( + db_session, + status=status, + ) + return PdfQueuePage( + items=items, + total_count=total_count, + limit=bounded_limit, + offset=bounded_offset, + ) diff --git a/app/services/domains/publications/queries.py b/app/services/domains/publications/queries.py index b55d567..292ec8e 100644 --- a/app/services/domains/publications/queries.py +++ b/app/services/domains/publications/queries.py @@ -36,7 +36,9 @@ def publications_query( mode: str, latest_run_id: int | None, scholar_profile_id: int | None, + favorite_only: bool, limit: int, + offset: int = 0, ) -> Select[tuple]: scholar_label = ScholarProfile.display_name stmt = ( @@ -53,6 +55,7 @@ def publications_query( Publication.doi, Publication.pdf_url, ScholarPublication.is_read, + ScholarPublication.is_favorite, ScholarPublication.first_seen_run_id, ScholarPublication.created_at, ) @@ -60,10 +63,13 @@ def publications_query( .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) .where(ScholarProfile.user_id == user_id) .order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) + .offset(max(int(offset), 0)) .limit(limit) ) if scholar_profile_id is not None: stmt = stmt.where(ScholarProfile.id == scholar_profile_id) + if favorite_only: + stmt = stmt.where(ScholarPublication.is_favorite.is_(True)) if mode == MODE_UNREAD: stmt = stmt.where(ScholarPublication.is_read.is_(False)) if mode == MODE_LATEST: @@ -93,6 +99,7 @@ def publication_query_for_user( Publication.doi, Publication.pdf_url, ScholarPublication.is_read, + ScholarPublication.is_favorite, ScholarPublication.first_seen_run_id, ScholarPublication.created_at, ) @@ -146,6 +153,7 @@ def publication_list_item_from_row( doi, pdf_url, is_read, + is_favorite, first_seen_run_id, created_at, ) = row @@ -161,6 +169,7 @@ def publication_list_item_from_row( doi=doi, pdf_url=pdf_url, is_read=bool(is_read), + is_favorite=bool(is_favorite), first_seen_at=created_at, is_new_in_latest_run=( latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id @@ -182,6 +191,7 @@ def unread_item_from_row(row: tuple) -> UnreadPublicationItem: doi, pdf_url, _is_read, + _is_favorite, _first_seen_run_id, _created_at, ) = row diff --git a/app/services/domains/publications/read_state.py b/app/services/domains/publications/read_state.py index 37a9815..41fc31a 100644 --- a/app/services/domains/publications/read_state.py +++ b/app/services/domains/publications/read_state.py @@ -16,16 +16,20 @@ def _normalized_selection_pairs(selections: list[tuple[int, int]]) -> set[tuple[ return pairs +def _scoped_scholar_ids_query(*, user_id: int): + return ( + select(ScholarProfile.id) + .where(ScholarProfile.user_id == user_id) + .scalar_subquery() + ) + + async def mark_all_unread_as_read_for_user( db_session: AsyncSession, *, user_id: int, ) -> int: - scholar_ids = ( - select(ScholarProfile.id) - .where(ScholarProfile.user_id == user_id) - .scalar_subquery() - ) + scholar_ids = _scoped_scholar_ids_query(user_id=user_id) stmt = ( update(ScholarPublication) .where( @@ -49,11 +53,7 @@ async def mark_selected_as_read_for_user( if not normalized_pairs: return 0 - scholar_ids = ( - select(ScholarProfile.id) - .where(ScholarProfile.user_id == user_id) - .scalar_subquery() - ) + scholar_ids = _scoped_scholar_ids_query(user_id=user_id) stmt = ( update(ScholarPublication) .where( @@ -69,3 +69,26 @@ async def mark_selected_as_read_for_user( result = await db_session.execute(stmt) await db_session.commit() return int(result.rowcount or 0) + + +async def set_publication_favorite_for_user( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, + publication_id: int, + is_favorite: bool, +) -> int: + scholar_ids = _scoped_scholar_ids_query(user_id=user_id) + stmt = ( + update(ScholarPublication) + .where( + ScholarPublication.scholar_profile_id.in_(scholar_ids), + ScholarPublication.scholar_profile_id == int(scholar_profile_id), + ScholarPublication.publication_id == int(publication_id), + ) + .values(is_favorite=bool(is_favorite)) + ) + result = await db_session.execute(stmt) + await db_session.commit() + return int(result.rowcount or 0) diff --git a/app/services/domains/publications/types.py b/app/services/domains/publications/types.py index f3a12c0..d1d4bad 100644 --- a/app/services/domains/publications/types.py +++ b/app/services/domains/publications/types.py @@ -19,6 +19,11 @@ class PublicationListItem: is_read: bool first_seen_at: datetime is_new_in_latest_run: bool + is_favorite: bool = False + pdf_status: str = "untracked" + pdf_attempt_count: int = 0 + pdf_failure_reason: str | None = None + pdf_failure_detail: str | None = None @dataclass(frozen=True) diff --git a/app/services/domains/unpaywall/__init__.py b/app/services/domains/unpaywall/__init__.py index 762e664..a1baa12 100644 --- a/app/services/domains/unpaywall/__init__.py +++ b/app/services/domains/unpaywall/__init__.py @@ -1,5 +1,10 @@ from __future__ import annotations -from app.services.domains.unpaywall.application import resolve_publication_pdf_urls + +async def resolve_publication_pdf_urls(*args, **kwargs): + from app.services.domains.unpaywall.application import resolve_publication_pdf_urls as _impl + + return await _impl(*args, **kwargs) + __all__ = ["resolve_publication_pdf_urls"] diff --git a/app/services/domains/unpaywall/application.py b/app/services/domains/unpaywall/application.py index 703b138..44c7560 100644 --- a/app/services/domains/unpaywall/application.py +++ b/app/services/domains/unpaywall/application.py @@ -1,21 +1,44 @@ from __future__ import annotations +from dataclasses import dataclass import logging import re +from typing import TYPE_CHECKING from urllib.parse import unquote from app.services.domains.crossref.application import discover_doi_for_publication -from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem from app.services.domains.doi.normalize import normalize_doi +from app.services.domains.unpaywall.pdf_discovery import ( + looks_like_pdf_url, + resolve_pdf_from_landing_page, +) +from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot from app.settings import settings +if TYPE_CHECKING: + from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem + DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I) DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I) DOI_URL_RE = re.compile(r"(?:https?://)?(?:dx\.)?doi\.org/(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I) UNPAYWALL_URL_TEMPLATE = "https://api.unpaywall.org/v2/{doi}" +FAILURE_MISSING_DOI = "missing_doi" +FAILURE_NO_RECORD = "no_unpaywall_record" +FAILURE_NO_PDF = "no_pdf_found" +FAILURE_RESOLUTION_EXCEPTION = "resolution_exception" logger = logging.getLogger(__name__) +@dataclass(frozen=True) +class OaResolutionOutcome: + publication_id: int + doi: str | None + pdf_url: str | None + failure_reason: str | None + source: str | None + used_crossref: bool + + def _extract_doi_candidate(text: str | None) -> str | None: if not text: return None @@ -57,21 +80,89 @@ def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str | ) -def _payload_pdf_url(payload: dict) -> str | None: +def _payload_locations(payload: dict) -> list[dict]: + locations: list[dict] = [] best = payload.get("best_oa_location") if isinstance(best, dict): - pdf_url = best.get("url_for_pdf") - if isinstance(pdf_url, str) and pdf_url.strip(): - return pdf_url.strip() - locations = payload.get("oa_locations") - if not isinstance(locations, list): + locations.append(best) + oa_locations = payload.get("oa_locations") + if isinstance(oa_locations, list): + locations.extend(location for location in oa_locations if isinstance(location, dict)) + return locations + + +def _location_value(location: dict, key: str) -> str | None: + value = location.get(key) + if not isinstance(value, str): return None - for location in locations: - if not isinstance(location, dict): + normalized = value.strip() + return normalized or None + + +def _payload_pdf_candidates(payload: dict) -> list[str]: + candidates: list[str] = [] + seen: set[str] = set() + for location in _payload_locations(payload): + candidate = _location_value(location, "url_for_pdf") + if candidate is None or candidate in seen: continue - pdf_url = location.get("url_for_pdf") - if isinstance(pdf_url, str) and pdf_url.strip(): - return pdf_url.strip() + seen.add(candidate) + candidates.append(candidate) + return candidates + + +def _payload_landing_candidates(payload: dict) -> list[str]: + candidates: list[str] = [] + seen: set[str] = set() + for location in _payload_locations(payload): + candidate = _location_value(location, "url") + if candidate is None or candidate in seen: + continue + seen.add(candidate) + candidates.append(candidate) + return candidates + + +def _crawl_targets( + *, + payload: dict, + pdf_candidates: list[str], +) -> list[str]: + targets = _payload_landing_candidates(payload) + seen = set(targets) + for candidate in pdf_candidates: + if candidate in seen: + continue + targets.append(candidate) + seen.add(candidate) + doi = normalize_doi(str(payload.get("doi") or "")) + doi_landing_url = f"https://doi.org/{doi}" if doi else None + if doi_landing_url and doi_landing_url not in seen: + targets.append(doi_landing_url) + return targets + + +def _has_direct_payload_pdf(payload: dict) -> bool: + return any(looks_like_pdf_url(candidate) for candidate in _payload_pdf_candidates(payload)) + + +async def _resolved_pdf_url_from_payload( + payload: dict, + *, + client, +) -> str | None: + pdf_candidates = _payload_pdf_candidates(payload) + for candidate in pdf_candidates: + if looks_like_pdf_url(candidate): + return candidate + for page_url in _crawl_targets(payload=payload, pdf_candidates=pdf_candidates)[:3]: + discovered = await resolve_pdf_from_landing_page(client, page_url=page_url) + if discovered: + logger.info( + "unpaywall.pdf_discovered_from_landing", + extra={"event": "unpaywall.pdf_discovered_from_landing", "landing_url": page_url}, + ) + return discovered return None @@ -81,6 +172,7 @@ async def _fetch_unpaywall_payload_by_doi( doi: str, email: str, ) -> dict | None: + await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds) response = await client.get( UNPAYWALL_URL_TEMPLATE.format(doi=doi), params={"email": email}, @@ -130,7 +222,7 @@ async def _resolve_item_payload( payload: dict | None = None if doi: payload = await _fetch_unpaywall_payload_by_doi(client=client, doi=doi, email=email) - if payload is not None and _payload_pdf_url(payload): + if payload is not None and _has_direct_payload_pdf(payload): return payload, False if not allow_crossref or not settings.crossref_enabled: return payload, False @@ -151,9 +243,13 @@ async def _resolve_item_payload( return payload, True -def _doi_and_pdf_from_payload(payload: dict) -> tuple[str | None, str | None]: +async def _doi_and_pdf_from_payload( + payload: dict, + *, + client, +) -> tuple[str | None, str | None]: doi = normalize_doi(str(payload.get("doi") or "")) - return doi, _payload_pdf_url(payload) + return doi, await _resolved_pdf_url_from_payload(payload, client=client) def _resolution_targets(items: list[PublicationListItem]) -> list[PublicationListItem]: @@ -164,11 +260,165 @@ def _crossref_budget_value() -> int: return max(int(settings.crossref_max_lookups_per_request), 0) +def _outcome_with_failure( + *, + item: PublicationListItem, + failure_reason: str, + used_crossref: bool, +) -> OaResolutionOutcome: + return OaResolutionOutcome( + publication_id=item.publication_id, + doi=_publication_doi(item), + pdf_url=None, + failure_reason=failure_reason, + source=None, + used_crossref=used_crossref, + ) + + +def _missing_payload_failure_reason(item: PublicationListItem, *, used_crossref: bool) -> str: + if _publication_doi(item): + return FAILURE_NO_RECORD + if used_crossref: + return FAILURE_NO_RECORD + return FAILURE_MISSING_DOI + + +def _source_name(*, used_crossref: bool) -> str: + return "crossref+unpaywall" if used_crossref else "unpaywall" + + +def _outcome_from_payload( + *, + item: PublicationListItem, + doi: str | None, + pdf_url: str | None, + used_crossref: bool, +) -> OaResolutionOutcome: + return OaResolutionOutcome( + publication_id=item.publication_id, + doi=doi, + pdf_url=pdf_url, + failure_reason=None if pdf_url else FAILURE_NO_PDF, + source=_source_name(used_crossref=used_crossref), + used_crossref=used_crossref, + ) + + +def _resolved_pdf_count(outcomes: dict[int, OaResolutionOutcome]) -> int: + return sum(1 for outcome in outcomes.values() if outcome.pdf_url) + + +def _outcome_metadata(outcomes: dict[int, OaResolutionOutcome]) -> dict[int, tuple[str | None, str | None]]: + return { + publication_id: (outcome.doi, outcome.pdf_url) + for publication_id, outcome in outcomes.items() + } + + +async def _resolve_outcome_for_item( + *, + client, + item: PublicationListItem, + email: str, + allow_crossref: bool, +) -> OaResolutionOutcome: + payload, used_crossref = await _resolve_item_payload( + client=client, + item=item, + email=email, + allow_crossref=allow_crossref, + ) + if not isinstance(payload, dict): + return _outcome_with_failure( + item=item, + failure_reason=_missing_payload_failure_reason(item, used_crossref=used_crossref), + used_crossref=used_crossref, + ) + doi, pdf_url = await _doi_and_pdf_from_payload(payload, client=client) + return _outcome_from_payload( + item=item, + doi=doi, + pdf_url=pdf_url, + used_crossref=used_crossref, + ) + + +def _doi_input_count(items: list[PublicationListItem]) -> int: + return len([item for item in items if _publication_doi(item)]) + + +def _search_attempt_count(*, targets: list[PublicationListItem]) -> int: + return len([item for item in targets if not _publication_doi(item)]) + + +async def _safe_outcome_for_item( + *, + client, + item: PublicationListItem, + email: str, + allow_crossref: bool, +) -> OaResolutionOutcome: + try: + return await _resolve_outcome_for_item( + client=client, + item=item, + email=email, + allow_crossref=allow_crossref, + ) + except Exception as exc: # pragma: no cover - defensive network boundary + logger.warning( + "unpaywall.resolve_item_failed", + extra={ + "event": "unpaywall.resolve_item_failed", + "publication_id": item.publication_id, + "error": str(exc), + }, + ) + return _outcome_with_failure( + item=item, + failure_reason=FAILURE_RESOLUTION_EXCEPTION, + used_crossref=allow_crossref and settings.crossref_enabled, + ) + + +async def _resolve_outcomes_with_client( + *, + client, + targets: list[PublicationListItem], + email: str, +) -> dict[int, OaResolutionOutcome]: + outcomes: dict[int, OaResolutionOutcome] = {} + crossref_budget = _crossref_budget_value() + crossref_lookups = 0 + for item in targets: + allow_crossref = crossref_budget > 0 and crossref_lookups < crossref_budget + outcome = await _safe_outcome_for_item( + client=client, + item=item, + email=email, + allow_crossref=allow_crossref, + ) + if outcome.used_crossref: + crossref_lookups += 1 + outcomes[item.publication_id] = outcome + return outcomes + + async def resolve_publication_oa_metadata( items: list[PublicationListItem], *, request_email: str | None = None, ) -> dict[int, tuple[str | None, str | None]]: + outcomes = await resolve_publication_oa_outcomes(items, request_email=request_email) + return _outcome_metadata(outcomes) + + +async def resolve_publication_oa_outcomes( + items: list[PublicationListItem], + *, + request_email: str | None = None, +) -> dict[int, OaResolutionOutcome]: if not settings.unpaywall_enabled: return {} email = _email_for_request(request_email) @@ -178,35 +428,21 @@ async def resolve_publication_oa_metadata( import httpx timeout_seconds = max(float(settings.unpaywall_timeout_seconds), 0.5) - resolved: dict[int, tuple[str | None, str | None]] = {} - crossref_budget = _crossref_budget_value() - crossref_lookups = 0 targets = _resolution_targets(items)[: max(int(settings.unpaywall_max_items_per_request), 0)] async with httpx.AsyncClient(timeout=timeout_seconds) as client: - for item in targets: - allow_crossref = crossref_budget > 0 and crossref_lookups < crossref_budget - payload, used_crossref = await _resolve_item_payload( - client=client, - item=item, - email=email, - allow_crossref=allow_crossref, - ) - if used_crossref: - crossref_lookups += 1 - if not isinstance(payload, dict): - continue - resolved[item.publication_id] = _doi_and_pdf_from_payload(payload) - resolved_count = sum(1 for _doi, pdf in resolved.values() if pdf) - doi_input_count = len([item for item in items if _publication_doi(item)]) - target_doi_count = len([item for item in targets if _publication_doi(item)]) + outcomes = await _resolve_outcomes_with_client( + client=client, + targets=targets, + email=email, + ) _log_resolution_summary( publication_count=len(items), - doi_input_count=doi_input_count, - search_attempt_count=max(0, len(targets) - target_doi_count), - resolved_pdf_count=resolved_count, + doi_input_count=_doi_input_count(items), + search_attempt_count=_search_attempt_count(targets=targets), + resolved_pdf_count=_resolved_pdf_count(outcomes), email=email, ) - return resolved + return outcomes async def resolve_publication_pdf_urls( diff --git a/app/services/domains/unpaywall/pdf_discovery.py b/app/services/domains/unpaywall/pdf_discovery.py new file mode 100644 index 0000000..5f82a58 --- /dev/null +++ b/app/services/domains/unpaywall/pdf_discovery.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +from html.parser import HTMLParser +import re +from urllib.parse import urljoin, urlparse + +from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot +from app.settings import settings + +PDF_MIME = "application/pdf" +URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.I) + + +class _LandingPdfParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.base_href: str | None = None + self.candidates: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + attrs_map = {key.lower(): (value or "").strip() for key, value in attrs} + if tag == "base": + self.base_href = attrs_map.get("href") or self.base_href + return + if tag == "meta": + self._append_meta_candidate(attrs_map) + return + if tag == "link": + self._append_link_candidate(attrs_map) + return + if tag == "a": + self._append_anchor_candidate(attrs_map) + + def _append_meta_candidate(self, attrs_map: dict[str, str]) -> None: + meta_name = (attrs_map.get("name") or attrs_map.get("property") or "").lower() + if meta_name != "citation_pdf_url": + return + content = attrs_map.get("content") + if content: + self.candidates.append(content) + + def _append_link_candidate(self, attrs_map: dict[str, str]) -> None: + href = attrs_map.get("href") + link_type = (attrs_map.get("type") or "").lower() + if href and "pdf" in link_type: + self.candidates.append(href) + + def _append_anchor_candidate(self, attrs_map: dict[str, str]) -> None: + href = attrs_map.get("href") + if href: + self.candidates.append(href) + + +def looks_like_pdf_url(url: str | None) -> bool: + if not isinstance(url, str): + return False + value = url.strip() + if not value: + return False + parsed = urlparse(value) + path = (parsed.path or "").lower() + query = (parsed.query or "").lower() + return path.endswith(".pdf") or ".pdf" in query + + +def _normalized_candidate_urls(*, page_url: str, html: str) -> list[str]: + parser = _LandingPdfParser() + parser.feed(html) + parser.close() + base_url = urljoin(page_url, parser.base_href) if parser.base_href else page_url + raw_candidates = [*parser.candidates, *_text_url_candidates(html)] + deduped: list[str] = [] + seen: set[str] = set() + for raw in raw_candidates: + absolute = urljoin(base_url, raw.strip()) + parsed = urlparse(absolute) + if parsed.scheme not in {"http", "https"}: + continue + if absolute in seen: + continue + seen.add(absolute) + deduped.append(absolute) + return sorted(deduped, key=_candidate_sort_key) + + +def _text_url_candidates(html: str) -> list[str]: + candidates: list[str] = [] + for match in URL_RE.findall(html): + cleaned = match.rstrip(".,);]}>") + if "pdf" not in cleaned.lower(): + continue + candidates.append(cleaned) + return candidates + + +def _candidate_sort_key(candidate: str) -> tuple[int, str]: + lowered = candidate.lower() + if looks_like_pdf_url(candidate): + return (0, lowered) + if any(token in lowered for token in ("doi", "full", "article", "download")): + return (1, lowered) + return (2, lowered) + + +def _is_html_response(response) -> bool: + content_type = str(response.headers.get("content-type") or "").lower() + return "text/html" in content_type or "application/xhtml+xml" in content_type + + +async def _fetch_page_html(client, *, page_url: str) -> str | None: + await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds) + response = await client.get(page_url, follow_redirects=True) + if response.status_code != 200 or not _is_html_response(response): + return None + text = response.text or "" + return text[: max(int(settings.unpaywall_pdf_discovery_max_html_bytes), 0)] + + +async def _candidate_is_pdf(client, *, candidate_url: str) -> bool: + if looks_like_pdf_url(candidate_url): + return True + await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds) + response = await client.get(candidate_url, follow_redirects=True) + content_type = str(response.headers.get("content-type") or "").lower() + return response.status_code == 200 and PDF_MIME in content_type + + +def _candidate_limit() -> int: + return max(int(settings.unpaywall_pdf_discovery_max_candidates), 1) + + +async def _resolve_pdf_from_candidate_page(client, *, candidate_url: str) -> str | None: + html = await _fetch_page_html(client, page_url=candidate_url) + if not html: + return None + nested_candidates = _normalized_candidate_urls(page_url=candidate_url, html=html) + for nested in nested_candidates[: _candidate_limit()]: + if await _candidate_is_pdf(client, candidate_url=nested): + return nested + return None + + +async def resolve_pdf_from_landing_page(client, *, page_url: str) -> str | None: + if not settings.unpaywall_pdf_discovery_enabled: + return None + html = await _fetch_page_html(client, page_url=page_url) + if not html: + return None + candidates = _normalized_candidate_urls(page_url=page_url, html=html) + for candidate in candidates[: _candidate_limit()]: + if await _candidate_is_pdf(client, candidate_url=candidate): + return candidate + nested_pdf = await _resolve_pdf_from_candidate_page(client, candidate_url=candidate) + if nested_pdf: + return nested_pdf + return None diff --git a/app/services/domains/unpaywall/rate_limit.py b/app/services/domains/unpaywall/rate_limit.py new file mode 100644 index 0000000..76d8a3d --- /dev/null +++ b/app/services/domains/unpaywall/rate_limit.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import asyncio +import time + +_REQUEST_LOCK = asyncio.Lock() +_LAST_REQUEST_AT = 0.0 + + +async def wait_for_unpaywall_slot(*, min_interval_seconds: float) -> None: + global _LAST_REQUEST_AT + interval = max(float(min_interval_seconds), 0.0) + async with _REQUEST_LOCK: + elapsed = time.monotonic() - _LAST_REQUEST_AT + remaining = interval - elapsed + if remaining > 0: + await asyncio.sleep(remaining) + _LAST_REQUEST_AT = time.monotonic() diff --git a/app/settings.py b/app/settings.py index 82988ac..43871cf 100644 --- a/app/settings.py +++ b/app/settings.py @@ -232,8 +232,21 @@ class Settings: unpaywall_enabled: bool = _env_bool("UNPAYWALL_ENABLED", True) unpaywall_email: str = _env_str("UNPAYWALL_EMAIL", "") unpaywall_timeout_seconds: float = _env_float("UNPAYWALL_TIMEOUT_SECONDS", 4.0) + unpaywall_min_interval_seconds: float = _env_float("UNPAYWALL_MIN_INTERVAL_SECONDS", 0.6) unpaywall_max_items_per_request: int = _env_int("UNPAYWALL_MAX_ITEMS_PER_REQUEST", 20) unpaywall_retry_cooldown_seconds: int = _env_int("UNPAYWALL_RETRY_COOLDOWN_SECONDS", 1800) + pdf_auto_retry_interval_seconds: int = _env_int( + "PDF_AUTO_RETRY_INTERVAL_SECONDS", + 86_400, + ) + pdf_auto_retry_first_interval_seconds: int = _env_int( + "PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS", + 3_600, + ) + pdf_auto_retry_max_attempts: int = _env_int("PDF_AUTO_RETRY_MAX_ATTEMPTS", 3) + unpaywall_pdf_discovery_enabled: bool = _env_bool("UNPAYWALL_PDF_DISCOVERY_ENABLED", True) + unpaywall_pdf_discovery_max_candidates: int = _env_int("UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES", 5) + unpaywall_pdf_discovery_max_html_bytes: int = _env_int("UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES", 500_000) crossref_enabled: bool = _env_bool("CROSSREF_ENABLED", True) crossref_max_rows: int = _env_int("CROSSREF_MAX_ROWS", 10) crossref_timeout_seconds: float = _env_float("CROSSREF_TIMEOUT_SECONDS", 8.0) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..33a4a56 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,30 @@ +# Documentation Index + +Use this directory for project documentation outside `README.md` and `agents.md`. + +## Deploy + +- `docs/deploy/quickstart.md`: deployment modes, local dev startup, and quality command reference. + +## Reference + +- `docs/reference/environment.md`: complete environment-variable catalog and contract/audit notes. +- `docs/reference/api_contract.md`: API envelope contract and endpoint semantics. + +## Architecture + +- `docs/architecture/domain_boundaries.md`: domain boundaries, user-scoped data rules, and frontend behavior notes. + +## Operations + +- `docs/ops/scrape_safety_runbook.md`: scrape cooldown and blocked-IP operations. +- `docs/ops/db_runbook.md`: backup, restore, and verification workflows. +- `docs/ops/migration_checklist.md`: migration planning and rollout checklist. + +## Frontend + +- `docs/frontend/theme_phase0_inventory.md`: theme-token inventory and adoption notes. + +## Contribution + +- `docs/contributing.md`: contribution policy and merge checklist. diff --git a/docs/architecture/domain_boundaries.md b/docs/architecture/domain_boundaries.md new file mode 100644 index 0000000..0a169e6 --- /dev/null +++ b/docs/architecture/domain_boundaries.md @@ -0,0 +1,25 @@ +# Domain Boundaries + +## Data Model Rules + +- Scholar tracking is user-scoped. +- Publications are global/deduplicated records. +- Read/favorite/visibility state stays on scholar-publication link rows. + +## Service Boundaries + +Canonical business logic belongs in `app/services/domains/*`. + +- `app/services/domains/ingestion/*`: run orchestration, continuation queue, scheduler, and scrape safety. +- `app/services/domains/scholar/*`: fail-fast scholar parsing and source fetch adapters. +- `app/services/domains/scholars/*`: scholar CRUD, profile image, and name-search controls. +- `app/services/domains/publications/*`: listing/read-state, favorite toggles, enrichment scheduling, and retry paths. +- `app/services/domains/crossref/*` + `app/services/domains/unpaywall/*`: DOI/OA enrichment with bounded pacing. +- `app/services/domains/runs/*`: run history and continuation queue operations. +- `app/services/domains/portability/*`: import/export workflows. + +## Frontend Behavior Notes + +- Mobile primary nav is in a left drawer and closes on route change or logout. +- Long list views use internal scroll containers. +- Name search remains intentionally constrained due upstream anti-bot behavior; production onboarding should prefer scholar ID/profile URL. diff --git a/CONTRIBUTING.md b/docs/contributing.md similarity index 100% rename from CONTRIBUTING.md rename to docs/contributing.md diff --git a/docs/deploy/quickstart.md b/docs/deploy/quickstart.md new file mode 100644 index 0000000..b959889 --- /dev/null +++ b/docs/deploy/quickstart.md @@ -0,0 +1,68 @@ +# Deploy and Dev Quickstart + +## Required Setup + +1. Copy `.env.example` to `.env`. +2. Set `POSTGRES_PASSWORD`. +3. Set `SESSION_SECRET_KEY`. + +## Deploy Method A: Prebuilt Image + +```bash +docker compose pull +docker compose up -d +``` + +Open: +- App/API: `http://localhost:8000` +- Health endpoint: `http://localhost:8000/healthz` + +Upgrade: + +```bash +docker compose pull +docker compose up -d +``` + +## Deploy Method B: Local Source + Dev Frontend + +```bash +docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build +``` + +Open: +- API: `http://localhost:8000` +- Frontend dev server: `http://localhost:5173` + +Stop: + +```bash +docker compose -f docker-compose.yml -f docker-compose.dev.yml down +``` + +## Quality Commands + +Backend: + +```bash +docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest tests/unit +docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app uv run pytest -m integration +``` + +Frontend: + +```bash +cd frontend +npm install +npm run typecheck +npm run test:run +npm run build +``` + +Repository checks: + +```bash +python3 scripts/check_frontend_api_contract.py +python3 scripts/check_env_contract.py +./scripts/check_no_generated_artifacts.sh +``` diff --git a/frontend/src/theme/THEME_PHASE0_INVENTORY.md b/docs/frontend/theme_phase0_inventory.md similarity index 100% rename from frontend/src/theme/THEME_PHASE0_INVENTORY.md rename to docs/frontend/theme_phase0_inventory.md diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..72a5e53 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,12 @@ +--- +slug: / +--- + +# scholarr Documentation + +- Start here: `docs/deploy/quickstart.md` +- Runtime config: `docs/reference/environment.md` +- API contract: `docs/reference/api_contract.md` +- Domain boundaries: `docs/architecture/domain_boundaries.md` +- Ops runbooks: `docs/ops/` +- Frontend theme inventory: `docs/frontend/theme_phase0_inventory.md` diff --git a/docs/ops/db_runbook.md b/docs/ops/db_runbook.md new file mode 100644 index 0000000..68cbbfc --- /dev/null +++ b/docs/ops/db_runbook.md @@ -0,0 +1,100 @@ +# Database Operations Runbook + +This runbook defines backup, restore, and verification procedures for the +PostgreSQL database used by `scholarr`. + +## Objectives + +- Keep data loss bounded via scheduled backups. +- Provide repeatable restore procedures. +- Verify backups by running regular restore drills. + +Recommended starting targets: + +- `RPO`: 24 hours (maximum acceptable data loss). +- `RTO`: 60 minutes (maximum acceptable restore time). + +## Backup Strategy + +Use logical backups from the running `db` compose service. + +- Custom-format backup (recommended for restore flexibility): + +```bash +scripts/db/backup_full.sh +``` + +- Plain SQL backup: + +```bash +scripts/db/backup_full.sh --plain +``` + +Optional environment variables: + +- `BACKUP_DIR`: destination directory (default: `/backups`) +- `BACKUP_PREFIX`: backup filename prefix (default: `scholarr`) +- `USE_DEV_COMPOSE=1`: include `docker-compose.dev.yml` + +## Restore Strategy + +Restore from a `.dump` (custom format) or `.sql` file into the running `db` +compose service. + +- Restore without schema wipe: + +```bash +scripts/db/restore_dump.sh --file backups/scholarr_20260220T120000Z.dump +``` + +- Restore with full `public` schema reset: + +```bash +scripts/db/restore_dump.sh --file backups/scholarr_20260220T120000Z.dump --wipe-public +``` + +## Safety Checklist Before Restore + +1. Pause writes (stop app/scheduler or set maintenance mode). +2. Create a fresh pre-restore backup. +3. Validate target dump checksum/size. +4. Confirm operator, scope, and rollback plan. + +## Post-Restore Verification + +1. Run migrations head check. +2. Run health endpoint checks. +3. Verify table row counts for core tables: + - `users` + - `scholar_profiles` + - `publications` + - `scholar_publications` + - `crawl_runs` +4. Run integrity report and require zero failures: + +```bash +python3 scripts/db/check_integrity.py +``` + +5. Run API smoke checks for scholars/publications/runs pages. + +## Data Cleanup and Repair + +Use the audited repair job for scholar-publication relinking cleanup: + +```bash +python3 scripts/db/repair_publication_links.py --user-id --requested-by "" --apply +``` + +Dry-run first, then re-run with `--apply` once scope and summary counts match expectation. + +If cleanup changes schema assumptions, follow `docs/ops/migration_checklist.md` before any migration rollout. + +## Restore Drill Cadence + +- Run at least one full restore drill per month. +- Record: + - backup artifact used, + - restore start/end timestamps, + - issues encountered, + - achieved `RTO`. diff --git a/docs/ops/migration_checklist.md b/docs/ops/migration_checklist.md new file mode 100644 index 0000000..106b9fc --- /dev/null +++ b/docs/ops/migration_checklist.md @@ -0,0 +1,48 @@ +# Migration Checklist + +Use this checklist for every schema/data migration. + +## 1. Design Review + +- Define change type: `expand`, `backfill`, `contract`. +- Document expected lock behavior and index impact. +- Confirm backward compatibility with currently deployed app version. +- Define rollback strategy before implementation. + +## 2. Pre-Migration Controls + +- Capture a fresh backup before applying migration. +- Confirm migration head revision and working tree cleanliness. +- Prepare validation queries for new/changed tables and indexes. +- Identify high-risk tables (large row count, hot write paths). + +## 3. Implementation Standards + +- Keep migrations idempotent when feasible. +- Prefer additive steps first (`nullable`, new index, new table). +- For destructive changes, separate into later contract migration. +- Avoid large blocking rewrites in a single deployment step. + +## 4. Verification + +- Apply migration in staging against production-like snapshot. +- Verify: + - expected tables/columns/indexes, + - app startup and health endpoint, + - affected API flows, + - data consistency queries. + +## 5. Rollout and Recovery + +- Apply migration during planned window. +- Monitor logs/errors and DB metrics during rollout. +- If rollback needed: + - follow downgrade/recovery runbook, + - restore from backup if downgrade is unsafe, + - document incident timeline. + +## 6. Post-Migration Tasks + +- Update `README.md` / `.env.example` / ops docs. +- Add/update integration tests for new schema assumptions. +- Record migration notes in changelog. diff --git a/docs/reference/api_contract.md b/docs/reference/api_contract.md new file mode 100644 index 0000000..1994943 --- /dev/null +++ b/docs/reference/api_contract.md @@ -0,0 +1,29 @@ +# API Contract + +## Envelope Invariant + +All API responses under `/api/v1` use one of these envelopes: + +- Success: `{"data": ..., "meta": {"request_id": "..."}}` +- Error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}` + +`meta.request_id` must be present for success and error responses. + +## Publications Semantics + +- `GET /api/v1/publications` supports `mode=all|unread|latest`. +- `mode=new` is currently accepted as a compatibility alias for `latest`. +- Pagination controls: `page`, `page_size` (with backward-compatible `limit`/`offset` support). +- Pagination fields in response: `page`, `page_size`, `has_prev`, `has_next`, `total_count`. +- `unread` represents read-state (`is_read=false`). +- `latest` represents discovery-state (`first seen in the latest completed run`). + +Publication payloads expose: +- `pub_url` (canonical scholar detail URL) +- `doi` (normalized DOI) +- `pdf_url` (resolved OA PDF when available) + +## Scholar Portability + +- `GET /api/v1/scholars/export` exports tracked scholars and scholar-publication link state. +- `POST /api/v1/scholars/import` imports that payload while preserving global publication deduplication. diff --git a/docs/reference/environment.md b/docs/reference/environment.md new file mode 100644 index 0000000..0dafb9b --- /dev/null +++ b/docs/reference/environment.md @@ -0,0 +1,68 @@ +# Environment Reference and Audit + +This document is the source-of-truth audit for what remains configurable versus internal defaults. + +## Decision Summary + +- Keep in `.env.example`: all runtime and compose controls currently exposed by `app/settings.py` plus compose/bootstrap extras. +- Move to internal defaults only: none at this time. +- Internal fallback behavior still exists in code to keep local/test startup resilient when values are omitted. + +## Configurable Variables (By Section) + +### Compose + Database + +`POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `DATABASE_URL`, `TEST_DATABASE_URL`, `SCHOLARR_IMAGE` + +### App Runtime + Networking + +`APP_NAME`, `APP_HOST`, `APP_PORT`, `APP_HOST_PORT`, `APP_RELOAD`, `MIGRATE_ON_START`, `FRONTEND_ENABLED`, `FRONTEND_DIST_DIR` + +### Database Pool + +`DATABASE_POOL_MODE`, `DATABASE_POOL_SIZE`, `DATABASE_POOL_MAX_OVERFLOW`, `DATABASE_POOL_TIMEOUT_SECONDS` + +### Frontend Dev Overrides + +`FRONTEND_HOST_PORT`, `CHOKIDAR_USEPOLLING`, `VITE_DEV_API_PROXY_TARGET` + +### Auth + Session + +`SESSION_SECRET_KEY`, `SESSION_COOKIE_SECURE`, `LOGIN_RATE_LIMIT_ATTEMPTS`, `LOGIN_RATE_LIMIT_WINDOW_SECONDS` + +### HTTP Security Headers + CSP + +`SECURITY_HEADERS_ENABLED`, `SECURITY_X_CONTENT_TYPE_OPTIONS`, `SECURITY_X_FRAME_OPTIONS`, `SECURITY_REFERRER_POLICY`, `SECURITY_PERMISSIONS_POLICY`, `SECURITY_CROSS_ORIGIN_OPENER_POLICY`, `SECURITY_CROSS_ORIGIN_RESOURCE_POLICY`, `SECURITY_CSP_ENABLED`, `SECURITY_CSP_POLICY`, `SECURITY_CSP_DOCS_POLICY`, `SECURITY_CSP_REPORT_ONLY`, `SECURITY_STRICT_TRANSPORT_SECURITY_ENABLED`, `SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE`, `SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS`, `SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD` + +### Logging + +`LOG_LEVEL`, `LOG_FORMAT`, `LOG_REQUESTS`, `LOG_UVICORN_ACCESS`, `LOG_REQUEST_SKIP_PATHS`, `LOG_REDACT_FIELDS` + +### Scheduler + Ingestion Safety + +`SCHEDULER_ENABLED`, `SCHEDULER_TICK_SECONDS`, `SCHEDULER_QUEUE_BATCH_SIZE`, `INGESTION_AUTOMATION_ALLOWED`, `INGESTION_MANUAL_RUN_ALLOWED`, `INGESTION_MIN_RUN_INTERVAL_MINUTES`, `INGESTION_MIN_REQUEST_DELAY_SECONDS`, `INGESTION_NETWORK_ERROR_RETRIES`, `INGESTION_RETRY_BACKOFF_SECONDS`, `INGESTION_MAX_PAGES_PER_SCHOLAR`, `INGESTION_PAGE_SIZE`, `INGESTION_ALERT_BLOCKED_FAILURE_THRESHOLD`, `INGESTION_ALERT_NETWORK_FAILURE_THRESHOLD`, `INGESTION_ALERT_RETRY_SCHEDULED_THRESHOLD`, `INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS`, `INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS`, `INGESTION_CONTINUATION_QUEUE_ENABLED`, `INGESTION_CONTINUATION_BASE_DELAY_SECONDS`, `INGESTION_CONTINUATION_MAX_DELAY_SECONDS`, `INGESTION_CONTINUATION_MAX_ATTEMPTS` + +### Scholar Images + Name Search Safety + +`SCHOLAR_IMAGE_UPLOAD_DIR`, `SCHOLAR_IMAGE_UPLOAD_MAX_BYTES`, `SCHOLAR_NAME_SEARCH_ENABLED`, `SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS`, `SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS`, `SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES`, `SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS`, `SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS`, `SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD`, `SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS`, `SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD`, `SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD` + +### OA Enrichment + PDF Resolution + +`UNPAYWALL_ENABLED`, `UNPAYWALL_EMAIL`, `UNPAYWALL_TIMEOUT_SECONDS`, `UNPAYWALL_MIN_INTERVAL_SECONDS`, `UNPAYWALL_MAX_ITEMS_PER_REQUEST`, `UNPAYWALL_RETRY_COOLDOWN_SECONDS`, `UNPAYWALL_PDF_DISCOVERY_ENABLED`, `UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES`, `UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES`, `PDF_AUTO_RETRY_INTERVAL_SECONDS`, `PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS`, `PDF_AUTO_RETRY_MAX_ATTEMPTS`, `CROSSREF_ENABLED`, `CROSSREF_MAX_ROWS`, `CROSSREF_TIMEOUT_SECONDS`, `CROSSREF_MIN_INTERVAL_SECONDS`, `CROSSREF_MAX_LOOKUPS_PER_REQUEST` + +### Startup Bootstrap + DB Wait + +`BOOTSTRAP_ADMIN_ON_START`, `BOOTSTRAP_ADMIN_EMAIL`, `BOOTSTRAP_ADMIN_PASSWORD`, `BOOTSTRAP_ADMIN_FORCE_PASSWORD`, `DB_WAIT_TIMEOUT_SECONDS`, `DB_WAIT_INTERVAL_SECONDS` + +## Drift Guard + +CI enforces env parity with: + +```bash +python3 scripts/check_env_contract.py +``` + +The check fails when: +- `app/settings.py` references a variable missing from `.env.example`. +- `.env.example` contains an unknown key (outside the approved compose/bootstrap extras). +- `.env.example` contains duplicate keys. diff --git a/frontend/index.html b/frontend/index.html index 91a3496..c23810e 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -3,6 +3,12 @@ + + + + + + scholarr + + + + diff --git a/frontend/src/components/ui/AppRefreshButton.vue b/frontend/src/components/ui/AppRefreshButton.vue new file mode 100644 index 0000000..f340217 --- /dev/null +++ b/frontend/src/components/ui/AppRefreshButton.vue @@ -0,0 +1,61 @@ + + + diff --git a/frontend/src/components/ui/AppTabs.vue b/frontend/src/components/ui/AppTabs.vue new file mode 100644 index 0000000..9f29b60 --- /dev/null +++ b/frontend/src/components/ui/AppTabs.vue @@ -0,0 +1,51 @@ + + + diff --git a/frontend/src/components/ui/AppThemePicker.vue b/frontend/src/components/ui/AppThemePicker.vue new file mode 100644 index 0000000..4fdf9f6 --- /dev/null +++ b/frontend/src/components/ui/AppThemePicker.vue @@ -0,0 +1,95 @@ + + + diff --git a/frontend/src/features/admin_dbops/index.ts b/frontend/src/features/admin_dbops/index.ts new file mode 100644 index 0000000..e3269e1 --- /dev/null +++ b/frontend/src/features/admin_dbops/index.ts @@ -0,0 +1,153 @@ +import { apiRequest } from "@/lib/api/client"; + +export interface AdminDbIntegrityCheck { + name: string; + count: number; + severity: "warning" | "failure" | "metric"; + message: string; +} + +export interface AdminDbIntegrityReport { + status: "ok" | "warning" | "failed"; + checked_at: string; + failures: string[]; + warnings: string[]; + checks: AdminDbIntegrityCheck[]; +} + +export interface AdminDbRepairJob { + id: number; + job_name: string; + requested_by: string | null; + dry_run: boolean; + status: string; + scope: Record; + summary: Record; + error_text: string | null; + started_at: string | null; + finished_at: string | null; + created_at: string; + updated_at: string; +} + +export interface AdminPdfQueueItem { + publication_id: number; + title: string; + doi: string | null; + pdf_url: string | null; + status: string; + attempt_count: number; + last_failure_reason: string | null; + last_failure_detail: string | null; + last_source: string | null; + requested_by_user_id: number | null; + requested_by_email: string | null; + queued_at: string | null; + last_attempt_at: string | null; + resolved_at: string | null; + updated_at: string; +} + +export interface AdminPdfQueuePage { + items: AdminPdfQueueItem[]; + total_count: number; + page: number; + page_size: number; + has_next: boolean; + has_prev: boolean; +} + +export interface AdminPdfQueueRequeueResult { + publication_id: number; + queued: boolean; + status: string; + message: string; +} + +export interface AdminPdfQueueBulkEnqueueResult { + requested_count: number; + queued_count: number; + message: string; +} + +export interface TriggerPublicationLinkRepairPayload { + scope_mode?: "single_user" | "all_users"; + user_id?: number; + scholar_profile_ids?: number[]; + dry_run?: boolean; + gc_orphan_publications?: boolean; + requested_by?: string; + confirmation_text?: string; +} + +export interface TriggerPublicationLinkRepairResult { + job_id: number; + status: string; + scope: Record; + summary: Record; +} + +export async function getAdminDbIntegrityReport(): Promise { + const response = await apiRequest("/admin/db/integrity", { method: "GET" }); + return response.data; +} + +export async function listAdminDbRepairJobs(limit = 30): Promise { + const parsedLimit = Number.isFinite(limit) ? Math.max(1, Math.min(200, Math.trunc(limit))) : 30; + const response = await apiRequest<{ jobs: AdminDbRepairJob[] }>( + `/admin/db/repair-jobs?limit=${parsedLimit}`, + { method: "GET" }, + ); + return response.data.jobs; +} + +export async function triggerPublicationLinkRepair( + payload: TriggerPublicationLinkRepairPayload, +): Promise { + const response = await apiRequest( + "/admin/db/repairs/publication-links", + { + method: "POST", + body: payload, + }, + ); + return response.data; +} + +export async function listAdminPdfQueue( + page = 1, + pageSize = 100, + status: string | null = null, +): Promise { + const parsedPage = Number.isFinite(page) ? Math.max(1, Math.trunc(page)) : 1; + const parsedPageSize = Number.isFinite(pageSize) ? Math.max(1, Math.min(500, Math.trunc(pageSize))) : 100; + const params = new URLSearchParams(); + params.set("page", String(parsedPage)); + params.set("page_size", String(parsedPageSize)); + if (status && status.trim().length > 0) { + params.set("status", status.trim().toLowerCase()); + } + const response = await apiRequest( + `/admin/db/pdf-queue?${params.toString()}`, + { method: "GET" }, + ); + return response.data; +} + +export async function requeueAdminPdfLookup(publicationId: number): Promise { + const id = Number.isFinite(publicationId) ? Math.trunc(publicationId) : 0; + const response = await apiRequest( + `/admin/db/pdf-queue/${Math.max(1, id)}/requeue`, + { method: "POST" }, + ); + return response.data; +} + +export async function requeueAllAdminPdfLookups(limit = 1000): Promise { + const parsedLimit = Number.isFinite(limit) ? Math.max(1, Math.min(5000, Math.trunc(limit))) : 1000; + const response = await apiRequest( + `/admin/db/pdf-queue/requeue-all?limit=${parsedLimit}`, + { method: "POST" }, + ); + return response.data; +} diff --git a/frontend/src/features/dashboard/index.ts b/frontend/src/features/dashboard/index.ts index ccf3ee0..ae4e538 100644 --- a/frontend/src/features/dashboard/index.ts +++ b/frontend/src/features/dashboard/index.ts @@ -41,7 +41,7 @@ function countQueueStatuses(statuses: string[]): QueueHealth { export async function fetchDashboardSnapshot(): Promise { const [publications, runsPayload, queueItems] = await Promise.all([ - listPublications({ mode: "latest", limit: 20 }), + listPublications({ mode: "latest", page: 1, pageSize: 20 }), listRuns({ limit: 20 }), listQueueItems(200), ]); diff --git a/frontend/src/features/publications/index.ts b/frontend/src/features/publications/index.ts index a36c7e5..2eddfbb 100644 --- a/frontend/src/features/publications/index.ts +++ b/frontend/src/features/publications/index.ts @@ -13,26 +13,39 @@ export interface PublicationItem { pub_url: string | null; doi: string | null; pdf_url: string | null; + pdf_status: "untracked" | "queued" | "running" | "resolved" | "failed"; + pdf_attempt_count: number; + pdf_failure_reason: string | null; + pdf_failure_detail: string | null; is_read: boolean; + is_favorite: boolean; first_seen_at: string; is_new_in_latest_run: boolean; } export interface PublicationsResult { mode: PublicationMode; + favorite_only: boolean; selected_scholar_profile_id: number | null; unread_count: number; + favorites_count: number; latest_count: number; // Compatibility alias for latest_count; retained while API clients migrate. new_count: number; total_count: number; + page: number; + page_size: number; + has_next: boolean; + has_prev: boolean; publications: PublicationItem[]; } export interface PublicationsQuery { mode?: PublicationMode; + favoriteOnly?: boolean; scholarProfileId?: number; - limit?: number; + page?: number; + pageSize?: number; } export interface PublicationSelection { @@ -46,12 +59,18 @@ export async function listPublications(query: PublicationsQuery = {}): Promise

( @@ -87,10 +106,16 @@ export async function markSelectedRead(selections: PublicationSelection[]): Prom export interface RetryPublicationPdfResult { message: string; + queued: boolean; resolved_pdf: boolean; publication: PublicationItem; } +export interface TogglePublicationFavoriteResult { + message: string; + publication: PublicationItem; +} + export async function retryPublicationPdf( publicationId: number, scholarProfileId: number, @@ -104,3 +129,21 @@ export async function retryPublicationPdf( ); return response.data; } + +export async function togglePublicationFavorite( + publicationId: number, + scholarProfileId: number, + isFavorite: boolean, +): Promise { + const response = await apiRequest( + `/publications/${publicationId}/favorite`, + { + method: "POST", + body: { + scholar_profile_id: scholarProfileId, + is_favorite: isFavorite, + }, + }, + ); + return response.data; +} diff --git a/frontend/src/features/settings/SettingsAdminPanel.vue b/frontend/src/features/settings/SettingsAdminPanel.vue new file mode 100644 index 0000000..7f5788e --- /dev/null +++ b/frontend/src/features/settings/SettingsAdminPanel.vue @@ -0,0 +1,826 @@ + + + diff --git a/frontend/src/logo.png b/frontend/src/logo.png new file mode 100644 index 0000000..5fbd13e Binary files /dev/null and b/frontend/src/logo.png differ diff --git a/frontend/src/pages/AdminUsersPage.vue b/frontend/src/pages/AdminUsersPage.vue deleted file mode 100644 index d942901..0000000 --- a/frontend/src/pages/AdminUsersPage.vue +++ /dev/null @@ -1,316 +0,0 @@ - - - diff --git a/frontend/src/pages/DashboardPage.vue b/frontend/src/pages/DashboardPage.vue index 8c26da7..e61df8d 100644 --- a/frontend/src/pages/DashboardPage.vue +++ b/frontend/src/pages/DashboardPage.vue @@ -1,5 +1,5 @@