From a5165dc3bae228fb8c41bd1816bbf236df2aa33d Mon Sep 17 00:00:00 2001 From: Justin Visser Date: Sun, 1 Mar 2026 18:14:22 +0100 Subject: [PATCH] after audit --- .claude/settings.json | 17 + .github/workflows/release.yml | 5 +- DECOMPOSITION_CLEANUP.md | 193 -- DECOMPOSITION_SLICES.md | 242 -- MODERNIZATION_PLAN.md | 628 ---- agents.md | 2 +- app/api/routers/run_manual.py | 18 +- app/api/routers/runs.py | 41 +- app/api/schemas/admin.py | 6 +- app/api/schemas/auth.py | 12 +- app/db/session.py | 2 +- app/http/middleware.py | 12 +- app/main.py | 9 +- app/security/csrf.py | 26 +- app/services/arxiv/rate_limit.py | 35 +- app/services/crossref/application.py | 3 +- app/services/dbops/near_duplicate_repair.py | 10 +- app/services/ingestion/application.py | 277 +- app/services/ingestion/enrichment.py | 26 +- app/services/ingestion/page_fetch.py | 12 +- app/services/ingestion/pagination.py | 6 +- app/services/ingestion/publication_upsert.py | 7 +- app/services/ingestion/queue_runner.py | 8 +- app/services/ingestion/run_completion.py | 28 + app/services/ingestion/run_enrichment.py | 110 + app/services/ingestion/run_guards.py | 135 + app/services/ingestion/scheduler.py | 18 +- app/services/ingestion/scholar_outcomes.py | 308 ++ app/services/ingestion/scholar_processing.py | 299 +- app/services/openalex/client.py | 27 +- app/services/publications/pdf_queue_common.py | 1 + .../publications/pdf_queue_queries.py | 11 +- .../publications/pdf_queue_resolution.py | 2 +- app/services/publications/queries.py | 14 +- app/services/runs/events.py | 24 +- app/services/scholar/profile_rows.py | 5 +- app/services/unpaywall/application.py | 2 +- frontend/package-lock.json | 700 +++++ frontend/package.json | 2 + .../src/composables/useRequestState.test.ts | 51 + frontend/src/composables/useRequestState.ts | 40 + .../components/PublicationTableRow.test.ts | 139 + .../components/PublicationTableRow.vue | 178 ++ .../components/PublicationToolbar.test.ts | 89 + .../components/PublicationToolbar.vue | 148 + .../composables/usePublicationData.test.ts | 118 + .../composables/usePublicationData.ts | 338 ++ .../components/ScholarBatchAdd.test.ts | 76 + .../scholars/components/ScholarBatchAdd.vue | 89 + .../components/ScholarNameSearch.test.ts | 43 + .../scholars/components/ScholarNameSearch.vue | 167 + .../components/ScholarSettingsModal.test.ts | 92 + .../components/ScholarSettingsModal.vue | 113 + .../features/settings/SettingsAdminPanel.vue | 1097 +------ .../components/AdminIntegritySection.test.ts | 72 + .../components/AdminIntegritySection.vue | 88 + .../components/AdminPdfQueueSection.test.ts | 104 + .../components/AdminPdfQueueSection.vue | 203 ++ .../components/AdminRepairsSection.test.ts | 91 + .../components/AdminRepairsSection.vue | 372 +++ .../components/AdminUsersSection.test.ts | 84 + .../settings/components/AdminUsersSection.vue | 211 ++ frontend/src/pages/PublicationsPage.vue | 1165 ++----- frontend/src/pages/ScholarsPage.vue | 1179 ++----- tests/integration/helpers.py | 114 + tests/integration/test_api_admin.py | 576 ++++ tests/integration/test_api_auth.py | 182 ++ tests/integration/test_api_publications.py | 573 ++++ .../test_api_publications_advanced.py | 337 ++ tests/integration/test_api_runs.py | 499 +++ tests/integration/test_api_scholars.py | 370 +++ tests/integration/test_api_settings.py | 183 ++ tests/integration/test_api_v1.py | 2721 ----------------- tests/integration/test_fixture_probe_runs.py | 2 +- .../test_run_lifecycle_consistency.py | 29 +- tests/unit/test_logging_policy.py | 40 + tests/unit/test_portability_import.py | 121 + tests/unit/test_portability_normalize.py | 193 ++ tests/unit/test_scholar_validators.py | 142 + tests/unit/test_user_validation.py | 79 + 80 files changed, 8489 insertions(+), 7302 deletions(-) create mode 100644 .claude/settings.json delete mode 100644 DECOMPOSITION_CLEANUP.md delete mode 100644 DECOMPOSITION_SLICES.md delete mode 100644 MODERNIZATION_PLAN.md create mode 100644 app/services/ingestion/run_enrichment.py create mode 100644 app/services/ingestion/run_guards.py create mode 100644 app/services/ingestion/scholar_outcomes.py create mode 100644 frontend/src/composables/useRequestState.test.ts create mode 100644 frontend/src/composables/useRequestState.ts create mode 100644 frontend/src/features/publications/components/PublicationTableRow.test.ts create mode 100644 frontend/src/features/publications/components/PublicationTableRow.vue create mode 100644 frontend/src/features/publications/components/PublicationToolbar.test.ts create mode 100644 frontend/src/features/publications/components/PublicationToolbar.vue create mode 100644 frontend/src/features/publications/composables/usePublicationData.test.ts create mode 100644 frontend/src/features/publications/composables/usePublicationData.ts create mode 100644 frontend/src/features/scholars/components/ScholarBatchAdd.test.ts create mode 100644 frontend/src/features/scholars/components/ScholarBatchAdd.vue create mode 100644 frontend/src/features/scholars/components/ScholarNameSearch.test.ts create mode 100644 frontend/src/features/scholars/components/ScholarNameSearch.vue create mode 100644 frontend/src/features/scholars/components/ScholarSettingsModal.test.ts create mode 100644 frontend/src/features/scholars/components/ScholarSettingsModal.vue create mode 100644 frontend/src/features/settings/components/AdminIntegritySection.test.ts create mode 100644 frontend/src/features/settings/components/AdminIntegritySection.vue create mode 100644 frontend/src/features/settings/components/AdminPdfQueueSection.test.ts create mode 100644 frontend/src/features/settings/components/AdminPdfQueueSection.vue create mode 100644 frontend/src/features/settings/components/AdminRepairsSection.test.ts create mode 100644 frontend/src/features/settings/components/AdminRepairsSection.vue create mode 100644 frontend/src/features/settings/components/AdminUsersSection.test.ts create mode 100644 frontend/src/features/settings/components/AdminUsersSection.vue create mode 100644 tests/integration/test_api_admin.py create mode 100644 tests/integration/test_api_auth.py create mode 100644 tests/integration/test_api_publications.py create mode 100644 tests/integration/test_api_publications_advanced.py create mode 100644 tests/integration/test_api_runs.py create mode 100644 tests/integration/test_api_scholars.py create mode 100644 tests/integration/test_api_settings.py delete mode 100644 tests/integration/test_api_v1.py create mode 100644 tests/unit/test_logging_policy.py create mode 100644 tests/unit/test_portability_import.py create mode 100644 tests/unit/test_portability_normalize.py create mode 100644 tests/unit/test_scholar_validators.py create mode 100644 tests/unit/test_user_validation.py diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..9fa7fd3 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,17 @@ +{ + "permissions": { + "allow": [ + "Bash(wc -l /home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src/features/settings/SettingsAdminPanel.vue /home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src/features/settings/components/Admin*.vue)", + "Read(//home/jv/src/personal/playground/scholar_scraper/scholar-scraper/**)", + "Bash(while read line rest)", + "Bash(do echo \"$line $rest\")", + "Bash(done)", + "Bash(grep -c 'def test_' tests/unit/*.py tests/unit/**/*.py tests/integration/*.py)", + "Bash(sort -t: -k2 -rn)", + "Bash(wc -l frontend/src/pages/*.vue)" + ], + "additionalDirectories": [ + "/home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src" + ] + } +} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0e0166c..9f39e29 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,9 +19,10 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - run: pip install python-semantic-release + - uses: astral-sh/setup-uv@v4 + - run: uv sync --extra dev - name: Semantic Release id: release - run: semantic-release publish + run: uv run semantic-release publish env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/DECOMPOSITION_CLEANUP.md b/DECOMPOSITION_CLEANUP.md deleted file mode 100644 index f95de24..0000000 --- a/DECOMPOSITION_CLEANUP.md +++ /dev/null @@ -1,193 +0,0 @@ -# Decomposition Cleanup — 3-Pass Fix Prompt - -You are working on the scholarr repository (`feat/decomposition` branch). A decomposition refactor split oversized files into focused modules. An audit found residual issues. Fix them in 3 sequential passes. - -**Read `agents.md` before starting.** Key rules: -- Function length: 50 lines max -- File length: 400 target, 600 hard ceiling (kwargs-heavy signatures are acceptable overage) -- No dead code, no duplicate boilerplate, no backward-compatibility shims -- All business logic in `app/services//` -- Use `structured_log()` for domain logging - -**Constraints for every pass:** -- Pure refactoring — no behavioral changes -- All tests must pass after each pass: `docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app python -m pytest` -- Commit after each pass using conventional commit format - ---- - -## Pass 1: Naming, DRY, and Dead Code - -**1a. Rename `_int_or_default` → `int_or_default`** - -This function in `app/services/ingestion/run_completion.py:38` is imported externally by `app/services/ingestion/application.py:25`, making it public API with a private-convention name. - -- Rename the definition in `run_completion.py` (line 38) -- Update all internal call sites in `run_completion.py` (lines 71, 350, 375) -- Update the import in `application.py` (line 25) and call sites (lines 138, 336, 342) - -**1b. Consolidate duplicate functions between `pdf_queue.py` and `pdf_queue_resolution.py`** - -Three functions exist with identical signatures in both files: - -| Function | `pdf_queue.py` | `pdf_queue_resolution.py` | -|---|---|---| -| `_utcnow()` | line 46 | line 33 | -| `_event_row()` | line 150 | line 39 | -| `_queued_job()` | line 165 | line 60 | - -Fix: Keep these in `pdf_queue.py` (the parent module). Have `pdf_queue_resolution.py` import them from `pdf_queue.py`. Since `pdf_queue.py` already imports from `pdf_queue_resolution.py`, check for circular imports — if importing `_utcnow`, `_event_row`, `_queued_job` from `pdf_queue` into `pdf_queue_resolution` would create a cycle, extract them to a shared `pdf_queue_common.py` instead. - -Also deduplicate the PDF status constants (`PDF_STATUS_QUEUED`, `PDF_STATUS_RUNNING`, `PDF_STATUS_RESOLVED`, `PDF_STATUS_FAILED`) that are duplicated between `pdf_queue.py` (lines 23-27) and `pdf_queue_resolution.py` (lines 20-23). Keep them in `pdf_queue.py` only. - -**1c. Consolidate `_effective_request_delay_seconds` duplication** - -`scheduler.py` (lines 32-42) and `queue_runner.py` (line 23-28) both define `_effective_request_delay_seconds` with slightly different signatures. The `queue_runner.py` version takes an explicit `floor` parameter (cleaner). Keep the `queue_runner.py` version. In `scheduler.py`, delete `_request_delay_floor_seconds()` and `_effective_request_delay_seconds()`, and instead import from `queue_runner`: - -```python -from app.services.ingestion.queue_runner import _effective_request_delay_seconds -``` - -At the call site in `scheduler.py`, pass `floor=_request_delay_floor_seconds_value` where needed (inline the floor computation or keep it as a one-liner). - -Actually — since `_effective_request_delay_seconds` would now be imported externally, rename it to `effective_request_delay_seconds` (drop the underscore) in `queue_runner.py`. - -Commit message: `refactor: fix naming, remove duplicate code from decomposition` - ---- - -## Pass 2: Function length — ingestion core (6 worst offenders) - -All 6 functions are >75 lines. Extract helpers to bring each under 50 lines. Below is specific guidance for each. - -**2a. `EnrichmentRunner.enrich_pending_publications` — 133 lines → ≤50** (`enrichment.py:47-179`) - -This function does 4 things: load publications, set up client, batch-loop with API calls, run dedup sweep. Extract: - -1. `_load_unenriched_publications(db_session, *, run_id) → list[Publication]` — lines 66-87 (query + return) -2. `_enrich_batch(self, db_session, *, batch, run_id, client, openalex_works, now) → bool` — lines 142-166 (per-publication enrichment loop; return False if canceled). This inner loop iterates `batch`, calls `_discover_identifiers_for_enrichment`, and applies the match. The caller handles the outer batch loop and API call. -3. `_flush_and_sweep_duplicates(db_session, *, run_id)` — lines 167-179 (flush + dedup sweep) - -The main function becomes: load pubs → early return → create client → batch loop (API call + `_enrich_batch`) → `_flush_and_sweep_duplicates`. - -**2b. `EnrichmentRunner.enrich_publications_with_openalex` — 64 lines → ≤50** (`enrichment.py:260-323`) - -Extract: -1. `_sanitize_titles(publications) → list[str]` — lines 279-286 (title cleaning loop) - -The outer batch loop + match logic stays in the main method, which should now fit in ~45 lines. - -**2c. `ScholarIngestionService.run_for_user` — 99 lines → ≤50** (`application.py:525-623`) - -This function calls `initialize_run`, builds paging/threshold kwargs, calls `run_scholar_iteration`, `complete_run_for_user`, handles enrichment, commits, and logs. Extract: - -1. `_run_iteration_and_complete(self, db_session, *, run, scholars, user_id, start_cstart_map, paging, thresholds, auto_queue_continuations, queue_delay_seconds, idempotency_key) → tuple[RunProgress, RunFailureSummary, RunAlertSummary]` — lines 579-599 (the iteration + completion block). This calls `run_scholar_iteration` and `complete_run_for_user` and returns their results. -2. `_inline_enrich_and_finalize(self, db_session, *, run, user_settings, intended_final_status) → None` — lines 604-614 (enrichment + status fixup + commit). This calls `enrich_pending_publications`, handles the exception, fixes status, and commits. - -**2d. `ScholarIngestionService.execute_run` — 88 lines → ≤50** (`application.py:374-461`) - -Extract: -1. `_execute_run_body(self, db_session, session_factory, *, run_id, user_id, scholars, start_cstart_map, paging, thresholds, auto_queue_continuations, queue_delay_seconds, idempotency_key) → None` — lines 411-457 (the try-body: prepare, iterate, complete, commit, log, spawn enrichment task). The outer `execute_run` handles kwargs resolution and the `async with session_factory()` + `except`. - -Or alternatively, the `_resolve_paging_kwargs` / `_threshold_kwargs` calls can be lifted into the caller or inlined since they're trivial dict builders — this alone may shave enough lines. Consider whether the kwargs signature itself is the bulk (it is — 22 lines of signature). If so, this function may be acceptable as-is per the kwargs-exception rule. Use your judgment: if the function body (excluding signature and kwargs resolution) is ≤50 lines, document it as acceptable and move on. - -**2e. `run_scholar_iteration` — 84 lines → ≤50** (`scholar_processing.py:531-614`) - -This has two clear phases: pass 1 (breadth-first, lines 560-584) and pass 2 (depth, lines 586-614). Extract: - -1. `_run_first_pass(db_session, *, scholars, pagination, run, user_id, start_cstart_map, scholar_kwargs, request_delay_seconds, queue_delay_seconds, progress) → dict[int, int]` — the breadth-first loop (lines 560-584), returns `first_pass_cstarts` -2. `_run_depth_pass(db_session, *, scholars, first_pass_cstarts, pagination, run, user_id, scholar_kwargs, request_delay_seconds, remaining_max, auto_queue_continuations, queue_delay_seconds, progress) → None` — the depth loop (lines 591-613) - -The main function becomes: build kwargs → call pass 1 → compute remaining → call pass 2 → return progress. - -**2f. `QueueJobRunner._finalize_queue_job_after_run` — 79 lines → ≤50** (`queue_runner.py:284-362`) - -Two branches: success path (lines 287-311) and failure path (lines 312-362). Extract: - -1. `_finalize_successful_queue_job(self, session, job, run_summary) → None` — success path -2. `_finalize_failed_queue_job(self, session, job, run_summary) → None` — failure path - -The main function opens a session, branches on `failed_count`, and delegates. - -Commit message: `refactor: extract helpers from long functions in ingestion core` - ---- - -## Pass 3: Function length — remaining violations (10 functions, 58-75 lines) - -**3a. `run_manual` — 105 lines → ≤50** (`app/api/routers/runs.py:183-287`) - -This route handler has a large try block. Extract: - -1. `_start_manual_run(db_session, *, current_user, ingest_service, idempotency_key) → tuple[CrawlRun, list, dict]` — lines 205-221 (initialize_run call with all the settings kwargs) -2. `_spawn_background_execution(ingest_service, *, run, current_user, scholars, start_cstart_map, user_settings, idempotency_key) → None` — lines 226-249 (create_task + background_tasks management) -3. `_manual_run_success_response(request, *, run, idempotency_key, db_session, current_user) → dict` — lines 251-265 (build success payload) - -The route handler becomes: load safety → check disabled → check idempotency → try: start → spawn → respond; except: handle errors. - -**3b. `PaginationEngine._paginate_loop` — 75 lines → ≤50** (`pagination.py:276-350`) - -Extract: -1. `_upsert_page_publications(self, db_session, *, run, scholar, publications, seen_canonical, state, upsert_publications_fn)` — the dedup + upsert block (appears twice: lines 294-302 and 339-347). Deduplicate by extracting a single helper called from both places. - -This should bring the loop body under 50 lines. - -**3c. `PaginationEngine.fetch_and_parse_all_pages` — 72 lines → ≤50** (`pagination.py:415-486`) - -Most of the length is the kwargs-heavy calls. Check if the function body (excluding the 17-line signature) is ≤50 lines — it should be ~55. Extract: -1. Move the `_short_circuit_initial_page` + `_build_loop_state` + `_paginate_loop` + `_result_from_pagination_state` sequence into a helper `_paginate_from_initial_page(self, *, scholar, run, db_session, state, ...) → PagedParseResult` if needed. Or accept this as kwargs-acceptable overage and document why. - -**3d. `PageFetcher.fetch_and_parse_with_retry` — 70 lines → ≤50** (`page_fetch.py:148-217`) - -Extract: -1. `_classify_attempt(parsed_page, *, network_attempts, rate_limit_attempts) → tuple[int, int, int]` — lines 173-183 (attempt counter logic, returns updated network_attempts, rate_limit_attempts, total_attempts) - -This plus the existing `_should_retry` and `_sleep_backoff` helpers should bring the loop under 50 lines. - -**3e. `complete_run_for_user` — 66 lines → ≤50** (`run_completion.py:265-330`) - -Extract: -1. `_log_alert_threshold_warnings(*, user_id, run, failure_summary, alert_summary) → None` — lines 284-313 (the 3 if-blocks that log threshold warnings) - -The main function becomes: summarize → build alerts → log warnings → apply safety → resolve status → finalize → return. ~35 lines. - -**3f. `process_scholar` — 61 lines → ≤50** (`scholar_processing.py:343-403`) - -The function body is mostly kwargs pass-through. Check if the actual body (excluding the 18-line signature) is ≤50 lines — it's ~43 lines of body. If so, this is acceptable under the kwargs exception. Document and move on. - -**3g. `_fetch_and_prepare_scholar_result` — 59 lines → ≤50** (`scholar_processing.py:406-464`) - -Same analysis: 16-line signature, 43-line body. If the body alone is ≤50, this is kwargs-acceptable. Document and move on. - -**3h. `_perform_live_author_search` — 60 lines → ≤50** (`author_search.py:459-518`) - -18-line signature, 42-line body. Kwargs-acceptable. Document and move on. - -**3i. `search_author_candidates` — 60 lines → ≤50** (`author_search.py:521-580`) - -19-line signature, 41-line body. Kwargs-acceptable. Document and move on. - -**3j. `ScholarIngestionService.initialize_run` — 58 lines → ≤50** (`application.py:315-372`) - -20-line signature, 38-line body. Kwargs-acceptable. Document and move on. - -Commit message: `refactor: extract helpers from remaining long functions` - ---- - -## Verification (after all 3 passes) - -```bash -# No function exceeds 50 lines (body, excluding kwargs signatures that exceed the limit) -# Run the AST checker from the audit to verify - -# All tests pass -docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app python -m pytest - -# Linting -docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app ruff check . -docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app ruff format --check . - -# Type checking -docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app mypy app/ -``` diff --git a/DECOMPOSITION_SLICES.md b/DECOMPOSITION_SLICES.md deleted file mode 100644 index 4442463..0000000 --- a/DECOMPOSITION_SLICES.md +++ /dev/null @@ -1,242 +0,0 @@ -# Scholarr Decomposition Slices - -## How to use this file - -This file contains self-contained decomposition prompts ("slices") to bring the codebase into compliance with `agents.md` file size rules. Each slice is an independent task for an LLM executor. - -**Rules from `agents.md`:** -- File length: 400 lines target, 600 lines hard ceiling -- Slightly exceeding the 600-line ceiling is acceptable when the excess is driven by verbose keyword-argument signatures (named parameters), not by business logic density -- Function length: 50 lines max -- All business logic in `app/services//` -- Read `agents.md` fully before starting any slice - -**Execution protocol:** -1. Pick the next incomplete slice in order (slices depend on prior ones) -2. Read the prompt section — it contains everything you need -3. Read all source files mentioned before making changes -4. Make the changes described — no business logic changes, only file reorganization -5. Verify: all imports resolve, all tests pass -6. Commit with the specified message - ---- - -## Progress - -| # | Slice | Status | -|---|-------|--------| -| 1 | Schema split | **DONE** | -| 2 | Ingestion: pagination + publication upsert | **DONE** | -| 3 | Ingestion: enrichment + scholar processing + run completion | **DONE** | -| 4 | Scholars service + PDF queue | **DONE** | -| 5 | Routers + scheduler | **DONE** | - ---- - -## Slice 1: Split `app/api/schemas.py` into domain packages — DONE - -Completed. The 945-line `app/api/schemas.py` was split into `app/api/schemas/` package: - -| File | Lines | Contents | -|---|---|---| -| `common.py` | 39 | `ApiMeta`, `ApiErrorData`, `ApiErrorEnvelope`, `MessageData`, `MessageEnvelope` | -| `auth.py` | 73 | `SessionUserData`, `AuthMe*`, `CsrfBootstrap*`, `Login*`, `ChangePasswordRequest` | -| `scholars.py` | 153 | `ScholarItem*`, `ScholarSearch*`, `ScholarExport*`, `DataExport*`, `DataImport*` | -| `publications.py` | 151 | `DisplayIdentifierData`, `PublicationItem*`, `MarkAll*`, `MarkSelected*`, `RetryPublication*`, `TogglePublication*` | -| `runs.py` | 226 | `RunListItem*`, `RunSummary*`, `RunDebug*`, `RunScholarResult*`, `RunDetail*`, `ManualRun*`, `Queue*`, `ScrapeSafety*` | -| `admin.py` | 289 | All `Admin*` models including PDF queue, repair, near-duplicate schemas | -| `settings.py` | 54 | `SettingsPolicy*`, `Settings*`, `SettingsUpdateRequest` | -| `__init__.py` | 7 | Wildcard re-exports — all `from app.api.schemas import X` imports unchanged | - ---- - -## Slice 2: Decompose `app/services/ingestion/application.py` — pagination + publication upsert — DONE - -Completed. Extracted three modules from `application.py` (2,955 → 2,084 lines): - -| File | Lines | Contents | -|---|---|---| -| `app/services/ingestion/page_fetch.py` | 219 | `PageFetcher` class — single-page fetch, parse-or-layout-error, retry with backoff | -| `app/services/ingestion/pagination.py` | 486 | `PaginationEngine` class — multi-page orchestration, loop state, short-circuit, fingerprint checks | -| `app/services/ingestion/publication_upsert.py` | 239 | Module-level functions — `resolve_publication`, `upsert_profile_publications`, find/create/update helpers | - -Also fixed two external import paths (`portability/normalize.py`, `portability/publication_import.py`) that imported `normalize_title`/`build_publication_url` from `application.py` — now correctly sourced from `fingerprints.py`. Updated one integration test to monkeypatch the new module-level function instead of the old instance method. - ---- - -## Slice 3: Decompose `app/services/ingestion/application.py` — enrichment, scholar processing, run completion — DONE - -Completed. Extracted four logical chunks from `application.py` (2,085 → 635 lines): - -| File | Lines | Contents | -|---|---|---| -| `app/services/ingestion/enrichment.py` | 323 | `EnrichmentRunner` class — OpenAlex enrichment, identifier discovery, dedup sweep | -| `app/services/ingestion/scholar_processing.py` | 614 | `process_scholar`, `run_scholar_iteration`, continuation queue, outcome resolution | -| `app/services/ingestion/run_completion.py` | 417 | `complete_run_for_user`, failure summary, alert summary, safety outcome, progress tracking | - -Also updated two tests (`test_ingestion_arxiv_rate_limit.py`, `test_ingestion_progress_reporting.py`) and one integration test (`test_deferred_enrichment.py`) to import from new modules. - -**Why:** Three remaining logical chunks. Extracting all three brings `application.py` down to ~500 lines (the orchestration spine). - -**Files to create:** `app/services/ingestion/enrichment.py`, `app/services/ingestion/scholar_processing.py`, `app/services/ingestion/run_completion.py` - -**Files to modify:** `app/services/ingestion/application.py` - -**Prompt:** - -You are working on the scholarr repository. Read `agents.md` for project conventions. Continue decomposing `app/services/ingestion/application.py` (slice 2 is complete — `page_fetch.py`, `pagination.py`, and `publication_upsert.py` already extracted). Extract three remaining chunks. - -**A. `app/services/ingestion/enrichment.py` (~300 lines)** - -Move these methods (post-run OpenAlex enrichment): - -- `_run_is_canceled`, `_enrich_pending_publications`, `_discover_identifiers_for_enrichment` -- `_publish_identifier_update_event`, `_enrich_publications_with_openalex` - -Create an `EnrichmentRunner` class that receives service dependencies (OpenAlex client, identifier service, dedup service, DB session factory) in its constructor. - -**B. `app/services/ingestion/scholar_processing.py` (~400 lines)** - -Move these methods (per-scholar outcome resolution): - -- `_assert_valid_paged_parse_result`, `_apply_first_page_profile_metadata`, `_build_result_entry` -- `_skipped_no_change_outcome`, `_upsert_publications_outcome`, `_upsert_success_or_exception` -- `_upsert_success`, `_upsert_exception_outcome`, `_parse_failure_outcome` -- `_sync_continuation_queue`, `_process_scholar`, `_process_scholar_inner` -- `_fetch_and_prepare_scholar_result`, `_resolve_scholar_outcome`, `_unexpected_scholar_exception_outcome` - -These become methods on a `ScholarProcessor` class or module-level functions. - -**C. `app/services/ingestion/run_completion.py` (~300 lines)** - -Move these methods (run finalization + alerting): - -- `_classify_failure_bucket` (standalone function) -- `_summarize_failures`, `_build_alert_summary`, `_apply_safety_outcome`, `_finalize_run_record` -- `_resolve_run_status`, `_resolve_continuation_queue_target`, `_build_failure_debug_context` -- `_complete_run_for_user` - -These become module-level functions accepting explicit parameters. - -**After all extractions, `application.py` should contain only:** - -- `ScholarIngestionService.__init__` and config helpers -- Run lifecycle: `initialize_run`, `execute_run`, `run_for_user` -- Scholar iteration orchestration: `_run_scholar_iteration` -- Progress tracking: `_result_counters`, `_adjust_progress_counts` -- Safety gate integration, background task management - -Target: `application.py` ≤ 500 lines. No new file exceeds 400 lines. All tests must pass unchanged. - -Commit message: `refactor: extract enrichment, scholar processing, and run completion from ingestion service` - ---- - -## Slice 4: Decompose `app/services/scholars/application.py` (996 lines) and `app/services/publications/pdf_queue.py` (969 lines) — DONE - -Completed. Extracted four modules from two oversized service files: - -| File | Lines | Contents | -|---|---|---| -| `app/services/scholars/author_search_cache.py` | 239 | Serialize/deserialize cache entries, cache get/set/prune | -| `app/services/scholars/author_search.py` | 580 | Cooldown, throttle, retry, circuit breaker, `search_author_candidates` | -| `app/services/scholars/application.py` | 221 | Scholar CRUD only (list, create, get, toggle, delete, image ops) | -| `app/services/publications/pdf_queue_queries.py` | 395 | SQL builders, row hydration, listing/counting/pagination, retry item builders | -| `app/services/publications/pdf_queue_resolution.py` | 311 | Task execution, outcome persistence, scheduling (`schedule_rows`) | -| `app/services/publications/pdf_queue.py` | 364 | Dataclasses, constants, enqueueing logic, public API | - -Also updated one test file (`test_publication_pdf_queue_policy.py`) to monkeypatch `pdf_queue_resolution` instead of `pdf_queue` for the 4 resolution-related tests, and updated `publications/application.py` and `publication_identifiers/application.py` imports. - -**Prompt:** - -You are working on the scholarr repository. Read `agents.md` for project conventions. Two service files exceed the 600-line ceiling. Decompose both. - -**A. `app/services/scholars/application.py` (996 lines)** - -This file mixes scholar CRUD (~350 lines) with the entire author search pipeline (~650 lines). Split them: - -Create `app/services/scholars/author_search_cache.py` (~150 lines): Move all `_serialize_*` and `_deserialize_*` functions. Move `_cache_get_author_search_result`, `_cache_set_author_search_result`, `_prune_author_search_cache`. These are pure functions operating on the DB cache table. - -Create `app/services/scholars/author_search.py` (~500 lines): Move remaining author search functions — cooldown management, throttle logic, retry wrappers, circuit breaker, lock acquisition, and `search_author_candidates`. Import cache functions from `author_search_cache.py`. Accept DB session and settings as explicit parameters. - -Update `application.py`: Keep only scholar CRUD methods. Import and delegate to `search_author_candidates` from `author_search.py`. Target: ≤ 350 lines. - -**B. `app/services/publications/pdf_queue.py` (969 lines)** - -Three concerns in one file. Split by concern: - -Create `app/services/publications/pdf_queue_queries.py` (~330 lines): Move SQL query builders (`_tracked_queue_select_base`, `_tracked_queue_select`, etc.), result hydration (`_queue_item_from_row`, `_hydrated_queue_items`), listing/counting/pagination (`list_pdf_queue_items`, `count_pdf_queue_items`, `list_pdf_queue_page`), retry item builders and missing-PDF candidate queries. - -Create `app/services/publications/pdf_queue_resolution.py` (~300 lines): Move task execution (`_mark_attempt_started`, `_failed_outcome`, `_fetch_outcome_for_row`), outcome persistence (`_apply_publication_update`, `_apply_job_outcome`, `_persist_outcome`), scheduling (`_resolve_publication_row`, `_run_resolution_task`, `_register_task`, `_drop_finished_task`, `_schedule_rows`). - -Keep in `pdf_queue.py` (~340 lines): dataclasses, constants, enqueueing logic, public API. - -No file should exceed 500 lines. All tests must pass unchanged. - -Commit message: `refactor: decompose scholars service and pdf_queue into focused modules` - ---- - -## Slice 5: Decompose router files and scheduler — DONE - -Completed. Extracted helpers from three oversized files: - -| File | Lines | Contents | -|---|---|---| -| `app/api/routers/run_serializers.py` | 238 | `serialize_run`, `serialize_queue_item`, `normalize_scholar_result`, `manual_run_payload_from_run`, `manual_run_success_payload`, value coercion helpers, idempotency key validation | -| `app/api/routers/run_manual.py` | 237 | `load_safety_state`, `raise_manual_runs_disabled`, `reused_manual_run_payload`, `run_ingestion_for_manual`, `recover_integrity_error`, `execute_manual_run`, safety/failure raise helpers | -| `app/api/routers/runs.py` | 440 | Route handlers only + imports | -| `app/api/routers/scholar_helpers.py` | 234 | `serialize_scholar`, `hydrate_scholar_metadata_if_needed`, `enqueue_initial_scrape_job_for_scholar`, `search_kwargs`, `search_response_data`, `require_user_profile`, `read_uploaded_image` | -| `app/api/routers/scholars.py` | 406 | Route handlers only + imports | -| `app/services/ingestion/queue_runner.py` | 379 | `QueueJobRunner` class — drain continuation queue, job lifecycle (drop/retry/reschedule), ingestion dispatch, finalization | -| `app/services/ingestion/scheduler.py` | 312 | `SchedulerService` — auto-run scheduling, `_run_loop`, `_tick_once`, candidate loading, PDF queue drain | - -Also updated one test file (`test_scholars_create_hydration.py`) to monkeypatch `scholar_helpers` instead of the `scholars` router for the 4 helper-related tests. - -**Files to create:** `app/api/routers/run_serializers.py`, `app/api/routers/run_manual.py`, `app/api/routers/scholar_helpers.py`, `app/services/ingestion/queue_runner.py` - -**Files to modify:** `app/api/routers/runs.py` (875 → ~405), `app/api/routers/scholars.py` (616 → ~396), `app/services/ingestion/scheduler.py` (622 → ~280) - -**Prompt:** - -You are working on the scholarr repository. Read `agents.md` for project conventions. Three files are slightly over the 600-line ceiling. Extract helpers from each. - -**A. `app/api/routers/runs.py` (875 lines) → 3 files** - -Create `app/api/routers/run_serializers.py` (~250 lines): Move all `_serialize_*`, `_normalize_*`, `_int_value`, `_str_value`, `_bool_value` helper functions. Move `_manual_run_payload_from_run` and `_manual_run_success_payload`. - -Create `app/api/routers/run_manual.py` (~220 lines): Move `_load_safety_state`, `_raise_manual_runs_disabled`, `_reused_manual_run_payload`, `_run_ingestion_for_manual`, `_recover_integrity_error`, `_execute_manual_run`, `_raise_manual_blocked_safety`, `_raise_manual_failed`. - -Keep in `runs.py` (~405 lines): only route handler functions + imports. - -**B. `app/api/routers/scholars.py` (616 lines) → 2 files** - -Create `app/api/routers/scholar_helpers.py` (~220 lines): Move `_needs_metadata_hydration`, `_is_create_hydration_rate_limited`, `_auto_enqueue_new_scholar_enabled`, `_enqueue_initial_scrape_job_for_scholar`, `_uploaded_image_media_path`, `_serialize_scholar`, `_hydrate_scholar_metadata_if_needed`, `_search_kwargs`, `_search_response_data`, `_read_uploaded_image`, `_require_user_profile`. - -Keep in `scholars.py` (~396 lines): only route handlers. - -**C. `app/services/ingestion/scheduler.py` (622 lines) → 2 files** - -Create `app/services/ingestion/queue_runner.py` (~350 lines): Move all continuation-queue job processing — `_drain_continuation_queue`, `_drop_queue_job_if_max_attempts`, `_mark_queue_job_retrying`, `_queue_job_has_available_scholar`, all `_reschedule_*` methods, `_run_ingestion_for_queue_job`, `_finalize_queue_job_after_run`, `_run_queue_job`. Create a `QueueJobRunner` class receiving config params in constructor. - -Keep in `scheduler.py` (~280 lines): auto-run scheduling, `_run_loop`, `_tick_once`, candidate loading. - -All tests must pass unchanged after each extraction. - -Commit message: `refactor: extract helpers from oversized router and scheduler files` - ---- - -## Verification (after all slices) - -```bash -# No Python file exceeds 600 lines -find app/ -name "*.py" -exec wc -l {} + | awk '$1 > 600 && !/total/ {print "FAIL:", $0}' - -# All tests pass -docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app python -m pytest - -# Linting -docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app ruff check . -docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app ruff format --check . -``` diff --git a/MODERNIZATION_PLAN.md b/MODERNIZATION_PLAN.md deleted file mode 100644 index d51aded..0000000 --- a/MODERNIZATION_PLAN.md +++ /dev/null @@ -1,628 +0,0 @@ -# Scholarr Modernization Plan — Slices 2-10 - -> **Slice 1 is complete** (commit `7736cab`): removed 114 stale `build/` files, fixed `.gitignore`, committed pending deletions, replaced `npm install` with `npm ci` in CI. -> -> Each slice below is a self-contained prompt designed for an LLM executor. Execute in order — each slice depends on the prior ones being complete. - ---- - -## Slice 2: Create `structured_log()` Utility - -**Why:** The codebase has 19 `_log_*` helper functions (422 lines) and 138 instances of duplicated `"event"` keys in logging calls. A single utility eliminates all of it. - -**Files to create:** -- `app/logging_utils.py` - -**Files to modify:** -- `tests/unit/test_logging.py` - -**Context files (read but don't modify):** -- `app/logging_config.py` — line 88 has `"event": getattr(record, "event", record.getMessage())` which means when no `event` key is in `extra`, it falls back to `getMessage()`. This is the mechanism that makes `structured_log()` work without duplicating the event name. -- `app/logging_context.py` - -**Prompt:** -``` -You are working on the scholarr repository. The current logging pattern has a DRY problem: - -1. Every log call duplicates the event name as both the message and in extra["event"]: - `logger.info("event.name", extra={"event": "event.name", "key": value})` - -2. There are 19 `_log_*` helper functions (422 lines total) that just wrap logger calls with typed signatures to build extra dicts. - -3. Metrics data (metric_name/metric_value) is mixed into log extra dicts but has no consumer. - -Create `app/logging_utils.py`: - -```python -"""Structured logging utility — eliminates boilerplate across domain services.""" - -from __future__ import annotations - -import logging -from typing import Any - - -def structured_log( - logger: logging.Logger, - level: str, - event: str, - /, - **fields: Any, -) -> None: - """Emit a structured log entry. - - The event name is passed as the log message. The JsonLogFormatter in - logging_config.py extracts it via record.getMessage() when no explicit - 'event' key exists in extra — so we do NOT duplicate it. - - Usage: - structured_log(logger, "info", "ingestion.run_started", user_id=1, scholar_count=5) - """ - fields.pop("metric_name", None) - fields.pop("metric_value", None) - - log_method = getattr(logger, level.lower()) - log_method(event, extra=fields) -``` - -Verify that `app/logging_config.py` line 88 already handles this correctly — when no `event` key is in extra, it falls back to getMessage() which returns the event string passed as the first argument. No changes to logging_config.py should be needed. - -Add tests in `tests/unit/test_logging.py` (append to existing file): -- Test that `structured_log()` produces a log record where the JsonLogFormatter outputs the event name correctly -- Test that `structured_log()` works with ConsoleLogFormatter -- Test that metric_name/metric_value fields are stripped from output -- Test that extra fields (user_id, scholar_id, etc.) appear in the formatted output - -Commit message: "refactor: add structured_log utility to eliminate logging boilerplate" -``` - ---- - -## Slice 3: Migrate Ingestion Logging to `structured_log()` - -**Why:** `app/services/ingestion/application.py` is 3,089 lines. 8 `_log_*` helpers consume ~239 lines. 34 inline logger calls duplicate event names and include dead metric fields. This slice removes ~300 lines. - -**Files to modify:** -- `app/services/ingestion/application.py` - -**Context files:** -- `app/logging_utils.py` (created in Slice 2) - -**Prompt:** -``` -You are working on the scholarr repository. The file `app/services/ingestion/application.py` (3,089 lines) has 8 `_log_*` helper functions consuming ~239 lines, plus 34 inline logger calls with duplicated `"event":` keys and mixed `metric_name`/`metric_value` fields. - -Migrate ALL logging in this file to use `structured_log()` from `app.logging_utils`. - -1. Add `from app.logging_utils import structured_log` at the top. - -2. Delete these 8 helper methods entirely: - - `_log_request_delay_coercion` (line ~123, 21 lines) - - `_log_run_started` (line ~310, 36 lines) - - `_log_scholar_parsed` (line ~383, 28 lines) - - `_log_alert_thresholds` (line ~1013, 47 lines) - - `_log_safety_transition` (line ~1093, 42 lines) - - `_log_run_completed` (line ~1178, 28 lines) - - `_attempt_log_entry` (line ~1837, 16 lines) - - `_page_log_entry` (line ~1999, 21 lines) - -3. Replace every call to these deleted helpers with an inline `structured_log()` call. Example: - - Before: - ```python - self._log_run_started( - user_id=user_id, - trigger_type=trigger_type, - scholar_count=scholar_count, - ... - ) - ``` - - After: - ```python - structured_log( - logger, "info", "ingestion.run_started", - user_id=user_id, - trigger_type=trigger_type.value, - scholar_count=scholar_count, - ... - ) - ``` - -4. For ALL remaining inline `logger.info/warning/debug/exception()` calls: - - Replace with `structured_log()` where the call uses `extra={"event": ..., ...}` pattern - - Do NOT convert `logger.exception()` calls — keep them but remove the duplicated `"event"` key from their extra dict - - Remove `metric_name` and `metric_value` from all calls - -5. Do NOT change any business logic. Only logging call sites. - -Expected outcome: ~300 lines removed. All existing tests must pass unchanged. - -Commit message: "refactor: migrate ingestion service logging to structured_log" -``` - ---- - -## Slice 4: Migrate All Remaining Services to `structured_log()` - -**Why:** 11 more `_log_*` helpers remain across 8 files (183 lines total), plus inline logger calls with the same duplication pattern in ~20 files. - -**Files to modify (all have `_log_*` helpers to delete):** -- `app/services/ingestion/scheduler.py` — `_log_queue_item_resolved` (line ~518, 21 lines) -- `app/services/arxiv/rate_limit.py` — `_log_request_scheduled` (199), `_log_request_completed` (216), `_log_cooldown_activated` (235) -- `app/services/arxiv/client.py` — `_log_cache_event` (283), `_log_request_skipped_for_cooldown` (299) -- `app/services/publications/pdf_resolution_pipeline.py` — `_log_arxiv_skip` (148) -- `app/services/unpaywall/application.py` — `_log_resolution_summary` (194) -- `app/api/routers/publications.py` — `_log_retry_pdf_result` (343) -- `app/api/routers/settings.py` — `_log_settings_update` (103) -- `app/main.py` — `_log_startup_build_marker` (53) - -**Files to modify (inline `extra={"event":...}` pattern only — no helpers to delete):** -- `app/http/middleware.py` -- `app/db/session.py` -- `app/auth/runtime.py` -- `app/security/csrf.py` -- `app/api/routers/scholars.py` -- `app/api/routers/runs.py` -- `app/api/routers/admin_dbops.py` -- `app/api/routers/admin.py` -- `app/api/routers/auth.py` -- `app/api/routers/publications.py` -- `app/services/scholars/application.py` -- `app/services/runs/events.py` -- `app/services/openalex/client.py` -- `app/services/openalex/matching.py` -- `app/services/crossref/application.py` -- `app/services/publications/dedup.py` -- `app/services/publications/enrichment.py` -- `app/services/publications/pdf_queue.py` -- `app/services/scholar/source.py` -- `app/services/arxiv/gateway.py` - -**Prompt:** -``` -You are working on the scholarr repository. Slice 3 migrated `ingestion/application.py` to `structured_log()`. Now do the same for ALL remaining files. - -1. Delete these `_log_*` helper functions and replace their call sites with inline `structured_log()`: - - - `app/services/ingestion/scheduler.py:~518` — `_log_queue_item_resolved` (21 lines) - - `app/services/arxiv/rate_limit.py:~199` — `_log_request_scheduled` (17 lines) - - `app/services/arxiv/rate_limit.py:~216` — `_log_request_completed` (19 lines) - - `app/services/arxiv/rate_limit.py:~235` — `_log_cooldown_activated` (13 lines) - - `app/services/arxiv/client.py:~283` — `_log_cache_event` (16 lines) - - `app/services/arxiv/client.py:~299` — `_log_request_skipped_for_cooldown` (13 lines) - - `app/services/publications/pdf_resolution_pipeline.py:~148` — `_log_arxiv_skip` (11 lines) - - `app/services/unpaywall/application.py:~194` — `_log_resolution_summary` (21 lines) - - `app/api/routers/publications.py:~343` — `_log_retry_pdf_result` (24 lines) - - `app/api/routers/settings.py:~103` — `_log_settings_update` (15 lines) - - `app/main.py:~53` — `_log_startup_build_marker` (13 lines) - -2. In ALL files under `app/` that have inline `logger.info/warning/debug("event.name", extra={"event": "event.name", ...})` calls: - - Replace with `structured_log(logger, "level", "event.name", key=value, ...)` - - Add `from app.logging_utils import structured_log` import - - Remove `metric_name`/`metric_value` from all calls - - Keep `logger.exception()` calls but remove the duplicated `"event"` key from their extra dicts - -3. In `app/services/openalex/client.py`, the lines that log `response.text` (lines ~87, ~137): - - Truncate: `response.text[:500]` - -4. Do NOT change any business logic. Only logging call sites. - -Verify: `grep -rn '"event":' app/ --include="*.py" | grep -v __pycache__` should return 0 matches. -Verify: `grep -rn 'def _log_' app/ --include="*.py"` should return 0 matches. - -Commit message: "refactor: migrate all remaining logging to structured_log, remove boilerplate helpers" -``` - ---- - -## Slice 5: Logging Relevance Audit — Reduce Noise, Improve Readability - -**Why:** Self-hosted users need clear, actionable logs. Current logging has verbose debug noise (per-HTTP-request, per-publication), overly long event names, and no console display optimization. - -**Files to modify:** -- `app/services/scholar/source.py` -- `app/services/ingestion/application.py` -- `app/logging_config.py` -- Any files with event names longer than ~40 chars - -**Prompt:** -``` -You are working on the scholarr repository. The logging has been migrated to `structured_log()` (slices 2-4). Now audit log RELEVANCE and READABILITY for self-hosted users running this in Docker. - -1. **Remove redundant debug logs:** - - `app/services/scholar/source.py`: Remove the 3 debug-level HTTP fetch events (fetch_started, search_fetch_started, publication_fetch_started). Keep only `fetch_succeeded` at DEBUG. The HTTP middleware already logs request.started/completed. - - `app/services/ingestion/application.py`: Remove per-publication `publication.discovered` and `publication.created` DEBUG logs. The run_completed summary already reports totals. - -2. **Shorten overly long event names** (search all structured_log calls in app/): - - `ingestion.request_delay_coerced_to_policy_floor` → `ingestion.delay_coerced` - - `ingestion.safety_cooldown_cleared` → `ingestion.cooldown_cleared` - - Any other names longer than ~40 chars — shorten while preserving meaning - -3. **Improve console readability** in `app/logging_config.py` `ConsoleLogFormatter.format()`: - - Add a short-name mapping for common context fields in console output ONLY (JSON keeps full names): - ```python - _CONSOLE_SHORT_KEYS = { - "user_id": "user", - "scholar_id": "scholar", - "crawl_run_id": "run", - "run_id": "run", - } - ``` - - Apply in the `for key in sorted(payload.keys())` loop - -4. Do NOT remove any WARNING or ERROR level logs. Only remove/demote DEBUG/INFO that are redundant. -5. Do NOT change business logic. - -Commit message: "refactor: reduce log noise, improve event naming and console readability" -``` - ---- - -## Slice 6: Add Ruff + Mypy to CI - -**Why:** No Python linting or type checking in CI. Code quality regressions go undetected. - -**Files to modify:** -- `pyproject.toml` -- `.github/workflows/ci.yml` - -**Prompt:** -``` -You are working on the scholarr repository. Add Python linting and type checking to CI. - -1. In `pyproject.toml`, add ruff configuration: - ```toml - [tool.ruff] - target-version = "py312" - line-length = 120 - - [tool.ruff.lint] - select = ["E", "F", "W", "I", "UP", "B", "SIM", "RUF"] - ignore = ["E501"] - - [tool.ruff.lint.isort] - known-first-party = ["app"] - ``` - -2. In `pyproject.toml`, add mypy configuration: - ```toml - [tool.mypy] - python_version = "3.12" - ignore_missing_imports = true - check_untyped_defs = false - warn_unused_ignores = true - ``` - -3. In `.github/workflows/ci.yml`, add a `lint` job after `repo-hygiene` and before `test`: - ```yaml - lint: - runs-on: ubuntu-latest - needs: [repo-hygiene] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - uses: astral-sh/setup-uv@v4 - - run: uv sync --extra dev - - name: Ruff check - run: uv run ruff check . - - name: Ruff format check - run: uv run ruff format --check . - - name: Mypy - run: uv run mypy app/ --ignore-missing-imports - ``` - -4. Update the `test` job's `needs` to include `lint`. - -5. Add `ruff` and `mypy` to dev dependencies in `pyproject.toml` under `[project.optional-dependencies]`. - -6. Run `ruff check .` locally and fix auto-fixable issues with `ruff check --fix .`. For unfixable issues, add targeted `noqa` comments only if fixing would change behavior. - -Commit message: "ci: add ruff linting and mypy type checking" -``` - ---- - -## Slice 7: Add CodeQL + Dependabot - -**Why:** No security scanning. Missed vulnerabilities, no automated dependency updates. - -**Files to create:** -- `.github/workflows/codeql.yml` -- `.github/dependabot.yml` - -**Prompt:** -``` -You are working on the scholarr repository. Add security scanning. - -1. Create `.github/workflows/codeql.yml`: - ```yaml - name: CodeQL - - on: - push: - branches: [main] - pull_request: - branches: [main] - schedule: - - cron: "30 5 * * 1" - - jobs: - analyze: - runs-on: ubuntu-latest - permissions: - security-events: write - contents: read - strategy: - matrix: - language: [python, javascript] - steps: - - uses: actions/checkout@v4 - - uses: github/codeql-action/init@v3 - with: - languages: ${{ matrix.language }} - - uses: github/codeql-action/autobuild@v3 - - uses: github/codeql-action/analyze@v3 - ``` - -2. Create `.github/dependabot.yml`: - ```yaml - version: 2 - updates: - - package-ecosystem: pip - directory: / - schedule: - interval: weekly - - package-ecosystem: npm - directory: /frontend - schedule: - interval: weekly - - package-ecosystem: github-actions - directory: / - schedule: - interval: weekly - ``` - -Commit message: "ci: add CodeQL security scanning and Dependabot" -``` - ---- - -## Slice 8: Fix Version Remnants + Adopt Semantic Release - -**Why:** Version `0.1.0` is hardcoded in 5 places (including `crossref/application.py:294`). `0.0.1` exists in `docs/website/package.json`. No CHANGELOG, no GitHub Releases. - -**Files to modify:** -- `pyproject.toml` -- `app/services/crossref/application.py` - -**Files to create:** -- `.github/workflows/release.yml` - -**Prompt:** -``` -You are working on the scholarr repository. Set up automated versioning with python-semantic-release. - -1. Fix hardcoded version in `app/services/crossref/application.py` (line ~294): - ```python - # Before: - etiquette = Etiquette(settings.app_name, "0.1.0", "https://scholarr.local", email) - - # After: - from importlib.metadata import version as pkg_version - _APP_VERSION = pkg_version("scholarr") - # then in function: - etiquette = Etiquette(settings.app_name, _APP_VERSION, "https://scholarr.local", email) - ``` - -2. In `pyproject.toml`, add semantic-release config: - ```toml - [tool.semantic_release] - version_toml = ["pyproject.toml:project.version"] - version_variables = ["frontend/package.json:version"] - branch = "main" - build_command = "" - commit_message = "chore(release): v{version}" - ``` - -3. Create `.github/workflows/release.yml`: - ```yaml - name: Release - - on: - push: - branches: [main] - - jobs: - release: - runs-on: ubuntu-latest - if: github.repository == 'JustinZeus/scholarr' - permissions: - contents: write - id-token: write - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - run: pip install python-semantic-release - - name: Semantic Release - id: release - run: semantic-release publish - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ``` - -4. Add `python-semantic-release` to dev dependencies in `pyproject.toml`. - -Note: `docs/website/package.json` version (`0.0.1`) is addressed in Slice 9 (docs rebuild). `frontend/package-lock.json` and `uv.lock` versions auto-update on next `npm install`/`uv sync`. - -Commit message: "ci: adopt python-semantic-release, fix hardcoded version strings" -``` - ---- - -## Slice 9: Docs — Delete and Rebuild from Scratch - -**Why:** Current docs have split authority (two api-contract.md files), orphaned pages, stale version (`0.0.1`), and missing critical docs (testing, deployment, configuration reference). Clean slate is faster than migration. - -**Files to delete:** -- `docs/` (entire directory) - -**Files to create:** -- Complete new `docs/` tree - -**Prompt:** -``` -You are working on the scholarr repository. Delete the entire `docs/` directory and rebuild from scratch. - -Target IA: - -docs/ -├── index.md # Landing page: what is scholarr, quick links -├── user/ -│ ├── overview.md # What scholarr does, key concepts -│ ├── getting-started.md # Install (docker compose), first run, add first scholar -│ └── configuration.md # ALL env vars from .env.example, organized by category -├── developer/ -│ ├── overview.md # Dev quickstart -│ ├── architecture.md # FastAPI + SQLAlchemy + Vue 3, domain service boundaries -│ ├── local-development.md # docker-compose.dev.yml, hot reload, running tests -│ ├── contributing.md # PR process, conventional commits, code standards from agents.md -│ ├── ingestion.md # Ingestion pipeline: parsing, rate limiting, safety gates -│ ├── frontend-theme-inventory.md # Theme tokens, Tailwind integration -│ └── testing.md # Test tiers (unit/integration/smoke), markers, fixtures, how to run -├── operations/ -│ ├── overview.md # Ops quickstart -│ ├── deployment.md # Production Docker, scaling, health checks -│ ├── database-runbook.md # Backup, restore, integrity checks, migration procedures -│ ├── scrape-safety-runbook.md # Rate limiting, cooldowns, CAPTCHA handling -│ └── arxiv-runbook.md # ArXiv rate limits, cache, query patterns -├── reference/ -│ ├── overview.md # Reference index -│ ├── api.md # CANONICAL: envelope spec + endpoints + DTO contract -│ ├── environment.md # Env var quick-reference (links to user/configuration.md) -│ └── changelog.md # Placeholder: "auto-generated by semantic-release" -└── website/ - ├── docusaurus.config.js - ├── sidebars.js # ALL pages listed, no orphans - ├── package.json # version: "0.1.0" - └── src/css/custom.css - -Content guidelines: -- Self-contained, scannable docs (headers, bullets, tables) -- Frontmatter: `title`, `sidebar_position` -- `configuration.md`: read `.env.example`, organize every variable with description, type, default, example -- `testing.md`: pytest markers from pyproject.toml (`integration`, `db`, `migrations`, `schema`, `smoke`), container-based runner from agents.md, fixture org under tests/fixtures/ -- `api.md`: merge old `developer/api-contract.md` (DTO structure) and `reference/api-contract.md` (envelope spec). Include exact envelope shapes: - - Success: `{"data": ..., "meta": {"request_id": "..."}}` - - Error: `{"error": {"code": "...", "message": "...", "details": ...}, "meta": {"request_id": "..."}}` - - Publication semantics (modes, pagination, identifiers) - - Scholar portability endpoints -- `deployment.md`: practical Docker commands from docker-compose.yml -- `database-runbook.md`: reference scripts/db/ (backup_full.sh, restore_dump.sh, check_integrity.py, repair_publication_links.py) - -Docusaurus config: -- url: "https://justinzeus.github.io" -- baseUrl: "/scholarr/" -- organizationName: "JustinZeus" -- projectName: "scholarr" -- onBrokenLinks: "throw" -- docs path: ".." (reads from docs/ parent) -- Exclude: "website/**", "README.md" - -Commit message: "docs: rebuild documentation from scratch with clean IA" -``` - ---- - -## Slice 10: Begin `ingestion/application.py` Decomposition - -**Why:** After logging cleanup (slices 3+5), the file is ~2,700-2,800 lines. Still the largest source file by 3x. This begins decomposition with the safest extract: safety gate logic. - -**Files to modify:** -- `app/services/ingestion/application.py` - -**Files to create:** -- `app/services/ingestion/safety.py` - -**Prompt:** -``` -You are working on the scholarr repository. `app/services/ingestion/application.py` is ~2,700-2,800 lines (after logging cleanup). It contains the entire ingestion orchestration in a single class. - -Extract the FIRST logical chunk: safety gate logic. - -1. Read `app/services/ingestion/application.py` fully. - -2. Identify safety/cooldown methods: - - `_enforce_safety_gate` - - `_raise_safety_blocked_start` - - Cooldown activation/clearing methods - - Alert threshold evaluation methods - -3. Create `app/services/ingestion/safety.py`: - - Move identified methods to standalone functions or a small class - - Accept explicit parameters (no implicit self.xxx state) - - Keep same function signatures where possible - -4. Update `application.py` to import from `safety.py`. - -5. Run full test suite: `docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app pytest tests/` - All tests must pass unchanged. - -Constraints from `agents.md`: -- Max 50 lines per function -- No flat files in `app/services/` root — all within `app/services/ingestion/` -- Fail fast, early returns, guard clauses - -Commit message: "refactor: extract ingestion safety gate logic to dedicated module" -``` - ---- - -## Verification (after all slices) - -Run these checks to confirm everything is clean: - -```bash -# 1. No tracked build artifacts -git ls-files build/ | wc -l # expect: 0 - -# 2. No duplicated event keys in logging -grep -rn '"event":' app/ --include="*.py" | grep -v __pycache__ | wc -l # expect: 0 - -# 3. No _log_ helper functions remain -grep -rn 'def _log_' app/ --include="*.py" | wc -l # expect: 0 - -# 4. No stale version strings -grep -rn '0\.0\.1' . --include="*.py" --include="*.json" --include="*.toml" \ - | grep -v node_modules | grep -v .venv | grep -v uv.lock | wc -l # expect: 0 - -# 5. Docs build -npm --prefix docs/website run build # expect: success, no broken links - -# 6. All tests pass -docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm app pytest tests/ -``` - ---- - -## Slice Summary - -| # | Slice | Files Changed | Estimated Impact | -|---|-------|---------------|------------------| -| ~~1~~ | ~~Repo hygiene~~ | ~~120 files~~ | ~~-18,380 lines~~ **DONE** | -| 2 | `structured_log()` utility | 2 files | +50 lines | -| 3 | Migrate ingestion logging | 1 file | -300 lines | -| 4 | Migrate remaining logging | ~20 files | -180 lines | -| 5 | Logging noise/readability | ~5 files | -30 lines, better output | -| 6 | Ruff + mypy in CI | 2 files | +50 lines config | -| 7 | CodeQL + Dependabot | 2 files | +40 lines config | -| 8 | Version fix + semantic-release | 3 files | +40 lines config | -| 9 | Docs rebuild | full replacement | ~20 new files | -| 10 | Ingestion decomposition | 2 files | net zero (move) | diff --git a/agents.md b/agents.md index 9f81a20..1767078 100644 --- a/agents.md +++ b/agents.md @@ -77,7 +77,7 @@ Types: `feat`, `fix`, `docs`, `ci`, `refactor`, `test`, `chore`, `perf`. ## 7. Testing & Quality Gates -All commands run inside containers. Do not run bare `npm`, `pytest`, or `ruff` on the host. +All **local development** commands run inside containers. Do not run bare `npm`, `pytest`, or `ruff` on the host. CI workflows (`.github/workflows/`) use bare runners with `uv` and `npm` directly — this is intentional since CI has no Docker-in-Docker dependency. ### Backend diff --git a/app/api/routers/run_manual.py b/app/api/routers/run_manual.py index 6360e34..0632e39 100644 --- a/app/api/routers/run_manual.py +++ b/app/api/routers/run_manual.py @@ -138,9 +138,11 @@ async def recover_integrity_error( original_exc: IntegrityError, ) -> dict[str, Any]: if idempotency_key is None: - logger.exception( + structured_log( + logger, + "exception", "api.runs.manual_integrity_error", - extra={"user_id": user_id}, + user_id=user_id, ) raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc existing_run = await run_service.get_manual_run_by_idempotency_key( @@ -149,9 +151,11 @@ async def recover_integrity_error( idempotency_key=idempotency_key, ) if existing_run is None: - logger.exception( + structured_log( + logger, + "exception", "api.runs.manual_integrity_error", - extra={"user_id": user_id}, + user_id=user_id, ) raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc if existing_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING): @@ -230,8 +234,10 @@ def raise_manual_blocked_safety(*, exc, user_id: int) -> None: def raise_manual_failed(*, exc: Exception, user_id: int) -> None: - logger.exception( + structured_log( + logger, + "exception", "api.runs.manual_failed", - extra={"user_id": user_id}, + user_id=user_id, ) raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from exc diff --git a/app/api/routers/runs.py b/app/api/routers/runs.py index 28779ef..15d86f3 100644 --- a/app/api/routers/runs.py +++ b/app/api/routers/runs.py @@ -37,7 +37,8 @@ from app.api.schemas import ( RunsListEnvelope, ) from app.db.models import RunStatus, RunTriggerType, User -from app.db.session import get_db_session +from app.db.session import get_db_session, get_session_factory +from app.logging_utils import structured_log from app.services.ingestion import application as ingestion_service from app.services.runs import application as run_service from app.services.runs.events import event_generator @@ -47,6 +48,14 @@ from app.settings import settings logger = logging.getLogger(__name__) _background_tasks: set[asyncio.Task[Any]] = set() + +def _drop_finished_task(task: asyncio.Task[Any]) -> None: + _background_tasks.discard(task) + try: + task.result() + except Exception: + structured_log(logger, "exception", "runs.background_task_failed") + router = APIRouter(prefix="/runs", tags=["api-runs"]) ACTIVE_RUN_STATUSES = {RunStatus.RUNNING, RunStatus.RESOLVING} @@ -234,7 +243,7 @@ def _spawn_background_execution( ) ) _background_tasks.add(task) - task.add_done_callback(_background_tasks.discard) + task.add_done_callback(_drop_finished_task) @router.post( @@ -247,15 +256,16 @@ async def run_manual( current_user: User = Depends(get_api_current_user), ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service), ): - safety_state = await load_safety_state(db_session, user_id=current_user.id) + user_id = int(current_user.id) + safety_state = await load_safety_state(db_session, user_id=user_id) if not settings.ingestion_manual_run_allowed: - raise_manual_runs_disabled(user_id=current_user.id, safety_state=safety_state) + raise_manual_runs_disabled(user_id=user_id, safety_state=safety_state) idempotency_key = normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER)) reused_payload = await reused_manual_run_payload( db_session, request=request, - user_id=current_user.id, + user_id=user_id, idempotency_key=idempotency_key, safety_state=safety_state, ) @@ -291,12 +301,12 @@ async def run_manual( "new_publication_count": 0, "reused_existing_run": False, "idempotency_key": idempotency_key, - "safety_state": await load_safety_state(db_session, user_id=current_user.id), + "safety_state": await load_safety_state(db_session, user_id=user_id), }, ) except ingestion_service.RunBlockedBySafetyPolicyError as exc: await db_session.rollback() - raise_manual_blocked_safety(exc=exc, user_id=current_user.id) + raise_manual_blocked_safety(exc=exc, user_id=user_id) except ingestion_service.RunAlreadyInProgressError as exc: await db_session.rollback() raise ApiException( @@ -309,13 +319,13 @@ async def run_manual( return await recover_integrity_error( db_session, request=request, - user_id=current_user.id, + user_id=user_id, idempotency_key=idempotency_key, original_exc=exc, ) except Exception as exc: await db_session.rollback() - raise_manual_failed(exc=exc, user_id=current_user.id) + raise_manual_failed(exc=exc, user_id=user_id) @router.get( @@ -454,14 +464,15 @@ async def clear_queue_item( @router.get("/{run_id}/stream") async def stream_run_events( run_id: int, - db_session: AsyncSession = Depends(get_db_session), current_user: User = Depends(get_api_current_user), ): - run = await run_service.get_run_for_user( - db_session, - user_id=current_user.id, - run_id=run_id, - ) + session_factory = get_session_factory() + async with session_factory() as db_session: + run = await run_service.get_run_for_user( + db_session, + user_id=current_user.id, + run_id=run_id, + ) if run is None: raise ApiException( status_code=404, diff --git a/app/api/schemas/admin.py b/app/api/schemas/admin.py index d9f2eb7..f5f73b5 100644 --- a/app/api/schemas/admin.py +++ b/app/api/schemas/admin.py @@ -34,8 +34,8 @@ class AdminUsersListEnvelope(BaseModel): class AdminUserCreateRequest(BaseModel): - email: str - password: str + email: str = Field(max_length=254) + password: str = Field(max_length=128) is_admin: bool = False model_config = ConfigDict(extra="forbid") @@ -55,7 +55,7 @@ class AdminUserEnvelope(BaseModel): class AdminResetPasswordRequest(BaseModel): - new_password: str + new_password: str = Field(max_length=128) model_config = ConfigDict(extra="forbid") diff --git a/app/api/schemas/auth.py b/app/api/schemas/auth.py index 174d0b7..f160b24 100644 --- a/app/api/schemas/auth.py +++ b/app/api/schemas/auth.py @@ -1,6 +1,6 @@ from __future__ import annotations -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field from app.api.schemas.common import ApiMeta @@ -44,8 +44,8 @@ class CsrfBootstrapEnvelope(BaseModel): class LoginRequest(BaseModel): - email: str - password: str + email: str = Field(max_length=254) + password: str = Field(max_length=128) model_config = ConfigDict(extra="forbid") @@ -66,8 +66,8 @@ class LoginEnvelope(BaseModel): class ChangePasswordRequest(BaseModel): - current_password: str - new_password: str - confirm_password: str + current_password: str = Field(max_length=128) + new_password: str = Field(max_length=128) + confirm_password: str = Field(max_length=128) model_config = ConfigDict(extra="forbid") diff --git a/app/db/session.py b/app/db/session.py index 6d286ca..59a1745 100644 --- a/app/db/session.py +++ b/app/db/session.py @@ -85,7 +85,7 @@ async def check_database() -> bool: result = await conn.execute(text("SELECT 1")) return result.scalar_one() == 1 except Exception: - logger.exception("db.healthcheck_failed") + structured_log(logger, "exception", "db.healthcheck_failed") return False diff --git a/app/http/middleware.py b/app/http/middleware.py index 00223e0..65607ed 100644 --- a/app/http/middleware.py +++ b/app/http/middleware.py @@ -74,13 +74,13 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware): response = await call_next(request) except Exception: duration_ms = int((time.perf_counter() - start) * 1000) - logger.exception( + structured_log( + logger, + "exception", "request.failed", - extra={ - "method": request.method, - "path": request.url.path, - "duration_ms": duration_ms, - }, + method=request.method, + path=request.url.path, + duration_ms=duration_ms, ) raise else: diff --git a/app/main.py b/app/main.py index cf03376..feda0c8 100644 --- a/app/main.py +++ b/app/main.py @@ -73,8 +73,13 @@ async def lifespan(_: FastAPI): ) await session.commit() structured_log(logger, "info", "app.startup_orphaned_runs_cleaned") - except Exception as e: - logger.error(f"Failed to clean orphaned runs at startup: {e}") + except Exception as exc: + structured_log( + logger, + "error", + "app.startup_orphaned_runs_cleanup_failed", + error=str(exc), + ) await scheduler_service.start() yield diff --git a/app/security/csrf.py b/app/security/csrf.py index 5c181c9..822be3b 100644 --- a/app/security/csrf.py +++ b/app/security/csrf.py @@ -6,9 +6,10 @@ from urllib.parse import parse_qs from starlette.middleware.base import BaseHTTPMiddleware from starlette.requests import Request -from starlette.responses import JSONResponse, PlainTextResponse, Response +from starlette.responses import PlainTextResponse, Response from starlette.types import Message +from app.api.responses import error_response from app.logging_utils import structured_log CSRF_SESSION_KEY = "csrf_token" @@ -88,7 +89,10 @@ class CSRFMiddleware(BaseHTTPMiddleware): content_type = request.headers.get("content-type", "") if not content_type.startswith("application/x-www-form-urlencoded"): return None - parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True) + try: + parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True) + except (UnicodeDecodeError, ValueError): + return None values = parsed.get(CSRF_FORM_FIELD) if not values: return None @@ -114,19 +118,11 @@ class CSRFMiddleware(BaseHTTPMiddleware): message: str, ) -> Response: if request.url.path.startswith("/api/"): - request_state = getattr(request, "state", None) - request_id = getattr(request_state, "request_id", None) if request_state else None - return JSONResponse( - { - "error": { - "code": code, - "message": message, - "details": None, - }, - "meta": { - "request_id": request_id, - }, - }, + return error_response( + request, status_code=403, + code=code, + message=message, + details=None, ) return PlainTextResponse(message, status_code=403) diff --git a/app/services/arxiv/rate_limit.py b/app/services/arxiv/rate_limit.py index 70d6b69..1f3ab3f 100644 --- a/app/services/arxiv/rate_limit.py +++ b/app/services/arxiv/rate_limit.py @@ -75,24 +75,31 @@ async def _run_serialized_fetch( runtime_state, source_path=source_path, ) - response = await fetch() + runtime_state.next_allowed_at = datetime.now(UTC) + timedelta(seconds=_min_interval_seconds()) + + if wait_seconds > 0: + await asyncio.sleep(wait_seconds) + response = await fetch() + + async with session_factory() as db_session, db_session.begin(): + runtime_state = await _load_runtime_state_for_update(db_session) hit_rate_limit = _record_post_response_state( runtime_state, response_status=int(response.status_code), source_path=source_path, ) - structured_log( - logger, - "info", - "arxiv.request_completed", - status_code=int(response.status_code), - wait_seconds=wait_seconds, - cooldown_remaining_seconds=_cooldown_remaining_seconds( - runtime_state.cooldown_until, now_utc=datetime.now(UTC) - ), - source_path=source_path, - ) - return response, hit_rate_limit + structured_log( + logger, + "info", + "arxiv.request_completed", + status_code=int(response.status_code), + wait_seconds=wait_seconds, + cooldown_remaining_seconds=_cooldown_remaining_seconds( + runtime_state.cooldown_until, now_utc=datetime.now(UTC) + ), + source_path=source_path, + ) + return response, hit_rate_limit async def _acquire_arxiv_lock(db_session: AsyncSession) -> None: @@ -144,8 +151,6 @@ async def _wait_for_allowed_slot_or_raise( source_path=source_path, cooldown_remaining_seconds=0.0, ) - if wait_seconds > 0: - await asyncio.sleep(wait_seconds) return wait_seconds diff --git a/app/services/crossref/application.py b/app/services/crossref/application.py index a9e46d5..7a2908b 100644 --- a/app/services/crossref/application.py +++ b/app/services/crossref/application.py @@ -350,7 +350,8 @@ async def _fetch_items( ), timeout=timeout, ) - except Exception: + except Exception as exc: + structured_log(logger, "warning", "crossref.fetch_failed", error=str(exc)) return [] diff --git a/app/services/dbops/near_duplicate_repair.py b/app/services/dbops/near_duplicate_repair.py index 76a295f..8d50570 100644 --- a/app/services/dbops/near_duplicate_repair.py +++ b/app/services/dbops/near_duplicate_repair.py @@ -6,12 +6,14 @@ from typing import Any from sqlalchemy.ext.asyncio import AsyncSession from app.db.models import DataRepairJob +from app.services.dbops.application import ( + REPAIR_STATUS_COMPLETED, + REPAIR_STATUS_FAILED, + REPAIR_STATUS_PLANNED, + REPAIR_STATUS_RUNNING, +) from app.services.publications import dedup as dedup_service -REPAIR_STATUS_PLANNED = "planned" -REPAIR_STATUS_RUNNING = "running" -REPAIR_STATUS_COMPLETED = "completed" -REPAIR_STATUS_FAILED = "failed" NEAR_DUP_JOB_NAME = "repair_publication_near_duplicates" NEAR_DUP_DEFAULT_MAX_CLUSTERS = 25 diff --git a/app/services/ingestion/application.py b/app/services/ingestion/application.py index 6087925..cd5e6ea 100644 --- a/app/services/ingestion/application.py +++ b/app/services/ingestion/application.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import logging from datetime import UTC, datetime from typing import Any @@ -17,21 +16,28 @@ from app.db.models import ( ) from app.logging_utils import structured_log from app.services.ingestion import queue as queue_service -from app.services.ingestion import safety as run_safety_service from app.services.ingestion.constants import RUN_LOCK_NAMESPACE from app.services.ingestion.enrichment import EnrichmentRunner from app.services.ingestion.pagination import PaginationEngine -from app.services.ingestion.preflight import check_scholar_reachable from app.services.ingestion.run_completion import ( complete_run_for_user, int_or_default, + log_run_completed, run_execution_summary, ) +from app.services.ingestion.run_enrichment import ( + inline_enrich_and_finalize, + spawn_background_enrichment_task, +) +from app.services.ingestion.run_guards import ( + load_user_settings_for_run, + run_preflight_guard, +) from app.services.ingestion.scholar_processing import run_scholar_iteration from app.services.ingestion.types import ( RunAlertSummary, RunAlreadyInProgressError, - RunBlockedBySafetyPolicyError, + RunBlockedBySafetyPolicyError, # noqa: F401 (re-exported) RunFailureSummary, RunProgress, ) @@ -42,8 +48,6 @@ from app.settings import settings logger = logging.getLogger(__name__) ACTIVE_RUN_INDEX_NAME = "uq_crawl_runs_user_active" -_background_tasks: set[asyncio.Task[Any]] = set() - def _is_active_run_integrity_error(exc: IntegrityError) -> bool: original_error = getattr(exc, "orig", None) @@ -97,34 +101,6 @@ def _threshold_kwargs( } -def _log_run_completed( - *, - run: CrawlRun, - user_id: int, - scholars: list[ScholarProfile], - progress: RunProgress, - failure_summary: RunFailureSummary, - alert_summary: RunAlertSummary, -) -> None: - structured_log( - logger, - "info", - "ingestion.run_completed", - user_id=user_id, - crawl_run_id=run.id, - status=run.status.value, - scholar_count=len(scholars), - succeeded_count=progress.succeeded_count, - failed_count=progress.failed_count, - partial_count=progress.partial_count, - new_publication_count=run.new_pub_count, - blocked_failure_count=alert_summary.blocked_failure_count, - network_failure_count=alert_summary.network_failure_count, - retries_scheduled_count=failure_summary.retries_scheduled_count, - alert_flags=alert_summary.alert_flags, - ) - - class ScholarIngestionService: def __init__(self, *, source: ScholarSource) -> None: self._source = source @@ -138,78 +114,6 @@ class ScholarIngestionService: ) return max(policy_minimum, int_or_default(value, policy_minimum)) - async def _load_user_settings_for_run( - self, - db_session: AsyncSession, - *, - user_id: int, - trigger_type: RunTriggerType, - ): - user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=user_id) - await self._enforce_safety_gate( - db_session, user_settings=user_settings, user_id=user_id, trigger_type=trigger_type - ) - return user_settings - - async def _enforce_safety_gate( - self, - db_session: AsyncSession, - *, - user_settings, - user_id: int, - trigger_type: RunTriggerType, - ) -> None: - now_utc = datetime.now(UTC) - previous = run_safety_service.get_safety_event_context(user_settings, now_utc=now_utc) - if run_safety_service.clear_expired_cooldown(user_settings, now_utc=now_utc): - await db_session.commit() - await db_session.refresh(user_settings) - structured_log( - logger, - "info", - "ingestion.cooldown_cleared", - user_id=user_id, - reason=previous.get("cooldown_reason"), - cooldown_until=previous.get("cooldown_until"), - ) - now_utc = datetime.now(UTC) - if run_safety_service.is_cooldown_active(user_settings, now_utc=now_utc): - await self._raise_safety_blocked_start( - db_session, - user_settings=user_settings, - user_id=user_id, - trigger_type=trigger_type, - now_utc=now_utc, - ) - - async def _raise_safety_blocked_start( - self, - db_session: AsyncSession, - *, - user_settings, - user_id: int, - trigger_type: RunTriggerType, - now_utc: datetime, - ) -> None: - safety_state = run_safety_service.register_cooldown_blocked_start(user_settings, now_utc=now_utc) - await db_session.commit() - structured_log( - logger, - "warning", - "ingestion.safety_policy_blocked_run_start", - user_id=user_id, - trigger_type=trigger_type.value, - reason=safety_state.get("cooldown_reason"), - cooldown_until=safety_state.get("cooldown_until"), - cooldown_remaining_seconds=safety_state.get("cooldown_remaining_seconds"), - blocked_start_count=((safety_state.get("counters") or {}).get("blocked_start_count")), - ) - raise RunBlockedBySafetyPolicyError( - code="scrape_cooldown_active", - message="Scrape safety cooldown is active; run start is temporarily blocked.", - safety_state=safety_state, - ) - async def _load_target_scholars( self, db_session: AsyncSession, @@ -232,49 +136,6 @@ class ScholarIngestionService: await queue_service.clear_job_for_scholar(db_session, user_id=user_id, scholar_profile_id=sid) return scholars - async def _run_preflight_guard( - self, - db_session: AsyncSession, - *, - user_settings: Any, - user_id: int, - scholars: list[ScholarProfile], - ) -> None: - if not scholars: - return - result = await check_scholar_reachable( - self._source, - scholar_id=scholars[0].scholar_id, - ) - if result.passed: - return - now_utc = datetime.now(UTC) - safety_state, _ = run_safety_service.apply_run_safety_outcome( - user_settings, - run_id=0, - blocked_failure_count=1, - network_failure_count=0, - blocked_failure_threshold=1, - network_failure_threshold=1, - blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds, - network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds, - now_utc=now_utc, - ) - await db_session.commit() - structured_log( - logger, - "warning", - "ingestion.cooldown_entered_preflight", - user_id=user_id, - block_reason=result.block_reason, - cooldown_until=safety_state.get("cooldown_until"), - ) - raise RunBlockedBySafetyPolicyError( - code="scrape_cooldown_active", - message=f"Preflight detected Scholar block ({result.block_reason}). Cooldown activated.", - safety_state=safety_state, - ) - async def _initialize_run_for_user( self, db_session: AsyncSession, @@ -287,7 +148,7 @@ class ScholarIngestionService: threshold_kwargs: dict[str, Any], idempotency_key: str | None, ) -> tuple[Any, CrawlRun, list[ScholarProfile], dict[int, int]]: - user_settings = await self._load_user_settings_for_run( + user_settings = await load_user_settings_for_run( db_session, user_id=user_id, trigger_type=trigger_type, @@ -301,8 +162,9 @@ class ScholarIngestionService: user_id=user_id, filtered_scholar_ids=filtered_scholar_ids, ) - await self._run_preflight_guard( + await run_preflight_guard( db_session, + self._source, user_settings=user_settings, user_id=user_id, scholars=scholars, @@ -486,7 +348,7 @@ class ScholarIngestionService: if intended_final_status not in (RunStatus.CANCELED,): run.status = RunStatus.RESOLVING await db_session.commit() - _log_run_completed( + log_run_completed( run=run, user_id=user_id, scholars=attached_scholars, @@ -495,19 +357,22 @@ class ScholarIngestionService: alert_summary=alert_summary, ) if intended_final_status not in (RunStatus.CANCELED,): - task = asyncio.create_task( - self._background_enrich( - session_factory, - run_id=run.id, - intended_final_status=intended_final_status, - openalex_api_key=getattr(user_settings, "openalex_api_key", None), - ) + spawn_background_enrichment_task( + session_factory, + self._enrichment, + run_id=run.id, + intended_final_status=intended_final_status, + openalex_api_key=getattr(user_settings, "openalex_api_key", None), ) - _background_tasks.add(task) - task.add_done_callback(_background_tasks.discard) except Exception as exc: await db_session.rollback() - logger.exception("ingestion.background_run_failed", extra={"run_id": run_id, "user_id": user_id}) + structured_log( + logger, + "exception", + "ingestion.background_run_failed", + run_id=run_id, + user_id=user_id, + ) await self._fail_run_in_background(session_factory, run_id, exc) async def _prepare_execute_run( @@ -530,47 +395,21 @@ class ScholarIngestionService: return run, user_settings, list(scholars_result.scalars().all()) async def _fail_run_in_background(self, session_factory: Any, run_id: int, exc: Exception) -> None: - async with session_factory() as cleanup_session: - run_to_fail = await cleanup_session.get(CrawlRun, run_id) - if run_to_fail: - run_to_fail.status = RunStatus.FAILED - run_to_fail.end_dt = datetime.now(UTC) - run_to_fail.error_log["terminal_exception"] = str(exc) - await cleanup_session.commit() - - async def _background_enrich( - self, - session_factory: Any, - *, - run_id: int, - intended_final_status: RunStatus, - openalex_api_key: str | None = None, - ) -> None: try: - async with session_factory() as session: - await self._enrichment.enrich_pending_publications( - session, - run_id=run_id, - openalex_api_key=openalex_api_key, - ) - run = await session.get(CrawlRun, run_id) - if run is not None and run.status == RunStatus.RESOLVING: - run.status = intended_final_status - await session.commit() - logger.info( - "ingestion.background_enrichment_complete", - extra={"run_id": run_id, "final_status": str(intended_final_status)}, - ) + async with session_factory() as cleanup_session: + run_to_fail = await cleanup_session.get(CrawlRun, run_id) + if run_to_fail: + run_to_fail.status = RunStatus.FAILED + run_to_fail.end_dt = datetime.now(UTC) + run_to_fail.error_log["terminal_exception"] = str(exc) + await cleanup_session.commit() except Exception: - logger.exception("ingestion.background_enrichment_failed", extra={"run_id": run_id}) - try: - async with session_factory() as fallback_session: - run = await fallback_session.get(CrawlRun, run_id) - if run is not None and run.status == RunStatus.RESOLVING: - run.status = intended_final_status - await fallback_session.commit() - except Exception: - logger.exception("ingestion.background_enrichment_fallback_failed", extra={"run_id": run_id}) + structured_log( + logger, + "exception", + "ingestion.fail_run_cleanup_failed", + run_id=run_id, + ) async def run_for_user( self, @@ -626,7 +465,7 @@ class ScholarIngestionService: alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, ) - progress, failure_summary, alert_summary = await self._run_iteration_and_complete( + progress, failure_summary, alert_summary, intended_final_status = await self._run_iteration_and_complete( db_session, run=run, scholars=scholars, @@ -639,10 +478,14 @@ class ScholarIngestionService: idempotency_key=idempotency_key, ) user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=user_id) - await self._inline_enrich_and_finalize( - db_session, run=run, user_settings=user_settings, intended_final_status=run.status + await inline_enrich_and_finalize( + db_session, + self._enrichment, + run=run, + user_settings=user_settings, + intended_final_status=intended_final_status, ) - _log_run_completed( + log_run_completed( run=run, user_id=user_id, scholars=scholars, @@ -665,7 +508,7 @@ class ScholarIngestionService: auto_queue_continuations: bool, queue_delay_seconds: int, idempotency_key: str | None, - ) -> tuple[RunProgress, RunFailureSummary, RunAlertSummary]: + ) -> tuple[RunProgress, RunFailureSummary, RunAlertSummary, RunStatus]: progress = await run_scholar_iteration( db_session, pagination=self._pagination, @@ -691,27 +534,7 @@ class ScholarIngestionService: if intended_final_status not in (RunStatus.CANCELED,): run.status = RunStatus.RESOLVING await db_session.commit() - return progress, failure_summary, alert_summary - - async def _inline_enrich_and_finalize( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - user_settings: Any, - intended_final_status: RunStatus, - ) -> None: - try: - await self._enrichment.enrich_pending_publications( - db_session, - run_id=run.id, - openalex_api_key=getattr(user_settings, "openalex_api_key", None), - ) - except Exception: - logger.exception("ingestion.enrichment_failed", extra={"run_id": run.id}) - if run.status == RunStatus.RESOLVING: - run.status = intended_final_status - await db_session.commit() + return progress, failure_summary, alert_summary, intended_final_status async def _try_acquire_user_lock( self, diff --git a/app/services/ingestion/enrichment.py b/app/services/ingestion/enrichment.py index 9232efb..1b66f58 100644 --- a/app/services/ingestion/enrichment.py +++ b/app/services/ingestion/enrichment.py @@ -96,7 +96,7 @@ class EnrichmentRunner: for p in batch: if await self.run_is_canceled(db_session, run_id=run_id): - logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id}) + structured_log(logger, "info", "ingestion.enrichment_aborted", run_id=run_id) return False, arxiv_lookup_allowed p.openalex_last_attempt_at = now arxiv_lookup_allowed = await self._discover_identifiers_for_enrichment( @@ -112,9 +112,9 @@ class EnrichmentRunner: candidates=openalex_works, ) if match: - p.year = match.publication_year or p.year - p.citation_count = match.cited_by_count or p.citation_count - p.pdf_url = match.oa_url or p.pdf_url + p.year = match.publication_year if match.publication_year is not None else p.year + p.citation_count = match.cited_by_count if match.cited_by_count is not None else p.citation_count + p.pdf_url = match.oa_url if match.oa_url is not None else p.pdf_url p.openalex_enriched = True return True, arxiv_lookup_allowed @@ -162,7 +162,7 @@ class EnrichmentRunner: for i in range(0, len(publications), batch_size): if await self.run_is_canceled(db_session, run_id=run_id): - logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id}) + structured_log(logger, "info", "ingestion.enrichment_aborted", run_id=run_id) return batch = publications[i : i + batch_size] titles = _sanitize_titles(batch) @@ -181,6 +181,8 @@ class EnrichmentRunner: continue except Exception as e: structured_log(logger, "warning", "ingestion.openalex_enrichment_failed", error=str(e), run_id=run_id) + for p in batch: + p.openalex_last_attempt_at = now continue should_continue, arxiv_lookup_allowed = await self._enrich_batch( db_session, @@ -302,9 +304,13 @@ class EnrichmentRunner: {"title.search": "|".join(titles)}, limit=batch_size * 3 ) except Exception as e: - logger.warning( + structured_log( + logger, + "warning", "ingestion.openalex_enrichment_failed", - extra={"error": str(e), "batch_size": len(batch), "scholar_id": scholar.id}, + error=str(e), + batch_size=len(batch), + scholar_id=scholar.id, ) openalex_works = [] @@ -320,11 +326,11 @@ class EnrichmentRunner: title=p.title, title_url=p.title_url, cluster_id=p.cluster_id, - year=match.publication_year or p.year, - citation_count=match.cited_by_count, + year=match.publication_year if match.publication_year is not None else p.year, + citation_count=match.cited_by_count if match.cited_by_count is not None else p.citation_count, authors_text=p.authors_text, venue_text=p.venue_text, - pdf_url=match.oa_url or p.pdf_url, + pdf_url=match.oa_url if match.oa_url is not None else p.pdf_url, ) enriched.append(new_p) else: diff --git a/app/services/ingestion/page_fetch.py b/app/services/ingestion/page_fetch.py index 19aff3f..3a57206 100644 --- a/app/services/ingestion/page_fetch.py +++ b/app/services/ingestion/page_fetch.py @@ -49,13 +49,13 @@ class PageFetcher: error="source_does_not_support_pagination", ) except Exception as exc: - logger.exception( + structured_log( + logger, + "exception", "ingestion.fetch_unexpected_error", - extra={ - "scholar_id": scholar_id, - "cstart": cstart, - "page_size": page_size, - }, + scholar_id=scholar_id, + cstart=cstart, + page_size=page_size, ) return FetchResult( requested_url=( diff --git a/app/services/ingestion/pagination.py b/app/services/ingestion/pagination.py index 5858a7d..91bc984 100644 --- a/app/services/ingestion/pagination.py +++ b/app/services/ingestion/pagination.py @@ -353,6 +353,9 @@ class PaginationEngine: page_attempt_log=next_attempt_log, ) + if self._handle_page_state_transition(state=state): + return + if next_parsed_page.publications: await self._upsert_page_publications( db_session, @@ -364,9 +367,6 @@ class PaginationEngine: upsert_publications_fn=upsert_publications_fn, ) - if self._handle_page_state_transition(state=state): - return - @staticmethod def _result_from_pagination_state( *, diff --git a/app/services/ingestion/publication_upsert.py b/app/services/ingestion/publication_upsert.py index 9d1a6b7..d2ba42b 100644 --- a/app/services/ingestion/publication_upsert.py +++ b/app/services/ingestion/publication_upsert.py @@ -8,7 +8,6 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.db.models import CrawlRun, Publication, ScholarProfile, ScholarPublication -from app.services.doi.normalize import first_doi_from_texts from app.services.ingestion.fingerprints import ( build_publication_fingerprint, build_publication_url, @@ -106,8 +105,9 @@ def update_existing_publication( ) -> None: if candidate.cluster_id and publication.cluster_id is None: publication.cluster_id = candidate.cluster_id - publication.title_raw = candidate.title - publication.title_normalized = normalize_title(candidate.title) + if not publication.title_raw: + publication.title_raw = candidate.title + publication.title_normalized = normalize_title(candidate.title) if candidate.year is not None: publication.year = candidate.year if candidate.citation_count is not None: @@ -118,7 +118,6 @@ def update_existing_publication( publication.venue_text = candidate.venue_text if candidate.title_url: publication.pub_url = build_publication_url(candidate.title_url) - first_doi_from_texts(candidate.title_url, candidate.venue_text, candidate.title) async def resolve_publication( diff --git a/app/services/ingestion/queue_runner.py b/app/services/ingestion/queue_runner.py index 45d9dd7..2220f58 100644 --- a/app/services/ingestion/queue_runner.py +++ b/app/services/ingestion/queue_runner.py @@ -229,7 +229,13 @@ class QueueJobRunner: error=str(exc), ) await recovery_session.commit() - logger.exception("scheduler.queue_item_run_failed", extra={"queue_item_id": job.id, "user_id": job.user_id}) + structured_log( + logger, + "exception", + "scheduler.queue_item_run_failed", + queue_item_id=job.id, + user_id=job.user_id, + ) async def _load_request_delay_for_user(self, user_id: int, *, floor: int) -> int: session_factory = get_session_factory() diff --git a/app/services/ingestion/run_completion.py b/app/services/ingestion/run_completion.py index 8f53f6a..d22b8fc 100644 --- a/app/services/ingestion/run_completion.py +++ b/app/services/ingestion/run_completion.py @@ -427,3 +427,31 @@ def run_execution_summary( partial_count=progress.partial_count, new_publication_count=run.new_pub_count, ) + + +def log_run_completed( + *, + run: CrawlRun, + user_id: int, + scholars: list[ScholarProfile], + progress: RunProgress, + failure_summary: RunFailureSummary, + alert_summary: RunAlertSummary, +) -> None: + structured_log( + logger, + "info", + "ingestion.run_completed", + user_id=user_id, + crawl_run_id=run.id, + status=run.status.value, + scholar_count=len(scholars), + succeeded_count=progress.succeeded_count, + failed_count=progress.failed_count, + partial_count=progress.partial_count, + new_publication_count=run.new_pub_count, + blocked_failure_count=alert_summary.blocked_failure_count, + network_failure_count=alert_summary.network_failure_count, + retries_scheduled_count=failure_summary.retries_scheduled_count, + alert_flags=alert_summary.alert_flags, + ) diff --git a/app/services/ingestion/run_enrichment.py b/app/services/ingestion/run_enrichment.py new file mode 100644 index 0000000..94ed0f4 --- /dev/null +++ b/app/services/ingestion/run_enrichment.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import CrawlRun, RunStatus +from app.logging_utils import structured_log +from app.services.ingestion.enrichment import EnrichmentRunner + +logger = logging.getLogger(__name__) + +_background_tasks: set[asyncio.Task[Any]] = set() + + +async def background_enrich( + session_factory: Any, + enrichment_runner: EnrichmentRunner, + *, + run_id: int, + intended_final_status: RunStatus, + openalex_api_key: str | None = None, +) -> None: + try: + async with session_factory() as session: + await enrichment_runner.enrich_pending_publications( + session, + run_id=run_id, + openalex_api_key=openalex_api_key, + ) + run = await session.get(CrawlRun, run_id) + if run is not None and run.status == RunStatus.RESOLVING: + run.status = intended_final_status + await session.commit() + structured_log( + logger, + "info", + "ingestion.background_enrichment_complete", + run_id=run_id, + final_status=str(intended_final_status), + ) + except Exception: + structured_log( + logger, + "exception", + "ingestion.background_enrichment_failed", + run_id=run_id, + ) + try: + async with session_factory() as fallback_session: + run = await fallback_session.get(CrawlRun, run_id) + if run is not None and run.status == RunStatus.RESOLVING: + run.status = intended_final_status + await fallback_session.commit() + except Exception: + structured_log( + logger, + "exception", + "ingestion.background_enrichment_fallback_failed", + run_id=run_id, + ) + + +async def inline_enrich_and_finalize( + db_session: AsyncSession, + enrichment_runner: EnrichmentRunner, + *, + run: CrawlRun, + user_settings: Any, + intended_final_status: RunStatus, +) -> None: + try: + await enrichment_runner.enrich_pending_publications( + db_session, + run_id=run.id, + openalex_api_key=getattr(user_settings, "openalex_api_key", None), + ) + except Exception: + structured_log( + logger, + "exception", + "ingestion.enrichment_failed", + run_id=run.id, + ) + if run.status == RunStatus.RESOLVING: + run.status = intended_final_status + await db_session.commit() + + +def spawn_background_enrichment_task( + session_factory: Any, + enrichment_runner: EnrichmentRunner, + *, + run_id: int, + intended_final_status: RunStatus, + openalex_api_key: str | None, +) -> None: + task = asyncio.create_task( + background_enrich( + session_factory, + enrichment_runner, + run_id=run_id, + intended_final_status=intended_final_status, + openalex_api_key=openalex_api_key, + ) + ) + _background_tasks.add(task) + task.add_done_callback(_background_tasks.discard) diff --git a/app/services/ingestion/run_guards.py b/app/services/ingestion/run_guards.py new file mode 100644 index 0000000..fafc405 --- /dev/null +++ b/app/services/ingestion/run_guards.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import logging +from datetime import UTC, datetime +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ( + RunTriggerType, + ScholarProfile, +) +from app.logging_utils import structured_log +from app.services.ingestion import safety as run_safety_service +from app.services.ingestion.preflight import check_scholar_reachable +from app.services.ingestion.types import RunBlockedBySafetyPolicyError +from app.services.scholar.source import ScholarSource +from app.services.settings import application as user_settings_service +from app.settings import settings + +logger = logging.getLogger(__name__) + + +async def load_user_settings_for_run( + db_session: AsyncSession, + *, + user_id: int, + trigger_type: RunTriggerType, +): + user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=user_id) + await enforce_safety_gate(db_session, user_settings=user_settings, user_id=user_id, trigger_type=trigger_type) + return user_settings + + +async def enforce_safety_gate( + db_session: AsyncSession, + *, + user_settings, + user_id: int, + trigger_type: RunTriggerType, +) -> None: + now_utc = datetime.now(UTC) + previous = run_safety_service.get_safety_event_context(user_settings, now_utc=now_utc) + if run_safety_service.clear_expired_cooldown(user_settings, now_utc=now_utc): + await db_session.commit() + await db_session.refresh(user_settings) + structured_log( + logger, + "info", + "ingestion.cooldown_cleared", + user_id=user_id, + reason=previous.get("cooldown_reason"), + cooldown_until=previous.get("cooldown_until"), + ) + now_utc = datetime.now(UTC) + if run_safety_service.is_cooldown_active(user_settings, now_utc=now_utc): + await raise_safety_blocked_start( + db_session, + user_settings=user_settings, + user_id=user_id, + trigger_type=trigger_type, + now_utc=now_utc, + ) + + +async def raise_safety_blocked_start( + db_session: AsyncSession, + *, + user_settings, + user_id: int, + trigger_type: RunTriggerType, + now_utc: datetime, +) -> None: + safety_state = run_safety_service.register_cooldown_blocked_start(user_settings, now_utc=now_utc) + await db_session.commit() + structured_log( + logger, + "warning", + "ingestion.safety_policy_blocked_run_start", + user_id=user_id, + trigger_type=trigger_type.value, + reason=safety_state.get("cooldown_reason"), + cooldown_until=safety_state.get("cooldown_until"), + cooldown_remaining_seconds=safety_state.get("cooldown_remaining_seconds"), + blocked_start_count=((safety_state.get("counters") or {}).get("blocked_start_count")), + ) + raise RunBlockedBySafetyPolicyError( + code="scrape_cooldown_active", + message="Scrape safety cooldown is active; run start is temporarily blocked.", + safety_state=safety_state, + ) + + +async def run_preflight_guard( + db_session: AsyncSession, + source: ScholarSource, + *, + user_settings: Any, + user_id: int, + scholars: list[ScholarProfile], +) -> None: + if not scholars: + return + result = await check_scholar_reachable( + source, + scholar_id=scholars[0].scholar_id, + ) + if result.passed: + return + now_utc = datetime.now(UTC) + safety_state, _ = run_safety_service.apply_run_safety_outcome( + user_settings, + run_id=0, + blocked_failure_count=1, + network_failure_count=0, + blocked_failure_threshold=1, + network_failure_threshold=1, + blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds, + network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds, + now_utc=now_utc, + ) + await db_session.commit() + structured_log( + logger, + "warning", + "ingestion.cooldown_entered_preflight", + user_id=user_id, + block_reason=result.block_reason, + cooldown_until=safety_state.get("cooldown_until"), + ) + raise RunBlockedBySafetyPolicyError( + code="scrape_cooldown_active", + message=f"Preflight detected Scholar block ({result.block_reason}). Cooldown activated.", + safety_state=safety_state, + ) diff --git a/app/services/ingestion/scheduler.py b/app/services/ingestion/scheduler.py index dbb0632..bf4e9e7 100644 --- a/app/services/ingestion/scheduler.py +++ b/app/services/ingestion/scheduler.py @@ -125,9 +125,10 @@ class SchedulerService: except asyncio.CancelledError: raise except Exception: - logger.exception( + structured_log( + logger, + "exception", "scheduler.tick_failed", - extra={}, ) await asyncio.sleep(float(self._tick_seconds)) @@ -263,7 +264,12 @@ class SchedulerService: return None except Exception: await session.rollback() - logger.exception("scheduler.run_failed", extra={"user_id": candidate.user_id}) + structured_log( + logger, + "exception", + "scheduler.run_failed", + user_id=candidate.user_id, + ) return None async def _run_candidate(self, candidate: _AutoRunCandidate) -> None: @@ -300,4 +306,8 @@ class SchedulerService: processed_count=processed, ) except Exception: - logger.exception("scheduler.pdf_queue_drain_failed", extra={}) + structured_log( + logger, + "exception", + "scheduler.pdf_queue_drain_failed", + ) diff --git a/app/services/ingestion/scholar_outcomes.py b/app/services/ingestion/scholar_outcomes.py new file mode 100644 index 0000000..0871273 --- /dev/null +++ b/app/services/ingestion/scholar_outcomes.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +import logging +from datetime import UTC, datetime +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ( + CrawlRun, + RunStatus, + ScholarProfile, +) +from app.logging_utils import structured_log +from app.services.ingestion.run_completion import build_failure_debug_context +from app.services.ingestion.types import ( + PagedParseResult, + ScholarProcessingOutcome, +) +from app.services.scholar.parser import ParseState + +logger = logging.getLogger(__name__) + + +def assert_valid_paged_parse_result( + *, + scholar_id: str, + paged_parse_result: PagedParseResult, +) -> None: + parsed_page = paged_parse_result.parsed_page + if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} and any( + code.startswith("layout_") for code in parsed_page.warnings + ): + raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.") + for publication in paged_parse_result.publications: + if not publication.title.strip(): + raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.") + if publication.citation_count is not None and int(publication.citation_count) < 0: + raise RuntimeError(f"Negative citation count for scholar_id={scholar_id}.") + + +def apply_first_page_profile_metadata( + *, + scholar: ScholarProfile, + paged_parse_result: PagedParseResult, + run_dt: datetime, +) -> None: + first_page = paged_parse_result.first_page_parsed_page + if first_page.profile_name and not (scholar.display_name or "").strip(): + scholar.display_name = first_page.profile_name + if first_page.profile_image_url: + scholar.profile_image_url = first_page.profile_image_url + scholar.last_initial_page_checked_at = run_dt + + +def build_result_entry( + *, + scholar: ScholarProfile, + start_cstart: int, + paged_parse_result: PagedParseResult, +) -> dict[str, Any]: + parsed_page = paged_parse_result.parsed_page + return { + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "outcome": "failed", + "attempt_count": len(paged_parse_result.attempt_log), + "publication_count": len(paged_parse_result.publications), + "start_cstart": start_cstart, + "articles_range": parsed_page.articles_range, + "warnings": parsed_page.warnings, + "has_show_more_button": parsed_page.has_show_more_button, + "pages_fetched": paged_parse_result.pages_fetched, + "pages_attempted": paged_parse_result.pages_attempted, + "has_more_remaining": paged_parse_result.has_more_remaining, + "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, + "continuation_cstart": paged_parse_result.continuation_cstart, + "skipped_no_change": paged_parse_result.skipped_no_change, + "initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, + } + + +def skipped_no_change_outcome( + *, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], +) -> ScholarProcessingOutcome: + first_page = paged_parse_result.first_page_parsed_page + scholar.last_run_status = RunStatus.SUCCESS + scholar.last_run_dt = run_dt + result_entry["state"] = first_page.state.value + result_entry["state_reason"] = "no_change_initial_page_signature" + result_entry["outcome"] = "success" + result_entry["publication_count"] = 0 + result_entry["warnings"] = first_page.warnings + result_entry["debug"] = { + "state_reason": "no_change_initial_page_signature", + "first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, + "attempt_log": paged_parse_result.attempt_log, + "page_logs": paged_parse_result.page_logs, + } + return ScholarProcessingOutcome( + result_entry=result_entry, + succeeded_count_delta=1, + failed_count_delta=0, + partial_count_delta=0, + discovered_publication_count=0, + ) + + +async def upsert_publications_outcome( + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], +) -> ScholarProcessingOutcome: + parsed_page = paged_parse_result.parsed_page + publications = paged_parse_result.publications + had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS} + has_partial_set = len(publications) > 0 and had_page_failure + if (not had_page_failure) or has_partial_set: + return await _upsert_success_or_exception( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + has_partial_publication_set=has_partial_set, + ) + return _parse_failure_outcome( + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + + +async def _upsert_success_or_exception( + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + has_partial_publication_set: bool, +) -> ScholarProcessingOutcome: + try: + return _upsert_success( + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + has_partial_publication_set=has_partial_publication_set, + ) + except Exception as exc: + return _upsert_exception_outcome( + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + exc=exc, + ) + + +def _upsert_success( + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + has_partial_publication_set: bool, +) -> ScholarProcessingOutcome: + discovered_count = paged_parse_result.discovered_publication_count + is_partial = ( + paged_parse_result.has_more_remaining + or paged_parse_result.pagination_truncated_reason is not None + or has_partial_publication_set + ) + scholar.last_run_status = RunStatus.PARTIAL_FAILURE if is_partial else RunStatus.SUCCESS + scholar.last_run_dt = run_dt + if not is_partial and paged_parse_result.first_page_fingerprint_sha256: + scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256 + result_entry["outcome"] = "partial" if is_partial else "success" + if is_partial: + result_entry["debug"] = build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, + ) + return ScholarProcessingOutcome( + result_entry=result_entry, + succeeded_count_delta=1, + failed_count_delta=0, + partial_count_delta=1 if is_partial else 0, + discovered_publication_count=discovered_count, + ) + + +def _upsert_exception_outcome( + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + exc: Exception, +) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = run_dt + result_entry["state"] = "ingestion_error" + result_entry["state_reason"] = "publication_upsert_exception" + result_entry["outcome"] = "failed" + result_entry["error"] = str(exc) + result_entry["debug"] = build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, + exception=exc, + ) + structured_log( + logger, + "exception", + "ingestion.scholar_failed", + crawl_run_id=run.id, + scholar_profile_id=scholar.id, + scholar_id=scholar.scholar_id, + ) + return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) + + +def _parse_failure_outcome( + *, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], +) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = run_dt + result_entry["debug"] = build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, + ) + structured_log( + logger, + "warning", + "ingestion.scholar_parse_failed", + scholar_profile_id=scholar.id, + scholar_id=scholar.scholar_id, + state=paged_parse_result.parsed_page.state.value, + state_reason=paged_parse_result.parsed_page.state_reason, + status_code=paged_parse_result.fetch_result.status_code, + ) + return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) + + +def unexpected_scholar_exception_outcome( + *, + run: CrawlRun, + scholar: ScholarProfile, + start_cstart: int, + exc: Exception, +) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = datetime.now(UTC) + structured_log( + logger, + "exception", + "ingestion.scholar_unexpected_failure", + crawl_run_id=run.id, + scholar_profile_id=scholar.id, + scholar_id=scholar.scholar_id, + ) + return ScholarProcessingOutcome( + result_entry={ + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": "ingestion_error", + "state_reason": "scholar_processing_exception", + "outcome": "failed", + "attempt_count": 0, + "publication_count": 0, + "start_cstart": start_cstart, + "warnings": [], + "error": str(exc), + "debug": {"exception_type": type(exc).__name__, "exception_message": str(exc)}, + }, + succeeded_count_delta=0, + failed_count_delta=1, + partial_count_delta=0, + discovered_publication_count=0, + ) diff --git a/app/services/ingestion/scholar_processing.py b/app/services/ingestion/scholar_processing.py index 37b47d3..f41d904 100644 --- a/app/services/ingestion/scholar_processing.py +++ b/app/services/ingestion/scholar_processing.py @@ -6,6 +6,7 @@ import random from datetime import UTC, datetime from typing import Any +from sqlalchemy.exc import InterfaceError, OperationalError from sqlalchemy.ext.asyncio import AsyncSession from app.db.models import ( @@ -21,7 +22,15 @@ from app.services.ingestion.constants import ( ) from app.services.ingestion.pagination import PaginationEngine from app.services.ingestion.publication_upsert import upsert_profile_publications -from app.services.ingestion.run_completion import apply_outcome_to_progress, build_failure_debug_context +from app.services.ingestion.run_completion import apply_outcome_to_progress +from app.services.ingestion.scholar_outcomes import ( + apply_first_page_profile_metadata, + assert_valid_paged_parse_result, + build_result_entry, + skipped_no_change_outcome, + unexpected_scholar_exception_outcome, + upsert_publications_outcome, +) from app.services.ingestion.types import ( PagedParseResult, RunProgress, @@ -33,254 +42,6 @@ from app.services.scholar.state_detection import is_hard_challenge_reason logger = logging.getLogger(__name__) -def assert_valid_paged_parse_result( - *, - scholar_id: str, - paged_parse_result: PagedParseResult, -) -> None: - parsed_page = paged_parse_result.parsed_page - if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} and any( - code.startswith("layout_") for code in parsed_page.warnings - ): - raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.") - for publication in paged_parse_result.publications: - if not publication.title.strip(): - raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.") - if publication.citation_count is not None and int(publication.citation_count) < 0: - raise RuntimeError(f"Negative citation count for scholar_id={scholar_id}.") - - -def apply_first_page_profile_metadata( - *, - scholar: ScholarProfile, - paged_parse_result: PagedParseResult, - run_dt: datetime, -) -> None: - first_page = paged_parse_result.first_page_parsed_page - if first_page.profile_name and not (scholar.display_name or "").strip(): - scholar.display_name = first_page.profile_name - if first_page.profile_image_url: - scholar.profile_image_url = first_page.profile_image_url - scholar.last_initial_page_checked_at = run_dt - - -def build_result_entry( - *, - scholar: ScholarProfile, - start_cstart: int, - paged_parse_result: PagedParseResult, -) -> dict[str, Any]: - parsed_page = paged_parse_result.parsed_page - return { - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - "state": parsed_page.state.value, - "state_reason": parsed_page.state_reason, - "outcome": "failed", - "attempt_count": len(paged_parse_result.attempt_log), - "publication_count": len(paged_parse_result.publications), - "start_cstart": start_cstart, - "articles_range": parsed_page.articles_range, - "warnings": parsed_page.warnings, - "has_show_more_button": parsed_page.has_show_more_button, - "pages_fetched": paged_parse_result.pages_fetched, - "pages_attempted": paged_parse_result.pages_attempted, - "has_more_remaining": paged_parse_result.has_more_remaining, - "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, - "continuation_cstart": paged_parse_result.continuation_cstart, - "skipped_no_change": paged_parse_result.skipped_no_change, - "initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, - } - - -def skipped_no_change_outcome( - *, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], -) -> ScholarProcessingOutcome: - first_page = paged_parse_result.first_page_parsed_page - scholar.last_run_status = RunStatus.SUCCESS - scholar.last_run_dt = run_dt - result_entry["state"] = first_page.state.value - result_entry["state_reason"] = "no_change_initial_page_signature" - result_entry["outcome"] = "success" - result_entry["publication_count"] = 0 - result_entry["warnings"] = first_page.warnings - result_entry["debug"] = { - "state_reason": "no_change_initial_page_signature", - "first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, - "attempt_log": paged_parse_result.attempt_log, - "page_logs": paged_parse_result.page_logs, - } - return ScholarProcessingOutcome( - result_entry=result_entry, - succeeded_count_delta=1, - failed_count_delta=0, - partial_count_delta=0, - discovered_publication_count=0, - ) - - -async def upsert_publications_outcome( - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], -) -> ScholarProcessingOutcome: - parsed_page = paged_parse_result.parsed_page - publications = paged_parse_result.publications - had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS} - has_partial_set = len(publications) > 0 and had_page_failure - if (not had_page_failure) or has_partial_set: - return await _upsert_success_or_exception( - db_session, - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - has_partial_publication_set=has_partial_set, - ) - return _parse_failure_outcome( - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - ) - - -async def _upsert_success_or_exception( - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - has_partial_publication_set: bool, -) -> ScholarProcessingOutcome: - try: - return _upsert_success( - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - has_partial_publication_set=has_partial_publication_set, - ) - except Exception as exc: - return _upsert_exception_outcome( - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - exc=exc, - ) - - -def _upsert_success( - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - has_partial_publication_set: bool, -) -> ScholarProcessingOutcome: - discovered_count = paged_parse_result.discovered_publication_count - is_partial = ( - paged_parse_result.has_more_remaining - or paged_parse_result.pagination_truncated_reason is not None - or has_partial_publication_set - ) - scholar.last_run_status = RunStatus.PARTIAL_FAILURE if is_partial else RunStatus.SUCCESS - scholar.last_run_dt = run_dt - if not is_partial and paged_parse_result.first_page_fingerprint_sha256: - scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256 - result_entry["outcome"] = "partial" if is_partial else "success" - if is_partial: - result_entry["debug"] = build_failure_debug_context( - fetch_result=paged_parse_result.fetch_result, - parsed_page=paged_parse_result.parsed_page, - attempt_log=paged_parse_result.attempt_log, - page_logs=paged_parse_result.page_logs, - ) - return ScholarProcessingOutcome( - result_entry=result_entry, - succeeded_count_delta=1, - failed_count_delta=0, - partial_count_delta=1 if is_partial else 0, - discovered_publication_count=discovered_count, - ) - - -def _upsert_exception_outcome( - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - exc: Exception, -) -> ScholarProcessingOutcome: - scholar.last_run_status = RunStatus.FAILED - scholar.last_run_dt = run_dt - result_entry["state"] = "ingestion_error" - result_entry["state_reason"] = "publication_upsert_exception" - result_entry["outcome"] = "failed" - result_entry["error"] = str(exc) - result_entry["debug"] = build_failure_debug_context( - fetch_result=paged_parse_result.fetch_result, - parsed_page=paged_parse_result.parsed_page, - attempt_log=paged_parse_result.attempt_log, - page_logs=paged_parse_result.page_logs, - exception=exc, - ) - logger.exception( - "ingestion.scholar_failed", - extra={ - "crawl_run_id": run.id, - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - }, - ) - return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) - - -def _parse_failure_outcome( - *, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], -) -> ScholarProcessingOutcome: - scholar.last_run_status = RunStatus.FAILED - scholar.last_run_dt = run_dt - result_entry["debug"] = build_failure_debug_context( - fetch_result=paged_parse_result.fetch_result, - parsed_page=paged_parse_result.parsed_page, - attempt_log=paged_parse_result.attempt_log, - page_logs=paged_parse_result.page_logs, - ) - structured_log( - logger, - "warning", - "ingestion.scholar_parse_failed", - scholar_profile_id=scholar.id, - scholar_id=scholar.scholar_id, - state=paged_parse_result.parsed_page.state.value, - state_reason=paged_parse_result.parsed_page.state_reason, - status_code=paged_parse_result.fetch_result.status_code, - ) - return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) - - async def sync_continuation_queue( db_session: AsyncSession, *, @@ -396,6 +157,8 @@ async def process_scholar( queue_delay_seconds=queue_delay_seconds, ) return outcome + except (OperationalError, InterfaceError): + raise except Exception as exc: return unexpected_scholar_exception_outcome( run=run, @@ -492,44 +255,6 @@ async def _resolve_scholar_outcome( ) -def unexpected_scholar_exception_outcome( - *, - run: CrawlRun, - scholar: ScholarProfile, - start_cstart: int, - exc: Exception, -) -> ScholarProcessingOutcome: - scholar.last_run_status = RunStatus.FAILED - scholar.last_run_dt = datetime.now(UTC) - logger.exception( - "ingestion.scholar_unexpected_failure", - extra={ - "crawl_run_id": run.id, - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - }, - ) - return ScholarProcessingOutcome( - result_entry={ - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - "state": "ingestion_error", - "state_reason": "scholar_processing_exception", - "outcome": "failed", - "attempt_count": 0, - "publication_count": 0, - "start_cstart": start_cstart, - "warnings": [], - "error": str(exc), - "debug": {"exception_type": type(exc).__name__, "exception_message": str(exc)}, - }, - succeeded_count_delta=0, - failed_count_delta=1, - partial_count_delta=0, - discovered_publication_count=0, - ) - - def _is_hard_challenge_outcome(outcome: ScholarProcessingOutcome) -> bool: entry = outcome.result_entry return str(entry.get("state", "")) == ParseState.BLOCKED_OR_CAPTCHA.value and is_hard_challenge_reason( diff --git a/app/services/openalex/client.py b/app/services/openalex/client.py index f418c1e..13db8ab 100644 --- a/app/services/openalex/client.py +++ b/app/services/openalex/client.py @@ -8,6 +8,7 @@ from tenacity import ( wait_exponential, ) +from app.logging_utils import structured_log from app.services.openalex.types import OpenAlexWork logger = logging.getLogger(__name__) @@ -82,7 +83,13 @@ class OpenAlexClient: raise OpenAlexBudgetExhaustedError("Daily API budget exhausted; retrying won't help until midnight UTC") raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex work by DOI") if response.status_code >= 400: - logger.warning("OpenAlex API error: %s %s", response.status_code, response.text[:500]) + structured_log( + logger, + "warning", + "openalex.api_error", + status_code=response.status_code, + response_preview=response.text[:500], + ) raise OpenAlexClientError(f"API Error {response.status_code}") data = response.json() @@ -130,7 +137,14 @@ class OpenAlexClient: raise OpenAlexBudgetExhaustedError("Daily API budget exhausted; retrying won't help until midnight UTC") raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex works list") if response.status_code >= 400: - logger.warning("OpenAlex API error (filters=%s): %s %s", filters, response.status_code, response.text[:500]) + structured_log( + logger, + "warning", + "openalex.api_error_with_filters", + filters=filters, + status_code=response.status_code, + response_preview=response.text[:500], + ) raise OpenAlexClientError(f"API Error {response.status_code}") data = response.json() @@ -140,8 +154,13 @@ class OpenAlexClient: for raw_work in results: try: parsed_works.append(OpenAlexWork.from_api_dict(raw_work)) - except Exception as e: - logger.warning("Failed to parse OpenAlex raw dict: %s", e) + except Exception as exc: + structured_log( + logger, + "warning", + "openalex.parse_failed", + error=str(exc), + ) continue return parsed_works diff --git a/app/services/publications/pdf_queue_common.py b/app/services/publications/pdf_queue_common.py index 46c2143..9eb95ed 100644 --- a/app/services/publications/pdf_queue_common.py +++ b/app/services/publications/pdf_queue_common.py @@ -4,6 +4,7 @@ from datetime import UTC, datetime from app.db.models import PublicationPdfJob, PublicationPdfJobEvent +PDF_STATUS_UNTRACKED = "untracked" PDF_STATUS_QUEUED = "queued" PDF_STATUS_RUNNING = "running" PDF_STATUS_RESOLVED = "resolved" diff --git a/app/services/publications/pdf_queue_queries.py b/app/services/publications/pdf_queue_queries.py index a0ee0b2..31260ea 100644 --- a/app/services/publications/pdf_queue_queries.py +++ b/app/services/publications/pdf_queue_queries.py @@ -16,14 +16,13 @@ from app.db.models import ( ) from app.services.publication_identifiers import application as identifier_service from app.services.publication_identifiers.types import DisplayIdentifier +from app.services.publications.pdf_queue_common import ( + PDF_STATUS_QUEUED, + PDF_STATUS_RUNNING, + PDF_STATUS_UNTRACKED, +) from app.services.publications.types import PublicationListItem -PDF_STATUS_UNTRACKED = "untracked" -PDF_STATUS_QUEUED = "queued" -PDF_STATUS_RUNNING = "running" -PDF_STATUS_RESOLVED = "resolved" -PDF_STATUS_FAILED = "failed" - def _bounded_limit(limit: int, *, max_value: int = 500) -> int: return max(1, min(int(limit), max_value)) diff --git a/app/services/publications/pdf_queue_resolution.py b/app/services/publications/pdf_queue_resolution.py index aa80761..aec3f74 100644 --- a/app/services/publications/pdf_queue_resolution.py +++ b/app/services/publications/pdf_queue_resolution.py @@ -251,7 +251,7 @@ def _drop_finished_task(task: asyncio.Task[None]) -> None: try: task.result() except Exception: - logger.exception("publications.pdf_queue.task_failed") + structured_log(logger, "exception", "publications.pdf_queue.task_failed") def schedule_rows( diff --git a/app/services/publications/queries.py b/app/services/publications/queries.py index 82763aa..f113080 100644 --- a/app/services/publications/queries.py +++ b/app/services/publications/queries.py @@ -16,6 +16,12 @@ from app.db.models import ( ScholarPublication, ) from app.services.publications.modes import MODE_LATEST, MODE_UNREAD +from app.services.publications.pdf_queue_common import ( + PDF_STATUS_FAILED, + PDF_STATUS_QUEUED, + PDF_STATUS_RESOLVED, + PDF_STATUS_RUNNING, +) from app.services.publications.types import PublicationListItem, UnreadPublicationItem @@ -29,10 +35,10 @@ def _normalized_citation_count(value: object) -> int: def _pdf_status_sort_rank(): return case( (Publication.pdf_url.is_not(None), 4), - (PublicationPdfJob.status == "resolved", 4), - (PublicationPdfJob.status == "running", 3), - (PublicationPdfJob.status == "queued", 2), - (PublicationPdfJob.status == "failed", 0), + (PublicationPdfJob.status == PDF_STATUS_RESOLVED, 4), + (PublicationPdfJob.status == PDF_STATUS_RUNNING, 3), + (PublicationPdfJob.status == PDF_STATUS_QUEUED, 2), + (PublicationPdfJob.status == PDF_STATUS_FAILED, 0), else_=1, ) diff --git a/app/services/runs/events.py b/app/services/runs/events.py index 7f4a71e..17baa62 100644 --- a/app/services/runs/events.py +++ b/app/services/runs/events.py @@ -4,6 +4,8 @@ import logging from collections.abc import AsyncGenerator from typing import Any +from app.logging_utils import structured_log + logger = logging.getLogger(__name__) @@ -17,7 +19,13 @@ class RunEventPublisher: self._subscribers[run_id] = set() queue: asyncio.Queue[Any] = asyncio.Queue() self._subscribers[run_id].add(queue) - logger.debug(f"New subscriber for run {run_id}. Total: {len(self._subscribers[run_id])}") + structured_log( + logger, + "debug", + "runs.event_subscriber_added", + run_id=run_id, + subscriber_count=len(self._subscribers[run_id]), + ) return queue def unsubscribe(self, run_id: int, queue: asyncio.Queue) -> None: @@ -37,7 +45,12 @@ class RunEventPublisher: try: queue.put_nowait(message) except asyncio.QueueFull: - logger.warning(f"Subscriber queue full for run {run_id}, dropping message") + structured_log( + logger, + "warning", + "runs.event_subscriber_queue_full", + run_id=run_id, + ) run_events = RunEventPublisher() @@ -54,7 +67,12 @@ async def event_generator(run_id: int) -> AsyncGenerator[str, None]: data_str = json.dumps(message["data"]) yield f"event: {event_type}\ndata: {data_str}\n\n" except asyncio.CancelledError: - logger.debug(f"Client disconnected from SSE stream for run {run_id}") + structured_log( + logger, + "debug", + "runs.event_stream_disconnected", + run_id=run_id, + ) raise finally: run_events.unsubscribe(run_id, queue) diff --git a/app/services/scholar/profile_rows.py b/app/services/scholar/profile_rows.py index bb0d83d..d291dbf 100644 --- a/app/services/scholar/profile_rows.py +++ b/app/services/scholar/profile_rows.py @@ -131,7 +131,10 @@ def parse_citation_count(parts: list[str]) -> int | None: text = normalize_space(" ".join(parts)) if not text: return 0 - digits = re.sub(r"\D+", "", text) + match = re.search(r"[\d,]+", text) + if not match: + return None + digits = match.group(0).replace(",", "") if not digits: return None return int(digits) diff --git a/app/services/unpaywall/application.py b/app/services/unpaywall/application.py index fa9b83b..f7d23f8 100644 --- a/app/services/unpaywall/application.py +++ b/app/services/unpaywall/application.py @@ -393,7 +393,7 @@ async def resolve_publication_oa_outcomes( return {} email = _email_for_request(request_email) if email is None: - logger.debug("unpaywall.resolve_skipped_missing_email") + structured_log(logger, "debug", "unpaywall.resolve_skipped_missing_email") return {} import httpx diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 81441d5..4e5d821 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -15,7 +15,9 @@ "devDependencies": { "@types/node": "^22.10.2", "@vitejs/plugin-vue": "^5.2.1", + "@vue/test-utils": "^2.4.6", "autoprefixer": "^10.4.21", + "happy-dom": "^20.7.0", "postcss": "^8.5.3", "tailwindcss": "^3.4.17", "typescript": "^5.7.2", @@ -474,6 +476,24 @@ "node": ">=12" } }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -550,6 +570,24 @@ "node": ">= 8" } }, + "node_modules/@one-ini/wasm": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@one-ini/wasm/-/wasm-0.1.1.tgz", + "integrity": "sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.57.1", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.57.1.tgz", @@ -917,6 +955,23 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/whatwg-mimetype": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/whatwg-mimetype/-/whatwg-mimetype-3.0.2.tgz", + "integrity": "sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@vitejs/plugin-vue": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-5.2.4.tgz", @@ -1227,6 +1282,27 @@ "integrity": "sha512-cfWa1fCGBxrvaHRhvV3Is0MgmrbSCxYTXCSCau2I0a1Xw1N1pHAvkWCiXPRAqjvToILvguNyEwjevUqAuBQWvQ==", "license": "MIT" }, + "node_modules/@vue/test-utils": { + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/@vue/test-utils/-/test-utils-2.4.6.tgz", + "integrity": "sha512-FMxEjOpYNYiFe0GkaHsnJPXFHxQ6m4t8vI/ElPGpMWxZKpmRvQ33OIrvRXemy6yha03RxhOlQuy+gZMC3CQSow==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-beautify": "^1.14.9", + "vue-component-type-helpers": "^2.0.0" + } + }, + "node_modules/abbrev": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-2.0.0.tgz", + "integrity": "sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, "node_modules/alien-signals": { "version": "1.0.13", "resolved": "https://registry.npmjs.org/alien-signals/-/alien-signals-1.0.13.tgz", @@ -1234,6 +1310,32 @@ "dev": true, "license": "MIT" }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/any-promise": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", @@ -1502,6 +1604,26 @@ "node": ">= 6" } }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, "node_modules/commander": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", @@ -1512,6 +1634,32 @@ "node": ">= 6" } }, + "node_modules/config-chain": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", + "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ini": "^1.3.4", + "proto-list": "~1.2.1" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/cssesc": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", @@ -1580,6 +1728,42 @@ "dev": true, "license": "MIT" }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, + "node_modules/editorconfig": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-1.0.7.tgz", + "integrity": "sha512-e0GOtq/aTQhVdNyDU9e02+wz9oDDM+SIOQxWME2QRjzRX5yyLAuHDE+0aE8vHb9XRC8XD37eO2u57+F09JqFhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@one-ini/wasm": "0.1.1", + "commander": "^10.0.0", + "minimatch": "^9.0.1", + "semver": "^7.5.3" + }, + "bin": { + "editorconfig": "bin/editorconfig" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/editorconfig/node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, "node_modules/electron-to-chromium": { "version": "1.5.286", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.286.tgz", @@ -1587,6 +1771,13 @@ "dev": true, "license": "ISC" }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, "node_modules/entities": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", @@ -1728,6 +1919,23 @@ "node": ">=8" } }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/fraction.js": { "version": "5.3.4", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.4.tgz", @@ -1767,6 +1975,28 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -1780,6 +2010,24 @@ "node": ">=10.13.0" } }, + "node_modules/happy-dom": { + "version": "20.7.0", + "resolved": "https://registry.npmjs.org/happy-dom/-/happy-dom-20.7.0.tgz", + "integrity": "sha512-hR/uLYQdngTyEfxnOoa+e6KTcfBFyc1hgFj/Cc144A5JJUuHFYqIEBDcD4FeGqUeKLRZqJ9eN9u7/GDjYEgS1g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": ">=20.0.0", + "@types/whatwg-mimetype": "^3.0.2", + "@types/ws": "^8.18.1", + "entities": "^7.0.1", + "whatwg-mimetype": "^3.0.0", + "ws": "^8.18.3" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -1803,6 +2051,13 @@ "he": "bin/he" } }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "license": "ISC" + }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -1842,6 +2097,16 @@ "node": ">=0.10.0" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -1865,6 +2130,29 @@ "node": ">=0.12.0" } }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/jiti": { "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", @@ -1875,6 +2163,38 @@ "jiti": "bin/jiti.js" } }, + "node_modules/js-beautify": { + "version": "1.15.4", + "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.15.4.tgz", + "integrity": "sha512-9/KXeZUKKJwqCXUdBxFJ3vPh467OCckSBmYDwSK/EtV090K+iMJ7zx2S3HLVDIWFQdqMIsZWbnaGiba18aWhaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "config-chain": "^1.1.13", + "editorconfig": "^1.0.4", + "glob": "^10.4.2", + "js-cookie": "^3.0.5", + "nopt": "^7.2.1" + }, + "bin": { + "css-beautify": "js/bin/css-beautify.js", + "html-beautify": "js/bin/html-beautify.js", + "js-beautify": "js/bin/js-beautify.js" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/js-cookie": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.5.tgz", + "integrity": "sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, "node_modules/lilconfig": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", @@ -1902,6 +2222,13 @@ "dev": true, "license": "MIT" }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -1951,6 +2278,16 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -2002,6 +2339,22 @@ "dev": true, "license": "MIT" }, + "node_modules/nopt": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-7.2.1.tgz", + "integrity": "sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w==", + "dev": true, + "license": "ISC", + "dependencies": { + "abbrev": "^2.0.0" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -2032,6 +2385,13 @@ "node": ">= 6" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, "node_modules/path-browserify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", @@ -2039,6 +2399,16 @@ "dev": true, "license": "MIT" }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", @@ -2046,6 +2416,23 @@ "dev": true, "license": "MIT" }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/pathe": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", @@ -2286,6 +2673,13 @@ "dev": true, "license": "MIT" }, + "node_modules/proto-list": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", + "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==", + "dev": true, + "license": "ISC" + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -2431,6 +2825,42 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/siginfo": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", @@ -2438,6 +2868,19 @@ "dev": true, "license": "ISC" }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -2461,6 +2904,110 @@ "dev": true, "license": "MIT" }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/sucrase": { "version": "3.35.1", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.1.tgz", @@ -2906,6 +3453,13 @@ } } }, + "node_modules/vue-component-type-helpers": { + "version": "2.2.12", + "resolved": "https://registry.npmjs.org/vue-component-type-helpers/-/vue-component-type-helpers-2.2.12.tgz", + "integrity": "sha512-YbGqHZ5/eW4SnkPNR44mKVc6ZKQoRs/Rux1sxC6rdwXb4qpbOSYfDr9DsTHolOTGmIKgM9j141mZbBeg05R1pw==", + "dev": true, + "license": "MIT" + }, "node_modules/vue-demi": { "version": "0.14.10", "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.10.tgz", @@ -2964,6 +3518,32 @@ "typescript": ">=5.0.0" } }, + "node_modules/whatwg-mimetype": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", + "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/why-is-node-running": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", @@ -2980,6 +3560,126 @@ "engines": { "node": ">=8" } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ws": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", + "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } } } } diff --git a/frontend/package.json b/frontend/package.json index 521cffc..912ec8a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -21,7 +21,9 @@ "devDependencies": { "@types/node": "^22.10.2", "@vitejs/plugin-vue": "^5.2.1", + "@vue/test-utils": "^2.4.6", "autoprefixer": "^10.4.21", + "happy-dom": "^20.7.0", "postcss": "^8.5.3", "tailwindcss": "^3.4.17", "typescript": "^5.7.2", diff --git a/frontend/src/composables/useRequestState.test.ts b/frontend/src/composables/useRequestState.test.ts new file mode 100644 index 0000000..d8e8168 --- /dev/null +++ b/frontend/src/composables/useRequestState.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest"; +import { ApiRequestError } from "@/lib/api/errors"; +import { useRequestState } from "@/composables/useRequestState"; + +describe("useRequestState", () => { + it("initializes with all values null", () => { + const { errorMessage, errorRequestId, successMessage } = useRequestState(); + expect(errorMessage.value).toBeNull(); + expect(errorRequestId.value).toBeNull(); + expect(successMessage.value).toBeNull(); + }); + + it("assignError extracts message and requestId from ApiRequestError", () => { + const { errorMessage, errorRequestId, assignError } = useRequestState(); + assignError( + new ApiRequestError({ status: 409, code: "conflict", message: "Conflict occurred", requestId: "req-42" }), + "fallback", + ); + expect(errorMessage.value).toBe("Conflict occurred"); + expect(errorRequestId.value).toBe("req-42"); + }); + + it("assignError uses Error.message for generic errors", () => { + const { errorMessage, errorRequestId, assignError } = useRequestState(); + assignError(new Error("something broke"), "fallback"); + expect(errorMessage.value).toBe("something broke"); + expect(errorRequestId.value).toBeNull(); + }); + + it("assignError uses fallback for non-Error values", () => { + const { errorMessage, assignError } = useRequestState(); + assignError("not an error object", "fallback text"); + expect(errorMessage.value).toBe("fallback text"); + }); + + it("setSuccess sets the success message", () => { + const { successMessage, setSuccess } = useRequestState(); + setSuccess("Operation completed"); + expect(successMessage.value).toBe("Operation completed"); + }); + + it("clearAlerts resets all messages", () => { + const { errorMessage, errorRequestId, successMessage, assignError, setSuccess, clearAlerts } = useRequestState(); + assignError(new ApiRequestError({ status: 500, code: "err", message: "fail", requestId: "r1" }), "fb"); + setSuccess("ok"); + clearAlerts(); + expect(errorMessage.value).toBeNull(); + expect(errorRequestId.value).toBeNull(); + expect(successMessage.value).toBeNull(); + }); +}); diff --git a/frontend/src/composables/useRequestState.ts b/frontend/src/composables/useRequestState.ts new file mode 100644 index 0000000..0e4c783 --- /dev/null +++ b/frontend/src/composables/useRequestState.ts @@ -0,0 +1,40 @@ +import { ref } from "vue"; +import { ApiRequestError } from "@/lib/api/errors"; + +export function useRequestState() { + const errorMessage = ref(null); + const errorRequestId = ref(null); + const successMessage = ref(null); + + function clearAlerts(): void { + errorMessage.value = null; + errorRequestId.value = null; + successMessage.value = null; + } + + function assignError(error: unknown, fallback: string): void { + if (error instanceof ApiRequestError) { + errorMessage.value = error.message; + errorRequestId.value = error.requestId; + return; + } + if (error instanceof Error && error.message) { + errorMessage.value = error.message; + return; + } + errorMessage.value = fallback; + } + + function setSuccess(message: string): void { + successMessage.value = message; + } + + return { + errorMessage, + errorRequestId, + successMessage, + clearAlerts, + assignError, + setSuccess, + }; +} diff --git a/frontend/src/features/publications/components/PublicationTableRow.test.ts b/frontend/src/features/publications/components/PublicationTableRow.test.ts new file mode 100644 index 0000000..3a1110d --- /dev/null +++ b/frontend/src/features/publications/components/PublicationTableRow.test.ts @@ -0,0 +1,139 @@ +// @vitest-environment happy-dom +import { describe, expect, it } from "vitest"; +import { mount } from "@vue/test-utils"; +import PublicationTableRow from "./PublicationTableRow.vue"; +import type { PublicationItem } from "@/features/publications"; + +function buildItem(overrides: Partial = {}): PublicationItem { + return { + publication_id: 1, + scholar_profile_id: 10, + title: "Test Publication", + year: 2025, + citation_count: 42, + pub_url: "https://example.com/pub", + pdf_url: null, + pdf_status: null, + is_read: false, + is_favorite: false, + is_new_in_latest_run: true, + scholar_label: "Dr. Test", + first_seen_at: "2025-01-15T00:00:00Z", + display_identifier: null, + ...overrides, + } as PublicationItem; +} + +const defaultProps = { + item: buildItem(), + itemKey: "pub-1", + selected: false, + favoriteUpdating: false, + retrying: false, + canRetry: false, +}; + +describe("PublicationTableRow", () => { + it("renders the publication title", () => { + const wrapper = mount(PublicationTableRow, { props: defaultProps }); + expect(wrapper.text()).toContain("Test Publication"); + }); + + it("renders scholar label", () => { + const wrapper = mount(PublicationTableRow, { props: defaultProps }); + expect(wrapper.text()).toContain("Dr. Test"); + }); + + it("renders year and citation count", () => { + const wrapper = mount(PublicationTableRow, { props: defaultProps }); + expect(wrapper.text()).toContain("2025"); + expect(wrapper.text()).toContain("42"); + }); + + it("shows New badge when is_new_in_latest_run is true", () => { + const wrapper = mount(PublicationTableRow, { props: defaultProps }); + expect(wrapper.text()).toContain("New"); + }); + + it("shows Seen badge when is_new_in_latest_run is false", () => { + const wrapper = mount(PublicationTableRow, { + props: { ...defaultProps, item: buildItem({ is_new_in_latest_run: false }) }, + }); + expect(wrapper.text()).toContain("Seen"); + }); + + it("shows Unread badge for unread items", () => { + const wrapper = mount(PublicationTableRow, { props: defaultProps }); + expect(wrapper.text()).toContain("Unread"); + }); + + it("shows Read badge for read items", () => { + const wrapper = mount(PublicationTableRow, { + props: { ...defaultProps, item: buildItem({ is_read: true }) }, + }); + expect(wrapper.text()).toContain("Read"); + }); + + it("disables checkbox when item is read", () => { + const wrapper = mount(PublicationTableRow, { + props: { ...defaultProps, item: buildItem({ is_read: true }) }, + }); + const checkbox = wrapper.find('input[type="checkbox"]'); + expect((checkbox.element as HTMLInputElement).disabled).toBe(true); + }); + + it("emits toggle-selection when checkbox is changed", async () => { + const wrapper = mount(PublicationTableRow, { props: defaultProps }); + await wrapper.find('input[type="checkbox"]').trigger("change"); + expect(wrapper.emitted("toggle-selection")).toHaveLength(1); + }); + + it("emits toggle-favorite when star button is clicked", async () => { + const wrapper = mount(PublicationTableRow, { props: defaultProps }); + await wrapper.find(".favorite-star-button").trigger("click"); + expect(wrapper.emitted("toggle-favorite")).toHaveLength(1); + }); + + it("shows filled star when item is favorite", () => { + const wrapper = mount(PublicationTableRow, { + props: { ...defaultProps, item: buildItem({ is_favorite: true }) }, + }); + expect(wrapper.find(".favorite-star-on").exists()).toBe(true); + expect(wrapper.text()).toContain("★"); + }); + + it("shows empty star when item is not favorite", () => { + const wrapper = mount(PublicationTableRow, { props: defaultProps }); + expect(wrapper.find(".favorite-star-off").exists()).toBe(true); + expect(wrapper.text()).toContain("☆"); + }); + + it("shows Available link when pdf_url is set", () => { + const wrapper = mount(PublicationTableRow, { + props: { ...defaultProps, item: buildItem({ pdf_url: "https://example.com/paper.pdf" }) }, + }); + expect(wrapper.text()).toContain("Available"); + }); + + it("shows retry button when canRetry is true and no pdf_url", () => { + const wrapper = mount(PublicationTableRow, { + props: { ...defaultProps, canRetry: true }, + }); + expect(wrapper.text()).toContain("Missing (Retry)"); + }); + + it("emits retry-pdf when retry button is clicked", async () => { + const wrapper = mount(PublicationTableRow, { + props: { ...defaultProps, canRetry: true }, + }); + await wrapper.find(".pdf-retry-button").trigger("click"); + expect(wrapper.emitted("retry-pdf")).toHaveLength(1); + }); + + it("shows Retrying... label when retrying", () => { + const wrapper = mount(PublicationTableRow, { + props: { ...defaultProps, canRetry: true, retrying: true }, + }); + expect(wrapper.text()).toContain("Retrying..."); + }); +}); diff --git a/frontend/src/features/publications/components/PublicationTableRow.vue b/frontend/src/features/publications/components/PublicationTableRow.vue new file mode 100644 index 0000000..01307ff --- /dev/null +++ b/frontend/src/features/publications/components/PublicationTableRow.vue @@ -0,0 +1,178 @@ + + + + + diff --git a/frontend/src/features/publications/components/PublicationToolbar.test.ts b/frontend/src/features/publications/components/PublicationToolbar.test.ts new file mode 100644 index 0000000..2d1a39c --- /dev/null +++ b/frontend/src/features/publications/components/PublicationToolbar.test.ts @@ -0,0 +1,89 @@ +// @vitest-environment happy-dom +import { describe, expect, it } from "vitest"; +import { mount } from "@vue/test-utils"; +import PublicationToolbar from "./PublicationToolbar.vue"; + +const defaultProps = { + mode: "all" as const, + selectedScholarFilter: "", + searchQuery: "", + favoriteOnly: false, + actionBusy: false, + loading: false, + scholars: [], + startRunDisabled: false, + startRunDisabledReason: null, + startRunButtonLabel: "Check now", +}; + +describe("PublicationToolbar", () => { + it("renders mode, scholar filter, and search inputs", () => { + const wrapper = mount(PublicationToolbar, { props: defaultProps }); + expect(wrapper.text()).toContain("Status"); + expect(wrapper.text()).toContain("Scholar"); + expect(wrapper.text()).toContain("Search"); + }); + + it("renders the start run button with provided label", () => { + const wrapper = mount(PublicationToolbar, { props: defaultProps }); + expect(wrapper.text()).toContain("Check now"); + }); + + it("disables start run button when startRunDisabled is true", () => { + const wrapper = mount(PublicationToolbar, { + props: { ...defaultProps, startRunDisabled: true, startRunDisabledReason: "Cooldown active" }, + }); + const buttons = wrapper.findAll("button"); + const runButton = buttons.find((b) => b.text().includes("Check now")); + expect(runButton?.attributes("disabled")).toBeDefined(); + }); + + it("emits start-run when start button is clicked", async () => { + const wrapper = mount(PublicationToolbar, { props: defaultProps }); + const buttons = wrapper.findAll("button"); + const runButton = buttons.find((b) => b.text().includes("Check now")); + await runButton?.trigger("click"); + expect(wrapper.emitted("start-run")).toHaveLength(1); + }); + + it("emits favorite-only-changed when star filter is clicked", async () => { + const wrapper = mount(PublicationToolbar, { props: defaultProps }); + await wrapper.find(".favorite-filter-button").trigger("click"); + expect(wrapper.emitted("favorite-only-changed")).toHaveLength(1); + }); + + it("shows active star filter style when favoriteOnly is true", () => { + const wrapper = mount(PublicationToolbar, { props: { ...defaultProps, favoriteOnly: true } }); + expect(wrapper.find(".favorite-filter-on").exists()).toBe(true); + }); + + it("shows inactive star filter style when favoriteOnly is false", () => { + const wrapper = mount(PublicationToolbar, { props: defaultProps }); + expect(wrapper.find(".favorite-filter-off").exists()).toBe(true); + }); + + it("renders scholar options from props", () => { + const scholars = [ + { id: 1, scholar_id: "abc123", display_name: "Alice", profile_image_url: null, profile_image_source: "none" as const, is_enabled: true, baseline_completed: true, last_run_dt: null, last_run_status: null }, + { id: 2, scholar_id: "def456", display_name: "", profile_image_url: null, profile_image_source: "none" as const, is_enabled: true, baseline_completed: true, last_run_dt: null, last_run_status: null }, + ]; + const wrapper = mount(PublicationToolbar, { props: { ...defaultProps, scholars } }); + expect(wrapper.text()).toContain("Alice"); + expect(wrapper.text()).toContain("def456"); + }); + + it("shows clear button only when search query is non-empty", () => { + const empty = mount(PublicationToolbar, { props: defaultProps }); + expect(empty.text()).not.toContain("Clear"); + + const withQuery = mount(PublicationToolbar, { props: { ...defaultProps, searchQuery: "test" } }); + expect(withQuery.text()).toContain("Clear"); + }); + + it("emits reset-search when clear button is clicked", async () => { + const wrapper = mount(PublicationToolbar, { props: { ...defaultProps, searchQuery: "test" } }); + const clearButton = wrapper.findAll("button").find((b) => b.text().includes("Clear")); + await clearButton?.trigger("click"); + expect(wrapper.emitted("reset-search")).toHaveLength(1); + }); +}); diff --git a/frontend/src/features/publications/components/PublicationToolbar.vue b/frontend/src/features/publications/components/PublicationToolbar.vue new file mode 100644 index 0000000..4f34939 --- /dev/null +++ b/frontend/src/features/publications/components/PublicationToolbar.vue @@ -0,0 +1,148 @@ + + + + + diff --git a/frontend/src/features/publications/composables/usePublicationData.test.ts b/frontend/src/features/publications/composables/usePublicationData.test.ts new file mode 100644 index 0000000..e06d9c7 --- /dev/null +++ b/frontend/src/features/publications/composables/usePublicationData.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it, vi, beforeEach } from "vitest"; +import { createPinia, setActivePinia } from "pinia"; + +vi.mock("vue-router", () => ({ + useRoute: vi.fn(() => ({ query: {} })), + useRouter: vi.fn(() => ({ replace: vi.fn() })), +})); + +vi.mock("@/features/publications", async (importOriginal) => { + const original = (await importOriginal()) as Record; + return { + ...original, + listPublications: vi.fn(), + }; +}); + +vi.mock("@/features/scholars", () => ({ + listScholars: vi.fn(), +})); + +import { listPublications } from "@/features/publications"; +import { listScholars } from "@/features/scholars"; +import { usePublicationData } from "./usePublicationData"; + +const mockedListPublications = vi.mocked(listPublications); +const mockedListScholars = vi.mocked(listScholars); + +describe("usePublicationData", () => { + beforeEach(() => { + setActivePinia(createPinia()); + mockedListPublications.mockReset(); + mockedListScholars.mockReset(); + }); + + it("initializes with default filter values", () => { + const data = usePublicationData(); + expect(data.mode.value).toBe("all"); + expect(data.favoriteOnly.value).toBe(false); + expect(data.searchQuery.value).toBe(""); + expect(data.selectedScholarFilter.value).toBe(""); + expect(data.loading.value).toBe(true); + }); + + it("loadScholarFilters calls listScholars", async () => { + mockedListScholars.mockResolvedValueOnce([ + { id: 1, scholar_id: "abc", display_name: "Test", profile_image_url: null, profile_image_source: "none" as const, is_enabled: true, baseline_completed: true, last_run_dt: null, last_run_status: null }, + ]); + const data = usePublicationData(); + await data.loadScholarFilters(); + expect(mockedListScholars).toHaveBeenCalled(); + expect(data.scholars.value).toHaveLength(1); + }); + + it("loadPublications calls listPublications with current filters", async () => { + mockedListPublications.mockResolvedValueOnce({ + publications: [], + mode: "all" as const, + favorite_only: false, + selected_scholar_profile_id: null, + unread_count: 0, + favorites_count: 0, + latest_count: 0, + new_count: 0, + total_count: 0, + page: 1, + page_size: 25, + snapshot: "", + has_next: false, + has_prev: false, + }); + const data = usePublicationData(); + await data.loadPublications(); + expect(mockedListPublications).toHaveBeenCalled(); + }); + + it("resetPageAndSnapshot resets page to 1", () => { + const data = usePublicationData(); + data.currentPage.value = 5; + data.resetPageAndSnapshot(); + expect(data.currentPage.value).toBe(1); + }); + + it("toggleSort reverses direction for same key", async () => { + mockedListPublications.mockResolvedValue({ + publications: [], + mode: "all" as const, + favorite_only: false, + selected_scholar_profile_id: null, + unread_count: 0, + favorites_count: 0, + latest_count: 0, + new_count: 0, + total_count: 0, + page: 1, + page_size: 25, + snapshot: "", + has_next: false, + has_prev: false, + }); + const data = usePublicationData(); + const initialDirection = data.sortDirection.value; + await data.toggleSort(data.sortKey.value); + expect(data.sortDirection.value).toBe(initialDirection === "asc" ? "desc" : "asc"); + }); + + it("sortMarker returns arrow indicators", () => { + const data = usePublicationData(); + expect(data.sortMarker(data.sortKey.value)).toMatch(/[↑↓]/); + expect(data.sortMarker("year" as never)).toBe("↕"); + }); + + it("computed pagination properties work correctly", () => { + const data = usePublicationData(); + expect(data.hasPrevPage.value).toBe(false); + expect(data.totalPages.value).toBe(1); + expect(data.totalCount.value).toBe(0); + }); +}); diff --git a/frontend/src/features/publications/composables/usePublicationData.ts b/frontend/src/features/publications/composables/usePublicationData.ts new file mode 100644 index 0000000..e48209e --- /dev/null +++ b/frontend/src/features/publications/composables/usePublicationData.ts @@ -0,0 +1,338 @@ +import { computed, ref, watch } from "vue"; +import { useRoute, useRouter } from "vue-router"; +import { + listPublications, + type PublicationItem, + type PublicationMode, + type PublicationsResult, +} from "@/features/publications"; +import { listScholars, type ScholarProfile } from "@/features/scholars"; +import { ApiRequestError } from "@/lib/api/errors"; +import { useRunStatusStore } from "@/stores/run_status"; + +export type PublicationSortKey = + | "title" + | "scholar" + | "year" + | "citations" + | "first_seen" + | "pdf_status"; + +export function publicationKey(item: PublicationItem): string { + return `${item.scholar_profile_id}:${item.publication_id}`; +} + +export function usePublicationData() { + const loading = ref(true); + const mode = ref("all"); + const favoriteOnly = ref(false); + const selectedScholarFilter = ref(""); + const searchQuery = ref(""); + const sortKey = ref("first_seen"); + const sortDirection = ref<"asc" | "desc">("desc"); + const currentPage = ref(1); + const pageSize = ref("50"); + const publicationSnapshot = ref(null); + + const scholars = ref([]); + const listState = ref(null); + + const errorMessage = ref(null); + const errorRequestId = ref(null); + const successMessage = ref(null); + const route = useRoute(); + const router = useRouter(); + const textCollator = new Intl.Collator(undefined, { sensitivity: "base", numeric: true }); + const runStatus = useRunStatusStore(); + + // --- Route sync --- + + function normalizeScholarFilterQuery(value: unknown): string { + if (Array.isArray(value)) return normalizeScholarFilterQuery(value[0]); + if (typeof value !== "string") return ""; + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? String(parsed) : ""; + } + + function normalizeFavoriteOnlyQuery(value: unknown): boolean { + if (Array.isArray(value)) return normalizeFavoriteOnlyQuery(value[0]); + if (typeof value !== "string") return false; + const normalized = value.trim().toLowerCase(); + return normalized === "1" || normalized === "true" || normalized === "yes"; + } + + function normalizePageQuery(value: unknown): number { + if (Array.isArray(value)) return normalizePageQuery(value[0]); + if (typeof value !== "string") return 1; + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : 1; + } + + function syncFiltersFromRoute(): boolean { + let changed = false; + const nextScholar = normalizeScholarFilterQuery(route.query.scholar); + const nextFavoriteOnly = normalizeFavoriteOnlyQuery(route.query.favorite); + const nextPage = normalizePageQuery(route.query.page); + if (selectedScholarFilter.value !== nextScholar) { selectedScholarFilter.value = nextScholar; changed = true; } + if (favoriteOnly.value !== nextFavoriteOnly) { favoriteOnly.value = nextFavoriteOnly; changed = true; } + if (currentPage.value !== nextPage) { currentPage.value = nextPage; changed = true; } + return changed; + } + + async function syncFiltersToRoute(): Promise { + const nextScholar = selectedScholarFilter.value.trim(); + const currentScholar = normalizeScholarFilterQuery(route.query.scholar); + const currentFavoriteOnly = normalizeFavoriteOnlyQuery(route.query.favorite); + const currentPageQuery = normalizePageQuery(route.query.page); + if ( + nextScholar === currentScholar + && favoriteOnly.value === currentFavoriteOnly + && currentPage.value === currentPageQuery + ) { + return; + } + await router.replace({ + query: { + ...route.query, + scholar: nextScholar || undefined, + favorite: favoriteOnly.value ? "1" : undefined, + page: currentPage.value > 1 ? String(currentPage.value) : undefined, + }, + }); + } + + // --- Sorting helpers --- + + function publicationSortValue(item: PublicationItem, key: PublicationSortKey): number | string { + if (key === "title") return item.title; + if (key === "scholar") return item.scholar_label; + if (key === "year") return item.year ?? -1; + if (key === "citations") return item.citation_count; + if (key === "pdf_status") { + if (item.pdf_url || item.pdf_status === "resolved") return 4; + if (item.pdf_status === "running") return 3; + if (item.pdf_status === "queued") return 2; + if (item.pdf_status === "failed") return 0; + return 1; + } + const timestamp = Date.parse(item.first_seen_at); + return Number.isNaN(timestamp) ? 0 : timestamp; + } + + // --- Computed --- + + const selectedScholarName = computed(() => { + const selectedId = Number(selectedScholarFilter.value); + if (!Number.isInteger(selectedId) || selectedId <= 0) return "all scholars"; + const profile = scholars.value.find((item) => item.id === selectedId); + return profile ? (profile.display_name || profile.scholar_id) : "the selected scholar"; + }); + + const filteredPublications = computed(() => { + let stream = [...runStatus.livePublications]; + if (favoriteOnly.value) stream = stream.filter((p) => p.is_favorite); + if (mode.value === "unread") stream = stream.filter((p) => !p.is_read); + const selectedScholarId = Number(selectedScholarFilter.value); + if (Number.isInteger(selectedScholarId) && selectedScholarId > 0) { + stream = stream.filter((p) => p.scholar_profile_id === selectedScholarId); + } + const base = listState.value?.publications ?? []; + const merged = [...stream, ...base]; + const seenKeys = new Set(); + const deduped: typeof base = []; + for (const item of merged) { + const key = publicationKey(item); + if (!seenKeys.has(key)) { seenKeys.add(key); deduped.push(item); } + } + return deduped; + }); + + const sortedPublications = computed(() => { + const direction = sortDirection.value === "asc" ? 1 : -1; + return [...filteredPublications.value].sort((left, right) => { + const leftValue = publicationSortValue(left, sortKey.value); + const rightValue = publicationSortValue(right, sortKey.value); + let comparison = 0; + if (typeof leftValue === "string" && typeof rightValue === "string") { + comparison = textCollator.compare(leftValue, rightValue); + } else { + comparison = Number(leftValue) - Number(rightValue); + } + if (comparison !== 0) return comparison * direction; + return right.publication_id - left.publication_id; + }); + }); + + const visibleUnreadKeys = computed(() => { + const keys = new Set(); + for (const item of sortedPublications.value) { + if (!item.is_read) keys.add(publicationKey(item)); + } + return keys; + }); + + const pageSizeValue = computed(() => { + const parsed = Number(pageSize.value); + if (!Number.isInteger(parsed)) return 50; + return Math.max(10, Math.min(200, parsed)); + }); + const hasNextPage = computed(() => Boolean(listState.value?.has_next)); + const hasPrevPage = computed(() => Boolean(listState.value?.has_prev)); + const totalPages = computed(() => { + if (!listState.value) return 1; + return Math.max(1, Math.ceil(listState.value.total_count / Math.max(listState.value.page_size, 1))); + }); + const totalCount = computed(() => listState.value?.total_count ?? 0); + const visibleCount = computed(() => sortedPublications.value.length); + const visibleUnreadCount = computed(() => visibleUnreadKeys.value.size); + const visibleFavoriteCount = computed( + () => sortedPublications.value.filter((item) => item.is_favorite).length, + ); + + // --- Data loading --- + + async function loadScholarFilters(): Promise { + try { scholars.value = await listScholars(); } catch { scholars.value = []; } + } + + function selectedScholarId(): number | undefined { + const parsed = Number(selectedScholarFilter.value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined; + } + + async function loadPublications(): Promise { + loading.value = true; + errorMessage.value = null; + errorRequestId.value = null; + try { + listState.value = await listPublications({ + mode: mode.value, + favoriteOnly: favoriteOnly.value, + scholarProfileId: selectedScholarId(), + search: searchQuery.value.trim() || undefined, + sortBy: sortKey.value, + sortDir: sortDirection.value, + page: currentPage.value, + pageSize: pageSizeValue.value, + snapshot: publicationSnapshot.value ?? undefined, + }); + publicationSnapshot.value = listState.value.snapshot; + currentPage.value = listState.value.page; + pageSize.value = String(listState.value.page_size); + } catch (error) { + listState.value = null; + if (error instanceof ApiRequestError) { + errorMessage.value = error.message; + errorRequestId.value = error.requestId; + } else { + errorMessage.value = "Unable to load publications."; + } + } finally { + loading.value = false; + } + } + + function resetPageAndSnapshot(): void { + currentPage.value = 1; + publicationSnapshot.value = null; + } + + async function toggleSort(nextKey: PublicationSortKey): Promise { + if (sortKey.value === nextKey) { + sortDirection.value = sortDirection.value === "asc" ? "desc" : "asc"; + } else { + sortKey.value = nextKey; + sortDirection.value = nextKey === "first_seen" || nextKey === "pdf_status" ? "desc" : "asc"; + } + resetPageAndSnapshot(); + await loadPublications(); + } + + function sortMarker(key: PublicationSortKey): string { + if (sortKey.value !== key) return "↕"; + return sortDirection.value === "asc" ? "↑" : "↓"; + } + + function replacePublication(updated: PublicationItem): void { + if (!listState.value) return; + listState.value.publications = listState.value.publications.map((item) => + publicationKey(item) !== publicationKey(updated) ? item : updated, + ); + } + + // --- Search debounce watcher setup --- + + let searchDebounceTimer: ReturnType | null = null; + watch(searchQuery, () => { + if (searchDebounceTimer !== null) clearTimeout(searchDebounceTimer); + searchDebounceTimer = setTimeout(() => { + resetPageAndSnapshot(); + void loadPublications(); + }, 300); + }); + + // --- Run-triggered refresh watcher --- + + watch( + () => runStatus.latestRun, + async (nextRun, previousRun) => { + const nextRunId = nextRun && (nextRun.status === "running" || nextRun.status === "resolving") ? nextRun.id : null; + const previousRunId = previousRun && (previousRun.status === "running" || previousRun.status === "resolving") ? previousRun.id : null; + if (nextRunId === null || nextRunId === previousRunId) return; + resetPageAndSnapshot(); + await loadPublications(); + }, + ); + + // --- Route watcher --- + + watch( + () => [route.query.scholar, route.query.favorite, route.query.page], + async () => { + const previousScholar = selectedScholarFilter.value; + const previousFavorite = favoriteOnly.value; + const changed = syncFiltersFromRoute(); + if (!changed) return; + if (selectedScholarFilter.value !== previousScholar || favoriteOnly.value !== previousFavorite) { + publicationSnapshot.value = null; + } + await loadPublications(); + }, + ); + + return { + loading, + mode, + favoriteOnly, + selectedScholarFilter, + searchQuery, + sortKey, + sortDirection, + currentPage, + pageSize, + scholars, + listState, + errorMessage, + errorRequestId, + successMessage, + selectedScholarName, + sortedPublications, + visibleUnreadKeys, + pageSizeValue, + hasNextPage, + hasPrevPage, + totalPages, + totalCount, + visibleCount, + visibleUnreadCount, + visibleFavoriteCount, + loadScholarFilters, + loadPublications, + resetPageAndSnapshot, + toggleSort, + sortMarker, + replacePublication, + syncFiltersFromRoute, + syncFiltersToRoute, + }; +} diff --git a/frontend/src/features/scholars/components/ScholarBatchAdd.test.ts b/frontend/src/features/scholars/components/ScholarBatchAdd.test.ts new file mode 100644 index 0000000..2d76eac --- /dev/null +++ b/frontend/src/features/scholars/components/ScholarBatchAdd.test.ts @@ -0,0 +1,76 @@ +// @vitest-environment happy-dom +import { describe, expect, it } from "vitest"; +import { mount } from "@vue/test-utils"; +import ScholarBatchAdd from "./ScholarBatchAdd.vue"; + +const defaultProps = { saving: false, loading: false }; + +describe("ScholarBatchAdd", () => { + it("renders the heading and input", () => { + const wrapper = mount(ScholarBatchAdd, { props: defaultProps }); + expect(wrapper.text()).toContain("Add Scholar Profiles"); + expect(wrapper.find("textarea").exists()).toBe(true); + }); + + it("shows parsed ID count as 0 for empty input", () => { + const wrapper = mount(ScholarBatchAdd, { props: defaultProps }); + expect(wrapper.text()).toContain("Parsed IDs:"); + }); + + it("parses bare scholar IDs from textarea input", async () => { + const wrapper = mount(ScholarBatchAdd, { props: defaultProps }); + await wrapper.find("textarea").setValue("A-UbBTPM15wL"); + expect(wrapper.text()).toContain("1"); + }); + + it("parses scholar IDs from Google Scholar URLs", async () => { + const wrapper = mount(ScholarBatchAdd, { props: defaultProps }); + await wrapper.find("textarea").setValue( + "https://scholar.google.com/citations?hl=en&user=A-UbBTPM15wL", + ); + expect(wrapper.text()).toContain("1"); + }); + + it("deduplicates IDs from mixed input", async () => { + const wrapper = mount(ScholarBatchAdd, { props: defaultProps }); + await wrapper.find("textarea").setValue( + "A-UbBTPM15wL\nhttps://scholar.google.com/citations?user=A-UbBTPM15wL", + ); + expect(wrapper.text()).toContain("1"); + }); + + it("parses multiple IDs separated by commas", async () => { + const wrapper = mount(ScholarBatchAdd, { props: defaultProps }); + await wrapper.find("textarea").setValue("A-UbBTPM15wL, B-UbBTPM15wL"); + expect(wrapper.text()).toContain("2"); + }); + + it("emits add-scholars with parsed IDs on submit", async () => { + const wrapper = mount(ScholarBatchAdd, { props: defaultProps }); + await wrapper.find("textarea").setValue("A-UbBTPM15wL"); + await wrapper.find("form").trigger("submit"); + const emitted = wrapper.emitted("add-scholars"); + expect(emitted).toHaveLength(1); + expect(emitted![0][0]).toEqual(["A-UbBTPM15wL"]); + }); + + it("clears textarea after successful submit", async () => { + const wrapper = mount(ScholarBatchAdd, { props: defaultProps }); + const textarea = wrapper.find("textarea"); + await textarea.setValue("A-UbBTPM15wL"); + await wrapper.find("form").trigger("submit"); + expect((textarea.element as HTMLTextAreaElement).value).toBe(""); + }); + + it("does not emit when input contains no valid IDs", async () => { + const wrapper = mount(ScholarBatchAdd, { props: defaultProps }); + await wrapper.find("textarea").setValue("not-a-valid-id"); + await wrapper.find("form").trigger("submit"); + expect(wrapper.emitted("add-scholars")).toBeUndefined(); + }); + + it("shows Adding... label when saving prop is true", () => { + const wrapper = mount(ScholarBatchAdd, { props: { ...defaultProps, saving: true } }); + expect(wrapper.text()).toContain("Adding..."); + }); +}); diff --git a/frontend/src/features/scholars/components/ScholarBatchAdd.vue b/frontend/src/features/scholars/components/ScholarBatchAdd.vue new file mode 100644 index 0000000..bfafbda --- /dev/null +++ b/frontend/src/features/scholars/components/ScholarBatchAdd.vue @@ -0,0 +1,89 @@ + + +