Compare commits

..

1 commit

Author SHA1 Message Date
dependabot[bot]
38acacca9b
chore(deps-dev): bump vite from 5.4.21 to 7.3.1 in /frontend
Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 5.4.21 to 7.3.1.
- [Release notes](https://github.com/vitejs/vite/releases)
- [Changelog](https://github.com/vitejs/vite/blob/main/packages/vite/CHANGELOG.md)
- [Commits](https://github.com/vitejs/vite/commits/v7.3.1/packages/vite)

---
updated-dependencies:
- dependency-name: vite
  dependency-version: 7.3.1
  dependency-type: direct:development
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-03-01 21:56:28 +00:00
62 changed files with 1576 additions and 20869 deletions

View file

@ -8,23 +8,7 @@
"Bash(done)", "Bash(done)",
"Bash(grep -c 'def test_' tests/unit/*.py tests/unit/**/*.py tests/integration/*.py)", "Bash(grep -c 'def test_' tests/unit/*.py tests/unit/**/*.py tests/integration/*.py)",
"Bash(sort -t: -k2 -rn)", "Bash(sort -t: -k2 -rn)",
"Bash(wc -l frontend/src/pages/*.vue)", "Bash(wc -l frontend/src/pages/*.vue)"
"Bash(uv run ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
"Bash(ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
"Bash(npm install --package-lock-only)",
"Bash(python -m ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
"Bash(pip show ruff)",
"Bash(pip install ruff)",
"Bash(uvx ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
"Bash(git add app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py docs/website/package-lock.json)",
"Bash(git push)",
"Bash(uvx ruff format --check app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
"Bash(test -f docs/website/package-lock.json)",
"Bash(gh run list --branch openalex-size-reduction-scholar-hydration --limit 3 --json databaseId,status,conclusion,name,event)",
"Bash(gh run list --limit 10 --json databaseId,status,conclusion,name,event,headBranch)",
"Bash(gh run view 22554585841 --log-failed)",
"Bash(gh run rerun 22554585841 --failed)",
"Bash(bash scripts/premerge.sh)"
], ],
"additionalDirectories": [ "additionalDirectories": [
"/home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src" "/home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src"

View file

@ -28,7 +28,6 @@ DATABASE_POOL_MODE=auto
DATABASE_POOL_SIZE=5 DATABASE_POOL_SIZE=5
DATABASE_POOL_MAX_OVERFLOW=10 DATABASE_POOL_MAX_OVERFLOW=10
DATABASE_POOL_TIMEOUT_SECONDS=30 DATABASE_POOL_TIMEOUT_SECONDS=30
DATABASE_RESERVED_API_CONNECTIONS=3
# ------------------------------ # ------------------------------
# Frontend Dev Overrides # Frontend Dev Overrides

2
.gitignore vendored
View file

@ -18,5 +18,3 @@ frontend/dist/
frontend/.vite/ frontend/.vite/
docs/website/node_modules/ docs/website/node_modules/
docs/website/build/ docs/website/build/
docs/website/.docusaurus/
.claude/

View file

@ -5,7 +5,7 @@ import logging
from fastapi import APIRouter, Depends, Path, Query, Request from fastapi import APIRouter, Depends, Path, Query, Request
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_admin_user, get_api_current_user from app.api.deps import get_api_admin_user
from app.api.errors import ApiException from app.api.errors import ApiException
from app.api.responses import success_payload from app.api.responses import success_payload
from app.api.schemas import ( from app.api.schemas import (
@ -57,7 +57,7 @@ def _requested_by_value(*, payload, admin_user: User) -> str:
return from_payload or admin_user.email return from_payload or admin_user.email
def _serialize_pdf_queue_item(item, *, is_admin: bool = False) -> dict[str, object]: def _serialize_pdf_queue_item(item) -> dict[str, object]:
return { return {
"publication_id": item.publication_id, "publication_id": item.publication_id,
"title": item.title, "title": item.title,
@ -69,7 +69,7 @@ def _serialize_pdf_queue_item(item, *, is_admin: bool = False) -> dict[str, obje
"last_failure_detail": item.last_failure_detail, "last_failure_detail": item.last_failure_detail,
"last_source": item.last_source, "last_source": item.last_source,
"requested_by_user_id": item.requested_by_user_id, "requested_by_user_id": item.requested_by_user_id,
"requested_by_email": item.requested_by_email if is_admin else None, "requested_by_email": item.requested_by_email,
"queued_at": item.queued_at, "queued_at": item.queued_at,
"last_attempt_at": item.last_attempt_at, "last_attempt_at": item.last_attempt_at,
"resolved_at": item.resolved_at, "resolved_at": item.resolved_at,
@ -185,7 +185,7 @@ async def get_pdf_queue(
offset: int | None = Query(default=None, ge=0), offset: int | None = Query(default=None, ge=0),
status: str | None = Query(default=None), status: str | None = Query(default=None),
db_session: AsyncSession = Depends(get_db_session), db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user), admin_user: User = Depends(get_api_admin_user),
): ):
normalized_status = (status or "").strip().lower() or None normalized_status = (status or "").strip().lower() or None
resolved_page, resolved_limit, resolved_offset = _resolve_pdf_queue_paging( resolved_page, resolved_limit, resolved_offset = _resolve_pdf_queue_paging(
@ -204,7 +204,7 @@ async def get_pdf_queue(
logger, logger,
"info", "info",
"api.admin.db.pdf_queue_listed", "api.admin.db.pdf_queue_listed",
user_id=int(current_user.id), admin_user_id=int(admin_user.id),
page=int(resolved_page), page=int(resolved_page),
page_size=int(resolved_limit), page_size=int(resolved_limit),
offset=int(resolved_offset), offset=int(resolved_offset),
@ -215,7 +215,7 @@ async def get_pdf_queue(
return success_payload( return success_payload(
request, request,
data={ data={
"items": [_serialize_pdf_queue_item(item, is_admin=current_user.is_admin) for item in queue_page.items], "items": [_serialize_pdf_queue_item(item) for item in queue_page.items],
**_pdf_queue_page_data( **_pdf_queue_page_data(
total_count=queue_page.total_count, total_count=queue_page.total_count,
page=resolved_page, page=resolved_page,

View file

@ -55,7 +55,7 @@ def _apply_scholar_http_settings(payload: AdminScholarHttpSettingsUpdateRequest)
max_length=_MAX_ACCEPT_LANGUAGE_LENGTH, max_length=_MAX_ACCEPT_LANGUAGE_LENGTH,
) )
cookie = _normalize_header_value( cookie = _normalize_header_value(
payload.cookie or "", payload.cookie,
field_name="cookie", field_name="cookie",
max_length=_MAX_COOKIE_LENGTH, max_length=_MAX_COOKIE_LENGTH,
) )

View file

@ -56,7 +56,6 @@ def _drop_finished_task(task: asyncio.Task[Any]) -> None:
except Exception: except Exception:
structured_log(logger, "exception", "runs.background_task_failed") structured_log(logger, "exception", "runs.background_task_failed")
router = APIRouter(prefix="/runs", tags=["api-runs"]) router = APIRouter(prefix="/runs", tags=["api-runs"])
ACTIVE_RUN_STATUSES = {RunStatus.RUNNING, RunStatus.RESOLVING} ACTIVE_RUN_STATUSES = {RunStatus.RUNNING, RunStatus.RESOLVING}
@ -221,11 +220,11 @@ def _spawn_background_execution(
user_settings: Any, user_settings: Any,
idempotency_key: str | None, idempotency_key: str | None,
) -> None: ) -> None:
from app.db.background_session import background_session from app.db.session import get_session_factory
task = asyncio.create_task( task = asyncio.create_task(
ingest_service.execute_run( ingest_service.execute_run(
session_factory=background_session, session_factory=get_session_factory(),
run_id=run.id, run_id=run.id,
user_id=current_user.id, user_id=current_user.id,
scholars=scholars, scholars=scholars,

View file

@ -23,9 +23,6 @@ from app.api.schemas import (
DataImportEnvelope, DataImportEnvelope,
DataImportRequest, DataImportRequest,
MessageEnvelope, MessageEnvelope,
ScholarBulkCountEnvelope,
ScholarBulkIdsRequest,
ScholarBulkToggleRequest,
ScholarCreateRequest, ScholarCreateRequest,
ScholarEnvelope, ScholarEnvelope,
ScholarImageUrlUpdateRequest, ScholarImageUrlUpdateRequest,
@ -45,22 +42,6 @@ logger = logging.getLogger(__name__)
router = APIRouter(prefix="/scholars", tags=["api-scholars"]) router = APIRouter(prefix="/scholars", tags=["api-scholars"])
def _parse_ids_param(ids: str | None) -> list[int] | None:
if not ids:
return None
parts = [p.strip() for p in ids.split(",") if p.strip()]
if not parts:
return None
try:
return [int(p) for p in parts]
except ValueError as exc:
raise ApiException(
status_code=400,
code="invalid_ids_param",
message="The 'ids' parameter must be a comma-separated list of integers.",
) from exc
@router.get( @router.get(
"", "",
response_model=ScholarsListEnvelope, response_model=ScholarsListEnvelope,
@ -88,81 +69,16 @@ async def list_scholars(
) )
async def export_scholars_and_publications( async def export_scholars_and_publications(
request: Request, request: Request,
ids: str | None = Query(None, description="Comma-separated scholar profile IDs to export"),
db_session: AsyncSession = Depends(get_db_session), db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user), current_user: User = Depends(get_api_current_user),
): ):
scholar_profile_ids = _parse_ids_param(ids)
data = await import_export_service.export_user_data( data = await import_export_service.export_user_data(
db_session, db_session,
user_id=current_user.id, user_id=current_user.id,
scholar_profile_ids=scholar_profile_ids,
) )
return success_payload(request, data=data) return success_payload(request, data=data)
@router.post(
"/bulk-delete",
response_model=ScholarBulkCountEnvelope,
)
async def bulk_delete_scholars(
payload: ScholarBulkIdsRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
try:
deleted_count = await scholar_service.bulk_delete_scholars(
db_session,
user_id=current_user.id,
scholar_profile_ids=payload.scholar_profile_ids,
upload_dir=settings.scholar_image_upload_dir,
)
except scholar_service.ScholarServiceError as exc:
raise ApiException(
status_code=409,
code="scholar_bulk_delete_failed",
message=str(exc),
) from exc
structured_log(
logger,
"info",
"scholars.bulk_delete",
user_id=current_user.id,
requested_ids=payload.scholar_profile_ids,
deleted_count=deleted_count,
)
return success_payload(request, data={"deleted_count": deleted_count, "updated_count": 0})
@router.post(
"/bulk-toggle",
response_model=ScholarBulkCountEnvelope,
)
async def bulk_toggle_scholars(
payload: ScholarBulkToggleRequest,
request: Request,
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
updated_count = await scholar_service.bulk_toggle_scholars(
db_session,
user_id=current_user.id,
scholar_profile_ids=payload.scholar_profile_ids,
is_enabled=payload.is_enabled,
)
structured_log(
logger,
"info",
"scholars.bulk_toggle",
user_id=current_user.id,
requested_ids=payload.scholar_profile_ids,
is_enabled=payload.is_enabled,
updated_count=updated_count,
)
return success_payload(request, data={"deleted_count": 0, "updated_count": updated_count})
@router.post( @router.post(
"/import", "/import",
response_model=DataImportEnvelope, response_model=DataImportEnvelope,
@ -228,17 +144,18 @@ async def create_scholar(
message=str(exc), message=str(exc),
) from exc ) from exc
structured_log(logger, "info", "api.scholars.created", user_id=current_user.id, scholar_profile_id=created.id) structured_log(logger, "info", "api.scholars.created", user_id=current_user.id, scholar_profile_id=created.id)
await enqueue_initial_scrape_job_for_scholar( did_queue_initial_scrape = await enqueue_initial_scrape_job_for_scholar(
db_session, db_session,
profile=created, profile=created,
user_id=current_user.id, user_id=current_user.id,
) )
created = await hydrate_scholar_metadata_if_needed( if not did_queue_initial_scrape:
db_session, created = await hydrate_scholar_metadata_if_needed(
profile=created, db_session,
source=source, profile=created,
user_id=current_user.id, source=source,
) user_id=current_user.id,
)
return success_payload( return success_payload(
request, request,
@ -345,18 +262,11 @@ async def delete_scholar(
code="scholar_not_found", code="scholar_not_found",
message="Scholar not found.", message="Scholar not found.",
) )
try: await scholar_service.delete_scholar(
await scholar_service.delete_scholar( db_session,
db_session, profile=profile,
profile=profile, upload_dir=settings.scholar_image_upload_dir,
upload_dir=settings.scholar_image_upload_dir, )
)
except scholar_service.ScholarServiceError as exc:
raise ApiException(
status_code=409,
code="scholar_delete_failed",
message=str(exc),
) from exc
structured_log( structured_log(
logger, "info", "api.scholars.deleted", user_id=current_user.id, scholar_profile_id=scholar_profile_id logger, "info", "api.scholars.deleted", user_id=current_user.id, scholar_profile_id=scholar_profile_id
) )

View file

@ -2,7 +2,7 @@ from __future__ import annotations
from datetime import datetime from datetime import datetime
from pydantic import BaseModel, ConfigDict, Field, model_validator from pydantic import BaseModel, ConfigDict, Field
from app.api.schemas.common import ApiMeta from app.api.schemas.common import ApiMeta
@ -126,49 +126,13 @@ class DataExportEnvelope(BaseModel):
model_config = ConfigDict(extra="forbid") model_config = ConfigDict(extra="forbid")
class ScholarBulkIdsRequest(BaseModel):
scholar_profile_ids: list[int] = Field(..., min_length=1, max_length=500)
model_config = ConfigDict(extra="forbid")
class ScholarBulkToggleRequest(BaseModel):
scholar_profile_ids: list[int] = Field(..., min_length=1, max_length=500)
is_enabled: bool
model_config = ConfigDict(extra="forbid")
class ScholarBulkCountData(BaseModel):
deleted_count: int = 0
updated_count: int = 0
model_config = ConfigDict(extra="forbid")
class ScholarBulkCountEnvelope(BaseModel):
data: ScholarBulkCountData
meta: ApiMeta
model_config = ConfigDict(extra="forbid")
class DataImportRequest(BaseModel): class DataImportRequest(BaseModel):
schema_version: int | None = None schema_version: int | None = None
exported_at: str | None = None
scholars: list[ScholarExportItemData] = Field(default_factory=list) scholars: list[ScholarExportItemData] = Field(default_factory=list)
publications: list[PublicationExportItemData] = Field(default_factory=list) publications: list[PublicationExportItemData] = Field(default_factory=list)
model_config = ConfigDict(extra="forbid") model_config = ConfigDict(extra="forbid")
@model_validator(mode="before")
@classmethod
def unwrap_export_envelope(cls, values: dict) -> dict:
"""Accept the full export envelope format (with data/meta wrapper)."""
if isinstance(values, dict) and "data" in values and isinstance(values["data"], dict):
return values["data"]
return values
class DataImportResultData(BaseModel): class DataImportResultData(BaseModel):
scholars_created: int scholars_created: int

View file

@ -1,57 +0,0 @@
"""Semaphore-gated DB sessions for background tasks.
Background tasks (ingestion, PDF resolution, scheduler) acquire a semaphore
permit before checking out a database connection. The semaphore cap is
``pool_size + max_overflow - reserved_for_api``, which guarantees that at
least *reserved_for_api* connections remain available for API request
handlers at all times.
"""
from __future__ import annotations
import asyncio
import logging
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.session import get_session_factory
from app.logging_utils import structured_log
from app.settings import settings
logger = logging.getLogger(__name__)
_semaphore: asyncio.Semaphore | None = None
def _build_semaphore() -> asyncio.Semaphore:
pool_capacity = max(1, settings.database_pool_size) + max(0, settings.database_pool_max_overflow)
reserved = max(0, settings.database_reserved_api_connections)
limit = max(1, pool_capacity - reserved)
structured_log(
logger,
"info",
"db.background_semaphore_initialized",
pool_capacity=pool_capacity,
reserved_for_api=reserved,
background_limit=limit,
)
return asyncio.Semaphore(limit)
def get_background_semaphore() -> asyncio.Semaphore:
global _semaphore
if _semaphore is None:
_semaphore = _build_semaphore()
return _semaphore
@asynccontextmanager
async def background_session() -> AsyncIterator[AsyncSession]:
"""Yield a DB session after acquiring the background semaphore."""
semaphore = get_background_semaphore()
async with semaphore:
session_factory = get_session_factory()
async with session_factory() as session:
yield session

View file

@ -99,7 +99,9 @@ async def _run_serialized_fetch(
"arxiv.request_completed", "arxiv.request_completed",
status_code=int(response.status_code), status_code=int(response.status_code),
wait_seconds=wait_seconds, wait_seconds=wait_seconds,
cooldown_remaining_seconds=_cooldown_remaining_seconds(cooldown_until_value, now_utc=datetime.now(UTC)), cooldown_remaining_seconds=_cooldown_remaining_seconds(
cooldown_until_value, now_utc=datetime.now(UTC)
),
source_path=source_path, source_path=source_path,
) )
return response, hit_rate_limit return response, hit_rate_limit

View file

@ -37,40 +37,6 @@ def _sanitize_titles(publications: list) -> list[str]:
return titles return titles
# OpenAlex (and most HTTP servers) struggle with very long query strings.
# Non-ASCII characters expand significantly when percent-encoded in URLs,
# so a fixed batch count is unreliable. Instead we cap the combined
# byte-length of the pipe-joined title filter value to stay well within
# safe URL limits (~4 KiB of filter text ≈ ~6 KiB when percent-encoded).
_MAX_TITLE_FILTER_BYTES = 4_000
def _chunk_titles_by_url_length(
titles: list[str],
max_bytes: int = _MAX_TITLE_FILTER_BYTES,
) -> list[list[str]]:
"""Split *titles* into chunks whose pipe-joined UTF-8 size stays under *max_bytes*."""
chunks: list[list[str]] = []
current_chunk: list[str] = []
current_bytes = 0
for title in titles:
title_bytes = len(title.encode("utf-8"))
# Account for the pipe separator between titles
separator = 3 if current_chunk else 0 # len("|".encode) but url-encoded as %7C = 3
if current_chunk and current_bytes + separator + title_bytes > max_bytes:
chunks.append(current_chunk)
current_chunk = [title]
current_bytes = title_bytes
else:
current_chunk.append(title)
current_bytes += separator + title_bytes
if current_chunk:
chunks.append(current_chunk)
return chunks
class EnrichmentRunner: class EnrichmentRunner:
"""Post-run OpenAlex enrichment logic. """Post-run OpenAlex enrichment logic.
@ -190,34 +156,21 @@ class EnrichmentRunner:
resolved_key = openalex_api_key or settings.openalex_api_key resolved_key = openalex_api_key or settings.openalex_api_key
client = OpenAlexClient(api_key=resolved_key, mailto=settings.crossref_api_mailto) client = OpenAlexClient(api_key=resolved_key, mailto=settings.crossref_api_mailto)
batch_size = 25
now = datetime.now(UTC) now = datetime.now(UTC)
arxiv_lookup_allowed = True arxiv_lookup_allowed = True
all_titles = _sanitize_titles(publications) for i in range(0, len(publications), batch_size):
title_chunks = _chunk_titles_by_url_length(all_titles)
# Build a mapping from sanitized title → publications for batch association
title_to_pubs: dict[str, list[Publication]] = {}
for p in publications:
raw = getattr(p, "title_raw", None) or getattr(p, "title", None)
if not raw or not raw.strip():
continue
safe = " ".join(re.sub(r"[^\w\s]", " ", raw).split())
if safe:
title_to_pubs.setdefault(safe, []).append(p)
for title_chunk in title_chunks:
if await self.run_is_canceled(db_session, run_id=run_id): if await self.run_is_canceled(db_session, run_id=run_id):
structured_log(logger, "info", "ingestion.enrichment_aborted", run_id=run_id) structured_log(logger, "info", "ingestion.enrichment_aborted", run_id=run_id)
return return
batch = [] batch = publications[i : i + batch_size]
for t in title_chunk: titles = _sanitize_titles(batch)
batch.extend(title_to_pubs.get(t, [])) if not titles:
if not batch:
continue continue
try: try:
openalex_works = await client.get_works_by_filter( openalex_works = await client.get_works_by_filter(
{"title.search": "|".join(title_chunk)}, limit=len(title_chunk) * 3 {"title.search": "|".join(titles)}, limit=batch_size * 3
) )
except OpenAlexBudgetExhaustedError: except OpenAlexBudgetExhaustedError:
structured_log(logger, "warning", "ingestion.openalex_budget_exhausted", run_id=run_id) structured_log(logger, "warning", "ingestion.openalex_budget_exhausted", run_id=run_id)
@ -336,37 +289,19 @@ class EnrichmentRunner:
client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto) client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto)
all_titles = _sanitize_titles(publications) batch_size = 25
title_chunks = _chunk_titles_by_url_length(all_titles)
# Build title → publication mapping for batch association
title_to_pubs: dict[str, list[PublicationCandidate]] = {}
for p in publications:
raw = getattr(p, "title", None)
if not raw or not raw.strip():
continue
safe = " ".join(re.sub(r"[^\w\s]", " ", raw).split())
if safe:
title_to_pubs.setdefault(safe, []).append(p)
# Collect publications that had no sanitizable title — pass through as-is
pubs_with_titles = set()
for titles in title_to_pubs.values():
for p in titles:
pubs_with_titles.add(id(p))
enriched: list[PublicationCandidate] = [] enriched: list[PublicationCandidate] = []
for title_chunk in title_chunks: for i in range(0, len(publications), batch_size):
batch = [] batch = publications[i : i + batch_size]
for t in title_chunk: titles = _sanitize_titles(batch)
batch.extend(title_to_pubs.get(t, [])) if not titles:
if not batch: enriched.extend(batch)
continue continue
try: try:
openalex_works = await client.get_works_by_filter( openalex_works = await client.get_works_by_filter(
{"title.search": "|".join(title_chunk)}, limit=len(title_chunk) * 3 {"title.search": "|".join(titles)}, limit=batch_size * 3
) )
except Exception as e: except Exception as e:
structured_log( structured_log(
@ -400,10 +335,4 @@ class EnrichmentRunner:
enriched.append(new_p) enriched.append(new_p)
else: else:
enriched.append(p) enriched.append(p)
# Append publications that had no sanitizable title (pass-through)
for p in publications:
if id(p) not in pubs_with_titles:
enriched.append(p)
return enriched return enriched

View file

@ -5,8 +5,8 @@ from datetime import UTC, datetime
from sqlalchemy import select from sqlalchemy import select
from app.db.background_session import background_session
from app.db.models import QueueItemStatus, RunTriggerType, ScholarProfile, UserSetting from app.db.models import QueueItemStatus, RunTriggerType, ScholarProfile, UserSetting
from app.db.session import get_session_factory
from app.logging_utils import structured_log from app.logging_utils import structured_log
from app.services.ingestion import queue as queue_service from app.services.ingestion import queue as queue_service
from app.services.ingestion.application import ( from app.services.ingestion.application import (
@ -57,7 +57,8 @@ class QueueJobRunner:
async def drain_continuation_queue(self) -> None: async def drain_continuation_queue(self) -> None:
now = datetime.now(UTC) now = datetime.now(UTC)
async with background_session() as session: session_factory = get_session_factory()
async with session_factory() as session:
jobs = await queue_service.list_due_jobs( jobs = await queue_service.list_due_jobs(
session, session,
now=now, now=now,
@ -72,7 +73,8 @@ class QueueJobRunner:
) -> bool: ) -> bool:
if job.attempt_count < self._continuation_max_attempts: if job.attempt_count < self._continuation_max_attempts:
return False return False
async with background_session() as session: session_factory = get_session_factory()
async with session_factory() as session:
dropped = await queue_service.mark_dropped( dropped = await queue_service.mark_dropped(
session, session,
job_id=job.id, job_id=job.id,
@ -96,7 +98,8 @@ class QueueJobRunner:
self, self,
job: queue_service.ContinuationQueueJob, job: queue_service.ContinuationQueueJob,
) -> bool: ) -> bool:
async with background_session() as session: session_factory = get_session_factory()
async with session_factory() as session:
queue_item = await queue_service.mark_retrying(session, job_id=job.id) queue_item = await queue_service.mark_retrying(session, job_id=job.id)
await session.commit() await session.commit()
if queue_item is None: if queue_item is None:
@ -107,7 +110,8 @@ class QueueJobRunner:
self, self,
job: queue_service.ContinuationQueueJob, job: queue_service.ContinuationQueueJob,
) -> bool: ) -> bool:
async with background_session() as session: session_factory = get_session_factory()
async with session_factory() as session:
scholar_result = await session.execute( scholar_result = await session.execute(
select(ScholarProfile.id).where( select(ScholarProfile.id).where(
ScholarProfile.user_id == job.user_id, ScholarProfile.user_id == job.user_id,
@ -136,7 +140,8 @@ class QueueJobRunner:
return False return False
async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None: async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
async with background_session() as recovery_session: session_factory = get_session_factory()
async with session_factory() as recovery_session:
await queue_service.reschedule_job( await queue_service.reschedule_job(
recovery_session, recovery_session,
job_id=job.id, job_id=job.id,
@ -162,7 +167,8 @@ class QueueJobRunner:
self._tick_seconds, self._tick_seconds,
int(exc.safety_state.get("cooldown_remaining_seconds") or 0), int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
) )
async with background_session() as recovery_session: session_factory = get_session_factory()
async with session_factory() as recovery_session:
await queue_service.reschedule_job( await queue_service.reschedule_job(
recovery_session, recovery_session,
job_id=job.id, job_id=job.id,
@ -187,7 +193,8 @@ class QueueJobRunner:
*, *,
exc: Exception, exc: Exception,
) -> None: ) -> None:
async with background_session() as recovery_session: session_factory = get_session_factory()
async with session_factory() as recovery_session:
queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id) queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
if queue_item is None: if queue_item is None:
await recovery_session.commit() await recovery_session.commit()
@ -231,7 +238,8 @@ class QueueJobRunner:
) )
async def _load_request_delay_for_user(self, user_id: int, *, floor: int) -> int: async def _load_request_delay_for_user(self, user_id: int, *, floor: int) -> int:
async with background_session() as session: session_factory = get_session_factory()
async with session_factory() as session:
result = await session.execute( result = await session.execute(
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id) select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
) )
@ -245,7 +253,8 @@ class QueueJobRunner:
request_delay_floor: int, request_delay_floor: int,
): ):
request_delay_seconds = await self._load_request_delay_for_user(job.user_id, floor=request_delay_floor) request_delay_seconds = await self._load_request_delay_for_user(job.user_id, floor=request_delay_floor)
async with background_session() as session: session_factory = get_session_factory()
async with session_factory() as session:
ingestion = ScholarIngestionService(source=self._source) ingestion = ScholarIngestionService(source=self._source)
try: try:
return await ingestion.run_for_user( return await ingestion.run_for_user(
@ -357,7 +366,8 @@ class QueueJobRunner:
) )
async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None: async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
async with background_session() as session: session_factory = get_session_factory()
async with session_factory() as session:
if int(run_summary.failed_count) <= 0: if int(run_summary.failed_count) <= 0:
await self._finalize_successful_queue_job(session, job, run_summary) await self._finalize_successful_queue_job(session, job, run_summary)
else: else:

View file

@ -8,13 +8,13 @@ from typing import Any
from sqlalchemy import select from sqlalchemy import select
from app.db.background_session import background_session
from app.db.models import ( from app.db.models import (
CrawlRun, CrawlRun,
RunTriggerType, RunTriggerType,
User, User,
UserSetting, UserSetting,
) )
from app.db.session import get_session_factory
from app.logging_utils import structured_log from app.logging_utils import structured_log
from app.services.ingestion.application import ( from app.services.ingestion.application import (
RunAlreadyInProgressError, RunAlreadyInProgressError,
@ -148,7 +148,8 @@ class SchedulerService:
await self._run_candidate(candidate) await self._run_candidate(candidate)
async def _load_candidate_rows(self) -> list[Any]: async def _load_candidate_rows(self) -> list[Any]:
async with background_session() as session: session_factory = get_session_factory()
async with session_factory() as session:
result = await session.execute( result = await session.execute(
select( select(
UserSetting.user_id, UserSetting.user_id,
@ -203,7 +204,8 @@ class SchedulerService:
return candidates return candidates
async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool: async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
async with background_session() as session: session_factory = get_session_factory()
async with session_factory() as session:
result = await session.execute( result = await session.execute(
select(CrawlRun.start_dt) select(CrawlRun.start_dt)
.where( .where(
@ -225,7 +227,8 @@ class SchedulerService:
*, *,
candidate: _AutoRunCandidate, candidate: _AutoRunCandidate,
): ):
async with background_session() as session: session_factory = get_session_factory()
async with session_factory() as session:
ingestion = ScholarIngestionService(source=self._source) ingestion = ScholarIngestionService(source=self._source)
try: try:
return await ingestion.run_for_user( return await ingestion.run_for_user(
@ -286,12 +289,9 @@ class SchedulerService:
async def _drain_pdf_queue(self) -> None: async def _drain_pdf_queue(self) -> None:
from app.services.publications.pdf_queue import drain_ready_jobs from app.services.publications.pdf_queue import drain_ready_jobs
from app.services.publications.pdf_queue_resolution import is_budget_cooldown_active
if is_budget_cooldown_active(): session_factory = get_session_factory()
return async with session_factory() as session:
async with background_session() as session:
try: try:
processed = await drain_ready_jobs( processed = await drain_ready_jobs(
session, session,

View file

@ -3,7 +3,6 @@ from __future__ import annotations
import asyncio import asyncio
import logging import logging
import random import random
from collections.abc import Callable, Coroutine
from datetime import UTC, datetime from datetime import UTC, datetime
from typing import Any from typing import Any
@ -275,7 +274,6 @@ async def _run_first_pass(
request_delay_seconds: int, request_delay_seconds: int,
queue_delay_seconds: int, queue_delay_seconds: int,
progress: RunProgress, progress: RunProgress,
on_progress: Callable[[int, int], Coroutine[Any, Any, None]] | None = None,
) -> dict[int, int]: ) -> dict[int, int]:
first_pass_cstarts: dict[int, int] = {} first_pass_cstarts: dict[int, int] = {}
for index, scholar in enumerate(scholars): for index, scholar in enumerate(scholars):
@ -312,8 +310,6 @@ async def _run_first_pass(
scholars_remaining=len(scholars) - index - 1, scholars_remaining=len(scholars) - index - 1,
) )
return first_pass_cstarts return first_pass_cstarts
if on_progress is not None:
await on_progress(1, 0)
resume_cstart = outcome.result_entry.get("continuation_cstart") resume_cstart = outcome.result_entry.get("continuation_cstart")
if resume_cstart is not None and int(resume_cstart) > start_cstart: if resume_cstart is not None and int(resume_cstart) > start_cstart:
first_pass_cstarts[int(scholar.id)] = int(resume_cstart) first_pass_cstarts[int(scholar.id)] = int(resume_cstart)
@ -334,7 +330,6 @@ async def _run_depth_pass(
auto_queue_continuations: bool, auto_queue_continuations: bool,
queue_delay_seconds: int, queue_delay_seconds: int,
progress: RunProgress, progress: RunProgress,
on_progress: Callable[[int, int], Coroutine[Any, Any, None]] | None = None,
) -> None: ) -> None:
for index, scholar in enumerate(scholars): for index, scholar in enumerate(scholars):
resume_cstart = first_pass_cstarts.get(int(scholar.id)) resume_cstart = first_pass_cstarts.get(int(scholar.id))
@ -372,8 +367,6 @@ async def _run_depth_pass(
scholars_remaining=len(scholars) - index - 1, scholars_remaining=len(scholars) - index - 1,
) )
break break
if on_progress is not None:
await on_progress(0, 1)
async def run_scholar_iteration( async def run_scholar_iteration(
@ -394,8 +387,6 @@ async def run_scholar_iteration(
auto_queue_continuations: bool, auto_queue_continuations: bool,
queue_delay_seconds: int, queue_delay_seconds: int,
) -> RunProgress: ) -> RunProgress:
from app.services.runs.events import run_events
progress = RunProgress() progress = RunProgress()
scholar_kwargs: dict[str, Any] = { scholar_kwargs: dict[str, Any] = {
"request_delay_seconds": request_delay_seconds, "request_delay_seconds": request_delay_seconds,
@ -405,22 +396,6 @@ async def run_scholar_iteration(
"rate_limit_backoff_seconds": rate_limit_backoff_seconds, "rate_limit_backoff_seconds": rate_limit_backoff_seconds,
"page_size": page_size, "page_size": page_size,
} }
visited = 0
finished = 0
total = len(scholars)
async def _emit(v: int = 0, f: int = 0) -> None:
nonlocal visited, finished
visited += v
finished += f
await run_events.publish(
run.id,
"scholar_progress",
{"visited": visited, "finished": finished, "total": total},
)
await _emit()
first_pass_cstarts = await _run_first_pass( first_pass_cstarts = await _run_first_pass(
db_session, db_session,
scholars=scholars, scholars=scholars,
@ -432,12 +407,8 @@ async def run_scholar_iteration(
request_delay_seconds=request_delay_seconds, request_delay_seconds=request_delay_seconds,
queue_delay_seconds=queue_delay_seconds, queue_delay_seconds=queue_delay_seconds,
progress=progress, progress=progress,
on_progress=_emit,
) )
remaining_max = max(max_pages_per_scholar - 1, 0) remaining_max = max(max_pages_per_scholar - 1, 0)
scholars_finished_in_first_pass = len(scholars) - len(first_pass_cstarts)
if scholars_finished_in_first_pass > 0:
await _emit(f=scholars_finished_in_first_pass)
if remaining_max <= 0: if remaining_max <= 0:
return progress return progress
await _run_depth_pass( await _run_depth_pass(
@ -453,6 +424,5 @@ async def run_scholar_iteration(
auto_queue_continuations=auto_queue_continuations, auto_queue_continuations=auto_queue_continuations,
queue_delay_seconds=queue_delay_seconds, queue_delay_seconds=queue_delay_seconds,
progress=progress, progress=progress,
on_progress=_emit,
) )
return progress return progress

View file

@ -56,14 +56,11 @@ async def export_user_data(
db_session: AsyncSession, db_session: AsyncSession,
*, *,
user_id: int, user_id: int,
scholar_profile_ids: list[int] | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
scholar_query = select(ScholarProfile).where(ScholarProfile.user_id == user_id) scholars_result = await db_session.execute(
if scholar_profile_ids: select(ScholarProfile).where(ScholarProfile.user_id == user_id).order_by(ScholarProfile.id.asc())
scholar_query = scholar_query.where(ScholarProfile.id.in_(scholar_profile_ids)) )
scholars_result = await db_session.execute(scholar_query.order_by(ScholarProfile.id.asc())) publication_result = await db_session.execute(
pub_query = (
select( select(
ScholarProfile.scholar_id, ScholarProfile.scholar_id,
Publication.cluster_id, Publication.cluster_id,
@ -80,13 +77,8 @@ async def export_user_data(
.join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id) .join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id)
.join(Publication, Publication.id == ScholarPublication.publication_id) .join(Publication, Publication.id == ScholarPublication.publication_id)
.where(ScholarProfile.user_id == user_id) .where(ScholarProfile.user_id == user_id)
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
) )
if scholar_profile_ids:
pub_query = pub_query.where(ScholarProfile.id.in_(scholar_profile_ids))
publication_result = await db_session.execute(
pub_query.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
)
scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()] scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()]
publications = [_serialize_export_publication(row) for row in publication_result.all()] publications = [_serialize_export_publication(row) for row in publication_result.all()]
return { return {

View file

@ -2,10 +2,9 @@ from __future__ import annotations
import asyncio import asyncio
import logging import logging
from datetime import UTC, datetime, timedelta
from app.db.background_session import background_session
from app.db.models import Publication, PublicationPdfJob from app.db.models import Publication, PublicationPdfJob
from app.db.session import get_session_factory
from app.logging_utils import structured_log from app.logging_utils import structured_log
from app.services.publication_identifiers import application as identifier_service from app.services.publication_identifiers import application as identifier_service
from app.services.publications.pdf_queue_common import ( from app.services.publications.pdf_queue_common import (
@ -30,20 +29,8 @@ PDF_EVENT_ATTEMPT_STARTED = "attempt_started"
PDF_EVENT_RESOLVED = "resolved" PDF_EVENT_RESOLVED = "resolved"
PDF_EVENT_FAILED = "failed" PDF_EVENT_FAILED = "failed"
_BUDGET_COOLDOWN_MINUTES = 15
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_scheduled_tasks: set[asyncio.Task[None]] = set() _scheduled_tasks: set[asyncio.Task[None]] = set()
_budget_cooldown_until: datetime | None = None
def is_budget_cooldown_active() -> bool:
return _budget_cooldown_until is not None and datetime.now(UTC) < _budget_cooldown_until
def _enter_budget_cooldown() -> None:
global _budget_cooldown_until
_budget_cooldown_until = datetime.now(UTC) + timedelta(minutes=_BUDGET_COOLDOWN_MINUTES)
async def _mark_attempt_started( async def _mark_attempt_started(
@ -51,7 +38,8 @@ async def _mark_attempt_started(
publication_id: int, publication_id: int,
user_id: int, user_id: int,
) -> None: ) -> None:
async with background_session() as db_session: session_factory = get_session_factory()
async with session_factory() as db_session:
job = await db_session.get(PublicationPdfJob, publication_id) job = await db_session.get(PublicationPdfJob, publication_id)
if job is None: if job is None:
job = queued_job(publication_id=publication_id, user_id=user_id) job = queued_job(publication_id=publication_id, user_id=user_id)
@ -137,7 +125,8 @@ async def _persist_outcome(
user_id: int, user_id: int,
outcome: OaResolutionOutcome, outcome: OaResolutionOutcome,
) -> None: ) -> None:
async with background_session() as db_session: session_factory = get_session_factory()
async with session_factory() as db_session:
publication = await db_session.get(Publication, publication_id) publication = await db_session.get(Publication, publication_id)
job = await db_session.get(PublicationPdfJob, publication_id) job = await db_session.get(PublicationPdfJob, publication_id)
if publication is None or job is None: if publication is None or job is None:
@ -218,7 +207,8 @@ async def _run_resolution_task(
openalex_api_key: str | None = None openalex_api_key: str | None = None
try: try:
async with background_session() as key_session: session_factory = get_session_factory()
async with session_factory() as key_session:
user_settings = await user_settings_service.get_or_create_settings(key_session, user_id=user_id) user_settings = await user_settings_service.get_or_create_settings(key_session, user_id=user_id)
openalex_api_key = getattr(user_settings, "openalex_api_key", None) or settings.openalex_api_key openalex_api_key = getattr(user_settings, "openalex_api_key", None) or settings.openalex_api_key
except Exception: except Exception:
@ -243,13 +233,11 @@ async def _run_resolution_task(
detail="arXiv temporarily disabled for remaining batch after rate limit", detail="arXiv temporarily disabled for remaining batch after rate limit",
) )
except OpenAlexBudgetExhaustedError: except OpenAlexBudgetExhaustedError:
_enter_budget_cooldown()
structured_log( structured_log(
logger, logger,
"warning", "warning",
"pdf_queue.budget_exhausted", "pdf_queue.budget_exhausted",
detail="Stopping PDF resolution batch — OpenAlex daily budget exhausted", detail="Stopping PDF resolution batch — OpenAlex daily budget exhausted",
cooldown_minutes=_BUDGET_COOLDOWN_MINUTES,
) )
break break
except Exception: except Exception:

View file

@ -1,7 +1,6 @@
from __future__ import annotations from __future__ import annotations
import os import os
from typing import Any
from uuid import uuid4 from uuid import uuid4
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
@ -11,11 +10,7 @@ from app.db.models import ScholarProfile
from app.services.scholar.parser import ScholarParserError, parse_profile_page from app.services.scholar.parser import ScholarParserError, parse_profile_page
from app.services.scholar.source import ScholarSource from app.services.scholar.source import ScholarSource
from app.services.scholars.author_search import search_author_candidates from app.services.scholars.author_search import search_author_candidates
from app.services.scholars.constants import ( from app.services.scholars.constants import ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES
ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES,
SEARCH_COOLDOWN_REASON,
SEARCH_DISABLED_REASON,
)
from app.services.scholars.exceptions import ScholarServiceError from app.services.scholars.exceptions import ScholarServiceError
from app.services.scholars.uploads import ( from app.services.scholars.uploads import (
_ensure_upload_root, _ensure_upload_root,
@ -28,66 +23,8 @@ from app.services.scholars.validators import (
validate_scholar_id, validate_scholar_id,
) )
async def bulk_delete_scholars(
db_session: AsyncSession,
*,
user_id: int,
scholar_profile_ids: list[int],
upload_dir: str | None = None,
) -> int:
from sqlalchemy import select
result = await db_session.execute(
select(ScholarProfile).where(
ScholarProfile.id.in_(scholar_profile_ids),
ScholarProfile.user_id == user_id,
)
)
profiles = list(result.scalars().all())
if not profiles:
return 0
if upload_dir:
upload_root = _ensure_upload_root(upload_dir, create=True)
for profile in profiles:
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
for profile in profiles:
await db_session.delete(profile)
try:
await db_session.commit()
except IntegrityError as exc:
await db_session.rollback()
raise ScholarServiceError("Unable to bulk-delete scholars due to a database constraint.") from exc
return len(profiles)
async def bulk_toggle_scholars(
db_session: AsyncSession,
*,
user_id: int,
scholar_profile_ids: list[int],
is_enabled: bool,
) -> int:
from sqlalchemy import CursorResult, update
cursor: CursorResult[Any] = await db_session.execute( # type: ignore[assignment]
update(ScholarProfile)
.where(
ScholarProfile.id.in_(scholar_profile_ids),
ScholarProfile.user_id == user_id,
)
.values(is_enabled=is_enabled)
)
await db_session.commit()
return int(cursor.rowcount or 0)
__all__ = [ __all__ = [
"SEARCH_COOLDOWN_REASON",
"SEARCH_DISABLED_REASON",
"ScholarServiceError", "ScholarServiceError",
"bulk_delete_scholars",
"bulk_toggle_scholars",
"clear_profile_image_customization", "clear_profile_image_customization",
"create_scholar_for_user", "create_scholar_for_user",
"delete_scholar", "delete_scholar",
@ -182,11 +119,7 @@ async def delete_scholar(
_safe_remove_upload(upload_root, profile.profile_image_upload_path) _safe_remove_upload(upload_root, profile.profile_image_upload_path)
await db_session.delete(profile) await db_session.delete(profile)
try: await db_session.commit()
await db_session.commit()
except IntegrityError as exc:
await db_session.rollback()
raise ScholarServiceError("Unable to delete scholar due to a database constraint.") from exc
async def hydrate_profile_metadata( async def hydrate_profile_metadata(

View file

@ -275,8 +275,6 @@ class Settings:
crossref_max_lookups_per_request: int = _env_int("CROSSREF_MAX_LOOKUPS_PER_REQUEST", 8) crossref_max_lookups_per_request: int = _env_int("CROSSREF_MAX_LOOKUPS_PER_REQUEST", 8)
openalex_api_key: str | None = os.getenv("OPENALEX_API_KEY") openalex_api_key: str | None = os.getenv("OPENALEX_API_KEY")
database_reserved_api_connections: int = _env_int("DATABASE_RESERVED_API_CONNECTIONS", 3)
crossref_api_token: str | None = os.getenv("CROSSREF_API_TOKEN") crossref_api_token: str | None = os.getenv("CROSSREF_API_TOKEN")
crossref_api_mailto: str | None = os.getenv("CROSSREF_API_MAILTO") crossref_api_mailto: str | None = os.getenv("CROSSREF_API_MAILTO")

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -27,7 +27,7 @@
"postcss": "^8.5.3", "postcss": "^8.5.3",
"tailwindcss": "^3.4.17", "tailwindcss": "^3.4.17",
"typescript": "^5.7.2", "typescript": "^5.7.2",
"vite": "^5.4.11", "vite": "^7.3.1",
"vitest": "^2.1.8", "vitest": "^2.1.8",
"vue-tsc": "^2.1.10" "vue-tsc": "^2.1.10"
} }

View file

@ -1,61 +0,0 @@
// @vitest-environment happy-dom
import { describe, expect, it } from "vitest";
import { mount } from "@vue/test-utils";
import ScrapeSafetyBadge from "./ScrapeSafetyBadge.vue";
import { createDefaultSafetyState, type ScrapeSafetyState } from "@/features/safety";
function buildState(overrides: Partial<ScrapeSafetyState> = {}): ScrapeSafetyState {
return { ...createDefaultSafetyState(), ...overrides };
}
describe("ScrapeSafetyBadge", () => {
it("shows ready tooltip when cooldown is inactive", () => {
const wrapper = mount(ScrapeSafetyBadge, {
props: { state: buildState({ cooldown_active: false }) },
});
expect(wrapper.text()).toContain("Safety ready");
const hint = wrapper.findComponent({ name: "AppHelpHint" });
expect(hint.exists()).toBe(true);
expect(hint.props("text")).toContain("No active cooldown");
});
it("shows cooldown tooltip with reason and action when active", () => {
const wrapper = mount(ScrapeSafetyBadge, {
props: {
state: buildState({
cooldown_active: true,
cooldown_reason: "blocked_failure_threshold_exceeded",
cooldown_reason_label: "Too many blocked requests",
recommended_action: "Wait for cooldown to expire",
cooldown_remaining_seconds: 120,
}),
},
});
expect(wrapper.text()).toContain("Safety cooldown");
const hint = wrapper.findComponent({ name: "AppHelpHint" });
expect(hint.exists()).toBe(true);
const text = hint.props("text") as string;
expect(text).toContain("Google Scholar rate-limits");
expect(text).toContain("Too many blocked requests");
expect(text).toContain("Wait for cooldown to expire");
});
it("shows cooldown tooltip without optional fields", () => {
const wrapper = mount(ScrapeSafetyBadge, {
props: {
state: buildState({
cooldown_active: true,
cooldown_reason: "some_reason",
cooldown_reason_label: null,
recommended_action: null,
cooldown_remaining_seconds: 60,
}),
},
});
const hint = wrapper.findComponent({ name: "AppHelpHint" });
const text = hint.props("text") as string;
expect(text).toContain("Google Scholar rate-limits");
expect(text).not.toContain("Why:");
expect(text).not.toContain("Action:");
});
});

View file

@ -1,7 +1,6 @@
<script setup lang="ts"> <script setup lang="ts">
import { computed } from "vue"; import { computed } from "vue";
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
import { type ScrapeSafetyState } from "@/features/safety"; import { type ScrapeSafetyState } from "@/features/safety";
const props = defineProps<{ const props = defineProps<{
@ -19,30 +18,10 @@ const toneClass = computed(() => {
} }
return "border-state-warning-border bg-state-warning-bg text-state-warning-text"; return "border-state-warning-border bg-state-warning-bg text-state-warning-text";
}); });
const READY_TOOLTIP = "No active cooldown. Scraping can proceed normally.";
const RATE_LIMIT_INTRO =
"Google Scholar rate-limits automated requests. The cooldown pauses scraping to avoid your IP being blocked.";
const tooltipText = computed(() => {
if (!props.state.cooldown_active) return READY_TOOLTIP;
const parts = [RATE_LIMIT_INTRO];
if (props.state.cooldown_reason_label) {
parts.push(`Why: ${props.state.cooldown_reason_label}`);
}
if (props.state.recommended_action) {
parts.push(`Action: ${props.state.recommended_action}`);
}
return parts.join(" \u2014 ");
});
</script> </script>
<template> <template>
<span class="inline-flex items-center gap-1"> <span class="inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-semibold" :class="toneClass">
<span class="inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-semibold" :class="toneClass"> {{ label }}
{{ label }}
</span>
<AppHelpHint :text="tooltipText" />
</span> </span>
</template> </template>

View file

@ -2,7 +2,6 @@
import { computed, onBeforeUnmount, onMounted, ref } from "vue"; import { computed, onBeforeUnmount, onMounted, ref } from "vue";
import AppAlert from "@/components/ui/AppAlert.vue"; import AppAlert from "@/components/ui/AppAlert.vue";
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
import { import {
formatCooldownCountdown, formatCooldownCountdown,
type ScrapeSafetyState, type ScrapeSafetyState,
@ -80,10 +79,6 @@ const actionText = computed(() => {
return props.safetyState.recommended_action; return props.safetyState.recommended_action;
}); });
const bannerHintText = computed(() => {
return "Google Scholar rate-limits automated requests. The cooldown pauses scraping to avoid your IP being blocked.";
});
onMounted(() => { onMounted(() => {
timer = setInterval(() => { timer = setInterval(() => {
now.value = Date.now(); now.value = Date.now();
@ -100,12 +95,7 @@ onBeforeUnmount(() => {
<template> <template>
<AppAlert v-if="isVisible" :tone="tone"> <AppAlert v-if="isVisible" :tone="tone">
<template #title> <template #title>{{ title }}</template>
<span class="inline-flex items-center gap-1">
{{ title }}
<AppHelpHint :text="bannerHintText" />
</span>
</template>
<p>{{ detailText }}</p> <p>{{ detailText }}</p>
<p v-if="actionText" class="text-secondary">{{ actionText }}</p> <p v-if="actionText" class="text-secondary">{{ actionText }}</p>
</AppAlert> </AppAlert>

View file

@ -1,28 +0,0 @@
<script setup lang="ts">
import AppButton from "@/components/ui/AppButton.vue";
import AppModal from "@/components/ui/AppModal.vue";
withDefaults(
defineProps<{
open: boolean;
title: string;
message: string;
confirmLabel?: string;
cancelLabel?: string;
variant?: "danger" | "default";
}>(),
{ confirmLabel: "Confirm", cancelLabel: "Cancel", variant: "default" },
);
const emit = defineEmits<{ confirm: []; cancel: [] }>();
</script>
<template>
<AppModal :open="open" :title="title" @close="emit('cancel')">
<p class="mb-6 text-sm text-secondary">{{ message }}</p>
<div class="flex justify-end gap-2">
<AppButton variant="secondary" @click="emit('cancel')">{{ cancelLabel }}</AppButton>
<AppButton :variant="variant === 'danger' ? 'danger' : 'primary'" @click="emit('confirm')">{{ confirmLabel }}</AppButton>
</div>
</AppModal>
</template>

View file

@ -277,16 +277,12 @@ export function usePublicationData() {
// --- Run-triggered refresh watcher --- // --- Run-triggered refresh watcher ---
let previousRunStatusKey: string | null = null;
watch( watch(
() => runStatus.latestRun, () => runStatus.latestRun,
async (nextRun) => { async (nextRun, previousRun) => {
const nextStatus = nextRun ? `${nextRun.id}:${nextRun.status}` : null; const nextRunId = nextRun && (nextRun.status === "running" || nextRun.status === "resolving") ? nextRun.id : null;
if (nextStatus === previousRunStatusKey) return; const previousRunId = previousRun && (previousRun.status === "running" || previousRun.status === "resolving") ? previousRun.id : null;
previousRunStatusKey = nextStatus; if (nextRunId === null || nextRunId === previousRunId) return;
const isActive = nextRun && (nextRun.status === "running" || nextRun.status === "resolving");
if (!isActive) return;
resetPageAndSnapshot(); resetPageAndSnapshot();
await loadPublications(); await loadPublications();
}, },

View file

@ -2,144 +2,10 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { mount } from "@vue/test-utils"; import { mount } from "@vue/test-utils";
import ScholarBatchAdd from "./ScholarBatchAdd.vue"; import ScholarBatchAdd from "./ScholarBatchAdd.vue";
import {
parseScholarIds,
parseScholarTokens,
extractScholarIdFromUrl,
validateTokenAsId,
} from "./scholar-batch-parsing";
const defaultProps = { saving: false, loading: false }; const defaultProps = { saving: false, loading: false };
describe("parseScholarIds (unit)", () => { describe("ScholarBatchAdd", () => {
it("parses a single bare ID", () => {
expect(parseScholarIds("A-UbBTPM15wL")).toEqual(["A-UbBTPM15wL"]);
});
it("parses multiple IDs separated by commas", () => {
expect(parseScholarIds("A-UbBTPM15wL, B-UbBTPM15wL")).toEqual(["A-UbBTPM15wL", "B-UbBTPM15wL"]);
});
it("deduplicates IDs", () => {
expect(parseScholarIds("A-UbBTPM15wL\nA-UbBTPM15wL")).toEqual(["A-UbBTPM15wL"]);
});
it("extracts ID from standard Google Scholar URL", () => {
expect(parseScholarIds("https://scholar.google.com/citations?hl=en&user=A-UbBTPM15wL")).toEqual(["A-UbBTPM15wL"]);
});
it("extracts ID from URL with trailing slash", () => {
expect(parseScholarIds("https://scholar.google.com/citations?user=A-UbBTPM15wL/")).toEqual(["A-UbBTPM15wL"]);
});
it("extracts ID from URL with fragment", () => {
expect(parseScholarIds("https://scholar.google.com/citations?user=A-UbBTPM15wL#section")).toEqual(["A-UbBTPM15wL"]);
});
it("extracts ID from URL with extra query params", () => {
expect(parseScholarIds("https://scholar.google.com/citations?hl=en&user=A-UbBTPM15wL&view_op=list_works&sortby=pubdate")).toEqual(["A-UbBTPM15wL"]);
});
it("handles URL-encoded characters in non-ID parts", () => {
expect(parseScholarIds("https://scholar.google.com/citations?hl=en&user=A-UbBTPM15wL&label=%E4%B8%AD%E6%96%87")).toEqual(["A-UbBTPM15wL"]);
});
it("rejects malformed IDs (too short)", () => {
expect(parseScholarIds("ABC123")).toEqual([]);
});
it("rejects malformed IDs (too long)", () => {
expect(parseScholarIds("ABCDEF1234567")).toEqual([]);
});
it("rejects IDs with special characters", () => {
expect(parseScholarIds("ABCDEF12345!")).toEqual([]);
});
it("rejects IDs with embedded whitespace", () => {
expect(parseScholarIds("ABC DEF12345")).toEqual([]);
});
it("handles mixed valid and invalid tokens", () => {
expect(parseScholarIds("A-UbBTPM15wL, invalid, B-UbBTPM15wL")).toEqual(["A-UbBTPM15wL", "B-UbBTPM15wL"]);
});
it("returns empty array for empty string", () => {
expect(parseScholarIds("")).toEqual([]);
});
it("returns empty array for whitespace only", () => {
expect(parseScholarIds(" ")).toEqual([]);
});
});
describe("extractScholarIdFromUrl", () => {
it("returns null for non-URL strings", () => {
expect(extractScholarIdFromUrl("not-a-url")).toBeNull();
});
it("returns null for URL without user param", () => {
expect(extractScholarIdFromUrl("https://scholar.google.com/citations?hl=en")).toBeNull();
});
it("extracts from URL with trailing slashes", () => {
expect(extractScholarIdFromUrl("https://scholar.google.com/citations?user=A-UbBTPM15wL///")).toBe("A-UbBTPM15wL");
});
it("strips fragment before parsing", () => {
expect(extractScholarIdFromUrl("https://scholar.google.com/citations?user=A-UbBTPM15wL#foo")).toBe("A-UbBTPM15wL");
});
it("returns null when user param is invalid length", () => {
expect(extractScholarIdFromUrl("https://scholar.google.com/citations?user=short")).toBeNull();
});
});
describe("validateTokenAsId", () => {
it("returns null for valid 12-char ID", () => {
expect(validateTokenAsId("ABCDEF123456")).toBeNull();
});
it("detects wrong length", () => {
expect(validateTokenAsId("ABC")).toContain("12 characters");
});
it("detects invalid characters", () => {
expect(validateTokenAsId("ABCDEF12345!")).toContain("invalid characters");
});
it("detects whitespace", () => {
expect(validateTokenAsId("ABC DEF12345")).toContain("whitespace");
});
it("detects empty input", () => {
expect(validateTokenAsId("")).toContain("empty");
});
});
describe("parseScholarTokens (per-token errors)", () => {
it("marks invalid tokens with error messages", () => {
const tokens = parseScholarTokens("A-UbBTPM15wL, short, B-UbBTPM15wL");
expect(tokens).toHaveLength(3);
expect(tokens[0].id).toBe("A-UbBTPM15wL");
expect(tokens[0].error).toBeNull();
expect(tokens[1].id).toBeNull();
expect(tokens[1].error).toContain("12 characters");
expect(tokens[2].id).toBe("B-UbBTPM15wL");
});
it("marks duplicate tokens", () => {
const tokens = parseScholarTokens("A-UbBTPM15wL, A-UbBTPM15wL");
expect(tokens[1].error).toBe("duplicate");
});
it("marks bad URL tokens", () => {
const tokens = parseScholarTokens("https://scholar.google.com/citations?hl=en");
expect(tokens[0].error).toContain("could not extract");
});
});
describe("ScholarBatchAdd (component)", () => {
it("renders the heading and input", () => { it("renders the heading and input", () => {
const wrapper = mount(ScholarBatchAdd, { props: defaultProps }); const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
expect(wrapper.text()).toContain("Add Scholar Profiles"); expect(wrapper.text()).toContain("Add Scholar Profiles");
@ -207,18 +73,4 @@ describe("ScholarBatchAdd (component)", () => {
const wrapper = mount(ScholarBatchAdd, { props: { ...defaultProps, saving: true } }); const wrapper = mount(ScholarBatchAdd, { props: { ...defaultProps, saving: true } });
expect(wrapper.text()).toContain("Adding..."); expect(wrapper.text()).toContain("Adding...");
}); });
it("shows validation errors for invalid tokens", async () => {
const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
await wrapper.find("textarea").setValue("A-UbBTPM15wL, bad!");
const errors = wrapper.find("[data-testid='validation-errors']");
expect(errors.exists()).toBe(true);
});
it("shows skipped count alongside valid count", async () => {
const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
await wrapper.find("textarea").setValue("A-UbBTPM15wL, tooshort");
expect(wrapper.text()).toContain("1 valid");
expect(wrapper.text()).toContain("1 skipped");
});
}); });

View file

@ -3,7 +3,6 @@ import { computed, ref } from "vue";
import AppButton from "@/components/ui/AppButton.vue"; import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue"; import AppCard from "@/components/ui/AppCard.vue";
import AppHelpHint from "@/components/ui/AppHelpHint.vue"; import AppHelpHint from "@/components/ui/AppHelpHint.vue";
import { parseScholarTokens } from "./scholar-batch-parsing";
defineProps<{ defineProps<{
saving: boolean; saving: boolean;
@ -16,13 +15,38 @@ const emit = defineEmits<{
const scholarBatchInput = ref(""); const scholarBatchInput = ref("");
const parsedTokens = computed(() => parseScholarTokens(scholarBatchInput.value)); const SCHOLAR_ID_PATTERN = /^[a-zA-Z0-9_-]{12}$/;
const validIds = computed(() => parsedTokens.value.filter((t) => t.id !== null)); const URL_USER_PARAM_PATTERN = /(?:\?|&)user=([a-zA-Z0-9_-]{12})(?:&|#|$)/i;
const invalidTokens = computed(() => parsedTokens.value.filter((t) => t.id === null));
const parsedBatchCount = computed(() => validIds.value.length); function parseScholarIds(raw: string): string[] {
const ordered: string[] = [];
const seen = new Set<string>();
const tokens = raw.split(/[\s,;]+/).map((v) => v.trim()).filter((v) => v.length > 0);
for (const token of tokens) {
let candidate: string | null = null;
if (SCHOLAR_ID_PATTERN.test(token)) candidate = token;
if (!candidate) {
const directParamMatch = token.match(URL_USER_PARAM_PATTERN);
if (directParamMatch) candidate = directParamMatch[1];
}
if (!candidate && token.includes("scholar.google")) {
try {
const parsed = new URL(token);
const userParam = parsed.searchParams.get("user");
if (userParam && SCHOLAR_ID_PATTERN.test(userParam)) candidate = userParam;
} catch (_error) { /* Ignore non-URL tokens. */ }
}
if (!candidate || seen.has(candidate)) continue;
seen.add(candidate);
ordered.push(candidate);
}
return ordered;
}
const parsedBatchCount = computed(() => parseScholarIds(scholarBatchInput.value).length);
function onSubmit(): void { function onSubmit(): void {
const ids = validIds.value.map((t) => t.id!); const ids = parseScholarIds(scholarBatchInput.value);
if (ids.length > 0) { if (ids.length > 0) {
emit("add-scholars", ids); emit("add-scholars", ids);
scholarBatchInput.value = ""; scholarBatchInput.value = "";
@ -52,33 +76,11 @@ function onSubmit(): void {
/> />
</label> </label>
<div v-if="parsedTokens.length > 0" class="space-y-1"> <div class="flex flex-wrap items-center justify-between gap-2">
<p class="text-xs text-secondary"> <p class="text-xs text-secondary">
<strong class="text-ink-primary">{{ parsedBatchCount }}</strong> valid ID{{ parsedBatchCount === 1 ? "" : "s" }} Parsed IDs: <strong class="text-ink-primary">{{ parsedBatchCount }}</strong>
<template v-if="invalidTokens.length > 0">
· <span class="text-state-warning-text">{{ invalidTokens.length }} skipped</span>
</template>
</p> </p>
<ul v-if="invalidTokens.length > 0" class="space-y-0.5" data-testid="validation-errors"> <AppButton type="submit" :disabled="saving || loading">
<li
v-for="item in invalidTokens.slice(0, 5)"
:key="item.index"
class="text-xs text-state-warning-text"
>
#{{ item.index }}: {{ item.error }}
<code class="ml-1 break-all text-ink-muted">{{ item.raw.length > 60 ? item.raw.slice(0, 57) + "..." : item.raw }}</code>
</li>
<li v-if="invalidTokens.length > 5" class="text-xs text-state-warning-text">
+{{ invalidTokens.length - 5 }} more skipped
</li>
</ul>
</div>
<div v-else class="text-xs text-secondary">
Parsed IDs: <strong class="text-ink-primary">0</strong>
</div>
<div class="flex flex-wrap items-center justify-end gap-2">
<AppButton type="submit" :disabled="saving || loading || parsedBatchCount === 0">
{{ saving ? "Adding..." : "Add scholars" }} {{ saving ? "Adding..." : "Add scholars" }}
</AppButton> </AppButton>
</div> </div>

View file

@ -47,14 +47,7 @@ function scholarPublicationsRoute(profile: ScholarProfile): { name: string; quer
:image-url="scholar.profile_image_url" :image-url="scholar.profile_image_url"
/> />
<div class="min-w-0 space-y-1"> <div class="min-w-0 space-y-1">
<p class="truncate text-sm font-semibold text-ink-primary"> <p class="truncate text-sm font-semibold text-ink-primary">{{ scholarLabel(scholar) }}</p>
{{ scholarLabel(scholar) }}
<span
v-if="!scholar.baseline_completed"
class="ml-1.5 inline-flex items-center rounded-full border border-state-warning-border bg-state-warning-bg px-1.5 py-0.5 text-[10px] font-medium text-state-warning-text"
title="This scholar hasn't been scraped yet. Publications will appear after the next run."
>Pending</span>
</p>
<p class="text-xs text-secondary">ID: <code>{{ scholar.scholar_id }}</code></p> <p class="text-xs text-secondary">ID: <code>{{ scholar.scholar_id }}</code></p>
<div class="flex flex-wrap items-center gap-3"> <div class="flex flex-wrap items-center gap-3">
<RouterLink :to="scholarPublicationsRoute(scholar)" class="link-inline text-xs"> <RouterLink :to="scholarPublicationsRoute(scholar)" class="link-inline text-xs">

View file

@ -1,29 +0,0 @@
<script setup lang="ts">
defineProps<{
baselineCompleted: boolean;
lastRunStatus: string | null;
}>();
</script>
<template>
<span
v-if="!baselineCompleted"
class="ml-1.5 inline-flex items-center rounded-full border border-state-warning-border bg-state-warning-bg px-1.5 py-0.5 text-[10px] font-medium text-state-warning-text"
title="This scholar hasn't been scraped yet. Publications will appear after the next run."
>Pending</span>
<span
v-if="lastRunStatus === 'failed'"
class="ml-1.5 inline-flex items-center rounded-full border border-state-danger-border bg-state-danger-bg px-1.5 py-0.5 text-[10px] font-medium text-state-danger-text"
title="The last scrape run for this scholar failed."
>Failed</span>
<span
v-else-if="lastRunStatus === 'partial'"
class="ml-1.5 inline-flex items-center rounded-full border border-state-warning-border bg-state-warning-bg px-1.5 py-0.5 text-[10px] font-medium text-state-warning-text"
title="The last scrape run for this scholar completed with partial results."
>Partial</span>
<span
v-else-if="lastRunStatus === 'success'"
class="ml-1.5 inline-flex items-center rounded-full border border-state-success-border bg-state-success-bg px-1.5 py-0.5 text-[10px] font-medium text-state-success-text"
title="The last scrape run for this scholar completed successfully."
>OK</span>
</template>

View file

@ -1,76 +0,0 @@
const SCHOLAR_ID_PATTERN = /^[a-zA-Z0-9_-]{12}$/;
export interface ParsedToken {
index: number;
raw: string;
id: string | null;
error: string | null;
}
export function extractScholarIdFromUrl(token: string): string | null {
const cleaned = token.replace(/\/+$/, "").replace(/#.*$/, "");
try {
const parsed = new URL(cleaned);
const userParam = parsed.searchParams.get("user");
if (!userParam) return null;
const decoded = decodeURIComponent(userParam).trim();
if (SCHOLAR_ID_PATTERN.test(decoded)) return decoded;
return null;
} catch {
return null;
}
}
export function validateTokenAsId(token: string): string | null {
const trimmed = token.trim();
if (!trimmed) return "empty input";
if (/\s/.test(trimmed)) return "contains whitespace";
if (trimmed.length !== 12) return `must be 12 characters (got ${trimmed.length})`;
if (!/^[a-zA-Z0-9_-]+$/.test(trimmed)) {
return "contains invalid characters (only a-z, A-Z, 0-9, _ and - allowed)";
}
return null;
}
export function parseScholarTokens(raw: string): ParsedToken[] {
const tokens = raw.split(/[\s,;]+/).map((v) => v.trim()).filter((v) => v.length > 0);
const results: ParsedToken[] = [];
const seen = new Set<string>();
for (let i = 0; i < tokens.length; i++) {
const token = tokens[i];
if (SCHOLAR_ID_PATTERN.test(token)) {
if (seen.has(token)) {
results.push({ index: i + 1, raw: token, id: null, error: "duplicate" });
continue;
}
seen.add(token);
results.push({ index: i + 1, raw: token, id: token, error: null });
continue;
}
if (token.includes("scholar.google") || token.startsWith("http")) {
const extracted = extractScholarIdFromUrl(token);
if (extracted) {
if (seen.has(extracted)) {
results.push({ index: i + 1, raw: token, id: null, error: "duplicate" });
continue;
}
seen.add(extracted);
results.push({ index: i + 1, raw: token, id: extracted, error: null });
continue;
}
results.push({ index: i + 1, raw: token, id: null, error: "could not extract scholar ID from URL" });
continue;
}
const reason = validateTokenAsId(token);
results.push({ index: i + 1, raw: token, id: null, error: reason ?? "invalid scholar ID" });
}
return results;
}
export function parseScholarIds(raw: string): string[] {
return parseScholarTokens(raw).filter((t) => t.id !== null).map((t) => t.id!);
}

View file

@ -1,148 +0,0 @@
// @vitest-environment happy-dom
import { describe, expect, it, vi } from "vitest";
import { nextTick, ref } from "vue";
import { useScholarBulkActions, type ScholarBulkAction } from "./useScholarBulkActions";
import type { ScholarProfile } from "@/features/scholars";
vi.mock("@/features/scholars", () => ({
bulkDeleteScholars: vi.fn(),
bulkToggleScholars: vi.fn(),
exportScholarData: vi.fn(),
}));
function makeProfile(id: number, name: string): ScholarProfile {
return {
id,
scholar_id: `scholar_${id}`,
display_name: name,
profile_image_url: null,
profile_image_source: "none",
is_enabled: true,
baseline_completed: false,
last_run_dt: null,
last_run_status: null,
};
}
function setup(profiles: ScholarProfile[] = []) {
const visibleScholars = ref(profiles);
const callbacks = {
clearMessages: vi.fn(),
assignError: vi.fn(),
setSuccess: vi.fn(),
reloadScholars: vi.fn(async () => {}),
};
const bulk = useScholarBulkActions(visibleScholars, callbacks);
return { visibleScholars, callbacks, bulk };
}
describe("useScholarBulkActions", () => {
it("starts with empty selection", () => {
const { bulk } = setup([makeProfile(1, "Alice")]);
expect(bulk.selectedIds.value.size).toBe(0);
expect(bulk.hasSelection.value).toBe(false);
});
it("toggles individual row selection", () => {
const { bulk } = setup([makeProfile(1, "Alice"), makeProfile(2, "Bob")]);
bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
expect(bulk.selectedIds.value.has(1)).toBe(true);
expect(bulk.selectedCount.value).toBe(1);
bulk.onToggleRow(1, { target: { checked: false } } as unknown as Event);
expect(bulk.selectedIds.value.has(1)).toBe(false);
expect(bulk.selectedCount.value).toBe(0);
});
it("toggles all visible scholars", () => {
const { bulk } = setup([makeProfile(1, "Alice"), makeProfile(2, "Bob")]);
bulk.onToggleAll({ target: { checked: true } } as unknown as Event);
expect(bulk.selectedIds.value.size).toBe(2);
expect(bulk.allVisibleSelected.value).toBe(true);
bulk.onToggleAll({ target: { checked: false } } as unknown as Event);
expect(bulk.selectedIds.value.size).toBe(0);
});
it("prunes stale selections when list changes", async () => {
const { visibleScholars, bulk } = setup([makeProfile(1, "Alice"), makeProfile(2, "Bob")]);
bulk.onToggleAll({ target: { checked: true } } as unknown as Event);
expect(bulk.selectedIds.value.size).toBe(2);
// Remove Bob from the visible list
visibleScholars.value = [makeProfile(1, "Alice")];
await nextTick();
expect(bulk.selectedIds.value.size).toBe(1);
expect(bulk.selectedIds.value.has(1)).toBe(true);
expect(bulk.selectedIds.value.has(2)).toBe(false);
});
it("shows correct bulk action options with and without selection", () => {
const { bulk } = setup([makeProfile(1, "Alice")]);
// Without selection
expect(bulk.bulkActionOptions.value.length).toBe(1);
expect(bulk.bulkActionOptions.value[0].value).toBe("select_all");
// With selection
bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
expect(bulk.bulkActionOptions.value.length).toBe(5);
const values = bulk.bulkActionOptions.value.map((o) => o.value);
expect(values).toContain("delete_selected");
expect(values).toContain("enable_selected");
expect(values).toContain("disable_selected");
expect(values).toContain("export_selected");
expect(values).toContain("clear_selection");
});
it("select all action selects all visible", async () => {
const { bulk } = setup([makeProfile(1, "Alice"), makeProfile(2, "Bob")]);
bulk.bulkAction.value = "select_all" as ScholarBulkAction;
await bulk.onApplyBulkAction();
expect(bulk.selectedIds.value.size).toBe(2);
});
it("clear selection action clears all", async () => {
const { bulk } = setup([makeProfile(1, "Alice")]);
bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
bulk.bulkAction.value = "clear_selection" as ScholarBulkAction;
await bulk.onApplyBulkAction();
expect(bulk.selectedIds.value.size).toBe(0);
});
it("bulk delete calls assignError on network failure", async () => {
const { bulkDeleteScholars } = await import("@/features/scholars");
vi.mocked(bulkDeleteScholars).mockRejectedValueOnce(new Error("Network error"));
const { bulk, callbacks } = setup([makeProfile(1, "Alice")]);
bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
bulk.bulkAction.value = "delete_selected" as ScholarBulkAction;
await bulk.onApplyBulkAction();
// Trigger the confirm callback
expect(bulk.confirmState.value.open).toBe(true);
await bulk.confirmState.value.onConfirm();
expect(callbacks.assignError).toHaveBeenCalledWith(
expect.any(Error),
"Unable to bulk delete scholars.",
);
expect(bulk.bulkBusy.value).toBe(false);
});
it("bulk toggle calls assignError on network failure", async () => {
const { bulkToggleScholars } = await import("@/features/scholars");
vi.mocked(bulkToggleScholars).mockRejectedValueOnce(new Error("Network error"));
const { bulk, callbacks } = setup([makeProfile(1, "Alice")]);
bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
bulk.bulkAction.value = "enable_selected" as ScholarBulkAction;
await bulk.onApplyBulkAction();
expect(callbacks.assignError).toHaveBeenCalledWith(
expect.any(Error),
"Unable to bulk enable scholars.",
);
expect(bulk.bulkBusy.value).toBe(false);
});
});

View file

@ -1,221 +0,0 @@
import { computed, ref, watch, type Ref } from "vue";
import {
bulkDeleteScholars,
bulkToggleScholars,
exportScholarData,
type ScholarProfile,
} from "@/features/scholars";
export type ScholarBulkAction =
| "delete_selected"
| "enable_selected"
| "disable_selected"
| "export_selected"
| "clear_selection"
| "select_all";
export interface ScholarBulkActionOption {
value: ScholarBulkAction;
label: string;
}
export interface ConfirmState {
open: boolean;
title: string;
message: string;
variant: "danger" | "default";
onConfirm: () => void;
}
export interface BulkActionCallbacks {
clearMessages: () => void;
assignError: (error: unknown, fallback: string) => void;
setSuccess: (msg: string) => void;
reloadScholars: () => Promise<void>;
}
export function useScholarBulkActions(
visibleScholars: Ref<ScholarProfile[]>,
callbacks: BulkActionCallbacks,
) {
const selectedIds = ref<Set<number>>(new Set());
const bulkAction = ref<ScholarBulkAction>("select_all");
const bulkBusy = ref(false);
const confirmState = ref<ConfirmState>({ open: false, title: "", message: "", variant: "default", onConfirm: () => {} });
const selectedCount = computed(() => selectedIds.value.size);
const hasSelection = computed(() => selectedCount.value > 0);
const allVisibleSelected = computed(() => {
if (visibleScholars.value.length === 0) return false;
for (const item of visibleScholars.value) {
if (!selectedIds.value.has(item.id)) return false;
}
return true;
});
const bulkActionOptions = computed<ScholarBulkActionOption[]>(() => {
if (!hasSelection.value) return [{ value: "select_all", label: "Select all" }];
const n = selectedCount.value;
return [
{ value: "delete_selected", label: `Delete selected (${n})` },
{ value: "enable_selected", label: `Enable selected (${n})` },
{ value: "disable_selected", label: `Disable selected (${n})` },
{ value: "export_selected", label: `Export selected (${n})` },
{ value: "clear_selection", label: "Clear selection" },
];
});
const bulkApplyLabel = computed(() => {
if (bulkBusy.value) return "Applying...";
if (bulkAction.value === "select_all") return "Select";
if (bulkAction.value === "clear_selection") return "Clear";
return "Apply";
});
const bulkApplyDisabled = computed(() => {
if (bulkBusy.value) return true;
if (bulkAction.value === "select_all") return visibleScholars.value.length === 0;
return selectedCount.value === 0;
});
// Prune stale selections when visible list changes
watch(visibleScholars, (items) => {
const validIds = new Set(items.map((item) => item.id));
const next = new Set<number>();
for (const id of selectedIds.value) {
if (validIds.has(id)) next.add(id);
}
if (next.size !== selectedIds.value.size) selectedIds.value = next;
});
// Reset bulk action dropdown when selection state changes
watch(hasSelection, (has) => {
const validValues = new Set(bulkActionOptions.value.map((o) => o.value));
if (validValues.has(bulkAction.value)) return;
bulkAction.value = has ? "delete_selected" : "select_all";
});
function onToggleAll(event: Event): void {
const checked = (event.target as HTMLInputElement).checked;
const next = new Set(selectedIds.value);
for (const item of visibleScholars.value) {
if (checked) { next.add(item.id); } else { next.delete(item.id); }
}
selectedIds.value = next;
}
function onToggleRow(id: number, event: Event): void {
const checked = (event.target as HTMLInputElement).checked;
const next = new Set(selectedIds.value);
if (checked) { next.add(id); } else { next.delete(id); }
selectedIds.value = next;
}
function downloadJsonFile(filename: string, payload: unknown): void {
const blob = new Blob([JSON.stringify(payload, null, 2)], { type: "application/json" });
const url = URL.createObjectURL(blob);
const anchor = document.createElement("a");
anchor.href = url;
anchor.download = filename;
anchor.click();
URL.revokeObjectURL(url);
}
async function onApplyBulkAction(): Promise<void> {
if (bulkApplyDisabled.value) return;
if (bulkAction.value === "select_all") {
selectedIds.value = new Set(visibleScholars.value.map((item) => item.id));
return;
}
if (bulkAction.value === "clear_selection") { selectedIds.value = new Set(); return; }
if (bulkAction.value === "delete_selected") { await onBulkDelete(); return; }
if (bulkAction.value === "enable_selected") { await onBulkToggle(true); return; }
if (bulkAction.value === "disable_selected") { await onBulkToggle(false); return; }
if (bulkAction.value === "export_selected") { await onBulkExport(); return; }
}
function dismissConfirm(): void {
confirmState.value = { ...confirmState.value, open: false };
}
function requestConfirm(title: string, message: string, variant: "danger" | "default", onConfirm: () => void): void {
confirmState.value = { open: true, title, message, variant, onConfirm };
}
function onBulkDelete(): void {
const ids = [...selectedIds.value];
requestConfirm(
`Delete ${ids.length} scholar(s)?`,
"This removes all linked publications and queue data. This action cannot be undone.",
"danger",
async () => {
dismissConfirm();
bulkBusy.value = true;
callbacks.clearMessages();
try {
const result = await bulkDeleteScholars(ids);
callbacks.setSuccess(`${result.deleted_count} scholar(s) deleted.`);
selectedIds.value = new Set();
await callbacks.reloadScholars();
} catch (error) {
callbacks.assignError(error, "Unable to bulk delete scholars.");
} finally {
bulkBusy.value = false;
}
},
);
}
async function onBulkToggle(isEnabled: boolean): Promise<void> {
bulkBusy.value = true;
callbacks.clearMessages();
try {
const ids = [...selectedIds.value];
const result = await bulkToggleScholars(ids, isEnabled);
const verb = isEnabled ? "enabled" : "disabled";
callbacks.setSuccess(`${result.updated_count} scholar(s) ${verb}.`);
selectedIds.value = new Set();
await callbacks.reloadScholars();
} catch (error) {
callbacks.assignError(error, `Unable to bulk ${isEnabled ? "enable" : "disable"} scholars.`);
} finally {
bulkBusy.value = false;
}
}
async function onBulkExport(): Promise<void> {
bulkBusy.value = true;
callbacks.clearMessages();
try {
const ids = [...selectedIds.value];
const payload = await exportScholarData(ids);
const dateSlug = payload.exported_at.slice(0, 10) || "unknown-date";
downloadJsonFile(`scholarr-export-${dateSlug}.json`, payload);
callbacks.setSuccess("Export complete.");
} catch (error) {
callbacks.assignError(error, "Unable to export selected scholars.");
} finally {
bulkBusy.value = false;
}
}
return {
selectedIds,
selectedCount,
hasSelection,
allVisibleSelected,
bulkAction,
bulkBusy,
bulkActionOptions,
bulkApplyLabel,
bulkApplyDisabled,
confirmState,
dismissConfirm,
requestConfirm,
onToggleAll,
onToggleRow,
onApplyBulkAction,
};
}

View file

@ -160,30 +160,8 @@ export async function clearScholarImage(
return response.data; return response.data;
} }
export interface BulkCountResult { export async function exportScholarData(): Promise<DataExportPayload> {
deleted_count: number; const response = await apiRequest<DataExportPayload>("/scholars/export", {
updated_count: number;
}
export async function bulkDeleteScholars(scholarProfileIds: number[]): Promise<BulkCountResult> {
const response = await apiRequest<BulkCountResult>("/scholars/bulk-delete", {
method: "POST",
body: { scholar_profile_ids: scholarProfileIds },
});
return response.data;
}
export async function bulkToggleScholars(scholarProfileIds: number[], isEnabled: boolean): Promise<BulkCountResult> {
const response = await apiRequest<BulkCountResult>("/scholars/bulk-toggle", {
method: "POST",
body: { scholar_profile_ids: scholarProfileIds, is_enabled: isEnabled },
});
return response.data;
}
export async function exportScholarData(ids?: number[]): Promise<DataExportPayload> {
const path = ids && ids.length > 0 ? `/scholars/export?ids=${ids.join(",")}` : "/scholars/export";
const response = await apiRequest<DataExportPayload>(path, {
method: "GET", method: "GET",
}); });
return response.data; return response.data;

View file

@ -1,94 +0,0 @@
// @vitest-environment happy-dom
import { describe, expect, it, vi, beforeEach } from "vitest";
import { mount, flushPromises } from "@vue/test-utils";
import { nextTick } from "vue";
import SettingsAdminPanel from "./SettingsAdminPanel.vue";
vi.mock("@/features/admin_users", () => ({
listAdminUsers: vi.fn().mockResolvedValue([]),
createAdminUser: vi.fn(),
setAdminUserActive: vi.fn(),
resetAdminUserPassword: vi.fn(),
}));
vi.mock("@/features/admin_dbops", () => ({
getAdminDbIntegrityReport: vi.fn().mockResolvedValue({
status: "ok",
warnings: [],
failures: [],
checked_at: null,
checks: [],
}),
listAdminPdfQueue: vi.fn().mockResolvedValue({
items: [],
total_count: 0,
has_next: false,
has_prev: false,
page: 1,
page_size: 50,
}),
requeueAdminPdfLookup: vi.fn(),
requeueAllAdminPdfLookups: vi.fn(),
}));
vi.mock("@/stores/auth", () => ({
useAuthStore: () => ({ isAdmin: true }),
}));
vi.mock("@/features/admin_repairs", () => ({
listAdminRepairTasks: vi.fn().mockResolvedValue([]),
runAdminRepairTask: vi.fn(),
}));
import { listAdminUsers } from "@/features/admin_users";
import { getAdminDbIntegrityReport, listAdminPdfQueue } from "@/features/admin_dbops";
const mockedListUsers = vi.mocked(listAdminUsers);
const mockedGetReport = vi.mocked(getAdminDbIntegrityReport);
const mockedListPdfQueue = vi.mocked(listAdminPdfQueue);
describe("SettingsAdminPanel", () => {
beforeEach(() => {
mockedListUsers.mockClear();
mockedGetReport.mockClear();
mockedListPdfQueue.mockClear();
});
it("calls load on users section after nextTick when section=users", async () => {
mount(SettingsAdminPanel, { props: { section: "users" } });
await nextTick();
await nextTick();
await flushPromises();
expect(mockedListUsers).toHaveBeenCalled();
});
it("calls load on integrity section after nextTick when section=integrity", async () => {
mount(SettingsAdminPanel, { props: { section: "integrity" } });
await nextTick();
await nextTick();
await flushPromises();
expect(mockedGetReport).toHaveBeenCalled();
});
it("calls load on new section when section prop changes", async () => {
const wrapper = mount(SettingsAdminPanel, { props: { section: "users" } });
await nextTick();
await nextTick();
await flushPromises();
mockedListUsers.mockClear();
mockedGetReport.mockClear();
await wrapper.setProps({ section: "integrity" });
await nextTick();
await flushPromises();
expect(mockedGetReport).toHaveBeenCalled();
});
it("calls load on pdf section when section=pdf", async () => {
mount(SettingsAdminPanel, { props: { section: "pdf" } });
await nextTick();
await nextTick();
await flushPromises();
expect(mockedListPdfQueue).toHaveBeenCalled();
});
});

View file

@ -1,7 +1,7 @@
<script setup lang="ts"> <script setup lang="ts">
import { nextTick, onMounted, watch } from "vue"; import { onMounted, ref, watch } from "vue";
import { ref } from "vue";
import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue"; import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue";
import AdminIntegritySection from "@/features/settings/components/AdminIntegritySection.vue"; import AdminIntegritySection from "@/features/settings/components/AdminIntegritySection.vue";
import AdminPdfQueueSection from "@/features/settings/components/AdminPdfQueueSection.vue"; import AdminPdfQueueSection from "@/features/settings/components/AdminPdfQueueSection.vue";
@ -18,6 +18,7 @@ const props = defineProps<{
section: "users" | "integrity" | "repairs" | "pdf"; section: "users" | "integrity" | "repairs" | "pdf";
}>(); }>();
const loading = ref(true);
const { errorMessage, errorRequestId, successMessage, clearAlerts, assignError } = useRequestState(); const { errorMessage, errorRequestId, successMessage, clearAlerts, assignError } = useRequestState();
const usersRef = ref<InstanceType<typeof AdminUsersSection> | null>(null); const usersRef = ref<InstanceType<typeof AdminUsersSection> | null>(null);
@ -25,33 +26,39 @@ const integrityRef = ref<InstanceType<typeof AdminIntegritySection> | null>(null
const repairsRef = ref<InstanceType<typeof AdminRepairsSection> | null>(null); const repairsRef = ref<InstanceType<typeof AdminRepairsSection> | null>(null);
const pdfQueueRef = ref<InstanceType<typeof AdminPdfQueueSection> | null>(null); const pdfQueueRef = ref<InstanceType<typeof AdminPdfQueueSection> | null>(null);
let loadGeneration = 0; async function refreshForSection(): Promise<void> {
if (props.section === SECTION_USERS && usersRef.value) {
async function loadSection(): Promise<void> { await usersRef.value.load();
const gen = ++loadGeneration; return;
clearAlerts(); }
try { if (props.section === SECTION_INTEGRITY && integrityRef.value) {
if (props.section === SECTION_USERS && usersRef.value) { await integrityRef.value.load();
await usersRef.value.load(); return;
} else if (props.section === SECTION_INTEGRITY && integrityRef.value) { }
await integrityRef.value.load(); if (props.section === SECTION_REPAIRS) {
} else if (props.section === SECTION_REPAIRS) { await usersRef.value?.load();
await usersRef.value?.load(); await repairsRef.value?.load();
if (gen !== loadGeneration) return; return;
await repairsRef.value?.load(); }
} else if (props.section === SECTION_PDF && pdfQueueRef.value) { if (props.section === SECTION_PDF && pdfQueueRef.value) {
await pdfQueueRef.value.load(); await pdfQueueRef.value.load();
}
} catch (error) {
if (gen !== loadGeneration) return;
assignError(error, "Unable to load admin data.");
} }
} }
onMounted(() => { async function loadSection(): Promise<void> {
nextTick(loadSection); loading.value = true;
}); clearAlerts();
watch(() => props.section, loadSection, { flush: "post" }); try {
await refreshForSection();
} catch (error) {
assignError(error, "Unable to load admin data.");
} finally {
loading.value = false;
}
}
onMounted(loadSection);
watch(() => props.section, loadSection);
</script> </script>
<template> <template>
@ -65,9 +72,11 @@ watch(() => props.section, loadSection, { flush: "post" });
@dismiss-success="successMessage = null" @dismiss-success="successMessage = null"
/> />
<AdminUsersSection v-if="props.section === SECTION_USERS || props.section === SECTION_REPAIRS" v-show="props.section === SECTION_USERS" ref="usersRef" /> <AsyncStateGate :loading="loading" :loading-lines="10" :show-empty="false">
<AdminIntegritySection v-if="props.section === SECTION_INTEGRITY" ref="integrityRef" /> <AdminUsersSection v-if="props.section === SECTION_USERS || props.section === SECTION_REPAIRS" v-show="props.section === SECTION_USERS" ref="usersRef" />
<AdminRepairsSection v-if="props.section === SECTION_REPAIRS" ref="repairsRef" :users="usersRef?.users ?? []" /> <AdminIntegritySection v-if="props.section === SECTION_INTEGRITY" ref="integrityRef" />
<AdminPdfQueueSection v-if="props.section === SECTION_PDF" ref="pdfQueueRef" /> <AdminRepairsSection v-if="props.section === SECTION_REPAIRS" ref="repairsRef" :users="usersRef?.users ?? []" />
<AdminPdfQueueSection v-if="props.section === SECTION_PDF" ref="pdfQueueRef" />
</AsyncStateGate>
</section> </section>
</template> </template>

View file

@ -1,8 +1,6 @@
// @vitest-environment happy-dom // @vitest-environment happy-dom
import { describe, expect, it, vi, beforeEach } from "vitest"; import { describe, expect, it, vi, beforeEach } from "vitest";
import { mount, flushPromises } from "@vue/test-utils"; import { mount, flushPromises } from "@vue/test-utils";
import { setActivePinia, createPinia } from "pinia";
import { useAuthStore } from "@/stores/auth";
import AdminPdfQueueSection from "./AdminPdfQueueSection.vue"; import AdminPdfQueueSection from "./AdminPdfQueueSection.vue";
vi.mock("@/features/admin_dbops", () => ({ vi.mock("@/features/admin_dbops", () => ({
@ -38,9 +36,6 @@ function buildQueueItem(overrides: Record<string, unknown> = {}) {
describe("AdminPdfQueueSection", () => { describe("AdminPdfQueueSection", () => {
beforeEach(() => { beforeEach(() => {
setActivePinia(createPinia());
const auth = useAuthStore();
auth.$patch({ state: "authenticated", user: { id: 1, email: "admin@test.com", is_admin: true, is_active: true } });
mockedListQueue.mockReset(); mockedListQueue.mockReset();
}); });

View file

@ -1,6 +1,5 @@
<script setup lang="ts"> <script setup lang="ts">
import { computed, ref } from "vue"; import { computed, ref } from "vue";
import { useAuthStore } from "@/stores/auth";
import AppBadge from "@/components/ui/AppBadge.vue"; import AppBadge from "@/components/ui/AppBadge.vue";
import AppButton from "@/components/ui/AppButton.vue"; import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue"; import AppCard from "@/components/ui/AppCard.vue";
@ -18,7 +17,6 @@ import {
import { useRequestState } from "@/composables/useRequestState"; import { useRequestState } from "@/composables/useRequestState";
const { errorMessage, errorRequestId, successMessage, clearAlerts, assignError, setSuccess } = useRequestState(); const { errorMessage, errorRequestId, successMessage, clearAlerts, assignError, setSuccess } = useRequestState();
const auth = useAuthStore();
const refreshingPdfQueue = ref(false); const refreshingPdfQueue = ref(false);
const requeueingPublicationId = ref<number | null>(null); const requeueingPublicationId = ref<number | null>(null);
@ -164,7 +162,7 @@ defineExpose({ load });
<option value="50">50 / page</option> <option value="50">50 / page</option>
<option value="100">100 / page</option> <option value="100">100 / page</option>
</AppSelect> </AppSelect>
<AppButton v-if="auth.isAdmin" variant="secondary" class="!min-h-8 whitespace-nowrap !px-2.5 !py-1 !text-xs" :disabled="requeueingAllPdfs" title="Queue all missing PDFs" @click="onRequeueAllPdfs"> <AppButton variant="secondary" class="!min-h-8 whitespace-nowrap !px-2.5 !py-1 !text-xs" :disabled="requeueingAllPdfs" title="Queue all missing PDFs" @click="onRequeueAllPdfs">
{{ requeueingAllPdfs ? "Queueing..." : "Queue all" }} {{ requeueingAllPdfs ? "Queueing..." : "Queue all" }}
</AppButton> </AppButton>
<AppRefreshButton variant="secondary" size="sm" :loading="refreshingPdfQueue" title="Refresh PDF queue" loading-title="Refreshing PDF queue" @click="refreshPdfQueue" /> <AppRefreshButton variant="secondary" size="sm" :loading="refreshingPdfQueue" title="Refresh PDF queue" loading-title="Refreshing PDF queue" @click="refreshPdfQueue" />
@ -196,7 +194,7 @@ defineExpose({ load });
<td>{{ formatTimestamp(item.last_attempt_at) }}</td> <td>{{ formatTimestamp(item.last_attempt_at) }}</td>
<td>{{ formatTimestamp(item.resolved_at) }}</td> <td>{{ formatTimestamp(item.resolved_at) }}</td>
<td> <td>
<AppButton v-if="auth.isAdmin" variant="ghost" :disabled="requeueingPublicationId === item.publication_id || !canRequeuePdf(item)" @click="onRequeuePdf(item)"> <AppButton variant="ghost" :disabled="requeueingPublicationId === item.publication_id || !canRequeuePdf(item)" @click="onRequeuePdf(item)">
{{ requeueingPublicationId === item.publication_id ? "Requeueing..." : "Requeue" }} {{ requeueingPublicationId === item.publication_id ? "Requeueing..." : "Requeue" }}
</AppButton> </AppButton>
</td> </td>

View file

@ -404,7 +404,7 @@ watch(
</div> </div>
<div class="flex items-center gap-2"> <div class="flex items-center gap-2">
<AppButton <AppButton
v-if="runStatus.isLikelyRunning" v-if="auth.isAdmin && runStatus.isLikelyRunning"
variant="danger" variant="danger"
:disabled="!activeRunId || isCancelAnimating" :disabled="!activeRunId || isCancelAnimating"
@click="onCancelRun" @click="onCancelRun"
@ -420,6 +420,7 @@ watch(
</span> </span>
</AppButton> </AppButton>
<AppButton <AppButton
v-if="auth.isAdmin"
:disabled="isStartBlocked" :disabled="isStartBlocked"
:title="startCheckDisabledReason || undefined" :title="startCheckDisabledReason || undefined"
:class="isStartCheckAnimating ? 'shadow-[0_0_0_1px_var(--color-state-info-border)]' : ''" :class="isStartCheckAnimating ? 'shadow-[0_0_0_1px_var(--color-state-info-border)]' : ''"
@ -485,21 +486,9 @@ watch(
</RouterLink> </RouterLink>
</div> </div>
<p class="mt-2 text-sm text-secondary"> <p class="mt-2 text-sm text-secondary">
Started {{ formatDate(displayedLatestRun.start_dt) }}. Started {{ formatDate(displayedLatestRun.start_dt) }}. Processed
<template v-if="runStatus.isLikelyRunning"> {{ displayedLatestRun.scholar_count }} scholars and discovered
{{ runStatus.scholarProgress?.visited ?? 0 }} / {{ runStatus.scholarProgress?.total ?? '…' }} visited {{ displayedLatestRun.new_publication_count }} new publications.
· {{ runStatus.scholarProgress?.finished ?? 0 }} finished
· {{ displayedLatestRun.new_publication_count }} new publications.
</template>
<template v-else>
Processed {{ displayedLatestRun.scholar_count }} scholars and discovered
{{ displayedLatestRun.new_publication_count }} new publications.
<template v-if="displayedLatestRun.failed_count > 0 || displayedLatestRun.partial_count > 0">
<span class="text-state-warning-text">
{{ displayedLatestRun.failed_count > 0 ? `${displayedLatestRun.failed_count} failed` : "" }}{{ displayedLatestRun.failed_count > 0 && displayedLatestRun.partial_count > 0 ? ", " : "" }}{{ displayedLatestRun.partial_count > 0 ? `${displayedLatestRun.partial_count} partial` : "" }}.
</span>
</template>
</template>
</p> </p>
</div> </div>
<AppEmptyState <AppEmptyState
@ -532,9 +521,6 @@ watch(
</div> </div>
<span class="text-xs text-secondary"> <span class="text-xs text-secondary">
{{ formatDate(run.start_dt) }} {{ run.new_publication_count }} new {{ formatDate(run.start_dt) }} {{ run.new_publication_count }} new
<template v-if="run.failed_count > 0">
<span class="text-state-warning-text">{{ run.failed_count }} failed</span>
</template>
</span> </span>
</li> </li>
</ul> </ul>

View file

@ -1,5 +1,5 @@
<script setup lang="ts"> <script setup lang="ts">
import { computed, onMounted, ref, watch } from "vue"; import { computed, onMounted, onUnmounted, ref, watch } from "vue";
import AppPage from "@/components/layout/AppPage.vue"; import AppPage from "@/components/layout/AppPage.vue";
import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue"; import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
@ -37,6 +37,23 @@ const selectedPublicationKeys = ref<Set<string>>(new Set());
const retryingPublicationKeys = ref<Set<string>>(new Set()); const retryingPublicationKeys = ref<Set<string>>(new Set());
const favoriteUpdatingKeys = ref<Set<string>>(new Set()); const favoriteUpdatingKeys = ref<Set<string>>(new Set());
const PUBLICATIONS_RUN_STATUS_SYNC_INTERVAL_MS = 5000;
let runStatusSyncTimer: ReturnType<typeof setInterval> | null = null;
function startRunStatusSyncLoop(): void {
if (runStatusSyncTimer !== null) return;
runStatusSyncTimer = setInterval(() => {
if (runStatus.isRunActive) return;
void runStatus.syncLatest();
}, PUBLICATIONS_RUN_STATUS_SYNC_INTERVAL_MS);
}
function stopRunStatusSyncLoop(): void {
if (runStatusSyncTimer === null) return;
clearInterval(runStatusSyncTimer);
runStatusSyncTimer = null;
}
// --- Computed --- // --- Computed ---
const selectedCount = computed(() => selectedPublicationKeys.value.size); const selectedCount = computed(() => selectedPublicationKeys.value.size);
@ -341,8 +358,13 @@ async function onStartRun(): Promise<void> {
onMounted(() => { onMounted(() => {
pub.syncFiltersFromRoute(); pub.syncFiltersFromRoute();
startRunStatusSyncLoop();
void Promise.all([pub.loadScholarFilters(), pub.loadPublications(), runStatus.syncLatest()]); void Promise.all([pub.loadScholarFilters(), pub.loadPublications(), runStatus.syncLatest()]);
}); });
onUnmounted(() => {
stopRunStatusSyncLoop();
});
</script> </script>
<template> <template>

View file

@ -6,7 +6,6 @@ import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue"; import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue";
import AppButton from "@/components/ui/AppButton.vue"; import AppButton from "@/components/ui/AppButton.vue";
import AppCard from "@/components/ui/AppCard.vue"; import AppCard from "@/components/ui/AppCard.vue";
import AppConfirmModal from "@/components/ui/AppConfirmModal.vue";
import AppEmptyState from "@/components/ui/AppEmptyState.vue"; import AppEmptyState from "@/components/ui/AppEmptyState.vue";
import AppHelpHint from "@/components/ui/AppHelpHint.vue"; import AppHelpHint from "@/components/ui/AppHelpHint.vue";
import AppInput from "@/components/ui/AppInput.vue"; import AppInput from "@/components/ui/AppInput.vue";
@ -28,12 +27,10 @@ import {
type ScholarProfile, type ScholarProfile,
type ScholarSearchCandidate, type ScholarSearchCandidate,
} from "@/features/scholars"; } from "@/features/scholars";
import { useScholarBulkActions } from "@/features/scholars/composables/useScholarBulkActions";
import ScholarAvatar from "@/features/scholars/components/ScholarAvatar.vue"; import ScholarAvatar from "@/features/scholars/components/ScholarAvatar.vue";
import ScholarBatchAdd from "@/features/scholars/components/ScholarBatchAdd.vue"; import ScholarBatchAdd from "@/features/scholars/components/ScholarBatchAdd.vue";
import ScholarNameSearch from "@/features/scholars/components/ScholarNameSearch.vue"; import ScholarNameSearch from "@/features/scholars/components/ScholarNameSearch.vue";
import ScholarSettingsModal from "@/features/scholars/components/ScholarSettingsModal.vue"; import ScholarSettingsModal from "@/features/scholars/components/ScholarSettingsModal.vue";
import ScholarStatusBadges from "@/features/scholars/components/ScholarStatusBadges.vue";
import { ApiRequestError } from "@/lib/api/errors"; import { ApiRequestError } from "@/lib/api/errors";
import { useRunStatusStore } from "@/stores/run_status"; import { useRunStatusStore } from "@/stores/run_status";
@ -273,71 +270,102 @@ async function onToggleScholar(): Promise<void> {
} }
} }
function onDeleteScholar(): void { async function onDeleteScholar(): Promise<void> {
const profile = activeScholarSettings.value; const profile = activeScholarSettings.value;
if (!profile) return; if (!profile) return;
const label = scholarLabel(profile); const label = scholarLabel(profile);
bulk.requestConfirm( if (!window.confirm(`Delete scholar ${label}? This removes all linked publications and queue data.`)) return;
`Delete ${label}?`, activeScholarId.value = profile.id;
"This removes all linked publications and queue data. This action cannot be undone.", clearMessages();
"danger", try {
async () => { await deleteScholar(profile.id);
bulk.dismissConfirm(); successMessage.value = `${label} deleted.`;
activeScholarId.value = profile.id; activeScholarSettingsId.value = null;
clearMessages(); await loadScholars();
try { } catch (error) {
await deleteScholar(profile.id); assignError(error, "Unable to delete scholar.");
successMessage.value = `${label} deleted.`; } finally {
activeScholarSettingsId.value = null; activeScholarId.value = null;
await loadScholars(); }
} catch (error) {
assignError(error, "Unable to delete scholar.");
} finally {
activeScholarId.value = null;
}
},
);
} }
async function onSaveImageUrl(): Promise<void> { async function onSaveImageUrl(): Promise<void> {
const p = activeScholarSettings.value; const profile = activeScholarSettings.value;
if (!p) return; if (!profile) return;
const url = (imageUrlDraftByScholarId.value[p.id] || "").trim(); const candidate = (imageUrlDraftByScholarId.value[profile.id] || "").trim();
if (!url) { errorMessage.value = "Enter an image URL before saving, or use Reset image."; return; } if (!candidate) { errorMessage.value = "Enter an image URL before saving, or use Reset image."; return; }
imageSavingScholarId.value = p.id; imageSavingScholarId.value = profile.id;
clearMessages(); clearMessages();
try { await setScholarImageUrl(p.id, url); successMessage.value = `Image URL updated for ${scholarLabel(p)}.`; await loadScholars(); } try {
catch (e) { assignError(e, "Unable to update scholar image URL."); } await setScholarImageUrl(profile.id, candidate);
finally { imageSavingScholarId.value = null; } successMessage.value = `Image URL updated for ${scholarLabel(profile)}.`;
await loadScholars();
} catch (error) {
assignError(error, "Unable to update scholar image URL.");
} finally {
imageSavingScholarId.value = null;
}
} }
async function onUploadImage(event: Event): Promise<void> { async function onUploadImage(event: Event): Promise<void> {
const p = activeScholarSettings.value; const profile = activeScholarSettings.value;
if (!p) return; if (!profile) return;
const input = event.target as HTMLInputElement | null; const input = event.target as HTMLInputElement | null;
const file = input?.files?.[0] ?? null; const file = input?.files?.[0] ?? null;
if (!file) return; if (!file) return;
imageUploadingScholarId.value = p.id; imageUploadingScholarId.value = profile.id;
clearMessages(); clearMessages();
try { await uploadScholarImage(p.id, file); successMessage.value = `Uploaded image for ${scholarLabel(p)}.`; await loadScholars(); } try {
catch (e) { assignError(e, "Unable to upload scholar image."); } await uploadScholarImage(profile.id, file);
finally { imageUploadingScholarId.value = null; if (input) input.value = ""; } successMessage.value = `Uploaded image for ${scholarLabel(profile)}.`;
await loadScholars();
} catch (error) {
assignError(error, "Unable to upload scholar image.");
} finally {
imageUploadingScholarId.value = null;
if (input) input.value = "";
}
} }
async function onResetImage(): Promise<void> { async function onResetImage(): Promise<void> {
const p = activeScholarSettings.value; const profile = activeScholarSettings.value;
if (!p) return; if (!profile) return;
imageSavingScholarId.value = p.id; imageSavingScholarId.value = profile.id;
clearMessages(); clearMessages();
try { await clearScholarImage(p.id); successMessage.value = `Image reset for ${scholarLabel(p)}.`; await loadScholars(); } try {
catch (e) { assignError(e, "Unable to reset scholar image."); } await clearScholarImage(profile.id);
finally { imageSavingScholarId.value = null; } successMessage.value = `Image reset for ${scholarLabel(profile)}.`;
await loadScholars();
} catch (error) {
assignError(error, "Unable to reset scholar image.");
} finally {
imageSavingScholarId.value = null;
}
} }
// --- Import/export --- // --- Import/export ---
function importSummary(r: DataImportResult): string { function suggestExportFilename(exportedAt: string): string {
return `Import complete. Scholars +${r.scholars_created} / updated ${r.scholars_updated}; publications +${r.publications_created} / updated ${r.publications_updated}; links +${r.links_created} / updated ${r.links_updated}; skipped ${r.skipped_records}.`; return `scholarr-export-${exportedAt.slice(0, 10) || "unknown-date"}.json`;
}
function downloadJsonFile(filename: string, payload: unknown): void {
const blob = new Blob([JSON.stringify(payload, null, 2)], { type: "application/json" });
const url = URL.createObjectURL(blob);
const anchor = document.createElement("a");
anchor.href = url;
anchor.download = filename;
anchor.click();
URL.revokeObjectURL(url);
}
function importSummary(result: DataImportResult): string {
return (
`Import complete. Scholars +${result.scholars_created}` +
` / updated ${result.scholars_updated}; publications +${result.publications_created}` +
` / updated ${result.publications_updated}; links +${result.links_created}` +
` / updated ${result.links_updated}; skipped ${result.skipped_records}.`
);
} }
async function onExportData(): Promise<void> { async function onExportData(): Promise<void> {
@ -345,13 +373,7 @@ async function onExportData(): Promise<void> {
clearMessages(); clearMessages();
try { try {
const payload = await exportScholarData(); const payload = await exportScholarData();
const blob = new Blob([JSON.stringify(payload, null, 2)], { type: "application/json" }); downloadJsonFile(suggestExportFilename(payload.exported_at), payload);
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = `scholarr-export-${payload.exported_at.slice(0, 10) || "unknown-date"}.json`;
a.click();
URL.revokeObjectURL(url);
successMessage.value = "Export complete."; successMessage.value = "Export complete.";
} catch (error) { } catch (error) {
assignError(error, "Unable to export scholars and publications."); assignError(error, "Unable to export scholars and publications.");
@ -360,6 +382,10 @@ async function onExportData(): Promise<void> {
} }
} }
function onOpenImportPicker(): void {
importFileInput.value?.click();
}
async function onImportFileSelected(event: Event): Promise<void> { async function onImportFileSelected(event: Event): Promise<void> {
const input = event.target as HTMLInputElement | null; const input = event.target as HTMLInputElement | null;
const file = input?.files?.[0] ?? null; const file = input?.files?.[0] ?? null;
@ -368,13 +394,12 @@ async function onImportFileSelected(event: Event): Promise<void> {
clearMessages(); clearMessages();
try { try {
const raw = await file.text(); const raw = await file.text();
let parsed = JSON.parse(raw); const parsed = JSON.parse(raw) as DataImportPayload;
if (parsed?.data && Array.isArray(parsed.data.scholars)) parsed = parsed.data; if (!parsed || !Array.isArray(parsed.scholars) || !Array.isArray(parsed.publications)) {
const payload = parsed as DataImportPayload;
if (!payload || !Array.isArray(payload.scholars) || !Array.isArray(payload.publications)) {
throw new Error("Invalid import file: expected scholars[] and publications[] arrays."); throw new Error("Invalid import file: expected scholars[] and publications[] arrays.");
} }
successMessage.value = importSummary(await importScholarData(payload)); const result = await importScholarData(parsed);
successMessage.value = importSummary(result);
await loadScholars(); await loadScholars();
} catch (error) { } catch (error) {
assignError(error, "Unable to import scholars and publications."); assignError(error, "Unable to import scholars and publications.");
@ -384,15 +409,6 @@ async function onImportFileSelected(event: Event): Promise<void> {
} }
} }
// --- Bulk actions ---
const bulk = useScholarBulkActions(visibleScholars, {
clearMessages,
assignError,
setSuccess: (msg: string) => { successMessage.value = msg; },
reloadScholars: loadScholars,
});
// --- Lifecycle --- // --- Lifecycle ---
onMounted(() => { void loadScholars(); }); onMounted(() => { void loadScholars(); });
@ -450,7 +466,7 @@ watch(
<AppButton variant="secondary" :disabled="loading || exportingData" @click="onExportData"> <AppButton variant="secondary" :disabled="loading || exportingData" @click="onExportData">
{{ exportingData ? "Exporting..." : "Export" }} {{ exportingData ? "Exporting..." : "Export" }}
</AppButton> </AppButton>
<AppButton variant="secondary" :disabled="loading || importingData" @click="importFileInput?.click()"> <AppButton variant="secondary" :disabled="loading || importingData" @click="onOpenImportPicker">
{{ importingData ? "Importing..." : "Import" }} {{ importingData ? "Importing..." : "Import" }}
</AppButton> </AppButton>
<AppRefreshButton variant="secondary" :disabled="saving" :loading="loading" title="Refresh scholars" loading-title="Refreshing scholars" @click="loadScholars" /> <AppRefreshButton variant="secondary" :disabled="saving" :loading="loading" title="Refresh scholars" loading-title="Refreshing scholars" @click="loadScholars" />
@ -475,34 +491,6 @@ watch(
</div> </div>
</div> </div>
<div class="flex flex-wrap items-center justify-between gap-2 border-t border-stroke-default pt-2">
<span class="text-xs text-secondary">
{{ trackedCountLabel }}
<template v-if="bulk.hasSelection.value"> · {{ bulk.selectedCount.value }} selected</template>
</span>
<div class="flex items-center gap-1">
<label for="scholars-bulk-action" class="sr-only">Bulk action</label>
<AppSelect
id="scholars-bulk-action"
v-model="bulk.bulkAction.value"
:disabled="bulk.bulkBusy.value || loading"
class="max-w-[14rem] !py-1.5 !text-xs"
>
<option v-for="option in bulk.bulkActionOptions.value" :key="option.value" :value="option.value">
{{ option.label }}
</option>
</AppSelect>
<AppButton
variant="secondary"
class="h-8 min-h-8 shrink-0 px-2 text-xs"
:disabled="bulk.bulkApplyDisabled.value"
@click="bulk.onApplyBulkAction"
>
{{ bulk.bulkApplyLabel.value }}
</AppButton>
</div>
</div>
<div class="min-h-0 flex-1 xl:overflow-hidden"> <div class="min-h-0 flex-1 xl:overflow-hidden">
<AsyncStateGate :loading="loading" :loading-lines="6" :empty="!hasTrackedScholars" :show-empty="!errorMessage" empty-title="No scholars tracked" empty-body="Add a Scholar ID or URL to start ingestion tracking."> <AsyncStateGate :loading="loading" :loading-lines="6" :empty="!hasTrackedScholars" :show-empty="!errorMessage" empty-title="No scholars tracked" empty-body="Add a Scholar ID or URL to start ingestion tracking.">
<AppEmptyState v-if="!hasVisibleScholars" title="No scholars match this filter" body="Clear or adjust the filter to see tracked scholars." /> <AppEmptyState v-if="!hasVisibleScholars" title="No scholars match this filter" body="Clear or adjust the filter to see tracked scholars." />
@ -510,19 +498,9 @@ watch(
<ul class="flex gap-3 overflow-x-auto p-1 lg:hidden"> <ul class="flex gap-3 overflow-x-auto p-1 lg:hidden">
<li v-for="item in visibleScholars" :key="item.id" class="rounded-xl border border-stroke-default bg-surface-card-muted/70 p-3"> <li v-for="item in visibleScholars" :key="item.id" class="rounded-xl border border-stroke-default bg-surface-card-muted/70 p-3">
<div class="flex items-start gap-3"> <div class="flex items-start gap-3">
<input
type="checkbox"
class="bulk-check mt-1 shrink-0"
:checked="bulk.selectedIds.value.has(item.id)"
:aria-label="`Select ${scholarLabel(item)}`"
@change="bulk.onToggleRow(item.id, $event)"
/>
<ScholarAvatar :label="item.display_name" :scholar-id="item.scholar_id" :image-url="item.profile_image_url" /> <ScholarAvatar :label="item.display_name" :scholar-id="item.scholar_id" :image-url="item.profile_image_url" />
<div class="min-w-0 flex-1 space-y-1"> <div class="min-w-0 flex-1 space-y-1">
<p class="truncate text-sm font-semibold text-ink-primary"> <p class="truncate text-sm font-semibold text-ink-primary">{{ scholarLabel(item) }}</p>
{{ scholarLabel(item) }}
<ScholarStatusBadges :baseline-completed="item.baseline_completed" :last-run-status="item.last_run_status" />
</p>
<div class="flex flex-wrap items-center gap-3"> <div class="flex flex-wrap items-center gap-3">
<RouterLink :to="scholarPublicationsRoute(item)" class="link-inline text-xs">Publications</RouterLink> <RouterLink :to="scholarPublicationsRoute(item)" class="link-inline text-xs">Publications</RouterLink>
<a :href="scholarProfileUrl(item.scholar_id)" target="_blank" rel="noreferrer" class="link-inline text-xs">Open profile</a> <a :href="scholarProfileUrl(item.scholar_id)" target="_blank" rel="noreferrer" class="link-inline text-xs">Open profile</a>
@ -536,39 +514,17 @@ watch(
<AppTable class="h-full overflow-y-scroll overscroll-contain" label="Tracked scholars table"> <AppTable class="h-full overflow-y-scroll overscroll-contain" label="Tracked scholars table">
<thead> <thead>
<tr> <tr>
<th scope="col" class="w-10">
<input
type="checkbox"
class="bulk-check"
:checked="bulk.allVisibleSelected.value"
:disabled="visibleScholars.length === 0"
aria-label="Select all visible scholars"
@change="bulk.onToggleAll"
/>
</th>
<th scope="col">Scholar</th> <th scope="col">Scholar</th>
<th scope="col" class="w-[11rem]">Manage</th> <th scope="col" class="w-[11rem]">Manage</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
<tr v-for="item in visibleScholars" :key="item.id"> <tr v-for="item in visibleScholars" :key="item.id">
<td>
<input
type="checkbox"
class="bulk-check"
:checked="bulk.selectedIds.value.has(item.id)"
:aria-label="`Select ${scholarLabel(item)}`"
@change="bulk.onToggleRow(item.id, $event)"
/>
</td>
<td> <td>
<div class="flex items-start gap-3"> <div class="flex items-start gap-3">
<ScholarAvatar :label="item.display_name" :scholar-id="item.scholar_id" :image-url="item.profile_image_url" /> <ScholarAvatar :label="item.display_name" :scholar-id="item.scholar_id" :image-url="item.profile_image_url" />
<div class="grid min-w-0 gap-1"> <div class="grid min-w-0 gap-1">
<strong class="truncate text-ink-primary"> <strong class="truncate text-ink-primary">{{ scholarLabel(item) }}</strong>
{{ scholarLabel(item) }}
<ScholarStatusBadges :baseline-completed="item.baseline_completed" :last-run-status="item.last_run_status" />
</strong>
<div class="flex flex-wrap items-center gap-3"> <div class="flex flex-wrap items-center gap-3">
<RouterLink :to="scholarPublicationsRoute(item)" class="link-inline text-xs">Publications</RouterLink> <RouterLink :to="scholarPublicationsRoute(item)" class="link-inline text-xs">Publications</RouterLink>
<a :href="scholarProfileUrl(item.scholar_id)" target="_blank" rel="noreferrer" class="link-inline text-xs">Open profile</a> <a :href="scholarProfileUrl(item.scholar_id)" target="_blank" rel="noreferrer" class="link-inline text-xs">Open profile</a>
@ -605,21 +561,5 @@ watch(
@toggle="onToggleScholar" @toggle="onToggleScholar"
@delete="onDeleteScholar" @delete="onDeleteScholar"
/> />
<AppConfirmModal
:open="bulk.confirmState.value.open"
:title="bulk.confirmState.value.title"
:message="bulk.confirmState.value.message"
:variant="bulk.confirmState.value.variant"
confirm-label="Delete"
@confirm="bulk.confirmState.value.onConfirm()"
@cancel="bulk.dismissConfirm()"
/>
</AppPage> </AppPage>
</template> </template>
<style scoped>
.bulk-check {
@apply h-4 w-4 rounded border-stroke-interactive bg-surface-input text-brand-600 focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset;
}
</style>

View file

@ -47,7 +47,7 @@ const savingScholarHttp = ref(false);
const updatingPassword = ref(false); const updatingPassword = ref(false);
const autoRunEnabled = ref(false); const autoRunEnabled = ref(false);
const runIntervalHours = ref("1"); const runIntervalMinutes = ref("60");
const requestDelaySeconds = ref("2"); const requestDelaySeconds = ref("2");
const navVisiblePages = ref<string[]>([]); const navVisiblePages = ref<string[]>([]);
const openalexApiKey = ref(""); const openalexApiKey = ref("");
@ -77,13 +77,13 @@ const tabItems = computed<AppTabItem[]>(() => {
const tabs: AppTabItem[] = [ const tabs: AppTabItem[] = [
{ id: TAB_CHECKING, label: "Checking" }, { id: TAB_CHECKING, label: "Checking" },
{ id: TAB_ACCOUNT, label: "Account" }, { id: TAB_ACCOUNT, label: "Account" },
{ id: TAB_ADMIN_PDF, label: "PDF Queue" },
]; ];
if (auth.isAdmin) { if (auth.isAdmin) {
tabs.push( tabs.push(
{ id: TAB_ADMIN_USERS, label: "Users" }, { id: TAB_ADMIN_USERS, label: "Users" },
{ id: TAB_ADMIN_INTEGRITY, label: "Integrity" }, { id: TAB_ADMIN_INTEGRITY, label: "Integrity" },
{ id: TAB_ADMIN_REPAIRS, label: "Repairs" }, { id: TAB_ADMIN_REPAIRS, label: "Repairs" },
{ id: TAB_ADMIN_PDF, label: "PDF Queue" },
{ id: TAB_ADMIN_INTEGRATIONS, label: "Integrations" }, { id: TAB_ADMIN_INTEGRATIONS, label: "Integrations" },
); );
} }
@ -138,7 +138,7 @@ function hydrateSettings(settings: UserSettings): void {
manualRunAllowed.value = Boolean(settings.policy?.manual_run_allowed ?? true); manualRunAllowed.value = Boolean(settings.policy?.manual_run_allowed ?? true);
autoRunEnabled.value = Boolean(settings.auto_run_enabled) && automationAllowed.value; autoRunEnabled.value = Boolean(settings.auto_run_enabled) && automationAllowed.value;
runIntervalHours.value = String(Number(settings.run_interval_minutes) / 60); runIntervalMinutes.value = String(settings.run_interval_minutes);
requestDelaySeconds.value = String(settings.request_delay_seconds); requestDelaySeconds.value = String(settings.request_delay_seconds);
navVisiblePages.value = normalizeUserNavVisiblePages(settings.nav_visible_pages); navVisiblePages.value = normalizeUserNavVisiblePages(settings.nav_visible_pages);
@ -168,23 +168,6 @@ function parseBoundedInteger(value: string, label: string, minimum: number): num
return parsed; return parsed;
} }
function formatHours(minutes: number): string {
const h = minutes / 60;
return Number.isInteger(h) ? String(h) : h.toFixed(2).replace(/\.?0+$/, "");
}
function parseHoursToMinutes(value: string, minMinutes: number): number {
const hours = Number(value);
if (!Number.isFinite(hours) || hours <= 0) {
throw new Error("Check interval must be a positive number of hours.");
}
const minutes = Math.round(hours * 60);
if (minutes < minMinutes) {
throw new Error(`Check interval must be at least ${formatHours(minMinutes)} hours.`);
}
return minutes;
}
async function loadSettings(): Promise<void> { async function loadSettings(): Promise<void> {
loading.value = true; loading.value = true;
errorMessage.value = null; errorMessage.value = null;
@ -242,8 +225,9 @@ async function onSaveSettings(): Promise<void> {
try { try {
const payload: UserSettingsUpdate = { const payload: UserSettingsUpdate = {
auto_run_enabled: autoRunEnabled.value, auto_run_enabled: autoRunEnabled.value,
run_interval_minutes: parseHoursToMinutes( run_interval_minutes: parseBoundedInteger(
runIntervalHours.value, runIntervalMinutes.value,
"Check interval (minutes)",
minCheckIntervalMinutes.value, minCheckIntervalMinutes.value,
), ),
request_delay_seconds: parseBoundedInteger( request_delay_seconds: parseBoundedInteger(
@ -373,11 +357,11 @@ onMounted(async () => {
<label class="grid gap-2 text-sm font-medium text-ink-secondary"> <label class="grid gap-2 text-sm font-medium text-ink-secondary">
<span class="inline-flex items-center gap-1"> <span class="inline-flex items-center gap-1">
Check interval (hours) Check interval (minutes)
<AppHelpHint text="Minimum is controlled by server policy." /> <AppHelpHint text="Minimum is controlled by server policy." />
</span> </span>
<AppInput v-model="runIntervalHours" inputmode="decimal" /> <AppInput v-model="runIntervalMinutes" inputmode="numeric" />
<span class="text-xs text-secondary">Minimum: {{ formatHours(minCheckIntervalMinutes) }} hours</span> <span class="text-xs text-secondary">Minimum: {{ minCheckIntervalMinutes }}</span>
</label> </label>
<label class="grid gap-2 text-sm font-medium text-ink-secondary"> <label class="grid gap-2 text-sm font-medium text-ink-secondary">

View file

@ -9,7 +9,7 @@ import {
import { type PublicationItem } from "@/features/publications"; import { type PublicationItem } from "@/features/publications";
import { ApiRequestError } from "@/lib/api/errors"; import { ApiRequestError } from "@/lib/api/errors";
export const RUN_STATUS_POLL_INTERVAL_MS = 15000; export const RUN_STATUS_POLL_INTERVAL_MS = 5000;
export const RUN_STATUS_STARTING_PHASE_MS = 1500; export const RUN_STATUS_STARTING_PHASE_MS = 1500;
export type StartManualCheckResult = export type StartManualCheckResult =
@ -168,7 +168,6 @@ export const useRunStatusStore = defineStore("runStatus", {
lastErrorRequestId: null as string | null, lastErrorRequestId: null as string | null,
lastSyncAt: null as number | null, lastSyncAt: null as number | null,
livePublications: [] as Array<PublicationItem>, livePublications: [] as Array<PublicationItem>,
scholarProgress: null as { visited: number; finished: number; total: number } | null,
}), }),
getters: { getters: {
isRunActive(state): boolean { isRunActive(state): boolean {
@ -258,7 +257,6 @@ export const useRunStatusStore = defineStore("runStatus", {
} }
activeStreamRunId = targetRunId; activeStreamRunId = targetRunId;
this.livePublications = []; this.livePublications = [];
this.scholarProgress = null;
eventSource = new EventSource(`/api/v1/runs/${targetRunId}/stream`); eventSource = new EventSource(`/api/v1/runs/${targetRunId}/stream`);
eventSource.addEventListener("publication_discovered", (e) => { eventSource.addEventListener("publication_discovered", (e) => {
try { try {
@ -314,19 +312,6 @@ export const useRunStatusStore = defineStore("runStatus", {
console.error("Failed to parse SSE event", err); console.error("Failed to parse SSE event", err);
} }
}); });
eventSource.addEventListener("scholar_progress", (e) => {
try {
const data = JSON.parse(e.data);
const visited = typeof data.visited === "number" ? Math.max(0, Math.trunc(data.visited)) : null;
const finished = typeof data.finished === "number" ? Math.max(0, Math.trunc(data.finished)) : null;
const total = typeof data.total === "number" ? Math.max(1, Math.trunc(data.total)) : null;
if (visited !== null && finished !== null && total !== null) {
this.scholarProgress = { visited, finished, total };
}
} catch (err) {
console.error("Failed to parse SSE event", err);
}
});
eventSource.onerror = () => { eventSource.onerror = () => {
// Reconnecting is handled automatically by EventSource, // Reconnecting is handled automatically by EventSource,
// but if it's permanently closed, we could do something here. // but if it's permanently closed, we could do something here.
@ -506,7 +491,6 @@ export const useRunStatusStore = defineStore("runStatus", {
this.lastSyncAt = null; this.lastSyncAt = null;
this.safetyState = createDefaultSafetyState(); this.safetyState = createDefaultSafetyState();
this.livePublications = []; this.livePublications = [];
this.scholarProgress = null;
}, },
}, },
}); });

View file

@ -1,58 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
DC="docker compose -f docker-compose.yml -f docker-compose.dev.yml"
passed=0
failed=0
failures=()
step() {
local label="$1"
shift
printf '\n\033[1;34m── %s\033[0m\n' "$label"
if "$@"; then
printf '\033[1;32m PASS\033[0m %s\n' "$label"
((passed++)) || true
else
printf '\033[1;31m FAIL\033[0m %s\n' "$label"
((failed++)) || true
failures+=("$label")
fi
}
# ── repo hygiene (host-side, no runtime deps) ────────────────────────────────
step "No generated artifacts" ./scripts/check_no_generated_artifacts.sh
step "Env contract parity" python3 scripts/check_env_contract.py
step "API contract drift" python3 scripts/check_frontend_api_contract.py
# ── backend lint + typecheck ─────────────────────────────────────────────────
step "Ruff check" $DC run --rm app ruff check .
step "Ruff format" $DC run --rm app ruff format --check .
step "Mypy" $DC run --rm app mypy app/ --ignore-missing-imports
# ── backend tests ────────────────────────────────────────────────────────────
step "Unit tests" $DC run --rm app python -m pytest tests/unit
step "Integration tests" $DC run --rm app python -m pytest -m integration
# ── frontend ─────────────────────────────────────────────────────────────────
step "Frontend theme tokens" $DC run --rm frontend npm run check:theme-tokens
step "Frontend typecheck" $DC run --rm frontend npm run typecheck
step "Frontend unit tests" $DC run --rm frontend npm run test:run
step "Frontend build" $DC run --rm frontend npm run build
# ── docs ─────────────────────────────────────────────────────────────────────
step "Docs build" npm --prefix docs/website run build
# ── summary ──────────────────────────────────────────────────────────────────
printf '\n\033[1;34m── Summary ─────────────────────────────────────────────\033[0m\n'
printf ' %s passed, %s failed\n' "$passed" "$failed"
if ((failed > 0)); then
printf '\n\033[1;31m Failed steps:\033[0m\n'
for f in "${failures[@]}"; do
printf ' - %s\n' "$f"
done
exit 1
else
printf '\n\033[1;32m All checks passed.\033[0m\n'
fi

View file

@ -340,11 +340,10 @@ async def test_api_admin_dbops_forbidden_for_non_admin_and_validates_scope(db_se
assert forbidden_integrity.status_code == 403 assert forbidden_integrity.status_code == 403
assert forbidden_integrity.json()["error"]["code"] == "forbidden" assert forbidden_integrity.json()["error"]["code"] == "forbidden"
# PDF queue listing is accessible to all authenticated users (not admin-only). forbidden_pdf_queue = client.get("/api/v1/admin/db/pdf-queue")
allowed_pdf_queue = client.get("/api/v1/admin/db/pdf-queue") assert forbidden_pdf_queue.status_code == 403
assert allowed_pdf_queue.status_code == 200 assert forbidden_pdf_queue.json()["error"]["code"] == "forbidden"
# But requeue actions still require admin.
forbidden_requeue = client.post( forbidden_requeue = client.post(
"/api/v1/admin/db/pdf-queue/1/requeue", "/api/v1/admin/db/pdf-queue/1/requeue",
headers=headers, headers=headers,

View file

@ -9,7 +9,6 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.api.runtime_deps import get_scholar_source from app.api.runtime_deps import get_scholar_source
from app.main import app from app.main import app
from app.services.scholar.rate_limit import reset_scholar_rate_limit_state_for_tests
from app.services.scholar.source import FetchResult from app.services.scholar.source import FetchResult
from app.settings import settings from app.settings import settings
from tests.integration.helpers import ( from tests.integration.helpers import (
@ -70,110 +69,6 @@ async def test_api_scholars_crud_flow(db_session: AsyncSession) -> None:
assert delete_response.json()["data"]["message"] == "Scholar deleted." assert delete_response.json()["data"]["message"] == "Scholar deleted."
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_delete_scholar_returns_404_for_nonexistent(db_session: AsyncSession) -> None:
await insert_user(
db_session,
email="api-delete-404@example.com",
password="api-password",
)
client = TestClient(app)
login_user(client, email="api-delete-404@example.com", password="api-password")
headers = api_csrf_headers(client)
response = client.delete("/api/v1/scholars/999999", headers=headers)
assert response.status_code == 404
assert response.json()["error"]["code"] == "scholar_not_found"
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_delete_scholar_cascades_linked_publications(db_session: AsyncSession) -> None:
user_id = await insert_user(
db_session,
email="api-delete-cascade@example.com",
password="api-password",
)
scholar_result = await db_session.execute(
text(
"""
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
VALUES (:user_id, :scholar_id, :display_name, true)
RETURNING id
"""
),
{"user_id": user_id, "scholar_id": "delCASCADE01", "display_name": "Cascade Test"},
)
scholar_profile_id = int(scholar_result.scalar_one())
pub_result = await db_session.execute(
text(
"""
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
VALUES (:fingerprint, :title_raw, :title_normalized, 0)
RETURNING id
"""
),
{
"fingerprint": f"{(user_id + 7000):064x}",
"title_raw": "Cascade Publication",
"title_normalized": "cascadepublication",
},
)
publication_id = int(pub_result.scalar_one())
await db_session.execute(
text(
"""
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
VALUES (:scholar_profile_id, :publication_id, false)
"""
),
{"scholar_profile_id": scholar_profile_id, "publication_id": publication_id},
)
await db_session.commit()
client = TestClient(app)
login_user(client, email="api-delete-cascade@example.com", password="api-password")
headers = api_csrf_headers(client)
delete_response = client.delete(f"/api/v1/scholars/{scholar_profile_id}", headers=headers)
assert delete_response.status_code == 200
assert delete_response.json()["data"]["message"] == "Scholar deleted."
link_result = await db_session.execute(
text("SELECT count(*) FROM scholar_publications WHERE scholar_profile_id = :id"),
{"id": scholar_profile_id},
)
assert int(link_result.scalar_one()) == 0
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_api_create_scholar_rejects_corrupted_id(db_session: AsyncSession) -> None:
await insert_user(
db_session,
email="api-bad-id@example.com",
password="api-password",
)
client = TestClient(app)
login_user(client, email="api-bad-id@example.com", password="api-password")
headers = api_csrf_headers(client)
for bad_id in ["short", "ABCDEF12345!", "", " ", "AB CD EF1234"]:
response = client.post(
"/api/v1/scholars",
json={"scholar_id": bad_id},
headers=headers,
)
assert response.status_code == 400, f"Expected 400 for scholar_id={bad_id!r}"
assert response.json()["error"]["code"] == "invalid_scholar"
@pytest.mark.integration @pytest.mark.integration
@pytest.mark.db @pytest.mark.db
@pytest.mark.asyncio @pytest.mark.asyncio
@ -246,7 +141,6 @@ async def test_api_scholars_search_and_profile_image_management(
assert candidate["scholar_id"] == "abcDEF123456" assert candidate["scholar_id"] == "abcDEF123456"
assert candidate["profile_image_url"] == "https://scholar.google.com/citations/images/avatar_scholar_256.png" assert candidate["profile_image_url"] == "https://scholar.google.com/citations/images/avatar_scholar_256.png"
reset_scholar_rate_limit_state_for_tests()
create_response = client.post( create_response = client.post(
"/api/v1/scholars", "/api/v1/scholars",
json={ json={

View file

@ -1,170 +0,0 @@
from __future__ import annotations
import pytest
from fastapi.testclient import TestClient
from sqlalchemy.ext.asyncio import AsyncSession
from app.main import app
from tests.integration.helpers import (
api_csrf_headers,
insert_user,
login_user,
)
def _create_scholar(client: TestClient, headers: dict, scholar_id: str) -> int:
resp = client.post("/api/v1/scholars", json={"scholar_id": scholar_id}, headers=headers)
assert resp.status_code == 201
return int(resp.json()["data"]["id"])
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_bulk_delete_with_valid_ids(db_session: AsyncSession) -> None:
await insert_user(db_session, email="bulk@example.com", password="pw123456")
client = TestClient(app)
login_user(client, email="bulk@example.com", password="pw123456")
headers = api_csrf_headers(client)
id1 = _create_scholar(client, headers, "aaaBBB111222")
id2 = _create_scholar(client, headers, "cccDDD333444")
resp = client.post(
"/api/v1/scholars/bulk-delete",
json={"scholar_profile_ids": [id1, id2]},
headers=headers,
)
assert resp.status_code == 200
assert resp.json()["data"]["deleted_count"] == 2
list_resp = client.get("/api/v1/scholars")
assert len(list_resp.json()["data"]["scholars"]) == 0
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_bulk_delete_only_deletes_own_scholars(db_session: AsyncSession) -> None:
await insert_user(db_session, email="user1@example.com", password="pw123456")
await insert_user(db_session, email="user2@example.com", password="pw123456")
client1 = TestClient(app)
login_user(client1, email="user1@example.com", password="pw123456")
headers1 = api_csrf_headers(client1)
client2 = TestClient(app)
login_user(client2, email="user2@example.com", password="pw123456")
headers2 = api_csrf_headers(client2)
id_user1 = _create_scholar(client1, headers1, "aaaBBB111222")
id_user2 = _create_scholar(client2, headers2, "cccDDD333444")
# User1 tries to delete both — should only delete own
resp = client1.post(
"/api/v1/scholars/bulk-delete",
json={"scholar_profile_ids": [id_user1, id_user2]},
headers=headers1,
)
assert resp.status_code == 200
assert resp.json()["data"]["deleted_count"] == 1
# User2's scholar still exists
list_resp = client2.get("/api/v1/scholars")
scholars = list_resp.json()["data"]["scholars"]
assert len(scholars) == 1
assert int(scholars[0]["id"]) == id_user2
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_bulk_toggle_enables_and_disables(db_session: AsyncSession) -> None:
await insert_user(db_session, email="toggle@example.com", password="pw123456")
client = TestClient(app)
login_user(client, email="toggle@example.com", password="pw123456")
headers = api_csrf_headers(client)
id1 = _create_scholar(client, headers, "aaaBBB111222")
id2 = _create_scholar(client, headers, "cccDDD333444")
# Disable both
resp = client.post(
"/api/v1/scholars/bulk-toggle",
json={"scholar_profile_ids": [id1, id2], "is_enabled": False},
headers=headers,
)
assert resp.status_code == 200
assert resp.json()["data"]["updated_count"] == 2
scholars = client.get("/api/v1/scholars").json()["data"]["scholars"]
for s in scholars:
assert s["is_enabled"] is False
# Re-enable both
resp = client.post(
"/api/v1/scholars/bulk-toggle",
json={"scholar_profile_ids": [id1, id2], "is_enabled": True},
headers=headers,
)
assert resp.status_code == 200
assert resp.json()["data"]["updated_count"] == 2
scholars = client.get("/api/v1/scholars").json()["data"]["scholars"]
for s in scholars:
assert s["is_enabled"] is True
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_export_with_ids_filter(db_session: AsyncSession) -> None:
await insert_user(db_session, email="export@example.com", password="pw123456")
client = TestClient(app)
login_user(client, email="export@example.com", password="pw123456")
headers = api_csrf_headers(client)
id1 = _create_scholar(client, headers, "aaaBBB111222")
_create_scholar(client, headers, "cccDDD333444")
# Export only id1
resp = client.get(f"/api/v1/scholars/export?ids={id1}")
assert resp.status_code == 200
data = resp.json()["data"]
assert len(data["scholars"]) == 1
assert data["scholars"][0]["scholar_id"] == "aaaBBB111222"
# Export all (no filter)
resp_all = client.get("/api/v1/scholars/export")
assert resp_all.status_code == 200
assert len(resp_all.json()["data"]["scholars"]) == 2
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_bulk_delete_rejects_without_csrf(db_session: AsyncSession) -> None:
await insert_user(db_session, email="csrf-bulk-del@example.com", password="pw123456")
client = TestClient(app)
login_user(client, email="csrf-bulk-del@example.com", password="pw123456")
response = client.post(
"/api/v1/scholars/bulk-delete",
json={"scholar_profile_ids": [1]},
)
assert response.status_code == 403
assert response.json()["error"]["code"] == "csrf_invalid"
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_bulk_toggle_rejects_without_csrf(db_session: AsyncSession) -> None:
await insert_user(db_session, email="csrf-bulk-tog@example.com", password="pw123456")
client = TestClient(app)
login_user(client, email="csrf-bulk-tog@example.com", password="pw123456")
response = client.post(
"/api/v1/scholars/bulk-toggle",
json={"scholar_profile_ids": [1], "is_enabled": False},
)
assert response.status_code == 403
assert response.json()["error"]["code"] == "csrf_invalid"

View file

@ -76,9 +76,6 @@ class FixtureScholarSource:
async def fetch_profile_html(self, scholar_id: str) -> FetchResult: async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
return await self.fetch_profile_page_html(scholar_id, cstart=0, pagesize=100) return await self.fetch_profile_page_html(scholar_id, cstart=0, pagesize=100)
async def fetch_author_search_html(self, query: str, *, start: int = 0) -> FetchResult:
raise NotImplementedError
@pytest.mark.integration @pytest.mark.integration
@pytest.mark.db @pytest.mark.db

View file

@ -1,176 +0,0 @@
from __future__ import annotations
import pytest
from fastapi.testclient import TestClient
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.runtime_deps import get_scholar_source
from app.main import app
from app.services.scholar.source import FetchResult
from tests.integration.helpers import (
api_csrf_headers,
insert_user,
login_user,
wait_for_run_complete,
)
SCHOLAR_ID = "newPubDetc01"
def _build_profile_html(publications: list[dict[str, str]]) -> str:
rows = []
for pub in publications:
rows.append(
f"""
<tr class="gsc_a_tr">
<td class="gsc_a_t">
<a class="gsc_a_at"
href="/citations?view_op=view_citation&amp;citation_for_view={SCHOLAR_ID}:{pub["cluster"]}"
>{pub["title"]}</a>
<div class="gs_gray">{pub.get("authors", "A Author")}</div>
<div class="gs_gray">{pub.get("venue", "Some Venue")}</div>
</td>
<td class="gsc_a_c"><a class="gsc_a_ac">{pub.get("citations", "0")}</a></td>
<td class="gsc_a_y"><span class="gsc_a_h">{pub.get("year", "2024")}</span></td>
</tr>"""
)
count = len(publications)
return f"""
<html>
<head>
<meta property="og:image" content="https://images.example.com/detect.png" />
</head>
<body>
<div id="gsc_prf_in">New Pub Detector</div>
<span id="gsc_a_nn">Articles 1-{count}</span>
<table>
<tbody id="gsc_a_b">{"".join(rows)}
</tbody>
</table>
</body>
</html>
"""
INITIAL_PUBLICATIONS = [
{"cluster": "aaa111", "title": "Existing Paper One", "year": "2023", "citations": "10"},
{"cluster": "bbb222", "title": "Existing Paper Two", "year": "2022", "citations": "5"},
]
UPDATED_PUBLICATIONS = [
*INITIAL_PUBLICATIONS,
{"cluster": "ccc333", "title": "Brand New Paper Three", "year": "2025", "citations": "0"},
]
class _MutableScholarSource:
"""Source that serves different HTML per phase to simulate page changes."""
def __init__(self, initial_html: str) -> None:
self.html = initial_html
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
return self._result(scholar_id)
async def fetch_profile_page_html(
self,
scholar_id: str,
*,
cstart: int,
pagesize: int,
) -> FetchResult:
return self._result(scholar_id)
def _result(self, scholar_id: str) -> FetchResult:
return FetchResult(
requested_url=f"https://scholar.google.com/citations?hl=en&user={scholar_id}",
status_code=200,
final_url=f"https://scholar.google.com/citations?hl=en&user={scholar_id}",
body=self.html,
error=None,
)
@pytest.mark.integration
@pytest.mark.db
@pytest.mark.asyncio
async def test_new_publication_detected_after_page_change(
db_session: AsyncSession,
) -> None:
"""Verify the fingerprint short-circuit lets new publications through.
Phase 1: Initial scrape discovers 2 publications.
Phase 2: Same page skipped via no_change fingerprint.
Phase 3: Page gains a 3rd publication fingerprint differs,
scrape runs and discovers the new entry.
"""
await insert_user(db_session, email="new-pub-detect@example.com", password="api-password")
source = _MutableScholarSource(_build_profile_html(INITIAL_PUBLICATIONS))
app.dependency_overrides[get_scholar_source] = lambda: source
try:
with TestClient(app) as client:
login_user(client, email="new-pub-detect@example.com", password="api-password")
headers = api_csrf_headers(client)
create_resp = client.post("/api/v1/scholars", json={"scholar_id": SCHOLAR_ID}, headers=headers)
assert create_resp.status_code == 201
# ── Phase 1: initial scrape ─────────────────────────────
run1 = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "detect-run-001"},
)
assert run1.status_code == 200
run1_id = int(run1.json()["data"]["run_id"])
run1_data = await wait_for_run_complete(client, run1_id)
assert run1_data["scholar_results"][0]["outcome"] == "success"
assert run1_data["scholar_results"][0]["publication_count"] == 2
assert run1_data["scholar_results"][0].get("state_reason") != "no_change_initial_page_signature"
pubs1 = client.get("/api/v1/publications?mode=latest").json()["data"]
assert pubs1["total_count"] == 2
assert all(p["is_new_in_latest_run"] for p in pubs1["publications"])
# ── Phase 2: unchanged page → skipped ──────────────────
run2 = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "detect-run-002"},
)
assert run2.status_code == 200
run2_id = int(run2.json()["data"]["run_id"])
run2_data = await wait_for_run_complete(client, run2_id)
assert run2_data["scholar_results"][0]["state_reason"] == "no_change_initial_page_signature"
assert run2_data["scholar_results"][0]["publication_count"] == 0
# ── Phase 3: new publication appears ────────────────────
source.html = _build_profile_html(UPDATED_PUBLICATIONS)
run3 = client.post(
"/api/v1/runs/manual",
headers={**headers, "Idempotency-Key": "detect-run-003"},
)
assert run3.status_code == 200
run3_id = int(run3.json()["data"]["run_id"])
run3_data = await wait_for_run_complete(client, run3_id)
assert run3_data["scholar_results"][0]["outcome"] == "success"
assert run3_data["scholar_results"][0].get("state_reason") != "no_change_initial_page_signature"
assert run3_data["scholar_results"][0]["publication_count"] == 3
pubs3 = client.get("/api/v1/publications?mode=latest").json()["data"]
assert pubs3["total_count"] == 3
titles = {p["title"] for p in pubs3["publications"]}
assert "Brand New Paper Three" in titles
new_pub = next(p for p in pubs3["publications"] if p["title"] == "Brand New Paper Three")
assert new_pub["is_new_in_latest_run"] is True
old_pubs = [p for p in pubs3["publications"] if p["title"] != "Brand New Paper Three"]
for p in old_pubs:
assert p["is_new_in_latest_run"] is False
finally:
app.dependency_overrides.pop(get_scholar_source, None)

View file

@ -30,12 +30,11 @@ class _PassthroughSource:
error=None, error=None,
) )
async def fetch_profile_page_html(self, scholar_id: str, *, cstart: int, pagesize: int) -> FetchResult: async def fetch_profile_page_html(
self, scholar_id: str, *, cstart: int, pagesize: int
) -> FetchResult:
return await self.fetch_profile_html(scholar_id) return await self.fetch_profile_html(scholar_id)
async def fetch_author_search_html(self, query: str, *, start: int = 0) -> FetchResult:
raise NotImplementedError
def _csrf_headers(client: TestClient) -> dict[str, str]: def _csrf_headers(client: TestClient) -> dict[str, str]:
response = client.get("/api/v1/auth/me") response = client.get("/api/v1/auth/me")
@ -349,7 +348,6 @@ async def test_partial_discovery_exception_keeps_new_pub_count_consistent(
) )
db_session.add_all([run, publication]) db_session.add_all([run, publication])
await db_session.commit() await db_session.commit()
run_id = int(run.id)
call_count = 0 call_count = 0
@ -401,8 +399,9 @@ async def test_partial_discovery_exception_keeps_new_pub_count_consistent(
publications=publications, publications=publications,
) )
refreshed_run = await db_session.get(CrawlRun, run_id) refreshed_run = await db_session.get(CrawlRun, int(run.id))
assert refreshed_run is not None assert refreshed_run is not None
assert int(refreshed_run.new_pub_count) == 1
link_count_result = await db_session.execute( link_count_result = await db_session.execute(
text( text(
""" """
@ -412,10 +411,9 @@ async def test_partial_discovery_exception_keeps_new_pub_count_consistent(
AND first_seen_run_id = :run_id AND first_seen_run_id = :run_id
""" """
), ),
{"scholar_profile_id": scholar_profile_id, "run_id": run_id}, {"scholar_profile_id": scholar_profile_id, "run_id": int(run.id)},
) )
link_count = int(link_count_result.scalar_one()) assert int(link_count_result.scalar_one()) == 1
assert int(refreshed_run.new_pub_count) == link_count
@pytest.mark.integration @pytest.mark.integration

View file

@ -1,17 +0,0 @@
from __future__ import annotations
import pytest
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@pytest.fixture
def patch_session_factory(db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch):
"""Patch get_session_factory in arxiv modules to use the test's DB engine.
Without this, the implementation creates its own engine via get_session_factory()
which may not share transaction visibility with the test's db_session fixture.
"""
factory = async_sessionmaker(db_session.bind, expire_on_commit=False)
monkeypatch.setattr("app.services.arxiv.rate_limit.get_session_factory", lambda: factory)
monkeypatch.setattr("app.services.arxiv.cache.get_session_factory", lambda: factory)
return factory

View file

@ -147,8 +147,8 @@ async def test_client_coalesces_concurrent_identical_search_requests() -> None:
async def test_client_logs_cache_hit_and_miss( async def test_client_logs_cache_hit_and_miss(
db_session: AsyncSession, db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
patch_session_factory,
) -> None: ) -> None:
_ = db_session
calls = {"count": 0} calls = {"count": 0}
logged: list[dict[str, object]] = [] logged: list[dict[str, object]] = []

View file

@ -1,6 +1,6 @@
from __future__ import annotations from __future__ import annotations
from typing import Any from types import SimpleNamespace
import pytest import pytest
@ -10,9 +10,7 @@ from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
from app.settings import settings from app.settings import settings
def _item(*, title: str = "A Test Paper", scholar_label: str = "Ada Lovelace") -> Any: def _item(*, title: str = "A Test Paper", scholar_label: str = "Ada Lovelace") -> SimpleNamespace:
from types import SimpleNamespace
return SimpleNamespace(title=title, scholar_label=scholar_label) return SimpleNamespace(title=title, scholar_label=scholar_label)
@ -73,7 +71,7 @@ async def test_http_gateway_returns_none_when_disabled(monkeypatch: pytest.Monke
fake_client = FakeClient() fake_client = FakeClient()
object.__setattr__(settings, "arxiv_enabled", False) object.__setattr__(settings, "arxiv_enabled", False)
try: try:
gateway = arxiv_gateway.HttpArxivGateway(client=fake_client) # type: ignore[arg-type] gateway = arxiv_gateway.HttpArxivGateway(client=fake_client)
result = await gateway.discover_arxiv_id_for_publication(item=_item()) result = await gateway.discover_arxiv_id_for_publication(item=_item())
assert result is None assert result is None
assert fake_client.calls == 0 assert fake_client.calls == 0
@ -104,7 +102,7 @@ async def test_http_gateway_uses_client_search_for_discovery() -> None:
) )
fake_client = FakeClient() fake_client = FakeClient()
gateway = arxiv_gateway.HttpArxivGateway(client=fake_client) # type: ignore[arg-type] gateway = arxiv_gateway.HttpArxivGateway(client=fake_client)
result = await gateway.discover_arxiv_id_for_publication( result = await gateway.discover_arxiv_id_for_publication(
item=_item(title="My Paper", scholar_label="Ada Lovelace"), item=_item(title="My Paper", scholar_label="Ada Lovelace"),
request_email="user@example.com", request_email="user@example.com",

View file

@ -16,7 +16,7 @@ from app.settings import settings
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_arxiv_rate_limit_respects_cooldown(db_session: AsyncSession, patch_session_factory) -> None: async def test_arxiv_rate_limit_respects_cooldown(db_session: AsyncSession) -> None:
db_session.add( db_session.add(
ArxivRuntimeState( ArxivRuntimeState(
state_key=ARXIV_RUNTIME_STATE_KEY, state_key=ARXIV_RUNTIME_STATE_KEY,
@ -37,7 +37,7 @@ async def test_arxiv_rate_limit_respects_cooldown(db_session: AsyncSession, patc
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_arxiv_rate_limit_persists_cooldown_after_429(db_session: AsyncSession, patch_session_factory) -> None: async def test_arxiv_rate_limit_persists_cooldown_after_429(db_session: AsyncSession) -> None:
previous_interval = settings.arxiv_min_interval_seconds previous_interval = settings.arxiv_min_interval_seconds
previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds
object.__setattr__(settings, "arxiv_min_interval_seconds", 0.0) object.__setattr__(settings, "arxiv_min_interval_seconds", 0.0)
@ -62,7 +62,7 @@ async def test_arxiv_rate_limit_persists_cooldown_after_429(db_session: AsyncSes
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_arxiv_rate_limit_serializes_concurrent_calls(db_session: AsyncSession, patch_session_factory) -> None: async def test_arxiv_rate_limit_serializes_concurrent_calls(db_session: AsyncSession) -> None:
previous_interval = settings.arxiv_min_interval_seconds previous_interval = settings.arxiv_min_interval_seconds
previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds
object.__setattr__(settings, "arxiv_min_interval_seconds", 0.2) object.__setattr__(settings, "arxiv_min_interval_seconds", 0.2)
@ -88,9 +88,7 @@ async def test_arxiv_rate_limit_serializes_concurrent_calls(db_session: AsyncSes
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_arxiv_rate_limit_logs_request_scheduled_and_completed( async def test_arxiv_rate_limit_logs_request_scheduled_and_completed(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
patch_session_factory,
) -> None: ) -> None:
logged: list[dict[str, object]] = [] logged: list[dict[str, object]] = []
@ -108,16 +106,14 @@ async def test_arxiv_rate_limit_logs_request_scheduled_and_completed(
assert scheduled assert scheduled
assert completed assert completed
assert float(str(scheduled[0]["wait_seconds"])) >= 0.0 assert float(scheduled[0]["wait_seconds"]) >= 0.0
assert int(str(completed[0]["status_code"])) == 200 assert int(completed[0]["status_code"]) == 200
assert completed[0]["source_path"] == "search" assert completed[0]["source_path"] == "search"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_arxiv_rate_limit_logs_cooldown_activation( async def test_arxiv_rate_limit_logs_cooldown_activation(
db_session: AsyncSession,
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
patch_session_factory,
) -> None: ) -> None:
logged_warning: list[dict[str, object]] = [] logged_warning: list[dict[str, object]] = []
previous_interval = settings.arxiv_min_interval_seconds previous_interval = settings.arxiv_min_interval_seconds
@ -142,11 +138,11 @@ async def test_arxiv_rate_limit_logs_cooldown_activation(
cooldown_events = [entry for entry in logged_warning if entry.get("event") == "arxiv.cooldown_activated"] cooldown_events = [entry for entry in logged_warning if entry.get("event") == "arxiv.cooldown_activated"]
assert cooldown_events assert cooldown_events
assert cooldown_events[0]["source_path"] == "lookup_ids" assert cooldown_events[0]["source_path"] == "lookup_ids"
assert float(str(cooldown_events[0]["cooldown_remaining_seconds"])) > 0.0 assert float(cooldown_events[0]["cooldown_remaining_seconds"]) > 0.0
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_get_arxiv_cooldown_status_reads_active_cooldown(db_session: AsyncSession, patch_session_factory) -> None: async def test_get_arxiv_cooldown_status_reads_active_cooldown(db_session: AsyncSession) -> None:
now_utc = datetime(2026, 2, 26, 13, 0, tzinfo=UTC) now_utc = datetime(2026, 2, 26, 13, 0, tzinfo=UTC)
existing = await db_session.get(ArxivRuntimeState, ARXIV_RUNTIME_STATE_KEY) existing = await db_session.get(ArxivRuntimeState, ARXIV_RUNTIME_STATE_KEY)
if existing is None: if existing is None:

View file

@ -1,7 +1,6 @@
from __future__ import annotations from __future__ import annotations
from types import SimpleNamespace from types import SimpleNamespace
from typing import Any, cast
import pytest import pytest
@ -39,8 +38,8 @@ async def test_discover_identifiers_for_enrichment_disables_arxiv_on_rate_limit(
monkeypatch.setattr(runner, "_publish_identifier_update_event", _publish_noop) monkeypatch.setattr(runner, "_publish_identifier_update_event", _publish_noop)
result = await runner._discover_identifiers_for_enrichment( result = await runner._discover_identifiers_for_enrichment(
cast(Any, object()), object(),
publication=cast(Any, publication), publication=publication,
run_id=321, run_id=321,
allow_arxiv_lookup=True, allow_arxiv_lookup=True,
) )

View file

@ -1,6 +1,5 @@
from __future__ import annotations from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock from unittest.mock import MagicMock
from app.services.portability.publication_import import ( from app.services.portability.publication_import import (
@ -10,14 +9,14 @@ from app.services.portability.publication_import import (
) )
def _mock_profile(scholar_id: str = "ABC123DEF456") -> Any: def _mock_profile(scholar_id: str = "ABC123DEF456") -> MagicMock:
profile = MagicMock() profile = MagicMock()
profile.id = 1 profile.id = 1
profile.scholar_id = scholar_id profile.scholar_id = scholar_id
return profile return profile
def _scholar_map(scholar_id: str = "ABC123DEF456") -> dict[str, Any]: def _scholar_map(scholar_id: str = "ABC123DEF456") -> dict[str, MagicMock]:
return {scholar_id: _mock_profile(scholar_id)} return {scholar_id: _mock_profile(scholar_id)}

View file

@ -111,9 +111,8 @@ class TestBuildFingerprint:
assert all(c in "0123456789abcdef" for c in fp) assert all(c in "0123456789abcdef" for c in fp)
def test_deterministic(self) -> None: def test_deterministic(self) -> None:
fp_a = _build_fingerprint(title="Test Title", year=2024, author_text="Smith", venue_text="ICML") kwargs = {"title": "Test Title", "year": 2024, "author_text": "Smith", "venue_text": "ICML"}
fp_b = _build_fingerprint(title="Test Title", year=2024, author_text="Smith", venue_text="ICML") assert _build_fingerprint(**kwargs) == _build_fingerprint(**kwargs)
assert fp_a == fp_b
def test_different_titles_produce_different_fingerprints(self) -> None: def test_different_titles_produce_different_fingerprints(self) -> None:
fp1 = _build_fingerprint(title="Title A", year=2024, author_text=None, venue_text=None) fp1 = _build_fingerprint(title="Title A", year=2024, author_text=None, venue_text=None)

View file

@ -1,9 +1,7 @@
from __future__ import annotations from __future__ import annotations
import contextlib
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from types import SimpleNamespace from types import SimpleNamespace
from typing import Any
import pytest import pytest
@ -29,7 +27,7 @@ def _job(
def _row( def _row(
*, pub_url: str | None = "https://scholar.google.com/citations?view_op=view_citation&citation_for_view=abc:xyz" *, pub_url: str | None = "https://scholar.google.com/citations?view_op=view_citation&citation_for_view=abc:xyz"
) -> Any: ) -> SimpleNamespace:
return SimpleNamespace( return SimpleNamespace(
publication_id=1, publication_id=1,
scholar_profile_id=1, scholar_profile_id=1,
@ -237,12 +235,10 @@ async def test_run_resolution_task_disables_arxiv_for_remaining_batch(
) -> None: ) -> None:
calls: list[tuple[int, bool]] = [] calls: list[tuple[int, bool]] = []
first = _row() first = _row()
second: Any = SimpleNamespace(**{**first.__dict__, "publication_id": 2}) second = SimpleNamespace(**{**first.__dict__, "publication_id": 2})
@contextlib.asynccontextmanager def _raise_session_factory_error():
async def _raise_background_session_error():
raise RuntimeError("skip user settings lookup in test") raise RuntimeError("skip user settings lookup in test")
yield # pragma: no cover
async def _fake_resolve_publication_row( async def _fake_resolve_publication_row(
*, *,
@ -256,7 +252,7 @@ async def test_run_resolution_task_disables_arxiv_for_remaining_batch(
calls.append((int(row.publication_id), bool(allow_arxiv_lookup))) calls.append((int(row.publication_id), bool(allow_arxiv_lookup)))
return row.publication_id == 1 return row.publication_id == 1
monkeypatch.setattr(pdf_queue_resolution, "background_session", _raise_background_session_error) monkeypatch.setattr(pdf_queue_resolution, "get_session_factory", _raise_session_factory_error)
monkeypatch.setattr(pdf_queue_resolution, "_resolve_publication_row", _fake_resolve_publication_row) monkeypatch.setattr(pdf_queue_resolution, "_resolve_publication_row", _fake_resolve_publication_row)
await pdf_queue_resolution._run_resolution_task( await pdf_queue_resolution._run_resolution_task(

View file

@ -2,7 +2,6 @@ from __future__ import annotations
from datetime import UTC, datetime from datetime import UTC, datetime
from types import SimpleNamespace from types import SimpleNamespace
from typing import Any
import pytest import pytest
@ -12,7 +11,7 @@ from app.services.publications import pdf_resolution_pipeline as pipeline
from app.services.unpaywall.application import OaResolutionOutcome from app.services.unpaywall.application import OaResolutionOutcome
def _row(*, display_identifier: DisplayIdentifier | None = None) -> Any: def _row(*, display_identifier: DisplayIdentifier | None = None) -> SimpleNamespace:
return SimpleNamespace( return SimpleNamespace(
publication_id=1, publication_id=1,
scholar_profile_id=1, scholar_profile_id=1,

View file

@ -23,12 +23,6 @@ class StubScholarSource:
index = min(self.calls - 1, len(self._fetch_results) - 1) index = min(self.calls - 1, len(self._fetch_results) - 1)
return self._fetch_results[index] return self._fetch_results[index]
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
raise NotImplementedError
async def fetch_profile_page_html(self, scholar_id: str, *, cstart: int, pagesize: int) -> FetchResult:
raise NotImplementedError
def _ok_author_search_fetch() -> FetchResult: def _ok_author_search_fetch() -> FetchResult:
body = ( body = (

View file

@ -42,34 +42,6 @@ class TestValidateScholarId:
with pytest.raises(ScholarServiceError, match="12"): with pytest.raises(ScholarServiceError, match="12"):
validate_scholar_id("ABCDEF12345!") validate_scholar_id("ABCDEF12345!")
def test_rejects_empty_string(self) -> None:
with pytest.raises(ScholarServiceError, match="12"):
validate_scholar_id("")
def test_rejects_whitespace_only(self) -> None:
with pytest.raises(ScholarServiceError, match="12"):
validate_scholar_id(" ")
def test_rejects_embedded_spaces(self) -> None:
with pytest.raises(ScholarServiceError, match="12"):
validate_scholar_id("ABCDEF 12345")
def test_rejects_tab_characters(self) -> None:
with pytest.raises(ScholarServiceError, match="12"):
validate_scholar_id("ABCDEF\t12345")
def test_rejects_url_as_id(self) -> None:
with pytest.raises(ScholarServiceError, match="12"):
validate_scholar_id("https://scholar.google.com/citations?user=ABCDEF123456")
def test_rejects_newline_in_id(self) -> None:
with pytest.raises(ScholarServiceError, match="12"):
validate_scholar_id("ABCDEF\n12345")
def test_rejects_unicode_characters(self) -> None:
with pytest.raises(ScholarServiceError, match="12"):
validate_scholar_id("ABCDEF12345\u00e9")
class TestNormalizeDisplayName: class TestNormalizeDisplayName:
def test_returns_stripped_name(self) -> None: def test_returns_stripped_name(self) -> None:

View file

@ -1,5 +1,4 @@
from types import SimpleNamespace from types import SimpleNamespace
from typing import Any, cast
import pytest import pytest
@ -43,9 +42,9 @@ async def test_create_hydration_skips_when_global_throttle_active(
) )
result = await scholar_helpers.hydrate_scholar_metadata_if_needed( result = await scholar_helpers.hydrate_scholar_metadata_if_needed(
db_session=cast(Any, None), db_session=None,
profile=profile, profile=profile,
source=cast(Any, object()), source=object(),
user_id=7, user_id=7,
) )
@ -78,9 +77,9 @@ async def test_create_hydration_runs_when_throttle_is_clear(
) )
result = await scholar_helpers.hydrate_scholar_metadata_if_needed( result = await scholar_helpers.hydrate_scholar_metadata_if_needed(
db_session=cast(Any, None), db_session=None,
profile=profile, profile=profile,
source=cast(Any, object()), source=object(),
user_id=8, user_id=8,
) )
@ -110,7 +109,7 @@ async def test_initial_scrape_job_enqueued_on_create(
) )
queued = await scholar_helpers.enqueue_initial_scrape_job_for_scholar( queued = await scholar_helpers.enqueue_initial_scrape_job_for_scholar(
cast(Any, session), session,
profile=profile, profile=profile,
user_id=9, user_id=9,
) )
@ -135,7 +134,7 @@ async def test_initial_scrape_job_not_enqueued_when_disabled(
) )
queued = await scholar_helpers.enqueue_initial_scrape_job_for_scholar( queued = await scholar_helpers.enqueue_initial_scrape_job_for_scholar(
cast(Any, session), session,
profile=profile, profile=profile,
user_id=11, user_id=11,
) )