Compare commits
1 commit
main
...
dependabot
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a901cb0ea3 |
61 changed files with 333 additions and 20706 deletions
|
|
@ -8,23 +8,7 @@
|
|||
"Bash(done)",
|
||||
"Bash(grep -c 'def test_' tests/unit/*.py tests/unit/**/*.py tests/integration/*.py)",
|
||||
"Bash(sort -t: -k2 -rn)",
|
||||
"Bash(wc -l frontend/src/pages/*.vue)",
|
||||
"Bash(uv run ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
|
||||
"Bash(ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
|
||||
"Bash(npm install --package-lock-only)",
|
||||
"Bash(python -m ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
|
||||
"Bash(pip show ruff)",
|
||||
"Bash(pip install ruff)",
|
||||
"Bash(uvx ruff format app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
|
||||
"Bash(git add app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py docs/website/package-lock.json)",
|
||||
"Bash(git push)",
|
||||
"Bash(uvx ruff format --check app/api/routers/runs.py app/services/arxiv/rate_limit.py tests/integration/test_run_lifecycle_consistency.py)",
|
||||
"Bash(test -f docs/website/package-lock.json)",
|
||||
"Bash(gh run list --branch openalex-size-reduction-scholar-hydration --limit 3 --json databaseId,status,conclusion,name,event)",
|
||||
"Bash(gh run list --limit 10 --json databaseId,status,conclusion,name,event,headBranch)",
|
||||
"Bash(gh run view 22554585841 --log-failed)",
|
||||
"Bash(gh run rerun 22554585841 --failed)",
|
||||
"Bash(bash scripts/premerge.sh)"
|
||||
"Bash(wc -l frontend/src/pages/*.vue)"
|
||||
],
|
||||
"additionalDirectories": [
|
||||
"/home/jv/src/personal/playground/scholar_scraper/scholar-scraper/frontend/src"
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ DATABASE_POOL_MODE=auto
|
|||
DATABASE_POOL_SIZE=5
|
||||
DATABASE_POOL_MAX_OVERFLOW=10
|
||||
DATABASE_POOL_TIMEOUT_SECONDS=30
|
||||
DATABASE_RESERVED_API_CONNECTIONS=3
|
||||
|
||||
# ------------------------------
|
||||
# Frontend Dev Overrides
|
||||
|
|
|
|||
2
.github/workflows/docs-pages.yml
vendored
2
.github/workflows/docs-pages.yml
vendored
|
|
@ -38,7 +38,7 @@ jobs:
|
|||
run: npm --prefix docs/website run build
|
||||
|
||||
- name: Upload pages artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
uses: actions/upload-pages-artifact@v4
|
||||
with:
|
||||
path: docs/website/build
|
||||
|
||||
|
|
|
|||
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -18,5 +18,3 @@ frontend/dist/
|
|||
frontend/.vite/
|
||||
docs/website/node_modules/
|
||||
docs/website/build/
|
||||
docs/website/.docusaurus/
|
||||
.claude/
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import logging
|
|||
from fastapi import APIRouter, Depends, Path, Query, Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_api_admin_user, get_api_current_user
|
||||
from app.api.deps import get_api_admin_user
|
||||
from app.api.errors import ApiException
|
||||
from app.api.responses import success_payload
|
||||
from app.api.schemas import (
|
||||
|
|
@ -57,7 +57,7 @@ def _requested_by_value(*, payload, admin_user: User) -> str:
|
|||
return from_payload or admin_user.email
|
||||
|
||||
|
||||
def _serialize_pdf_queue_item(item, *, is_admin: bool = False) -> dict[str, object]:
|
||||
def _serialize_pdf_queue_item(item) -> dict[str, object]:
|
||||
return {
|
||||
"publication_id": item.publication_id,
|
||||
"title": item.title,
|
||||
|
|
@ -69,7 +69,7 @@ def _serialize_pdf_queue_item(item, *, is_admin: bool = False) -> dict[str, obje
|
|||
"last_failure_detail": item.last_failure_detail,
|
||||
"last_source": item.last_source,
|
||||
"requested_by_user_id": item.requested_by_user_id,
|
||||
"requested_by_email": item.requested_by_email if is_admin else None,
|
||||
"requested_by_email": item.requested_by_email,
|
||||
"queued_at": item.queued_at,
|
||||
"last_attempt_at": item.last_attempt_at,
|
||||
"resolved_at": item.resolved_at,
|
||||
|
|
@ -185,7 +185,7 @@ async def get_pdf_queue(
|
|||
offset: int | None = Query(default=None, ge=0),
|
||||
status: str | None = Query(default=None),
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
admin_user: User = Depends(get_api_admin_user),
|
||||
):
|
||||
normalized_status = (status or "").strip().lower() or None
|
||||
resolved_page, resolved_limit, resolved_offset = _resolve_pdf_queue_paging(
|
||||
|
|
@ -204,7 +204,7 @@ async def get_pdf_queue(
|
|||
logger,
|
||||
"info",
|
||||
"api.admin.db.pdf_queue_listed",
|
||||
user_id=int(current_user.id),
|
||||
admin_user_id=int(admin_user.id),
|
||||
page=int(resolved_page),
|
||||
page_size=int(resolved_limit),
|
||||
offset=int(resolved_offset),
|
||||
|
|
@ -215,7 +215,7 @@ async def get_pdf_queue(
|
|||
return success_payload(
|
||||
request,
|
||||
data={
|
||||
"items": [_serialize_pdf_queue_item(item, is_admin=current_user.is_admin) for item in queue_page.items],
|
||||
"items": [_serialize_pdf_queue_item(item) for item in queue_page.items],
|
||||
**_pdf_queue_page_data(
|
||||
total_count=queue_page.total_count,
|
||||
page=resolved_page,
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ def _apply_scholar_http_settings(payload: AdminScholarHttpSettingsUpdateRequest)
|
|||
max_length=_MAX_ACCEPT_LANGUAGE_LENGTH,
|
||||
)
|
||||
cookie = _normalize_header_value(
|
||||
payload.cookie or "",
|
||||
payload.cookie,
|
||||
field_name="cookie",
|
||||
max_length=_MAX_COOKIE_LENGTH,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -56,7 +56,6 @@ def _drop_finished_task(task: asyncio.Task[Any]) -> None:
|
|||
except Exception:
|
||||
structured_log(logger, "exception", "runs.background_task_failed")
|
||||
|
||||
|
||||
router = APIRouter(prefix="/runs", tags=["api-runs"])
|
||||
ACTIVE_RUN_STATUSES = {RunStatus.RUNNING, RunStatus.RESOLVING}
|
||||
|
||||
|
|
@ -221,11 +220,11 @@ def _spawn_background_execution(
|
|||
user_settings: Any,
|
||||
idempotency_key: str | None,
|
||||
) -> None:
|
||||
from app.db.background_session import background_session
|
||||
from app.db.session import get_session_factory
|
||||
|
||||
task = asyncio.create_task(
|
||||
ingest_service.execute_run(
|
||||
session_factory=background_session,
|
||||
session_factory=get_session_factory(),
|
||||
run_id=run.id,
|
||||
user_id=current_user.id,
|
||||
scholars=scholars,
|
||||
|
|
|
|||
|
|
@ -23,9 +23,6 @@ from app.api.schemas import (
|
|||
DataImportEnvelope,
|
||||
DataImportRequest,
|
||||
MessageEnvelope,
|
||||
ScholarBulkCountEnvelope,
|
||||
ScholarBulkIdsRequest,
|
||||
ScholarBulkToggleRequest,
|
||||
ScholarCreateRequest,
|
||||
ScholarEnvelope,
|
||||
ScholarImageUrlUpdateRequest,
|
||||
|
|
@ -45,22 +42,6 @@ logger = logging.getLogger(__name__)
|
|||
router = APIRouter(prefix="/scholars", tags=["api-scholars"])
|
||||
|
||||
|
||||
def _parse_ids_param(ids: str | None) -> list[int] | None:
|
||||
if not ids:
|
||||
return None
|
||||
parts = [p.strip() for p in ids.split(",") if p.strip()]
|
||||
if not parts:
|
||||
return None
|
||||
try:
|
||||
return [int(p) for p in parts]
|
||||
except ValueError as exc:
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_ids_param",
|
||||
message="The 'ids' parameter must be a comma-separated list of integers.",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"",
|
||||
response_model=ScholarsListEnvelope,
|
||||
|
|
@ -88,81 +69,16 @@ async def list_scholars(
|
|||
)
|
||||
async def export_scholars_and_publications(
|
||||
request: Request,
|
||||
ids: str | None = Query(None, description="Comma-separated scholar profile IDs to export"),
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
scholar_profile_ids = _parse_ids_param(ids)
|
||||
data = await import_export_service.export_user_data(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
scholar_profile_ids=scholar_profile_ids,
|
||||
)
|
||||
return success_payload(request, data=data)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/bulk-delete",
|
||||
response_model=ScholarBulkCountEnvelope,
|
||||
)
|
||||
async def bulk_delete_scholars(
|
||||
payload: ScholarBulkIdsRequest,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
try:
|
||||
deleted_count = await scholar_service.bulk_delete_scholars(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
scholar_profile_ids=payload.scholar_profile_ids,
|
||||
upload_dir=settings.scholar_image_upload_dir,
|
||||
)
|
||||
except scholar_service.ScholarServiceError as exc:
|
||||
raise ApiException(
|
||||
status_code=409,
|
||||
code="scholar_bulk_delete_failed",
|
||||
message=str(exc),
|
||||
) from exc
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"scholars.bulk_delete",
|
||||
user_id=current_user.id,
|
||||
requested_ids=payload.scholar_profile_ids,
|
||||
deleted_count=deleted_count,
|
||||
)
|
||||
return success_payload(request, data={"deleted_count": deleted_count, "updated_count": 0})
|
||||
|
||||
|
||||
@router.post(
|
||||
"/bulk-toggle",
|
||||
response_model=ScholarBulkCountEnvelope,
|
||||
)
|
||||
async def bulk_toggle_scholars(
|
||||
payload: ScholarBulkToggleRequest,
|
||||
request: Request,
|
||||
db_session: AsyncSession = Depends(get_db_session),
|
||||
current_user: User = Depends(get_api_current_user),
|
||||
):
|
||||
updated_count = await scholar_service.bulk_toggle_scholars(
|
||||
db_session,
|
||||
user_id=current_user.id,
|
||||
scholar_profile_ids=payload.scholar_profile_ids,
|
||||
is_enabled=payload.is_enabled,
|
||||
)
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"scholars.bulk_toggle",
|
||||
user_id=current_user.id,
|
||||
requested_ids=payload.scholar_profile_ids,
|
||||
is_enabled=payload.is_enabled,
|
||||
updated_count=updated_count,
|
||||
)
|
||||
return success_payload(request, data={"deleted_count": 0, "updated_count": updated_count})
|
||||
|
||||
|
||||
@router.post(
|
||||
"/import",
|
||||
response_model=DataImportEnvelope,
|
||||
|
|
@ -228,11 +144,12 @@ async def create_scholar(
|
|||
message=str(exc),
|
||||
) from exc
|
||||
structured_log(logger, "info", "api.scholars.created", user_id=current_user.id, scholar_profile_id=created.id)
|
||||
await enqueue_initial_scrape_job_for_scholar(
|
||||
did_queue_initial_scrape = await enqueue_initial_scrape_job_for_scholar(
|
||||
db_session,
|
||||
profile=created,
|
||||
user_id=current_user.id,
|
||||
)
|
||||
if not did_queue_initial_scrape:
|
||||
created = await hydrate_scholar_metadata_if_needed(
|
||||
db_session,
|
||||
profile=created,
|
||||
|
|
@ -345,18 +262,11 @@ async def delete_scholar(
|
|||
code="scholar_not_found",
|
||||
message="Scholar not found.",
|
||||
)
|
||||
try:
|
||||
await scholar_service.delete_scholar(
|
||||
db_session,
|
||||
profile=profile,
|
||||
upload_dir=settings.scholar_image_upload_dir,
|
||||
)
|
||||
except scholar_service.ScholarServiceError as exc:
|
||||
raise ApiException(
|
||||
status_code=409,
|
||||
code="scholar_delete_failed",
|
||||
message=str(exc),
|
||||
) from exc
|
||||
structured_log(
|
||||
logger, "info", "api.scholars.deleted", user_id=current_user.id, scholar_profile_id=scholar_profile_id
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from app.api.schemas.common import ApiMeta
|
||||
|
||||
|
|
@ -126,49 +126,13 @@ class DataExportEnvelope(BaseModel):
|
|||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarBulkIdsRequest(BaseModel):
|
||||
scholar_profile_ids: list[int] = Field(..., min_length=1, max_length=500)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarBulkToggleRequest(BaseModel):
|
||||
scholar_profile_ids: list[int] = Field(..., min_length=1, max_length=500)
|
||||
is_enabled: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarBulkCountData(BaseModel):
|
||||
deleted_count: int = 0
|
||||
updated_count: int = 0
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarBulkCountEnvelope(BaseModel):
|
||||
data: ScholarBulkCountData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataImportRequest(BaseModel):
|
||||
schema_version: int | None = None
|
||||
exported_at: str | None = None
|
||||
scholars: list[ScholarExportItemData] = Field(default_factory=list)
|
||||
publications: list[PublicationExportItemData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def unwrap_export_envelope(cls, values: dict) -> dict:
|
||||
"""Accept the full export envelope format (with data/meta wrapper)."""
|
||||
if isinstance(values, dict) and "data" in values and isinstance(values["data"], dict):
|
||||
return values["data"]
|
||||
return values
|
||||
|
||||
|
||||
class DataImportResultData(BaseModel):
|
||||
scholars_created: int
|
||||
|
|
|
|||
|
|
@ -1,57 +0,0 @@
|
|||
"""Semaphore-gated DB sessions for background tasks.
|
||||
|
||||
Background tasks (ingestion, PDF resolution, scheduler) acquire a semaphore
|
||||
permit before checking out a database connection. The semaphore cap is
|
||||
``pool_size + max_overflow - reserved_for_api``, which guarantees that at
|
||||
least *reserved_for_api* connections remain available for API request
|
||||
handlers at all times.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.session import get_session_factory
|
||||
from app.logging_utils import structured_log
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_semaphore: asyncio.Semaphore | None = None
|
||||
|
||||
|
||||
def _build_semaphore() -> asyncio.Semaphore:
|
||||
pool_capacity = max(1, settings.database_pool_size) + max(0, settings.database_pool_max_overflow)
|
||||
reserved = max(0, settings.database_reserved_api_connections)
|
||||
limit = max(1, pool_capacity - reserved)
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"db.background_semaphore_initialized",
|
||||
pool_capacity=pool_capacity,
|
||||
reserved_for_api=reserved,
|
||||
background_limit=limit,
|
||||
)
|
||||
return asyncio.Semaphore(limit)
|
||||
|
||||
|
||||
def get_background_semaphore() -> asyncio.Semaphore:
|
||||
global _semaphore
|
||||
if _semaphore is None:
|
||||
_semaphore = _build_semaphore()
|
||||
return _semaphore
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def background_session() -> AsyncIterator[AsyncSession]:
|
||||
"""Yield a DB session after acquiring the background semaphore."""
|
||||
semaphore = get_background_semaphore()
|
||||
async with semaphore:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
yield session
|
||||
|
|
@ -99,7 +99,9 @@ async def _run_serialized_fetch(
|
|||
"arxiv.request_completed",
|
||||
status_code=int(response.status_code),
|
||||
wait_seconds=wait_seconds,
|
||||
cooldown_remaining_seconds=_cooldown_remaining_seconds(cooldown_until_value, now_utc=datetime.now(UTC)),
|
||||
cooldown_remaining_seconds=_cooldown_remaining_seconds(
|
||||
cooldown_until_value, now_utc=datetime.now(UTC)
|
||||
),
|
||||
source_path=source_path,
|
||||
)
|
||||
return response, hit_rate_limit
|
||||
|
|
|
|||
|
|
@ -37,40 +37,6 @@ def _sanitize_titles(publications: list) -> list[str]:
|
|||
return titles
|
||||
|
||||
|
||||
# OpenAlex (and most HTTP servers) struggle with very long query strings.
|
||||
# Non-ASCII characters expand significantly when percent-encoded in URLs,
|
||||
# so a fixed batch count is unreliable. Instead we cap the combined
|
||||
# byte-length of the pipe-joined title filter value to stay well within
|
||||
# safe URL limits (~4 KiB of filter text ≈ ~6 KiB when percent-encoded).
|
||||
_MAX_TITLE_FILTER_BYTES = 4_000
|
||||
|
||||
|
||||
def _chunk_titles_by_url_length(
|
||||
titles: list[str],
|
||||
max_bytes: int = _MAX_TITLE_FILTER_BYTES,
|
||||
) -> list[list[str]]:
|
||||
"""Split *titles* into chunks whose pipe-joined UTF-8 size stays under *max_bytes*."""
|
||||
chunks: list[list[str]] = []
|
||||
current_chunk: list[str] = []
|
||||
current_bytes = 0
|
||||
|
||||
for title in titles:
|
||||
title_bytes = len(title.encode("utf-8"))
|
||||
# Account for the pipe separator between titles
|
||||
separator = 3 if current_chunk else 0 # len("|".encode) but url-encoded as %7C = 3
|
||||
if current_chunk and current_bytes + separator + title_bytes > max_bytes:
|
||||
chunks.append(current_chunk)
|
||||
current_chunk = [title]
|
||||
current_bytes = title_bytes
|
||||
else:
|
||||
current_chunk.append(title)
|
||||
current_bytes += separator + title_bytes
|
||||
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
return chunks
|
||||
|
||||
|
||||
class EnrichmentRunner:
|
||||
"""Post-run OpenAlex enrichment logic.
|
||||
|
||||
|
|
@ -190,34 +156,21 @@ class EnrichmentRunner:
|
|||
|
||||
resolved_key = openalex_api_key or settings.openalex_api_key
|
||||
client = OpenAlexClient(api_key=resolved_key, mailto=settings.crossref_api_mailto)
|
||||
batch_size = 25
|
||||
now = datetime.now(UTC)
|
||||
arxiv_lookup_allowed = True
|
||||
|
||||
all_titles = _sanitize_titles(publications)
|
||||
title_chunks = _chunk_titles_by_url_length(all_titles)
|
||||
|
||||
# Build a mapping from sanitized title → publications for batch association
|
||||
title_to_pubs: dict[str, list[Publication]] = {}
|
||||
for p in publications:
|
||||
raw = getattr(p, "title_raw", None) or getattr(p, "title", None)
|
||||
if not raw or not raw.strip():
|
||||
continue
|
||||
safe = " ".join(re.sub(r"[^\w\s]", " ", raw).split())
|
||||
if safe:
|
||||
title_to_pubs.setdefault(safe, []).append(p)
|
||||
|
||||
for title_chunk in title_chunks:
|
||||
for i in range(0, len(publications), batch_size):
|
||||
if await self.run_is_canceled(db_session, run_id=run_id):
|
||||
structured_log(logger, "info", "ingestion.enrichment_aborted", run_id=run_id)
|
||||
return
|
||||
batch = []
|
||||
for t in title_chunk:
|
||||
batch.extend(title_to_pubs.get(t, []))
|
||||
if not batch:
|
||||
batch = publications[i : i + batch_size]
|
||||
titles = _sanitize_titles(batch)
|
||||
if not titles:
|
||||
continue
|
||||
try:
|
||||
openalex_works = await client.get_works_by_filter(
|
||||
{"title.search": "|".join(title_chunk)}, limit=len(title_chunk) * 3
|
||||
{"title.search": "|".join(titles)}, limit=batch_size * 3
|
||||
)
|
||||
except OpenAlexBudgetExhaustedError:
|
||||
structured_log(logger, "warning", "ingestion.openalex_budget_exhausted", run_id=run_id)
|
||||
|
|
@ -336,37 +289,19 @@ class EnrichmentRunner:
|
|||
|
||||
client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto)
|
||||
|
||||
all_titles = _sanitize_titles(publications)
|
||||
title_chunks = _chunk_titles_by_url_length(all_titles)
|
||||
|
||||
# Build title → publication mapping for batch association
|
||||
title_to_pubs: dict[str, list[PublicationCandidate]] = {}
|
||||
for p in publications:
|
||||
raw = getattr(p, "title", None)
|
||||
if not raw or not raw.strip():
|
||||
continue
|
||||
safe = " ".join(re.sub(r"[^\w\s]", " ", raw).split())
|
||||
if safe:
|
||||
title_to_pubs.setdefault(safe, []).append(p)
|
||||
|
||||
# Collect publications that had no sanitizable title — pass through as-is
|
||||
pubs_with_titles = set()
|
||||
for titles in title_to_pubs.values():
|
||||
for p in titles:
|
||||
pubs_with_titles.add(id(p))
|
||||
|
||||
batch_size = 25
|
||||
enriched: list[PublicationCandidate] = []
|
||||
|
||||
for title_chunk in title_chunks:
|
||||
batch = []
|
||||
for t in title_chunk:
|
||||
batch.extend(title_to_pubs.get(t, []))
|
||||
if not batch:
|
||||
for i in range(0, len(publications), batch_size):
|
||||
batch = publications[i : i + batch_size]
|
||||
titles = _sanitize_titles(batch)
|
||||
if not titles:
|
||||
enriched.extend(batch)
|
||||
continue
|
||||
|
||||
try:
|
||||
openalex_works = await client.get_works_by_filter(
|
||||
{"title.search": "|".join(title_chunk)}, limit=len(title_chunk) * 3
|
||||
{"title.search": "|".join(titles)}, limit=batch_size * 3
|
||||
)
|
||||
except Exception as e:
|
||||
structured_log(
|
||||
|
|
@ -400,10 +335,4 @@ class EnrichmentRunner:
|
|||
enriched.append(new_p)
|
||||
else:
|
||||
enriched.append(p)
|
||||
|
||||
# Append publications that had no sanitizable title (pass-through)
|
||||
for p in publications:
|
||||
if id(p) not in pubs_with_titles:
|
||||
enriched.append(p)
|
||||
|
||||
return enriched
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ from datetime import UTC, datetime
|
|||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db.background_session import background_session
|
||||
from app.db.models import QueueItemStatus, RunTriggerType, ScholarProfile, UserSetting
|
||||
from app.db.session import get_session_factory
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.ingestion import queue as queue_service
|
||||
from app.services.ingestion.application import (
|
||||
|
|
@ -57,7 +57,8 @@ class QueueJobRunner:
|
|||
|
||||
async def drain_continuation_queue(self) -> None:
|
||||
now = datetime.now(UTC)
|
||||
async with background_session() as session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
jobs = await queue_service.list_due_jobs(
|
||||
session,
|
||||
now=now,
|
||||
|
|
@ -72,7 +73,8 @@ class QueueJobRunner:
|
|||
) -> bool:
|
||||
if job.attempt_count < self._continuation_max_attempts:
|
||||
return False
|
||||
async with background_session() as session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
dropped = await queue_service.mark_dropped(
|
||||
session,
|
||||
job_id=job.id,
|
||||
|
|
@ -96,7 +98,8 @@ class QueueJobRunner:
|
|||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
async with background_session() as session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
queue_item = await queue_service.mark_retrying(session, job_id=job.id)
|
||||
await session.commit()
|
||||
if queue_item is None:
|
||||
|
|
@ -107,7 +110,8 @@ class QueueJobRunner:
|
|||
self,
|
||||
job: queue_service.ContinuationQueueJob,
|
||||
) -> bool:
|
||||
async with background_session() as session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
scholar_result = await session.execute(
|
||||
select(ScholarProfile.id).where(
|
||||
ScholarProfile.user_id == job.user_id,
|
||||
|
|
@ -136,7 +140,8 @@ class QueueJobRunner:
|
|||
return False
|
||||
|
||||
async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
|
||||
async with background_session() as recovery_session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
|
|
@ -162,7 +167,8 @@ class QueueJobRunner:
|
|||
self._tick_seconds,
|
||||
int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
|
||||
)
|
||||
async with background_session() as recovery_session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
await queue_service.reschedule_job(
|
||||
recovery_session,
|
||||
job_id=job.id,
|
||||
|
|
@ -187,7 +193,8 @@ class QueueJobRunner:
|
|||
*,
|
||||
exc: Exception,
|
||||
) -> None:
|
||||
async with background_session() as recovery_session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as recovery_session:
|
||||
queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
|
||||
if queue_item is None:
|
||||
await recovery_session.commit()
|
||||
|
|
@ -231,7 +238,8 @@ class QueueJobRunner:
|
|||
)
|
||||
|
||||
async def _load_request_delay_for_user(self, user_id: int, *, floor: int) -> int:
|
||||
async with background_session() as session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
|
||||
)
|
||||
|
|
@ -245,7 +253,8 @@ class QueueJobRunner:
|
|||
request_delay_floor: int,
|
||||
):
|
||||
request_delay_seconds = await self._load_request_delay_for_user(job.user_id, floor=request_delay_floor)
|
||||
async with background_session() as session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
ingestion = ScholarIngestionService(source=self._source)
|
||||
try:
|
||||
return await ingestion.run_for_user(
|
||||
|
|
@ -357,7 +366,8 @@ class QueueJobRunner:
|
|||
)
|
||||
|
||||
async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
|
||||
async with background_session() as session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
if int(run_summary.failed_count) <= 0:
|
||||
await self._finalize_successful_queue_job(session, job, run_summary)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -8,13 +8,13 @@ from typing import Any
|
|||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db.background_session import background_session
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
RunTriggerType,
|
||||
User,
|
||||
UserSetting,
|
||||
)
|
||||
from app.db.session import get_session_factory
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.ingestion.application import (
|
||||
RunAlreadyInProgressError,
|
||||
|
|
@ -148,7 +148,8 @@ class SchedulerService:
|
|||
await self._run_candidate(candidate)
|
||||
|
||||
async def _load_candidate_rows(self) -> list[Any]:
|
||||
async with background_session() as session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(
|
||||
UserSetting.user_id,
|
||||
|
|
@ -203,7 +204,8 @@ class SchedulerService:
|
|||
return candidates
|
||||
|
||||
async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
|
||||
async with background_session() as session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
result = await session.execute(
|
||||
select(CrawlRun.start_dt)
|
||||
.where(
|
||||
|
|
@ -225,7 +227,8 @@ class SchedulerService:
|
|||
*,
|
||||
candidate: _AutoRunCandidate,
|
||||
):
|
||||
async with background_session() as session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
ingestion = ScholarIngestionService(source=self._source)
|
||||
try:
|
||||
return await ingestion.run_for_user(
|
||||
|
|
@ -286,12 +289,9 @@ class SchedulerService:
|
|||
|
||||
async def _drain_pdf_queue(self) -> None:
|
||||
from app.services.publications.pdf_queue import drain_ready_jobs
|
||||
from app.services.publications.pdf_queue_resolution import is_budget_cooldown_active
|
||||
|
||||
if is_budget_cooldown_active():
|
||||
return
|
||||
|
||||
async with background_session() as session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as session:
|
||||
try:
|
||||
processed = await drain_ready_jobs(
|
||||
session,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
from collections.abc import Callable, Coroutine
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -275,7 +274,6 @@ async def _run_first_pass(
|
|||
request_delay_seconds: int,
|
||||
queue_delay_seconds: int,
|
||||
progress: RunProgress,
|
||||
on_progress: Callable[[int, int], Coroutine[Any, Any, None]] | None = None,
|
||||
) -> dict[int, int]:
|
||||
first_pass_cstarts: dict[int, int] = {}
|
||||
for index, scholar in enumerate(scholars):
|
||||
|
|
@ -312,8 +310,6 @@ async def _run_first_pass(
|
|||
scholars_remaining=len(scholars) - index - 1,
|
||||
)
|
||||
return first_pass_cstarts
|
||||
if on_progress is not None:
|
||||
await on_progress(1, 0)
|
||||
resume_cstart = outcome.result_entry.get("continuation_cstart")
|
||||
if resume_cstart is not None and int(resume_cstart) > start_cstart:
|
||||
first_pass_cstarts[int(scholar.id)] = int(resume_cstart)
|
||||
|
|
@ -334,7 +330,6 @@ async def _run_depth_pass(
|
|||
auto_queue_continuations: bool,
|
||||
queue_delay_seconds: int,
|
||||
progress: RunProgress,
|
||||
on_progress: Callable[[int, int], Coroutine[Any, Any, None]] | None = None,
|
||||
) -> None:
|
||||
for index, scholar in enumerate(scholars):
|
||||
resume_cstart = first_pass_cstarts.get(int(scholar.id))
|
||||
|
|
@ -372,8 +367,6 @@ async def _run_depth_pass(
|
|||
scholars_remaining=len(scholars) - index - 1,
|
||||
)
|
||||
break
|
||||
if on_progress is not None:
|
||||
await on_progress(0, 1)
|
||||
|
||||
|
||||
async def run_scholar_iteration(
|
||||
|
|
@ -394,8 +387,6 @@ async def run_scholar_iteration(
|
|||
auto_queue_continuations: bool,
|
||||
queue_delay_seconds: int,
|
||||
) -> RunProgress:
|
||||
from app.services.runs.events import run_events
|
||||
|
||||
progress = RunProgress()
|
||||
scholar_kwargs: dict[str, Any] = {
|
||||
"request_delay_seconds": request_delay_seconds,
|
||||
|
|
@ -405,22 +396,6 @@ async def run_scholar_iteration(
|
|||
"rate_limit_backoff_seconds": rate_limit_backoff_seconds,
|
||||
"page_size": page_size,
|
||||
}
|
||||
|
||||
visited = 0
|
||||
finished = 0
|
||||
total = len(scholars)
|
||||
|
||||
async def _emit(v: int = 0, f: int = 0) -> None:
|
||||
nonlocal visited, finished
|
||||
visited += v
|
||||
finished += f
|
||||
await run_events.publish(
|
||||
run.id,
|
||||
"scholar_progress",
|
||||
{"visited": visited, "finished": finished, "total": total},
|
||||
)
|
||||
|
||||
await _emit()
|
||||
first_pass_cstarts = await _run_first_pass(
|
||||
db_session,
|
||||
scholars=scholars,
|
||||
|
|
@ -432,12 +407,8 @@ async def run_scholar_iteration(
|
|||
request_delay_seconds=request_delay_seconds,
|
||||
queue_delay_seconds=queue_delay_seconds,
|
||||
progress=progress,
|
||||
on_progress=_emit,
|
||||
)
|
||||
remaining_max = max(max_pages_per_scholar - 1, 0)
|
||||
scholars_finished_in_first_pass = len(scholars) - len(first_pass_cstarts)
|
||||
if scholars_finished_in_first_pass > 0:
|
||||
await _emit(f=scholars_finished_in_first_pass)
|
||||
if remaining_max <= 0:
|
||||
return progress
|
||||
await _run_depth_pass(
|
||||
|
|
@ -453,6 +424,5 @@ async def run_scholar_iteration(
|
|||
auto_queue_continuations=auto_queue_continuations,
|
||||
queue_delay_seconds=queue_delay_seconds,
|
||||
progress=progress,
|
||||
on_progress=_emit,
|
||||
)
|
||||
return progress
|
||||
|
|
|
|||
|
|
@ -56,14 +56,11 @@ async def export_user_data(
|
|||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_ids: list[int] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
scholar_query = select(ScholarProfile).where(ScholarProfile.user_id == user_id)
|
||||
if scholar_profile_ids:
|
||||
scholar_query = scholar_query.where(ScholarProfile.id.in_(scholar_profile_ids))
|
||||
scholars_result = await db_session.execute(scholar_query.order_by(ScholarProfile.id.asc()))
|
||||
|
||||
pub_query = (
|
||||
scholars_result = await db_session.execute(
|
||||
select(ScholarProfile).where(ScholarProfile.user_id == user_id).order_by(ScholarProfile.id.asc())
|
||||
)
|
||||
publication_result = await db_session.execute(
|
||||
select(
|
||||
ScholarProfile.scholar_id,
|
||||
Publication.cluster_id,
|
||||
|
|
@ -80,13 +77,8 @@ async def export_user_data(
|
|||
.join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id)
|
||||
.join(Publication, Publication.id == ScholarPublication.publication_id)
|
||||
.where(ScholarProfile.user_id == user_id)
|
||||
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
|
||||
)
|
||||
if scholar_profile_ids:
|
||||
pub_query = pub_query.where(ScholarProfile.id.in_(scholar_profile_ids))
|
||||
publication_result = await db_session.execute(
|
||||
pub_query.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
|
||||
)
|
||||
|
||||
scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()]
|
||||
publications = [_serialize_export_publication(row) for row in publication_result.all()]
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -2,10 +2,9 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from app.db.background_session import background_session
|
||||
from app.db.models import Publication, PublicationPdfJob
|
||||
from app.db.session import get_session_factory
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.publication_identifiers import application as identifier_service
|
||||
from app.services.publications.pdf_queue_common import (
|
||||
|
|
@ -30,20 +29,8 @@ PDF_EVENT_ATTEMPT_STARTED = "attempt_started"
|
|||
PDF_EVENT_RESOLVED = "resolved"
|
||||
PDF_EVENT_FAILED = "failed"
|
||||
|
||||
_BUDGET_COOLDOWN_MINUTES = 15
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_scheduled_tasks: set[asyncio.Task[None]] = set()
|
||||
_budget_cooldown_until: datetime | None = None
|
||||
|
||||
|
||||
def is_budget_cooldown_active() -> bool:
|
||||
return _budget_cooldown_until is not None and datetime.now(UTC) < _budget_cooldown_until
|
||||
|
||||
|
||||
def _enter_budget_cooldown() -> None:
|
||||
global _budget_cooldown_until
|
||||
_budget_cooldown_until = datetime.now(UTC) + timedelta(minutes=_BUDGET_COOLDOWN_MINUTES)
|
||||
|
||||
|
||||
async def _mark_attempt_started(
|
||||
|
|
@ -51,7 +38,8 @@ async def _mark_attempt_started(
|
|||
publication_id: int,
|
||||
user_id: int,
|
||||
) -> None:
|
||||
async with background_session() as db_session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as db_session:
|
||||
job = await db_session.get(PublicationPdfJob, publication_id)
|
||||
if job is None:
|
||||
job = queued_job(publication_id=publication_id, user_id=user_id)
|
||||
|
|
@ -137,7 +125,8 @@ async def _persist_outcome(
|
|||
user_id: int,
|
||||
outcome: OaResolutionOutcome,
|
||||
) -> None:
|
||||
async with background_session() as db_session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as db_session:
|
||||
publication = await db_session.get(Publication, publication_id)
|
||||
job = await db_session.get(PublicationPdfJob, publication_id)
|
||||
if publication is None or job is None:
|
||||
|
|
@ -218,7 +207,8 @@ async def _run_resolution_task(
|
|||
|
||||
openalex_api_key: str | None = None
|
||||
try:
|
||||
async with background_session() as key_session:
|
||||
session_factory = get_session_factory()
|
||||
async with session_factory() as key_session:
|
||||
user_settings = await user_settings_service.get_or_create_settings(key_session, user_id=user_id)
|
||||
openalex_api_key = getattr(user_settings, "openalex_api_key", None) or settings.openalex_api_key
|
||||
except Exception:
|
||||
|
|
@ -243,13 +233,11 @@ async def _run_resolution_task(
|
|||
detail="arXiv temporarily disabled for remaining batch after rate limit",
|
||||
)
|
||||
except OpenAlexBudgetExhaustedError:
|
||||
_enter_budget_cooldown()
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"pdf_queue.budget_exhausted",
|
||||
detail="Stopping PDF resolution batch — OpenAlex daily budget exhausted",
|
||||
cooldown_minutes=_BUDGET_COOLDOWN_MINUTES,
|
||||
)
|
||||
break
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
|
@ -11,11 +10,7 @@ from app.db.models import ScholarProfile
|
|||
from app.services.scholar.parser import ScholarParserError, parse_profile_page
|
||||
from app.services.scholar.source import ScholarSource
|
||||
from app.services.scholars.author_search import search_author_candidates
|
||||
from app.services.scholars.constants import (
|
||||
ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES,
|
||||
SEARCH_COOLDOWN_REASON,
|
||||
SEARCH_DISABLED_REASON,
|
||||
)
|
||||
from app.services.scholars.constants import ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES
|
||||
from app.services.scholars.exceptions import ScholarServiceError
|
||||
from app.services.scholars.uploads import (
|
||||
_ensure_upload_root,
|
||||
|
|
@ -28,66 +23,8 @@ from app.services.scholars.validators import (
|
|||
validate_scholar_id,
|
||||
)
|
||||
|
||||
|
||||
async def bulk_delete_scholars(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_ids: list[int],
|
||||
upload_dir: str | None = None,
|
||||
) -> int:
|
||||
from sqlalchemy import select
|
||||
|
||||
result = await db_session.execute(
|
||||
select(ScholarProfile).where(
|
||||
ScholarProfile.id.in_(scholar_profile_ids),
|
||||
ScholarProfile.user_id == user_id,
|
||||
)
|
||||
)
|
||||
profiles = list(result.scalars().all())
|
||||
if not profiles:
|
||||
return 0
|
||||
if upload_dir:
|
||||
upload_root = _ensure_upload_root(upload_dir, create=True)
|
||||
for profile in profiles:
|
||||
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
|
||||
for profile in profiles:
|
||||
await db_session.delete(profile)
|
||||
try:
|
||||
await db_session.commit()
|
||||
except IntegrityError as exc:
|
||||
await db_session.rollback()
|
||||
raise ScholarServiceError("Unable to bulk-delete scholars due to a database constraint.") from exc
|
||||
return len(profiles)
|
||||
|
||||
|
||||
async def bulk_toggle_scholars(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar_profile_ids: list[int],
|
||||
is_enabled: bool,
|
||||
) -> int:
|
||||
from sqlalchemy import CursorResult, update
|
||||
|
||||
cursor: CursorResult[Any] = await db_session.execute( # type: ignore[assignment]
|
||||
update(ScholarProfile)
|
||||
.where(
|
||||
ScholarProfile.id.in_(scholar_profile_ids),
|
||||
ScholarProfile.user_id == user_id,
|
||||
)
|
||||
.values(is_enabled=is_enabled)
|
||||
)
|
||||
await db_session.commit()
|
||||
return int(cursor.rowcount or 0)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SEARCH_COOLDOWN_REASON",
|
||||
"SEARCH_DISABLED_REASON",
|
||||
"ScholarServiceError",
|
||||
"bulk_delete_scholars",
|
||||
"bulk_toggle_scholars",
|
||||
"clear_profile_image_customization",
|
||||
"create_scholar_for_user",
|
||||
"delete_scholar",
|
||||
|
|
@ -182,11 +119,7 @@ async def delete_scholar(
|
|||
_safe_remove_upload(upload_root, profile.profile_image_upload_path)
|
||||
|
||||
await db_session.delete(profile)
|
||||
try:
|
||||
await db_session.commit()
|
||||
except IntegrityError as exc:
|
||||
await db_session.rollback()
|
||||
raise ScholarServiceError("Unable to delete scholar due to a database constraint.") from exc
|
||||
|
||||
|
||||
async def hydrate_profile_metadata(
|
||||
|
|
|
|||
|
|
@ -275,8 +275,6 @@ class Settings:
|
|||
crossref_max_lookups_per_request: int = _env_int("CROSSREF_MAX_LOOKUPS_PER_REQUEST", 8)
|
||||
|
||||
openalex_api_key: str | None = os.getenv("OPENALEX_API_KEY")
|
||||
database_reserved_api_connections: int = _env_int("DATABASE_RESERVED_API_CONNECTIONS", 3)
|
||||
|
||||
crossref_api_token: str | None = os.getenv("CROSSREF_API_TOKEN")
|
||||
crossref_api_mailto: str | None = os.getenv("CROSSREF_API_MAILTO")
|
||||
|
||||
|
|
|
|||
18461
docs/website/package-lock.json
generated
18461
docs/website/package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,61 +0,0 @@
|
|||
// @vitest-environment happy-dom
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { mount } from "@vue/test-utils";
|
||||
import ScrapeSafetyBadge from "./ScrapeSafetyBadge.vue";
|
||||
import { createDefaultSafetyState, type ScrapeSafetyState } from "@/features/safety";
|
||||
|
||||
function buildState(overrides: Partial<ScrapeSafetyState> = {}): ScrapeSafetyState {
|
||||
return { ...createDefaultSafetyState(), ...overrides };
|
||||
}
|
||||
|
||||
describe("ScrapeSafetyBadge", () => {
|
||||
it("shows ready tooltip when cooldown is inactive", () => {
|
||||
const wrapper = mount(ScrapeSafetyBadge, {
|
||||
props: { state: buildState({ cooldown_active: false }) },
|
||||
});
|
||||
expect(wrapper.text()).toContain("Safety ready");
|
||||
const hint = wrapper.findComponent({ name: "AppHelpHint" });
|
||||
expect(hint.exists()).toBe(true);
|
||||
expect(hint.props("text")).toContain("No active cooldown");
|
||||
});
|
||||
|
||||
it("shows cooldown tooltip with reason and action when active", () => {
|
||||
const wrapper = mount(ScrapeSafetyBadge, {
|
||||
props: {
|
||||
state: buildState({
|
||||
cooldown_active: true,
|
||||
cooldown_reason: "blocked_failure_threshold_exceeded",
|
||||
cooldown_reason_label: "Too many blocked requests",
|
||||
recommended_action: "Wait for cooldown to expire",
|
||||
cooldown_remaining_seconds: 120,
|
||||
}),
|
||||
},
|
||||
});
|
||||
expect(wrapper.text()).toContain("Safety cooldown");
|
||||
const hint = wrapper.findComponent({ name: "AppHelpHint" });
|
||||
expect(hint.exists()).toBe(true);
|
||||
const text = hint.props("text") as string;
|
||||
expect(text).toContain("Google Scholar rate-limits");
|
||||
expect(text).toContain("Too many blocked requests");
|
||||
expect(text).toContain("Wait for cooldown to expire");
|
||||
});
|
||||
|
||||
it("shows cooldown tooltip without optional fields", () => {
|
||||
const wrapper = mount(ScrapeSafetyBadge, {
|
||||
props: {
|
||||
state: buildState({
|
||||
cooldown_active: true,
|
||||
cooldown_reason: "some_reason",
|
||||
cooldown_reason_label: null,
|
||||
recommended_action: null,
|
||||
cooldown_remaining_seconds: 60,
|
||||
}),
|
||||
},
|
||||
});
|
||||
const hint = wrapper.findComponent({ name: "AppHelpHint" });
|
||||
const text = hint.props("text") as string;
|
||||
expect(text).toContain("Google Scholar rate-limits");
|
||||
expect(text).not.toContain("Why:");
|
||||
expect(text).not.toContain("Action:");
|
||||
});
|
||||
});
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
<script setup lang="ts">
|
||||
import { computed } from "vue";
|
||||
|
||||
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
||||
import { type ScrapeSafetyState } from "@/features/safety";
|
||||
|
||||
const props = defineProps<{
|
||||
|
|
@ -19,30 +18,10 @@ const toneClass = computed(() => {
|
|||
}
|
||||
return "border-state-warning-border bg-state-warning-bg text-state-warning-text";
|
||||
});
|
||||
|
||||
const READY_TOOLTIP = "No active cooldown. Scraping can proceed normally.";
|
||||
const RATE_LIMIT_INTRO =
|
||||
"Google Scholar rate-limits automated requests. The cooldown pauses scraping to avoid your IP being blocked.";
|
||||
|
||||
const tooltipText = computed(() => {
|
||||
if (!props.state.cooldown_active) return READY_TOOLTIP;
|
||||
|
||||
const parts = [RATE_LIMIT_INTRO];
|
||||
if (props.state.cooldown_reason_label) {
|
||||
parts.push(`Why: ${props.state.cooldown_reason_label}`);
|
||||
}
|
||||
if (props.state.recommended_action) {
|
||||
parts.push(`Action: ${props.state.recommended_action}`);
|
||||
}
|
||||
return parts.join(" \u2014 ");
|
||||
});
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<span class="inline-flex items-center gap-1">
|
||||
<span class="inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-semibold" :class="toneClass">
|
||||
{{ label }}
|
||||
</span>
|
||||
<AppHelpHint :text="tooltipText" />
|
||||
</span>
|
||||
</template>
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
import { computed, onBeforeUnmount, onMounted, ref } from "vue";
|
||||
|
||||
import AppAlert from "@/components/ui/AppAlert.vue";
|
||||
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
||||
import {
|
||||
formatCooldownCountdown,
|
||||
type ScrapeSafetyState,
|
||||
|
|
@ -80,10 +79,6 @@ const actionText = computed(() => {
|
|||
return props.safetyState.recommended_action;
|
||||
});
|
||||
|
||||
const bannerHintText = computed(() => {
|
||||
return "Google Scholar rate-limits automated requests. The cooldown pauses scraping to avoid your IP being blocked.";
|
||||
});
|
||||
|
||||
onMounted(() => {
|
||||
timer = setInterval(() => {
|
||||
now.value = Date.now();
|
||||
|
|
@ -100,12 +95,7 @@ onBeforeUnmount(() => {
|
|||
|
||||
<template>
|
||||
<AppAlert v-if="isVisible" :tone="tone">
|
||||
<template #title>
|
||||
<span class="inline-flex items-center gap-1">
|
||||
{{ title }}
|
||||
<AppHelpHint :text="bannerHintText" />
|
||||
</span>
|
||||
</template>
|
||||
<template #title>{{ title }}</template>
|
||||
<p>{{ detailText }}</p>
|
||||
<p v-if="actionText" class="text-secondary">{{ actionText }}</p>
|
||||
</AppAlert>
|
||||
|
|
|
|||
|
|
@ -1,28 +0,0 @@
|
|||
<script setup lang="ts">
|
||||
import AppButton from "@/components/ui/AppButton.vue";
|
||||
import AppModal from "@/components/ui/AppModal.vue";
|
||||
|
||||
withDefaults(
|
||||
defineProps<{
|
||||
open: boolean;
|
||||
title: string;
|
||||
message: string;
|
||||
confirmLabel?: string;
|
||||
cancelLabel?: string;
|
||||
variant?: "danger" | "default";
|
||||
}>(),
|
||||
{ confirmLabel: "Confirm", cancelLabel: "Cancel", variant: "default" },
|
||||
);
|
||||
|
||||
const emit = defineEmits<{ confirm: []; cancel: [] }>();
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<AppModal :open="open" :title="title" @close="emit('cancel')">
|
||||
<p class="mb-6 text-sm text-secondary">{{ message }}</p>
|
||||
<div class="flex justify-end gap-2">
|
||||
<AppButton variant="secondary" @click="emit('cancel')">{{ cancelLabel }}</AppButton>
|
||||
<AppButton :variant="variant === 'danger' ? 'danger' : 'primary'" @click="emit('confirm')">{{ confirmLabel }}</AppButton>
|
||||
</div>
|
||||
</AppModal>
|
||||
</template>
|
||||
|
|
@ -277,16 +277,12 @@ export function usePublicationData() {
|
|||
|
||||
// --- Run-triggered refresh watcher ---
|
||||
|
||||
let previousRunStatusKey: string | null = null;
|
||||
|
||||
watch(
|
||||
() => runStatus.latestRun,
|
||||
async (nextRun) => {
|
||||
const nextStatus = nextRun ? `${nextRun.id}:${nextRun.status}` : null;
|
||||
if (nextStatus === previousRunStatusKey) return;
|
||||
previousRunStatusKey = nextStatus;
|
||||
const isActive = nextRun && (nextRun.status === "running" || nextRun.status === "resolving");
|
||||
if (!isActive) return;
|
||||
async (nextRun, previousRun) => {
|
||||
const nextRunId = nextRun && (nextRun.status === "running" || nextRun.status === "resolving") ? nextRun.id : null;
|
||||
const previousRunId = previousRun && (previousRun.status === "running" || previousRun.status === "resolving") ? previousRun.id : null;
|
||||
if (nextRunId === null || nextRunId === previousRunId) return;
|
||||
resetPageAndSnapshot();
|
||||
await loadPublications();
|
||||
},
|
||||
|
|
|
|||
|
|
@ -2,144 +2,10 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { mount } from "@vue/test-utils";
|
||||
import ScholarBatchAdd from "./ScholarBatchAdd.vue";
|
||||
import {
|
||||
parseScholarIds,
|
||||
parseScholarTokens,
|
||||
extractScholarIdFromUrl,
|
||||
validateTokenAsId,
|
||||
} from "./scholar-batch-parsing";
|
||||
|
||||
const defaultProps = { saving: false, loading: false };
|
||||
|
||||
describe("parseScholarIds (unit)", () => {
|
||||
it("parses a single bare ID", () => {
|
||||
expect(parseScholarIds("A-UbBTPM15wL")).toEqual(["A-UbBTPM15wL"]);
|
||||
});
|
||||
|
||||
it("parses multiple IDs separated by commas", () => {
|
||||
expect(parseScholarIds("A-UbBTPM15wL, B-UbBTPM15wL")).toEqual(["A-UbBTPM15wL", "B-UbBTPM15wL"]);
|
||||
});
|
||||
|
||||
it("deduplicates IDs", () => {
|
||||
expect(parseScholarIds("A-UbBTPM15wL\nA-UbBTPM15wL")).toEqual(["A-UbBTPM15wL"]);
|
||||
});
|
||||
|
||||
it("extracts ID from standard Google Scholar URL", () => {
|
||||
expect(parseScholarIds("https://scholar.google.com/citations?hl=en&user=A-UbBTPM15wL")).toEqual(["A-UbBTPM15wL"]);
|
||||
});
|
||||
|
||||
it("extracts ID from URL with trailing slash", () => {
|
||||
expect(parseScholarIds("https://scholar.google.com/citations?user=A-UbBTPM15wL/")).toEqual(["A-UbBTPM15wL"]);
|
||||
});
|
||||
|
||||
it("extracts ID from URL with fragment", () => {
|
||||
expect(parseScholarIds("https://scholar.google.com/citations?user=A-UbBTPM15wL#section")).toEqual(["A-UbBTPM15wL"]);
|
||||
});
|
||||
|
||||
it("extracts ID from URL with extra query params", () => {
|
||||
expect(parseScholarIds("https://scholar.google.com/citations?hl=en&user=A-UbBTPM15wL&view_op=list_works&sortby=pubdate")).toEqual(["A-UbBTPM15wL"]);
|
||||
});
|
||||
|
||||
it("handles URL-encoded characters in non-ID parts", () => {
|
||||
expect(parseScholarIds("https://scholar.google.com/citations?hl=en&user=A-UbBTPM15wL&label=%E4%B8%AD%E6%96%87")).toEqual(["A-UbBTPM15wL"]);
|
||||
});
|
||||
|
||||
it("rejects malformed IDs (too short)", () => {
|
||||
expect(parseScholarIds("ABC123")).toEqual([]);
|
||||
});
|
||||
|
||||
it("rejects malformed IDs (too long)", () => {
|
||||
expect(parseScholarIds("ABCDEF1234567")).toEqual([]);
|
||||
});
|
||||
|
||||
it("rejects IDs with special characters", () => {
|
||||
expect(parseScholarIds("ABCDEF12345!")).toEqual([]);
|
||||
});
|
||||
|
||||
it("rejects IDs with embedded whitespace", () => {
|
||||
expect(parseScholarIds("ABC DEF12345")).toEqual([]);
|
||||
});
|
||||
|
||||
it("handles mixed valid and invalid tokens", () => {
|
||||
expect(parseScholarIds("A-UbBTPM15wL, invalid, B-UbBTPM15wL")).toEqual(["A-UbBTPM15wL", "B-UbBTPM15wL"]);
|
||||
});
|
||||
|
||||
it("returns empty array for empty string", () => {
|
||||
expect(parseScholarIds("")).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns empty array for whitespace only", () => {
|
||||
expect(parseScholarIds(" ")).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractScholarIdFromUrl", () => {
|
||||
it("returns null for non-URL strings", () => {
|
||||
expect(extractScholarIdFromUrl("not-a-url")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null for URL without user param", () => {
|
||||
expect(extractScholarIdFromUrl("https://scholar.google.com/citations?hl=en")).toBeNull();
|
||||
});
|
||||
|
||||
it("extracts from URL with trailing slashes", () => {
|
||||
expect(extractScholarIdFromUrl("https://scholar.google.com/citations?user=A-UbBTPM15wL///")).toBe("A-UbBTPM15wL");
|
||||
});
|
||||
|
||||
it("strips fragment before parsing", () => {
|
||||
expect(extractScholarIdFromUrl("https://scholar.google.com/citations?user=A-UbBTPM15wL#foo")).toBe("A-UbBTPM15wL");
|
||||
});
|
||||
|
||||
it("returns null when user param is invalid length", () => {
|
||||
expect(extractScholarIdFromUrl("https://scholar.google.com/citations?user=short")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("validateTokenAsId", () => {
|
||||
it("returns null for valid 12-char ID", () => {
|
||||
expect(validateTokenAsId("ABCDEF123456")).toBeNull();
|
||||
});
|
||||
|
||||
it("detects wrong length", () => {
|
||||
expect(validateTokenAsId("ABC")).toContain("12 characters");
|
||||
});
|
||||
|
||||
it("detects invalid characters", () => {
|
||||
expect(validateTokenAsId("ABCDEF12345!")).toContain("invalid characters");
|
||||
});
|
||||
|
||||
it("detects whitespace", () => {
|
||||
expect(validateTokenAsId("ABC DEF12345")).toContain("whitespace");
|
||||
});
|
||||
|
||||
it("detects empty input", () => {
|
||||
expect(validateTokenAsId("")).toContain("empty");
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseScholarTokens (per-token errors)", () => {
|
||||
it("marks invalid tokens with error messages", () => {
|
||||
const tokens = parseScholarTokens("A-UbBTPM15wL, short, B-UbBTPM15wL");
|
||||
expect(tokens).toHaveLength(3);
|
||||
expect(tokens[0].id).toBe("A-UbBTPM15wL");
|
||||
expect(tokens[0].error).toBeNull();
|
||||
expect(tokens[1].id).toBeNull();
|
||||
expect(tokens[1].error).toContain("12 characters");
|
||||
expect(tokens[2].id).toBe("B-UbBTPM15wL");
|
||||
});
|
||||
|
||||
it("marks duplicate tokens", () => {
|
||||
const tokens = parseScholarTokens("A-UbBTPM15wL, A-UbBTPM15wL");
|
||||
expect(tokens[1].error).toBe("duplicate");
|
||||
});
|
||||
|
||||
it("marks bad URL tokens", () => {
|
||||
const tokens = parseScholarTokens("https://scholar.google.com/citations?hl=en");
|
||||
expect(tokens[0].error).toContain("could not extract");
|
||||
});
|
||||
});
|
||||
|
||||
describe("ScholarBatchAdd (component)", () => {
|
||||
describe("ScholarBatchAdd", () => {
|
||||
it("renders the heading and input", () => {
|
||||
const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
|
||||
expect(wrapper.text()).toContain("Add Scholar Profiles");
|
||||
|
|
@ -207,18 +73,4 @@ describe("ScholarBatchAdd (component)", () => {
|
|||
const wrapper = mount(ScholarBatchAdd, { props: { ...defaultProps, saving: true } });
|
||||
expect(wrapper.text()).toContain("Adding...");
|
||||
});
|
||||
|
||||
it("shows validation errors for invalid tokens", async () => {
|
||||
const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
|
||||
await wrapper.find("textarea").setValue("A-UbBTPM15wL, bad!");
|
||||
const errors = wrapper.find("[data-testid='validation-errors']");
|
||||
expect(errors.exists()).toBe(true);
|
||||
});
|
||||
|
||||
it("shows skipped count alongside valid count", async () => {
|
||||
const wrapper = mount(ScholarBatchAdd, { props: defaultProps });
|
||||
await wrapper.find("textarea").setValue("A-UbBTPM15wL, tooshort");
|
||||
expect(wrapper.text()).toContain("1 valid");
|
||||
expect(wrapper.text()).toContain("1 skipped");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import { computed, ref } from "vue";
|
|||
import AppButton from "@/components/ui/AppButton.vue";
|
||||
import AppCard from "@/components/ui/AppCard.vue";
|
||||
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
||||
import { parseScholarTokens } from "./scholar-batch-parsing";
|
||||
|
||||
defineProps<{
|
||||
saving: boolean;
|
||||
|
|
@ -16,13 +15,38 @@ const emit = defineEmits<{
|
|||
|
||||
const scholarBatchInput = ref("");
|
||||
|
||||
const parsedTokens = computed(() => parseScholarTokens(scholarBatchInput.value));
|
||||
const validIds = computed(() => parsedTokens.value.filter((t) => t.id !== null));
|
||||
const invalidTokens = computed(() => parsedTokens.value.filter((t) => t.id === null));
|
||||
const parsedBatchCount = computed(() => validIds.value.length);
|
||||
const SCHOLAR_ID_PATTERN = /^[a-zA-Z0-9_-]{12}$/;
|
||||
const URL_USER_PARAM_PATTERN = /(?:\?|&)user=([a-zA-Z0-9_-]{12})(?:&|#|$)/i;
|
||||
|
||||
function parseScholarIds(raw: string): string[] {
|
||||
const ordered: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
const tokens = raw.split(/[\s,;]+/).map((v) => v.trim()).filter((v) => v.length > 0);
|
||||
for (const token of tokens) {
|
||||
let candidate: string | null = null;
|
||||
if (SCHOLAR_ID_PATTERN.test(token)) candidate = token;
|
||||
if (!candidate) {
|
||||
const directParamMatch = token.match(URL_USER_PARAM_PATTERN);
|
||||
if (directParamMatch) candidate = directParamMatch[1];
|
||||
}
|
||||
if (!candidate && token.includes("scholar.google")) {
|
||||
try {
|
||||
const parsed = new URL(token);
|
||||
const userParam = parsed.searchParams.get("user");
|
||||
if (userParam && SCHOLAR_ID_PATTERN.test(userParam)) candidate = userParam;
|
||||
} catch (_error) { /* Ignore non-URL tokens. */ }
|
||||
}
|
||||
if (!candidate || seen.has(candidate)) continue;
|
||||
seen.add(candidate);
|
||||
ordered.push(candidate);
|
||||
}
|
||||
return ordered;
|
||||
}
|
||||
|
||||
const parsedBatchCount = computed(() => parseScholarIds(scholarBatchInput.value).length);
|
||||
|
||||
function onSubmit(): void {
|
||||
const ids = validIds.value.map((t) => t.id!);
|
||||
const ids = parseScholarIds(scholarBatchInput.value);
|
||||
if (ids.length > 0) {
|
||||
emit("add-scholars", ids);
|
||||
scholarBatchInput.value = "";
|
||||
|
|
@ -52,33 +76,11 @@ function onSubmit(): void {
|
|||
/>
|
||||
</label>
|
||||
|
||||
<div v-if="parsedTokens.length > 0" class="space-y-1">
|
||||
<div class="flex flex-wrap items-center justify-between gap-2">
|
||||
<p class="text-xs text-secondary">
|
||||
<strong class="text-ink-primary">{{ parsedBatchCount }}</strong> valid ID{{ parsedBatchCount === 1 ? "" : "s" }}
|
||||
<template v-if="invalidTokens.length > 0">
|
||||
· <span class="text-state-warning-text">{{ invalidTokens.length }} skipped</span>
|
||||
</template>
|
||||
Parsed IDs: <strong class="text-ink-primary">{{ parsedBatchCount }}</strong>
|
||||
</p>
|
||||
<ul v-if="invalidTokens.length > 0" class="space-y-0.5" data-testid="validation-errors">
|
||||
<li
|
||||
v-for="item in invalidTokens.slice(0, 5)"
|
||||
:key="item.index"
|
||||
class="text-xs text-state-warning-text"
|
||||
>
|
||||
#{{ item.index }}: {{ item.error }}
|
||||
<code class="ml-1 break-all text-ink-muted">{{ item.raw.length > 60 ? item.raw.slice(0, 57) + "..." : item.raw }}</code>
|
||||
</li>
|
||||
<li v-if="invalidTokens.length > 5" class="text-xs text-state-warning-text">
|
||||
+{{ invalidTokens.length - 5 }} more skipped
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div v-else class="text-xs text-secondary">
|
||||
Parsed IDs: <strong class="text-ink-primary">0</strong>
|
||||
</div>
|
||||
|
||||
<div class="flex flex-wrap items-center justify-end gap-2">
|
||||
<AppButton type="submit" :disabled="saving || loading || parsedBatchCount === 0">
|
||||
<AppButton type="submit" :disabled="saving || loading">
|
||||
{{ saving ? "Adding..." : "Add scholars" }}
|
||||
</AppButton>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -47,14 +47,7 @@ function scholarPublicationsRoute(profile: ScholarProfile): { name: string; quer
|
|||
:image-url="scholar.profile_image_url"
|
||||
/>
|
||||
<div class="min-w-0 space-y-1">
|
||||
<p class="truncate text-sm font-semibold text-ink-primary">
|
||||
{{ scholarLabel(scholar) }}
|
||||
<span
|
||||
v-if="!scholar.baseline_completed"
|
||||
class="ml-1.5 inline-flex items-center rounded-full border border-state-warning-border bg-state-warning-bg px-1.5 py-0.5 text-[10px] font-medium text-state-warning-text"
|
||||
title="This scholar hasn't been scraped yet. Publications will appear after the next run."
|
||||
>Pending</span>
|
||||
</p>
|
||||
<p class="truncate text-sm font-semibold text-ink-primary">{{ scholarLabel(scholar) }}</p>
|
||||
<p class="text-xs text-secondary">ID: <code>{{ scholar.scholar_id }}</code></p>
|
||||
<div class="flex flex-wrap items-center gap-3">
|
||||
<RouterLink :to="scholarPublicationsRoute(scholar)" class="link-inline text-xs">
|
||||
|
|
|
|||
|
|
@ -1,29 +0,0 @@
|
|||
<script setup lang="ts">
|
||||
defineProps<{
|
||||
baselineCompleted: boolean;
|
||||
lastRunStatus: string | null;
|
||||
}>();
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<span
|
||||
v-if="!baselineCompleted"
|
||||
class="ml-1.5 inline-flex items-center rounded-full border border-state-warning-border bg-state-warning-bg px-1.5 py-0.5 text-[10px] font-medium text-state-warning-text"
|
||||
title="This scholar hasn't been scraped yet. Publications will appear after the next run."
|
||||
>Pending</span>
|
||||
<span
|
||||
v-if="lastRunStatus === 'failed'"
|
||||
class="ml-1.5 inline-flex items-center rounded-full border border-state-danger-border bg-state-danger-bg px-1.5 py-0.5 text-[10px] font-medium text-state-danger-text"
|
||||
title="The last scrape run for this scholar failed."
|
||||
>Failed</span>
|
||||
<span
|
||||
v-else-if="lastRunStatus === 'partial'"
|
||||
class="ml-1.5 inline-flex items-center rounded-full border border-state-warning-border bg-state-warning-bg px-1.5 py-0.5 text-[10px] font-medium text-state-warning-text"
|
||||
title="The last scrape run for this scholar completed with partial results."
|
||||
>Partial</span>
|
||||
<span
|
||||
v-else-if="lastRunStatus === 'success'"
|
||||
class="ml-1.5 inline-flex items-center rounded-full border border-state-success-border bg-state-success-bg px-1.5 py-0.5 text-[10px] font-medium text-state-success-text"
|
||||
title="The last scrape run for this scholar completed successfully."
|
||||
>OK</span>
|
||||
</template>
|
||||
|
|
@ -1,76 +0,0 @@
|
|||
const SCHOLAR_ID_PATTERN = /^[a-zA-Z0-9_-]{12}$/;
|
||||
|
||||
export interface ParsedToken {
|
||||
index: number;
|
||||
raw: string;
|
||||
id: string | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export function extractScholarIdFromUrl(token: string): string | null {
|
||||
const cleaned = token.replace(/\/+$/, "").replace(/#.*$/, "");
|
||||
try {
|
||||
const parsed = new URL(cleaned);
|
||||
const userParam = parsed.searchParams.get("user");
|
||||
if (!userParam) return null;
|
||||
const decoded = decodeURIComponent(userParam).trim();
|
||||
if (SCHOLAR_ID_PATTERN.test(decoded)) return decoded;
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function validateTokenAsId(token: string): string | null {
|
||||
const trimmed = token.trim();
|
||||
if (!trimmed) return "empty input";
|
||||
if (/\s/.test(trimmed)) return "contains whitespace";
|
||||
if (trimmed.length !== 12) return `must be 12 characters (got ${trimmed.length})`;
|
||||
if (!/^[a-zA-Z0-9_-]+$/.test(trimmed)) {
|
||||
return "contains invalid characters (only a-z, A-Z, 0-9, _ and - allowed)";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function parseScholarTokens(raw: string): ParsedToken[] {
|
||||
const tokens = raw.split(/[\s,;]+/).map((v) => v.trim()).filter((v) => v.length > 0);
|
||||
const results: ParsedToken[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (let i = 0; i < tokens.length; i++) {
|
||||
const token = tokens[i];
|
||||
|
||||
if (SCHOLAR_ID_PATTERN.test(token)) {
|
||||
if (seen.has(token)) {
|
||||
results.push({ index: i + 1, raw: token, id: null, error: "duplicate" });
|
||||
continue;
|
||||
}
|
||||
seen.add(token);
|
||||
results.push({ index: i + 1, raw: token, id: token, error: null });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (token.includes("scholar.google") || token.startsWith("http")) {
|
||||
const extracted = extractScholarIdFromUrl(token);
|
||||
if (extracted) {
|
||||
if (seen.has(extracted)) {
|
||||
results.push({ index: i + 1, raw: token, id: null, error: "duplicate" });
|
||||
continue;
|
||||
}
|
||||
seen.add(extracted);
|
||||
results.push({ index: i + 1, raw: token, id: extracted, error: null });
|
||||
continue;
|
||||
}
|
||||
results.push({ index: i + 1, raw: token, id: null, error: "could not extract scholar ID from URL" });
|
||||
continue;
|
||||
}
|
||||
|
||||
const reason = validateTokenAsId(token);
|
||||
results.push({ index: i + 1, raw: token, id: null, error: reason ?? "invalid scholar ID" });
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
export function parseScholarIds(raw: string): string[] {
|
||||
return parseScholarTokens(raw).filter((t) => t.id !== null).map((t) => t.id!);
|
||||
}
|
||||
|
|
@ -1,148 +0,0 @@
|
|||
// @vitest-environment happy-dom
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { nextTick, ref } from "vue";
|
||||
import { useScholarBulkActions, type ScholarBulkAction } from "./useScholarBulkActions";
|
||||
import type { ScholarProfile } from "@/features/scholars";
|
||||
|
||||
vi.mock("@/features/scholars", () => ({
|
||||
bulkDeleteScholars: vi.fn(),
|
||||
bulkToggleScholars: vi.fn(),
|
||||
exportScholarData: vi.fn(),
|
||||
}));
|
||||
|
||||
function makeProfile(id: number, name: string): ScholarProfile {
|
||||
return {
|
||||
id,
|
||||
scholar_id: `scholar_${id}`,
|
||||
display_name: name,
|
||||
profile_image_url: null,
|
||||
profile_image_source: "none",
|
||||
is_enabled: true,
|
||||
baseline_completed: false,
|
||||
last_run_dt: null,
|
||||
last_run_status: null,
|
||||
};
|
||||
}
|
||||
|
||||
function setup(profiles: ScholarProfile[] = []) {
|
||||
const visibleScholars = ref(profiles);
|
||||
const callbacks = {
|
||||
clearMessages: vi.fn(),
|
||||
assignError: vi.fn(),
|
||||
setSuccess: vi.fn(),
|
||||
reloadScholars: vi.fn(async () => {}),
|
||||
};
|
||||
const bulk = useScholarBulkActions(visibleScholars, callbacks);
|
||||
return { visibleScholars, callbacks, bulk };
|
||||
}
|
||||
|
||||
describe("useScholarBulkActions", () => {
|
||||
it("starts with empty selection", () => {
|
||||
const { bulk } = setup([makeProfile(1, "Alice")]);
|
||||
expect(bulk.selectedIds.value.size).toBe(0);
|
||||
expect(bulk.hasSelection.value).toBe(false);
|
||||
});
|
||||
|
||||
it("toggles individual row selection", () => {
|
||||
const { bulk } = setup([makeProfile(1, "Alice"), makeProfile(2, "Bob")]);
|
||||
bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
|
||||
expect(bulk.selectedIds.value.has(1)).toBe(true);
|
||||
expect(bulk.selectedCount.value).toBe(1);
|
||||
|
||||
bulk.onToggleRow(1, { target: { checked: false } } as unknown as Event);
|
||||
expect(bulk.selectedIds.value.has(1)).toBe(false);
|
||||
expect(bulk.selectedCount.value).toBe(0);
|
||||
});
|
||||
|
||||
it("toggles all visible scholars", () => {
|
||||
const { bulk } = setup([makeProfile(1, "Alice"), makeProfile(2, "Bob")]);
|
||||
bulk.onToggleAll({ target: { checked: true } } as unknown as Event);
|
||||
expect(bulk.selectedIds.value.size).toBe(2);
|
||||
expect(bulk.allVisibleSelected.value).toBe(true);
|
||||
|
||||
bulk.onToggleAll({ target: { checked: false } } as unknown as Event);
|
||||
expect(bulk.selectedIds.value.size).toBe(0);
|
||||
});
|
||||
|
||||
it("prunes stale selections when list changes", async () => {
|
||||
const { visibleScholars, bulk } = setup([makeProfile(1, "Alice"), makeProfile(2, "Bob")]);
|
||||
bulk.onToggleAll({ target: { checked: true } } as unknown as Event);
|
||||
expect(bulk.selectedIds.value.size).toBe(2);
|
||||
|
||||
// Remove Bob from the visible list
|
||||
visibleScholars.value = [makeProfile(1, "Alice")];
|
||||
await nextTick();
|
||||
|
||||
expect(bulk.selectedIds.value.size).toBe(1);
|
||||
expect(bulk.selectedIds.value.has(1)).toBe(true);
|
||||
expect(bulk.selectedIds.value.has(2)).toBe(false);
|
||||
});
|
||||
|
||||
it("shows correct bulk action options with and without selection", () => {
|
||||
const { bulk } = setup([makeProfile(1, "Alice")]);
|
||||
// Without selection
|
||||
expect(bulk.bulkActionOptions.value.length).toBe(1);
|
||||
expect(bulk.bulkActionOptions.value[0].value).toBe("select_all");
|
||||
|
||||
// With selection
|
||||
bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
|
||||
expect(bulk.bulkActionOptions.value.length).toBe(5);
|
||||
const values = bulk.bulkActionOptions.value.map((o) => o.value);
|
||||
expect(values).toContain("delete_selected");
|
||||
expect(values).toContain("enable_selected");
|
||||
expect(values).toContain("disable_selected");
|
||||
expect(values).toContain("export_selected");
|
||||
expect(values).toContain("clear_selection");
|
||||
});
|
||||
|
||||
it("select all action selects all visible", async () => {
|
||||
const { bulk } = setup([makeProfile(1, "Alice"), makeProfile(2, "Bob")]);
|
||||
bulk.bulkAction.value = "select_all" as ScholarBulkAction;
|
||||
await bulk.onApplyBulkAction();
|
||||
expect(bulk.selectedIds.value.size).toBe(2);
|
||||
});
|
||||
|
||||
it("clear selection action clears all", async () => {
|
||||
const { bulk } = setup([makeProfile(1, "Alice")]);
|
||||
bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
|
||||
bulk.bulkAction.value = "clear_selection" as ScholarBulkAction;
|
||||
await bulk.onApplyBulkAction();
|
||||
expect(bulk.selectedIds.value.size).toBe(0);
|
||||
});
|
||||
|
||||
it("bulk delete calls assignError on network failure", async () => {
|
||||
const { bulkDeleteScholars } = await import("@/features/scholars");
|
||||
vi.mocked(bulkDeleteScholars).mockRejectedValueOnce(new Error("Network error"));
|
||||
|
||||
const { bulk, callbacks } = setup([makeProfile(1, "Alice")]);
|
||||
bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
|
||||
bulk.bulkAction.value = "delete_selected" as ScholarBulkAction;
|
||||
await bulk.onApplyBulkAction();
|
||||
|
||||
// Trigger the confirm callback
|
||||
expect(bulk.confirmState.value.open).toBe(true);
|
||||
await bulk.confirmState.value.onConfirm();
|
||||
|
||||
expect(callbacks.assignError).toHaveBeenCalledWith(
|
||||
expect.any(Error),
|
||||
"Unable to bulk delete scholars.",
|
||||
);
|
||||
expect(bulk.bulkBusy.value).toBe(false);
|
||||
});
|
||||
|
||||
it("bulk toggle calls assignError on network failure", async () => {
|
||||
const { bulkToggleScholars } = await import("@/features/scholars");
|
||||
vi.mocked(bulkToggleScholars).mockRejectedValueOnce(new Error("Network error"));
|
||||
|
||||
const { bulk, callbacks } = setup([makeProfile(1, "Alice")]);
|
||||
bulk.onToggleRow(1, { target: { checked: true } } as unknown as Event);
|
||||
bulk.bulkAction.value = "enable_selected" as ScholarBulkAction;
|
||||
await bulk.onApplyBulkAction();
|
||||
|
||||
expect(callbacks.assignError).toHaveBeenCalledWith(
|
||||
expect.any(Error),
|
||||
"Unable to bulk enable scholars.",
|
||||
);
|
||||
expect(bulk.bulkBusy.value).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,221 +0,0 @@
|
|||
import { computed, ref, watch, type Ref } from "vue";
|
||||
|
||||
import {
|
||||
bulkDeleteScholars,
|
||||
bulkToggleScholars,
|
||||
exportScholarData,
|
||||
type ScholarProfile,
|
||||
} from "@/features/scholars";
|
||||
|
||||
export type ScholarBulkAction =
|
||||
| "delete_selected"
|
||||
| "enable_selected"
|
||||
| "disable_selected"
|
||||
| "export_selected"
|
||||
| "clear_selection"
|
||||
| "select_all";
|
||||
|
||||
export interface ScholarBulkActionOption {
|
||||
value: ScholarBulkAction;
|
||||
label: string;
|
||||
}
|
||||
|
||||
export interface ConfirmState {
|
||||
open: boolean;
|
||||
title: string;
|
||||
message: string;
|
||||
variant: "danger" | "default";
|
||||
onConfirm: () => void;
|
||||
}
|
||||
|
||||
export interface BulkActionCallbacks {
|
||||
clearMessages: () => void;
|
||||
assignError: (error: unknown, fallback: string) => void;
|
||||
setSuccess: (msg: string) => void;
|
||||
reloadScholars: () => Promise<void>;
|
||||
}
|
||||
|
||||
export function useScholarBulkActions(
|
||||
visibleScholars: Ref<ScholarProfile[]>,
|
||||
callbacks: BulkActionCallbacks,
|
||||
) {
|
||||
const selectedIds = ref<Set<number>>(new Set());
|
||||
const bulkAction = ref<ScholarBulkAction>("select_all");
|
||||
const bulkBusy = ref(false);
|
||||
const confirmState = ref<ConfirmState>({ open: false, title: "", message: "", variant: "default", onConfirm: () => {} });
|
||||
|
||||
const selectedCount = computed(() => selectedIds.value.size);
|
||||
const hasSelection = computed(() => selectedCount.value > 0);
|
||||
|
||||
const allVisibleSelected = computed(() => {
|
||||
if (visibleScholars.value.length === 0) return false;
|
||||
for (const item of visibleScholars.value) {
|
||||
if (!selectedIds.value.has(item.id)) return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
const bulkActionOptions = computed<ScholarBulkActionOption[]>(() => {
|
||||
if (!hasSelection.value) return [{ value: "select_all", label: "Select all" }];
|
||||
const n = selectedCount.value;
|
||||
return [
|
||||
{ value: "delete_selected", label: `Delete selected (${n})` },
|
||||
{ value: "enable_selected", label: `Enable selected (${n})` },
|
||||
{ value: "disable_selected", label: `Disable selected (${n})` },
|
||||
{ value: "export_selected", label: `Export selected (${n})` },
|
||||
{ value: "clear_selection", label: "Clear selection" },
|
||||
];
|
||||
});
|
||||
|
||||
const bulkApplyLabel = computed(() => {
|
||||
if (bulkBusy.value) return "Applying...";
|
||||
if (bulkAction.value === "select_all") return "Select";
|
||||
if (bulkAction.value === "clear_selection") return "Clear";
|
||||
return "Apply";
|
||||
});
|
||||
|
||||
const bulkApplyDisabled = computed(() => {
|
||||
if (bulkBusy.value) return true;
|
||||
if (bulkAction.value === "select_all") return visibleScholars.value.length === 0;
|
||||
return selectedCount.value === 0;
|
||||
});
|
||||
|
||||
// Prune stale selections when visible list changes
|
||||
watch(visibleScholars, (items) => {
|
||||
const validIds = new Set(items.map((item) => item.id));
|
||||
const next = new Set<number>();
|
||||
for (const id of selectedIds.value) {
|
||||
if (validIds.has(id)) next.add(id);
|
||||
}
|
||||
if (next.size !== selectedIds.value.size) selectedIds.value = next;
|
||||
});
|
||||
|
||||
// Reset bulk action dropdown when selection state changes
|
||||
watch(hasSelection, (has) => {
|
||||
const validValues = new Set(bulkActionOptions.value.map((o) => o.value));
|
||||
if (validValues.has(bulkAction.value)) return;
|
||||
bulkAction.value = has ? "delete_selected" : "select_all";
|
||||
});
|
||||
|
||||
function onToggleAll(event: Event): void {
|
||||
const checked = (event.target as HTMLInputElement).checked;
|
||||
const next = new Set(selectedIds.value);
|
||||
for (const item of visibleScholars.value) {
|
||||
if (checked) { next.add(item.id); } else { next.delete(item.id); }
|
||||
}
|
||||
selectedIds.value = next;
|
||||
}
|
||||
|
||||
function onToggleRow(id: number, event: Event): void {
|
||||
const checked = (event.target as HTMLInputElement).checked;
|
||||
const next = new Set(selectedIds.value);
|
||||
if (checked) { next.add(id); } else { next.delete(id); }
|
||||
selectedIds.value = next;
|
||||
}
|
||||
|
||||
function downloadJsonFile(filename: string, payload: unknown): void {
|
||||
const blob = new Blob([JSON.stringify(payload, null, 2)], { type: "application/json" });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const anchor = document.createElement("a");
|
||||
anchor.href = url;
|
||||
anchor.download = filename;
|
||||
anchor.click();
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
async function onApplyBulkAction(): Promise<void> {
|
||||
if (bulkApplyDisabled.value) return;
|
||||
if (bulkAction.value === "select_all") {
|
||||
selectedIds.value = new Set(visibleScholars.value.map((item) => item.id));
|
||||
return;
|
||||
}
|
||||
if (bulkAction.value === "clear_selection") { selectedIds.value = new Set(); return; }
|
||||
if (bulkAction.value === "delete_selected") { await onBulkDelete(); return; }
|
||||
if (bulkAction.value === "enable_selected") { await onBulkToggle(true); return; }
|
||||
if (bulkAction.value === "disable_selected") { await onBulkToggle(false); return; }
|
||||
if (bulkAction.value === "export_selected") { await onBulkExport(); return; }
|
||||
}
|
||||
|
||||
function dismissConfirm(): void {
|
||||
confirmState.value = { ...confirmState.value, open: false };
|
||||
}
|
||||
|
||||
function requestConfirm(title: string, message: string, variant: "danger" | "default", onConfirm: () => void): void {
|
||||
confirmState.value = { open: true, title, message, variant, onConfirm };
|
||||
}
|
||||
|
||||
function onBulkDelete(): void {
|
||||
const ids = [...selectedIds.value];
|
||||
requestConfirm(
|
||||
`Delete ${ids.length} scholar(s)?`,
|
||||
"This removes all linked publications and queue data. This action cannot be undone.",
|
||||
"danger",
|
||||
async () => {
|
||||
dismissConfirm();
|
||||
bulkBusy.value = true;
|
||||
callbacks.clearMessages();
|
||||
try {
|
||||
const result = await bulkDeleteScholars(ids);
|
||||
callbacks.setSuccess(`${result.deleted_count} scholar(s) deleted.`);
|
||||
selectedIds.value = new Set();
|
||||
await callbacks.reloadScholars();
|
||||
} catch (error) {
|
||||
callbacks.assignError(error, "Unable to bulk delete scholars.");
|
||||
} finally {
|
||||
bulkBusy.value = false;
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function onBulkToggle(isEnabled: boolean): Promise<void> {
|
||||
bulkBusy.value = true;
|
||||
callbacks.clearMessages();
|
||||
try {
|
||||
const ids = [...selectedIds.value];
|
||||
const result = await bulkToggleScholars(ids, isEnabled);
|
||||
const verb = isEnabled ? "enabled" : "disabled";
|
||||
callbacks.setSuccess(`${result.updated_count} scholar(s) ${verb}.`);
|
||||
selectedIds.value = new Set();
|
||||
await callbacks.reloadScholars();
|
||||
} catch (error) {
|
||||
callbacks.assignError(error, `Unable to bulk ${isEnabled ? "enable" : "disable"} scholars.`);
|
||||
} finally {
|
||||
bulkBusy.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function onBulkExport(): Promise<void> {
|
||||
bulkBusy.value = true;
|
||||
callbacks.clearMessages();
|
||||
try {
|
||||
const ids = [...selectedIds.value];
|
||||
const payload = await exportScholarData(ids);
|
||||
const dateSlug = payload.exported_at.slice(0, 10) || "unknown-date";
|
||||
downloadJsonFile(`scholarr-export-${dateSlug}.json`, payload);
|
||||
callbacks.setSuccess("Export complete.");
|
||||
} catch (error) {
|
||||
callbacks.assignError(error, "Unable to export selected scholars.");
|
||||
} finally {
|
||||
bulkBusy.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
selectedIds,
|
||||
selectedCount,
|
||||
hasSelection,
|
||||
allVisibleSelected,
|
||||
bulkAction,
|
||||
bulkBusy,
|
||||
bulkActionOptions,
|
||||
bulkApplyLabel,
|
||||
bulkApplyDisabled,
|
||||
confirmState,
|
||||
dismissConfirm,
|
||||
requestConfirm,
|
||||
onToggleAll,
|
||||
onToggleRow,
|
||||
onApplyBulkAction,
|
||||
};
|
||||
}
|
||||
|
|
@ -160,30 +160,8 @@ export async function clearScholarImage(
|
|||
return response.data;
|
||||
}
|
||||
|
||||
export interface BulkCountResult {
|
||||
deleted_count: number;
|
||||
updated_count: number;
|
||||
}
|
||||
|
||||
export async function bulkDeleteScholars(scholarProfileIds: number[]): Promise<BulkCountResult> {
|
||||
const response = await apiRequest<BulkCountResult>("/scholars/bulk-delete", {
|
||||
method: "POST",
|
||||
body: { scholar_profile_ids: scholarProfileIds },
|
||||
});
|
||||
return response.data;
|
||||
}
|
||||
|
||||
export async function bulkToggleScholars(scholarProfileIds: number[], isEnabled: boolean): Promise<BulkCountResult> {
|
||||
const response = await apiRequest<BulkCountResult>("/scholars/bulk-toggle", {
|
||||
method: "POST",
|
||||
body: { scholar_profile_ids: scholarProfileIds, is_enabled: isEnabled },
|
||||
});
|
||||
return response.data;
|
||||
}
|
||||
|
||||
export async function exportScholarData(ids?: number[]): Promise<DataExportPayload> {
|
||||
const path = ids && ids.length > 0 ? `/scholars/export?ids=${ids.join(",")}` : "/scholars/export";
|
||||
const response = await apiRequest<DataExportPayload>(path, {
|
||||
export async function exportScholarData(): Promise<DataExportPayload> {
|
||||
const response = await apiRequest<DataExportPayload>("/scholars/export", {
|
||||
method: "GET",
|
||||
});
|
||||
return response.data;
|
||||
|
|
|
|||
|
|
@ -1,94 +0,0 @@
|
|||
// @vitest-environment happy-dom
|
||||
import { describe, expect, it, vi, beforeEach } from "vitest";
|
||||
import { mount, flushPromises } from "@vue/test-utils";
|
||||
import { nextTick } from "vue";
|
||||
import SettingsAdminPanel from "./SettingsAdminPanel.vue";
|
||||
|
||||
vi.mock("@/features/admin_users", () => ({
|
||||
listAdminUsers: vi.fn().mockResolvedValue([]),
|
||||
createAdminUser: vi.fn(),
|
||||
setAdminUserActive: vi.fn(),
|
||||
resetAdminUserPassword: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/features/admin_dbops", () => ({
|
||||
getAdminDbIntegrityReport: vi.fn().mockResolvedValue({
|
||||
status: "ok",
|
||||
warnings: [],
|
||||
failures: [],
|
||||
checked_at: null,
|
||||
checks: [],
|
||||
}),
|
||||
listAdminPdfQueue: vi.fn().mockResolvedValue({
|
||||
items: [],
|
||||
total_count: 0,
|
||||
has_next: false,
|
||||
has_prev: false,
|
||||
page: 1,
|
||||
page_size: 50,
|
||||
}),
|
||||
requeueAdminPdfLookup: vi.fn(),
|
||||
requeueAllAdminPdfLookups: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/stores/auth", () => ({
|
||||
useAuthStore: () => ({ isAdmin: true }),
|
||||
}));
|
||||
|
||||
vi.mock("@/features/admin_repairs", () => ({
|
||||
listAdminRepairTasks: vi.fn().mockResolvedValue([]),
|
||||
runAdminRepairTask: vi.fn(),
|
||||
}));
|
||||
|
||||
import { listAdminUsers } from "@/features/admin_users";
|
||||
import { getAdminDbIntegrityReport, listAdminPdfQueue } from "@/features/admin_dbops";
|
||||
|
||||
const mockedListUsers = vi.mocked(listAdminUsers);
|
||||
const mockedGetReport = vi.mocked(getAdminDbIntegrityReport);
|
||||
const mockedListPdfQueue = vi.mocked(listAdminPdfQueue);
|
||||
|
||||
describe("SettingsAdminPanel", () => {
|
||||
beforeEach(() => {
|
||||
mockedListUsers.mockClear();
|
||||
mockedGetReport.mockClear();
|
||||
mockedListPdfQueue.mockClear();
|
||||
});
|
||||
|
||||
it("calls load on users section after nextTick when section=users", async () => {
|
||||
mount(SettingsAdminPanel, { props: { section: "users" } });
|
||||
await nextTick();
|
||||
await nextTick();
|
||||
await flushPromises();
|
||||
expect(mockedListUsers).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("calls load on integrity section after nextTick when section=integrity", async () => {
|
||||
mount(SettingsAdminPanel, { props: { section: "integrity" } });
|
||||
await nextTick();
|
||||
await nextTick();
|
||||
await flushPromises();
|
||||
expect(mockedGetReport).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("calls load on new section when section prop changes", async () => {
|
||||
const wrapper = mount(SettingsAdminPanel, { props: { section: "users" } });
|
||||
await nextTick();
|
||||
await nextTick();
|
||||
await flushPromises();
|
||||
mockedListUsers.mockClear();
|
||||
mockedGetReport.mockClear();
|
||||
|
||||
await wrapper.setProps({ section: "integrity" });
|
||||
await nextTick();
|
||||
await flushPromises();
|
||||
expect(mockedGetReport).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("calls load on pdf section when section=pdf", async () => {
|
||||
mount(SettingsAdminPanel, { props: { section: "pdf" } });
|
||||
await nextTick();
|
||||
await nextTick();
|
||||
await flushPromises();
|
||||
expect(mockedListPdfQueue).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
<script setup lang="ts">
|
||||
import { nextTick, onMounted, watch } from "vue";
|
||||
import { ref } from "vue";
|
||||
import { onMounted, ref, watch } from "vue";
|
||||
|
||||
import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
|
||||
import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue";
|
||||
import AdminIntegritySection from "@/features/settings/components/AdminIntegritySection.vue";
|
||||
import AdminPdfQueueSection from "@/features/settings/components/AdminPdfQueueSection.vue";
|
||||
|
|
@ -18,6 +18,7 @@ const props = defineProps<{
|
|||
section: "users" | "integrity" | "repairs" | "pdf";
|
||||
}>();
|
||||
|
||||
const loading = ref(true);
|
||||
const { errorMessage, errorRequestId, successMessage, clearAlerts, assignError } = useRequestState();
|
||||
|
||||
const usersRef = ref<InstanceType<typeof AdminUsersSection> | null>(null);
|
||||
|
|
@ -25,33 +26,39 @@ const integrityRef = ref<InstanceType<typeof AdminIntegritySection> | null>(null
|
|||
const repairsRef = ref<InstanceType<typeof AdminRepairsSection> | null>(null);
|
||||
const pdfQueueRef = ref<InstanceType<typeof AdminPdfQueueSection> | null>(null);
|
||||
|
||||
let loadGeneration = 0;
|
||||
|
||||
async function loadSection(): Promise<void> {
|
||||
const gen = ++loadGeneration;
|
||||
clearAlerts();
|
||||
try {
|
||||
async function refreshForSection(): Promise<void> {
|
||||
if (props.section === SECTION_USERS && usersRef.value) {
|
||||
await usersRef.value.load();
|
||||
} else if (props.section === SECTION_INTEGRITY && integrityRef.value) {
|
||||
await integrityRef.value.load();
|
||||
} else if (props.section === SECTION_REPAIRS) {
|
||||
await usersRef.value?.load();
|
||||
if (gen !== loadGeneration) return;
|
||||
await repairsRef.value?.load();
|
||||
} else if (props.section === SECTION_PDF && pdfQueueRef.value) {
|
||||
await pdfQueueRef.value.load();
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
if (gen !== loadGeneration) return;
|
||||
assignError(error, "Unable to load admin data.");
|
||||
if (props.section === SECTION_INTEGRITY && integrityRef.value) {
|
||||
await integrityRef.value.load();
|
||||
return;
|
||||
}
|
||||
if (props.section === SECTION_REPAIRS) {
|
||||
await usersRef.value?.load();
|
||||
await repairsRef.value?.load();
|
||||
return;
|
||||
}
|
||||
if (props.section === SECTION_PDF && pdfQueueRef.value) {
|
||||
await pdfQueueRef.value.load();
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
nextTick(loadSection);
|
||||
});
|
||||
watch(() => props.section, loadSection, { flush: "post" });
|
||||
async function loadSection(): Promise<void> {
|
||||
loading.value = true;
|
||||
clearAlerts();
|
||||
try {
|
||||
await refreshForSection();
|
||||
} catch (error) {
|
||||
assignError(error, "Unable to load admin data.");
|
||||
} finally {
|
||||
loading.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(loadSection);
|
||||
watch(() => props.section, loadSection);
|
||||
</script>
|
||||
|
||||
<template>
|
||||
|
|
@ -65,9 +72,11 @@ watch(() => props.section, loadSection, { flush: "post" });
|
|||
@dismiss-success="successMessage = null"
|
||||
/>
|
||||
|
||||
<AsyncStateGate :loading="loading" :loading-lines="10" :show-empty="false">
|
||||
<AdminUsersSection v-if="props.section === SECTION_USERS || props.section === SECTION_REPAIRS" v-show="props.section === SECTION_USERS" ref="usersRef" />
|
||||
<AdminIntegritySection v-if="props.section === SECTION_INTEGRITY" ref="integrityRef" />
|
||||
<AdminRepairsSection v-if="props.section === SECTION_REPAIRS" ref="repairsRef" :users="usersRef?.users ?? []" />
|
||||
<AdminPdfQueueSection v-if="props.section === SECTION_PDF" ref="pdfQueueRef" />
|
||||
</AsyncStateGate>
|
||||
</section>
|
||||
</template>
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
// @vitest-environment happy-dom
|
||||
import { describe, expect, it, vi, beforeEach } from "vitest";
|
||||
import { mount, flushPromises } from "@vue/test-utils";
|
||||
import { setActivePinia, createPinia } from "pinia";
|
||||
import { useAuthStore } from "@/stores/auth";
|
||||
import AdminPdfQueueSection from "./AdminPdfQueueSection.vue";
|
||||
|
||||
vi.mock("@/features/admin_dbops", () => ({
|
||||
|
|
@ -38,9 +36,6 @@ function buildQueueItem(overrides: Record<string, unknown> = {}) {
|
|||
|
||||
describe("AdminPdfQueueSection", () => {
|
||||
beforeEach(() => {
|
||||
setActivePinia(createPinia());
|
||||
const auth = useAuthStore();
|
||||
auth.$patch({ state: "authenticated", user: { id: 1, email: "admin@test.com", is_admin: true, is_active: true } });
|
||||
mockedListQueue.mockReset();
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
<script setup lang="ts">
|
||||
import { computed, ref } from "vue";
|
||||
import { useAuthStore } from "@/stores/auth";
|
||||
import AppBadge from "@/components/ui/AppBadge.vue";
|
||||
import AppButton from "@/components/ui/AppButton.vue";
|
||||
import AppCard from "@/components/ui/AppCard.vue";
|
||||
|
|
@ -18,7 +17,6 @@ import {
|
|||
import { useRequestState } from "@/composables/useRequestState";
|
||||
|
||||
const { errorMessage, errorRequestId, successMessage, clearAlerts, assignError, setSuccess } = useRequestState();
|
||||
const auth = useAuthStore();
|
||||
|
||||
const refreshingPdfQueue = ref(false);
|
||||
const requeueingPublicationId = ref<number | null>(null);
|
||||
|
|
@ -164,7 +162,7 @@ defineExpose({ load });
|
|||
<option value="50">50 / page</option>
|
||||
<option value="100">100 / page</option>
|
||||
</AppSelect>
|
||||
<AppButton v-if="auth.isAdmin" variant="secondary" class="!min-h-8 whitespace-nowrap !px-2.5 !py-1 !text-xs" :disabled="requeueingAllPdfs" title="Queue all missing PDFs" @click="onRequeueAllPdfs">
|
||||
<AppButton variant="secondary" class="!min-h-8 whitespace-nowrap !px-2.5 !py-1 !text-xs" :disabled="requeueingAllPdfs" title="Queue all missing PDFs" @click="onRequeueAllPdfs">
|
||||
{{ requeueingAllPdfs ? "Queueing..." : "Queue all" }}
|
||||
</AppButton>
|
||||
<AppRefreshButton variant="secondary" size="sm" :loading="refreshingPdfQueue" title="Refresh PDF queue" loading-title="Refreshing PDF queue" @click="refreshPdfQueue" />
|
||||
|
|
@ -196,7 +194,7 @@ defineExpose({ load });
|
|||
<td>{{ formatTimestamp(item.last_attempt_at) }}</td>
|
||||
<td>{{ formatTimestamp(item.resolved_at) }}</td>
|
||||
<td>
|
||||
<AppButton v-if="auth.isAdmin" variant="ghost" :disabled="requeueingPublicationId === item.publication_id || !canRequeuePdf(item)" @click="onRequeuePdf(item)">
|
||||
<AppButton variant="ghost" :disabled="requeueingPublicationId === item.publication_id || !canRequeuePdf(item)" @click="onRequeuePdf(item)">
|
||||
{{ requeueingPublicationId === item.publication_id ? "Requeueing..." : "Requeue" }}
|
||||
</AppButton>
|
||||
</td>
|
||||
|
|
|
|||
|
|
@ -404,7 +404,7 @@ watch(
|
|||
</div>
|
||||
<div class="flex items-center gap-2">
|
||||
<AppButton
|
||||
v-if="runStatus.isLikelyRunning"
|
||||
v-if="auth.isAdmin && runStatus.isLikelyRunning"
|
||||
variant="danger"
|
||||
:disabled="!activeRunId || isCancelAnimating"
|
||||
@click="onCancelRun"
|
||||
|
|
@ -420,6 +420,7 @@ watch(
|
|||
</span>
|
||||
</AppButton>
|
||||
<AppButton
|
||||
v-if="auth.isAdmin"
|
||||
:disabled="isStartBlocked"
|
||||
:title="startCheckDisabledReason || undefined"
|
||||
:class="isStartCheckAnimating ? 'shadow-[0_0_0_1px_var(--color-state-info-border)]' : ''"
|
||||
|
|
@ -485,21 +486,9 @@ watch(
|
|||
</RouterLink>
|
||||
</div>
|
||||
<p class="mt-2 text-sm text-secondary">
|
||||
Started {{ formatDate(displayedLatestRun.start_dt) }}.
|
||||
<template v-if="runStatus.isLikelyRunning">
|
||||
{{ runStatus.scholarProgress?.visited ?? 0 }} / {{ runStatus.scholarProgress?.total ?? '…' }} visited
|
||||
· {{ runStatus.scholarProgress?.finished ?? 0 }} finished
|
||||
· {{ displayedLatestRun.new_publication_count }} new publications.
|
||||
</template>
|
||||
<template v-else>
|
||||
Processed {{ displayedLatestRun.scholar_count }} scholars and discovered
|
||||
Started {{ formatDate(displayedLatestRun.start_dt) }}. Processed
|
||||
{{ displayedLatestRun.scholar_count }} scholars and discovered
|
||||
{{ displayedLatestRun.new_publication_count }} new publications.
|
||||
<template v-if="displayedLatestRun.failed_count > 0 || displayedLatestRun.partial_count > 0">
|
||||
<span class="text-state-warning-text">
|
||||
{{ displayedLatestRun.failed_count > 0 ? `${displayedLatestRun.failed_count} failed` : "" }}{{ displayedLatestRun.failed_count > 0 && displayedLatestRun.partial_count > 0 ? ", " : "" }}{{ displayedLatestRun.partial_count > 0 ? `${displayedLatestRun.partial_count} partial` : "" }}.
|
||||
</span>
|
||||
</template>
|
||||
</template>
|
||||
</p>
|
||||
</div>
|
||||
<AppEmptyState
|
||||
|
|
@ -532,9 +521,6 @@ watch(
|
|||
</div>
|
||||
<span class="text-xs text-secondary">
|
||||
{{ formatDate(run.start_dt) }} • {{ run.new_publication_count }} new
|
||||
<template v-if="run.failed_count > 0">
|
||||
• <span class="text-state-warning-text">{{ run.failed_count }} failed</span>
|
||||
</template>
|
||||
</span>
|
||||
</li>
|
||||
</ul>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<script setup lang="ts">
|
||||
import { computed, onMounted, ref, watch } from "vue";
|
||||
import { computed, onMounted, onUnmounted, ref, watch } from "vue";
|
||||
|
||||
import AppPage from "@/components/layout/AppPage.vue";
|
||||
import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
|
||||
|
|
@ -37,6 +37,23 @@ const selectedPublicationKeys = ref<Set<string>>(new Set());
|
|||
const retryingPublicationKeys = ref<Set<string>>(new Set());
|
||||
const favoriteUpdatingKeys = ref<Set<string>>(new Set());
|
||||
|
||||
const PUBLICATIONS_RUN_STATUS_SYNC_INTERVAL_MS = 5000;
|
||||
let runStatusSyncTimer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
function startRunStatusSyncLoop(): void {
|
||||
if (runStatusSyncTimer !== null) return;
|
||||
runStatusSyncTimer = setInterval(() => {
|
||||
if (runStatus.isRunActive) return;
|
||||
void runStatus.syncLatest();
|
||||
}, PUBLICATIONS_RUN_STATUS_SYNC_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function stopRunStatusSyncLoop(): void {
|
||||
if (runStatusSyncTimer === null) return;
|
||||
clearInterval(runStatusSyncTimer);
|
||||
runStatusSyncTimer = null;
|
||||
}
|
||||
|
||||
// --- Computed ---
|
||||
|
||||
const selectedCount = computed(() => selectedPublicationKeys.value.size);
|
||||
|
|
@ -341,8 +358,13 @@ async function onStartRun(): Promise<void> {
|
|||
|
||||
onMounted(() => {
|
||||
pub.syncFiltersFromRoute();
|
||||
startRunStatusSyncLoop();
|
||||
void Promise.all([pub.loadScholarFilters(), pub.loadPublications(), runStatus.syncLatest()]);
|
||||
});
|
||||
|
||||
onUnmounted(() => {
|
||||
stopRunStatusSyncLoop();
|
||||
});
|
||||
</script>
|
||||
|
||||
<template>
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue";
|
|||
import RequestStateAlerts from "@/components/patterns/RequestStateAlerts.vue";
|
||||
import AppButton from "@/components/ui/AppButton.vue";
|
||||
import AppCard from "@/components/ui/AppCard.vue";
|
||||
import AppConfirmModal from "@/components/ui/AppConfirmModal.vue";
|
||||
import AppEmptyState from "@/components/ui/AppEmptyState.vue";
|
||||
import AppHelpHint from "@/components/ui/AppHelpHint.vue";
|
||||
import AppInput from "@/components/ui/AppInput.vue";
|
||||
|
|
@ -28,12 +27,10 @@ import {
|
|||
type ScholarProfile,
|
||||
type ScholarSearchCandidate,
|
||||
} from "@/features/scholars";
|
||||
import { useScholarBulkActions } from "@/features/scholars/composables/useScholarBulkActions";
|
||||
import ScholarAvatar from "@/features/scholars/components/ScholarAvatar.vue";
|
||||
import ScholarBatchAdd from "@/features/scholars/components/ScholarBatchAdd.vue";
|
||||
import ScholarNameSearch from "@/features/scholars/components/ScholarNameSearch.vue";
|
||||
import ScholarSettingsModal from "@/features/scholars/components/ScholarSettingsModal.vue";
|
||||
import ScholarStatusBadges from "@/features/scholars/components/ScholarStatusBadges.vue";
|
||||
import { ApiRequestError } from "@/lib/api/errors";
|
||||
import { useRunStatusStore } from "@/stores/run_status";
|
||||
|
||||
|
|
@ -273,16 +270,11 @@ async function onToggleScholar(): Promise<void> {
|
|||
}
|
||||
}
|
||||
|
||||
function onDeleteScholar(): void {
|
||||
async function onDeleteScholar(): Promise<void> {
|
||||
const profile = activeScholarSettings.value;
|
||||
if (!profile) return;
|
||||
const label = scholarLabel(profile);
|
||||
bulk.requestConfirm(
|
||||
`Delete ${label}?`,
|
||||
"This removes all linked publications and queue data. This action cannot be undone.",
|
||||
"danger",
|
||||
async () => {
|
||||
bulk.dismissConfirm();
|
||||
if (!window.confirm(`Delete scholar ${label}? This removes all linked publications and queue data.`)) return;
|
||||
activeScholarId.value = profile.id;
|
||||
clearMessages();
|
||||
try {
|
||||
|
|
@ -295,49 +287,85 @@ function onDeleteScholar(): void {
|
|||
} finally {
|
||||
activeScholarId.value = null;
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function onSaveImageUrl(): Promise<void> {
|
||||
const p = activeScholarSettings.value;
|
||||
if (!p) return;
|
||||
const url = (imageUrlDraftByScholarId.value[p.id] || "").trim();
|
||||
if (!url) { errorMessage.value = "Enter an image URL before saving, or use Reset image."; return; }
|
||||
imageSavingScholarId.value = p.id;
|
||||
const profile = activeScholarSettings.value;
|
||||
if (!profile) return;
|
||||
const candidate = (imageUrlDraftByScholarId.value[profile.id] || "").trim();
|
||||
if (!candidate) { errorMessage.value = "Enter an image URL before saving, or use Reset image."; return; }
|
||||
imageSavingScholarId.value = profile.id;
|
||||
clearMessages();
|
||||
try { await setScholarImageUrl(p.id, url); successMessage.value = `Image URL updated for ${scholarLabel(p)}.`; await loadScholars(); }
|
||||
catch (e) { assignError(e, "Unable to update scholar image URL."); }
|
||||
finally { imageSavingScholarId.value = null; }
|
||||
try {
|
||||
await setScholarImageUrl(profile.id, candidate);
|
||||
successMessage.value = `Image URL updated for ${scholarLabel(profile)}.`;
|
||||
await loadScholars();
|
||||
} catch (error) {
|
||||
assignError(error, "Unable to update scholar image URL.");
|
||||
} finally {
|
||||
imageSavingScholarId.value = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function onUploadImage(event: Event): Promise<void> {
|
||||
const p = activeScholarSettings.value;
|
||||
if (!p) return;
|
||||
const profile = activeScholarSettings.value;
|
||||
if (!profile) return;
|
||||
const input = event.target as HTMLInputElement | null;
|
||||
const file = input?.files?.[0] ?? null;
|
||||
if (!file) return;
|
||||
imageUploadingScholarId.value = p.id;
|
||||
imageUploadingScholarId.value = profile.id;
|
||||
clearMessages();
|
||||
try { await uploadScholarImage(p.id, file); successMessage.value = `Uploaded image for ${scholarLabel(p)}.`; await loadScholars(); }
|
||||
catch (e) { assignError(e, "Unable to upload scholar image."); }
|
||||
finally { imageUploadingScholarId.value = null; if (input) input.value = ""; }
|
||||
try {
|
||||
await uploadScholarImage(profile.id, file);
|
||||
successMessage.value = `Uploaded image for ${scholarLabel(profile)}.`;
|
||||
await loadScholars();
|
||||
} catch (error) {
|
||||
assignError(error, "Unable to upload scholar image.");
|
||||
} finally {
|
||||
imageUploadingScholarId.value = null;
|
||||
if (input) input.value = "";
|
||||
}
|
||||
}
|
||||
|
||||
async function onResetImage(): Promise<void> {
|
||||
const p = activeScholarSettings.value;
|
||||
if (!p) return;
|
||||
imageSavingScholarId.value = p.id;
|
||||
const profile = activeScholarSettings.value;
|
||||
if (!profile) return;
|
||||
imageSavingScholarId.value = profile.id;
|
||||
clearMessages();
|
||||
try { await clearScholarImage(p.id); successMessage.value = `Image reset for ${scholarLabel(p)}.`; await loadScholars(); }
|
||||
catch (e) { assignError(e, "Unable to reset scholar image."); }
|
||||
finally { imageSavingScholarId.value = null; }
|
||||
try {
|
||||
await clearScholarImage(profile.id);
|
||||
successMessage.value = `Image reset for ${scholarLabel(profile)}.`;
|
||||
await loadScholars();
|
||||
} catch (error) {
|
||||
assignError(error, "Unable to reset scholar image.");
|
||||
} finally {
|
||||
imageSavingScholarId.value = null;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Import/export ---
|
||||
|
||||
function importSummary(r: DataImportResult): string {
|
||||
return `Import complete. Scholars +${r.scholars_created} / updated ${r.scholars_updated}; publications +${r.publications_created} / updated ${r.publications_updated}; links +${r.links_created} / updated ${r.links_updated}; skipped ${r.skipped_records}.`;
|
||||
function suggestExportFilename(exportedAt: string): string {
|
||||
return `scholarr-export-${exportedAt.slice(0, 10) || "unknown-date"}.json`;
|
||||
}
|
||||
|
||||
function downloadJsonFile(filename: string, payload: unknown): void {
|
||||
const blob = new Blob([JSON.stringify(payload, null, 2)], { type: "application/json" });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const anchor = document.createElement("a");
|
||||
anchor.href = url;
|
||||
anchor.download = filename;
|
||||
anchor.click();
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
function importSummary(result: DataImportResult): string {
|
||||
return (
|
||||
`Import complete. Scholars +${result.scholars_created}` +
|
||||
` / updated ${result.scholars_updated}; publications +${result.publications_created}` +
|
||||
` / updated ${result.publications_updated}; links +${result.links_created}` +
|
||||
` / updated ${result.links_updated}; skipped ${result.skipped_records}.`
|
||||
);
|
||||
}
|
||||
|
||||
async function onExportData(): Promise<void> {
|
||||
|
|
@ -345,13 +373,7 @@ async function onExportData(): Promise<void> {
|
|||
clearMessages();
|
||||
try {
|
||||
const payload = await exportScholarData();
|
||||
const blob = new Blob([JSON.stringify(payload, null, 2)], { type: "application/json" });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = `scholarr-export-${payload.exported_at.slice(0, 10) || "unknown-date"}.json`;
|
||||
a.click();
|
||||
URL.revokeObjectURL(url);
|
||||
downloadJsonFile(suggestExportFilename(payload.exported_at), payload);
|
||||
successMessage.value = "Export complete.";
|
||||
} catch (error) {
|
||||
assignError(error, "Unable to export scholars and publications.");
|
||||
|
|
@ -360,6 +382,10 @@ async function onExportData(): Promise<void> {
|
|||
}
|
||||
}
|
||||
|
||||
function onOpenImportPicker(): void {
|
||||
importFileInput.value?.click();
|
||||
}
|
||||
|
||||
async function onImportFileSelected(event: Event): Promise<void> {
|
||||
const input = event.target as HTMLInputElement | null;
|
||||
const file = input?.files?.[0] ?? null;
|
||||
|
|
@ -368,13 +394,12 @@ async function onImportFileSelected(event: Event): Promise<void> {
|
|||
clearMessages();
|
||||
try {
|
||||
const raw = await file.text();
|
||||
let parsed = JSON.parse(raw);
|
||||
if (parsed?.data && Array.isArray(parsed.data.scholars)) parsed = parsed.data;
|
||||
const payload = parsed as DataImportPayload;
|
||||
if (!payload || !Array.isArray(payload.scholars) || !Array.isArray(payload.publications)) {
|
||||
const parsed = JSON.parse(raw) as DataImportPayload;
|
||||
if (!parsed || !Array.isArray(parsed.scholars) || !Array.isArray(parsed.publications)) {
|
||||
throw new Error("Invalid import file: expected scholars[] and publications[] arrays.");
|
||||
}
|
||||
successMessage.value = importSummary(await importScholarData(payload));
|
||||
const result = await importScholarData(parsed);
|
||||
successMessage.value = importSummary(result);
|
||||
await loadScholars();
|
||||
} catch (error) {
|
||||
assignError(error, "Unable to import scholars and publications.");
|
||||
|
|
@ -384,15 +409,6 @@ async function onImportFileSelected(event: Event): Promise<void> {
|
|||
}
|
||||
}
|
||||
|
||||
// --- Bulk actions ---
|
||||
|
||||
const bulk = useScholarBulkActions(visibleScholars, {
|
||||
clearMessages,
|
||||
assignError,
|
||||
setSuccess: (msg: string) => { successMessage.value = msg; },
|
||||
reloadScholars: loadScholars,
|
||||
});
|
||||
|
||||
// --- Lifecycle ---
|
||||
|
||||
onMounted(() => { void loadScholars(); });
|
||||
|
|
@ -450,7 +466,7 @@ watch(
|
|||
<AppButton variant="secondary" :disabled="loading || exportingData" @click="onExportData">
|
||||
{{ exportingData ? "Exporting..." : "Export" }}
|
||||
</AppButton>
|
||||
<AppButton variant="secondary" :disabled="loading || importingData" @click="importFileInput?.click()">
|
||||
<AppButton variant="secondary" :disabled="loading || importingData" @click="onOpenImportPicker">
|
||||
{{ importingData ? "Importing..." : "Import" }}
|
||||
</AppButton>
|
||||
<AppRefreshButton variant="secondary" :disabled="saving" :loading="loading" title="Refresh scholars" loading-title="Refreshing scholars" @click="loadScholars" />
|
||||
|
|
@ -475,34 +491,6 @@ watch(
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex flex-wrap items-center justify-between gap-2 border-t border-stroke-default pt-2">
|
||||
<span class="text-xs text-secondary">
|
||||
{{ trackedCountLabel }}
|
||||
<template v-if="bulk.hasSelection.value"> · {{ bulk.selectedCount.value }} selected</template>
|
||||
</span>
|
||||
<div class="flex items-center gap-1">
|
||||
<label for="scholars-bulk-action" class="sr-only">Bulk action</label>
|
||||
<AppSelect
|
||||
id="scholars-bulk-action"
|
||||
v-model="bulk.bulkAction.value"
|
||||
:disabled="bulk.bulkBusy.value || loading"
|
||||
class="max-w-[14rem] !py-1.5 !text-xs"
|
||||
>
|
||||
<option v-for="option in bulk.bulkActionOptions.value" :key="option.value" :value="option.value">
|
||||
{{ option.label }}
|
||||
</option>
|
||||
</AppSelect>
|
||||
<AppButton
|
||||
variant="secondary"
|
||||
class="h-8 min-h-8 shrink-0 px-2 text-xs"
|
||||
:disabled="bulk.bulkApplyDisabled.value"
|
||||
@click="bulk.onApplyBulkAction"
|
||||
>
|
||||
{{ bulk.bulkApplyLabel.value }}
|
||||
</AppButton>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="min-h-0 flex-1 xl:overflow-hidden">
|
||||
<AsyncStateGate :loading="loading" :loading-lines="6" :empty="!hasTrackedScholars" :show-empty="!errorMessage" empty-title="No scholars tracked" empty-body="Add a Scholar ID or URL to start ingestion tracking.">
|
||||
<AppEmptyState v-if="!hasVisibleScholars" title="No scholars match this filter" body="Clear or adjust the filter to see tracked scholars." />
|
||||
|
|
@ -510,19 +498,9 @@ watch(
|
|||
<ul class="flex gap-3 overflow-x-auto p-1 lg:hidden">
|
||||
<li v-for="item in visibleScholars" :key="item.id" class="rounded-xl border border-stroke-default bg-surface-card-muted/70 p-3">
|
||||
<div class="flex items-start gap-3">
|
||||
<input
|
||||
type="checkbox"
|
||||
class="bulk-check mt-1 shrink-0"
|
||||
:checked="bulk.selectedIds.value.has(item.id)"
|
||||
:aria-label="`Select ${scholarLabel(item)}`"
|
||||
@change="bulk.onToggleRow(item.id, $event)"
|
||||
/>
|
||||
<ScholarAvatar :label="item.display_name" :scholar-id="item.scholar_id" :image-url="item.profile_image_url" />
|
||||
<div class="min-w-0 flex-1 space-y-1">
|
||||
<p class="truncate text-sm font-semibold text-ink-primary">
|
||||
{{ scholarLabel(item) }}
|
||||
<ScholarStatusBadges :baseline-completed="item.baseline_completed" :last-run-status="item.last_run_status" />
|
||||
</p>
|
||||
<p class="truncate text-sm font-semibold text-ink-primary">{{ scholarLabel(item) }}</p>
|
||||
<div class="flex flex-wrap items-center gap-3">
|
||||
<RouterLink :to="scholarPublicationsRoute(item)" class="link-inline text-xs">Publications</RouterLink>
|
||||
<a :href="scholarProfileUrl(item.scholar_id)" target="_blank" rel="noreferrer" class="link-inline text-xs">Open profile</a>
|
||||
|
|
@ -536,39 +514,17 @@ watch(
|
|||
<AppTable class="h-full overflow-y-scroll overscroll-contain" label="Tracked scholars table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col" class="w-10">
|
||||
<input
|
||||
type="checkbox"
|
||||
class="bulk-check"
|
||||
:checked="bulk.allVisibleSelected.value"
|
||||
:disabled="visibleScholars.length === 0"
|
||||
aria-label="Select all visible scholars"
|
||||
@change="bulk.onToggleAll"
|
||||
/>
|
||||
</th>
|
||||
<th scope="col">Scholar</th>
|
||||
<th scope="col" class="w-[11rem]">Manage</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="item in visibleScholars" :key="item.id">
|
||||
<td>
|
||||
<input
|
||||
type="checkbox"
|
||||
class="bulk-check"
|
||||
:checked="bulk.selectedIds.value.has(item.id)"
|
||||
:aria-label="`Select ${scholarLabel(item)}`"
|
||||
@change="bulk.onToggleRow(item.id, $event)"
|
||||
/>
|
||||
</td>
|
||||
<td>
|
||||
<div class="flex items-start gap-3">
|
||||
<ScholarAvatar :label="item.display_name" :scholar-id="item.scholar_id" :image-url="item.profile_image_url" />
|
||||
<div class="grid min-w-0 gap-1">
|
||||
<strong class="truncate text-ink-primary">
|
||||
{{ scholarLabel(item) }}
|
||||
<ScholarStatusBadges :baseline-completed="item.baseline_completed" :last-run-status="item.last_run_status" />
|
||||
</strong>
|
||||
<strong class="truncate text-ink-primary">{{ scholarLabel(item) }}</strong>
|
||||
<div class="flex flex-wrap items-center gap-3">
|
||||
<RouterLink :to="scholarPublicationsRoute(item)" class="link-inline text-xs">Publications</RouterLink>
|
||||
<a :href="scholarProfileUrl(item.scholar_id)" target="_blank" rel="noreferrer" class="link-inline text-xs">Open profile</a>
|
||||
|
|
@ -605,21 +561,5 @@ watch(
|
|||
@toggle="onToggleScholar"
|
||||
@delete="onDeleteScholar"
|
||||
/>
|
||||
|
||||
<AppConfirmModal
|
||||
:open="bulk.confirmState.value.open"
|
||||
:title="bulk.confirmState.value.title"
|
||||
:message="bulk.confirmState.value.message"
|
||||
:variant="bulk.confirmState.value.variant"
|
||||
confirm-label="Delete"
|
||||
@confirm="bulk.confirmState.value.onConfirm()"
|
||||
@cancel="bulk.dismissConfirm()"
|
||||
/>
|
||||
</AppPage>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.bulk-check {
|
||||
@apply h-4 w-4 rounded border-stroke-interactive bg-surface-input text-brand-600 focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:ring-offset-2 focus-visible:ring-offset-focus-offset;
|
||||
}
|
||||
</style>
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ const savingScholarHttp = ref(false);
|
|||
const updatingPassword = ref(false);
|
||||
|
||||
const autoRunEnabled = ref(false);
|
||||
const runIntervalHours = ref("1");
|
||||
const runIntervalMinutes = ref("60");
|
||||
const requestDelaySeconds = ref("2");
|
||||
const navVisiblePages = ref<string[]>([]);
|
||||
const openalexApiKey = ref("");
|
||||
|
|
@ -77,13 +77,13 @@ const tabItems = computed<AppTabItem[]>(() => {
|
|||
const tabs: AppTabItem[] = [
|
||||
{ id: TAB_CHECKING, label: "Checking" },
|
||||
{ id: TAB_ACCOUNT, label: "Account" },
|
||||
{ id: TAB_ADMIN_PDF, label: "PDF Queue" },
|
||||
];
|
||||
if (auth.isAdmin) {
|
||||
tabs.push(
|
||||
{ id: TAB_ADMIN_USERS, label: "Users" },
|
||||
{ id: TAB_ADMIN_INTEGRITY, label: "Integrity" },
|
||||
{ id: TAB_ADMIN_REPAIRS, label: "Repairs" },
|
||||
{ id: TAB_ADMIN_PDF, label: "PDF Queue" },
|
||||
{ id: TAB_ADMIN_INTEGRATIONS, label: "Integrations" },
|
||||
);
|
||||
}
|
||||
|
|
@ -138,7 +138,7 @@ function hydrateSettings(settings: UserSettings): void {
|
|||
manualRunAllowed.value = Boolean(settings.policy?.manual_run_allowed ?? true);
|
||||
|
||||
autoRunEnabled.value = Boolean(settings.auto_run_enabled) && automationAllowed.value;
|
||||
runIntervalHours.value = String(Number(settings.run_interval_minutes) / 60);
|
||||
runIntervalMinutes.value = String(settings.run_interval_minutes);
|
||||
requestDelaySeconds.value = String(settings.request_delay_seconds);
|
||||
navVisiblePages.value = normalizeUserNavVisiblePages(settings.nav_visible_pages);
|
||||
|
||||
|
|
@ -168,23 +168,6 @@ function parseBoundedInteger(value: string, label: string, minimum: number): num
|
|||
return parsed;
|
||||
}
|
||||
|
||||
function formatHours(minutes: number): string {
|
||||
const h = minutes / 60;
|
||||
return Number.isInteger(h) ? String(h) : h.toFixed(2).replace(/\.?0+$/, "");
|
||||
}
|
||||
|
||||
function parseHoursToMinutes(value: string, minMinutes: number): number {
|
||||
const hours = Number(value);
|
||||
if (!Number.isFinite(hours) || hours <= 0) {
|
||||
throw new Error("Check interval must be a positive number of hours.");
|
||||
}
|
||||
const minutes = Math.round(hours * 60);
|
||||
if (minutes < minMinutes) {
|
||||
throw new Error(`Check interval must be at least ${formatHours(minMinutes)} hours.`);
|
||||
}
|
||||
return minutes;
|
||||
}
|
||||
|
||||
async function loadSettings(): Promise<void> {
|
||||
loading.value = true;
|
||||
errorMessage.value = null;
|
||||
|
|
@ -242,8 +225,9 @@ async function onSaveSettings(): Promise<void> {
|
|||
try {
|
||||
const payload: UserSettingsUpdate = {
|
||||
auto_run_enabled: autoRunEnabled.value,
|
||||
run_interval_minutes: parseHoursToMinutes(
|
||||
runIntervalHours.value,
|
||||
run_interval_minutes: parseBoundedInteger(
|
||||
runIntervalMinutes.value,
|
||||
"Check interval (minutes)",
|
||||
minCheckIntervalMinutes.value,
|
||||
),
|
||||
request_delay_seconds: parseBoundedInteger(
|
||||
|
|
@ -373,11 +357,11 @@ onMounted(async () => {
|
|||
|
||||
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
||||
<span class="inline-flex items-center gap-1">
|
||||
Check interval (hours)
|
||||
Check interval (minutes)
|
||||
<AppHelpHint text="Minimum is controlled by server policy." />
|
||||
</span>
|
||||
<AppInput v-model="runIntervalHours" inputmode="decimal" />
|
||||
<span class="text-xs text-secondary">Minimum: {{ formatHours(minCheckIntervalMinutes) }} hours</span>
|
||||
<AppInput v-model="runIntervalMinutes" inputmode="numeric" />
|
||||
<span class="text-xs text-secondary">Minimum: {{ minCheckIntervalMinutes }}</span>
|
||||
</label>
|
||||
|
||||
<label class="grid gap-2 text-sm font-medium text-ink-secondary">
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import {
|
|||
import { type PublicationItem } from "@/features/publications";
|
||||
import { ApiRequestError } from "@/lib/api/errors";
|
||||
|
||||
export const RUN_STATUS_POLL_INTERVAL_MS = 15000;
|
||||
export const RUN_STATUS_POLL_INTERVAL_MS = 5000;
|
||||
export const RUN_STATUS_STARTING_PHASE_MS = 1500;
|
||||
|
||||
export type StartManualCheckResult =
|
||||
|
|
@ -168,7 +168,6 @@ export const useRunStatusStore = defineStore("runStatus", {
|
|||
lastErrorRequestId: null as string | null,
|
||||
lastSyncAt: null as number | null,
|
||||
livePublications: [] as Array<PublicationItem>,
|
||||
scholarProgress: null as { visited: number; finished: number; total: number } | null,
|
||||
}),
|
||||
getters: {
|
||||
isRunActive(state): boolean {
|
||||
|
|
@ -258,7 +257,6 @@ export const useRunStatusStore = defineStore("runStatus", {
|
|||
}
|
||||
activeStreamRunId = targetRunId;
|
||||
this.livePublications = [];
|
||||
this.scholarProgress = null;
|
||||
eventSource = new EventSource(`/api/v1/runs/${targetRunId}/stream`);
|
||||
eventSource.addEventListener("publication_discovered", (e) => {
|
||||
try {
|
||||
|
|
@ -314,19 +312,6 @@ export const useRunStatusStore = defineStore("runStatus", {
|
|||
console.error("Failed to parse SSE event", err);
|
||||
}
|
||||
});
|
||||
eventSource.addEventListener("scholar_progress", (e) => {
|
||||
try {
|
||||
const data = JSON.parse(e.data);
|
||||
const visited = typeof data.visited === "number" ? Math.max(0, Math.trunc(data.visited)) : null;
|
||||
const finished = typeof data.finished === "number" ? Math.max(0, Math.trunc(data.finished)) : null;
|
||||
const total = typeof data.total === "number" ? Math.max(1, Math.trunc(data.total)) : null;
|
||||
if (visited !== null && finished !== null && total !== null) {
|
||||
this.scholarProgress = { visited, finished, total };
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Failed to parse SSE event", err);
|
||||
}
|
||||
});
|
||||
eventSource.onerror = () => {
|
||||
// Reconnecting is handled automatically by EventSource,
|
||||
// but if it's permanently closed, we could do something here.
|
||||
|
|
@ -506,7 +491,6 @@ export const useRunStatusStore = defineStore("runStatus", {
|
|||
this.lastSyncAt = null;
|
||||
this.safetyState = createDefaultSafetyState();
|
||||
this.livePublications = [];
|
||||
this.scholarProgress = null;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,58 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
DC="docker compose -f docker-compose.yml -f docker-compose.dev.yml"
|
||||
|
||||
passed=0
|
||||
failed=0
|
||||
failures=()
|
||||
|
||||
step() {
|
||||
local label="$1"
|
||||
shift
|
||||
printf '\n\033[1;34m── %s\033[0m\n' "$label"
|
||||
if "$@"; then
|
||||
printf '\033[1;32m PASS\033[0m %s\n' "$label"
|
||||
((passed++)) || true
|
||||
else
|
||||
printf '\033[1;31m FAIL\033[0m %s\n' "$label"
|
||||
((failed++)) || true
|
||||
failures+=("$label")
|
||||
fi
|
||||
}
|
||||
|
||||
# ── repo hygiene (host-side, no runtime deps) ────────────────────────────────
|
||||
step "No generated artifacts" ./scripts/check_no_generated_artifacts.sh
|
||||
step "Env contract parity" python3 scripts/check_env_contract.py
|
||||
step "API contract drift" python3 scripts/check_frontend_api_contract.py
|
||||
|
||||
# ── backend lint + typecheck ─────────────────────────────────────────────────
|
||||
step "Ruff check" $DC run --rm app ruff check .
|
||||
step "Ruff format" $DC run --rm app ruff format --check .
|
||||
step "Mypy" $DC run --rm app mypy app/ --ignore-missing-imports
|
||||
|
||||
# ── backend tests ────────────────────────────────────────────────────────────
|
||||
step "Unit tests" $DC run --rm app python -m pytest tests/unit
|
||||
step "Integration tests" $DC run --rm app python -m pytest -m integration
|
||||
|
||||
# ── frontend ─────────────────────────────────────────────────────────────────
|
||||
step "Frontend theme tokens" $DC run --rm frontend npm run check:theme-tokens
|
||||
step "Frontend typecheck" $DC run --rm frontend npm run typecheck
|
||||
step "Frontend unit tests" $DC run --rm frontend npm run test:run
|
||||
step "Frontend build" $DC run --rm frontend npm run build
|
||||
|
||||
# ── docs ─────────────────────────────────────────────────────────────────────
|
||||
step "Docs build" npm --prefix docs/website run build
|
||||
|
||||
# ── summary ──────────────────────────────────────────────────────────────────
|
||||
printf '\n\033[1;34m── Summary ─────────────────────────────────────────────\033[0m\n'
|
||||
printf ' %s passed, %s failed\n' "$passed" "$failed"
|
||||
if ((failed > 0)); then
|
||||
printf '\n\033[1;31m Failed steps:\033[0m\n'
|
||||
for f in "${failures[@]}"; do
|
||||
printf ' - %s\n' "$f"
|
||||
done
|
||||
exit 1
|
||||
else
|
||||
printf '\n\033[1;32m All checks passed.\033[0m\n'
|
||||
fi
|
||||
|
|
@ -340,11 +340,10 @@ async def test_api_admin_dbops_forbidden_for_non_admin_and_validates_scope(db_se
|
|||
assert forbidden_integrity.status_code == 403
|
||||
assert forbidden_integrity.json()["error"]["code"] == "forbidden"
|
||||
|
||||
# PDF queue listing is accessible to all authenticated users (not admin-only).
|
||||
allowed_pdf_queue = client.get("/api/v1/admin/db/pdf-queue")
|
||||
assert allowed_pdf_queue.status_code == 200
|
||||
forbidden_pdf_queue = client.get("/api/v1/admin/db/pdf-queue")
|
||||
assert forbidden_pdf_queue.status_code == 403
|
||||
assert forbidden_pdf_queue.json()["error"]["code"] == "forbidden"
|
||||
|
||||
# But requeue actions still require admin.
|
||||
forbidden_requeue = client.post(
|
||||
"/api/v1/admin/db/pdf-queue/1/requeue",
|
||||
headers=headers,
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||
|
||||
from app.api.runtime_deps import get_scholar_source
|
||||
from app.main import app
|
||||
from app.services.scholar.rate_limit import reset_scholar_rate_limit_state_for_tests
|
||||
from app.services.scholar.source import FetchResult
|
||||
from app.settings import settings
|
||||
from tests.integration.helpers import (
|
||||
|
|
@ -70,110 +69,6 @@ async def test_api_scholars_crud_flow(db_session: AsyncSession) -> None:
|
|||
assert delete_response.json()["data"]["message"] == "Scholar deleted."
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_delete_scholar_returns_404_for_nonexistent(db_session: AsyncSession) -> None:
|
||||
await insert_user(
|
||||
db_session,
|
||||
email="api-delete-404@example.com",
|
||||
password="api-password",
|
||||
)
|
||||
|
||||
client = TestClient(app)
|
||||
login_user(client, email="api-delete-404@example.com", password="api-password")
|
||||
headers = api_csrf_headers(client)
|
||||
|
||||
response = client.delete("/api/v1/scholars/999999", headers=headers)
|
||||
assert response.status_code == 404
|
||||
assert response.json()["error"]["code"] == "scholar_not_found"
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_delete_scholar_cascades_linked_publications(db_session: AsyncSession) -> None:
|
||||
user_id = await insert_user(
|
||||
db_session,
|
||||
email="api-delete-cascade@example.com",
|
||||
password="api-password",
|
||||
)
|
||||
scholar_result = await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO scholar_profiles (user_id, scholar_id, display_name, is_enabled)
|
||||
VALUES (:user_id, :scholar_id, :display_name, true)
|
||||
RETURNING id
|
||||
"""
|
||||
),
|
||||
{"user_id": user_id, "scholar_id": "delCASCADE01", "display_name": "Cascade Test"},
|
||||
)
|
||||
scholar_profile_id = int(scholar_result.scalar_one())
|
||||
pub_result = await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO publications (fingerprint_sha256, title_raw, title_normalized, citation_count)
|
||||
VALUES (:fingerprint, :title_raw, :title_normalized, 0)
|
||||
RETURNING id
|
||||
"""
|
||||
),
|
||||
{
|
||||
"fingerprint": f"{(user_id + 7000):064x}",
|
||||
"title_raw": "Cascade Publication",
|
||||
"title_normalized": "cascadepublication",
|
||||
},
|
||||
)
|
||||
publication_id = int(pub_result.scalar_one())
|
||||
await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO scholar_publications (scholar_profile_id, publication_id, is_read)
|
||||
VALUES (:scholar_profile_id, :publication_id, false)
|
||||
"""
|
||||
),
|
||||
{"scholar_profile_id": scholar_profile_id, "publication_id": publication_id},
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
client = TestClient(app)
|
||||
login_user(client, email="api-delete-cascade@example.com", password="api-password")
|
||||
headers = api_csrf_headers(client)
|
||||
|
||||
delete_response = client.delete(f"/api/v1/scholars/{scholar_profile_id}", headers=headers)
|
||||
assert delete_response.status_code == 200
|
||||
assert delete_response.json()["data"]["message"] == "Scholar deleted."
|
||||
|
||||
link_result = await db_session.execute(
|
||||
text("SELECT count(*) FROM scholar_publications WHERE scholar_profile_id = :id"),
|
||||
{"id": scholar_profile_id},
|
||||
)
|
||||
assert int(link_result.scalar_one()) == 0
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_create_scholar_rejects_corrupted_id(db_session: AsyncSession) -> None:
|
||||
await insert_user(
|
||||
db_session,
|
||||
email="api-bad-id@example.com",
|
||||
password="api-password",
|
||||
)
|
||||
|
||||
client = TestClient(app)
|
||||
login_user(client, email="api-bad-id@example.com", password="api-password")
|
||||
headers = api_csrf_headers(client)
|
||||
|
||||
for bad_id in ["short", "ABCDEF12345!", "", " ", "AB CD EF1234"]:
|
||||
response = client.post(
|
||||
"/api/v1/scholars",
|
||||
json={"scholar_id": bad_id},
|
||||
headers=headers,
|
||||
)
|
||||
assert response.status_code == 400, f"Expected 400 for scholar_id={bad_id!r}"
|
||||
assert response.json()["error"]["code"] == "invalid_scholar"
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -246,7 +141,6 @@ async def test_api_scholars_search_and_profile_image_management(
|
|||
assert candidate["scholar_id"] == "abcDEF123456"
|
||||
assert candidate["profile_image_url"] == "https://scholar.google.com/citations/images/avatar_scholar_256.png"
|
||||
|
||||
reset_scholar_rate_limit_state_for_tests()
|
||||
create_response = client.post(
|
||||
"/api/v1/scholars",
|
||||
json={
|
||||
|
|
|
|||
|
|
@ -1,170 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.main import app
|
||||
from tests.integration.helpers import (
|
||||
api_csrf_headers,
|
||||
insert_user,
|
||||
login_user,
|
||||
)
|
||||
|
||||
|
||||
def _create_scholar(client: TestClient, headers: dict, scholar_id: str) -> int:
|
||||
resp = client.post("/api/v1/scholars", json={"scholar_id": scholar_id}, headers=headers)
|
||||
assert resp.status_code == 201
|
||||
return int(resp.json()["data"]["id"])
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_delete_with_valid_ids(db_session: AsyncSession) -> None:
|
||||
await insert_user(db_session, email="bulk@example.com", password="pw123456")
|
||||
client = TestClient(app)
|
||||
login_user(client, email="bulk@example.com", password="pw123456")
|
||||
headers = api_csrf_headers(client)
|
||||
|
||||
id1 = _create_scholar(client, headers, "aaaBBB111222")
|
||||
id2 = _create_scholar(client, headers, "cccDDD333444")
|
||||
|
||||
resp = client.post(
|
||||
"/api/v1/scholars/bulk-delete",
|
||||
json={"scholar_profile_ids": [id1, id2]},
|
||||
headers=headers,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["deleted_count"] == 2
|
||||
|
||||
list_resp = client.get("/api/v1/scholars")
|
||||
assert len(list_resp.json()["data"]["scholars"]) == 0
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_delete_only_deletes_own_scholars(db_session: AsyncSession) -> None:
|
||||
await insert_user(db_session, email="user1@example.com", password="pw123456")
|
||||
await insert_user(db_session, email="user2@example.com", password="pw123456")
|
||||
|
||||
client1 = TestClient(app)
|
||||
login_user(client1, email="user1@example.com", password="pw123456")
|
||||
headers1 = api_csrf_headers(client1)
|
||||
|
||||
client2 = TestClient(app)
|
||||
login_user(client2, email="user2@example.com", password="pw123456")
|
||||
headers2 = api_csrf_headers(client2)
|
||||
|
||||
id_user1 = _create_scholar(client1, headers1, "aaaBBB111222")
|
||||
id_user2 = _create_scholar(client2, headers2, "cccDDD333444")
|
||||
|
||||
# User1 tries to delete both — should only delete own
|
||||
resp = client1.post(
|
||||
"/api/v1/scholars/bulk-delete",
|
||||
json={"scholar_profile_ids": [id_user1, id_user2]},
|
||||
headers=headers1,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["deleted_count"] == 1
|
||||
|
||||
# User2's scholar still exists
|
||||
list_resp = client2.get("/api/v1/scholars")
|
||||
scholars = list_resp.json()["data"]["scholars"]
|
||||
assert len(scholars) == 1
|
||||
assert int(scholars[0]["id"]) == id_user2
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_toggle_enables_and_disables(db_session: AsyncSession) -> None:
|
||||
await insert_user(db_session, email="toggle@example.com", password="pw123456")
|
||||
client = TestClient(app)
|
||||
login_user(client, email="toggle@example.com", password="pw123456")
|
||||
headers = api_csrf_headers(client)
|
||||
|
||||
id1 = _create_scholar(client, headers, "aaaBBB111222")
|
||||
id2 = _create_scholar(client, headers, "cccDDD333444")
|
||||
|
||||
# Disable both
|
||||
resp = client.post(
|
||||
"/api/v1/scholars/bulk-toggle",
|
||||
json={"scholar_profile_ids": [id1, id2], "is_enabled": False},
|
||||
headers=headers,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["updated_count"] == 2
|
||||
|
||||
scholars = client.get("/api/v1/scholars").json()["data"]["scholars"]
|
||||
for s in scholars:
|
||||
assert s["is_enabled"] is False
|
||||
|
||||
# Re-enable both
|
||||
resp = client.post(
|
||||
"/api/v1/scholars/bulk-toggle",
|
||||
json={"scholar_profile_ids": [id1, id2], "is_enabled": True},
|
||||
headers=headers,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["data"]["updated_count"] == 2
|
||||
|
||||
scholars = client.get("/api/v1/scholars").json()["data"]["scholars"]
|
||||
for s in scholars:
|
||||
assert s["is_enabled"] is True
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
async def test_export_with_ids_filter(db_session: AsyncSession) -> None:
|
||||
await insert_user(db_session, email="export@example.com", password="pw123456")
|
||||
client = TestClient(app)
|
||||
login_user(client, email="export@example.com", password="pw123456")
|
||||
headers = api_csrf_headers(client)
|
||||
|
||||
id1 = _create_scholar(client, headers, "aaaBBB111222")
|
||||
_create_scholar(client, headers, "cccDDD333444")
|
||||
|
||||
# Export only id1
|
||||
resp = client.get(f"/api/v1/scholars/export?ids={id1}")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()["data"]
|
||||
assert len(data["scholars"]) == 1
|
||||
assert data["scholars"][0]["scholar_id"] == "aaaBBB111222"
|
||||
|
||||
# Export all (no filter)
|
||||
resp_all = client.get("/api/v1/scholars/export")
|
||||
assert resp_all.status_code == 200
|
||||
assert len(resp_all.json()["data"]["scholars"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_delete_rejects_without_csrf(db_session: AsyncSession) -> None:
|
||||
await insert_user(db_session, email="csrf-bulk-del@example.com", password="pw123456")
|
||||
client = TestClient(app)
|
||||
login_user(client, email="csrf-bulk-del@example.com", password="pw123456")
|
||||
response = client.post(
|
||||
"/api/v1/scholars/bulk-delete",
|
||||
json={"scholar_profile_ids": [1]},
|
||||
)
|
||||
assert response.status_code == 403
|
||||
assert response.json()["error"]["code"] == "csrf_invalid"
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_toggle_rejects_without_csrf(db_session: AsyncSession) -> None:
|
||||
await insert_user(db_session, email="csrf-bulk-tog@example.com", password="pw123456")
|
||||
client = TestClient(app)
|
||||
login_user(client, email="csrf-bulk-tog@example.com", password="pw123456")
|
||||
response = client.post(
|
||||
"/api/v1/scholars/bulk-toggle",
|
||||
json={"scholar_profile_ids": [1], "is_enabled": False},
|
||||
)
|
||||
assert response.status_code == 403
|
||||
assert response.json()["error"]["code"] == "csrf_invalid"
|
||||
|
|
@ -76,9 +76,6 @@ class FixtureScholarSource:
|
|||
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
|
||||
return await self.fetch_profile_page_html(scholar_id, cstart=0, pagesize=100)
|
||||
|
||||
async def fetch_author_search_html(self, query: str, *, start: int = 0) -> FetchResult:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
|
|
|
|||
|
|
@ -1,176 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.runtime_deps import get_scholar_source
|
||||
from app.main import app
|
||||
from app.services.scholar.source import FetchResult
|
||||
from tests.integration.helpers import (
|
||||
api_csrf_headers,
|
||||
insert_user,
|
||||
login_user,
|
||||
wait_for_run_complete,
|
||||
)
|
||||
|
||||
SCHOLAR_ID = "newPubDetc01"
|
||||
|
||||
|
||||
def _build_profile_html(publications: list[dict[str, str]]) -> str:
|
||||
rows = []
|
||||
for pub in publications:
|
||||
rows.append(
|
||||
f"""
|
||||
<tr class="gsc_a_tr">
|
||||
<td class="gsc_a_t">
|
||||
<a class="gsc_a_at"
|
||||
href="/citations?view_op=view_citation&citation_for_view={SCHOLAR_ID}:{pub["cluster"]}"
|
||||
>{pub["title"]}</a>
|
||||
<div class="gs_gray">{pub.get("authors", "A Author")}</div>
|
||||
<div class="gs_gray">{pub.get("venue", "Some Venue")}</div>
|
||||
</td>
|
||||
<td class="gsc_a_c"><a class="gsc_a_ac">{pub.get("citations", "0")}</a></td>
|
||||
<td class="gsc_a_y"><span class="gsc_a_h">{pub.get("year", "2024")}</span></td>
|
||||
</tr>"""
|
||||
)
|
||||
count = len(publications)
|
||||
return f"""
|
||||
<html>
|
||||
<head>
|
||||
<meta property="og:image" content="https://images.example.com/detect.png" />
|
||||
</head>
|
||||
<body>
|
||||
<div id="gsc_prf_in">New Pub Detector</div>
|
||||
<span id="gsc_a_nn">Articles 1-{count}</span>
|
||||
<table>
|
||||
<tbody id="gsc_a_b">{"".join(rows)}
|
||||
</tbody>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
INITIAL_PUBLICATIONS = [
|
||||
{"cluster": "aaa111", "title": "Existing Paper One", "year": "2023", "citations": "10"},
|
||||
{"cluster": "bbb222", "title": "Existing Paper Two", "year": "2022", "citations": "5"},
|
||||
]
|
||||
|
||||
UPDATED_PUBLICATIONS = [
|
||||
*INITIAL_PUBLICATIONS,
|
||||
{"cluster": "ccc333", "title": "Brand New Paper Three", "year": "2025", "citations": "0"},
|
||||
]
|
||||
|
||||
|
||||
class _MutableScholarSource:
|
||||
"""Source that serves different HTML per phase to simulate page changes."""
|
||||
|
||||
def __init__(self, initial_html: str) -> None:
|
||||
self.html = initial_html
|
||||
|
||||
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
|
||||
return self._result(scholar_id)
|
||||
|
||||
async def fetch_profile_page_html(
|
||||
self,
|
||||
scholar_id: str,
|
||||
*,
|
||||
cstart: int,
|
||||
pagesize: int,
|
||||
) -> FetchResult:
|
||||
return self._result(scholar_id)
|
||||
|
||||
def _result(self, scholar_id: str) -> FetchResult:
|
||||
return FetchResult(
|
||||
requested_url=f"https://scholar.google.com/citations?hl=en&user={scholar_id}",
|
||||
status_code=200,
|
||||
final_url=f"https://scholar.google.com/citations?hl=en&user={scholar_id}",
|
||||
body=self.html,
|
||||
error=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_publication_detected_after_page_change(
|
||||
db_session: AsyncSession,
|
||||
) -> None:
|
||||
"""Verify the fingerprint short-circuit lets new publications through.
|
||||
|
||||
Phase 1: Initial scrape discovers 2 publications.
|
||||
Phase 2: Same page → skipped via no_change fingerprint.
|
||||
Phase 3: Page gains a 3rd publication → fingerprint differs,
|
||||
scrape runs and discovers the new entry.
|
||||
"""
|
||||
await insert_user(db_session, email="new-pub-detect@example.com", password="api-password")
|
||||
|
||||
source = _MutableScholarSource(_build_profile_html(INITIAL_PUBLICATIONS))
|
||||
app.dependency_overrides[get_scholar_source] = lambda: source
|
||||
try:
|
||||
with TestClient(app) as client:
|
||||
login_user(client, email="new-pub-detect@example.com", password="api-password")
|
||||
headers = api_csrf_headers(client)
|
||||
|
||||
create_resp = client.post("/api/v1/scholars", json={"scholar_id": SCHOLAR_ID}, headers=headers)
|
||||
assert create_resp.status_code == 201
|
||||
|
||||
# ── Phase 1: initial scrape ─────────────────────────────
|
||||
run1 = client.post(
|
||||
"/api/v1/runs/manual",
|
||||
headers={**headers, "Idempotency-Key": "detect-run-001"},
|
||||
)
|
||||
assert run1.status_code == 200
|
||||
run1_id = int(run1.json()["data"]["run_id"])
|
||||
run1_data = await wait_for_run_complete(client, run1_id)
|
||||
|
||||
assert run1_data["scholar_results"][0]["outcome"] == "success"
|
||||
assert run1_data["scholar_results"][0]["publication_count"] == 2
|
||||
assert run1_data["scholar_results"][0].get("state_reason") != "no_change_initial_page_signature"
|
||||
|
||||
pubs1 = client.get("/api/v1/publications?mode=latest").json()["data"]
|
||||
assert pubs1["total_count"] == 2
|
||||
assert all(p["is_new_in_latest_run"] for p in pubs1["publications"])
|
||||
|
||||
# ── Phase 2: unchanged page → skipped ──────────────────
|
||||
run2 = client.post(
|
||||
"/api/v1/runs/manual",
|
||||
headers={**headers, "Idempotency-Key": "detect-run-002"},
|
||||
)
|
||||
assert run2.status_code == 200
|
||||
run2_id = int(run2.json()["data"]["run_id"])
|
||||
run2_data = await wait_for_run_complete(client, run2_id)
|
||||
|
||||
assert run2_data["scholar_results"][0]["state_reason"] == "no_change_initial_page_signature"
|
||||
assert run2_data["scholar_results"][0]["publication_count"] == 0
|
||||
|
||||
# ── Phase 3: new publication appears ────────────────────
|
||||
source.html = _build_profile_html(UPDATED_PUBLICATIONS)
|
||||
|
||||
run3 = client.post(
|
||||
"/api/v1/runs/manual",
|
||||
headers={**headers, "Idempotency-Key": "detect-run-003"},
|
||||
)
|
||||
assert run3.status_code == 200
|
||||
run3_id = int(run3.json()["data"]["run_id"])
|
||||
run3_data = await wait_for_run_complete(client, run3_id)
|
||||
|
||||
assert run3_data["scholar_results"][0]["outcome"] == "success"
|
||||
assert run3_data["scholar_results"][0].get("state_reason") != "no_change_initial_page_signature"
|
||||
assert run3_data["scholar_results"][0]["publication_count"] == 3
|
||||
|
||||
pubs3 = client.get("/api/v1/publications?mode=latest").json()["data"]
|
||||
assert pubs3["total_count"] == 3
|
||||
|
||||
titles = {p["title"] for p in pubs3["publications"]}
|
||||
assert "Brand New Paper Three" in titles
|
||||
|
||||
new_pub = next(p for p in pubs3["publications"] if p["title"] == "Brand New Paper Three")
|
||||
assert new_pub["is_new_in_latest_run"] is True
|
||||
|
||||
old_pubs = [p for p in pubs3["publications"] if p["title"] != "Brand New Paper Three"]
|
||||
for p in old_pubs:
|
||||
assert p["is_new_in_latest_run"] is False
|
||||
finally:
|
||||
app.dependency_overrides.pop(get_scholar_source, None)
|
||||
|
|
@ -30,12 +30,11 @@ class _PassthroughSource:
|
|||
error=None,
|
||||
)
|
||||
|
||||
async def fetch_profile_page_html(self, scholar_id: str, *, cstart: int, pagesize: int) -> FetchResult:
|
||||
async def fetch_profile_page_html(
|
||||
self, scholar_id: str, *, cstart: int, pagesize: int
|
||||
) -> FetchResult:
|
||||
return await self.fetch_profile_html(scholar_id)
|
||||
|
||||
async def fetch_author_search_html(self, query: str, *, start: int = 0) -> FetchResult:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _csrf_headers(client: TestClient) -> dict[str, str]:
|
||||
response = client.get("/api/v1/auth/me")
|
||||
|
|
@ -349,7 +348,6 @@ async def test_partial_discovery_exception_keeps_new_pub_count_consistent(
|
|||
)
|
||||
db_session.add_all([run, publication])
|
||||
await db_session.commit()
|
||||
run_id = int(run.id)
|
||||
|
||||
call_count = 0
|
||||
|
||||
|
|
@ -401,8 +399,9 @@ async def test_partial_discovery_exception_keeps_new_pub_count_consistent(
|
|||
publications=publications,
|
||||
)
|
||||
|
||||
refreshed_run = await db_session.get(CrawlRun, run_id)
|
||||
refreshed_run = await db_session.get(CrawlRun, int(run.id))
|
||||
assert refreshed_run is not None
|
||||
assert int(refreshed_run.new_pub_count) == 1
|
||||
link_count_result = await db_session.execute(
|
||||
text(
|
||||
"""
|
||||
|
|
@ -412,10 +411,9 @@ async def test_partial_discovery_exception_keeps_new_pub_count_consistent(
|
|||
AND first_seen_run_id = :run_id
|
||||
"""
|
||||
),
|
||||
{"scholar_profile_id": scholar_profile_id, "run_id": run_id},
|
||||
{"scholar_profile_id": scholar_profile_id, "run_id": int(run.id)},
|
||||
)
|
||||
link_count = int(link_count_result.scalar_one())
|
||||
assert int(refreshed_run.new_pub_count) == link_count
|
||||
assert int(link_count_result.scalar_one()) == 1
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
|
|
|
|||
|
|
@ -1,17 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patch_session_factory(db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch):
|
||||
"""Patch get_session_factory in arxiv modules to use the test's DB engine.
|
||||
|
||||
Without this, the implementation creates its own engine via get_session_factory()
|
||||
which may not share transaction visibility with the test's db_session fixture.
|
||||
"""
|
||||
factory = async_sessionmaker(db_session.bind, expire_on_commit=False)
|
||||
monkeypatch.setattr("app.services.arxiv.rate_limit.get_session_factory", lambda: factory)
|
||||
monkeypatch.setattr("app.services.arxiv.cache.get_session_factory", lambda: factory)
|
||||
return factory
|
||||
|
|
@ -147,8 +147,8 @@ async def test_client_coalesces_concurrent_identical_search_requests() -> None:
|
|||
async def test_client_logs_cache_hit_and_miss(
|
||||
db_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
patch_session_factory,
|
||||
) -> None:
|
||||
_ = db_session
|
||||
calls = {"count": 0}
|
||||
logged: list[dict[str, object]] = []
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -10,9 +10,7 @@ from app.services.arxiv.types import ArxivEntry, ArxivFeed, ArxivOpenSearchMeta
|
|||
from app.settings import settings
|
||||
|
||||
|
||||
def _item(*, title: str = "A Test Paper", scholar_label: str = "Ada Lovelace") -> Any:
|
||||
from types import SimpleNamespace
|
||||
|
||||
def _item(*, title: str = "A Test Paper", scholar_label: str = "Ada Lovelace") -> SimpleNamespace:
|
||||
return SimpleNamespace(title=title, scholar_label=scholar_label)
|
||||
|
||||
|
||||
|
|
@ -73,7 +71,7 @@ async def test_http_gateway_returns_none_when_disabled(monkeypatch: pytest.Monke
|
|||
fake_client = FakeClient()
|
||||
object.__setattr__(settings, "arxiv_enabled", False)
|
||||
try:
|
||||
gateway = arxiv_gateway.HttpArxivGateway(client=fake_client) # type: ignore[arg-type]
|
||||
gateway = arxiv_gateway.HttpArxivGateway(client=fake_client)
|
||||
result = await gateway.discover_arxiv_id_for_publication(item=_item())
|
||||
assert result is None
|
||||
assert fake_client.calls == 0
|
||||
|
|
@ -104,7 +102,7 @@ async def test_http_gateway_uses_client_search_for_discovery() -> None:
|
|||
)
|
||||
|
||||
fake_client = FakeClient()
|
||||
gateway = arxiv_gateway.HttpArxivGateway(client=fake_client) # type: ignore[arg-type]
|
||||
gateway = arxiv_gateway.HttpArxivGateway(client=fake_client)
|
||||
result = await gateway.discover_arxiv_id_for_publication(
|
||||
item=_item(title="My Paper", scholar_label="Ada Lovelace"),
|
||||
request_email="user@example.com",
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from app.settings import settings
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arxiv_rate_limit_respects_cooldown(db_session: AsyncSession, patch_session_factory) -> None:
|
||||
async def test_arxiv_rate_limit_respects_cooldown(db_session: AsyncSession) -> None:
|
||||
db_session.add(
|
||||
ArxivRuntimeState(
|
||||
state_key=ARXIV_RUNTIME_STATE_KEY,
|
||||
|
|
@ -37,7 +37,7 @@ async def test_arxiv_rate_limit_respects_cooldown(db_session: AsyncSession, patc
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arxiv_rate_limit_persists_cooldown_after_429(db_session: AsyncSession, patch_session_factory) -> None:
|
||||
async def test_arxiv_rate_limit_persists_cooldown_after_429(db_session: AsyncSession) -> None:
|
||||
previous_interval = settings.arxiv_min_interval_seconds
|
||||
previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds
|
||||
object.__setattr__(settings, "arxiv_min_interval_seconds", 0.0)
|
||||
|
|
@ -62,7 +62,7 @@ async def test_arxiv_rate_limit_persists_cooldown_after_429(db_session: AsyncSes
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arxiv_rate_limit_serializes_concurrent_calls(db_session: AsyncSession, patch_session_factory) -> None:
|
||||
async def test_arxiv_rate_limit_serializes_concurrent_calls(db_session: AsyncSession) -> None:
|
||||
previous_interval = settings.arxiv_min_interval_seconds
|
||||
previous_cooldown = settings.arxiv_rate_limit_cooldown_seconds
|
||||
object.__setattr__(settings, "arxiv_min_interval_seconds", 0.2)
|
||||
|
|
@ -88,9 +88,7 @@ async def test_arxiv_rate_limit_serializes_concurrent_calls(db_session: AsyncSes
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arxiv_rate_limit_logs_request_scheduled_and_completed(
|
||||
db_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
patch_session_factory,
|
||||
) -> None:
|
||||
logged: list[dict[str, object]] = []
|
||||
|
||||
|
|
@ -108,16 +106,14 @@ async def test_arxiv_rate_limit_logs_request_scheduled_and_completed(
|
|||
|
||||
assert scheduled
|
||||
assert completed
|
||||
assert float(str(scheduled[0]["wait_seconds"])) >= 0.0
|
||||
assert int(str(completed[0]["status_code"])) == 200
|
||||
assert float(scheduled[0]["wait_seconds"]) >= 0.0
|
||||
assert int(completed[0]["status_code"]) == 200
|
||||
assert completed[0]["source_path"] == "search"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arxiv_rate_limit_logs_cooldown_activation(
|
||||
db_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
patch_session_factory,
|
||||
) -> None:
|
||||
logged_warning: list[dict[str, object]] = []
|
||||
previous_interval = settings.arxiv_min_interval_seconds
|
||||
|
|
@ -142,11 +138,11 @@ async def test_arxiv_rate_limit_logs_cooldown_activation(
|
|||
cooldown_events = [entry for entry in logged_warning if entry.get("event") == "arxiv.cooldown_activated"]
|
||||
assert cooldown_events
|
||||
assert cooldown_events[0]["source_path"] == "lookup_ids"
|
||||
assert float(str(cooldown_events[0]["cooldown_remaining_seconds"])) > 0.0
|
||||
assert float(cooldown_events[0]["cooldown_remaining_seconds"]) > 0.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_arxiv_cooldown_status_reads_active_cooldown(db_session: AsyncSession, patch_session_factory) -> None:
|
||||
async def test_get_arxiv_cooldown_status_reads_active_cooldown(db_session: AsyncSession) -> None:
|
||||
now_utc = datetime(2026, 2, 26, 13, 0, tzinfo=UTC)
|
||||
existing = await db_session.get(ArxivRuntimeState, ARXIV_RUNTIME_STATE_KEY)
|
||||
if existing is None:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -39,8 +38,8 @@ async def test_discover_identifiers_for_enrichment_disables_arxiv_on_rate_limit(
|
|||
monkeypatch.setattr(runner, "_publish_identifier_update_event", _publish_noop)
|
||||
|
||||
result = await runner._discover_identifiers_for_enrichment(
|
||||
cast(Any, object()),
|
||||
publication=cast(Any, publication),
|
||||
object(),
|
||||
publication=publication,
|
||||
run_id=321,
|
||||
allow_arxiv_lookup=True,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.services.portability.publication_import import (
|
||||
|
|
@ -10,14 +9,14 @@ from app.services.portability.publication_import import (
|
|||
)
|
||||
|
||||
|
||||
def _mock_profile(scholar_id: str = "ABC123DEF456") -> Any:
|
||||
def _mock_profile(scholar_id: str = "ABC123DEF456") -> MagicMock:
|
||||
profile = MagicMock()
|
||||
profile.id = 1
|
||||
profile.scholar_id = scholar_id
|
||||
return profile
|
||||
|
||||
|
||||
def _scholar_map(scholar_id: str = "ABC123DEF456") -> dict[str, Any]:
|
||||
def _scholar_map(scholar_id: str = "ABC123DEF456") -> dict[str, MagicMock]:
|
||||
return {scholar_id: _mock_profile(scholar_id)}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -111,9 +111,8 @@ class TestBuildFingerprint:
|
|||
assert all(c in "0123456789abcdef" for c in fp)
|
||||
|
||||
def test_deterministic(self) -> None:
|
||||
fp_a = _build_fingerprint(title="Test Title", year=2024, author_text="Smith", venue_text="ICML")
|
||||
fp_b = _build_fingerprint(title="Test Title", year=2024, author_text="Smith", venue_text="ICML")
|
||||
assert fp_a == fp_b
|
||||
kwargs = {"title": "Test Title", "year": 2024, "author_text": "Smith", "venue_text": "ICML"}
|
||||
assert _build_fingerprint(**kwargs) == _build_fingerprint(**kwargs)
|
||||
|
||||
def test_different_titles_produce_different_fingerprints(self) -> None:
|
||||
fp1 = _build_fingerprint(title="Title A", year=2024, author_text=None, venue_text=None)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -29,7 +27,7 @@ def _job(
|
|||
|
||||
def _row(
|
||||
*, pub_url: str | None = "https://scholar.google.com/citations?view_op=view_citation&citation_for_view=abc:xyz"
|
||||
) -> Any:
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
publication_id=1,
|
||||
scholar_profile_id=1,
|
||||
|
|
@ -237,12 +235,10 @@ async def test_run_resolution_task_disables_arxiv_for_remaining_batch(
|
|||
) -> None:
|
||||
calls: list[tuple[int, bool]] = []
|
||||
first = _row()
|
||||
second: Any = SimpleNamespace(**{**first.__dict__, "publication_id": 2})
|
||||
second = SimpleNamespace(**{**first.__dict__, "publication_id": 2})
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _raise_background_session_error():
|
||||
def _raise_session_factory_error():
|
||||
raise RuntimeError("skip user settings lookup in test")
|
||||
yield # pragma: no cover
|
||||
|
||||
async def _fake_resolve_publication_row(
|
||||
*,
|
||||
|
|
@ -256,7 +252,7 @@ async def test_run_resolution_task_disables_arxiv_for_remaining_batch(
|
|||
calls.append((int(row.publication_id), bool(allow_arxiv_lookup)))
|
||||
return row.publication_id == 1
|
||||
|
||||
monkeypatch.setattr(pdf_queue_resolution, "background_session", _raise_background_session_error)
|
||||
monkeypatch.setattr(pdf_queue_resolution, "get_session_factory", _raise_session_factory_error)
|
||||
monkeypatch.setattr(pdf_queue_resolution, "_resolve_publication_row", _fake_resolve_publication_row)
|
||||
|
||||
await pdf_queue_resolution._run_resolution_task(
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ from __future__ import annotations
|
|||
|
||||
from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -12,7 +11,7 @@ from app.services.publications import pdf_resolution_pipeline as pipeline
|
|||
from app.services.unpaywall.application import OaResolutionOutcome
|
||||
|
||||
|
||||
def _row(*, display_identifier: DisplayIdentifier | None = None) -> Any:
|
||||
def _row(*, display_identifier: DisplayIdentifier | None = None) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
publication_id=1,
|
||||
scholar_profile_id=1,
|
||||
|
|
|
|||
|
|
@ -23,12 +23,6 @@ class StubScholarSource:
|
|||
index = min(self.calls - 1, len(self._fetch_results) - 1)
|
||||
return self._fetch_results[index]
|
||||
|
||||
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def fetch_profile_page_html(self, scholar_id: str, *, cstart: int, pagesize: int) -> FetchResult:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _ok_author_search_fetch() -> FetchResult:
|
||||
body = (
|
||||
|
|
|
|||
|
|
@ -42,34 +42,6 @@ class TestValidateScholarId:
|
|||
with pytest.raises(ScholarServiceError, match="12"):
|
||||
validate_scholar_id("ABCDEF12345!")
|
||||
|
||||
def test_rejects_empty_string(self) -> None:
|
||||
with pytest.raises(ScholarServiceError, match="12"):
|
||||
validate_scholar_id("")
|
||||
|
||||
def test_rejects_whitespace_only(self) -> None:
|
||||
with pytest.raises(ScholarServiceError, match="12"):
|
||||
validate_scholar_id(" ")
|
||||
|
||||
def test_rejects_embedded_spaces(self) -> None:
|
||||
with pytest.raises(ScholarServiceError, match="12"):
|
||||
validate_scholar_id("ABCDEF 12345")
|
||||
|
||||
def test_rejects_tab_characters(self) -> None:
|
||||
with pytest.raises(ScholarServiceError, match="12"):
|
||||
validate_scholar_id("ABCDEF\t12345")
|
||||
|
||||
def test_rejects_url_as_id(self) -> None:
|
||||
with pytest.raises(ScholarServiceError, match="12"):
|
||||
validate_scholar_id("https://scholar.google.com/citations?user=ABCDEF123456")
|
||||
|
||||
def test_rejects_newline_in_id(self) -> None:
|
||||
with pytest.raises(ScholarServiceError, match="12"):
|
||||
validate_scholar_id("ABCDEF\n12345")
|
||||
|
||||
def test_rejects_unicode_characters(self) -> None:
|
||||
with pytest.raises(ScholarServiceError, match="12"):
|
||||
validate_scholar_id("ABCDEF12345\u00e9")
|
||||
|
||||
|
||||
class TestNormalizeDisplayName:
|
||||
def test_returns_stripped_name(self) -> None:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -43,9 +42,9 @@ async def test_create_hydration_skips_when_global_throttle_active(
|
|||
)
|
||||
|
||||
result = await scholar_helpers.hydrate_scholar_metadata_if_needed(
|
||||
db_session=cast(Any, None),
|
||||
db_session=None,
|
||||
profile=profile,
|
||||
source=cast(Any, object()),
|
||||
source=object(),
|
||||
user_id=7,
|
||||
)
|
||||
|
||||
|
|
@ -78,9 +77,9 @@ async def test_create_hydration_runs_when_throttle_is_clear(
|
|||
)
|
||||
|
||||
result = await scholar_helpers.hydrate_scholar_metadata_if_needed(
|
||||
db_session=cast(Any, None),
|
||||
db_session=None,
|
||||
profile=profile,
|
||||
source=cast(Any, object()),
|
||||
source=object(),
|
||||
user_id=8,
|
||||
)
|
||||
|
||||
|
|
@ -110,7 +109,7 @@ async def test_initial_scrape_job_enqueued_on_create(
|
|||
)
|
||||
|
||||
queued = await scholar_helpers.enqueue_initial_scrape_job_for_scholar(
|
||||
cast(Any, session),
|
||||
session,
|
||||
profile=profile,
|
||||
user_id=9,
|
||||
)
|
||||
|
|
@ -135,7 +134,7 @@ async def test_initial_scrape_job_not_enqueued_when_disabled(
|
|||
)
|
||||
|
||||
queued = await scholar_helpers.enqueue_initial_scrape_job_for_scholar(
|
||||
cast(Any, session),
|
||||
session,
|
||||
profile=profile,
|
||||
user_id=11,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue