diff --git a/app/api/routers/admin_dbops.py b/app/api/routers/admin_dbops.py index 0f4d1f6..048be70 100644 --- a/app/api/routers/admin_dbops.py +++ b/app/api/routers/admin_dbops.py @@ -5,7 +5,7 @@ import logging from fastapi import APIRouter, Depends, Path, Query, Request from sqlalchemy.ext.asyncio import AsyncSession -from app.api.deps import get_api_admin_user +from app.api.deps import get_api_admin_user, get_api_current_user from app.api.errors import ApiException from app.api.responses import success_payload from app.api.schemas import ( @@ -57,7 +57,7 @@ def _requested_by_value(*, payload, admin_user: User) -> str: return from_payload or admin_user.email -def _serialize_pdf_queue_item(item) -> dict[str, object]: +def _serialize_pdf_queue_item(item, *, is_admin: bool = False) -> dict[str, object]: return { "publication_id": item.publication_id, "title": item.title, @@ -69,7 +69,7 @@ def _serialize_pdf_queue_item(item) -> dict[str, object]: "last_failure_detail": item.last_failure_detail, "last_source": item.last_source, "requested_by_user_id": item.requested_by_user_id, - "requested_by_email": item.requested_by_email, + "requested_by_email": item.requested_by_email if is_admin else None, "queued_at": item.queued_at, "last_attempt_at": item.last_attempt_at, "resolved_at": item.resolved_at, @@ -185,7 +185,7 @@ async def get_pdf_queue( offset: int | None = Query(default=None, ge=0), status: str | None = Query(default=None), db_session: AsyncSession = Depends(get_db_session), - admin_user: User = Depends(get_api_admin_user), + current_user: User = Depends(get_api_current_user), ): normalized_status = (status or "").strip().lower() or None resolved_page, resolved_limit, resolved_offset = _resolve_pdf_queue_paging( @@ -204,7 +204,7 @@ async def get_pdf_queue( logger, "info", "api.admin.db.pdf_queue_listed", - admin_user_id=int(admin_user.id), + user_id=int(current_user.id), page=int(resolved_page), page_size=int(resolved_limit), offset=int(resolved_offset), @@ -215,7 +215,7 @@ async def get_pdf_queue( return success_payload( request, data={ - "items": [_serialize_pdf_queue_item(item) for item in queue_page.items], + "items": [_serialize_pdf_queue_item(item, is_admin=current_user.is_admin) for item in queue_page.items], **_pdf_queue_page_data( total_count=queue_page.total_count, page=resolved_page, diff --git a/app/api/routers/scholars.py b/app/api/routers/scholars.py index 7bbc642..2a0c276 100644 --- a/app/api/routers/scholars.py +++ b/app/api/routers/scholars.py @@ -23,6 +23,9 @@ from app.api.schemas import ( DataImportEnvelope, DataImportRequest, MessageEnvelope, + ScholarBulkCountEnvelope, + ScholarBulkIdsRequest, + ScholarBulkToggleRequest, ScholarCreateRequest, ScholarEnvelope, ScholarImageUrlUpdateRequest, @@ -42,6 +45,22 @@ logger = logging.getLogger(__name__) router = APIRouter(prefix="/scholars", tags=["api-scholars"]) +def _parse_ids_param(ids: str | None) -> list[int] | None: + if not ids: + return None + parts = [p.strip() for p in ids.split(",") if p.strip()] + if not parts: + return None + try: + return [int(p) for p in parts] + except ValueError as exc: + raise ApiException( + status_code=400, + code="invalid_ids_param", + message="The 'ids' parameter must be a comma-separated list of integers.", + ) from exc + + @router.get( "", response_model=ScholarsListEnvelope, @@ -69,16 +88,81 @@ async def list_scholars( ) async def export_scholars_and_publications( request: Request, + ids: str | None = Query(None, description="Comma-separated scholar profile IDs to export"), db_session: AsyncSession = Depends(get_db_session), current_user: User = Depends(get_api_current_user), ): + scholar_profile_ids = _parse_ids_param(ids) data = await import_export_service.export_user_data( db_session, user_id=current_user.id, + scholar_profile_ids=scholar_profile_ids, ) return success_payload(request, data=data) +@router.post( + "/bulk-delete", + response_model=ScholarBulkCountEnvelope, +) +async def bulk_delete_scholars( + payload: ScholarBulkIdsRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + try: + deleted_count = await scholar_service.bulk_delete_scholars( + db_session, + user_id=current_user.id, + scholar_profile_ids=payload.scholar_profile_ids, + upload_dir=settings.scholar_image_upload_dir, + ) + except scholar_service.ScholarServiceError as exc: + raise ApiException( + status_code=409, + code="scholar_bulk_delete_failed", + message=str(exc), + ) from exc + structured_log( + logger, + "info", + "scholars.bulk_delete", + user_id=current_user.id, + requested_ids=payload.scholar_profile_ids, + deleted_count=deleted_count, + ) + return success_payload(request, data={"deleted_count": deleted_count, "updated_count": 0}) + + +@router.post( + "/bulk-toggle", + response_model=ScholarBulkCountEnvelope, +) +async def bulk_toggle_scholars( + payload: ScholarBulkToggleRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + updated_count = await scholar_service.bulk_toggle_scholars( + db_session, + user_id=current_user.id, + scholar_profile_ids=payload.scholar_profile_ids, + is_enabled=payload.is_enabled, + ) + structured_log( + logger, + "info", + "scholars.bulk_toggle", + user_id=current_user.id, + requested_ids=payload.scholar_profile_ids, + is_enabled=payload.is_enabled, + updated_count=updated_count, + ) + return success_payload(request, data={"deleted_count": 0, "updated_count": updated_count}) + + @router.post( "/import", response_model=DataImportEnvelope, @@ -261,11 +345,18 @@ async def delete_scholar( code="scholar_not_found", message="Scholar not found.", ) - await scholar_service.delete_scholar( - db_session, - profile=profile, - upload_dir=settings.scholar_image_upload_dir, - ) + try: + await scholar_service.delete_scholar( + db_session, + profile=profile, + upload_dir=settings.scholar_image_upload_dir, + ) + except scholar_service.ScholarServiceError as exc: + raise ApiException( + status_code=409, + code="scholar_delete_failed", + message=str(exc), + ) from exc structured_log( logger, "info", "api.scholars.deleted", user_id=current_user.id, scholar_profile_id=scholar_profile_id ) diff --git a/app/api/schemas/scholars.py b/app/api/schemas/scholars.py index 91cc459..bc1035d 100644 --- a/app/api/schemas/scholars.py +++ b/app/api/schemas/scholars.py @@ -126,6 +126,33 @@ class DataExportEnvelope(BaseModel): model_config = ConfigDict(extra="forbid") +class ScholarBulkIdsRequest(BaseModel): + scholar_profile_ids: list[int] = Field(..., min_length=1, max_length=500) + + model_config = ConfigDict(extra="forbid") + + +class ScholarBulkToggleRequest(BaseModel): + scholar_profile_ids: list[int] = Field(..., min_length=1, max_length=500) + is_enabled: bool + + model_config = ConfigDict(extra="forbid") + + +class ScholarBulkCountData(BaseModel): + deleted_count: int = 0 + updated_count: int = 0 + + model_config = ConfigDict(extra="forbid") + + +class ScholarBulkCountEnvelope(BaseModel): + data: ScholarBulkCountData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + class DataImportRequest(BaseModel): schema_version: int | None = None exported_at: str | None = None diff --git a/app/services/ingestion/scholar_processing.py b/app/services/ingestion/scholar_processing.py index f41d904..1735915 100644 --- a/app/services/ingestion/scholar_processing.py +++ b/app/services/ingestion/scholar_processing.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import logging import random +from collections.abc import Callable, Coroutine from datetime import UTC, datetime from typing import Any @@ -274,6 +275,7 @@ async def _run_first_pass( request_delay_seconds: int, queue_delay_seconds: int, progress: RunProgress, + on_progress: Callable[[int, int], Coroutine[Any, Any, None]] | None = None, ) -> dict[int, int]: first_pass_cstarts: dict[int, int] = {} for index, scholar in enumerate(scholars): @@ -310,6 +312,8 @@ async def _run_first_pass( scholars_remaining=len(scholars) - index - 1, ) return first_pass_cstarts + if on_progress is not None: + await on_progress(1, 0) resume_cstart = outcome.result_entry.get("continuation_cstart") if resume_cstart is not None and int(resume_cstart) > start_cstart: first_pass_cstarts[int(scholar.id)] = int(resume_cstart) @@ -330,6 +334,7 @@ async def _run_depth_pass( auto_queue_continuations: bool, queue_delay_seconds: int, progress: RunProgress, + on_progress: Callable[[int, int], Coroutine[Any, Any, None]] | None = None, ) -> None: for index, scholar in enumerate(scholars): resume_cstart = first_pass_cstarts.get(int(scholar.id)) @@ -367,6 +372,8 @@ async def _run_depth_pass( scholars_remaining=len(scholars) - index - 1, ) break + if on_progress is not None: + await on_progress(0, 1) async def run_scholar_iteration( @@ -387,6 +394,8 @@ async def run_scholar_iteration( auto_queue_continuations: bool, queue_delay_seconds: int, ) -> RunProgress: + from app.services.runs.events import run_events + progress = RunProgress() scholar_kwargs: dict[str, Any] = { "request_delay_seconds": request_delay_seconds, @@ -396,6 +405,22 @@ async def run_scholar_iteration( "rate_limit_backoff_seconds": rate_limit_backoff_seconds, "page_size": page_size, } + + visited = 0 + finished = 0 + total = len(scholars) + + async def _emit(v: int = 0, f: int = 0) -> None: + nonlocal visited, finished + visited += v + finished += f + await run_events.publish( + run.id, + "scholar_progress", + {"visited": visited, "finished": finished, "total": total}, + ) + + await _emit() first_pass_cstarts = await _run_first_pass( db_session, scholars=scholars, @@ -407,8 +432,12 @@ async def run_scholar_iteration( request_delay_seconds=request_delay_seconds, queue_delay_seconds=queue_delay_seconds, progress=progress, + on_progress=_emit, ) remaining_max = max(max_pages_per_scholar - 1, 0) + scholars_finished_in_first_pass = len(scholars) - len(first_pass_cstarts) + if scholars_finished_in_first_pass > 0: + await _emit(f=scholars_finished_in_first_pass) if remaining_max <= 0: return progress await _run_depth_pass( @@ -424,5 +453,6 @@ async def run_scholar_iteration( auto_queue_continuations=auto_queue_continuations, queue_delay_seconds=queue_delay_seconds, progress=progress, + on_progress=_emit, ) return progress diff --git a/app/services/portability/exporting.py b/app/services/portability/exporting.py index 0589498..a11eaef 100644 --- a/app/services/portability/exporting.py +++ b/app/services/portability/exporting.py @@ -56,11 +56,14 @@ async def export_user_data( db_session: AsyncSession, *, user_id: int, + scholar_profile_ids: list[int] | None = None, ) -> dict[str, Any]: - scholars_result = await db_session.execute( - select(ScholarProfile).where(ScholarProfile.user_id == user_id).order_by(ScholarProfile.id.asc()) - ) - publication_result = await db_session.execute( + scholar_query = select(ScholarProfile).where(ScholarProfile.user_id == user_id) + if scholar_profile_ids: + scholar_query = scholar_query.where(ScholarProfile.id.in_(scholar_profile_ids)) + scholars_result = await db_session.execute(scholar_query.order_by(ScholarProfile.id.asc())) + + pub_query = ( select( ScholarProfile.scholar_id, Publication.cluster_id, @@ -77,8 +80,13 @@ async def export_user_data( .join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id) .join(Publication, Publication.id == ScholarPublication.publication_id) .where(ScholarProfile.user_id == user_id) - .order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) ) + if scholar_profile_ids: + pub_query = pub_query.where(ScholarProfile.id.in_(scholar_profile_ids)) + publication_result = await db_session.execute( + pub_query.order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) + ) + scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()] publications = [_serialize_export_publication(row) for row in publication_result.all()] return { diff --git a/app/services/scholars/application.py b/app/services/scholars/application.py index 028e3fc..3496732 100644 --- a/app/services/scholars/application.py +++ b/app/services/scholars/application.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +from typing import Any from uuid import uuid4 from sqlalchemy.exc import IntegrityError @@ -27,10 +28,66 @@ from app.services.scholars.validators import ( validate_scholar_id, ) + +async def bulk_delete_scholars( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_ids: list[int], + upload_dir: str | None = None, +) -> int: + from sqlalchemy import select + + result = await db_session.execute( + select(ScholarProfile).where( + ScholarProfile.id.in_(scholar_profile_ids), + ScholarProfile.user_id == user_id, + ) + ) + profiles = list(result.scalars().all()) + if not profiles: + return 0 + if upload_dir: + upload_root = _ensure_upload_root(upload_dir, create=True) + for profile in profiles: + _safe_remove_upload(upload_root, profile.profile_image_upload_path) + for profile in profiles: + await db_session.delete(profile) + try: + await db_session.commit() + except IntegrityError as exc: + await db_session.rollback() + raise ScholarServiceError("Unable to bulk-delete scholars due to a database constraint.") from exc + return len(profiles) + + +async def bulk_toggle_scholars( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_ids: list[int], + is_enabled: bool, +) -> int: + from sqlalchemy import CursorResult, update + + cursor: CursorResult[Any] = await db_session.execute( # type: ignore[assignment] + update(ScholarProfile) + .where( + ScholarProfile.id.in_(scholar_profile_ids), + ScholarProfile.user_id == user_id, + ) + .values(is_enabled=is_enabled) + ) + await db_session.commit() + return int(cursor.rowcount or 0) + + __all__ = [ "SEARCH_COOLDOWN_REASON", "SEARCH_DISABLED_REASON", "ScholarServiceError", + "bulk_delete_scholars", + "bulk_toggle_scholars", "clear_profile_image_customization", "create_scholar_for_user", "delete_scholar", @@ -125,7 +182,11 @@ async def delete_scholar( _safe_remove_upload(upload_root, profile.profile_image_upload_path) await db_session.delete(profile) - await db_session.commit() + try: + await db_session.commit() + except IntegrityError as exc: + await db_session.rollback() + raise ScholarServiceError("Unable to delete scholar due to a database constraint.") from exc async def hydrate_profile_metadata( diff --git a/frontend/src/components/patterns/ScrapeSafetyBadge.test.ts b/frontend/src/components/patterns/ScrapeSafetyBadge.test.ts new file mode 100644 index 0000000..8d17969 --- /dev/null +++ b/frontend/src/components/patterns/ScrapeSafetyBadge.test.ts @@ -0,0 +1,61 @@ +// @vitest-environment happy-dom +import { describe, expect, it } from "vitest"; +import { mount } from "@vue/test-utils"; +import ScrapeSafetyBadge from "./ScrapeSafetyBadge.vue"; +import { createDefaultSafetyState, type ScrapeSafetyState } from "@/features/safety"; + +function buildState(overrides: Partial = {}): ScrapeSafetyState { + return { ...createDefaultSafetyState(), ...overrides }; +} + +describe("ScrapeSafetyBadge", () => { + it("shows ready tooltip when cooldown is inactive", () => { + const wrapper = mount(ScrapeSafetyBadge, { + props: { state: buildState({ cooldown_active: false }) }, + }); + expect(wrapper.text()).toContain("Safety ready"); + const hint = wrapper.findComponent({ name: "AppHelpHint" }); + expect(hint.exists()).toBe(true); + expect(hint.props("text")).toContain("No active cooldown"); + }); + + it("shows cooldown tooltip with reason and action when active", () => { + const wrapper = mount(ScrapeSafetyBadge, { + props: { + state: buildState({ + cooldown_active: true, + cooldown_reason: "blocked_failure_threshold_exceeded", + cooldown_reason_label: "Too many blocked requests", + recommended_action: "Wait for cooldown to expire", + cooldown_remaining_seconds: 120, + }), + }, + }); + expect(wrapper.text()).toContain("Safety cooldown"); + const hint = wrapper.findComponent({ name: "AppHelpHint" }); + expect(hint.exists()).toBe(true); + const text = hint.props("text") as string; + expect(text).toContain("Google Scholar rate-limits"); + expect(text).toContain("Too many blocked requests"); + expect(text).toContain("Wait for cooldown to expire"); + }); + + it("shows cooldown tooltip without optional fields", () => { + const wrapper = mount(ScrapeSafetyBadge, { + props: { + state: buildState({ + cooldown_active: true, + cooldown_reason: "some_reason", + cooldown_reason_label: null, + recommended_action: null, + cooldown_remaining_seconds: 60, + }), + }, + }); + const hint = wrapper.findComponent({ name: "AppHelpHint" }); + const text = hint.props("text") as string; + expect(text).toContain("Google Scholar rate-limits"); + expect(text).not.toContain("Why:"); + expect(text).not.toContain("Action:"); + }); +}); diff --git a/frontend/src/components/patterns/ScrapeSafetyBadge.vue b/frontend/src/components/patterns/ScrapeSafetyBadge.vue index b40c1c9..845fb2f 100644 --- a/frontend/src/components/patterns/ScrapeSafetyBadge.vue +++ b/frontend/src/components/patterns/ScrapeSafetyBadge.vue @@ -1,6 +1,7 @@ diff --git a/frontend/src/components/patterns/ScrapeSafetyBanner.vue b/frontend/src/components/patterns/ScrapeSafetyBanner.vue index 300b05a..6ea343d 100644 --- a/frontend/src/components/patterns/ScrapeSafetyBanner.vue +++ b/frontend/src/components/patterns/ScrapeSafetyBanner.vue @@ -2,6 +2,7 @@ import { computed, onBeforeUnmount, onMounted, ref } from "vue"; import AppAlert from "@/components/ui/AppAlert.vue"; +import AppHelpHint from "@/components/ui/AppHelpHint.vue"; import { formatCooldownCountdown, type ScrapeSafetyState, @@ -79,6 +80,10 @@ const actionText = computed(() => { return props.safetyState.recommended_action; }); +const bannerHintText = computed(() => { + return "Google Scholar rate-limits automated requests. The cooldown pauses scraping to avoid your IP being blocked."; +}); + onMounted(() => { timer = setInterval(() => { now.value = Date.now(); @@ -95,7 +100,12 @@ onBeforeUnmount(() => {