diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a93c27..f82312a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,7 @@ jobs: - name: Install frontend dependencies working-directory: frontend - run: npm install + run: npm ci - name: Frontend theme token policy working-directory: frontend @@ -124,7 +124,7 @@ jobs: node-version: "20" - name: Install docs dependencies - run: npm install --prefix docs/website + run: npm ci --prefix docs/website - name: Docs build run: npm --prefix docs/website run build diff --git a/.github/workflows/docs-pages.yml b/.github/workflows/docs-pages.yml index 1e699e5..24ebe7a 100644 --- a/.github/workflows/docs-pages.yml +++ b/.github/workflows/docs-pages.yml @@ -32,7 +32,7 @@ jobs: node-version: "20" - name: Install docs dependencies - run: npm install --prefix docs/website + run: npm ci --prefix docs/website - name: Build docs site run: npm --prefix docs/website run build diff --git a/.gitignore b/.gitignore index 88c4ac6..68a3f87 100644 --- a/.gitignore +++ b/.gitignore @@ -12,9 +12,9 @@ htmlcov/ *.log .DS_Store planning/ +build/ frontend/node_modules/ frontend/dist/ frontend/.vite/ docs/website/node_modules/ docs/website/build/ -AGENTS.MD diff --git a/build/lib/app/__init__.py b/build/lib/app/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/build/lib/app/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/build/lib/app/api/__init__.py b/build/lib/app/api/__init__.py deleted file mode 100644 index 3ce55a6..0000000 --- a/build/lib/app/api/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from __future__ import annotations - diff --git a/build/lib/app/api/deps.py b/build/lib/app/api/deps.py deleted file mode 100644 index f9c5377..0000000 --- a/build/lib/app/api/deps.py +++ /dev/null @@ -1,35 +0,0 @@ -from __future__ import annotations - -from fastapi import Depends, Request -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.errors import ApiException -from app.auth import runtime as auth_runtime -from app.db.models import User -from app.db.session import get_db_session - - -async def get_api_current_user( - request: Request, - db_session: AsyncSession = Depends(get_db_session), -) -> User: - current_user = await auth_runtime.get_authenticated_user(request, db_session) - if current_user is None: - raise ApiException( - status_code=401, - code="auth_required", - message="Authentication required.", - ) - return current_user - - -async def get_api_admin_user( - current_user: User = Depends(get_api_current_user), -) -> User: - if not current_user.is_admin: - raise ApiException( - status_code=403, - code="forbidden", - message="Admin access required.", - ) - return current_user diff --git a/build/lib/app/api/errors.py b/build/lib/app/api/errors.py deleted file mode 100644 index 976ccb8..0000000 --- a/build/lib/app/api/errors.py +++ /dev/null @@ -1,85 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from fastapi import FastAPI, HTTPException, Request -from fastapi.exception_handlers import ( - http_exception_handler as fastapi_http_exception_handler, -) -from fastapi.exception_handlers import ( - request_validation_exception_handler as fastapi_validation_exception_handler, -) -from fastapi.exceptions import RequestValidationError - -from app.api.responses import error_response - - -def _is_api_path(path: str) -> bool: - return path.startswith("/api/") - - -def _status_code_to_error_code(status_code: int) -> str: - mapping = { - 400: "bad_request", - 401: "auth_required", - 403: "forbidden", - 404: "not_found", - 409: "conflict", - 422: "validation_error", - 429: "rate_limited", - 500: "internal_error", - } - return mapping.get(status_code, "error") - - -class ApiException(Exception): - def __init__( - self, - *, - status_code: int, - code: str, - message: str, - details: Any | None = None, - ) -> None: - super().__init__(message) - self.status_code = status_code - self.code = code - self.message = message - self.details = details - - -def register_api_exception_handlers(app: FastAPI) -> None: - @app.exception_handler(ApiException) - async def _handle_api_exception(request: Request, exc: ApiException): - return error_response( - request, - status_code=exc.status_code, - code=exc.code, - message=exc.message, - details=exc.details, - ) - - @app.exception_handler(HTTPException) - async def _handle_http_exception(request: Request, exc: HTTPException): - if not _is_api_path(request.url.path): - return await fastapi_http_exception_handler(request, exc) - return error_response( - request, - status_code=exc.status_code, - code=_status_code_to_error_code(exc.status_code), - message=str(exc.detail) if exc.detail is not None else "Request failed.", - details=exc.detail if isinstance(exc.detail, (dict, list)) else None, - ) - - @app.exception_handler(RequestValidationError) - async def _handle_validation_exception(request: Request, exc: RequestValidationError): - if not _is_api_path(request.url.path): - return await fastapi_validation_exception_handler(request, exc) - return error_response( - request, - status_code=422, - code="validation_error", - message="Request validation failed.", - details=exc.errors(), - ) - diff --git a/build/lib/app/api/media.py b/build/lib/app/api/media.py deleted file mode 100644 index 00943bd..0000000 --- a/build/lib/app/api/media.py +++ /dev/null @@ -1,63 +0,0 @@ -from __future__ import annotations - -import mimetypes - -from fastapi import APIRouter, Depends -from fastapi.responses import FileResponse -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.deps import get_api_current_user -from app.api.errors import ApiException -from app.db.models import User -from app.db.session import get_db_session -from app.services.domains.scholars import application as scholar_service -from app.settings import settings - -router = APIRouter(tags=["media"]) - - -@router.get("/scholar-images/{scholar_profile_id}/upload") -async def get_uploaded_scholar_image( - scholar_profile_id: int, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - profile = await scholar_service.get_user_scholar_by_id( - db_session, - user_id=current_user.id, - scholar_profile_id=scholar_profile_id, - ) - if profile is None: - raise ApiException( - status_code=404, - code="scholar_not_found", - message="Scholar not found.", - ) - if not profile.profile_image_upload_path: - raise ApiException( - status_code=404, - code="scholar_image_not_found", - message="Scholar image not found.", - ) - - try: - image_path = scholar_service.resolve_upload_file_path( - upload_dir=settings.scholar_image_upload_dir, - relative_path=profile.profile_image_upload_path, - ) - except scholar_service.ScholarServiceError as exc: - raise ApiException( - status_code=404, - code="scholar_image_not_found", - message="Scholar image not found.", - ) from exc - - if not image_path.exists() or not image_path.is_file(): - raise ApiException( - status_code=404, - code="scholar_image_not_found", - message="Scholar image not found.", - ) - - media_type = mimetypes.guess_type(str(image_path))[0] or "application/octet-stream" - return FileResponse(path=image_path, media_type=media_type) diff --git a/build/lib/app/api/responses.py b/build/lib/app/api/responses.py deleted file mode 100644 index 9976aba..0000000 --- a/build/lib/app/api/responses.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from fastapi.encoders import jsonable_encoder -from starlette.requests import Request -from starlette.responses import JSONResponse - - -def _request_id(request: Request) -> str | None: - request_state = getattr(request, "state", None) - if request_state is None: - return None - return getattr(request_state, "request_id", None) - - -def success_payload( - request: Request, - *, - data: Any, -) -> dict[str, Any]: - return { - "data": data, - "meta": { - "request_id": _request_id(request), - }, - } - - -def success_response( - request: Request, - *, - data: Any, - status_code: int = 200, -) -> JSONResponse: - payload = success_payload(request, data=data) - return JSONResponse( - status_code=status_code, - content=jsonable_encoder(payload), - ) - - -def error_response( - request: Request, - *, - status_code: int, - code: str, - message: str, - details: Any | None = None, -) -> JSONResponse: - payload = { - "error": { - "code": code, - "message": message, - "details": details, - }, - "meta": { - "request_id": _request_id(request), - }, - } - return JSONResponse( - status_code=status_code, - content=jsonable_encoder(payload), - ) diff --git a/build/lib/app/api/router.py b/build/lib/app/api/router.py deleted file mode 100644 index a11d98a..0000000 --- a/build/lib/app/api/router.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import annotations - -from fastapi import APIRouter - -from app.api.routers import admin, admin_dbops, auth, publications, runs, scholars, settings - -router = APIRouter(prefix="/api/v1") -router.include_router(auth.router) -router.include_router(admin.router) -router.include_router(admin_dbops.router) -router.include_router(scholars.router) -router.include_router(settings.router) -router.include_router(runs.router) -router.include_router(publications.router) diff --git a/build/lib/app/api/routers/__init__.py b/build/lib/app/api/routers/__init__.py deleted file mode 100644 index 3ce55a6..0000000 --- a/build/lib/app/api/routers/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from __future__ import annotations - diff --git a/build/lib/app/api/routers/admin.py b/build/lib/app/api/routers/admin.py deleted file mode 100644 index 9844c3e..0000000 --- a/build/lib/app/api/routers/admin.py +++ /dev/null @@ -1,202 +0,0 @@ -from __future__ import annotations - -import logging - -from fastapi import APIRouter, Depends, Request -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.deps import get_api_admin_user -from app.api.errors import ApiException -from app.api.responses import success_payload -from app.api.schemas import ( - AdminResetPasswordRequest, - AdminUserActiveUpdateRequest, - AdminUserCreateRequest, - AdminUserEnvelope, - AdminUsersListEnvelope, - MessageEnvelope, -) -from app.auth.deps import get_auth_service -from app.auth.service import AuthService -from app.db.models import User -from app.db.session import get_db_session -from app.services.domains.users import application as user_service - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/admin/users", tags=["api-admin-users"]) - - -def _serialize_user(user: User) -> dict[str, object]: - return { - "id": int(user.id), - "email": user.email, - "is_active": bool(user.is_active), - "is_admin": bool(user.is_admin), - "created_at": user.created_at, - "updated_at": user.updated_at, - } - - -@router.get( - "", - response_model=AdminUsersListEnvelope, -) -async def list_users( - request: Request, - db_session: AsyncSession = Depends(get_db_session), - admin_user: User = Depends(get_api_admin_user), -): - users = await user_service.list_users(db_session) - logger.info( - "api.admin.users_listed", - extra={ - "event": "api.admin.users_listed", - "admin_user_id": int(admin_user.id), - "user_count": len(users), - }, - ) - return success_payload( - request, - data={ - "users": [_serialize_user(user) for user in users], - }, - ) - - -@router.post( - "", - response_model=AdminUserEnvelope, - status_code=201, -) -async def create_user( - payload: AdminUserCreateRequest, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - auth_service: AuthService = Depends(get_auth_service), - admin_user: User = Depends(get_api_admin_user), -): - try: - validated_email = user_service.validate_email(payload.email) - validated_password = user_service.validate_password(payload.password) - created_user = await user_service.create_user( - db_session, - email=validated_email, - password_hash=auth_service.hash_password(validated_password), - is_admin=bool(payload.is_admin), - ) - except user_service.UserServiceError as exc: - raise ApiException( - status_code=400, - code="invalid_user_input", - message=str(exc), - ) from exc - - logger.info( - "api.admin.user_created", - extra={ - "event": "api.admin.user_created", - "admin_user_id": int(admin_user.id), - "target_user_id": int(created_user.id), - "target_is_admin": bool(created_user.is_admin), - }, - ) - return success_payload( - request, - data=_serialize_user(created_user), - ) - - -@router.patch( - "/{user_id}/active", - response_model=AdminUserEnvelope, -) -async def set_user_active( - user_id: int, - payload: AdminUserActiveUpdateRequest, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - admin_user: User = Depends(get_api_admin_user), -): - target_user = await user_service.get_user_by_id(db_session, user_id) - if target_user is None: - raise ApiException( - status_code=404, - code="user_not_found", - message="User not found.", - ) - if ( - int(target_user.id) == int(admin_user.id) - and bool(target_user.is_active) - and not bool(payload.is_active) - ): - raise ApiException( - status_code=400, - code="cannot_deactivate_self", - message="You cannot deactivate your own account.", - ) - updated_user = await user_service.set_user_active( - db_session, - user=target_user, - is_active=bool(payload.is_active), - ) - logger.info( - "api.admin.user_active_updated", - extra={ - "event": "api.admin.user_active_updated", - "admin_user_id": int(admin_user.id), - "target_user_id": int(updated_user.id), - "is_active": bool(updated_user.is_active), - }, - ) - return success_payload( - request, - data=_serialize_user(updated_user), - ) - - -@router.post( - "/{user_id}/reset-password", - response_model=MessageEnvelope, -) -async def reset_user_password( - user_id: int, - payload: AdminResetPasswordRequest, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - auth_service: AuthService = Depends(get_auth_service), - admin_user: User = Depends(get_api_admin_user), -): - target_user = await user_service.get_user_by_id(db_session, user_id) - if target_user is None: - raise ApiException( - status_code=404, - code="user_not_found", - message="User not found.", - ) - try: - validated_password = user_service.validate_password(payload.new_password) - except user_service.UserServiceError as exc: - raise ApiException( - status_code=400, - code="invalid_password", - message=str(exc), - ) from exc - - await user_service.set_user_password_hash( - db_session, - user=target_user, - password_hash=auth_service.hash_password(validated_password), - ) - logger.info( - "api.admin.user_password_reset", - extra={ - "event": "api.admin.user_password_reset", - "admin_user_id": int(admin_user.id), - "target_user_id": int(target_user.id), - }, - ) - return success_payload( - request, - data={"message": f"Password reset: {target_user.email}"}, - ) diff --git a/build/lib/app/api/routers/admin_dbops.py b/build/lib/app/api/routers/admin_dbops.py deleted file mode 100644 index 87fc3b8..0000000 --- a/build/lib/app/api/routers/admin_dbops.py +++ /dev/null @@ -1,349 +0,0 @@ -from __future__ import annotations - -import logging - -from fastapi import APIRouter, Depends, Path, Query, Request -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.deps import get_api_admin_user -from app.api.errors import ApiException -from app.api.responses import success_payload -from app.api.schemas import ( - AdminDbIntegrityEnvelope, - AdminPdfQueueBulkEnqueueEnvelope, - AdminPdfQueueRequeueEnvelope, - AdminPdfQueueEnvelope, - AdminDbRepairJobsEnvelope, - AdminRepairPublicationLinksEnvelope, - AdminRepairPublicationLinksRequest, -) -from app.db.models import DataRepairJob, User -from app.db.session import get_db_session -from app.services.domains.dbops import ( - collect_integrity_report, - list_repair_jobs, - run_publication_link_repair, -) -from app.services.domains.publications import application as publication_service - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/admin/db", tags=["api-admin-dbops"]) - - -def _serialize_repair_job(job: DataRepairJob) -> dict[str, object]: - return { - "id": int(job.id), - "job_name": job.job_name, - "requested_by": job.requested_by, - "dry_run": bool(job.dry_run), - "status": job.status, - "scope": dict(job.scope or {}), - "summary": dict(job.summary or {}), - "error_text": job.error_text, - "started_at": job.started_at, - "finished_at": job.finished_at, - "created_at": job.created_at, - "updated_at": job.updated_at, - } - - -def _requested_by_value(*, payload: AdminRepairPublicationLinksRequest, admin_user: User) -> str: - from_payload = (payload.requested_by or "").strip() - return from_payload or admin_user.email - - -def _serialize_pdf_queue_item(item) -> dict[str, object]: - return { - "publication_id": item.publication_id, - "title": item.title, - "display_identifier": _serialize_display_identifier(item.display_identifier), - "pdf_url": item.pdf_url, - "status": item.status, - "attempt_count": item.attempt_count, - "last_failure_reason": item.last_failure_reason, - "last_failure_detail": item.last_failure_detail, - "last_source": item.last_source, - "requested_by_user_id": item.requested_by_user_id, - "requested_by_email": item.requested_by_email, - "queued_at": item.queued_at, - "last_attempt_at": item.last_attempt_at, - "resolved_at": item.resolved_at, - "updated_at": item.updated_at, - } - - -def _serialize_display_identifier(value) -> dict[str, object] | None: - if value is None: - return None - return { - "kind": value.kind, - "value": value.value, - "label": value.label, - "url": value.url, - "confidence_score": float(value.confidence_score), - } - - -def _requeue_response_state(*, queued: bool) -> tuple[str, str]: - if queued: - return "queued", "PDF lookup queued." - return "blocked", "PDF lookup is already queued or currently running." - - -def _bulk_enqueue_message(*, queued_count: int, requested_count: int) -> str: - if requested_count == 0: - return "No missing-PDF publications were found." - if queued_count == 0: - return "No publications were queued; all candidates were already in-flight." - return f"Queued {queued_count} publication(s) for PDF lookup." - - -def _resolve_pdf_queue_paging( - *, - page: int, - page_size: int, - limit: int | None, - offset: int | None, -) -> tuple[int, int, int]: - resolved_limit = int(limit) if limit is not None else int(page_size) - resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0) - resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1) - return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0) - - -def _pdf_queue_page_data(*, total_count: int, page: int, page_size: int, offset: int) -> dict[str, object]: - return { - "total_count": int(total_count), - "page": int(page), - "page_size": int(page_size), - "has_prev": int(offset) > 0, - "has_next": int(offset) + int(page_size) < int(total_count), - } - - -@router.get( - "/integrity", - response_model=AdminDbIntegrityEnvelope, -) -async def get_integrity_report( - request: Request, - db_session: AsyncSession = Depends(get_db_session), - admin_user: User = Depends(get_api_admin_user), -): - report = await collect_integrity_report(db_session) - logger.info( - "api.admin.db.integrity_checked", - extra={ - "event": "api.admin.db.integrity_checked", - "admin_user_id": int(admin_user.id), - "status": report.get("status"), - "failure_count": len(report.get("failures", [])), - "warning_count": len(report.get("warnings", [])), - }, - ) - return success_payload(request, data=report) - - -@router.get( - "/repair-jobs", - response_model=AdminDbRepairJobsEnvelope, -) -async def get_repair_jobs( - request: Request, - limit: int = Query(default=50, ge=1, le=200), - db_session: AsyncSession = Depends(get_db_session), - admin_user: User = Depends(get_api_admin_user), -): - jobs = await list_repair_jobs(db_session, limit=limit) - logger.info( - "api.admin.db.repair_jobs_listed", - extra={ - "event": "api.admin.db.repair_jobs_listed", - "admin_user_id": int(admin_user.id), - "limit": int(limit), - "job_count": len(jobs), - }, - ) - return success_payload( - request, - data={"jobs": [_serialize_repair_job(job) for job in jobs]}, - ) - - -@router.get( - "/pdf-queue", - response_model=AdminPdfQueueEnvelope, -) -async def get_pdf_queue( - request: Request, - page: int = Query(default=1, ge=1), - page_size: int = Query(default=100, ge=1, le=500), - limit: int | None = Query(default=None, ge=1, le=500), - offset: int | None = Query(default=None, ge=0), - status: str | None = Query(default=None), - db_session: AsyncSession = Depends(get_db_session), - admin_user: User = Depends(get_api_admin_user), -): - normalized_status = (status or "").strip().lower() or None - resolved_page, resolved_limit, resolved_offset = _resolve_pdf_queue_paging( - page=page, - page_size=page_size, - limit=limit, - offset=offset, - ) - queue_page = await publication_service.list_pdf_queue_page( - db_session, - limit=resolved_limit, - offset=resolved_offset, - status=normalized_status, - ) - logger.info( - "api.admin.db.pdf_queue_listed", - extra={ - "event": "api.admin.db.pdf_queue_listed", - "admin_user_id": int(admin_user.id), - "page": int(resolved_page), - "page_size": int(resolved_limit), - "offset": int(resolved_offset), - "status": normalized_status, - "item_count": len(queue_page.items), - "total_count": int(queue_page.total_count), - }, - ) - return success_payload( - request, - data={ - "items": [_serialize_pdf_queue_item(item) for item in queue_page.items], - **_pdf_queue_page_data( - total_count=queue_page.total_count, - page=resolved_page, - page_size=resolved_limit, - offset=resolved_offset, - ), - }, - ) - - -@router.post( - "/pdf-queue/{publication_id}/requeue", - response_model=AdminPdfQueueRequeueEnvelope, -) -async def requeue_pdf_lookup( - request: Request, - publication_id: int = Path(ge=1), - db_session: AsyncSession = Depends(get_db_session), - admin_user: User = Depends(get_api_admin_user), -): - result = await publication_service.enqueue_retry_pdf_job_for_publication_id( - db_session, - user_id=int(admin_user.id), - request_email=admin_user.email, - publication_id=publication_id, - ) - if not result.publication_exists: - raise ApiException( - status_code=404, - code="publication_not_found", - message="Publication not found.", - ) - status, message = _requeue_response_state(queued=result.queued) - logger.info( - "api.admin.db.pdf_queue_requeue_requested", - extra={ - "event": "api.admin.db.pdf_queue_requeue_requested", - "admin_user_id": int(admin_user.id), - "publication_id": int(publication_id), - "queued": bool(result.queued), - }, - ) - return success_payload( - request, - data={ - "publication_id": int(publication_id), - "queued": bool(result.queued), - "status": status, - "message": message, - }, - ) - - -@router.post( - "/pdf-queue/requeue-all", - response_model=AdminPdfQueueBulkEnqueueEnvelope, -) -async def requeue_all_missing_pdfs( - request: Request, - limit: int = Query(default=1000, ge=1, le=5000), - db_session: AsyncSession = Depends(get_db_session), - admin_user: User = Depends(get_api_admin_user), -): - result = await publication_service.enqueue_all_missing_pdf_jobs( - db_session, - user_id=int(admin_user.id), - request_email=admin_user.email, - limit=limit, - ) - logger.info( - "api.admin.db.pdf_queue_requeue_all", - extra={ - "event": "api.admin.db.pdf_queue_requeue_all", - "admin_user_id": int(admin_user.id), - "limit": int(limit), - "requested_count": int(result.requested_count), - "queued_count": int(result.queued_count), - }, - ) - return success_payload( - request, - data={ - "requested_count": int(result.requested_count), - "queued_count": int(result.queued_count), - "message": _bulk_enqueue_message( - queued_count=int(result.queued_count), - requested_count=int(result.requested_count), - ), - }, - ) - - -@router.post( - "/repairs/publication-links", - response_model=AdminRepairPublicationLinksEnvelope, -) -async def trigger_publication_link_repair( - payload: AdminRepairPublicationLinksRequest, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - admin_user: User = Depends(get_api_admin_user), -): - try: - result = await run_publication_link_repair( - db_session, - scope_mode=payload.scope_mode, - user_id=payload.user_id, - scholar_profile_ids=payload.scholar_profile_ids, - dry_run=bool(payload.dry_run), - gc_orphan_publications=bool(payload.gc_orphan_publications), - requested_by=_requested_by_value(payload=payload, admin_user=admin_user), - ) - except ValueError as exc: - raise ApiException( - status_code=400, - code="invalid_repair_scope", - message=str(exc), - ) from exc - logger.info( - "api.admin.db.publication_link_repair_triggered", - extra={ - "event": "api.admin.db.publication_link_repair_triggered", - "admin_user_id": int(admin_user.id), - "scope_mode": payload.scope_mode, - "target_user_id": int(payload.user_id) if payload.user_id is not None else None, - "dry_run": bool(payload.dry_run), - "gc_orphan_publications": bool(payload.gc_orphan_publications), - "job_id": int(result["job_id"]), - "status": result["status"], - }, - ) - return success_payload(request, data=result) diff --git a/build/lib/app/api/routers/auth.py b/build/lib/app/api/routers/auth.py deleted file mode 100644 index 8f8da17..0000000 --- a/build/lib/app/api/routers/auth.py +++ /dev/null @@ -1,237 +0,0 @@ -from __future__ import annotations - -import logging - -from fastapi import APIRouter, Depends, Request -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.errors import ApiException -from app.api.deps import get_api_current_user -from app.api.responses import success_payload -from app.api.schemas import ( - AuthMeEnvelope, - ChangePasswordRequest, - CsrfBootstrapEnvelope, - LoginEnvelope, - LoginRequest, - MessageEnvelope, -) -from app.auth.deps import get_auth_service, get_login_rate_limiter -from app.auth.rate_limit import SlidingWindowRateLimiter -from app.auth import runtime as auth_runtime -from app.auth.service import AuthService -from app.auth.session import set_session_user -from app.db.models import User -from app.db.session import get_db_session -from app.security.csrf import ensure_csrf_token -from app.services.domains.users import application as user_service - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/auth", tags=["api-auth"]) - - -def _login_limiter_key_and_email(request: Request, payload: LoginRequest) -> tuple[str, str]: - return auth_runtime.login_rate_limit_key(request, payload.email), payload.email.strip().lower() - - -def _raise_rate_limited(normalized_email: str, retry_after_seconds: int) -> None: - logger.warning( - "api.auth.login_rate_limited", - extra={ - "event": "api.auth.login_rate_limited", - "email": normalized_email, - "retry_after_seconds": retry_after_seconds, - }, - ) - raise ApiException( - status_code=429, - code="rate_limited", - message="Too many login attempts. Please try again later.", - details={"retry_after_seconds": retry_after_seconds}, - ) - - -def _serialize_user_payload(user: User) -> dict[str, object]: - return { - "id": int(user.id), - "email": user.email, - "is_admin": bool(user.is_admin), - "is_active": bool(user.is_active), - } - - -def _raise_invalid_credentials(*, normalized_email: str) -> None: - logger.info( - "api.auth.login_failed", - extra={"event": "api.auth.login_failed", "email": normalized_email}, - ) - raise ApiException( - status_code=401, - code="invalid_credentials", - message="Invalid email or password.", - ) - - -@router.post( - "/login", - response_model=LoginEnvelope, -) -async def login( - payload: LoginRequest, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - auth_service: AuthService = Depends(get_auth_service), - rate_limiter: SlidingWindowRateLimiter = Depends(get_login_rate_limiter), -): - limiter_key, normalized_email = _login_limiter_key_and_email(request, payload) - decision = rate_limiter.check(limiter_key) - if not decision.allowed: - _raise_rate_limited(normalized_email, int(decision.retry_after_seconds)) - - user = await auth_service.authenticate_user( - db_session, - email=payload.email, - password=payload.password, - ) - if user is None: - rate_limiter.record_failure(limiter_key) - _raise_invalid_credentials(normalized_email=normalized_email) - - rate_limiter.reset(limiter_key) - set_session_user( - request, - user_id=user.id, - email=user.email, - is_admin=user.is_admin, - ) - logger.info( - "api.auth.login_succeeded", - extra={ - "event": "api.auth.login_succeeded", - "user_id": user.id, - "is_admin": user.is_admin, - }, - ) - return success_payload( - request, - data={ - "authenticated": True, - "csrf_token": ensure_csrf_token(request), - "user": _serialize_user_payload(user), - }, - ) - - -@router.get( - "/me", - response_model=AuthMeEnvelope, -) -async def get_current_session( - request: Request, - current_user: User = Depends(get_api_current_user), -): - return success_payload( - request, - data={ - "authenticated": True, - "csrf_token": ensure_csrf_token(request), - "user": _serialize_user_payload(current_user), - }, - ) - - -@router.get( - "/csrf", - response_model=CsrfBootstrapEnvelope, -) -async def get_csrf_bootstrap( - request: Request, - db_session: AsyncSession = Depends(get_db_session), -): - current_user = await auth_runtime.get_authenticated_user(request, db_session) - return success_payload( - request, - data={ - "csrf_token": ensure_csrf_token(request), - "authenticated": current_user is not None, - }, - ) - - -@router.post( - "/change-password", - response_model=MessageEnvelope, -) -async def change_password( - payload: ChangePasswordRequest, - request: Request, - current_user: User = Depends(get_api_current_user), - db_session: AsyncSession = Depends(get_db_session), - auth_service: AuthService = Depends(get_auth_service), -): - if not auth_service.verify_password( - password_hash=current_user.password_hash, - password=payload.current_password, - ): - raise ApiException( - status_code=400, - code="invalid_current_password", - message="Current password is incorrect.", - ) - if payload.new_password != payload.confirm_password: - raise ApiException( - status_code=400, - code="password_confirmation_mismatch", - message="New password and confirmation do not match.", - ) - try: - validated_password = user_service.validate_password(payload.new_password) - except user_service.UserServiceError as exc: - raise ApiException( - status_code=400, - code="invalid_password", - message=str(exc), - ) from exc - - await user_service.set_user_password_hash( - db_session, - user=current_user, - password_hash=auth_service.hash_password(validated_password), - ) - logger.info( - "api.auth.password_changed", - extra={ - "event": "api.auth.password_changed", - "user_id": int(current_user.id), - }, - ) - return success_payload( - request, - data={"message": "Password updated successfully."}, - ) - - -@router.post( - "/logout", - response_model=MessageEnvelope, -) -async def logout( - request: Request, - db_session: AsyncSession = Depends(get_db_session), -): - current_user = await auth_runtime.get_authenticated_user(request, db_session) - auth_runtime.invalidate_session(request) - logger.info( - "api.auth.logout", - extra={ - "event": "api.auth.logout", - "user_id": int(current_user.id) if current_user is not None else None, - }, - ) - return success_payload( - request, - data={ - "message": "Logged out.", - }, - ) diff --git a/build/lib/app/api/routers/publications.py b/build/lib/app/api/routers/publications.py deleted file mode 100644 index 2a96b8b..0000000 --- a/build/lib/app/api/routers/publications.py +++ /dev/null @@ -1,528 +0,0 @@ -from __future__ import annotations - -import logging -from typing import Literal - -from fastapi import APIRouter, Depends, Path, Query, Request -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.deps import get_api_current_user -from app.api.errors import ApiException -from app.api.responses import success_payload -from app.api.schemas import ( - MarkAllReadEnvelope, - MarkSelectedReadEnvelope, - MarkSelectedReadRequest, - PublicationsListEnvelope, - RetryPublicationPdfEnvelope, - RetryPublicationPdfRequest, - TogglePublicationFavoriteEnvelope, - TogglePublicationFavoriteRequest, -) -from app.db.models import User -from app.db.session import get_db_session -from app.services.domains.publication_identifiers import application as identifier_service -from app.services.domains.publications import application as publication_service -from app.services.domains.scholars import application as scholar_service -from app.settings import settings - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/publications", tags=["api-publications"]) - - -async def _require_selected_profile( - db_session: AsyncSession, - *, - user_id: int, - selected_scholar_id: int | None, -) -> None: - if selected_scholar_id is None: - return - selected_profile = await scholar_service.get_user_scholar_by_id( - db_session, - user_id=user_id, - scholar_profile_id=selected_scholar_id, - ) - if selected_profile is None: - raise ApiException( - status_code=404, - code="scholar_not_found", - message="Scholar filter not found.", - ) - - -def _serialize_publication_item(item) -> dict[str, object]: - return { - "publication_id": item.publication_id, - "scholar_profile_id": item.scholar_profile_id, - "scholar_label": item.scholar_label, - "title": item.title, - "year": item.year, - "citation_count": item.citation_count, - "venue_text": item.venue_text, - "pub_url": item.pub_url, - "display_identifier": _serialize_display_identifier(item.display_identifier), - "pdf_url": item.pdf_url, - "pdf_status": item.pdf_status, - "pdf_attempt_count": item.pdf_attempt_count, - "pdf_failure_reason": item.pdf_failure_reason, - "pdf_failure_detail": item.pdf_failure_detail, - "is_read": item.is_read, - "is_favorite": item.is_favorite, - "first_seen_at": item.first_seen_at, - "is_new_in_latest_run": item.is_new_in_latest_run, - } - - -def _serialize_display_identifier(value) -> dict[str, object] | None: - if value is None: - return None - return { - "kind": value.kind, - "value": value.value, - "label": value.label, - "url": value.url, - "confidence_score": float(value.confidence_score), - } - - -async def _publication_counts( - db_session: AsyncSession, - *, - user_id: int, - selected_scholar_id: int | None, - favorite_only: bool, -) -> tuple[int, int, int, int]: - unread_count = await publication_service.count_unread_for_user( - db_session, - user_id=user_id, - scholar_profile_id=selected_scholar_id, - favorite_only=favorite_only, - ) - favorites_count = await publication_service.count_favorite_for_user( - db_session, - user_id=user_id, - scholar_profile_id=selected_scholar_id, - ) - latest_count = await publication_service.count_latest_for_user( - db_session, - user_id=user_id, - scholar_profile_id=selected_scholar_id, - favorite_only=favorite_only, - ) - total_count = await publication_service.count_for_user( - db_session, - user_id=user_id, - mode=publication_service.MODE_ALL, - scholar_profile_id=selected_scholar_id, - favorite_only=favorite_only, - ) - return unread_count, favorites_count, latest_count, total_count - - -async def _list_publications_for_request( - db_session: AsyncSession, - *, - current_user: User, - mode: Literal["all", "unread", "latest", "new"] | None, - favorite_only: bool, - scholar_profile_id: int | None, - limit: int, - offset: int, -) -> tuple[str, int | None, list]: - resolved_mode = publication_service.resolve_publication_view_mode(mode) - selected_scholar_id = scholar_profile_id - await _require_selected_profile( - db_session, - user_id=current_user.id, - selected_scholar_id=selected_scholar_id, - ) - publications = await publication_service.list_for_user( - db_session, - user_id=current_user.id, - mode=resolved_mode, - scholar_profile_id=selected_scholar_id, - favorite_only=favorite_only, - limit=limit, - offset=offset, - ) - await publication_service.schedule_missing_pdf_enrichment_for_user( - db_session, - user_id=current_user.id, - request_email=current_user.email, - items=publications, - max_items=settings.unpaywall_max_items_per_request, - ) - hydrated = await publication_service.hydrate_pdf_enrichment_state( - db_session, - items=publications, - ) - return resolved_mode, selected_scholar_id, hydrated - - -def _resolve_publications_paging( - *, - page: int, - page_size: int, - limit: int | None, - offset: int | None, -) -> tuple[int, int, int]: - resolved_limit = int(limit) if limit is not None else int(page_size) - resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0) - resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1) - return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0) - - -def _publications_list_data( - *, - mode: str, - favorite_only: bool, - selected_scholar_id: int | None, - unread_count: int, - favorites_count: int, - latest_count: int, - total_count: int, - publications: list, - page: int, - page_size: int, - offset: int, -) -> dict[str, object]: - return { - "mode": mode, - "favorite_only": favorite_only, - "selected_scholar_profile_id": selected_scholar_id, - "unread_count": unread_count, - "favorites_count": favorites_count, - "latest_count": latest_count, - "new_count": latest_count, - "total_count": total_count, - "page": int(page), - "page_size": int(page_size), - "has_prev": int(offset) > 0, - "has_next": int(offset) + int(page_size) < int(total_count), - "publications": [_serialize_publication_item(item) for item in publications], - } - - -def _retry_pdf_message(*, queued: bool, resolved_pdf: bool, pdf_status: str) -> str: - if resolved_pdf: - return "Open-access PDF link already resolved." - if queued: - return "PDF lookup queued." - if pdf_status in {"queued", "running"}: - return "PDF lookup is already queued." - return "No open-access PDF link found." - - -def _favorite_message(*, is_favorite: bool) -> str: - if is_favorite: - return "Publication marked as favorite." - return "Publication removed from favorites." - - -async def _retry_publication_state( - db_session: AsyncSession, - *, - current_user: User, - scholar_profile_id: int, - publication_id: int, -): - publication = await publication_service.retry_pdf_for_user( - db_session, - user_id=current_user.id, - scholar_profile_id=scholar_profile_id, - publication_id=publication_id, - ) - if publication is None: - raise ApiException( - status_code=404, - code="publication_not_found", - message="Publication not found.", - ) - queued = False - if not publication.pdf_url: - queued = await publication_service.schedule_retry_pdf_enrichment_for_row( - db_session, - user_id=current_user.id, - request_email=current_user.email, - item=publication, - ) - hydrated = await publication_service.hydrate_pdf_enrichment_state( - db_session, - items=[publication], - ) - current = hydrated[0] if hydrated else publication - return current, queued - - -async def _favorite_publication_state( - db_session: AsyncSession, - *, - current_user: User, - scholar_profile_id: int, - publication_id: int, - is_favorite: bool, -): - updated_count = await publication_service.set_publication_favorite_for_user( - db_session, - user_id=current_user.id, - scholar_profile_id=scholar_profile_id, - publication_id=publication_id, - is_favorite=is_favorite, - ) - if updated_count <= 0: - raise ApiException( - status_code=404, - code="publication_not_found", - message="Publication not found.", - ) - publication = await publication_service.get_publication_item_for_user( - db_session, - user_id=current_user.id, - scholar_profile_id=scholar_profile_id, - publication_id=publication_id, - ) - if publication is None: - raise ApiException( - status_code=404, - code="publication_not_found", - message="Publication not found.", - ) - hydrated = await publication_service.hydrate_pdf_enrichment_state( - db_session, - items=[publication], - ) - current = hydrated[0] if hydrated else publication - identifiers = await identifier_service.overlay_publication_items_with_display_identifiers( - db_session, - items=[current], - ) - return identifiers[0] if identifiers else current - - -def _log_retry_pdf_result( - *, - current_user: User, - scholar_profile_id: int, - publication_id: int, - queued: bool, - resolved_pdf: bool, - pdf_status: str, - doi: str | None, -) -> None: - logger.info( - "api.publications.retry_pdf", - extra={ - "event": "api.publications.retry_pdf", - "user_id": current_user.id, - "scholar_profile_id": scholar_profile_id, - "publication_id": publication_id, - "queued": queued, - "resolved_pdf": resolved_pdf, - "pdf_status": pdf_status, - }, - ) - - -@router.get( - "", - response_model=PublicationsListEnvelope, -) -async def list_publications( - request: Request, - mode: Literal["all", "unread", "latest", "new"] | None = Query(default=None), - favorite_only: bool = Query(default=False), - scholar_profile_id: int | None = Query(default=None, ge=1), - page: int = Query(default=1, ge=1), - page_size: int = Query(default=100, ge=1, le=500), - limit: int | None = Query(default=None, ge=1, le=1000), - offset: int | None = Query(default=None, ge=0), - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - resolved_page, resolved_limit, resolved_offset = _resolve_publications_paging( - page=page, - page_size=page_size, - limit=limit, - offset=offset, - ) - resolved_mode, selected_scholar_id, publications = await _list_publications_for_request( - db_session, - current_user=current_user, - mode=mode, - favorite_only=favorite_only, - scholar_profile_id=scholar_profile_id, - limit=resolved_limit, - offset=resolved_offset, - ) - unread_count, favorites_count, latest_count, total_count = await _publication_counts( - db_session, - user_id=current_user.id, - selected_scholar_id=selected_scholar_id, - favorite_only=favorite_only, - ) - data = _publications_list_data( - mode=resolved_mode, - favorite_only=favorite_only, - selected_scholar_id=selected_scholar_id, - unread_count=unread_count, - favorites_count=favorites_count, - latest_count=latest_count, - total_count=total_count, - publications=publications, - page=resolved_page, - page_size=resolved_limit, - offset=resolved_offset, - ) - return success_payload(request, data=data) - - -@router.post( - "/mark-all-read", - response_model=MarkAllReadEnvelope, -) -async def mark_all_publications_read( - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - updated_count = await publication_service.mark_all_unread_as_read_for_user( - db_session, - user_id=current_user.id, - ) - logger.info( - "api.publications.mark_all_read", - extra={ - "event": "api.publications.mark_all_read", - "user_id": current_user.id, - "updated_count": updated_count, - }, - ) - return success_payload( - request, - data={ - "message": "Marked all unread publications as read.", - "updated_count": updated_count, - }, - ) - - -@router.post( - "/mark-read", - response_model=MarkSelectedReadEnvelope, -) -async def mark_selected_publications_read( - payload: MarkSelectedReadRequest, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - selection_pairs = sorted( - { - (int(item.scholar_profile_id), int(item.publication_id)) - for item in payload.selections - } - ) - updated_count = await publication_service.mark_selected_as_read_for_user( - db_session, - user_id=current_user.id, - selections=selection_pairs, - ) - logger.info( - "api.publications.mark_selected_read", - extra={ - "event": "api.publications.mark_selected_read", - "user_id": current_user.id, - "requested_count": len(selection_pairs), - "updated_count": updated_count, - }, - ) - return success_payload( - request, - data={ - "message": "Marked selected publications as read.", - "requested_count": len(selection_pairs), - "updated_count": updated_count, - }, - ) - - -@router.post( - "/{publication_id}/retry-pdf", - response_model=RetryPublicationPdfEnvelope, -) -async def retry_publication_pdf( - payload: RetryPublicationPdfRequest, - request: Request, - publication_id: int = Path(ge=1), - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - current, queued = await _retry_publication_state( - db_session, - current_user=current_user, - scholar_profile_id=payload.scholar_profile_id, - publication_id=publication_id, - ) - resolved_pdf = bool(current.pdf_url) - message = _retry_pdf_message( - queued=queued, - resolved_pdf=resolved_pdf, - pdf_status=current.pdf_status, - ) - _log_retry_pdf_result( - current_user=current_user, - scholar_profile_id=payload.scholar_profile_id, - publication_id=publication_id, - queued=queued, - resolved_pdf=resolved_pdf, - pdf_status=current.pdf_status, - doi=None, - ) - return success_payload( - request, - data={ - "message": message, - "queued": queued, - "resolved_pdf": resolved_pdf, - "publication": _serialize_publication_item(current), - }, - ) - - -@router.post( - "/{publication_id}/favorite", - response_model=TogglePublicationFavoriteEnvelope, -) -async def toggle_publication_favorite( - payload: TogglePublicationFavoriteRequest, - request: Request, - publication_id: int = Path(ge=1), - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - current = await _favorite_publication_state( - db_session, - current_user=current_user, - scholar_profile_id=payload.scholar_profile_id, - publication_id=publication_id, - is_favorite=payload.is_favorite, - ) - logger.info( - "api.publications.favorite", - extra={ - "event": "api.publications.favorite", - "user_id": current_user.id, - "scholar_profile_id": payload.scholar_profile_id, - "publication_id": publication_id, - "is_favorite": bool(payload.is_favorite), - }, - ) - return success_payload( - request, - data={ - "message": _favorite_message(is_favorite=bool(payload.is_favorite)), - "publication": _serialize_publication_item(current), - }, - ) diff --git a/build/lib/app/api/routers/runs.py b/build/lib/app/api/routers/runs.py deleted file mode 100644 index 4e85af4..0000000 --- a/build/lib/app/api/routers/runs.py +++ /dev/null @@ -1,845 +0,0 @@ -from __future__ import annotations - -import logging -import re -from typing import Any - -from fastapi import APIRouter, Depends, Query, Request -from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.deps import get_api_current_user -from app.api.errors import ApiException -from app.api.responses import success_payload -from fastapi.responses import StreamingResponse -from app.api.schemas import ( - ManualRunEnvelope, - QueueClearEnvelope, - QueueItemEnvelope, - QueueListEnvelope, - RunDetailEnvelope, - RunsListEnvelope, -) -from app.db.models import RunStatus, RunTriggerType, User -from app.db.session import get_db_session -from app.services.domains.ingestion import application as ingestion_service -from app.services.domains.ingestion import safety as run_safety_service -from app.services.domains.runs import application as run_service -from app.services.domains.runs.events import event_generator -from app.services.domains.settings import application as user_settings_service -from app.settings import settings -from app.api.runtime_deps import get_ingestion_service - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/runs", tags=["api-runs"]) - -IDEMPOTENCY_HEADER = "Idempotency-Key" -IDEMPOTENCY_MAX_LENGTH = 128 -IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$") - - -def _int_value(value: Any, default: int = 0) -> int: - try: - return int(value) - except (TypeError, ValueError): - return default - - -def _str_value(value: Any) -> str | None: - if value is None: - return None - text = str(value).strip() - return text if text else None - - -def _bool_value(value: Any, default: bool = False) -> bool: - if isinstance(value, bool): - return value - if isinstance(value, str): - lowered = value.strip().lower() - if lowered in {"1", "true", "yes", "on"}: - return True - if lowered in {"0", "false", "no", "off"}: - return False - return default - - -def _normalize_idempotency_key(raw_value: str | None) -> str | None: - if raw_value is None: - return None - candidate = raw_value.strip() - if not candidate: - return None - if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate): - raise ApiException( - status_code=400, - code="invalid_idempotency_key", - message=( - "Invalid Idempotency-Key. Use 8-128 characters from: " - "A-Z a-z 0-9 . _ : -" - ), - ) - return candidate - - -def _serialize_run(run) -> dict[str, Any]: - summary = run_service.extract_run_summary(run.error_log) - return { - "id": int(run.id), - "trigger_type": run.trigger_type.value, - "status": run.status.value, - "start_dt": run.start_dt, - "end_dt": run.end_dt, - "scholar_count": int(run.scholar_count or 0), - "new_publication_count": int(run.new_pub_count or 0), - "failed_count": int(summary["failed_count"]), - "partial_count": int(summary["partial_count"]), - } - - -def _serialize_queue_item(item) -> dict[str, Any]: - return { - "id": int(item.id), - "scholar_profile_id": int(item.scholar_profile_id), - "scholar_label": item.scholar_label, - "status": item.status, - "reason": item.reason, - "dropped_reason": item.dropped_reason, - "attempt_count": int(item.attempt_count), - "resume_cstart": int(item.resume_cstart), - "next_attempt_dt": item.next_attempt_dt, - "updated_at": item.updated_at, - "last_error": item.last_error, - "last_run_id": item.last_run_id, - } - - -def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]: - if not isinstance(value, list): - return [] - normalized: list[dict[str, Any]] = [] - for item in value: - if not isinstance(item, dict): - continue - normalized.append( - { - "attempt": _int_value(item.get("attempt"), 0), - "cstart": _int_value(item.get("cstart"), 0), - "state": _str_value(item.get("state")), - "state_reason": _str_value(item.get("state_reason")), - "status_code": ( - _int_value(item.get("status_code")) - if item.get("status_code") is not None - else None - ), - "fetch_error": _str_value(item.get("fetch_error")), - } - ) - return normalized - - -def _normalize_page_logs(value: Any) -> list[dict[str, Any]]: - if not isinstance(value, list): - return [] - normalized: list[dict[str, Any]] = [] - for item in value: - if not isinstance(item, dict): - continue - warning_codes = item.get("warning_codes") - normalized.append( - { - "page": _int_value(item.get("page"), 0), - "cstart": _int_value(item.get("cstart"), 0), - "state": _str_value(item.get("state")) or "unknown", - "state_reason": _str_value(item.get("state_reason")), - "status_code": ( - _int_value(item.get("status_code")) - if item.get("status_code") is not None - else None - ), - "publication_count": _int_value(item.get("publication_count"), 0), - "attempt_count": _int_value(item.get("attempt_count"), 0), - "has_show_more_button": _bool_value(item.get("has_show_more_button"), False), - "articles_range": _str_value(item.get("articles_range")), - "warning_codes": [ - str(code) - for code in (warning_codes if isinstance(warning_codes, list) else []) - ], - } - ) - return normalized - - -def _normalize_debug(value: Any) -> dict[str, Any] | None: - if not isinstance(value, dict): - return None - marker_counts = value.get("marker_counts_nonzero") - warning_codes = value.get("warning_codes") - return { - "status_code": ( - _int_value(value.get("status_code")) - if value.get("status_code") is not None - else None - ), - "final_url": _str_value(value.get("final_url")), - "fetch_error": _str_value(value.get("fetch_error")), - "body_sha256": _str_value(value.get("body_sha256")), - "body_length": ( - _int_value(value.get("body_length")) - if value.get("body_length") is not None - else None - ), - "has_show_more_button": ( - _bool_value(value.get("has_show_more_button"), False) - if value.get("has_show_more_button") is not None - else None - ), - "articles_range": _str_value(value.get("articles_range")), - "state_reason": _str_value(value.get("state_reason")), - "warning_codes": [ - str(code) - for code in (warning_codes if isinstance(warning_codes, list) else []) - ], - "marker_counts_nonzero": { - str(key): _int_value(count, 0) - for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else []) - }, - "page_logs": _normalize_page_logs(value.get("page_logs")), - "attempt_log": _normalize_attempt_log(value.get("attempt_log")), - } - - -def _normalize_scholar_result(value: Any) -> dict[str, Any]: - if not isinstance(value, dict): - return { - "scholar_profile_id": 0, - "scholar_id": "unknown", - "state": "unknown", - "state_reason": None, - "outcome": "failed", - "attempt_count": 0, - "publication_count": 0, - "start_cstart": 0, - "continuation_cstart": None, - "continuation_enqueued": False, - "continuation_cleared": False, - "warnings": [], - "error": None, - "debug": None, - } - warnings = value.get("warnings") - return { - "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0), - "scholar_id": _str_value(value.get("scholar_id")) or "unknown", - "state": _str_value(value.get("state")) or "unknown", - "state_reason": _str_value(value.get("state_reason")), - "outcome": _str_value(value.get("outcome")) or "failed", - "attempt_count": _int_value(value.get("attempt_count"), 0), - "publication_count": _int_value(value.get("publication_count"), 0), - "start_cstart": _int_value(value.get("start_cstart"), 0), - "continuation_cstart": ( - _int_value(value.get("continuation_cstart")) - if value.get("continuation_cstart") is not None - else None - ), - "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False), - "continuation_cleared": _bool_value(value.get("continuation_cleared"), False), - "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])], - "error": _str_value(value.get("error")), - "debug": _normalize_debug(value.get("debug")), - } - - -def _manual_run_payload_from_run( - run, - *, - idempotency_key: str | None, - reused_existing_run: bool, - safety_state: dict[str, Any], -) -> dict[str, Any]: - summary = run_service.extract_run_summary(run.error_log) - return { - "run_id": int(run.id), - "status": run.status.value, - "scholar_count": int(run.scholar_count or 0), - "succeeded_count": int(summary["succeeded_count"]), - "failed_count": int(summary["failed_count"]), - "partial_count": int(summary["partial_count"]), - "new_publication_count": int(run.new_pub_count or 0), - "reused_existing_run": reused_existing_run, - "idempotency_key": idempotency_key, - "safety_state": safety_state, - } - - -async def _load_safety_state( - db_session: AsyncSession, - *, - user_id: int, -) -> dict[str, Any]: - user_settings = await user_settings_service.get_or_create_settings( - db_session, - user_id=user_id, - ) - previous_safety_state = run_safety_service.get_safety_event_context(user_settings) - if run_safety_service.clear_expired_cooldown(user_settings): - await db_session.commit() - await db_session.refresh(user_settings) - logger.info( - "api.runs.safety_cooldown_cleared", - extra={ - "event": "api.runs.safety_cooldown_cleared", - "user_id": user_id, - "reason": previous_safety_state.get("cooldown_reason"), - "cooldown_until": previous_safety_state.get("cooldown_until"), - "metric_name": "api_runs_safety_cooldown_cleared_total", - "metric_value": 1, - }, - ) - return run_safety_service.get_safety_state_payload(user_settings) - - -def _raise_manual_runs_disabled(*, user_id: int, safety_state: dict[str, Any]) -> None: - logger.warning( - "api.runs.manual_blocked_policy", - extra={ - "event": "api.runs.manual_blocked_policy", - "user_id": user_id, - "policy": {"manual_run_allowed": False}, - "safety_state": safety_state, - "metric_name": "api_runs_manual_blocked_policy_total", - "metric_value": 1, - }, - ) - raise ApiException( - status_code=403, - code="manual_runs_disabled", - message="Manual checks are disabled by server policy.", - details={ - "policy": {"manual_run_allowed": False}, - "safety_state": safety_state, - }, - ) - - -async def _reused_manual_run_payload( - db_session: AsyncSession, - *, - request: Request, - user_id: int, - idempotency_key: str | None, - safety_state: dict[str, Any], -) -> dict[str, Any] | None: - if idempotency_key is None: - return None - previous_run = await run_service.get_manual_run_by_idempotency_key( - db_session, - user_id=user_id, - idempotency_key=idempotency_key, - ) - if previous_run is None: - return None - if previous_run.status == RunStatus.RUNNING: - raise ApiException( - status_code=409, - code="run_in_progress", - message="A run with this idempotency key is still in progress.", - details={"run_id": int(previous_run.id), "idempotency_key": idempotency_key}, - ) - return success_payload( - request, - data=_manual_run_payload_from_run( - previous_run, - idempotency_key=idempotency_key, - reused_existing_run=True, - safety_state=safety_state, - ), - ) - - -async def _run_ingestion_for_manual( - db_session: AsyncSession, - *, - ingest_service: ingestion_service.ScholarIngestionService, - user_id: int, - idempotency_key: str | None, -): - user_settings = await user_settings_service.get_or_create_settings( - db_session, - user_id=user_id, - ) - return await ingest_service.run_for_user( - db_session, - user_id=user_id, - trigger_type=RunTriggerType.MANUAL, - request_delay_seconds=user_settings.request_delay_seconds, - network_error_retries=settings.ingestion_network_error_retries, - retry_backoff_seconds=settings.ingestion_retry_backoff_seconds, - max_pages_per_scholar=settings.ingestion_max_pages_per_scholar, - page_size=settings.ingestion_page_size, - auto_queue_continuations=settings.ingestion_continuation_queue_enabled, - queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds, - idempotency_key=idempotency_key, - alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, - alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, - alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, - ) - - -async def _recover_integrity_error( - db_session: AsyncSession, - *, - request: Request, - user_id: int, - idempotency_key: str | None, - original_exc: IntegrityError, -) -> dict[str, Any]: - if idempotency_key is None: - logger.exception( - "api.runs.manual_integrity_error", - extra={"event": "api.runs.manual_integrity_error", "user_id": user_id}, - ) - raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc - existing_run = await run_service.get_manual_run_by_idempotency_key( - db_session, - user_id=user_id, - idempotency_key=idempotency_key, - ) - if existing_run is None: - logger.exception( - "api.runs.manual_integrity_error", - extra={"event": "api.runs.manual_integrity_error", "user_id": user_id}, - ) - raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc - if existing_run.status == RunStatus.RUNNING: - raise ApiException( - status_code=409, - code="run_in_progress", - message="A run with this idempotency key is still in progress.", - details={"run_id": int(existing_run.id), "idempotency_key": idempotency_key}, - ) from original_exc - return success_payload( - request, - data=_manual_run_payload_from_run( - existing_run, - idempotency_key=idempotency_key, - reused_existing_run=True, - safety_state=await _load_safety_state(db_session, user_id=user_id), - ), - ) - - -async def _execute_manual_run( - db_session: AsyncSession, - *, - request: Request, - ingest_service: ingestion_service.ScholarIngestionService, - user_id: int, - idempotency_key: str | None, -): - try: - return await _run_ingestion_for_manual( - db_session, - ingest_service=ingest_service, - user_id=user_id, - idempotency_key=idempotency_key, - ) - except ingestion_service.RunAlreadyInProgressError as exc: - await db_session.rollback() - raise ApiException( - status_code=409, - code="run_in_progress", - message="A run is already in progress for this account.", - ) from exc - except ingestion_service.RunBlockedBySafetyPolicyError as exc: - await db_session.rollback() - _raise_manual_blocked_safety(exc=exc, user_id=user_id) - except IntegrityError as exc: - await db_session.rollback() - return await _recover_integrity_error( - db_session, - request=request, - user_id=user_id, - idempotency_key=idempotency_key, - original_exc=exc, - ) - except Exception as exc: - await db_session.rollback() - _raise_manual_failed(exc=exc, user_id=user_id) - - -def _raise_manual_blocked_safety(*, exc, user_id: int) -> None: - logger.info( - "api.runs.manual_blocked_safety", - extra={ - "event": "api.runs.manual_blocked_safety", - "user_id": user_id, - "reason": exc.safety_state.get("cooldown_reason"), - "cooldown_until": exc.safety_state.get("cooldown_until"), - "cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"), - "metric_name": "api_runs_manual_blocked_safety_total", - "metric_value": 1, - }, - ) - raise ApiException( - status_code=429, - code=exc.code, - message=exc.message, - details={"safety_state": exc.safety_state}, - ) from exc - - -def _raise_manual_failed(*, exc: Exception, user_id: int) -> None: - logger.exception( - "api.runs.manual_failed", - extra={"event": "api.runs.manual_failed", "user_id": user_id}, - ) - raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from exc - - -def _manual_run_success_payload( - *, - run_summary, - idempotency_key: str | None, - safety_state: dict[str, Any], -) -> dict[str, Any]: - return { - "run_id": run_summary.crawl_run_id, - "status": run_summary.status.value, - "scholar_count": run_summary.scholar_count, - "succeeded_count": run_summary.succeeded_count, - "failed_count": run_summary.failed_count, - "partial_count": run_summary.partial_count, - "new_publication_count": run_summary.new_publication_count, - "reused_existing_run": False, - "idempotency_key": idempotency_key, - "safety_state": safety_state, - } - - -@router.get( - "", - response_model=RunsListEnvelope, -) -async def list_runs( - request: Request, - failed_only: bool = Query(default=False), - limit: int = Query(default=100, ge=1, le=500), - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - runs = await run_service.list_runs_for_user( - db_session, - user_id=current_user.id, - limit=limit, - failed_only=failed_only, - ) - safety_state = await _load_safety_state( - db_session, - user_id=current_user.id, - ) - return success_payload( - request, - data={ - "runs": [_serialize_run(run) for run in runs], - "safety_state": safety_state, - }, - ) - - -@router.get( - "/{run_id}", - response_model=RunDetailEnvelope, -) -async def get_run( - run_id: int, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - run = await run_service.get_run_for_user( - db_session, - user_id=current_user.id, - run_id=run_id, - ) - if run is None: - raise ApiException( - status_code=404, - code="run_not_found", - message="Run not found.", - ) - error_log = run.error_log if isinstance(run.error_log, dict) else {} - scholar_results = error_log.get("scholar_results") - if not isinstance(scholar_results, list): - scholar_results = [] - safety_state = await _load_safety_state( - db_session, - user_id=current_user.id, - ) - return success_payload( - request, - data={ - "run": _serialize_run(run), - "summary": run_service.extract_run_summary(error_log), - "scholar_results": [_normalize_scholar_result(item) for item in scholar_results], - "safety_state": safety_state, - }, - ) - - -@router.post( - "/{run_id}/cancel", - response_model=RunDetailEnvelope, -) -async def cancel_run( - run_id: int, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - run = await run_service.get_run_for_user( - db_session, - user_id=current_user.id, - run_id=run_id, - ) - if run is None: - raise ApiException( - status_code=404, - code="run_not_found", - message="Run not found.", - ) - - if run.status == RunStatus.RUNNING: - run.status = RunStatus.CANCELED - await db_session.commit() - await db_session.refresh(run) - - error_log = run.error_log if isinstance(run.error_log, dict) else {} - scholar_results = error_log.get("scholar_results") - if not isinstance(scholar_results, list): - scholar_results = [] - - safety_state = await _load_safety_state( - db_session, - user_id=current_user.id, - ) - - return success_payload( - request, - data={ - "run": _serialize_run(run), - "summary": run_service.extract_run_summary(error_log), - "scholar_results": [_normalize_scholar_result(item) for item in scholar_results], - "safety_state": safety_state, - }, - ) - - -@router.post( - "/manual", - response_model=ManualRunEnvelope, -) -async def run_manual( - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), - ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service), -): - safety_state = await _load_safety_state( - db_session, - user_id=current_user.id, - ) - if not settings.ingestion_manual_run_allowed: - _raise_manual_runs_disabled(user_id=current_user.id, safety_state=safety_state) - - idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER)) - reused_payload = await _reused_manual_run_payload( - db_session, - request=request, - user_id=current_user.id, - idempotency_key=idempotency_key, - safety_state=safety_state, - ) - if reused_payload is not None: - return reused_payload - - run_summary = await _execute_manual_run( - db_session, - request=request, - ingest_service=ingest_service, - user_id=current_user.id, - idempotency_key=idempotency_key, - ) - if isinstance(run_summary, dict): - return run_summary - - current_safety_state = await _load_safety_state( - db_session, - user_id=current_user.id, - ) - return success_payload( - request, - data=_manual_run_success_payload( - run_summary=run_summary, - idempotency_key=idempotency_key, - safety_state=current_safety_state, - ), - ) - - - -@router.get( - "/queue/items", - response_model=QueueListEnvelope, -) -async def list_queue_items( - request: Request, - limit: int = Query(default=200, ge=1, le=500), - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - items = await run_service.list_queue_items_for_user( - db_session, - user_id=current_user.id, - limit=limit, - ) - return success_payload( - request, - data={ - "queue_items": [_serialize_queue_item(item) for item in items], - }, - ) - - -@router.post( - "/queue/{queue_item_id}/retry", - response_model=QueueItemEnvelope, -) -async def retry_queue_item( - queue_item_id: int, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - try: - queue_item = await run_service.retry_queue_item_for_user( - db_session, - user_id=current_user.id, - queue_item_id=queue_item_id, - ) - except run_service.QueueTransitionError as exc: - raise ApiException( - status_code=409, - code=exc.code, - message=exc.message, - details={"current_status": exc.current_status}, - ) from exc - if queue_item is None: - raise ApiException( - status_code=404, - code="queue_item_not_found", - message="Queue item not found.", - ) - return success_payload( - request, - data=_serialize_queue_item(queue_item), - ) - - -@router.post( - "/queue/{queue_item_id}/drop", - response_model=QueueItemEnvelope, -) -async def drop_queue_item( - queue_item_id: int, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - try: - dropped = await run_service.drop_queue_item_for_user( - db_session, - user_id=current_user.id, - queue_item_id=queue_item_id, - ) - except run_service.QueueTransitionError as exc: - raise ApiException( - status_code=409, - code=exc.code, - message=exc.message, - details={"current_status": exc.current_status}, - ) from exc - if dropped is None: - raise ApiException( - status_code=404, - code="queue_item_not_found", - message="Queue item not found.", - ) - return success_payload( - request, - data=_serialize_queue_item(dropped), - ) - - -@router.delete( - "/queue/{queue_item_id}", - response_model=QueueClearEnvelope, -) -async def clear_queue_item( - queue_item_id: int, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - try: - deleted = await run_service.clear_queue_item_for_user( - db_session, - user_id=current_user.id, - queue_item_id=queue_item_id, - ) - except run_service.QueueTransitionError as exc: - raise ApiException( - status_code=409, - code=exc.code, - message=exc.message, - details={"current_status": exc.current_status}, - ) from exc - if deleted is None: - raise ApiException( - status_code=404, - code="queue_item_not_found", - message="Queue item not found.", - ) - return success_payload( - request, - data={ - "queue_item_id": deleted.queue_item_id, - "previous_status": deleted.previous_status, - "status": "cleared", - "message": "Queue item cleared.", - }, - ) - - -@router.get("/{run_id}/stream") -async def stream_run_events( - run_id: int, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - run = await run_service.get_run_for_user( - db_session, - user_id=current_user.id, - run_id=run_id, - ) - if run is None: - raise ApiException( - status_code=404, - code="run_not_found", - message="Run not found.", - ) - return StreamingResponse( - event_generator(run_id), - media_type="text/event-stream" - ) diff --git a/build/lib/app/api/routers/scholars.py b/build/lib/app/api/routers/scholars.py deleted file mode 100644 index f4be187..0000000 --- a/build/lib/app/api/routers/scholars.py +++ /dev/null @@ -1,557 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging - -from fastapi import APIRouter, Depends, File, Query, Request, UploadFile -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.deps import get_api_current_user -from app.api.errors import ApiException -from app.api.responses import success_payload -from app.api.runtime_deps import get_scholar_source -from app.api.schemas import ( - DataExportEnvelope, - DataImportEnvelope, - DataImportRequest, - MessageEnvelope, - ScholarCreateRequest, - ScholarEnvelope, - ScholarImageUrlUpdateRequest, - ScholarSearchEnvelope, - ScholarsListEnvelope, -) -from app.db.models import User -from app.db.session import get_db_session -from app.services.domains.portability import application as import_export_service -from app.services.domains.scholars import application as scholar_service -from app.services.domains.scholar.source import ScholarSource -from app.settings import settings - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/scholars", tags=["api-scholars"]) - - -def _uploaded_image_media_path(scholar_profile_id: int) -> str: - return f"/scholar-images/{scholar_profile_id}/upload" - - -def _serialize_scholar(profile) -> dict[str, object]: - uploaded_image_url = None - if profile.profile_image_upload_path: - uploaded_image_url = _uploaded_image_media_path(int(profile.id)) - - profile_image_url, profile_image_source = scholar_service.resolve_profile_image( - profile, - uploaded_image_url=uploaded_image_url, - ) - - return { - "id": int(profile.id), - "scholar_id": profile.scholar_id, - "display_name": profile.display_name, - "profile_image_url": profile_image_url, - "profile_image_source": profile_image_source, - "is_enabled": bool(profile.is_enabled), - "baseline_completed": bool(profile.baseline_completed), - "last_run_dt": profile.last_run_dt, - "last_run_status": ( - profile.last_run_status.value if profile.last_run_status is not None else None - ), - } - - -async def _hydrate_scholar_metadata_if_needed( - db_session: AsyncSession, - *, - profile, - source: ScholarSource, - user_id: int, -): - try: - if not profile.profile_image_url or not (profile.display_name or "").strip(): - return await asyncio.wait_for( - scholar_service.hydrate_profile_metadata( - db_session, - profile=profile, - source=source, - ), - timeout=5.0, - ) - except Exception: - logger.warning( - "api.scholars.create_metadata_hydration_failed", - extra={ - "event": "api.scholars.create_metadata_hydration_failed", - "user_id": user_id, - "scholar_profile_id": profile.id, - }, - ) - return profile - - -def _search_kwargs() -> dict[str, object]: - return { - "network_error_retries": settings.ingestion_network_error_retries, - "retry_backoff_seconds": settings.ingestion_retry_backoff_seconds, - "search_enabled": settings.scholar_name_search_enabled, - "cache_ttl_seconds": settings.scholar_name_search_cache_ttl_seconds, - "blocked_cache_ttl_seconds": settings.scholar_name_search_blocked_cache_ttl_seconds, - "cache_max_entries": settings.scholar_name_search_cache_max_entries, - "min_interval_seconds": settings.scholar_name_search_min_interval_seconds, - "interval_jitter_seconds": settings.scholar_name_search_interval_jitter_seconds, - "cooldown_block_threshold": settings.scholar_name_search_cooldown_block_threshold, - "cooldown_seconds": settings.scholar_name_search_cooldown_seconds, - "retry_alert_threshold": settings.scholar_name_search_alert_retry_count_threshold, - "cooldown_rejection_alert_threshold": ( - settings.scholar_name_search_alert_cooldown_rejections_threshold - ), - } - - -def _search_response_data(query: str, parsed) -> dict[str, object]: - return { - "query": query.strip(), - "state": parsed.state.value, - "state_reason": parsed.state_reason, - "action_hint": scholar_service.scrape_state_hint( - state=parsed.state, - state_reason=parsed.state_reason, - ), - "candidates": [ - { - "scholar_id": item.scholar_id, - "display_name": item.display_name, - "affiliation": item.affiliation, - "email_domain": item.email_domain, - "cited_by_count": item.cited_by_count, - "interests": item.interests, - "profile_url": item.profile_url, - "profile_image_url": item.profile_image_url, - } - for item in parsed.candidates - ], - "warnings": parsed.warnings, - } - - -async def _read_uploaded_image(image: UploadFile) -> bytes: - try: - return await image.read() - finally: - await image.close() - - -async def _require_user_profile( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int, -): - profile = await scholar_service.get_user_scholar_by_id( - db_session, - user_id=user_id, - scholar_profile_id=scholar_profile_id, - ) - if profile is None: - raise ApiException( - status_code=404, - code="scholar_not_found", - message="Scholar not found.", - ) - return profile - - -@router.get( - "", - response_model=ScholarsListEnvelope, -) -async def list_scholars( - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - scholars = await scholar_service.list_scholars_for_user( - db_session, - user_id=current_user.id, - ) - return success_payload( - request, - data={ - "scholars": [_serialize_scholar(profile) for profile in scholars], - }, - ) - - -@router.get( - "/export", - response_model=DataExportEnvelope, -) -async def export_scholars_and_publications( - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - data = await import_export_service.export_user_data( - db_session, - user_id=current_user.id, - ) - return success_payload(request, data=data) - - -@router.post( - "/import", - response_model=DataImportEnvelope, -) -async def import_scholars_and_publications( - payload: DataImportRequest, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - if ( - payload.schema_version is not None - and int(payload.schema_version) != import_export_service.EXPORT_SCHEMA_VERSION - ): - raise ApiException( - status_code=400, - code="invalid_import_schema_version", - message=( - "Import schema version is not supported. " - f"Expected {import_export_service.EXPORT_SCHEMA_VERSION}." - ), - ) - try: - result = await import_export_service.import_user_data( - db_session, - user_id=current_user.id, - scholars=[item.model_dump() for item in payload.scholars], - publications=[item.model_dump() for item in payload.publications], - ) - except import_export_service.ImportExportError as exc: - raise ApiException( - status_code=400, - code="invalid_import_payload", - message=str(exc), - ) from exc - logger.info( - "api.scholars.imported", - extra={ - "event": "api.scholars.imported", - "user_id": current_user.id, - **result, - }, - ) - return success_payload(request, data=result) - - -@router.post( - "", - response_model=ScholarEnvelope, - status_code=201, -) -async def create_scholar( - payload: ScholarCreateRequest, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - source: ScholarSource = Depends(get_scholar_source), - current_user: User = Depends(get_api_current_user), -): - try: - created = await scholar_service.create_scholar_for_user( - db_session, - user_id=current_user.id, - scholar_id=payload.scholar_id, - display_name="", - profile_image_url=payload.profile_image_url, - ) - except scholar_service.ScholarServiceError as exc: - raise ApiException( - status_code=400, - code="invalid_scholar", - message=str(exc), - ) from exc - logger.info( - "api.scholars.created", - extra={ - "event": "api.scholars.created", - "user_id": current_user.id, - "scholar_profile_id": created.id, - }, - ) - created = await _hydrate_scholar_metadata_if_needed( - db_session, - profile=created, - source=source, - user_id=current_user.id, - ) - - return success_payload( - request, - data=_serialize_scholar(created), - ) - - -@router.get( - "/search", - response_model=ScholarSearchEnvelope, -) -async def search_scholars( - request: Request, - query: str = Query(..., min_length=2, max_length=120), - limit: int = Query(10, ge=1, le=25), - db_session: AsyncSession = Depends(get_db_session), - source: ScholarSource = Depends(get_scholar_source), - current_user: User = Depends(get_api_current_user), -): - try: - parsed = await scholar_service.search_author_candidates( - source=source, - db_session=db_session, - query=query, - limit=limit, - **_search_kwargs(), - ) - except scholar_service.ScholarServiceError as exc: - raise ApiException( - status_code=400, - code="invalid_scholar_search", - message=str(exc), - ) from exc - - logger.info( - "api.scholars.search.completed", - extra={ - "event": "api.scholars.search.completed", - "user_id": current_user.id, - "query": query.strip(), - "candidate_count": len(parsed.candidates), - "state": parsed.state.value, - }, - ) - return success_payload( - request, - data=_search_response_data(query, parsed), - ) - - -@router.patch( - "/{scholar_profile_id}/toggle", - response_model=ScholarEnvelope, -) -async def toggle_scholar( - scholar_profile_id: int, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - profile = await scholar_service.get_user_scholar_by_id( - db_session, - user_id=current_user.id, - scholar_profile_id=scholar_profile_id, - ) - if profile is None: - raise ApiException( - status_code=404, - code="scholar_not_found", - message="Scholar not found.", - ) - updated = await scholar_service.toggle_scholar_enabled(db_session, profile=profile) - logger.info( - "api.scholars.toggled", - extra={ - "event": "api.scholars.toggled", - "user_id": current_user.id, - "scholar_profile_id": updated.id, - "is_enabled": updated.is_enabled, - }, - ) - return success_payload( - request, - data=_serialize_scholar(updated), - ) - - -@router.delete( - "/{scholar_profile_id}", - response_model=MessageEnvelope, -) -async def delete_scholar( - scholar_profile_id: int, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - profile = await scholar_service.get_user_scholar_by_id( - db_session, - user_id=current_user.id, - scholar_profile_id=scholar_profile_id, - ) - if profile is None: - raise ApiException( - status_code=404, - code="scholar_not_found", - message="Scholar not found.", - ) - await scholar_service.delete_scholar( - db_session, - profile=profile, - upload_dir=settings.scholar_image_upload_dir, - ) - logger.info( - "api.scholars.deleted", - extra={ - "event": "api.scholars.deleted", - "user_id": current_user.id, - "scholar_profile_id": scholar_profile_id, - }, - ) - return success_payload( - request, - data={"message": "Scholar deleted."}, - ) - - -@router.put( - "/{scholar_profile_id}/image/url", - response_model=ScholarEnvelope, -) -async def update_scholar_image_url( - scholar_profile_id: int, - payload: ScholarImageUrlUpdateRequest, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - profile = await scholar_service.get_user_scholar_by_id( - db_session, - user_id=current_user.id, - scholar_profile_id=scholar_profile_id, - ) - if profile is None: - raise ApiException( - status_code=404, - code="scholar_not_found", - message="Scholar not found.", - ) - - try: - updated = await scholar_service.set_profile_image_override_url( - db_session, - profile=profile, - image_url=payload.image_url, - upload_dir=settings.scholar_image_upload_dir, - ) - except scholar_service.ScholarServiceError as exc: - raise ApiException( - status_code=400, - code="invalid_scholar_image", - message=str(exc), - ) from exc - - logger.info( - "api.scholars.image_url_updated", - extra={ - "event": "api.scholars.image_url_updated", - "user_id": current_user.id, - "scholar_profile_id": updated.id, - }, - ) - return success_payload( - request, - data=_serialize_scholar(updated), - ) - - -@router.post( - "/{scholar_profile_id}/image/upload", - response_model=ScholarEnvelope, -) -async def upload_scholar_image( - scholar_profile_id: int, - request: Request, - image: UploadFile = File(...), - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - profile = await _require_user_profile( - db_session, - user_id=current_user.id, - scholar_profile_id=scholar_profile_id, - ) - - image_bytes = await _read_uploaded_image(image) - try: - updated = await scholar_service.set_profile_image_upload( - db_session, - profile=profile, - content_type=image.content_type, - image_bytes=image_bytes, - upload_dir=settings.scholar_image_upload_dir, - max_upload_bytes=settings.scholar_image_upload_max_bytes, - ) - except scholar_service.ScholarServiceError as exc: - raise ApiException( - status_code=400, - code="invalid_scholar_image", - message=str(exc), - ) from exc - - image_size = len(image_bytes) - logger.info( - "api.scholars.image_uploaded", - extra={ - "event": "api.scholars.image_uploaded", - "user_id": current_user.id, - "scholar_profile_id": updated.id, - "content_type": image.content_type, - "size_bytes": image_size, - }, - ) - return success_payload( - request, - data=_serialize_scholar(updated), - ) - - -@router.delete( - "/{scholar_profile_id}/image", - response_model=ScholarEnvelope, -) -async def clear_scholar_image_customization( - scholar_profile_id: int, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - profile = await scholar_service.get_user_scholar_by_id( - db_session, - user_id=current_user.id, - scholar_profile_id=scholar_profile_id, - ) - if profile is None: - raise ApiException( - status_code=404, - code="scholar_not_found", - message="Scholar not found.", - ) - - updated = await scholar_service.clear_profile_image_customization( - db_session, - profile=profile, - upload_dir=settings.scholar_image_upload_dir, - ) - logger.info( - "api.scholars.image_customization_cleared", - extra={ - "event": "api.scholars.image_customization_cleared", - "user_id": current_user.id, - "scholar_profile_id": updated.id, - }, - ) - return success_payload( - request, - data=_serialize_scholar(updated), - ) - diff --git a/build/lib/app/api/routers/settings.py b/build/lib/app/api/routers/settings.py deleted file mode 100644 index 5a2ef91..0000000 --- a/build/lib/app/api/routers/settings.py +++ /dev/null @@ -1,189 +0,0 @@ -from __future__ import annotations - -import logging - -from fastapi import APIRouter, Depends, Request -from sqlalchemy.ext.asyncio import AsyncSession - -from app.api.deps import get_api_current_user -from app.api.errors import ApiException -from app.api.responses import success_payload -from app.api.schemas import SettingsEnvelope, SettingsUpdateRequest -from app.db.models import User -from app.db.session import get_db_session -from app.services.domains.ingestion import safety as run_safety_service -from app.services.domains.settings import application as user_settings_service -from app.settings import settings as settings_module - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/settings", tags=["api-settings"]) - - -def _serialize_settings(user_settings) -> dict[str, object]: - min_run_interval_minutes, min_request_delay_seconds = _minimum_policy() - return { - "auto_run_enabled": bool(user_settings.auto_run_enabled) and settings_module.ingestion_automation_allowed, - "run_interval_minutes": int(user_settings.run_interval_minutes), - "request_delay_seconds": int(user_settings.request_delay_seconds), - "nav_visible_pages": list(user_settings.nav_visible_pages or []), - "policy": { - "min_run_interval_minutes": min_run_interval_minutes, - "min_request_delay_seconds": min_request_delay_seconds, - "automation_allowed": bool(settings_module.ingestion_automation_allowed), - "manual_run_allowed": bool(settings_module.ingestion_manual_run_allowed), - "blocked_failure_threshold": max(1, int(settings_module.ingestion_alert_blocked_failure_threshold)), - "network_failure_threshold": max(1, int(settings_module.ingestion_alert_network_failure_threshold)), - "cooldown_blocked_seconds": max(60, int(settings_module.ingestion_safety_cooldown_blocked_seconds)), - "cooldown_network_seconds": max(60, int(settings_module.ingestion_safety_cooldown_network_seconds)), - }, - "safety_state": run_safety_service.get_safety_state_payload(user_settings), - "openalex_api_key": user_settings.openalex_api_key, - "crossref_api_token": user_settings.crossref_api_token, - "crossref_api_mailto": user_settings.crossref_api_mailto, - } - - -def _minimum_policy() -> tuple[int, int]: - min_run_interval_minutes = user_settings_service.resolve_run_interval_minimum( - settings_module.ingestion_min_run_interval_minutes - ) - min_request_delay_seconds = user_settings_service.resolve_request_delay_minimum( - settings_module.ingestion_min_request_delay_seconds - ) - return min_run_interval_minutes, min_request_delay_seconds - - -def _parse_settings_payload(payload: SettingsUpdateRequest, user_settings) -> tuple[int, int, list[str]]: - min_run_interval_minutes, min_request_delay_seconds = _minimum_policy() - parsed_interval = user_settings_service.parse_run_interval_minutes( - str(payload.run_interval_minutes), - minimum=min_run_interval_minutes, - ) - parsed_delay = user_settings_service.parse_request_delay_seconds( - str(payload.request_delay_seconds), - minimum=min_request_delay_seconds, - ) - parsed_nav_visible_pages = user_settings_service.parse_nav_visible_pages( - payload.nav_visible_pages - if payload.nav_visible_pages is not None - else list(user_settings.nav_visible_pages or user_settings_service.DEFAULT_NAV_VISIBLE_PAGES) - ) - return parsed_interval, parsed_delay, parsed_nav_visible_pages - - -async def _clear_expired_cooldown_with_log( - db_session: AsyncSession, - *, - user_id: int, - user_settings, -) -> None: - previous_safety_state = run_safety_service.get_safety_event_context(user_settings) - if not run_safety_service.clear_expired_cooldown(user_settings): - return - await db_session.commit() - await db_session.refresh(user_settings) - logger.info( - "api.settings.safety_cooldown_cleared", - extra={ - "event": "api.settings.safety_cooldown_cleared", - "user_id": user_id, - "reason": previous_safety_state.get("cooldown_reason"), - "cooldown_until": previous_safety_state.get("cooldown_until"), - "metric_name": "api_settings_safety_cooldown_cleared_total", - "metric_value": 1, - }, - ) - - -def _effective_auto_run_enabled(payload: SettingsUpdateRequest) -> bool: - return bool(payload.auto_run_enabled) and settings_module.ingestion_automation_allowed - - -def _log_settings_update(*, user_id: int, updated) -> None: - logger.info( - "api.settings.updated", - extra={ - "event": "api.settings.updated", - "user_id": user_id, - "auto_run_enabled": updated.auto_run_enabled, - "run_interval_minutes": updated.run_interval_minutes, - "request_delay_seconds": updated.request_delay_seconds, - "nav_visible_pages": updated.nav_visible_pages, - "openalex_api_key": "SET" if updated.openalex_api_key else "UNSET", - }, - ) - - -@router.get( - "", - response_model=SettingsEnvelope, -) -async def get_settings( - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - user_settings = await user_settings_service.get_or_create_settings( - db_session, - user_id=current_user.id, - ) - await _clear_expired_cooldown_with_log( - db_session, - user_id=current_user.id, - user_settings=user_settings, - ) - return success_payload( - request, - data=_serialize_settings(user_settings), - ) - - -@router.put( - "", - response_model=SettingsEnvelope, -) -async def update_settings( - payload: SettingsUpdateRequest, - request: Request, - db_session: AsyncSession = Depends(get_db_session), - current_user: User = Depends(get_api_current_user), -): - user_settings = await user_settings_service.get_or_create_settings( - db_session, - user_id=current_user.id, - ) - - try: - parsed_interval, parsed_delay, parsed_nav_visible_pages = _parse_settings_payload( - payload, - user_settings, - ) - except user_settings_service.UserSettingsServiceError as exc: - raise ApiException( - status_code=400, - code="invalid_settings", - message=str(exc), - ) from exc - - updated = await user_settings_service.update_settings( - db_session, - settings=user_settings, - auto_run_enabled=_effective_auto_run_enabled(payload), - run_interval_minutes=parsed_interval, - request_delay_seconds=parsed_delay, - nav_visible_pages=parsed_nav_visible_pages, - openalex_api_key=payload.openalex_api_key, - crossref_api_token=payload.crossref_api_token, - crossref_api_mailto=payload.crossref_api_mailto, - ) - await _clear_expired_cooldown_with_log( - db_session, - user_id=current_user.id, - user_settings=updated, - ) - _log_settings_update(user_id=current_user.id, updated=updated) - return success_payload( - request, - data=_serialize_settings(updated), - ) diff --git a/build/lib/app/api/runtime_deps.py b/build/lib/app/api/runtime_deps.py deleted file mode 100644 index 3a692a3..0000000 --- a/build/lib/app/api/runtime_deps.py +++ /dev/null @@ -1,16 +0,0 @@ -from __future__ import annotations - -from fastapi import Depends - -from app.services.domains.ingestion import application as ingestion_service -from app.services.domains.scholar import source as scholar_source_service - - -def get_scholar_source() -> scholar_source_service.ScholarSource: - return scholar_source_service.LiveScholarSource() - - -def get_ingestion_service( - source: scholar_source_service.ScholarSource = Depends(get_scholar_source), -) -> ingestion_service.ScholarIngestionService: - return ingestion_service.ScholarIngestionService(source=source) diff --git a/build/lib/app/api/schemas.py b/build/lib/app/api/schemas.py deleted file mode 100644 index bac3bb9..0000000 --- a/build/lib/app/api/schemas.py +++ /dev/null @@ -1,884 +0,0 @@ -from __future__ import annotations - -from datetime import datetime -from typing import Any, Literal - -from pydantic import BaseModel, ConfigDict, Field, model_validator - - -class ApiMeta(BaseModel): - request_id: str | None = None - - model_config = ConfigDict(extra="forbid") - - -class ApiErrorData(BaseModel): - code: str - message: str - details: Any | None = None - - model_config = ConfigDict(extra="forbid") - - -class ApiErrorEnvelope(BaseModel): - error: ApiErrorData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class MessageData(BaseModel): - message: str - - model_config = ConfigDict(extra="forbid") - - -class MessageEnvelope(BaseModel): - data: MessageData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class SessionUserData(BaseModel): - id: int - email: str - is_admin: bool - is_active: bool - - model_config = ConfigDict(extra="forbid") - - -class AuthMeData(BaseModel): - authenticated: bool - csrf_token: str - user: SessionUserData - - model_config = ConfigDict(extra="forbid") - - -class AuthMeEnvelope(BaseModel): - data: AuthMeData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class CsrfBootstrapData(BaseModel): - csrf_token: str - authenticated: bool - - model_config = ConfigDict(extra="forbid") - - -class CsrfBootstrapEnvelope(BaseModel): - data: CsrfBootstrapData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class LoginRequest(BaseModel): - email: str - password: str - - model_config = ConfigDict(extra="forbid") - - -class LoginData(BaseModel): - authenticated: bool - csrf_token: str - user: SessionUserData - - model_config = ConfigDict(extra="forbid") - - -class LoginEnvelope(BaseModel): - data: LoginData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class ChangePasswordRequest(BaseModel): - current_password: str - new_password: str - confirm_password: str - - model_config = ConfigDict(extra="forbid") - - -class ScholarItemData(BaseModel): - id: int - scholar_id: str - display_name: str | None - profile_image_url: str | None - profile_image_source: str - is_enabled: bool - baseline_completed: bool - last_run_dt: datetime | None - last_run_status: str | None - - model_config = ConfigDict(extra="forbid") - - -class ScholarsListData(BaseModel): - scholars: list[ScholarItemData] - - model_config = ConfigDict(extra="forbid") - - -class ScholarsListEnvelope(BaseModel): - data: ScholarsListData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class ScholarCreateRequest(BaseModel): - scholar_id: str - profile_image_url: str | None = None - - model_config = ConfigDict(extra="forbid") - - -class ScholarEnvelope(BaseModel): - data: ScholarItemData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class ScholarSearchCandidateData(BaseModel): - scholar_id: str - display_name: str - affiliation: str | None - email_domain: str | None - cited_by_count: int | None - interests: list[str] = Field(default_factory=list) - profile_url: str - profile_image_url: str | None - - model_config = ConfigDict(extra="forbid") - - -class ScholarSearchData(BaseModel): - query: str - state: str - state_reason: str - action_hint: str | None = None - candidates: list[ScholarSearchCandidateData] - warnings: list[str] = Field(default_factory=list) - - model_config = ConfigDict(extra="forbid") - - -class ScholarSearchEnvelope(BaseModel): - data: ScholarSearchData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class ScholarImageUrlUpdateRequest(BaseModel): - image_url: str - - model_config = ConfigDict(extra="forbid") - - -class ScholarExportItemData(BaseModel): - scholar_id: str - display_name: str | None = None - is_enabled: bool = True - profile_image_override_url: str | None = None - - model_config = ConfigDict(extra="forbid") - - -class PublicationExportItemData(BaseModel): - scholar_id: str - cluster_id: str | None = None - fingerprint_sha256: str | None = None - title: str - year: int | None = None - citation_count: int = 0 - author_text: str | None = None - venue_text: str | None = None - pub_url: str | None = None - pdf_url: str | None = None - is_read: bool = False - - model_config = ConfigDict(extra="forbid") - - -class DataExportData(BaseModel): - schema_version: int - exported_at: str - scholars: list[ScholarExportItemData] - publications: list[PublicationExportItemData] - - model_config = ConfigDict(extra="forbid") - - -class DataExportEnvelope(BaseModel): - data: DataExportData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class DataImportRequest(BaseModel): - schema_version: int | None = None - scholars: list[ScholarExportItemData] = Field(default_factory=list) - publications: list[PublicationExportItemData] = Field(default_factory=list) - - model_config = ConfigDict(extra="forbid") - - -class DataImportResultData(BaseModel): - scholars_created: int - scholars_updated: int - publications_created: int - publications_updated: int - links_created: int - links_updated: int - skipped_records: int - - model_config = ConfigDict(extra="forbid") - - -class DataImportEnvelope(BaseModel): - data: DataImportResultData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class RunListItemData(BaseModel): - id: int - trigger_type: str - status: str - start_dt: datetime - end_dt: datetime | None - scholar_count: int - new_publication_count: int - failed_count: int - partial_count: int - - model_config = ConfigDict(extra="forbid") - - -class RunsListData(BaseModel): - runs: list[RunListItemData] - safety_state: ScrapeSafetyStateData - - model_config = ConfigDict(extra="forbid") - - -class RunsListEnvelope(BaseModel): - data: RunsListData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class RunSummaryData(BaseModel): - succeeded_count: int - failed_count: int - partial_count: int - failed_state_counts: dict[str, int] = Field(default_factory=dict) - failed_reason_counts: dict[str, int] = Field(default_factory=dict) - scrape_failure_counts: dict[str, int] = Field(default_factory=dict) - retry_counts: dict[str, int] = Field(default_factory=dict) - alert_thresholds: dict[str, int] = Field(default_factory=dict) - alert_flags: dict[str, bool] = Field(default_factory=dict) - - model_config = ConfigDict(extra="forbid") - - -class ScrapeSafetyCountersData(BaseModel): - consecutive_blocked_runs: int = 0 - consecutive_network_runs: int = 0 - cooldown_entry_count: int = 0 - blocked_start_count: int = 0 - last_blocked_failure_count: int = 0 - last_network_failure_count: int = 0 - last_evaluated_run_id: int | None = None - - model_config = ConfigDict(extra="forbid") - - -class ScrapeSafetyStateData(BaseModel): - cooldown_active: bool - cooldown_reason: str | None = None - cooldown_reason_label: str | None = None - cooldown_until: datetime | None = None - cooldown_remaining_seconds: int = 0 - recommended_action: str | None = None - counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData) - - model_config = ConfigDict(extra="forbid") - - -class RunAttemptLogData(BaseModel): - attempt: int - cstart: int - state: str | None = None - state_reason: str | None = None - status_code: int | None = None - fetch_error: str | None = None - - model_config = ConfigDict(extra="forbid") - - -class RunPageLogData(BaseModel): - page: int - cstart: int - state: str - state_reason: str | None = None - status_code: int | None = None - publication_count: int = 0 - attempt_count: int = 0 - has_show_more_button: bool = False - articles_range: str | None = None - warning_codes: list[str] = Field(default_factory=list) - - model_config = ConfigDict(extra="forbid") - - -class RunDebugData(BaseModel): - status_code: int | None = None - final_url: str | None = None - fetch_error: str | None = None - body_sha256: str | None = None - body_length: int | None = None - has_show_more_button: bool | None = None - articles_range: str | None = None - state_reason: str | None = None - warning_codes: list[str] = Field(default_factory=list) - marker_counts_nonzero: dict[str, int] = Field(default_factory=dict) - page_logs: list[RunPageLogData] = Field(default_factory=list) - attempt_log: list[RunAttemptLogData] = Field(default_factory=list) - - model_config = ConfigDict(extra="forbid") - - -class RunScholarResultData(BaseModel): - scholar_profile_id: int - scholar_id: str - state: str - state_reason: str | None = None - outcome: str - attempt_count: int = 0 - publication_count: int = 0 - start_cstart: int = 0 - continuation_cstart: int | None = None - continuation_enqueued: bool = False - continuation_cleared: bool = False - warnings: list[str] = Field(default_factory=list) - error: str | None = None - debug: RunDebugData | None = None - - model_config = ConfigDict(extra="forbid") - - -class RunDetailData(BaseModel): - run: RunListItemData - summary: RunSummaryData - scholar_results: list[RunScholarResultData] - safety_state: ScrapeSafetyStateData - - model_config = ConfigDict(extra="forbid") - - -class RunDetailEnvelope(BaseModel): - data: RunDetailData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class ManualRunData(BaseModel): - run_id: int - status: str - scholar_count: int - succeeded_count: int - failed_count: int - partial_count: int - new_publication_count: int - reused_existing_run: bool - idempotency_key: str | None = None - safety_state: ScrapeSafetyStateData - - model_config = ConfigDict(extra="forbid") - - -class ManualRunEnvelope(BaseModel): - data: ManualRunData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class QueueItemData(BaseModel): - id: int - scholar_profile_id: int - scholar_label: str - status: str - reason: str - dropped_reason: str | None - attempt_count: int - resume_cstart: int - next_attempt_dt: datetime | None - updated_at: datetime - last_error: str | None - last_run_id: int | None - - model_config = ConfigDict(extra="forbid") - - -class QueueListData(BaseModel): - queue_items: list[QueueItemData] - - model_config = ConfigDict(extra="forbid") - - -class QueueListEnvelope(BaseModel): - data: QueueListData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class QueueItemEnvelope(BaseModel): - data: QueueItemData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class QueueClearData(BaseModel): - queue_item_id: int - previous_status: str - status: str - message: str - - model_config = ConfigDict(extra="forbid") - - -class QueueClearEnvelope(BaseModel): - data: QueueClearData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminUserData(BaseModel): - id: int - email: str - is_active: bool - is_admin: bool - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(extra="forbid") - - -class AdminUsersListData(BaseModel): - users: list[AdminUserData] - - model_config = ConfigDict(extra="forbid") - - -class AdminUsersListEnvelope(BaseModel): - data: AdminUsersListData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminUserCreateRequest(BaseModel): - email: str - password: str - is_admin: bool = False - - model_config = ConfigDict(extra="forbid") - - -class AdminUserActiveUpdateRequest(BaseModel): - is_active: bool - - model_config = ConfigDict(extra="forbid") - - -class AdminUserEnvelope(BaseModel): - data: AdminUserData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminResetPasswordRequest(BaseModel): - new_password: str - - model_config = ConfigDict(extra="forbid") - - -class AdminDbIntegrityCheckData(BaseModel): - name: str - count: int - severity: str - message: str - - model_config = ConfigDict(extra="forbid") - - -class AdminDbIntegrityData(BaseModel): - status: str - checked_at: datetime - failures: list[str] = Field(default_factory=list) - warnings: list[str] = Field(default_factory=list) - checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list) - - model_config = ConfigDict(extra="forbid") - - -class AdminDbIntegrityEnvelope(BaseModel): - data: AdminDbIntegrityData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminDbRepairJobData(BaseModel): - id: int - job_name: str - requested_by: str | None - dry_run: bool - status: str - scope: dict[str, Any] = Field(default_factory=dict) - summary: dict[str, Any] = Field(default_factory=dict) - error_text: str | None - started_at: datetime | None - finished_at: datetime | None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(extra="forbid") - - -class AdminDbRepairJobsData(BaseModel): - jobs: list[AdminDbRepairJobData] = Field(default_factory=list) - - model_config = ConfigDict(extra="forbid") - - -class AdminDbRepairJobsEnvelope(BaseModel): - data: AdminDbRepairJobsData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class DisplayIdentifierData(BaseModel): - kind: str - value: str - label: str - url: str | None - confidence_score: float = Field(ge=0.0, le=1.0) - - model_config = ConfigDict(extra="forbid") - - -class AdminPdfQueueItemData(BaseModel): - publication_id: int - title: str - display_identifier: DisplayIdentifierData | None = None - pdf_url: str | None - status: str - attempt_count: int - last_failure_reason: str | None - last_failure_detail: str | None - last_source: str | None - requested_by_user_id: int | None - requested_by_email: str | None - queued_at: datetime | None - last_attempt_at: datetime | None - resolved_at: datetime | None - updated_at: datetime - - model_config = ConfigDict(extra="forbid") - - -class AdminPdfQueueData(BaseModel): - items: list[AdminPdfQueueItemData] = Field(default_factory=list) - total_count: int = 0 - page: int = 1 - page_size: int = 100 - has_next: bool = False - has_prev: bool = False - - model_config = ConfigDict(extra="forbid") - - -class AdminPdfQueueEnvelope(BaseModel): - data: AdminPdfQueueData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminPdfQueueRequeueData(BaseModel): - publication_id: int - queued: bool - status: str - message: str - - model_config = ConfigDict(extra="forbid") - - -class AdminPdfQueueRequeueEnvelope(BaseModel): - data: AdminPdfQueueRequeueData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminPdfQueueBulkEnqueueData(BaseModel): - requested_count: int - queued_count: int - message: str - - model_config = ConfigDict(extra="forbid") - - -class AdminPdfQueueBulkEnqueueEnvelope(BaseModel): - data: AdminPdfQueueBulkEnqueueData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminRepairPublicationLinksRequest(BaseModel): - scope_mode: Literal["single_user", "all_users"] = "single_user" - user_id: int | None = Field(default=None, ge=1) - scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200) - dry_run: bool = True - gc_orphan_publications: bool = False - requested_by: str | None = None - confirmation_text: str | None = None - - model_config = ConfigDict(extra="forbid") - - @model_validator(mode="after") - def validate_scope(self) -> "AdminRepairPublicationLinksRequest": - if self.scope_mode == "single_user" and self.user_id is None: - raise ValueError("user_id is required when scope_mode=single_user.") - if self.scope_mode == "all_users" and self.user_id is not None: - raise ValueError("user_id must be omitted when scope_mode=all_users.") - if not self.dry_run and self.scope_mode == "all_users": - expected = "REPAIR ALL USERS" - provided = (self.confirmation_text or "").strip() - if provided != expected: - raise ValueError( - "confirmation_text must equal 'REPAIR ALL USERS' " - "when applying a repair to all users." - ) - return self - - -class AdminRepairPublicationLinksResultData(BaseModel): - job_id: int - status: str - scope: dict[str, Any] = Field(default_factory=dict) - summary: dict[str, Any] = Field(default_factory=dict) - - model_config = ConfigDict(extra="forbid") - - -class AdminRepairPublicationLinksEnvelope(BaseModel): - data: AdminRepairPublicationLinksResultData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class SettingsPolicyData(BaseModel): - min_run_interval_minutes: int - min_request_delay_seconds: int - automation_allowed: bool - manual_run_allowed: bool - blocked_failure_threshold: int - network_failure_threshold: int - cooldown_blocked_seconds: int - cooldown_network_seconds: int - - model_config = ConfigDict(extra="forbid") - - -class SettingsData(BaseModel): - auto_run_enabled: bool - run_interval_minutes: int - request_delay_seconds: int - nav_visible_pages: list[str] - policy: SettingsPolicyData - safety_state: ScrapeSafetyStateData - - openalex_api_key: str | None = None - crossref_api_token: str | None = None - crossref_api_mailto: str | None = None - - model_config = ConfigDict(extra="forbid") - - -class SettingsEnvelope(BaseModel): - data: SettingsData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class SettingsUpdateRequest(BaseModel): - auto_run_enabled: bool - run_interval_minutes: int - request_delay_seconds: int - nav_visible_pages: list[str] | None = None - - openalex_api_key: str | None = None - crossref_api_token: str | None = None - crossref_api_mailto: str | None = None - - model_config = ConfigDict(extra="forbid") - - -class PublicationItemData(BaseModel): - publication_id: int - scholar_profile_id: int - scholar_label: str - title: str - year: int | None - citation_count: int - venue_text: str | None - pub_url: str | None - display_identifier: DisplayIdentifierData | None = None - pdf_url: str | None - pdf_status: str = "untracked" - pdf_attempt_count: int = 0 - pdf_failure_reason: str | None = None - pdf_failure_detail: str | None = None - is_read: bool - is_favorite: bool = False - first_seen_at: datetime - is_new_in_latest_run: bool - - model_config = ConfigDict(extra="forbid") - - -class PublicationsListData(BaseModel): - mode: str - favorite_only: bool = False - selected_scholar_profile_id: int | None - unread_count: int - favorites_count: int - latest_count: int - new_count: int - total_count: int - page: int - page_size: int - has_next: bool = False - has_prev: bool = False - publications: list[PublicationItemData] - - model_config = ConfigDict(extra="forbid") - - -class PublicationsListEnvelope(BaseModel): - data: PublicationsListData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class MarkAllReadData(BaseModel): - message: str - updated_count: int - - model_config = ConfigDict(extra="forbid") - - -class MarkAllReadEnvelope(BaseModel): - data: MarkAllReadData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class PublicationSelectionItem(BaseModel): - scholar_profile_id: int - publication_id: int - - model_config = ConfigDict(extra="forbid") - - -class MarkSelectedReadRequest(BaseModel): - selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500) - - model_config = ConfigDict(extra="forbid") - - -class MarkSelectedReadData(BaseModel): - message: str - requested_count: int - updated_count: int - - model_config = ConfigDict(extra="forbid") - - -class MarkSelectedReadEnvelope(BaseModel): - data: MarkSelectedReadData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class RetryPublicationPdfRequest(BaseModel): - scholar_profile_id: int = Field(ge=1) - - model_config = ConfigDict(extra="forbid") - - -class RetryPublicationPdfData(BaseModel): - message: str - queued: bool - resolved_pdf: bool - publication: PublicationItemData - - model_config = ConfigDict(extra="forbid") - - -class RetryPublicationPdfEnvelope(BaseModel): - data: RetryPublicationPdfData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class TogglePublicationFavoriteRequest(BaseModel): - scholar_profile_id: int = Field(ge=1) - is_favorite: bool - - model_config = ConfigDict(extra="forbid") - - -class TogglePublicationFavoriteData(BaseModel): - message: str - publication: PublicationItemData - - model_config = ConfigDict(extra="forbid") - - -class TogglePublicationFavoriteEnvelope(BaseModel): - data: TogglePublicationFavoriteData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") diff --git a/build/lib/app/auth/__init__.py b/build/lib/app/auth/__init__.py deleted file mode 100644 index 79644a4..0000000 --- a/build/lib/app/auth/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Authentication package for scholarr.""" - diff --git a/build/lib/app/auth/deps.py b/build/lib/app/auth/deps.py deleted file mode 100644 index 50424ae..0000000 --- a/build/lib/app/auth/deps.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import annotations - -from functools import lru_cache - -from app.auth.rate_limit import SlidingWindowRateLimiter -from app.auth.security import PasswordService -from app.auth.service import AuthService -from app.settings import settings - - -@lru_cache -def get_password_service() -> PasswordService: - return PasswordService() - - -@lru_cache -def get_auth_service() -> AuthService: - return AuthService(password_service=get_password_service()) - - -@lru_cache -def get_login_rate_limiter() -> SlidingWindowRateLimiter: - return SlidingWindowRateLimiter( - max_attempts=settings.login_rate_limit_attempts, - window_seconds=settings.login_rate_limit_window_seconds, - ) - diff --git a/build/lib/app/auth/rate_limit.py b/build/lib/app/auth/rate_limit.py deleted file mode 100644 index c3f5e8a..0000000 --- a/build/lib/app/auth/rate_limit.py +++ /dev/null @@ -1,69 +0,0 @@ -from __future__ import annotations - -from collections import deque -from collections.abc import Callable -from dataclasses import dataclass -from math import ceil -from threading import Lock -from time import monotonic - - -@dataclass(frozen=True) -class RateLimitDecision: - allowed: bool - retry_after_seconds: int = 0 - - -class SlidingWindowRateLimiter: - def __init__( - self, - *, - max_attempts: int, - window_seconds: int, - now: Callable[[], float] = monotonic, - ) -> None: - self._max_attempts = max_attempts - self._window_seconds = window_seconds - self._now = now - self._attempts: dict[str, deque[float]] = {} - self._lock = Lock() - - def check(self, key: str) -> RateLimitDecision: - with self._lock: - now_value = self._now() - attempts = self._attempts.get(key) - if attempts is None: - return RateLimitDecision(allowed=True) - self._trim_expired(attempts, now_value) - if not attempts: - self._attempts.pop(key, None) - return RateLimitDecision(allowed=True) - if len(attempts) >= self._max_attempts: - retry_after = self._window_seconds - (now_value - attempts[0]) - return RateLimitDecision( - allowed=False, - retry_after_seconds=max(1, ceil(retry_after)), - ) - return RateLimitDecision(allowed=True) - - def record_failure(self, key: str) -> None: - with self._lock: - now_value = self._now() - attempts = self._attempts.get(key) - if attempts is None: - attempts = deque() - self._attempts[key] = attempts - self._trim_expired(attempts, now_value) - attempts.append(now_value) - - def reset(self, key: str) -> None: - with self._lock: - self._attempts.pop(key, None) - - def clear_all(self) -> None: - with self._lock: - self._attempts.clear() - - def _trim_expired(self, attempts: deque[float], now_value: float) -> None: - while attempts and now_value - attempts[0] >= self._window_seconds: - attempts.popleft() diff --git a/build/lib/app/auth/runtime.py b/build/lib/app/auth/runtime.py deleted file mode 100644 index 777e799..0000000 --- a/build/lib/app/auth/runtime.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import annotations - -import logging - -from fastapi import Request -from sqlalchemy.ext.asyncio import AsyncSession - -from app.auth.session import clear_session_user, get_session_user, set_session_user -from app.db.models import User -from app.security.csrf import CSRF_SESSION_KEY -from app.services.domains.users import application as user_service - -logger = logging.getLogger(__name__) - - -def invalidate_session(request: Request) -> None: - clear_session_user(request) - request.session.pop(CSRF_SESSION_KEY, None) - - -async def get_authenticated_user( - request: Request, - db_session: AsyncSession, -) -> User | None: - session_user = get_session_user(request) - if session_user is None: - return None - - user = await user_service.get_user_by_id(db_session, session_user.id) - if user is None or not user.is_active: - logger.info( - "auth.session_invalidated", - extra={ - "event": "auth.session_invalidated", - "session_user_id": session_user.id, - }, - ) - invalidate_session(request) - return None - - if user.email != session_user.email or user.is_admin != session_user.is_admin: - set_session_user( - request, - user_id=user.id, - email=user.email, - is_admin=user.is_admin, - ) - - return user - - -def login_rate_limit_key(request: Request, email: str) -> str: - client_host = request.client.host if request.client is not None else "unknown" - normalized_email = email.strip().lower() - return f"{client_host}:{normalized_email or ''}" diff --git a/build/lib/app/auth/security.py b/build/lib/app/auth/security.py deleted file mode 100644 index f89b122..0000000 --- a/build/lib/app/auth/security.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import annotations - -from argon2 import PasswordHasher -from argon2.exceptions import InvalidHashError, VerificationError - - -class PasswordService: - def __init__(self, hasher: PasswordHasher | None = None) -> None: - self._hasher = hasher or PasswordHasher() - - def hash_password(self, password: str) -> str: - return self._hasher.hash(password) - - def verify_password(self, password_hash: str, password: str) -> bool: - try: - return bool(self._hasher.verify(password_hash, password)) - except (InvalidHashError, VerificationError): - return False - diff --git a/build/lib/app/auth/service.py b/build/lib/app/auth/service.py deleted file mode 100644 index 9772358..0000000 --- a/build/lib/app/auth/service.py +++ /dev/null @@ -1,38 +0,0 @@ -from __future__ import annotations - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.auth.security import PasswordService -from app.db.models import User - - -class AuthService: - def __init__(self, password_service: PasswordService) -> None: - self._password_service = password_service - - async def authenticate_user( - self, - db_session: AsyncSession, - *, - email: str, - password: str, - ) -> User | None: - normalized_email = email.strip().lower() - if not normalized_email or not password: - return None - result = await db_session.execute( - select(User).where(User.email == normalized_email) - ) - user = result.scalar_one_or_none() - if user is None or not user.is_active: - return None - if not self._password_service.verify_password(user.password_hash, password): - return None - return user - - def hash_password(self, password: str) -> str: - return self._password_service.hash_password(password) - - def verify_password(self, *, password_hash: str, password: str) -> bool: - return self._password_service.verify_password(password_hash, password) diff --git a/build/lib/app/auth/session.py b/build/lib/app/auth/session.py deleted file mode 100644 index 8c83197..0000000 --- a/build/lib/app/auth/session.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from starlette.requests import Request - - -SESSION_USER_ID_KEY = "auth_user_id" -SESSION_USER_EMAIL_KEY = "auth_user_email" -SESSION_USER_IS_ADMIN_KEY = "auth_user_is_admin" - - -@dataclass(frozen=True) -class SessionUser: - id: int - email: str - is_admin: bool - - -def get_session_user(request: Request) -> SessionUser | None: - user_id = request.session.get(SESSION_USER_ID_KEY) - email = request.session.get(SESSION_USER_EMAIL_KEY) - is_admin = request.session.get(SESSION_USER_IS_ADMIN_KEY) - if user_id is None or email is None or is_admin is None: - return None - try: - parsed_user_id = int(user_id) - except (TypeError, ValueError): - return None - if not isinstance(email, str): - return None - return SessionUser(id=parsed_user_id, email=email, is_admin=bool(is_admin)) - - -def set_session_user( - request: Request, - *, - user_id: int, - email: str, - is_admin: bool, -) -> None: - request.session[SESSION_USER_ID_KEY] = int(user_id) - request.session[SESSION_USER_EMAIL_KEY] = email - request.session[SESSION_USER_IS_ADMIN_KEY] = bool(is_admin) - - -def clear_session_user(request: Request) -> None: - request.session.pop(SESSION_USER_ID_KEY, None) - request.session.pop(SESSION_USER_EMAIL_KEY, None) - request.session.pop(SESSION_USER_IS_ADMIN_KEY, None) diff --git a/build/lib/app/db/__init__.py b/build/lib/app/db/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/build/lib/app/db/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/build/lib/app/db/base.py b/build/lib/app/db/base.py deleted file mode 100644 index 6fa2049..0000000 --- a/build/lib/app/db/base.py +++ /dev/null @@ -1,27 +0,0 @@ -from datetime import datetime, timezone - -from sqlalchemy import MetaData, event -from sqlalchemy.orm import DeclarativeBase - - -NAMING_CONVENTION = { - "ix": "ix_%(column_0_label)s", - "uq": "uq_%(table_name)s_%(column_0_name)s", - "ck": "ck_%(table_name)s_%(constraint_name)s", - "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", - "pk": "pk_%(table_name)s", -} - - -class Base(DeclarativeBase): - metadata = MetaData(naming_convention=NAMING_CONVENTION) - - -@event.listens_for(Base, "before_update", propagate=True) -def _set_updated_at_before_update(_mapper, _connection, target) -> None: - # Keep audit timestamps current for ORM-managed updates. - if hasattr(target, "updated_at"): - target.updated_at = datetime.now(timezone.utc) - - -metadata = Base.metadata diff --git a/build/lib/app/db/models.py b/build/lib/app/db/models.py deleted file mode 100644 index 1a8ad8a..0000000 --- a/build/lib/app/db/models.py +++ /dev/null @@ -1,546 +0,0 @@ -from __future__ import annotations - -from datetime import datetime -from enum import StrEnum - -from sqlalchemy import ( - Boolean, - CheckConstraint, - DateTime, - Enum, - Float, - ForeignKey, - Index, - Integer, - String, - Text, - UniqueConstraint, - func, - text, -) -from sqlalchemy.dialects.postgresql import JSONB -from sqlalchemy.orm import Mapped, mapped_column - -from app.db.base import Base - - -class RunTriggerType(StrEnum): - MANUAL = "manual" - SCHEDULED = "scheduled" - - -class RunStatus(StrEnum): - RUNNING = "running" - SUCCESS = "success" - PARTIAL_FAILURE = "partial_failure" - FAILED = "failed" - CANCELED = "canceled" - - -class QueueItemStatus(StrEnum): - QUEUED = "queued" - RETRYING = "retrying" - DROPPED = "dropped" - - -RUN_STATUS_DB_ENUM = Enum( - RunStatus, - name="run_status", - values_callable=lambda members: [member.value for member in members], -) -RUN_TRIGGER_TYPE_DB_ENUM = Enum( - RunTriggerType, - name="run_trigger_type", - values_callable=lambda members: [member.value for member in members], -) - - -class User(Base): - __tablename__ = "users" - - id: Mapped[int] = mapped_column(primary_key=True) - email: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) - password_hash: Mapped[str] = mapped_column(String(255), nullable=False) - is_active: Mapped[bool] = mapped_column( - Boolean, nullable=False, server_default=text("true") - ) - is_admin: Mapped[bool] = mapped_column( - Boolean, nullable=False, server_default=text("false") - ) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - - -class UserSetting(Base): - __tablename__ = "user_settings" - __table_args__ = ( - CheckConstraint( - "run_interval_minutes >= 15", - name="run_interval_minutes_min_15", - ), - CheckConstraint( - "request_delay_seconds >= 2", - name="request_delay_seconds_min_2", - ), - ) - - user_id: Mapped[int] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), primary_key=True - ) - auto_run_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, server_default=text("false") - ) - run_interval_minutes: Mapped[int] = mapped_column( - Integer, nullable=False, server_default=text("1440") - ) - request_delay_seconds: Mapped[int] = mapped_column( - Integer, nullable=False, server_default=text("10") - ) - nav_visible_pages: Mapped[list[str]] = mapped_column( - JSONB, - nullable=False, - server_default=text( - '\'["dashboard","scholars","publications","settings","style-guide","runs","users"]\'::jsonb' - ), - ) - scrape_safety_state: Mapped[dict] = mapped_column( - JSONB, - nullable=False, - server_default=text("'{}'::jsonb"), - ) - scrape_cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - scrape_cooldown_reason: Mapped[str | None] = mapped_column(String(64)) - - openalex_api_key: Mapped[str | None] = mapped_column(String(255)) - crossref_api_token: Mapped[str | None] = mapped_column(String(255)) - crossref_api_mailto: Mapped[str | None] = mapped_column(String(255)) - - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - - -class ScholarProfile(Base): - __tablename__ = "scholar_profiles" - __table_args__ = ( - UniqueConstraint("user_id", "scholar_id", name="uq_scholar_profiles_user_scholar"), - Index("ix_scholar_profiles_user_enabled", "user_id", "is_enabled"), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - user_id: Mapped[int] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), nullable=False - ) - scholar_id: Mapped[str] = mapped_column(String(64), nullable=False) - display_name: Mapped[str | None] = mapped_column(String(255)) - profile_image_url: Mapped[str | None] = mapped_column(Text) - profile_image_override_url: Mapped[str | None] = mapped_column(Text) - profile_image_upload_path: Mapped[str | None] = mapped_column(Text) - last_initial_page_fingerprint_sha256: Mapped[str | None] = mapped_column(String(64)) - last_initial_page_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - is_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, server_default=text("true") - ) - baseline_completed: Mapped[bool] = mapped_column( - Boolean, nullable=False, server_default=text("false") - ) - last_run_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - last_run_status: Mapped[RunStatus | None] = mapped_column( - RUN_STATUS_DB_ENUM, - ) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - - -class CrawlRun(Base): - __tablename__ = "crawl_runs" - __table_args__ = ( - Index("ix_crawl_runs_user_start", "user_id", "start_dt"), - Index( - "uq_crawl_runs_user_manual_idempotency_key", - "user_id", - "idempotency_key", - unique=True, - postgresql_where=text( - "idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type" - ), - ), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - user_id: Mapped[int] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), nullable=False - ) - trigger_type: Mapped[RunTriggerType] = mapped_column( - RUN_TRIGGER_TYPE_DB_ENUM, nullable=False - ) - status: Mapped[RunStatus] = mapped_column( - RUN_STATUS_DB_ENUM, nullable=False - ) - start_dt: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - end_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - scholar_count: Mapped[int] = mapped_column( - Integer, nullable=False, server_default=text("0") - ) - new_pub_count: Mapped[int] = mapped_column( - Integer, nullable=False, server_default=text("0") - ) - idempotency_key: Mapped[str | None] = mapped_column(String(128)) - error_log: Mapped[dict] = mapped_column( - JSONB, nullable=False, server_default=text("'{}'::jsonb") - ) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - - -class Publication(Base): - __tablename__ = "publications" - __table_args__ = ( - UniqueConstraint("fingerprint_sha256", name="uq_publications_fingerprint"), - Index( - "uq_publications_cluster_id_not_null", - "cluster_id", - unique=True, - postgresql_where=text("cluster_id IS NOT NULL"), - ), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - cluster_id: Mapped[str | None] = mapped_column(String(64)) - fingerprint_sha256: Mapped[str] = mapped_column(String(64), nullable=False) - title_raw: Mapped[str] = mapped_column(Text, nullable=False) - title_normalized: Mapped[str] = mapped_column(Text, nullable=False) - year: Mapped[int | None] = mapped_column(Integer) - citation_count: Mapped[int] = mapped_column( - Integer, nullable=False, server_default=text("0") - ) - author_text: Mapped[str | None] = mapped_column(Text) - venue_text: Mapped[str | None] = mapped_column(Text) - pub_url: Mapped[str | None] = mapped_column(Text) - pdf_url: Mapped[str | None] = mapped_column(Text) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - - -class PublicationIdentifier(Base): - __tablename__ = "publication_identifiers" - __table_args__ = ( - UniqueConstraint( - "publication_id", - "kind", - "value_normalized", - name="uq_publication_identifiers_publication_kind_value", - ), - CheckConstraint( - "confidence_score >= 0 AND confidence_score <= 1", - name="publication_identifiers_confidence_score_range", - ), - Index( - "ix_publication_identifiers_kind_value", - "kind", - "value_normalized", - ), - Index( - "ix_publication_identifiers_publication_id", - "publication_id", - ), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - publication_id: Mapped[int] = mapped_column( - ForeignKey("publications.id", ondelete="CASCADE"), - nullable=False, - ) - kind: Mapped[str] = mapped_column(String(32), nullable=False) - value_raw: Mapped[str] = mapped_column(Text, nullable=False) - value_normalized: Mapped[str] = mapped_column(String(255), nullable=False) - source: Mapped[str] = mapped_column(String(64), nullable=False) - confidence_score: Mapped[float] = mapped_column( - Float, - nullable=False, - server_default=text("0"), - ) - evidence_url: Mapped[str | None] = mapped_column(Text) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - - -class PublicationPdfJob(Base): - __tablename__ = "publication_pdf_jobs" - __table_args__ = ( - CheckConstraint( - "status IN ('queued', 'running', 'resolved', 'failed')", - name="publication_pdf_jobs_status_valid", - ), - Index("ix_publication_pdf_jobs_status", "status"), - Index("ix_publication_pdf_jobs_updated_at", "updated_at"), - Index("ix_publication_pdf_jobs_queued_at", "queued_at"), - ) - - publication_id: Mapped[int] = mapped_column( - ForeignKey("publications.id", ondelete="CASCADE"), - primary_key=True, - ) - status: Mapped[str] = mapped_column( - String(16), - nullable=False, - server_default=text("'queued'"), - ) - attempt_count: Mapped[int] = mapped_column( - Integer, - nullable=False, - server_default=text("0"), - ) - queued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - last_failure_reason: Mapped[str | None] = mapped_column(String(64)) - last_failure_detail: Mapped[str | None] = mapped_column(Text) - last_source: Mapped[str | None] = mapped_column(String(32)) - last_requested_by_user_id: Mapped[int | None] = mapped_column( - ForeignKey("users.id", ondelete="SET NULL"), - ) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - - -class PublicationPdfJobEvent(Base): - __tablename__ = "publication_pdf_job_events" - __table_args__ = ( - Index("ix_publication_pdf_job_events_publication_created", "publication_id", "created_at"), - Index("ix_publication_pdf_job_events_created_at", "created_at"), - Index("ix_publication_pdf_job_events_event_type", "event_type"), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - publication_id: Mapped[int] = mapped_column( - ForeignKey("publications.id", ondelete="CASCADE"), - nullable=False, - ) - user_id: Mapped[int | None] = mapped_column( - ForeignKey("users.id", ondelete="SET NULL"), - ) - event_type: Mapped[str] = mapped_column(String(32), nullable=False) - status: Mapped[str | None] = mapped_column(String(16)) - source: Mapped[str | None] = mapped_column(String(32)) - failure_reason: Mapped[str | None] = mapped_column(String(64)) - message: Mapped[str | None] = mapped_column(Text) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - - -class ScholarPublication(Base): - __tablename__ = "scholar_publications" - __table_args__ = ( - Index("ix_scholar_publications_is_read", "is_read"), - Index("ix_scholar_publications_is_favorite", "is_favorite"), - ) - - scholar_profile_id: Mapped[int] = mapped_column( - ForeignKey("scholar_profiles.id", ondelete="CASCADE"), - primary_key=True, - ) - publication_id: Mapped[int] = mapped_column( - ForeignKey("publications.id", ondelete="CASCADE"), - primary_key=True, - ) - is_read: Mapped[bool] = mapped_column( - Boolean, nullable=False, server_default=text("false") - ) - is_favorite: Mapped[bool] = mapped_column( - Boolean, nullable=False, server_default=text("false") - ) - first_seen_run_id: Mapped[int | None] = mapped_column( - ForeignKey("crawl_runs.id", ondelete="SET NULL") - ) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - - -class IngestionQueueItem(Base): - __tablename__ = "ingestion_queue_items" - __table_args__ = ( - UniqueConstraint( - "user_id", - "scholar_profile_id", - name="uq_ingestion_queue_user_scholar", - ), - CheckConstraint( - "status IN ('queued', 'retrying', 'dropped')", - name="ingestion_queue_status_valid", - ), - Index("ix_ingestion_queue_next_attempt", "next_attempt_dt"), - Index("ix_ingestion_queue_status_next_attempt", "status", "next_attempt_dt"), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - user_id: Mapped[int] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - ) - scholar_profile_id: Mapped[int] = mapped_column( - ForeignKey("scholar_profiles.id", ondelete="CASCADE"), - nullable=False, - ) - resume_cstart: Mapped[int] = mapped_column( - Integer, - nullable=False, - server_default=text("0"), - ) - reason: Mapped[str] = mapped_column(String(64), nullable=False) - status: Mapped[str] = mapped_column( - String(16), - nullable=False, - server_default=text("'queued'"), - ) - attempt_count: Mapped[int] = mapped_column( - Integer, - nullable=False, - server_default=text("0"), - ) - next_attempt_dt: Mapped[datetime] = mapped_column( - DateTime(timezone=True), - nullable=False, - server_default=func.now(), - ) - last_run_id: Mapped[int | None] = mapped_column( - ForeignKey("crawl_runs.id", ondelete="SET NULL"), - ) - last_error: Mapped[str | None] = mapped_column(Text) - dropped_reason: Mapped[str | None] = mapped_column(String(128)) - dropped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - - -class DataRepairJob(Base): - __tablename__ = "data_repair_jobs" - __table_args__ = ( - CheckConstraint( - "status IN ('planned', 'running', 'completed', 'failed')", - name="data_repair_jobs_status_valid", - ), - Index("ix_data_repair_jobs_created_at", "created_at"), - Index("ix_data_repair_jobs_job_name_created_at", "job_name", "created_at"), - ) - - id: Mapped[int] = mapped_column(primary_key=True) - job_name: Mapped[str] = mapped_column(String(64), nullable=False) - requested_by: Mapped[str | None] = mapped_column(String(255)) - scope: Mapped[dict] = mapped_column( - JSONB, - nullable=False, - server_default=text("'{}'::jsonb"), - ) - dry_run: Mapped[bool] = mapped_column( - Boolean, - nullable=False, - server_default=text("true"), - ) - status: Mapped[str] = mapped_column( - String(16), - nullable=False, - server_default=text("'planned'"), - ) - summary: Mapped[dict] = mapped_column( - JSONB, - nullable=False, - server_default=text("'{}'::jsonb"), - ) - error_text: Mapped[str | None] = mapped_column(Text) - started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - - -class AuthorSearchRuntimeState(Base): - __tablename__ = "author_search_runtime_state" - - state_key: Mapped[str] = mapped_column(String(64), primary_key=True) - last_live_request_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - consecutive_blocked_count: Mapped[int] = mapped_column( - Integer, - nullable=False, - server_default=text("0"), - ) - cooldown_rejection_count: Mapped[int] = mapped_column( - Integer, - nullable=False, - server_default=text("0"), - ) - cooldown_alert_emitted: Mapped[bool] = mapped_column( - Boolean, - nullable=False, - server_default=text("false"), - ) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - - -class AuthorSearchCacheEntry(Base): - __tablename__ = "author_search_cache_entries" - __table_args__ = ( - Index("ix_author_search_cache_expires_at", "expires_at"), - Index("ix_author_search_cache_cached_at", "cached_at"), - ) - - query_key: Mapped[str] = mapped_column(String(256), primary_key=True) - payload: Mapped[dict] = mapped_column( - JSONB, - nullable=False, - ) - expires_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), - nullable=False, - ) - cached_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, server_default=func.now() - ) diff --git a/build/lib/app/db/session.py b/build/lib/app/db/session.py deleted file mode 100644 index c05266e..0000000 --- a/build/lib/app/db/session.py +++ /dev/null @@ -1,99 +0,0 @@ -from collections.abc import AsyncIterator -import logging -import os - -from sqlalchemy import text -from sqlalchemy.ext.asyncio import ( - AsyncEngine, - AsyncSession, - async_sessionmaker, - create_async_engine, -) -from sqlalchemy.pool import NullPool - -from app.settings import settings - -logger = logging.getLogger(__name__) - -_engine: AsyncEngine | None = None -_session_factory: async_sessionmaker[AsyncSession] | None = None - - -def _normalized_pool_mode(raw_mode: str) -> str: - mode = (raw_mode or "").strip().lower() - if mode == "auto": - if os.getenv("PYTEST_CURRENT_TEST"): - return "null" - app_env = (os.getenv("APP_ENV") or "").strip().lower() - if app_env in {"test", "development", "dev", "local"}: - return "null" - return "queue" - if mode in {"null", "queue"}: - return mode - logger.warning( - "db.invalid_pool_mode_fallback", - extra={ - "event": "db.invalid_pool_mode_fallback", - "database_pool_mode": raw_mode, - "fallback_mode": "queue", - }, - ) - return "queue" - - -def get_engine() -> AsyncEngine: - global _engine - if _engine is None: - pool_mode = _normalized_pool_mode(settings.database_pool_mode) - engine_kwargs: dict[str, object] = { - "pool_pre_ping": True, - } - if pool_mode == "null": - engine_kwargs["poolclass"] = NullPool - else: - engine_kwargs["pool_size"] = max(1, int(settings.database_pool_size)) - engine_kwargs["max_overflow"] = max(0, int(settings.database_pool_max_overflow)) - engine_kwargs["pool_timeout"] = max(1, int(settings.database_pool_timeout_seconds)) - - _engine = create_async_engine(settings.database_url, **engine_kwargs) - logger.info( - "db.engine_initialized", - extra={ - "event": "db.engine_initialized", - "pool_mode": pool_mode, - }, - ) - return _engine - - -def get_session_factory() -> async_sessionmaker[AsyncSession]: - global _session_factory - if _session_factory is None: - _session_factory = async_sessionmaker(get_engine(), expire_on_commit=False) - return _session_factory - - -async def get_db_session() -> AsyncIterator[AsyncSession]: - session_factory = get_session_factory() - async with session_factory() as session: - yield session - - -async def check_database() -> bool: - engine = get_engine() - try: - async with engine.connect() as conn: - result = await conn.execute(text("SELECT 1")) - return result.scalar_one() == 1 - except Exception: - logger.exception("db.healthcheck_failed", extra={"event": "db.healthcheck_failed"}) - return False - - -async def close_engine() -> None: - global _engine, _session_factory - if _engine is not None: - await _engine.dispose() - logger.info("db.engine_disposed", extra={"event": "db.engine_disposed"}) - _engine = None - _session_factory = None diff --git a/build/lib/app/http/__init__.py b/build/lib/app/http/__init__.py deleted file mode 100644 index 9d48db4..0000000 --- a/build/lib/app/http/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from __future__ import annotations diff --git a/build/lib/app/http/middleware.py b/build/lib/app/http/middleware.py deleted file mode 100644 index ebe72f8..0000000 --- a/build/lib/app/http/middleware.py +++ /dev/null @@ -1,205 +0,0 @@ -from __future__ import annotations - -from secrets import token_urlsafe -import logging -import time - -from starlette.middleware.base import BaseHTTPMiddleware -from starlette.requests import Request -from starlette.responses import Response - -from app.logging_context import set_request_id - -REQUEST_ID_HEADER = "X-Request-ID" -DEFAULT_PERMISSIONS_POLICY = ( - "accelerometer=(), autoplay=(), camera=(), display-capture=(), " - "geolocation=(), gyroscope=(), microphone=(), payment=(), usb=()" -) -DEFAULT_CSP_POLICY = ( - "default-src 'self'; " - "base-uri 'self'; " - "form-action 'self'; " - "frame-ancestors 'none'; " - "img-src 'self' data: https:; " - "script-src 'self'; " - "style-src 'self' 'unsafe-inline'; " - "font-src 'self' data:; " - "connect-src 'self'; " - "object-src 'none'" -) -DEFAULT_CSP_DOCS_POLICY = ( - "default-src 'self'; " - "img-src 'self' data: https:; " - "script-src 'self' 'unsafe-inline'; " - "style-src 'self' 'unsafe-inline'; " - "font-src 'self' data:; " - "connect-src 'self' https:; " - "object-src 'none'; " - "frame-ancestors 'none'" -) - -logger = logging.getLogger(__name__) - - -class RequestLoggingMiddleware(BaseHTTPMiddleware): - def __init__( - self, - app, - *, - log_requests: bool = True, - skip_paths: tuple[str, ...] = (), - ) -> None: - super().__init__(app) - self._log_requests = log_requests - self._skip_paths = tuple(path for path in skip_paths if path) - - async def dispatch(self, request: Request, call_next) -> Response: - request_id = request.headers.get(REQUEST_ID_HEADER) or token_urlsafe(12) - request.state.request_id = request_id - set_request_id(request_id) - - start = time.perf_counter() - should_log = self._log_requests and not self._is_skipped_path(request.url.path) - if should_log: - logger.debug( - "request.started", - extra={ - "event": "request.started", - "method": request.method, - "path": request.url.path, - }, - ) - - try: - response = await call_next(request) - except Exception: - duration_ms = int((time.perf_counter() - start) * 1000) - logger.exception( - "request.failed", - extra={ - "event": "request.failed", - "method": request.method, - "path": request.url.path, - "duration_ms": duration_ms, - }, - ) - raise - else: - duration_ms = int((time.perf_counter() - start) * 1000) - response.headers[REQUEST_ID_HEADER] = request_id - if should_log: - logger.debug( - "request.completed", - extra={ - "event": "request.completed", - "method": request.method, - "path": request.url.path, - "status_code": response.status_code, - "duration_ms": duration_ms, - }, - ) - return response - finally: - set_request_id(None) - - def _is_skipped_path(self, path: str) -> bool: - return any(path.startswith(prefix) for prefix in self._skip_paths) - - -class SecurityHeadersMiddleware(BaseHTTPMiddleware): - def __init__( - self, - app, - *, - enabled: bool = True, - x_content_type_options: str = "nosniff", - x_frame_options: str = "DENY", - referrer_policy: str = "strict-origin-when-cross-origin", - permissions_policy: str = DEFAULT_PERMISSIONS_POLICY, - cross_origin_opener_policy: str = "same-origin", - cross_origin_resource_policy: str = "same-origin", - content_security_policy_enabled: bool = True, - content_security_policy: str = DEFAULT_CSP_POLICY, - content_security_policy_docs: str = DEFAULT_CSP_DOCS_POLICY, - content_security_policy_report_only: bool = False, - strict_transport_security_enabled: bool = False, - strict_transport_security_max_age: int = 31_536_000, - strict_transport_security_include_subdomains: bool = True, - strict_transport_security_preload: bool = False, - ) -> None: - super().__init__(app) - self._enabled = enabled - self._x_content_type_options = x_content_type_options.strip() - self._x_frame_options = x_frame_options.strip() - self._referrer_policy = referrer_policy.strip() - self._permissions_policy = permissions_policy.strip() - self._cross_origin_opener_policy = cross_origin_opener_policy.strip() - self._cross_origin_resource_policy = cross_origin_resource_policy.strip() - self._csp_enabled = content_security_policy_enabled - self._csp_policy = content_security_policy.strip() - self._csp_docs_policy = content_security_policy_docs.strip() - self._csp_report_only = content_security_policy_report_only - self._hsts_enabled = strict_transport_security_enabled - self._hsts_max_age = max(0, strict_transport_security_max_age) - self._hsts_include_subdomains = strict_transport_security_include_subdomains - self._hsts_preload = strict_transport_security_preload - - async def dispatch(self, request: Request, call_next) -> Response: - response = await call_next(request) - if not self._enabled: - return response - - if self._x_content_type_options: - response.headers.setdefault("X-Content-Type-Options", self._x_content_type_options) - if self._x_frame_options: - response.headers.setdefault("X-Frame-Options", self._x_frame_options) - if self._referrer_policy: - response.headers.setdefault("Referrer-Policy", self._referrer_policy) - if self._permissions_policy: - response.headers.setdefault("Permissions-Policy", self._permissions_policy) - if self._cross_origin_opener_policy: - response.headers.setdefault( - "Cross-Origin-Opener-Policy", - self._cross_origin_opener_policy, - ) - if self._cross_origin_resource_policy: - response.headers.setdefault( - "Cross-Origin-Resource-Policy", - self._cross_origin_resource_policy, - ) - - csp_policy = self._csp_policy_for_path(request.url.path) - if self._csp_enabled and csp_policy: - csp_header = ( - "Content-Security-Policy-Report-Only" - if self._csp_report_only - else "Content-Security-Policy" - ) - response.headers.setdefault(csp_header, csp_policy) - - hsts = self._strict_transport_security_value() - if hsts: - response.headers.setdefault("Strict-Transport-Security", hsts) - - return response - - def _csp_policy_for_path(self, path: str) -> str: - if path.startswith("/docs") or path.startswith("/redoc"): - return self._csp_docs_policy or self._csp_policy - return self._csp_policy - - def _strict_transport_security_value(self) -> str: - if not self._hsts_enabled: - return "" - - directives = [f"max-age={self._hsts_max_age}"] - if self._hsts_include_subdomains: - directives.append("includeSubDomains") - if self._hsts_preload: - directives.append("preload") - return "; ".join(directives) - - -def parse_skip_paths(raw_value: str) -> tuple[str, ...]: - parts = [part.strip() for part in raw_value.split(",")] - return tuple(part for part in parts if part) diff --git a/build/lib/app/logging_config.py b/build/lib/app/logging_config.py deleted file mode 100644 index 22907be..0000000 --- a/build/lib/app/logging_config.py +++ /dev/null @@ -1,183 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timezone -import json -import logging -import sys -from typing import Any - -from app.logging_context import get_request_id - -DEFAULT_REDACT_FIELDS = { - "authorization", - "cookie", - "csrf_token", - "new_password", - "password", - "password_hash", - "session", - "session_secret_key", -} - -_BASE_RECORD = logging.makeLogRecord({}) -_STANDARD_RECORD_FIELDS = set(_BASE_RECORD.__dict__.keys()) | {"message", "asctime"} -_NOISY_RECORD_FIELDS = {"color_message"} - - -def parse_redact_fields(raw: str | None) -> set[str]: - fields = {field.strip().lower() for field in (raw or "").split(",") if field.strip()} - return DEFAULT_REDACT_FIELDS | fields - - -def configure_logging( - *, - level: str, - log_format: str, - redact_fields: set[str], - include_uvicorn_access: bool, -) -> None: - normalized_level = _normalize_level(level) - root_logger = logging.getLogger() - root_logger.handlers.clear() - root_logger.setLevel(normalized_level) - - handler = logging.StreamHandler(stream=sys.stdout) - handler.setLevel(normalized_level) - handler.addFilter(RequestContextFilter()) - - normalized_format = log_format.strip().lower() - if normalized_format == "json": - handler.setFormatter(JsonLogFormatter(redact_fields=redact_fields)) - else: - handler.setFormatter(ConsoleLogFormatter(redact_fields=redact_fields)) - - root_logger.addHandler(handler) - - # Route server/framework logs through our single root handler. - for logger_name in ("uvicorn", "uvicorn.error"): - framework_logger = logging.getLogger(logger_name) - framework_logger.handlers.clear() - framework_logger.propagate = True - framework_logger.setLevel(normalized_level) - - access_logger = logging.getLogger("uvicorn.access") - access_logger.handlers.clear() - access_logger.propagate = True - access_logger.setLevel(normalized_level if include_uvicorn_access else logging.WARNING) - - -class RequestContextFilter(logging.Filter): - def filter(self, record: logging.LogRecord) -> bool: - if not getattr(record, "request_id", None): - request_id = get_request_id() - if request_id: - record.request_id = request_id - return True - - -class JsonLogFormatter(logging.Formatter): - def __init__(self, *, redact_fields: set[str]) -> None: - super().__init__() - self._redact_fields = {field.lower() for field in redact_fields} - - def format(self, record: logging.LogRecord) -> str: - payload: dict[str, Any] = { - "timestamp": _format_timestamp(record.created), - "level": record.levelname.lower(), - "logger": record.name, - "event": getattr(record, "event", record.getMessage()), - } - request_id = getattr(record, "request_id", None) - if request_id: - payload["request_id"] = request_id - payload.update(self._redact_mapping(_extra_fields(record))) - if record.exc_info: - payload["exception"] = self.formatException(record.exc_info) - return json.dumps(payload, ensure_ascii=True, default=str) - - def _redact_mapping(self, value: dict[str, Any]) -> dict[str, Any]: - return {key: self._redact_value(key, item) for key, item in value.items()} - - def _redact_value(self, key: str, value: Any) -> Any: - if key.lower() in self._redact_fields: - return "[REDACTED]" - if isinstance(value, dict): - return {nested_key: self._redact_value(nested_key, nested_value) for nested_key, nested_value in value.items()} - if isinstance(value, (list, tuple)): - return [self._redact_value(key, item) for item in value] - return value - - -class ConsoleLogFormatter(logging.Formatter): - def __init__(self, *, redact_fields: set[str]) -> None: - super().__init__() - self._json_formatter = JsonLogFormatter(redact_fields=redact_fields) - - def format(self, record: logging.LogRecord) -> str: - payload = json.loads(self._json_formatter.format(record)) - timestamp = payload.get("timestamp", "") - level = _short_level(payload.get("level", "info")) - logger_name = str(payload.get("logger", "app")) - event = str(payload.get("event", "")) - - parts = [timestamp, level, logger_name, event] - - request_id = payload.pop("request_id", None) - method = payload.pop("method", None) - path = payload.pop("path", None) - status_code = payload.pop("status_code", None) - duration_ms = payload.pop("duration_ms", None) - - if request_id: - parts.append(f"rid={request_id}") - if method and path: - parts.append(f"{method} {path}") - if status_code is not None: - parts.append(str(status_code)) - if duration_ms is not None: - parts.append(f"{duration_ms}ms") - - for key in sorted(payload.keys()): - if key in {"timestamp", "level", "logger", "event", "exception"}: - continue - parts.append(f"{key}={payload[key]}") - - if "exception" in payload: - parts.append(f"exception={payload['exception']}") - - return " | ".join(str(part) for part in parts if part) - - -def _extra_fields(record: logging.LogRecord) -> dict[str, Any]: - extras: dict[str, Any] = {} - for key, value in record.__dict__.items(): - if key in _STANDARD_RECORD_FIELDS or key.startswith("_"): - continue - if key in _NOISY_RECORD_FIELDS: - continue - extras[key] = value - return extras - - -def _normalize_level(level: str) -> int: - normalized = level.strip().upper() - mapping = logging.getLevelNamesMapping() - if normalized not in mapping: - return logging.INFO - return mapping[normalized] - - -def _format_timestamp(created_ts: float) -> str: - dt = datetime.fromtimestamp(created_ts, tz=timezone.utc) - return dt.strftime("%Y-%m-%d %H:%M:%SZ") - - -def _short_level(level: str) -> str: - mapping = { - "debug": "DBG", - "info": "INF", - "warning": "WRN", - "error": "ERR", - "critical": "CRT", - } - return mapping.get(level.lower(), level[:3].upper()) diff --git a/build/lib/app/logging_context.py b/build/lib/app/logging_context.py deleted file mode 100644 index ee578fb..0000000 --- a/build/lib/app/logging_context.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations - -from contextvars import ContextVar - - -_request_id_ctx: ContextVar[str | None] = ContextVar("request_id", default=None) - - -def get_request_id() -> str | None: - return _request_id_ctx.get() - - -def set_request_id(value: str | None) -> None: - _request_id_ctx.set(value) - diff --git a/build/lib/app/main.py b/build/lib/app/main.py deleted file mode 100644 index e628bd9..0000000 --- a/build/lib/app/main.py +++ /dev/null @@ -1,168 +0,0 @@ -from __future__ import annotations - -from contextlib import asynccontextmanager -import logging -from pathlib import Path - -from fastapi import FastAPI, HTTPException -from fastapi.responses import FileResponse -from fastapi.staticfiles import StaticFiles -from starlette.middleware.sessions import SessionMiddleware - -from app.api.errors import register_api_exception_handlers -from app.api.media import router as media_router -from app.api.router import router as api_router -from app.api.runtime_deps import get_ingestion_service, get_scholar_source -from app.db.session import check_database -from app.db.session import close_engine -from app.http.middleware import ( - RequestLoggingMiddleware, - SecurityHeadersMiddleware, - parse_skip_paths, -) -from app.logging_config import configure_logging, parse_redact_fields -from app.security.csrf import CSRFMiddleware -from app.services.domains.ingestion.scheduler import SchedulerService -from app.settings import settings - -logger = logging.getLogger(__name__) -BUILD_MARKER = "2026-02-19.phase2.direct-pdf-import-export-dashboard-sync" - -configure_logging( - level=settings.log_level, - log_format=settings.log_format, - redact_fields=parse_redact_fields(settings.log_redact_fields), - include_uvicorn_access=settings.log_uvicorn_access, -) - -scheduler_service = SchedulerService( - enabled=settings.scheduler_enabled, - tick_seconds=settings.scheduler_tick_seconds, - network_error_retries=settings.ingestion_network_error_retries, - retry_backoff_seconds=settings.ingestion_retry_backoff_seconds, - max_pages_per_scholar=settings.ingestion_max_pages_per_scholar, - page_size=settings.ingestion_page_size, - continuation_queue_enabled=settings.ingestion_continuation_queue_enabled, - continuation_base_delay_seconds=settings.ingestion_continuation_base_delay_seconds, - continuation_max_delay_seconds=settings.ingestion_continuation_max_delay_seconds, - continuation_max_attempts=settings.ingestion_continuation_max_attempts, - queue_batch_size=settings.scheduler_queue_batch_size, -) - - -def _log_startup_build_marker() -> None: - logger.info( - "app.startup_build_marker", - extra={ - "event": "app.startup_build_marker", - "build_marker": BUILD_MARKER, - "frontend_enabled": settings.frontend_enabled, - "scheduler_enabled": settings.scheduler_enabled, - "log_format": settings.log_format, - }, - ) - - -@asynccontextmanager -async def lifespan(_: FastAPI): - _log_startup_build_marker() - - from app.db.session import get_session_factory - from sqlalchemy import text - try: - session_factory = get_session_factory() - async with session_factory() as session: - await session.execute(text("UPDATE crawl_runs SET status = 'failed' WHERE status = 'running'")) - await session.commit() - logger.info("app.startup_orphaned_runs_cleaned", extra={"event": "app.startup_orphaned_runs_cleaned"}) - except Exception as e: - logger.error(f"Failed to clean orphaned runs at startup: {e}") - - await scheduler_service.start() - yield - await scheduler_service.stop() - await close_engine() - - -app = FastAPI(title=settings.app_name, lifespan=lifespan) -register_api_exception_handlers(app) -app.add_middleware(CSRFMiddleware) -app.add_middleware( - SessionMiddleware, - secret_key=settings.session_secret_key, - same_site="lax", - https_only=settings.session_cookie_secure, -) -app.add_middleware( - RequestLoggingMiddleware, - log_requests=settings.log_requests, - skip_paths=parse_skip_paths(settings.log_request_skip_paths), -) -app.add_middleware( - SecurityHeadersMiddleware, - enabled=settings.security_headers_enabled, - x_content_type_options=settings.security_x_content_type_options, - x_frame_options=settings.security_x_frame_options, - referrer_policy=settings.security_referrer_policy, - permissions_policy=settings.security_permissions_policy, - cross_origin_opener_policy=settings.security_cross_origin_opener_policy, - cross_origin_resource_policy=settings.security_cross_origin_resource_policy, - content_security_policy_enabled=settings.security_csp_enabled, - content_security_policy=settings.security_csp_policy, - content_security_policy_docs=settings.security_csp_docs_policy, - content_security_policy_report_only=settings.security_csp_report_only, - strict_transport_security_enabled=settings.security_strict_transport_security_enabled, - strict_transport_security_max_age=settings.security_strict_transport_security_max_age, - strict_transport_security_include_subdomains=( - settings.security_strict_transport_security_include_subdomains - ), - strict_transport_security_preload=settings.security_strict_transport_security_preload, -) -app.include_router(api_router) -app.include_router(media_router) - - -@app.get("/healthz") -async def healthz() -> dict[str, str]: - if await check_database(): - return {"status": "ok"} - raise HTTPException(status_code=500, detail="database unavailable") - - -def _configure_frontend_routes(application: FastAPI) -> None: - if not settings.frontend_enabled: - return - - dist_dir = Path(settings.frontend_dist_dir) - index_file = dist_dir / "index.html" - - if not index_file.is_file(): - return - - assets_dir = dist_dir / "assets" - if assets_dir.is_dir(): - application.mount("/assets", StaticFiles(directory=assets_dir), name="frontend-assets") - - @application.get("/", include_in_schema=False) - async def frontend_index() -> FileResponse: - return FileResponse(index_file) - - @application.get("/{full_path:path}", include_in_schema=False) - async def frontend_entry(full_path: str) -> FileResponse: - normalized = full_path.lstrip("/") - if normalized in {"healthz", "api"} or normalized.startswith("api/"): - raise HTTPException(status_code=404, detail="Not Found") - - candidate = (dist_dir / normalized).resolve() - try: - candidate.relative_to(dist_dir.resolve()) - except ValueError as error: - raise HTTPException(status_code=404, detail="Not Found") from error - - if candidate.is_file(): - return FileResponse(candidate) - - return FileResponse(index_file) - - -_configure_frontend_routes(app) diff --git a/build/lib/app/security/__init__.py b/build/lib/app/security/__init__.py deleted file mode 100644 index 4033692..0000000 --- a/build/lib/app/security/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Security middleware and helpers for scholarr.""" - diff --git a/build/lib/app/security/csrf.py b/build/lib/app/security/csrf.py deleted file mode 100644 index 70eecac..0000000 --- a/build/lib/app/security/csrf.py +++ /dev/null @@ -1,133 +0,0 @@ -from __future__ import annotations - -import logging -from secrets import compare_digest, token_urlsafe -from urllib.parse import parse_qs - -from starlette.middleware.base import BaseHTTPMiddleware -from starlette.requests import Request -from starlette.responses import JSONResponse, PlainTextResponse, Response -from starlette.types import Message - - -CSRF_SESSION_KEY = "csrf_token" -CSRF_FORM_FIELD = "csrf_token" -CSRF_HEADER_NAME = "x-csrf-token" -SAFE_METHODS = {"GET", "HEAD", "OPTIONS", "TRACE"} -logger = logging.getLogger(__name__) - - -def ensure_csrf_token(request: Request) -> str: - token = request.session.get(CSRF_SESSION_KEY) - if token is None: - token = token_urlsafe(32) - request.session[CSRF_SESSION_KEY] = token - return str(token) - - -class CSRFMiddleware(BaseHTTPMiddleware): - def __init__(self, app, *, exempt_paths: set[str] | None = None) -> None: - super().__init__(app) - self._exempt_paths = exempt_paths or set() - - async def dispatch(self, request: Request, call_next) -> Response: - if self._should_skip(request): - return await call_next(request) - - session_token = request.session.get(CSRF_SESSION_KEY) - if not session_token: - logger.warning( - "csrf.missing_session_token", - extra={ - "event": "csrf.missing_session_token", - "method": request.method, - "path": request.url.path, - }, - ) - return self._csrf_error_response( - request, - code="csrf_missing", - message="CSRF token missing.", - ) - - request_token = request.headers.get(CSRF_HEADER_NAME) - if request_token is None and self._is_form_payload(request): - body = await request.body() - request_token = self._token_from_form_body(request, body) - request._receive = self._build_receive(body) # type: ignore[attr-defined] - - if not request_token or not compare_digest(str(session_token), str(request_token)): - logger.warning( - "csrf.invalid_token", - extra={ - "event": "csrf.invalid_token", - "method": request.method, - "path": request.url.path, - }, - ) - return self._csrf_error_response( - request, - code="csrf_invalid", - message="CSRF token invalid.", - ) - - return await call_next(request) - - def _should_skip(self, request: Request) -> bool: - if request.method in SAFE_METHODS: - return True - path = request.url.path - return path in self._exempt_paths - - def _is_form_payload(self, request: Request) -> bool: - content_type = request.headers.get("content-type", "") - return content_type.startswith("application/x-www-form-urlencoded") or ( - content_type.startswith("multipart/form-data") - ) - - def _token_from_form_body(self, request: Request, body: bytes) -> str | None: - content_type = request.headers.get("content-type", "") - if not content_type.startswith("application/x-www-form-urlencoded"): - return None - parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True) - values = parsed.get(CSRF_FORM_FIELD) - if not values: - return None - return values[0] - - def _build_receive(self, body: bytes): - consumed = False - - async def receive() -> Message: - nonlocal consumed - if consumed: - return {"type": "http.request", "body": b"", "more_body": False} - consumed = True - return {"type": "http.request", "body": body, "more_body": False} - - return receive - - def _csrf_error_response( - self, - request: Request, - *, - code: str, - message: str, - ) -> Response: - if request.url.path.startswith("/api/"): - request_state = getattr(request, "state", None) - request_id = getattr(request_state, "request_id", None) if request_state else None - return JSONResponse( - { - "error": { - "code": code, - "message": message, - "details": None, - }, - "meta": { - "request_id": request_id, - }, - }, - status_code=403, - ) - return PlainTextResponse(message, status_code=403) diff --git a/build/lib/app/services/__init__.py b/build/lib/app/services/__init__.py deleted file mode 100644 index 167f869..0000000 --- a/build/lib/app/services/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Service layer for scholarr application workflows.""" - diff --git a/build/lib/app/services/domains/__init__.py b/build/lib/app/services/domains/__init__.py deleted file mode 100644 index 9d48db4..0000000 --- a/build/lib/app/services/domains/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from __future__ import annotations diff --git a/build/lib/app/services/domains/arxiv/application.py b/build/lib/app/services/domains/arxiv/application.py deleted file mode 100644 index 1581a7d..0000000 --- a/build/lib/app/services/domains/arxiv/application.py +++ /dev/null @@ -1,84 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging -from typing import TYPE_CHECKING -import xml.etree.ElementTree as ET - -import httpx - -from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id -from app.settings import settings - -if TYPE_CHECKING: - from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem - -logger = logging.getLogger(__name__) - - -def _build_arxiv_query(title: str, author_surname: str | None) -> str | None: - parts = [] - if title: - # arXiv api allows strict title searching using ti: - clean_title = title.replace('"', '').replace("'", "") - parts.append(f'ti:"{clean_title}"') - if author_surname: - parts.append(f'au:"{author_surname}"') - - if not parts: - return None - return " AND ".join(parts) - - -async def discover_arxiv_id_for_publication( - *, - item: PublicationListItem | UnreadPublicationItem, - request_email: str | None = None, - timeout_seconds: float = 3.0, -) -> str | None: - title = (item.title or "").strip() - if not title: - return None - - author_surname = None - if item.scholar_label: - tokens = [t for t in item.scholar_label.strip().split() if t] - if tokens: - author_surname = tokens[-1].lower() - - query = _build_arxiv_query(title, author_surname) - if not query: - return None - - url = "https://export.arxiv.org/api/query" - params = { - "search_query": query, - "start": 0, - "max_results": 3, - } - - headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{request_email or settings.crossref_api_mailto or 'unknown@example.com'})"} - - try: - async with httpx.AsyncClient(timeout=timeout_seconds, follow_redirects=True, headers=headers) as client: - response = await client.get(url, params=params) - response.raise_for_status() - - root = ET.fromstring(response.text) - namespace = {'atom': 'http://www.w3.org/2005/Atom'} - - for entry in root.findall('atom:entry', namespace): - id_elem = entry.find('atom:id', namespace) - if id_elem is not None and id_elem.text: - candidate = str(id_elem.text) - if '/abs/' in candidate: - candidate = candidate.split('/abs/')[-1] - normalized = normalize_arxiv_id(candidate) - if normalized: - logger.debug("arxiv.id_discovered", extra={"event": "arxiv.id_discovered", "arxiv_id": normalized}) - return normalized - - except Exception as exc: - logger.debug(f"Failed to query arXiv API: {exc}") - - return None diff --git a/build/lib/app/services/domains/crossref/__init__.py b/build/lib/app/services/domains/crossref/__init__.py deleted file mode 100644 index 8adb7ba..0000000 --- a/build/lib/app/services/domains/crossref/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import annotations - -from app.services.domains.crossref.application import discover_doi_for_publication - -__all__ = ["discover_doi_for_publication"] diff --git a/build/lib/app/services/domains/crossref/application.py b/build/lib/app/services/domains/crossref/application.py deleted file mode 100644 index f0e1311..0000000 --- a/build/lib/app/services/domains/crossref/application.py +++ /dev/null @@ -1,382 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging -import re -import threading -import time -from typing import TYPE_CHECKING - -from crossref.restful import Etiquette, Works - -from app.services.domains.doi.normalize import normalize_doi -from app.settings import settings - -if TYPE_CHECKING: - from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem - -TOKEN_RE = re.compile(r"[a-z0-9]+") -NON_ALNUM_RE = re.compile(r"[^a-z0-9\\s]+") -STOP_WORDS = {"the", "and", "for", "with", "from", "method", "study", "analysis"} -_RATE_LOCK = threading.Lock() -_LAST_REQUEST_AT = 0.0 -logger = logging.getLogger(__name__) -STRICT_TITLE_MATCH_THRESHOLD = 0.75 -RELAXED_TITLE_MATCH_THRESHOLD = 0.85 - - -def _rate_limit_wait(min_interval_seconds: float) -> None: - global _LAST_REQUEST_AT - interval = max(float(min_interval_seconds), 0.0) - with _RATE_LOCK: - elapsed = time.monotonic() - _LAST_REQUEST_AT - remaining = interval - elapsed - if remaining > 0: - time.sleep(remaining) - _LAST_REQUEST_AT = time.monotonic() - - -def _normalized_tokens(value: str) -> list[str]: - lowered = value.lower().replace("’", "'").replace("“", "\"").replace("”", "\"") - lowered = NON_ALNUM_RE.sub(" ", lowered) - return [token for token in TOKEN_RE.findall(lowered) if len(token) >= 3] - - -def _normalized_query(value: str) -> str: - tokens = [token for token in _normalized_tokens(value) if token not in STOP_WORDS] - if len(tokens) < 3: - tokens = _normalized_tokens(value) - if len(tokens) < 3: - return "" - return " ".join(tokens[:12]).strip() - - -def _query_author(value: str) -> str | None: - tokens = [token for token in value.strip().split() if token] - if len(tokens) < 2: - return None - return " ".join(tokens[:2])[:64] - - -def _author_surname(value: str) -> str | None: - tokens = [token for token in value.strip().split() if token] - if not tokens: - return None - return NON_ALNUM_RE.sub("", tokens[-1].lower()) or None - - -def _query_filters(year: int | None) -> list[tuple[str, str] | None]: - if year is None: - return [None] - return [ - (f"{year - 1}-01-01", f"{year + 1}-12-31"), - (f"{year}-01-01", f"{year}-12-31"), - None, - ] - - -def _candidate_title(item: dict) -> str: - titles = item.get("title") - if isinstance(titles, list) and titles: - return str(titles[0] or "") - return str(item.get("title") or "") - - -def _title_match_score(source: str, candidate: str) -> float: - source_tokens = {token for token in _normalized_tokens(source) if len(token) >= 3} - candidate_tokens = {token for token in _normalized_tokens(candidate) if len(token) >= 3} - if not source_tokens or not candidate_tokens: - return 0.0 - return len(source_tokens & candidate_tokens) / float(len(source_tokens)) - - -def _candidate_year(item: dict) -> int | None: - issued = item.get("issued") - if not isinstance(issued, dict): - return None - date_parts = issued.get("date-parts") - if not isinstance(date_parts, list) or not date_parts: - return None - first = date_parts[0] - if not isinstance(first, list) or not first: - return None - try: - return int(first[0]) - except (TypeError, ValueError): - return None - - -def _candidate_author_match(item: dict, surname: str | None) -> bool: - if not surname: - return True - authors = item.get("author") - if not isinstance(authors, list): - return False - for author in authors: - if not isinstance(author, dict): - continue - family = NON_ALNUM_RE.sub("", str(author.get("family") or "").lower()) - if family and family == surname: - return True - return False - - -def _candidate_rank(*, title: str, year: int | None, item: dict) -> tuple[float, str | None]: - doi = normalize_doi(str(item.get("DOI") or "")) - if doi is None: - return 0.0, None - score = _title_match_score(title, _candidate_title(item)) - candidate_year = _candidate_year(item) - if year is not None and candidate_year is not None: - if abs(year - candidate_year) > 1: - return 0.0, None - score += 0.1 - return score, doi - - -def _year_delta(source_year: int | None, candidate_year: int | None) -> int | None: - if source_year is None or candidate_year is None: - return None - return abs(int(source_year) - int(candidate_year)) - - -def _candidate_rank_relaxed( - *, - title: str, - year: int | None, - item: dict, - author_surname: str | None, -) -> tuple[float, str | None]: - doi = normalize_doi(str(item.get("DOI") or "")) - if doi is None: - return 0.0, None - score = _title_match_score(title, _candidate_title(item)) - if score <= 0: - return 0.0, None - candidate_year = _candidate_year(item) - delta = _year_delta(year, candidate_year) - if delta is not None: - if delta <= 1: - score += 0.05 - elif delta <= 3: - score += 0.0 - elif delta <= 5: - score -= 0.03 - else: - score -= 0.08 - if _candidate_author_match(item, author_surname): - score += 0.03 - return score, doi - - -def _best_candidate_doi_strict( - *, - title: str, - year: int | None, - items: list[dict], - author_surname: str | None, -) -> str | None: - best_score = 0.0 - best_doi: str | None = None - best_year: int | None = None - for item in items: - if not isinstance(item, dict): - continue - if not _candidate_author_match(item, author_surname): - continue - score, doi = _candidate_rank(title=title, year=year, item=item) - candidate_year = _candidate_year(item) - if doi is None or score < STRICT_TITLE_MATCH_THRESHOLD: - continue - if score > best_score: - best_score = score - best_doi = doi - best_year = candidate_year - continue - if abs(score - best_score) > 0.02: - continue - if best_year is None or candidate_year is None: - continue - if candidate_year < best_year: - best_doi = doi - best_year = candidate_year - return best_doi - - -def _best_candidate_doi_relaxed( - *, - title: str, - year: int | None, - items: list[dict], - author_surname: str | None, -) -> str | None: - best_score = 0.0 - best_doi: str | None = None - best_author_match = False - best_delta: int | None = None - best_year: int | None = None - for item in items: - if not isinstance(item, dict): - continue - score, doi = _candidate_rank_relaxed( - title=title, - year=year, - item=item, - author_surname=author_surname, - ) - if doi is None or score < RELAXED_TITLE_MATCH_THRESHOLD: - continue - candidate_year = _candidate_year(item) - candidate_author_match = _candidate_author_match(item, author_surname) - candidate_delta = _year_delta(year, candidate_year) - if score > best_score: - best_score = score - best_doi = doi - best_author_match = candidate_author_match - best_delta = candidate_delta - best_year = candidate_year - continue - if abs(score - best_score) > 0.02: - continue - if candidate_author_match and not best_author_match: - best_doi = doi - best_author_match = True - best_delta = candidate_delta - best_year = candidate_year - continue - if best_delta is None and candidate_delta is not None: - best_doi = doi - best_author_match = candidate_author_match - best_delta = candidate_delta - best_year = candidate_year - continue - if best_delta is not None and candidate_delta is not None and candidate_delta < best_delta: - best_doi = doi - best_author_match = candidate_author_match - best_delta = candidate_delta - best_year = candidate_year - continue - if best_year is None or candidate_year is None: - continue - if candidate_year < best_year: - best_doi = doi - best_author_match = candidate_author_match - best_delta = candidate_delta - best_year = candidate_year - return best_doi - - -def _best_candidate_doi( - *, - title: str, - year: int | None, - items: list[dict], - author_surname: str | None, -) -> str | None: - strict_match = _best_candidate_doi_strict( - title=title, - year=year, - items=items, - author_surname=author_surname, - ) - if strict_match: - return strict_match - return _best_candidate_doi_relaxed( - title=title, - year=year, - items=items, - author_surname=author_surname, - ) - - -def _works_client(email: str | None) -> Works: - if email: - etiquette = Etiquette(settings.app_name, "0.1.0", "https://scholarr.local", email) - return Works(etiquette=etiquette) - return Works() - - -def _fetch_items_sync( - *, - query: str, - author: str | None, - date_range: tuple[str, str] | None, - max_rows: int, - email: str | None, - min_interval_seconds: float, -) -> list[dict]: - _rate_limit_wait(min_interval_seconds) - works = _works_client(email) - params = {"bibliographic": query} - if author: - params["author"] = author - request = works.query(**params) - if date_range is not None: - from_date, until_date = date_range - request = request.filter(from_pub_date=from_date, until_pub_date=until_date) - request = request.select(["DOI", "title", "issued", "score", "author"]) - items: list[dict] = [] - for entry in request: - if isinstance(entry, dict): - items.append(entry) - if len(items) >= max(max_rows, 1): - break - return items - - -async def _fetch_items( - *, - query: str, - author: str | None, - date_range: tuple[str, str] | None, - max_rows: int, - email: str | None, -) -> list[dict]: - timeout = max(float(settings.crossref_timeout_seconds), 0.5) - try: - return await asyncio.wait_for( - asyncio.to_thread( - _fetch_items_sync, - query=query, - author=author, - date_range=date_range, - max_rows=max_rows, - email=email, - min_interval_seconds=settings.crossref_min_interval_seconds, - ), - timeout=timeout, - ) - except Exception: - return [] - - -async def discover_doi_for_publication( - *, - item: PublicationListItem | UnreadPublicationItem, - max_rows: int = 10, - email: str | None = None, -) -> str | None: - title = (item.title or "").strip() - query = _normalized_query(title) - if not query: - return None - author = _query_author(item.scholar_label) - author_surname = _author_surname(item.scholar_label) - for date_range in _query_filters(item.year): - items = await _fetch_items( - query=query, - author=author, - date_range=date_range, - max_rows=max_rows, - email=email, - ) - doi = _best_candidate_doi( - title=title, - year=item.year, - items=items, - author_surname=author_surname, - ) - if doi: - logger.debug("crossref.doi_discovered", extra={"event": "crossref.doi_discovered"}) - return doi - return None diff --git a/build/lib/app/services/domains/dbops/__init__.py b/build/lib/app/services/domains/dbops/__init__.py deleted file mode 100644 index f5b02e4..0000000 --- a/build/lib/app/services/domains/dbops/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from app.services.domains.dbops.application import run_publication_link_repair -from app.services.domains.dbops.integrity import collect_integrity_report -from app.services.domains.dbops.query import list_repair_jobs - -__all__ = ["collect_integrity_report", "list_repair_jobs", "run_publication_link_repair"] diff --git a/build/lib/app/services/domains/dbops/application.py b/build/lib/app/services/domains/dbops/application.py deleted file mode 100644 index e7ace77..0000000 --- a/build/lib/app/services/domains/dbops/application.py +++ /dev/null @@ -1,364 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timezone -from typing import Any - -from sqlalchemy import delete, exists, func, select, update -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import DataRepairJob, IngestionQueueItem, Publication, ScholarProfile, ScholarPublication - - -REPAIR_STATUS_PLANNED = "planned" -REPAIR_STATUS_RUNNING = "running" -REPAIR_STATUS_COMPLETED = "completed" -REPAIR_STATUS_FAILED = "failed" -SCOPE_MODE_SINGLE_USER = "single_user" -SCOPE_MODE_ALL_USERS = "all_users" - - -def _utcnow() -> datetime: - return datetime.now(timezone.utc) - - -def _normalize_scope_mode(scope_mode: str) -> str: - normalized = scope_mode.strip().lower() - if normalized in {SCOPE_MODE_SINGLE_USER, SCOPE_MODE_ALL_USERS}: - return normalized - raise ValueError("Unknown scope mode.") - - -def _scope_user_id(*, scope_mode: str, user_id: int | None) -> int | None: - if scope_mode == SCOPE_MODE_SINGLE_USER: - if user_id is None: - raise ValueError("user_id is required when scope_mode=single_user.") - return int(user_id) - if user_id is not None: - raise ValueError("user_id must be omitted when scope_mode=all_users.") - return None - - -def _scope_payload( - *, - scope_mode: str, - user_id: int | None, - target_scholar_profile_ids: list[int], - orphan_gc: bool, -) -> dict[str, Any]: - payload: dict[str, Any] = { - "scope_mode": scope_mode, - "scholar_profile_ids": [int(value) for value in target_scholar_profile_ids], - "gc_orphan_publications": bool(orphan_gc), - } - if user_id is not None: - payload["user_id"] = int(user_id) - return payload - - -async def _target_scholar_profile_ids( - db_session: AsyncSession, - *, - scope_mode: str, - user_id: int | None, - scholar_profile_ids: list[int] | None, -) -> list[int]: - stmt = select(ScholarProfile.id) - if scope_mode == SCOPE_MODE_SINGLE_USER: - stmt = stmt.where(ScholarProfile.user_id == user_id) - if scholar_profile_ids: - normalized_ids = [int(value) for value in scholar_profile_ids] - stmt = stmt.where(ScholarProfile.id.in_(normalized_ids)) - result = await db_session.execute(stmt.order_by(ScholarProfile.id.asc())) - ids = [int(row[0]) for row in result.all()] - if not ids: - raise ValueError("No target scholar profiles found for the requested scope.") - return ids - - -async def _count_scope( - db_session: AsyncSession, - *, - user_id: int | None, - target_scholar_profile_ids: list[int], -) -> dict[str, int]: - links_result = await db_session.execute( - select(func.count()) - .select_from(ScholarPublication) - .where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids)) - ) - queue_stmt = ( - select(func.count()) - .select_from(IngestionQueueItem) - .where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids)) - ) - if user_id is not None: - queue_stmt = queue_stmt.where(IngestionQueueItem.user_id == user_id) - queue_result = await db_session.execute(queue_stmt) - return { - "target_scholar_count": len(target_scholar_profile_ids), - "links_in_scope": int(links_result.scalar_one() or 0), - "queue_items_in_scope": int(queue_result.scalar_one() or 0), - } - - -async def _count_orphan_publications(db_session: AsyncSession) -> int: - stmt = ( - select(func.count()) - .select_from(Publication) - .where( - ~exists( - select(1).where(ScholarPublication.publication_id == Publication.id) - ) - ) - ) - result = await db_session.execute(stmt) - return int(result.scalar_one() or 0) - - -async def _create_job( - db_session: AsyncSession, - *, - requested_by: str | None, - scope: dict[str, Any], - dry_run: bool, -) -> DataRepairJob: - job = DataRepairJob( - job_name="repair_publication_links", - requested_by=(requested_by or "").strip() or None, - scope=scope, - dry_run=dry_run, - status=REPAIR_STATUS_PLANNED, - summary={}, - ) - db_session.add(job) - await db_session.flush() - return job - - -def _job_summary( - *, - counts: dict[str, int], - dry_run: bool, - links_deleted: int, - queue_items_deleted: int, - scholars_reset: int, - orphan_publications_before: int, - orphan_publications_deleted: int, -) -> dict[str, Any]: - return { - **counts, - "dry_run": bool(dry_run), - "links_deleted": int(links_deleted), - "queue_items_deleted": int(queue_items_deleted), - "scholars_reset": int(scholars_reset), - "orphan_publications_before": int(orphan_publications_before), - "orphan_publications_deleted": int(orphan_publications_deleted), - } - - -async def _delete_links_for_targets(db_session: AsyncSession, *, target_scholar_profile_ids: list[int]) -> int: - result = await db_session.execute( - delete(ScholarPublication).where( - ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids) - ) - ) - return int(result.rowcount or 0) - - -async def _delete_queue_for_targets( - db_session: AsyncSession, - *, - user_id: int | None, - target_scholar_profile_ids: list[int], -) -> int: - stmt = delete(IngestionQueueItem).where( - IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids) - ) - if user_id is not None: - stmt = stmt.where(IngestionQueueItem.user_id == user_id) - result = await db_session.execute(stmt) - return int(result.rowcount or 0) - - -async def _reset_scholar_tracking_state( - db_session: AsyncSession, - *, - user_id: int | None, - target_scholar_profile_ids: list[int], -) -> int: - stmt = update(ScholarProfile).where(ScholarProfile.id.in_(target_scholar_profile_ids)) - if user_id is not None: - stmt = stmt.where(ScholarProfile.user_id == user_id) - result = await db_session.execute( - stmt.values( - baseline_completed=False, - last_initial_page_fingerprint_sha256=None, - last_initial_page_checked_at=None, - last_run_dt=None, - last_run_status=None, - ) - ) - return int(result.rowcount or 0) - - -async def _delete_orphan_publications(db_session: AsyncSession) -> int: - result = await db_session.execute( - delete(Publication).where( - ~exists(select(1).where(ScholarPublication.publication_id == Publication.id)) - ) - ) - return int(result.rowcount or 0) - - -async def _mutation_counts( - db_session: AsyncSession, - *, - user_id: int | None, - target_ids: list[int], - dry_run: bool, - gc_orphan_publications: bool, -) -> tuple[int, int, int, int]: - if dry_run: - return 0, 0, 0, 0 - links_deleted = await _delete_links_for_targets( - db_session, - target_scholar_profile_ids=target_ids, - ) - queue_deleted = await _delete_queue_for_targets( - db_session, - user_id=user_id, - target_scholar_profile_ids=target_ids, - ) - scholars_reset = await _reset_scholar_tracking_state( - db_session, - user_id=user_id, - target_scholar_profile_ids=target_ids, - ) - orphan_deleted = 0 - if gc_orphan_publications: - orphan_deleted = await _delete_orphan_publications(db_session) - return links_deleted, queue_deleted, scholars_reset, orphan_deleted - - -def _result_payload(*, job: DataRepairJob, scope: dict[str, Any], summary: dict[str, Any]) -> dict[str, Any]: - return { - "job_id": int(job.id), - "status": job.status, - "scope": scope, - "summary": summary, - } - - -async def _complete_job( - db_session: AsyncSession, - *, - job: DataRepairJob, - summary: dict[str, Any], - scope: dict[str, Any], -) -> dict[str, Any]: - job.summary = summary - job.status = REPAIR_STATUS_COMPLETED - job.finished_at = _utcnow() - await db_session.commit() - return _result_payload(job=job, scope=scope, summary=summary) - - -async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None: - await db_session.rollback() - job.status = REPAIR_STATUS_FAILED - job.error_text = str(error) - job.finished_at = _utcnow() - db_session.add(job) - await db_session.commit() - - -async def _prepare_repair_job( - db_session: AsyncSession, - *, - scope_mode: str, - user_id: int | None, - scholar_profile_ids: list[int] | None, - dry_run: bool, - gc_orphan_publications: bool, - requested_by: str | None, -) -> tuple[int | None, list[int], dict[str, Any], DataRepairJob]: - normalized_scope = _normalize_scope_mode(scope_mode) - scope_user_id = _scope_user_id(scope_mode=normalized_scope, user_id=user_id) - target_ids = await _target_scholar_profile_ids( - db_session, - scope_mode=normalized_scope, - user_id=scope_user_id, - scholar_profile_ids=scholar_profile_ids, - ) - scope = _scope_payload( - scope_mode=normalized_scope, - user_id=scope_user_id, - target_scholar_profile_ids=target_ids, - orphan_gc=gc_orphan_publications, - ) - job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run) - job.status = REPAIR_STATUS_RUNNING - job.started_at = _utcnow() - return scope_user_id, target_ids, scope, job - - -async def _build_repair_summary( - db_session: AsyncSession, - *, - scope_user_id: int | None, - target_ids: list[int], - dry_run: bool, - gc_orphan_publications: bool, -) -> dict[str, Any]: - counts = await _count_scope(db_session, user_id=scope_user_id, target_scholar_profile_ids=target_ids) - orphan_before = await _count_orphan_publications(db_session) - links_deleted, queue_deleted, scholars_reset, orphan_deleted = await _mutation_counts( - db_session, - user_id=scope_user_id, - target_ids=target_ids, - dry_run=dry_run, - gc_orphan_publications=gc_orphan_publications, - ) - return _job_summary( - counts=counts, - dry_run=dry_run, - links_deleted=links_deleted, - queue_items_deleted=queue_deleted, - scholars_reset=scholars_reset, - orphan_publications_before=orphan_before, - orphan_publications_deleted=orphan_deleted, - ) - - -async def run_publication_link_repair( - db_session: AsyncSession, - *, - scope_mode: str = SCOPE_MODE_SINGLE_USER, - user_id: int | None = None, - scholar_profile_ids: list[int] | None = None, - dry_run: bool = True, - gc_orphan_publications: bool = False, - requested_by: str | None = None, -) -> dict[str, Any]: - scope_user_id, target_ids, scope, job = await _prepare_repair_job( - db_session, - scope_mode=scope_mode, - user_id=user_id, - scholar_profile_ids=scholar_profile_ids, - dry_run=dry_run, - gc_orphan_publications=gc_orphan_publications, - requested_by=requested_by, - ) - - try: - summary = await _build_repair_summary( - db_session, - scope_user_id=scope_user_id, - target_ids=target_ids, - dry_run=dry_run, - gc_orphan_publications=gc_orphan_publications, - ) - return await _complete_job(db_session, job=job, summary=summary, scope=scope) - except Exception as exc: - await _fail_job(db_session, job=job, error=exc) - raise diff --git a/build/lib/app/services/domains/dbops/integrity.py b/build/lib/app/services/domains/dbops/integrity.py deleted file mode 100644 index 4d9c43c..0000000 --- a/build/lib/app/services/domains/dbops/integrity.py +++ /dev/null @@ -1,175 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timezone -from typing import Any - -from sqlalchemy import func, select, text -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import Publication - -INTEGRITY_CHECK_DEFS = ( - ( - "legacy_cluster_id_format", - "warning", - "Publications with non-namespaced cluster IDs.", - ), - ( - "negative_citation_count", - "failure", - "Publications with negative citation counts.", - ), - ( - "orphan_publications", - "warning", - "Publications with no scholar links.", - ), - ( - "orphan_scholar_publication_links", - "failure", - "Link rows missing parent scholar/publication.", - ), - ( - "duplicate_fingerprint_keys", - "failure", - "Duplicate publication fingerprint keys.", - ), - ( - "duplicate_cluster_ids", - "failure", - "Duplicate non-null publication cluster IDs.", - ), - ( - "missing_pdf_url", - "metric", - "Publications without a resolved PDF URL.", - ), -) - - -async def _legacy_cluster_id_count(db_session: AsyncSession) -> int: - stmt = ( - select(func.count()) - .select_from(Publication) - .where(Publication.cluster_id.is_not(None)) - .where(~Publication.cluster_id.like("cfv:%")) - .where(~Publication.cluster_id.like("cluster:%")) - ) - result = await db_session.execute(stmt) - return int(result.scalar_one() or 0) - - -async def _negative_citation_count(db_session: AsyncSession) -> int: - result = await db_session.execute( - select(func.count()).select_from(Publication).where(Publication.citation_count < 0) - ) - return int(result.scalar_one() or 0) - - -async def _missing_pdf_url_count(db_session: AsyncSession) -> int: - result = await db_session.execute( - select(func.count()).select_from(Publication).where(Publication.pdf_url.is_(None)) - ) - return int(result.scalar_one() or 0) - - -async def _count_from_sql(db_session: AsyncSession, *, sql: str) -> int: - result = await db_session.execute(text(sql)) - return int(result.scalar_one() or 0) - - -def _issues_for_severity(*, checks: list[dict[str, Any]], severity: str) -> list[str]: - return [row["name"] for row in checks if row["severity"] == severity and row["count"] > 0] - - -def _check_row(*, name: str, count: int, severity: str, message: str) -> dict[str, Any]: - return { - "name": name, - "count": int(count), - "severity": severity, - "message": message, - } - - -async def _collect_counts(db_session: AsyncSession) -> dict[str, int]: - orphan_publications = await _count_from_sql( - db_session, - sql=( - "SELECT count(*) FROM publications p " - "LEFT JOIN scholar_publications sp ON sp.publication_id = p.id " - "WHERE sp.publication_id IS NULL" - ), - ) - orphan_links = await _count_from_sql( - db_session, - sql=( - "SELECT count(*) FROM scholar_publications sp " - "LEFT JOIN publications p ON p.id = sp.publication_id " - "LEFT JOIN scholar_profiles s ON s.id = sp.scholar_profile_id " - "WHERE p.id IS NULL OR s.id IS NULL" - ), - ) - duplicate_fingerprints = await _count_from_sql( - db_session, - sql=( - "SELECT count(*) FROM (" - "SELECT fingerprint_sha256 FROM publications " - "GROUP BY fingerprint_sha256 HAVING count(*) > 1" - ") dup" - ), - ) - duplicate_cluster_ids = await _count_from_sql( - db_session, - sql=( - "SELECT count(*) FROM (" - "SELECT cluster_id FROM publications " - "WHERE cluster_id IS NOT NULL " - "GROUP BY cluster_id HAVING count(*) > 1" - ") dup" - ), - ) - return { - "legacy_cluster_id_format": await _legacy_cluster_id_count(db_session), - "negative_citation_count": await _negative_citation_count(db_session), - "orphan_publications": orphan_publications, - "orphan_scholar_publication_links": orphan_links, - "duplicate_fingerprint_keys": duplicate_fingerprints, - "duplicate_cluster_ids": duplicate_cluster_ids, - "missing_pdf_url": await _missing_pdf_url_count(db_session), - } - - -def _build_checks(*, counts: dict[str, int]) -> list[dict[str, Any]]: - return [ - _check_row( - name=name, - count=counts[name], - severity=severity, - message=message, - ) - for name, severity, message in INTEGRITY_CHECK_DEFS - ] - - -def _status_from_issues(*, failures: list[str], warnings: list[str]) -> str: - status = "ok" - if failures: - status = "failed" - elif warnings: - status = "warning" - return status - - -async def collect_integrity_report(db_session: AsyncSession) -> dict[str, Any]: - counts = await _collect_counts(db_session) - checks = _build_checks(counts=counts) - failures = _issues_for_severity(checks=checks, severity="failure") - warnings = _issues_for_severity(checks=checks, severity="warning") - - return { - "status": _status_from_issues(failures=failures, warnings=warnings), - "checked_at": datetime.now(timezone.utc).isoformat(), - "failures": failures, - "warnings": warnings, - "checks": checks, - } diff --git a/build/lib/app/services/domains/dbops/query.py b/build/lib/app/services/domains/dbops/query.py deleted file mode 100644 index 2e95e31..0000000 --- a/build/lib/app/services/domains/dbops/query.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import annotations - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import DataRepairJob - - -def _bounded_limit(limit: int) -> int: - return max(1, min(int(limit), 200)) - - -async def list_repair_jobs( - db_session: AsyncSession, - *, - limit: int = 50, -) -> list[DataRepairJob]: - bounded = _bounded_limit(limit) - result = await db_session.execute( - select(DataRepairJob).order_by(DataRepairJob.created_at.desc()).limit(bounded) - ) - return list(result.scalars()) diff --git a/build/lib/app/services/domains/doi/normalize.py b/build/lib/app/services/domains/doi/normalize.py deleted file mode 100644 index 4719cf3..0000000 --- a/build/lib/app/services/domains/doi/normalize.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - -import re -from urllib.parse import unquote - -DOI_RE = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I) - - -def normalize_doi(value: str | None) -> str | None: - if not value: - return None - match = DOI_RE.search(unquote(value)) - if not match: - return None - return match.group(0).rstrip(" .;,)").lower() - - -def first_doi_from_texts(*values: str | None) -> str | None: - for value in values: - doi = normalize_doi(value) - if doi: - return doi - return None diff --git a/build/lib/app/services/domains/ingestion/__init__.py b/build/lib/app/services/domains/ingestion/__init__.py deleted file mode 100644 index 9d48db4..0000000 --- a/build/lib/app/services/domains/ingestion/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from __future__ import annotations diff --git a/build/lib/app/services/domains/ingestion/application.py b/build/lib/app/services/domains/ingestion/application.py deleted file mode 100644 index 63656b4..0000000 --- a/build/lib/app/services/domains/ingestion/application.py +++ /dev/null @@ -1,2413 +0,0 @@ -from __future__ import annotations - -import asyncio -from datetime import datetime, timezone -import hashlib -import logging -from typing import Any - -from sqlalchemy import select, text -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import ( - CrawlRun, - Publication, - RunStatus, - RunTriggerType, - ScholarProfile, - ScholarPublication, -) -from app.services.domains.ingestion.constants import ( - FAILED_STATES, - FAILURE_BUCKET_BLOCKED, - FAILURE_BUCKET_INGESTION, - FAILURE_BUCKET_LAYOUT, - FAILURE_BUCKET_NETWORK, - FAILURE_BUCKET_OTHER, - RESUMABLE_PARTIAL_REASON_PREFIXES, - RESUMABLE_PARTIAL_REASONS, - RUN_LOCK_NAMESPACE, -) -from app.services.domains.doi.normalize import first_doi_from_texts -from app.services.domains.publication_identifiers import application as identifier_service -from app.services.domains.ingestion.fingerprints import ( - _build_body_excerpt, - _dedupe_publication_candidates, - _next_cstart_value, - build_initial_page_fingerprint, - build_publication_fingerprint, - build_publication_url, - normalize_title, -) -from app.services.domains.ingestion import queue as queue_service -from app.services.domains.ingestion import safety as run_safety_service -from app.services.domains.ingestion.types import ( - PagedLoopState, - PagedParseResult, - RunAlertSummary, - RunAlreadyInProgressError, - RunBlockedBySafetyPolicyError, - RunExecutionSummary, - RunFailureSummary, - RunProgress, - ScholarProcessingOutcome, -) -from app.services.domains.settings import application as user_settings_service -from app.services.domains.runs.events import run_events -from app.services.domains.scholar.parser import ( - ParseState, - ParsedProfilePage, - PublicationCandidate, - ScholarParserError, - parse_profile_page, -) -from app.services.domains.scholar.source import FetchResult, ScholarSource -from app.settings import settings - -logger = logging.getLogger(__name__) - - -def _int_or_default(value: Any, default: int = 0) -> int: - try: - return int(value) - except (TypeError, ValueError): - return default - - -def _classify_failure_bucket(*, state: str, state_reason: str) -> str: - reason = state_reason.strip().lower() - normalized_state = state.strip().lower() - - if normalized_state == ParseState.BLOCKED_OR_CAPTCHA.value or reason.startswith("blocked_"): - return FAILURE_BUCKET_BLOCKED - if normalized_state == ParseState.NETWORK_ERROR.value or reason.startswith("network_"): - return FAILURE_BUCKET_NETWORK - if normalized_state == ParseState.LAYOUT_CHANGED.value: - return FAILURE_BUCKET_LAYOUT - if normalized_state == "ingestion_error": - return FAILURE_BUCKET_INGESTION - return FAILURE_BUCKET_OTHER - - -class ScholarIngestionService: - def __init__(self, *, source: ScholarSource) -> None: - self._source = source - - @staticmethod - def _effective_request_delay_seconds(value: int) -> int: - policy_minimum = user_settings_service.resolve_request_delay_minimum( - settings.ingestion_min_request_delay_seconds - ) - return max(policy_minimum, _int_or_default(value, policy_minimum)) - - @staticmethod - def _log_request_delay_coercion( - *, - user_id: int, - requested_request_delay_seconds: int, - effective_request_delay_seconds: int, - ) -> None: - logger.warning( - "ingestion.request_delay_coerced_to_policy_floor", - extra={ - "event": "ingestion.request_delay_coerced_to_policy_floor", - "user_id": user_id, - "requested_request_delay_seconds": requested_request_delay_seconds, - "effective_request_delay_seconds": effective_request_delay_seconds, - "policy_minimum_request_delay_seconds": user_settings_service.resolve_request_delay_minimum( - settings.ingestion_min_request_delay_seconds - ), - "metric_name": "ingestion_request_delay_coerced_total", - "metric_value": 1, - }, - ) - - async def _load_user_settings_for_run( - self, - db_session: AsyncSession, - *, - user_id: int, - trigger_type: RunTriggerType, - ): - user_settings = await user_settings_service.get_or_create_settings( - db_session, - user_id=user_id, - ) - await self._enforce_safety_gate( - db_session, - user_settings=user_settings, - user_id=user_id, - trigger_type=trigger_type, - ) - return user_settings - - async def _enforce_safety_gate( - self, - db_session: AsyncSession, - *, - user_settings, - user_id: int, - trigger_type: RunTriggerType, - ) -> None: - now_utc = datetime.now(timezone.utc) - previous = run_safety_service.get_safety_event_context(user_settings, now_utc=now_utc) - if run_safety_service.clear_expired_cooldown(user_settings, now_utc=now_utc): - await db_session.commit() - await db_session.refresh(user_settings) - logger.info( - "ingestion.safety_cooldown_cleared", - extra={ - "event": "ingestion.safety_cooldown_cleared", - "user_id": user_id, - "reason": previous.get("cooldown_reason"), - "cooldown_until": previous.get("cooldown_until"), - "metric_name": "ingestion_safety_cooldown_cleared_total", - "metric_value": 1, - }, - ) - now_utc = datetime.now(timezone.utc) - if run_safety_service.is_cooldown_active(user_settings, now_utc=now_utc): - await self._raise_safety_blocked_start( - db_session, - user_settings=user_settings, - user_id=user_id, - trigger_type=trigger_type, - now_utc=now_utc, - ) - - async def _raise_safety_blocked_start( - self, - db_session: AsyncSession, - *, - user_settings, - user_id: int, - trigger_type: RunTriggerType, - now_utc: datetime, - ) -> None: - safety_state = run_safety_service.register_cooldown_blocked_start( - user_settings, - now_utc=now_utc, - ) - await db_session.commit() - logger.warning( - "ingestion.safety_policy_blocked_run_start", - extra={ - "event": "ingestion.safety_policy_blocked_run_start", - "user_id": user_id, - "trigger_type": trigger_type.value, - "reason": safety_state.get("cooldown_reason"), - "cooldown_until": safety_state.get("cooldown_until"), - "cooldown_remaining_seconds": safety_state.get("cooldown_remaining_seconds"), - "blocked_start_count": ((safety_state.get("counters") or {}).get("blocked_start_count")), - "metric_name": "ingestion_safety_run_start_blocked_total", - "metric_value": 1, - }, - ) - raise RunBlockedBySafetyPolicyError( - code="scrape_cooldown_active", - message="Scrape safety cooldown is active; run start is temporarily blocked.", - safety_state=safety_state, - ) - - @staticmethod - def _normalize_run_targets( - *, - scholar_profile_ids: set[int] | None, - start_cstart_by_scholar_id: dict[int, int] | None, - ) -> tuple[set[int] | None, dict[int, int]]: - filtered_scholar_ids = ( - {int(value) for value in scholar_profile_ids} - if scholar_profile_ids is not None - else None - ) - start_cstart_map = { - int(key): max(0, int(value)) - for key, value in (start_cstart_by_scholar_id or {}).items() - } - return filtered_scholar_ids, start_cstart_map - - async def _load_target_scholars( - self, - db_session: AsyncSession, - *, - user_id: int, - filtered_scholar_ids: set[int] | None, - ) -> list[ScholarProfile]: - scholars_stmt = ( - select(ScholarProfile) - .where(ScholarProfile.user_id == user_id, ScholarProfile.is_enabled.is_(True)) - .order_by(ScholarProfile.created_at.asc(), ScholarProfile.id.asc()) - ) - if filtered_scholar_ids is not None: - scholars_stmt = scholars_stmt.where(ScholarProfile.id.in_(filtered_scholar_ids)) - scholars_result = await db_session.execute(scholars_stmt) - scholars = list(scholars_result.scalars().all()) - await self._clear_missing_filtered_jobs( - db_session, - user_id=user_id, - filtered_scholar_ids=filtered_scholar_ids, - scholars=scholars, - ) - return scholars - - async def _clear_missing_filtered_jobs( - self, - db_session: AsyncSession, - *, - user_id: int, - filtered_scholar_ids: set[int] | None, - scholars: list[ScholarProfile], - ) -> None: - if filtered_scholar_ids is None: - return - found_ids = {int(scholar.id) for scholar in scholars} - missing_ids = filtered_scholar_ids - found_ids - for scholar_profile_id in missing_ids: - await queue_service.clear_job_for_scholar( - db_session, - user_id=user_id, - scholar_profile_id=scholar_profile_id, - ) - - @staticmethod - def _create_running_run( - *, - user_id: int, - trigger_type: RunTriggerType, - scholar_count: int, - idempotency_key: str | None, - ) -> CrawlRun: - return CrawlRun( - user_id=user_id, - trigger_type=trigger_type, - status=RunStatus.RUNNING, - scholar_count=scholar_count, - new_pub_count=0, - idempotency_key=idempotency_key, - error_log={}, - ) - - @staticmethod - def _log_run_started( - *, - user_id: int, - trigger_type: RunTriggerType, - scholar_count: int, - filtered: bool, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - idempotency_key: str | None, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, - ) -> None: - logger.info( - "ingestion.run_started", - extra={ - "event": "ingestion.run_started", - "user_id": user_id, - "trigger_type": trigger_type.value, - "scholar_count": scholar_count, - "is_filtered_run": filtered, - "request_delay_seconds": request_delay_seconds, - "network_error_retries": network_error_retries, - "retry_backoff_seconds": retry_backoff_seconds, - "max_pages_per_scholar": max_pages_per_scholar, - "page_size": page_size, - "idempotency_key": idempotency_key, - "alert_blocked_failure_threshold": alert_blocked_failure_threshold, - "alert_network_failure_threshold": alert_network_failure_threshold, - "alert_retry_scheduled_threshold": alert_retry_scheduled_threshold, - }, - ) - - @staticmethod - async def _wait_between_scholars(*, index: int, request_delay_seconds: int) -> None: - if index <= 0 or request_delay_seconds <= 0: - return - await asyncio.sleep(float(request_delay_seconds)) - - @staticmethod - def _assert_valid_paged_parse_result( - *, - scholar_id: str, - paged_parse_result: PagedParseResult, - ) -> None: - parsed_page = paged_parse_result.parsed_page - if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS}: - if any(code.startswith("layout_") for code in parsed_page.warnings): - raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.") - for publication in paged_parse_result.publications: - if not publication.title.strip(): - raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.") - if publication.citation_count is not None and int(publication.citation_count) < 0: - raise RuntimeError(f"Negative citation count for scholar_id={scholar_id}.") - - @staticmethod - def _apply_first_page_profile_metadata( - *, - scholar: ScholarProfile, - paged_parse_result: PagedParseResult, - run_dt: datetime, - ) -> None: - first_page = paged_parse_result.first_page_parsed_page - if first_page.profile_name and not (scholar.display_name or "").strip(): - scholar.display_name = first_page.profile_name - if first_page.profile_image_url: - scholar.profile_image_url = first_page.profile_image_url - scholar.last_initial_page_checked_at = run_dt - - @staticmethod - def _log_scholar_parsed( - *, - user_id: int, - run_id: int, - scholar: ScholarProfile, - paged_parse_result: PagedParseResult, - ) -> None: - parsed_page = paged_parse_result.parsed_page - logger.info( - "ingestion.scholar_parsed", - extra={ - "event": "ingestion.scholar_parsed", - "user_id": user_id, - "crawl_run_id": run_id, - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - "state": parsed_page.state.value, - "publication_count": len(paged_parse_result.publications), - "has_show_more_button": parsed_page.has_show_more_button, - "pages_fetched": paged_parse_result.pages_fetched, - "pages_attempted": paged_parse_result.pages_attempted, - "has_more_remaining": paged_parse_result.has_more_remaining, - "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, - "warning_count": len(parsed_page.warnings), - "skipped_no_change": paged_parse_result.skipped_no_change, - }, - ) - - @staticmethod - def _build_result_entry( - *, - scholar: ScholarProfile, - start_cstart: int, - paged_parse_result: PagedParseResult, - ) -> dict[str, Any]: - parsed_page = paged_parse_result.parsed_page - return { - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - "state": parsed_page.state.value, - "state_reason": parsed_page.state_reason, - "outcome": "failed", - "attempt_count": len(paged_parse_result.attempt_log), - "publication_count": len(paged_parse_result.publications), - "start_cstart": start_cstart, - "articles_range": parsed_page.articles_range, - "warnings": parsed_page.warnings, - "has_show_more_button": parsed_page.has_show_more_button, - "pages_fetched": paged_parse_result.pages_fetched, - "pages_attempted": paged_parse_result.pages_attempted, - "has_more_remaining": paged_parse_result.has_more_remaining, - "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, - "continuation_cstart": paged_parse_result.continuation_cstart, - "skipped_no_change": paged_parse_result.skipped_no_change, - "initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, - } - - def _skipped_no_change_outcome( - self, - *, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - ) -> ScholarProcessingOutcome: - first_page = paged_parse_result.first_page_parsed_page - scholar.last_run_status = RunStatus.SUCCESS - scholar.last_run_dt = run_dt - result_entry["state"] = first_page.state.value - result_entry["state_reason"] = "no_change_initial_page_signature" - result_entry["outcome"] = "success" - result_entry["publication_count"] = 0 - result_entry["warnings"] = first_page.warnings - result_entry["debug"] = { - "state_reason": "no_change_initial_page_signature", - "first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, - "attempt_log": paged_parse_result.attempt_log, - "page_logs": paged_parse_result.page_logs, - } - return ScholarProcessingOutcome( - result_entry=result_entry, - succeeded_count_delta=1, - failed_count_delta=0, - partial_count_delta=0, - discovered_publication_count=0, - ) - - async def _upsert_publications_outcome( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - ) -> ScholarProcessingOutcome: - parsed_page = paged_parse_result.parsed_page - publications = paged_parse_result.publications - had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS} - has_partial_set = len(publications) > 0 and had_page_failure - if (not had_page_failure) or has_partial_set: - return await self._upsert_success_or_exception( - db_session, - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - has_partial_publication_set=has_partial_set, - ) - return self._parse_failure_outcome( - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - ) - - async def _upsert_success_or_exception( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - has_partial_publication_set: bool, - ) -> ScholarProcessingOutcome: - try: - return await self._upsert_success( - db_session, - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - has_partial_publication_set=has_partial_publication_set, - ) - except Exception as exc: - return self._upsert_exception_outcome( - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - exc=exc, - ) - - async def _upsert_success( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - has_partial_publication_set: bool, - ) -> ScholarProcessingOutcome: - # We no longer aggregate and upsert the publications here. - # Eager DB insertions have already saved and committed them inside `_fetch_and_parse_all_pages_with_retry` and `_paginate_loop`. - discovered_count = paged_parse_result.discovered_publication_count - is_partial = ( - paged_parse_result.has_more_remaining - or paged_parse_result.pagination_truncated_reason is not None - or has_partial_publication_set - ) - scholar.last_run_status = RunStatus.PARTIAL_FAILURE if is_partial else RunStatus.SUCCESS - scholar.last_run_dt = run_dt - if not is_partial and paged_parse_result.first_page_fingerprint_sha256: - scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256 - result_entry["outcome"] = "partial" if is_partial else "success" - if is_partial: - result_entry["debug"] = self._build_failure_debug_context( - fetch_result=paged_parse_result.fetch_result, - parsed_page=paged_parse_result.parsed_page, - attempt_log=paged_parse_result.attempt_log, - page_logs=paged_parse_result.page_logs, - ) - return ScholarProcessingOutcome( - result_entry=result_entry, - succeeded_count_delta=1, - failed_count_delta=0, - partial_count_delta=1 if is_partial else 0, - discovered_publication_count=discovered_count, - ) - - def _upsert_exception_outcome( - self, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - exc: Exception, - ) -> ScholarProcessingOutcome: - scholar.last_run_status = RunStatus.FAILED - scholar.last_run_dt = run_dt - result_entry["state"] = "ingestion_error" - result_entry["state_reason"] = "publication_upsert_exception" - result_entry["outcome"] = "failed" - result_entry["error"] = str(exc) - result_entry["debug"] = self._build_failure_debug_context( - fetch_result=paged_parse_result.fetch_result, - parsed_page=paged_parse_result.parsed_page, - attempt_log=paged_parse_result.attempt_log, - page_logs=paged_parse_result.page_logs, - exception=exc, - ) - logger.exception( - "ingestion.scholar_failed", - extra={ - "event": "ingestion.scholar_failed", - "crawl_run_id": run.id, - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - }, - ) - return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) - - def _parse_failure_outcome( - self, - *, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - ) -> ScholarProcessingOutcome: - scholar.last_run_status = RunStatus.FAILED - scholar.last_run_dt = run_dt - result_entry["debug"] = self._build_failure_debug_context( - fetch_result=paged_parse_result.fetch_result, - parsed_page=paged_parse_result.parsed_page, - attempt_log=paged_parse_result.attempt_log, - page_logs=paged_parse_result.page_logs, - ) - logger.warning( - "ingestion.scholar_parse_failed", - extra={ - "event": "ingestion.scholar_parse_failed", - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - "state": paged_parse_result.parsed_page.state.value, - "state_reason": paged_parse_result.parsed_page.state_reason, - "status_code": paged_parse_result.fetch_result.status_code, - }, - ) - return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) - - async def _sync_continuation_queue( - self, - db_session: AsyncSession, - *, - user_id: int, - scholar: ScholarProfile, - run: CrawlRun, - start_cstart: int, - result_entry: dict[str, Any], - paged_parse_result: PagedParseResult, - auto_queue_continuations: bool, - queue_delay_seconds: int, - ) -> None: - queue_reason, queue_cstart = self._resolve_continuation_queue_target( - outcome=str(result_entry.get("outcome", "")), - state=str(result_entry.get("state", "")), - pagination_truncated_reason=paged_parse_result.pagination_truncated_reason, - continuation_cstart=paged_parse_result.continuation_cstart, - fallback_cstart=start_cstart, - ) - if auto_queue_continuations and queue_reason is not None: - await queue_service.upsert_job( - db_session, - user_id=user_id, - scholar_profile_id=scholar.id, - resume_cstart=queue_cstart, - reason=queue_reason, - run_id=run.id, - delay_seconds=queue_delay_seconds, - ) - result_entry["continuation_enqueued"] = True - result_entry["continuation_reason"] = queue_reason - result_entry["continuation_cstart"] = queue_cstart - return - if await queue_service.clear_job_for_scholar(db_session, user_id=user_id, scholar_profile_id=scholar.id): - result_entry["continuation_cleared"] = True - - async def _process_scholar( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - user_id: int, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - start_cstart: int, - auto_queue_continuations: bool, - queue_delay_seconds: int, - ) -> ScholarProcessingOutcome: - try: - return await self._process_scholar_inner( - db_session, - run=run, - scholar=scholar, - user_id=user_id, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, - start_cstart=start_cstart, - auto_queue_continuations=auto_queue_continuations, - queue_delay_seconds=queue_delay_seconds, - ) - except Exception as exc: - return self._unexpected_scholar_exception_outcome( - run=run, - scholar=scholar, - start_cstart=start_cstart, - exc=exc, - ) - - async def _process_scholar_inner( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - user_id: int, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - start_cstart: int, - auto_queue_continuations: bool, - queue_delay_seconds: int, - ) -> ScholarProcessingOutcome: - run_dt, paged_parse_result, result_entry = await self._fetch_and_prepare_scholar_result( - db_session, - run=run, - scholar=scholar, - user_id=user_id, - start_cstart=start_cstart, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, - ) - outcome = await self._resolve_scholar_outcome( - db_session, - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - ) - await self._sync_continuation_queue( - db_session, - user_id=user_id, - scholar=scholar, - run=run, - start_cstart=start_cstart, - result_entry=outcome.result_entry, - paged_parse_result=paged_parse_result, - auto_queue_continuations=auto_queue_continuations, - queue_delay_seconds=queue_delay_seconds, - ) - return outcome - - async def _fetch_and_prepare_scholar_result( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - user_id: int, - start_cstart: int, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - ) -> tuple[datetime, PagedParseResult, dict[str, Any]]: - run_dt = datetime.now(timezone.utc) - paged_parse_result = await self._fetch_and_parse_all_pages_with_retry( - scholar=scholar, - run=run, - db_session=db_session, - start_cstart=start_cstart, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds, - max_pages=max_pages_per_scholar, - page_size=page_size, - previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256, - ) - self._assert_valid_paged_parse_result(scholar_id=scholar.scholar_id, paged_parse_result=paged_parse_result) - self._apply_first_page_profile_metadata(scholar=scholar, paged_parse_result=paged_parse_result, run_dt=run_dt) - self._log_scholar_parsed(user_id=user_id, run_id=run.id, scholar=scholar, paged_parse_result=paged_parse_result) - result_entry = self._build_result_entry( - scholar=scholar, - start_cstart=start_cstart, - paged_parse_result=paged_parse_result, - ) - return run_dt, paged_parse_result, result_entry - - async def _resolve_scholar_outcome( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - ) -> ScholarProcessingOutcome: - if paged_parse_result.skipped_no_change: - return self._skipped_no_change_outcome( - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - ) - return await self._upsert_publications_outcome( - db_session, - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - ) - - @staticmethod - def _apply_outcome_to_progress( - *, - progress: RunProgress, - run: CrawlRun, - outcome: ScholarProcessingOutcome, - ) -> None: - progress.succeeded_count += outcome.succeeded_count_delta - progress.failed_count += outcome.failed_count_delta - progress.partial_count += outcome.partial_count_delta - run.new_pub_count = int(run.new_pub_count or 0) + outcome.discovered_publication_count - progress.scholar_results.append(outcome.result_entry) - - def _unexpected_scholar_exception_outcome( - self, - *, - run: CrawlRun, - scholar: ScholarProfile, - start_cstart: int, - exc: Exception, - ) -> ScholarProcessingOutcome: - scholar.last_run_status = RunStatus.FAILED - scholar.last_run_dt = datetime.now(timezone.utc) - logger.exception( - "ingestion.scholar_unexpected_failure", - extra={ - "event": "ingestion.scholar_unexpected_failure", - "crawl_run_id": run.id, - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - }, - ) - return ScholarProcessingOutcome( - result_entry={ - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - "state": "ingestion_error", - "state_reason": "scholar_processing_exception", - "outcome": "failed", - "attempt_count": 0, - "publication_count": 0, - "start_cstart": start_cstart, - "warnings": [], - "error": str(exc), - "debug": {"exception_type": type(exc).__name__, "exception_message": str(exc)}, - }, - succeeded_count_delta=0, - failed_count_delta=1, - partial_count_delta=0, - discovered_publication_count=0, - ) - - @staticmethod - def _summarize_failures( - *, - scholar_results: list[dict[str, Any]], - ) -> RunFailureSummary: - failed_state_counts: dict[str, int] = {} - failed_reason_counts: dict[str, int] = {} - scrape_failure_counts: dict[str, int] = {} - retries_scheduled_count = 0 - scholars_with_retries_count = 0 - retry_exhausted_count = 0 - for entry in scholar_results: - retries_for_entry = max(0, _int_or_default(entry.get("attempt_count"), 0) - 1) - if retries_for_entry > 0: - retries_scheduled_count += retries_for_entry - scholars_with_retries_count += 1 - if str(entry.get("outcome", "")) != "failed": - continue - state = str(entry.get("state", "")).strip() - if state not in FAILED_STATES: - continue - failed_state_counts[state] = failed_state_counts.get(state, 0) + 1 - reason = str(entry.get("state_reason", "")).strip() - if reason: - failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1 - bucket = _classify_failure_bucket(state=state, state_reason=reason) - scrape_failure_counts[bucket] = scrape_failure_counts.get(bucket, 0) + 1 - if state == ParseState.NETWORK_ERROR.value and retries_for_entry > 0: - retry_exhausted_count += 1 - return RunFailureSummary( - failed_state_counts=failed_state_counts, - failed_reason_counts=failed_reason_counts, - scrape_failure_counts=scrape_failure_counts, - retries_scheduled_count=retries_scheduled_count, - scholars_with_retries_count=scholars_with_retries_count, - retry_exhausted_count=retry_exhausted_count, - ) - - @staticmethod - def _build_alert_summary( - *, - failure_summary: RunFailureSummary, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, - ) -> RunAlertSummary: - blocked_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_BLOCKED, 0)) - network_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_NETWORK, 0)) - blocked_threshold = max(1, int(alert_blocked_failure_threshold)) - network_threshold = max(1, int(alert_network_failure_threshold)) - retry_threshold = max(1, int(alert_retry_scheduled_threshold)) - alert_flags = { - "blocked_failure_threshold_exceeded": blocked_failure_count >= blocked_threshold, - "network_failure_threshold_exceeded": network_failure_count >= network_threshold, - "retry_scheduled_threshold_exceeded": failure_summary.retries_scheduled_count >= retry_threshold, - } - return RunAlertSummary( - blocked_failure_count=blocked_failure_count, - network_failure_count=network_failure_count, - blocked_failure_threshold=blocked_threshold, - network_failure_threshold=network_threshold, - retry_scheduled_threshold=retry_threshold, - alert_flags=alert_flags, - ) - - @staticmethod - def _log_alert_thresholds( - *, - user_id: int, - run_id: int, - failure_summary: RunFailureSummary, - alert_summary: RunAlertSummary, - ) -> None: - if alert_summary.alert_flags["blocked_failure_threshold_exceeded"]: - logger.warning( - "ingestion.alert_blocked_failure_threshold_exceeded", - extra={ - "event": "ingestion.alert_blocked_failure_threshold_exceeded", - "user_id": user_id, - "crawl_run_id": run_id, - "blocked_failure_count": alert_summary.blocked_failure_count, - "threshold": alert_summary.blocked_failure_threshold, - "metric_name": "ingestion_blocked_failure_threshold_exceeded_total", - "metric_value": 1, - }, - ) - if alert_summary.alert_flags["network_failure_threshold_exceeded"]: - logger.warning( - "ingestion.alert_network_failure_threshold_exceeded", - extra={ - "event": "ingestion.alert_network_failure_threshold_exceeded", - "user_id": user_id, - "crawl_run_id": run_id, - "network_failure_count": alert_summary.network_failure_count, - "threshold": alert_summary.network_failure_threshold, - "metric_name": "ingestion_network_failure_threshold_exceeded_total", - "metric_value": 1, - }, - ) - if alert_summary.alert_flags["retry_scheduled_threshold_exceeded"]: - logger.warning( - "ingestion.alert_retry_scheduled_threshold_exceeded", - extra={ - "event": "ingestion.alert_retry_scheduled_threshold_exceeded", - "user_id": user_id, - "crawl_run_id": run_id, - "retries_scheduled_count": failure_summary.retries_scheduled_count, - "threshold": alert_summary.retry_scheduled_threshold, - "metric_name": "ingestion_retry_scheduled_threshold_exceeded_total", - "metric_value": 1, - }, - ) - - @staticmethod - def _apply_safety_outcome( - *, - user_settings, - run: CrawlRun, - user_id: int, - alert_summary: RunAlertSummary, - ) -> None: - pre_apply_state = run_safety_service.get_safety_event_context( - user_settings, - now_utc=datetime.now(timezone.utc), - ) - safety_state, cooldown_trigger_reason = run_safety_service.apply_run_safety_outcome( - user_settings, - run_id=int(run.id), - blocked_failure_count=alert_summary.blocked_failure_count, - network_failure_count=alert_summary.network_failure_count, - blocked_failure_threshold=alert_summary.blocked_failure_threshold, - network_failure_threshold=alert_summary.network_failure_threshold, - blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds, - network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds, - now_utc=datetime.now(timezone.utc), - ) - ScholarIngestionService._log_safety_transition( - user_id=user_id, - run_id=int(run.id), - alert_summary=alert_summary, - pre_apply_state=pre_apply_state, - safety_state=safety_state, - cooldown_trigger_reason=cooldown_trigger_reason, - ) - - @staticmethod - def _log_safety_transition( - *, - user_id: int, - run_id: int, - alert_summary: RunAlertSummary, - pre_apply_state: dict[str, Any], - safety_state: dict[str, Any], - cooldown_trigger_reason: str | None, - ) -> None: - if cooldown_trigger_reason is not None: - logger.warning( - "ingestion.safety_cooldown_entered", - extra={ - "event": "ingestion.safety_cooldown_entered", - "user_id": user_id, - "crawl_run_id": run_id, - "reason": cooldown_trigger_reason, - "blocked_failure_count": alert_summary.blocked_failure_count, - "network_failure_count": alert_summary.network_failure_count, - "blocked_failure_threshold": alert_summary.blocked_failure_threshold, - "network_failure_threshold": alert_summary.network_failure_threshold, - "cooldown_until": safety_state.get("cooldown_until"), - "cooldown_remaining_seconds": safety_state.get("cooldown_remaining_seconds"), - "safety_counters": safety_state.get("counters", {}), - "metric_name": "ingestion_safety_cooldown_entered_total", - "metric_value": 1, - }, - ) - elif pre_apply_state.get("cooldown_active") and not safety_state.get("cooldown_active"): - logger.info( - "ingestion.safety_cooldown_cleared", - extra={ - "event": "ingestion.safety_cooldown_cleared", - "user_id": user_id, - "crawl_run_id": run_id, - "reason": pre_apply_state.get("cooldown_reason"), - "cooldown_until": pre_apply_state.get("cooldown_until"), - "metric_name": "ingestion_safety_cooldown_cleared_total", - "metric_value": 1, - }, - ) - - def _finalize_run_record( - self, - *, - run: CrawlRun, - scholars: list[ScholarProfile], - progress: RunProgress, - failure_summary: RunFailureSummary, - alert_summary: RunAlertSummary, - idempotency_key: str | None, - ) -> None: - run.end_dt = datetime.now(timezone.utc) - if run.status != RunStatus.CANCELED: - run.status = self._resolve_run_status( - scholar_count=len(scholars), - succeeded_count=progress.succeeded_count, - failed_count=progress.failed_count, - partial_count=progress.partial_count, - ) - run.error_log = { - "scholar_results": progress.scholar_results, - "summary": { - "succeeded_count": progress.succeeded_count, - "failed_count": progress.failed_count, - "partial_count": progress.partial_count, - "failed_state_counts": failure_summary.failed_state_counts, - "failed_reason_counts": failure_summary.failed_reason_counts, - "scrape_failure_counts": failure_summary.scrape_failure_counts, - "retry_counts": { - "retries_scheduled_count": failure_summary.retries_scheduled_count, - "scholars_with_retries_count": failure_summary.scholars_with_retries_count, - "retry_exhausted_count": failure_summary.retry_exhausted_count, - }, - "alert_thresholds": { - "blocked_failure_threshold": alert_summary.blocked_failure_threshold, - "network_failure_threshold": alert_summary.network_failure_threshold, - "retry_scheduled_threshold": alert_summary.retry_scheduled_threshold, - }, - "alert_flags": alert_summary.alert_flags, - }, - "meta": {"idempotency_key": idempotency_key} if idempotency_key else {}, - } - - @staticmethod - def _log_run_completed( - *, - user_id: int, - run: CrawlRun, - scholars: list[ScholarProfile], - progress: RunProgress, - alert_summary: RunAlertSummary, - failure_summary: RunFailureSummary, - ) -> None: - logger.info( - "ingestion.run_completed", - extra={ - "event": "ingestion.run_completed", - "user_id": user_id, - "crawl_run_id": run.id, - "status": run.status.value, - "scholar_count": len(scholars), - "succeeded_count": progress.succeeded_count, - "failed_count": progress.failed_count, - "partial_count": progress.partial_count, - "new_publication_count": run.new_pub_count, - "blocked_failure_count": alert_summary.blocked_failure_count, - "network_failure_count": alert_summary.network_failure_count, - "retries_scheduled_count": failure_summary.retries_scheduled_count, - "alert_flags": alert_summary.alert_flags, - }, - ) - - @staticmethod - def _paging_kwargs( - *, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - ) -> dict[str, Any]: - return { - "request_delay_seconds": request_delay_seconds, - "network_error_retries": network_error_retries, - "retry_backoff_seconds": retry_backoff_seconds, - "rate_limit_retries": rate_limit_retries, - "rate_limit_backoff_seconds": rate_limit_backoff_seconds, - "max_pages_per_scholar": max_pages_per_scholar, - "page_size": page_size, - } - - @staticmethod - def _threshold_kwargs( - *, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, - ) -> dict[str, Any]: - return { - "alert_blocked_failure_threshold": alert_blocked_failure_threshold, - "alert_network_failure_threshold": alert_network_failure_threshold, - "alert_retry_scheduled_threshold": alert_retry_scheduled_threshold, - } - - @staticmethod - def _run_execution_summary( - *, - run: CrawlRun, - scholars: list[ScholarProfile], - progress: RunProgress, - ) -> RunExecutionSummary: - return RunExecutionSummary( - crawl_run_id=run.id, - status=run.status, - scholar_count=len(scholars), - succeeded_count=progress.succeeded_count, - failed_count=progress.failed_count, - partial_count=progress.partial_count, - new_publication_count=run.new_pub_count, - ) - - async def _initialize_run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, scholar_profile_ids: set[int] | None, start_cstart_by_scholar_id: dict[int, int] | None, request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, rate_limit_retries: int, rate_limit_backoff_seconds: float, max_pages_per_scholar: int, page_size: int, idempotency_key: str | None, alert_blocked_failure_threshold: int, alert_network_failure_threshold: int, alert_retry_scheduled_threshold: int) -> tuple[Any, CrawlRun, list[ScholarProfile], dict[int, int]]: - user_settings = await self._load_user_settings_for_run( - db_session, - user_id=user_id, - trigger_type=trigger_type, - ) - if not await self._try_acquire_user_lock(db_session, user_id=user_id): - raise RunAlreadyInProgressError(f"Run already in progress for user_id={user_id}.") - filtered_scholar_ids, start_cstart_map = self._normalize_run_targets( - scholar_profile_ids=scholar_profile_ids, - start_cstart_by_scholar_id=start_cstart_by_scholar_id, - ) - scholars = await self._load_target_scholars( - db_session, - user_id=user_id, - filtered_scholar_ids=filtered_scholar_ids, - ) - run = await self._start_run_record_for_targets( - db_session, - user_id=user_id, - trigger_type=trigger_type, - scholars=scholars, - filtered=filtered_scholar_ids is not None, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, - idempotency_key=idempotency_key, - alert_blocked_failure_threshold=alert_blocked_failure_threshold, - alert_network_failure_threshold=alert_network_failure_threshold, - alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, - ) - return user_settings, run, scholars, start_cstart_map - - async def _start_run_record_for_targets( - self, - db_session: AsyncSession, - *, - user_id: int, - trigger_type: RunTriggerType, - scholars: list[ScholarProfile], - filtered: bool, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - idempotency_key: str | None, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, - ) -> CrawlRun: - self._log_run_started( - user_id=user_id, - trigger_type=trigger_type, - scholar_count=len(scholars), - filtered=filtered, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, - idempotency_key=idempotency_key, - alert_blocked_failure_threshold=alert_blocked_failure_threshold, - alert_network_failure_threshold=alert_network_failure_threshold, - alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, - ) - run = self._create_running_run( - user_id=user_id, - trigger_type=trigger_type, - scholar_count=len(scholars), - idempotency_key=idempotency_key, - ) - db_session.add(run) - await db_session.flush() - return run - - async def _run_scholar_iteration( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholars: list[ScholarProfile], - user_id: int, - start_cstart_map: dict[int, int], - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - auto_queue_continuations: bool, - queue_delay_seconds: int, - ) -> RunProgress: - from app.db.models import RunStatus - - progress = RunProgress() - for index, scholar in enumerate(scholars): - await db_session.refresh(run) - if run.status == RunStatus.CANCELED: - logger.info( - "ingestion.run_canceled", - extra={"event": "ingestion.run_canceled", "run_id": run.id, "user_id": user_id}, - ) - break - await self._wait_between_scholars(index=index, request_delay_seconds=request_delay_seconds) - start_cstart = int(start_cstart_map.get(int(scholar.id), 0)) - outcome = await self._process_scholar( - db_session, - run=run, - scholar=scholar, - user_id=user_id, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, - start_cstart=start_cstart, - auto_queue_continuations=auto_queue_continuations, - queue_delay_seconds=queue_delay_seconds, - ) - self._apply_outcome_to_progress(progress=progress, run=run, outcome=outcome) - return progress - - def _complete_run_for_user( - self, - *, - user_settings: Any, - run: CrawlRun, - scholars: list[ScholarProfile], - user_id: int, - progress: RunProgress, - idempotency_key: str | None, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, - ) -> tuple[RunFailureSummary, RunAlertSummary]: - failure_summary = self._summarize_failures(scholar_results=progress.scholar_results) - alert_summary = self._build_alert_summary( - failure_summary=failure_summary, - alert_blocked_failure_threshold=alert_blocked_failure_threshold, - alert_network_failure_threshold=alert_network_failure_threshold, - alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, - ) - self._log_alert_thresholds( - user_id=user_id, - run_id=int(run.id), - failure_summary=failure_summary, - alert_summary=alert_summary, - ) - self._apply_safety_outcome(user_settings=user_settings, run=run, user_id=user_id, alert_summary=alert_summary) - self._finalize_run_record( - run=run, - scholars=scholars, - progress=progress, - failure_summary=failure_summary, - alert_summary=alert_summary, - idempotency_key=idempotency_key, - ) - return failure_summary, alert_summary - - async def run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, request_delay_seconds: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, max_pages_per_scholar: int = 30, page_size: int = 100, scholar_profile_ids: set[int] | None = None, start_cstart_by_scholar_id: dict[int, int] | None = None, auto_queue_continuations: bool = True, queue_delay_seconds: int = 60, idempotency_key: str | None = None, alert_blocked_failure_threshold: int = 1, alert_network_failure_threshold: int = 2, alert_retry_scheduled_threshold: int = 3) -> RunExecutionSummary: - effective_request_delay_seconds = self._effective_request_delay_seconds(request_delay_seconds) - if effective_request_delay_seconds != _int_or_default(request_delay_seconds, effective_request_delay_seconds): - self._log_request_delay_coercion( - user_id=user_id, - requested_request_delay_seconds=_int_or_default(request_delay_seconds, 0), - effective_request_delay_seconds=effective_request_delay_seconds, - ) - paging_kwargs = self._paging_kwargs(request_delay_seconds=effective_request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, rate_limit_retries=settings.ingestion_rate_limit_retries, rate_limit_backoff_seconds=settings.ingestion_rate_limit_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size) - threshold_kwargs = self._threshold_kwargs(alert_blocked_failure_threshold=alert_blocked_failure_threshold, alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold) - user_settings, run, scholars, start_cstart_map = await self._initialize_run_for_user( - db_session, - user_id=user_id, - trigger_type=trigger_type, - scholar_profile_ids=scholar_profile_ids, - start_cstart_by_scholar_id=start_cstart_by_scholar_id, - idempotency_key=idempotency_key, - **paging_kwargs, - **threshold_kwargs, - ) - progress = await self._run_scholar_iteration( - db_session, - run=run, - scholars=scholars, - user_id=user_id, - start_cstart_map=start_cstart_map, - auto_queue_continuations=auto_queue_continuations, - queue_delay_seconds=queue_delay_seconds, - **paging_kwargs, - ) - failure_summary, alert_summary = self._complete_run_for_user( - user_settings=user_settings, - run=run, - scholars=scholars, - user_id=user_id, - progress=progress, - idempotency_key=idempotency_key, - **threshold_kwargs, - ) - await db_session.commit() - self._log_run_completed(user_id=user_id, run=run, scholars=scholars, progress=progress, alert_summary=alert_summary, failure_summary=failure_summary) - return self._run_execution_summary(run=run, scholars=scholars, progress=progress) - - async def _fetch_profile_page( - self, - *, - scholar_id: str, - cstart: int, - page_size: int, - ) -> FetchResult: - try: - page_fetcher = getattr(self._source, "fetch_profile_page_html", None) - if callable(page_fetcher): - return await page_fetcher( - scholar_id, - cstart=cstart, - pagesize=page_size, - ) - if cstart <= 0: - return await self._source.fetch_profile_html(scholar_id) - return FetchResult( - requested_url=( - "https://scholar.google.com/citations" - f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}" - ), - status_code=None, - final_url=None, - body="", - error="source_does_not_support_pagination", - ) - except Exception as exc: - logger.exception( - "ingestion.fetch_unexpected_error", - extra={ - "event": "ingestion.fetch_unexpected_error", - "scholar_id": scholar_id, - "cstart": cstart, - "page_size": page_size, - }, - ) - return FetchResult( - requested_url=( - "https://scholar.google.com/citations" - f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}" - ), - status_code=None, - final_url=None, - body="", - error=str(exc), - ) - - @staticmethod - def _attempt_log_entry( - *, - attempt: int, - cstart: int, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - ) -> dict[str, Any]: - return { - "attempt": attempt, - "cstart": cstart, - "state": parsed_page.state.value, - "state_reason": parsed_page.state_reason, - "status_code": fetch_result.status_code, - "fetch_error": fetch_result.error, - } - - @staticmethod - def _should_retry_page_fetch( - *, - parsed_page: ParsedProfilePage, - network_attempt_count: int, - rate_limit_attempt_count: int, - network_error_retries: int, - rate_limit_retries: int, - ) -> bool: - if parsed_page.state == ParseState.NETWORK_ERROR: - return network_attempt_count <= network_error_retries - if parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA and parsed_page.state_reason == "blocked_http_429_rate_limited": - return rate_limit_attempt_count <= rate_limit_retries - return False - - @staticmethod - async def _sleep_retry_backoff( - *, - scholar_id: str, - cstart: int, - network_attempt_count: int, - rate_limit_attempt_count: int, - retry_backoff_seconds: float, - rate_limit_backoff_seconds: float, - state_reason: str, - ) -> None: - if state_reason == "blocked_http_429_rate_limited": - sleep_seconds = rate_limit_backoff_seconds * rate_limit_attempt_count - attempt_label = rate_limit_attempt_count - else: - sleep_seconds = retry_backoff_seconds * (2 ** (network_attempt_count - 1)) - attempt_label = network_attempt_count - - logger.warning( - "ingestion.scholar_retry_scheduled", - extra={ - "event": "ingestion.scholar_retry_scheduled", - "scholar_id": scholar_id, - "cstart": cstart, - "attempt_count": attempt_label, - "sleep_seconds": sleep_seconds, - "state_reason": state_reason, - }, - ) - if sleep_seconds > 0: - await asyncio.sleep(sleep_seconds) - - async def _fetch_and_parse_page_with_retry( - self, - *, - scholar_id: str, - cstart: int, - page_size: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]: - network_attempts = 0 - rate_limit_attempts = 0 - attempt_log: list[dict[str, Any]] = [] - fetch_result: FetchResult | None = None - parsed_page: ParsedProfilePage | None = None - - while True: - fetch_result = await self._fetch_profile_page( - scholar_id=scholar_id, - cstart=cstart, - page_size=page_size, - ) - parsed_page = self._parse_profile_page_or_layout_error(fetch_result=fetch_result) - - if parsed_page.state == ParseState.NETWORK_ERROR: - network_attempts += 1 - total_attempts = network_attempts - elif parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA and parsed_page.state_reason == "blocked_http_429_rate_limited": - rate_limit_attempts += 1 - total_attempts = rate_limit_attempts - else: - total_attempts = network_attempts + rate_limit_attempts + 1 - - attempt_log.append( - self._attempt_log_entry( - attempt=total_attempts, - cstart=cstart, - fetch_result=fetch_result, - parsed_page=parsed_page, - ) - ) - - if not self._should_retry_page_fetch( - parsed_page=parsed_page, - network_attempt_count=network_attempts, - rate_limit_attempt_count=rate_limit_attempts, - network_error_retries=network_error_retries, - rate_limit_retries=rate_limit_retries, - ): - break - - await self._sleep_retry_backoff( - scholar_id=scholar_id, - cstart=cstart, - network_attempt_count=network_attempts, - rate_limit_attempt_count=rate_limit_attempts, - retry_backoff_seconds=max(float(retry_backoff_seconds), 0.0), - rate_limit_backoff_seconds=max(float(rate_limit_backoff_seconds), 0.0), - state_reason=parsed_page.state_reason, - ) - - if fetch_result is None or parsed_page is None: - raise RuntimeError("Fetch-and-parse retry loop produced no result.") - return fetch_result, parsed_page, attempt_log - - def _parse_profile_page_or_layout_error( - self, - *, - fetch_result: FetchResult, - ) -> ParsedProfilePage: - try: - return parse_profile_page(fetch_result) - except ScholarParserError as exc: - return self._parsed_page_from_parser_error( - fetch_result=fetch_result, - code=exc.code, - ) - - @staticmethod - def _parsed_page_from_parser_error( - *, - fetch_result: FetchResult, - code: str, - ) -> ParsedProfilePage: - return ParsedProfilePage( - state=ParseState.LAYOUT_CHANGED, - state_reason=code, - profile_name=None, - profile_image_url=None, - publications=[], - marker_counts={}, - warnings=[code], - has_show_more_button=False, - has_operation_error_banner=False, - articles_range=None, - ) - - @staticmethod - def _page_log_entry( - *, - page_number: int, - cstart: int, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - attempt_count: int, - ) -> dict[str, Any]: - return { - "page": page_number, - "cstart": cstart, - "state": parsed_page.state.value, - "state_reason": parsed_page.state_reason, - "status_code": fetch_result.status_code, - "publication_count": len(parsed_page.publications), - "articles_range": parsed_page.articles_range, - "has_show_more_button": parsed_page.has_show_more_button, - "warning_codes": parsed_page.warnings, - "attempt_count": attempt_count, - } - - @staticmethod - def _should_skip_no_change( - *, - start_cstart: int, - first_page_fingerprint_sha256: str | None, - previous_initial_page_fingerprint_sha256: str | None, - parsed_page: ParsedProfilePage, - ) -> bool: - return ( - start_cstart <= 0 - and first_page_fingerprint_sha256 is not None - and previous_initial_page_fingerprint_sha256 is not None - and first_page_fingerprint_sha256 == previous_initial_page_fingerprint_sha256 - and parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} - ) - - @staticmethod - def _skip_no_change_result( - *, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - first_page_fingerprint_sha256: str | None, - attempt_log: list[dict[str, Any]], - page_logs: list[dict[str, Any]], - ) -> PagedParseResult: - return PagedParseResult( - fetch_result=fetch_result, - parsed_page=parsed_page, - first_page_fetch_result=fetch_result, - first_page_parsed_page=parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - publications=[], - attempt_log=attempt_log, - page_logs=page_logs, - pages_fetched=1, - pages_attempted=1, - has_more_remaining=False, - pagination_truncated_reason=None, - continuation_cstart=None, - skipped_no_change=True, - discovered_publication_count=0, - ) - - @staticmethod - def _initial_failure_result( - *, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - first_page_fingerprint_sha256: str | None, - start_cstart: int, - attempt_log: list[dict[str, Any]], - page_logs: list[dict[str, Any]], - ) -> PagedParseResult: - continuation_cstart = start_cstart if parsed_page.state == ParseState.NETWORK_ERROR else None - return PagedParseResult( - fetch_result=fetch_result, - parsed_page=parsed_page, - first_page_fetch_result=fetch_result, - first_page_parsed_page=parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - publications=[], - attempt_log=attempt_log, - page_logs=page_logs, - pages_fetched=0, - pages_attempted=1, - has_more_remaining=False, - pagination_truncated_reason=None, - continuation_cstart=continuation_cstart, - skipped_no_change=False, - discovered_publication_count=0, - ) - - @staticmethod - def _build_loop_state( - *, - start_cstart: int, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - attempt_log: list[dict[str, Any]], - page_logs: list[dict[str, Any]], - ) -> PagedLoopState: - next_cstart = _next_cstart_value( - articles_range=parsed_page.articles_range, - fallback=start_cstart + len(parsed_page.publications), - ) - return PagedLoopState( - fetch_result=fetch_result, - parsed_page=parsed_page, - attempt_log=attempt_log, - page_logs=page_logs, - publications=list(parsed_page.publications), - pages_fetched=1, - pages_attempted=1, - current_cstart=start_cstart, - next_cstart=next_cstart, - ) - - @staticmethod - def _set_truncated_state( - *, - state: PagedLoopState, - reason: str, - continuation_cstart: int, - ) -> None: - state.has_more_remaining = True - state.pagination_truncated_reason = reason - state.continuation_cstart = continuation_cstart - - def _should_stop_pagination(self, *, state: PagedLoopState, bounded_max_pages: int) -> bool: - if state.pages_fetched >= bounded_max_pages: - self._set_truncated_state( - state=state, - reason="max_pages_reached", - continuation_cstart=( - state.next_cstart if state.next_cstart > state.current_cstart else state.current_cstart - ), - ) - return True - if state.next_cstart <= state.current_cstart: - self._set_truncated_state( - state=state, - reason="pagination_cursor_stalled", - continuation_cstart=state.current_cstart, - ) - return True - return False - - async def _fetch_next_page( - self, - *, - scholar_id: str, - state: PagedLoopState, - request_delay_seconds: int, - bounded_page_size: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]: - if request_delay_seconds > 0: - await asyncio.sleep(float(request_delay_seconds)) - state.current_cstart = state.next_cstart - return await self._fetch_and_parse_page_with_retry( - scholar_id=scholar_id, - cstart=state.current_cstart, - page_size=bounded_page_size, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds, - ) - - @staticmethod - def _record_next_page( - *, - state: PagedLoopState, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - page_attempt_log: list[dict[str, Any]], - ) -> None: - state.pages_attempted += 1 - state.attempt_log.extend(page_attempt_log) - state.page_logs.append( - ScholarIngestionService._page_log_entry( - page_number=state.pages_attempted, - cstart=state.current_cstart, - fetch_result=fetch_result, - parsed_page=parsed_page, - attempt_count=len(page_attempt_log), - ) - ) - state.fetch_result = fetch_result - state.parsed_page = parsed_page - - @staticmethod - def _handle_page_state_transition(*, state: PagedLoopState) -> bool: - if state.parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: - ScholarIngestionService._set_truncated_state( - state=state, - reason=f"page_state_{state.parsed_page.state.value}", - continuation_cstart=state.current_cstart, - ) - return True - if state.parsed_page.state == ParseState.NO_RESULTS and len(state.parsed_page.publications) == 0: - state.pages_fetched += 1 - return True - state.pages_fetched += 1 - state.publications.extend(state.parsed_page.publications) - state.next_cstart = _next_cstart_value( - articles_range=state.parsed_page.articles_range, - fallback=state.current_cstart + len(state.parsed_page.publications), - ) - return False - - async def _fetch_initial_page_context( - self, - *, - scholar_id: str, - start_cstart: int, - bounded_page_size: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - ) -> tuple[FetchResult, ParsedProfilePage, str | None, list[dict[str, Any]], list[dict[str, Any]]]: - fetch_result, parsed_page, first_attempt_log = await self._fetch_and_parse_page_with_retry( - scholar_id=scholar_id, - cstart=start_cstart, - page_size=bounded_page_size, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds, - ) - first_page_fingerprint_sha256 = build_initial_page_fingerprint(parsed_page) - attempt_log = list(first_attempt_log) - page_logs = [ - self._page_log_entry( - page_number=1, - cstart=start_cstart, - fetch_result=fetch_result, - parsed_page=parsed_page, - attempt_count=len(first_attempt_log), - ) - ] - return fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs - - async def _paginate_loop( - self, - *, - scholar: ScholarProfile, - run: CrawlRun, - db_session: AsyncSession, - state: PagedLoopState, - bounded_max_pages: int, - request_delay_seconds: int, - bounded_page_size: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - ) -> None: - if state.parsed_page.publications: - discovered_count = await self._upsert_profile_publications( - db_session, run=run, scholar=scholar, publications=list(state.parsed_page.publications) - ) - state.discovered_publication_count += discovered_count - await db_session.commit() - - while state.parsed_page.has_show_more_button: - from app.db.models import RunStatus - await db_session.refresh(run) - if run.status == RunStatus.CANCELED: - logger.info( - "ingestion.pagination_canceled", - extra={"event": "ingestion.pagination_canceled", "run_id": run.id}, - ) - self._set_truncated_state( - state=state, - reason="run_canceled", - continuation_cstart=state.current_cstart, - ) - return - - if self._should_stop_pagination(state=state, bounded_max_pages=bounded_max_pages): - return - next_fetch_result, next_parsed_page, next_attempt_log = await self._fetch_next_page( - scholar_id=scholar.scholar_id, - state=state, - request_delay_seconds=request_delay_seconds, - bounded_page_size=bounded_page_size, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds, - ) - self._record_next_page( - state=state, - fetch_result=next_fetch_result, - parsed_page=next_parsed_page, - page_attempt_log=next_attempt_log, - ) - - # Immediately commit the new page's publications to the DB - if next_parsed_page.publications: - discovered_count = await self._upsert_profile_publications( - db_session, run=run, scholar=scholar, publications=list(next_parsed_page.publications) - ) - state.discovered_publication_count += discovered_count - await db_session.commit() - - if self._handle_page_state_transition(state=state): - return - - @staticmethod - def _result_from_pagination_state( - *, - state: PagedLoopState, - first_page_fetch_result: FetchResult, - first_page_parsed_page: ParsedProfilePage, - first_page_fingerprint_sha256: str | None, - ) -> PagedParseResult: - return PagedParseResult( - fetch_result=state.fetch_result, - parsed_page=state.parsed_page, - first_page_fetch_result=first_page_fetch_result, - first_page_parsed_page=first_page_parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - publications=_dedupe_publication_candidates(state.publications), - attempt_log=state.attempt_log, - page_logs=state.page_logs, - pages_fetched=state.pages_fetched, - pages_attempted=state.pages_attempted, - has_more_remaining=state.has_more_remaining, - pagination_truncated_reason=state.pagination_truncated_reason, - continuation_cstart=state.continuation_cstart, - skipped_no_change=False, - discovered_publication_count=state.discovered_publication_count, - ) - - def _short_circuit_initial_page( - self, - *, - start_cstart: int, - previous_initial_page_fingerprint_sha256: str | None, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - first_page_fingerprint_sha256: str | None, - attempt_log: list[dict[str, Any]], - page_logs: list[dict[str, Any]], - ) -> PagedParseResult | None: - if self._should_skip_no_change( - start_cstart=start_cstart, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256, - parsed_page=parsed_page, - ): - return self._skip_no_change_result( - fetch_result=fetch_result, - parsed_page=parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - attempt_log=attempt_log, - page_logs=page_logs, - ) - if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: - return self._initial_failure_result( - fetch_result=fetch_result, - parsed_page=parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - start_cstart=start_cstart, - attempt_log=attempt_log, - page_logs=page_logs, - ) - return None - - async def _fetch_and_parse_all_pages_with_retry( - self, - *, - scholar: ScholarProfile, - run: CrawlRun, - db_session: AsyncSession, - start_cstart: int, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - max_pages: int, - page_size: int, - previous_initial_page_fingerprint_sha256: str | None = None - ) -> PagedParseResult: - bounded_max_pages = max(1, int(max_pages)) - bounded_page_size = max(1, int(page_size)) - fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs = ( - await self._fetch_initial_page_context( - scholar_id=scholar.scholar_id, - start_cstart=start_cstart, - bounded_page_size=bounded_page_size, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds, - )) - shortcut_result = self._short_circuit_initial_page( - start_cstart=start_cstart, - previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256, - fetch_result=fetch_result, - parsed_page=parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - attempt_log=attempt_log, - page_logs=page_logs, - ) - if shortcut_result is not None: - return shortcut_result - state = self._build_loop_state( - start_cstart=start_cstart, - fetch_result=fetch_result, - parsed_page=parsed_page, - attempt_log=attempt_log, - page_logs=page_logs, - ) - await self._paginate_loop( - scholar=scholar, - run=run, - db_session=db_session, - state=state, - bounded_max_pages=bounded_max_pages, - request_delay_seconds=request_delay_seconds, - bounded_page_size=bounded_page_size, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds, - ) - return self._result_from_pagination_state( - state=state, - first_page_fetch_result=fetch_result, - first_page_parsed_page=parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - ) - - async def _enrich_publications_with_openalex( - self, - scholar: ScholarProfile, - publications: list[PublicationCandidate], - ) -> list[PublicationCandidate]: - if not publications: - return publications - - from app.services.domains.openalex.client import OpenAlexClient - from app.services.domains.openalex.matching import find_best_match - - client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto) - - # Batch requests by 25 to stay within URL length limits - batch_size = 25 - enriched: list[PublicationCandidate] = [] - - import re - for i in range(0, len(publications), batch_size): - batch = publications[i:i + batch_size] - - titles = [] - for p in batch: - if not p.title: - continue - # Strip all non-alphanumeric words to prevent OpenAlex API injection crashes - # and minimize punctuation-based duplicate misses. - safe_title = re.sub(r"[^\w\s]", " ", p.title) - safe_title = " ".join(safe_title.split()) - if safe_title: - titles.append(safe_title) - - if not titles: - enriched.extend(batch) - continue - - query = "|".join(t for t in titles) - try: - openalex_works = await client.get_works_by_filter( - {"title.search": query}, limit=batch_size * 3 - ) - except Exception as e: - logger.warning( - "ingestion.openalex_enrichment_failed", - extra={"error": str(e), "batch_size": len(batch), "scholar_id": scholar.id} - ) - openalex_works = [] - - for p in batch: - match = find_best_match( - target_title=p.title, - target_year=p.year, - target_authors=p.authors_text or scholar.name, - candidates=openalex_works - ) - if match: - new_p = PublicationCandidate( - title=p.title, - title_url=p.title_url, - cluster_id=p.cluster_id, - year=match.publication_year or p.year, - citation_count=match.cited_by_count, - authors_text=p.authors_text, - venue_text=p.venue_text, - pdf_url=match.oa_url or p.pdf_url, - ) - enriched.append(new_p) - else: - enriched.append(p) - - return enriched - - async def _upsert_profile_publications( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - publications: list[PublicationCandidate], - ) -> int: - publications = await self._enrich_publications_with_openalex(scholar, publications) - - seen_publication_ids: set[int] = set() - discovered_count = 0 - - for candidate in publications: - publication = await self._resolve_publication(db_session, candidate) - if publication.id in seen_publication_ids: - continue - seen_publication_ids.add(publication.id) - - link_result = await db_session.execute( - select(ScholarPublication).where( - ScholarPublication.scholar_profile_id == scholar.id, - ScholarPublication.publication_id == publication.id, - ) - ) - link = link_result.scalar_one_or_none() - if link is not None: - continue - - link = ScholarPublication( - scholar_profile_id=scholar.id, - publication_id=publication.id, - is_read=False, - first_seen_run_id=run.id, - ) - db_session.add(link) - discovered_count += 1 - - logger.debug( - "ingestion.publication_discovered", - extra={ - "event": "ingestion.publication_discovered", - "scholar_profile_id": scholar.id, - "publication_id": publication.id, - "crawl_run_id": run.id, - }, - ) - - # Commit immediately to ensure the row becomes visible to the UI immediately - await db_session.commit() - - # Emit Server-Sent Event for live UI progress updates - await run_events.publish( - run_id=run.id, - event_type="publication_discovered", - data={ - "publication_id": publication.id, - "title": publication.title_raw, - "pub_url": publication.pub_url, - "scholar_profile_id": scholar.id, - "scholar_label": scholar.display_name or scholar.scholar_id, - "first_seen_at": datetime.now(timezone.utc).isoformat(), - } - ) - - if not scholar.baseline_completed: - scholar.baseline_completed = True - - return discovered_count - - @staticmethod - def _validate_publication_candidate(candidate: PublicationCandidate) -> None: - if not candidate.title.strip(): - raise RuntimeError("Publication candidate is missing title.") - if candidate.citation_count is not None and int(candidate.citation_count) < 0: - raise RuntimeError("Publication candidate has negative citation_count.") - - async def _find_publication_by_cluster( - self, - db_session: AsyncSession, - *, - cluster_id: str | None, - ) -> Publication | None: - if not cluster_id: - return None - result = await db_session.execute( - select(Publication).where(Publication.cluster_id == cluster_id) - ) - return result.scalar_one_or_none() - - async def _find_publication_by_fingerprint( - self, - db_session: AsyncSession, - *, - fingerprint: str, - ) -> Publication | None: - result = await db_session.execute( - select(Publication).where(Publication.fingerprint_sha256 == fingerprint) - ) - return result.scalar_one_or_none() - - @staticmethod - def _select_existing_publication( - *, - cluster_publication: Publication | None, - fingerprint_publication: Publication | None, - ) -> Publication | None: - if cluster_publication is not None: - return cluster_publication - return fingerprint_publication - - async def _create_publication( - self, - db_session: AsyncSession, - *, - candidate: PublicationCandidate, - fingerprint: str, - ) -> Publication: - publication = Publication( - cluster_id=candidate.cluster_id, - fingerprint_sha256=fingerprint, - title_raw=candidate.title, - title_normalized=normalize_title(candidate.title), - year=candidate.year, - citation_count=int(candidate.citation_count or 0), - author_text=candidate.authors_text, - venue_text=candidate.venue_text, - pub_url=build_publication_url(candidate.title_url), - pdf_url=None, - ) - db_session.add(publication) - await db_session.flush() - logger.debug( - "ingestion.publication_created", - extra={ - "event": "ingestion.publication_created", - "publication_id": publication.id, - "cluster_id": publication.cluster_id, - }, - ) - return publication - - @staticmethod - def _update_existing_publication( - *, - publication: Publication, - candidate: PublicationCandidate, - ) -> None: - if candidate.cluster_id and publication.cluster_id is None: - publication.cluster_id = candidate.cluster_id - publication.title_raw = candidate.title - publication.title_normalized = normalize_title(candidate.title) - if candidate.year is not None: - publication.year = candidate.year - if candidate.citation_count is not None: - publication.citation_count = int(candidate.citation_count) - if candidate.authors_text: - publication.author_text = candidate.authors_text - if candidate.venue_text: - publication.venue_text = candidate.venue_text - if candidate.title_url: - publication.pub_url = build_publication_url(candidate.title_url) - local_doi = first_doi_from_texts(candidate.title_url, candidate.venue_text, candidate.title) - - async def _resolve_publication( - self, - db_session: AsyncSession, - candidate: PublicationCandidate, - ) -> Publication: - self._validate_publication_candidate(candidate) - fingerprint = build_publication_fingerprint(candidate) - cluster_publication = await self._find_publication_by_cluster( - db_session, - cluster_id=candidate.cluster_id, - ) - fingerprint_publication = await self._find_publication_by_fingerprint( - db_session, - fingerprint=fingerprint, - ) - publication = self._select_existing_publication( - cluster_publication=cluster_publication, - fingerprint_publication=fingerprint_publication, - ) - if publication is None: - created = await self._create_publication( - db_session, - candidate=candidate, - fingerprint=fingerprint, - ) - await identifier_service.discover_and_sync_identifiers_for_publication( - db_session, - publication=created, - scholar_label=candidate.authors_text or "", - ) - return created - self._update_existing_publication( - publication=publication, - candidate=candidate, - ) - await identifier_service.discover_and_sync_identifiers_for_publication( - db_session, - publication=publication, - scholar_label=candidate.authors_text or "", - ) - return publication - - def _resolve_run_status( - self, - *, - scholar_count: int, - succeeded_count: int, - failed_count: int, - partial_count: int, - ) -> RunStatus: - if scholar_count == 0: - return RunStatus.SUCCESS - if failed_count == scholar_count: - return RunStatus.FAILED - if failed_count > 0 or partial_count > 0: - return RunStatus.PARTIAL_FAILURE - if succeeded_count > 0: - return RunStatus.SUCCESS - return RunStatus.FAILED - - def _resolve_continuation_queue_target( - self, - *, - outcome: str, - state: str, - pagination_truncated_reason: str | None, - continuation_cstart: int | None, - fallback_cstart: int, - ) -> tuple[str | None, int]: - if outcome == "partial": - reason = (pagination_truncated_reason or "").strip() - if reason in RESUMABLE_PARTIAL_REASONS or reason.startswith( - RESUMABLE_PARTIAL_REASON_PREFIXES - ): - return reason, queue_service.normalize_cstart( - continuation_cstart if continuation_cstart is not None else fallback_cstart - ) - return None, queue_service.normalize_cstart(fallback_cstart) - - if outcome == "failed" and state == ParseState.NETWORK_ERROR.value: - return "network_error_retry", queue_service.normalize_cstart( - continuation_cstart if continuation_cstart is not None else fallback_cstart - ) - - return None, queue_service.normalize_cstart(fallback_cstart) - - def _build_failure_debug_context( - self, - *, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - attempt_log: list[dict[str, Any]], - page_logs: list[dict[str, Any]] | None = None, - exception: Exception | None = None, - ) -> dict[str, Any]: - context: dict[str, Any] = { - "requested_url": fetch_result.requested_url, - "final_url": fetch_result.final_url, - "status_code": fetch_result.status_code, - "fetch_error": fetch_result.error, - "state_reason": parsed_page.state_reason, - "profile_name": parsed_page.profile_name, - "profile_image_url": parsed_page.profile_image_url, - "articles_range": parsed_page.articles_range, - "has_show_more_button": parsed_page.has_show_more_button, - "has_operation_error_banner": parsed_page.has_operation_error_banner, - "warning_codes": parsed_page.warnings, - "marker_counts_nonzero": { - key: value for key, value in parsed_page.marker_counts.items() if value > 0 - }, - "body_length": len(fetch_result.body), - "body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest() - if fetch_result.body - else None, - "body_excerpt": _build_body_excerpt(fetch_result.body), - "attempt_log": attempt_log, - } - if page_logs: - context["page_logs"] = page_logs - if exception is not None: - context["exception_type"] = type(exception).__name__ - context["exception_message"] = str(exception) - return context - - async def _try_acquire_user_lock( - self, - db_session: AsyncSession, - *, - user_id: int, - ) -> bool: - result = await db_session.execute( - text( - "SELECT pg_try_advisory_xact_lock(:namespace, :user_key)" - ), - { - "namespace": RUN_LOCK_NAMESPACE, - "user_key": int(user_id), - }, - ) - return bool(result.scalar_one()) diff --git a/build/lib/app/services/domains/ingestion/constants.py b/build/lib/app/services/domains/ingestion/constants.py deleted file mode 100644 index cfe2ba0..0000000 --- a/build/lib/app/services/domains/ingestion/constants.py +++ /dev/null @@ -1,31 +0,0 @@ -from __future__ import annotations - -import re - -from app.services.domains.scholar.parser import ParseState - -TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+") -WORD_RE = re.compile(r"[a-z0-9]+") -HTML_TAG_RE = re.compile(r"<[^>]+>", re.S) -SPACE_RE = re.compile(r"\s+") - -FAILED_STATES = { - ParseState.BLOCKED_OR_CAPTCHA.value, - ParseState.LAYOUT_CHANGED.value, - ParseState.NETWORK_ERROR.value, - "ingestion_error", -} - -FAILURE_BUCKET_BLOCKED = "blocked_or_captcha" -FAILURE_BUCKET_NETWORK = "network_error" -FAILURE_BUCKET_LAYOUT = "layout_changed" -FAILURE_BUCKET_INGESTION = "ingestion_error" -FAILURE_BUCKET_OTHER = "other_failure" - -RUN_LOCK_NAMESPACE = 8217 -RESUMABLE_PARTIAL_REASONS = { - "max_pages_reached", - "pagination_cursor_stalled", -} -RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",) -INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS = 30 diff --git a/build/lib/app/services/domains/ingestion/fingerprints.py b/build/lib/app/services/domains/ingestion/fingerprints.py deleted file mode 100644 index b59710e..0000000 --- a/build/lib/app/services/domains/ingestion/fingerprints.py +++ /dev/null @@ -1,175 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import re -from typing import Any -from urllib.parse import urljoin - -from app.services.domains.ingestion.constants import ( - HTML_TAG_RE, - INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS, - SPACE_RE, - TITLE_ALNUM_RE, - WORD_RE, -) -from app.services.domains.scholar.parser import ParseState, ParsedProfilePage, PublicationCandidate - - -def normalize_title(value: str) -> str: - lowered = value.lower() - return TITLE_ALNUM_RE.sub("", lowered) - - -def _first_author_last_name(authors_text: str | None) -> str: - if not authors_text: - return "" - first_author = authors_text.split(",", maxsplit=1)[0].strip().lower() - words = WORD_RE.findall(first_author) - if not words: - return "" - return words[-1] - - -def _first_venue_word(venue_text: str | None) -> str: - if not venue_text: - return "" - words = WORD_RE.findall(venue_text.lower()) - if not words: - return "" - return words[0] - - -def build_publication_fingerprint(candidate: PublicationCandidate) -> str: - canonical = "|".join( - [ - normalize_title(candidate.title), - str(candidate.year) if candidate.year is not None else "", - _first_author_last_name(candidate.authors_text), - _first_venue_word(candidate.venue_text), - ] - ) - return hashlib.sha256(canonical.encode("utf-8")).hexdigest() - - -def build_initial_page_fingerprint(parsed_page: ParsedProfilePage) -> str | None: - if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: - return None - - normalized_rows: list[dict[str, Any]] = [] - for publication in parsed_page.publications[:INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS]: - normalized_rows.append( - { - "cluster_id": publication.cluster_id or "", - "title_normalized": normalize_title(publication.title), - "year": publication.year, - "citation_count": publication.citation_count, - } - ) - - payload = { - "state": parsed_page.state.value, - "articles_range": parsed_page.articles_range or "", - "has_show_more_button": parsed_page.has_show_more_button, - "profile_name": parsed_page.profile_name or "", - "publications": normalized_rows, - } - canonical = json.dumps( - payload, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=True, - ) - return hashlib.sha256(canonical.encode("utf-8")).hexdigest() - - -def build_publication_url(path_or_url: str | None) -> str | None: - if not path_or_url: - return None - return urljoin("https://scholar.google.com", path_or_url) - - -def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int: - if articles_range: - numbers = re.findall(r"\d+", articles_range) - if len(numbers) >= 2: - try: - return int(numbers[1]) - except ValueError: - pass - return int(fallback) - - -def _title_tokens(value: str) -> set[str]: - """Extract normalized word tokens for fuzzy title comparison.""" - return set(WORD_RE.findall(value.lower())) - - -def fuzzy_titles_match( - title_a: str, - title_b: str, - *, - threshold: float = 0.85, -) -> bool: - """Return True if two titles are near-duplicates by token-level Jaccard similarity. - - A threshold of 0.85 catches common academic duplicate patterns: - differences in punctuation, minor word variations, subtitle changes. - """ - tokens_a = _title_tokens(title_a) - tokens_b = _title_tokens(title_b) - if not tokens_a or not tokens_b: - return False - intersection = tokens_a & tokens_b - union = tokens_a | tokens_b - return (len(intersection) / len(union)) >= threshold - - -def _dedupe_publication_candidates( - publications: list[PublicationCandidate], -) -> list[PublicationCandidate]: - deduped: list[PublicationCandidate] = [] - seen: set[str] = set() - seen_titles: list[tuple[str, int]] = [] # (normalized_title, index into deduped) - - for publication in publications: - if publication.cluster_id: - identity = f"cluster:{publication.cluster_id}" - else: - identity = "|".join( - [ - "fallback", - normalize_title(publication.title), - str(publication.year) if publication.year is not None else "", - _first_author_last_name(publication.authors_text), - _first_venue_word(publication.venue_text), - ] - ) - if identity in seen: - continue - - # Fuzzy title check — catch near-identical titles not caught by exact fingerprint - norm_title = normalize_title(publication.title) - is_fuzzy_dup = False - for existing_title, _idx in seen_titles: - if fuzzy_titles_match(norm_title, existing_title): - is_fuzzy_dup = True - break - if is_fuzzy_dup: - continue - - seen.add(identity) - seen_titles.append((norm_title, len(deduped))) - deduped.append(publication) - return deduped - - -def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None: - if not body: - return None - flattened = SPACE_RE.sub(" ", HTML_TAG_RE.sub(" ", body)).strip() - if not flattened: - return None - if len(flattened) <= max_chars: - return flattened - return f"{flattened[:max_chars - 1]}..." diff --git a/build/lib/app/services/domains/ingestion/queue.py b/build/lib/app/services/domains/ingestion/queue.py deleted file mode 100644 index 7494420..0000000 --- a/build/lib/app/services/domains/ingestion/queue.py +++ /dev/null @@ -1,348 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import IngestionQueueItem, QueueItemStatus - - -@dataclass(frozen=True) -class ContinuationQueueJob: - id: int - user_id: int - scholar_profile_id: int - resume_cstart: int - reason: str - status: str - attempt_count: int - next_attempt_dt: datetime - - -ACTIVE_QUEUE_STATUSES: tuple[str, ...] = ( - QueueItemStatus.QUEUED.value, - QueueItemStatus.RETRYING.value, -) - - -def normalize_cstart(value: int | None) -> int: - if value is None: - return 0 - return max(0, int(value)) - - -def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_seconds: int) -> int: - base = max(1, int(base_seconds)) - attempts = max(1, int(attempt_count)) - maximum = max(base, int(max_seconds)) - seconds = base * (2 ** max(0, attempts - 1)) - return min(seconds, maximum) - - -async def _get_item_for_user_scholar( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int, -) -> IngestionQueueItem | None: - result = await db_session.execute( - select(IngestionQueueItem).where( - IngestionQueueItem.user_id == user_id, - IngestionQueueItem.scholar_profile_id == scholar_profile_id, - ) - ) - return result.scalar_one_or_none() - - -def _build_queue_item( - *, - now: datetime, - user_id: int, - scholar_profile_id: int, - normalized_cstart: int, - reason: str, - run_id: int | None, - next_attempt_dt: datetime, -) -> IngestionQueueItem: - return IngestionQueueItem( - user_id=user_id, - scholar_profile_id=scholar_profile_id, - resume_cstart=normalized_cstart, - reason=reason, - status=QueueItemStatus.QUEUED.value, - attempt_count=0, - next_attempt_dt=next_attempt_dt, - last_run_id=run_id, - last_error=None, - dropped_reason=None, - dropped_at=None, - created_at=now, - updated_at=now, - ) - - -def _update_existing_queue_item( - *, - item: IngestionQueueItem, - now: datetime, - normalized_cstart: int, - reason: str, - run_id: int | None, - next_attempt_dt: datetime, -) -> None: - item.resume_cstart = normalized_cstart - item.reason = reason - if item.status == QueueItemStatus.DROPPED.value: - item.attempt_count = 0 - item.status = QueueItemStatus.QUEUED.value - item.next_attempt_dt = next_attempt_dt - item.last_run_id = run_id - item.last_error = None - item.dropped_reason = None - item.dropped_at = None - item.updated_at = now - - -async def upsert_job( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int, - resume_cstart: int, - reason: str, - run_id: int | None, - delay_seconds: int, -) -> IngestionQueueItem: - now = datetime.now(timezone.utc) - next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds))) - item = await _get_item_for_user_scholar( - db_session, - user_id=user_id, - scholar_profile_id=scholar_profile_id, - ) - normalized_cstart = normalize_cstart(resume_cstart) - if item is None: - item = _build_queue_item( - now=now, - user_id=user_id, - scholar_profile_id=scholar_profile_id, - normalized_cstart=normalized_cstart, - reason=reason, - run_id=run_id, - next_attempt_dt=next_attempt_dt, - ) - db_session.add(item) - return item - - _update_existing_queue_item( - item=item, - now=now, - normalized_cstart=normalized_cstart, - reason=reason, - run_id=run_id, - next_attempt_dt=next_attempt_dt, - ) - return item - - -async def clear_job_for_scholar( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int, -) -> bool: - result = await db_session.execute( - select(IngestionQueueItem).where( - IngestionQueueItem.user_id == user_id, - IngestionQueueItem.scholar_profile_id == scholar_profile_id, - ) - ) - item = result.scalar_one_or_none() - if item is None: - return False - await db_session.delete(item) - return True - - -async def delete_job_by_id( - db_session: AsyncSession, - *, - job_id: int, -) -> bool: - result = await db_session.execute( - select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) - ) - item = result.scalar_one_or_none() - if item is None: - return False - await db_session.delete(item) - return True - - -async def list_due_jobs( - db_session: AsyncSession, - *, - now: datetime, - limit: int, -) -> list[ContinuationQueueJob]: - result = await db_session.execute( - select(IngestionQueueItem) - .where( - IngestionQueueItem.next_attempt_dt <= now, - IngestionQueueItem.status.in_(ACTIVE_QUEUE_STATUSES), - ) - .order_by( - IngestionQueueItem.next_attempt_dt.asc(), - IngestionQueueItem.id.asc(), - ) - .limit(limit) - ) - rows = list(result.scalars().all()) - jobs: list[ContinuationQueueJob] = [] - for row in rows: - jobs.append( - ContinuationQueueJob( - id=int(row.id), - user_id=int(row.user_id), - scholar_profile_id=int(row.scholar_profile_id), - resume_cstart=normalize_cstart(row.resume_cstart), - reason=row.reason, - status=row.status, - attempt_count=int(row.attempt_count), - next_attempt_dt=row.next_attempt_dt, - ) - ) - return jobs - - -async def increment_attempt_count( - db_session: AsyncSession, - *, - job_id: int, -) -> IngestionQueueItem | None: - now = datetime.now(timezone.utc) - result = await db_session.execute( - select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) - ) - item = result.scalar_one_or_none() - if item is None: - return None - item.attempt_count = int(item.attempt_count or 0) + 1 - item.updated_at = now - return item - - -async def reset_attempt_count( - db_session: AsyncSession, - *, - job_id: int, -) -> IngestionQueueItem | None: - now = datetime.now(timezone.utc) - result = await db_session.execute( - select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) - ) - item = result.scalar_one_or_none() - if item is None: - return None - item.attempt_count = 0 - item.updated_at = now - return item - - -async def reschedule_job( - db_session: AsyncSession, - *, - job_id: int, - delay_seconds: int, - reason: str, - error: str | None = None, -) -> IngestionQueueItem | None: - now = datetime.now(timezone.utc) - result = await db_session.execute( - select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) - ) - item = result.scalar_one_or_none() - if item is None: - return None - item.next_attempt_dt = now + timedelta(seconds=max(1, int(delay_seconds))) - item.status = QueueItemStatus.QUEUED.value - item.reason = reason - item.last_error = error - item.dropped_reason = None - item.dropped_at = None - item.updated_at = now - return item - - -async def mark_retrying( - db_session: AsyncSession, - *, - job_id: int, - reason: str | None = None, -) -> IngestionQueueItem | None: - now = datetime.now(timezone.utc) - result = await db_session.execute( - select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) - ) - item = result.scalar_one_or_none() - if item is None: - return None - if item.status == QueueItemStatus.DROPPED.value: - return item - item.status = QueueItemStatus.RETRYING.value - if reason: - item.reason = reason - item.updated_at = now - return item - - -async def mark_dropped( - db_session: AsyncSession, - *, - job_id: int, - reason: str, - error: str | None = None, -) -> IngestionQueueItem | None: - now = datetime.now(timezone.utc) - result = await db_session.execute( - select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) - ) - item = result.scalar_one_or_none() - if item is None: - return None - item.status = QueueItemStatus.DROPPED.value - item.reason = "dropped" - item.dropped_reason = reason - item.dropped_at = now - if error is not None: - item.last_error = error - item.updated_at = now - return item - - -async def mark_queued_now( - db_session: AsyncSession, - *, - job_id: int, - reason: str, - reset_attempt_count: bool = False, -) -> IngestionQueueItem | None: - now = datetime.now(timezone.utc) - result = await db_session.execute( - select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) - ) - item = result.scalar_one_or_none() - if item is None: - return None - item.status = QueueItemStatus.QUEUED.value - item.reason = reason - item.next_attempt_dt = now - if reset_attempt_count: - item.attempt_count = 0 - item.last_error = None - item.dropped_reason = None - item.dropped_at = None - item.updated_at = now - return item diff --git a/build/lib/app/services/domains/ingestion/safety.py b/build/lib/app/services/domains/ingestion/safety.py deleted file mode 100644 index debb8c5..0000000 --- a/build/lib/app/services/domains/ingestion/safety.py +++ /dev/null @@ -1,282 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timedelta, timezone -from typing import Any - -from app.db.models import UserSetting - -COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD = "blocked_failure_threshold_exceeded" -COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD = "network_failure_threshold_exceeded" - -_COUNTER_CONSECUTIVE_BLOCKED_RUNS = "consecutive_blocked_runs" -_COUNTER_CONSECUTIVE_NETWORK_RUNS = "consecutive_network_runs" -_COUNTER_COOLDOWN_ENTRY_COUNT = "cooldown_entry_count" -_COUNTER_BLOCKED_START_COUNT = "blocked_start_count" -_COUNTER_LAST_BLOCKED_FAILURE_COUNT = "last_blocked_failure_count" -_COUNTER_LAST_NETWORK_FAILURE_COUNT = "last_network_failure_count" -_COUNTER_LAST_EVALUATED_RUN_ID = "last_evaluated_run_id" - - -def _utcnow() -> datetime: - return datetime.now(timezone.utc) - - -def _safe_int(value: Any, default: int = 0) -> int: - try: - return int(value) - except (TypeError, ValueError): - return default - - -def _safe_optional_int(value: Any) -> int | None: - if value is None: - return None - try: - return int(value) - except (TypeError, ValueError): - return None - - -def _normalize_datetime(value: datetime | None) -> datetime | None: - if value is None: - return None - if value.tzinfo is None: - return value.replace(tzinfo=timezone.utc) - return value.astimezone(timezone.utc) - - -def _state_dict(settings: UserSetting) -> dict[str, Any]: - state = settings.scrape_safety_state - if isinstance(state, dict): - return state - return {} - - -def _counters_from_state(settings: UserSetting) -> dict[str, Any]: - state = _state_dict(settings) - return { - _COUNTER_CONSECUTIVE_BLOCKED_RUNS: max( - 0, - _safe_int(state.get(_COUNTER_CONSECUTIVE_BLOCKED_RUNS), 0), - ), - _COUNTER_CONSECUTIVE_NETWORK_RUNS: max( - 0, - _safe_int(state.get(_COUNTER_CONSECUTIVE_NETWORK_RUNS), 0), - ), - _COUNTER_COOLDOWN_ENTRY_COUNT: max( - 0, - _safe_int(state.get(_COUNTER_COOLDOWN_ENTRY_COUNT), 0), - ), - _COUNTER_BLOCKED_START_COUNT: max( - 0, - _safe_int(state.get(_COUNTER_BLOCKED_START_COUNT), 0), - ), - _COUNTER_LAST_BLOCKED_FAILURE_COUNT: max( - 0, - _safe_int(state.get(_COUNTER_LAST_BLOCKED_FAILURE_COUNT), 0), - ), - _COUNTER_LAST_NETWORK_FAILURE_COUNT: max( - 0, - _safe_int(state.get(_COUNTER_LAST_NETWORK_FAILURE_COUNT), 0), - ), - _COUNTER_LAST_EVALUATED_RUN_ID: _safe_optional_int( - state.get(_COUNTER_LAST_EVALUATED_RUN_ID), - ), - } - - -def _cooldown_reason_label(reason: str | None) -> str | None: - if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD: - return "Blocked responses exceeded safety threshold" - if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD: - return "Network failures exceeded safety threshold" - return None - - -def _recommended_action(reason: str | None) -> str | None: - if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD: - return ( - "Google Scholar appears to be blocking requests. Wait for cooldown to expire, " - "increase request delay, and avoid repeated manual retries." - ) - if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD: - return ( - "Network failures crossed the threshold. Verify connectivity and retry after cooldown." - ) - return None - - -def is_cooldown_active( - settings: UserSetting, - *, - now_utc: datetime | None = None, -) -> bool: - now = now_utc or _utcnow() - cooldown_until = _normalize_datetime(settings.scrape_cooldown_until) - if cooldown_until is None: - return False - return cooldown_until > now - - -def clear_expired_cooldown( - settings: UserSetting, - *, - now_utc: datetime | None = None, -) -> bool: - now = now_utc or _utcnow() - cooldown_until = _normalize_datetime(settings.scrape_cooldown_until) - if cooldown_until is None: - return False - if cooldown_until > now: - return False - settings.scrape_cooldown_until = None - settings.scrape_cooldown_reason = None - return True - - -def register_cooldown_blocked_start( - settings: UserSetting, - *, - now_utc: datetime | None = None, -) -> dict[str, Any]: - now = now_utc or _utcnow() - counters = _counters_from_state(settings) - counters[_COUNTER_BLOCKED_START_COUNT] = int(counters[_COUNTER_BLOCKED_START_COUNT]) + 1 - settings.scrape_safety_state = counters - return get_safety_state_payload(settings, now_utc=now) - - -def _update_run_counters( - *, - counters: dict[str, Any], - run_id: int, - blocked_failure_count: int, - network_failure_count: int, -) -> tuple[int, int]: - bounded_blocked_failures = max(0, int(blocked_failure_count)) - bounded_network_failures = max(0, int(network_failure_count)) - counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures - counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures - counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id) - counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = ( - int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1 - if bounded_blocked_failures > 0 - else 0 - ) - counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = ( - int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1 - if bounded_network_failures > 0 - else 0 - ) - return bounded_blocked_failures, bounded_network_failures - - -def _resolve_cooldown_trigger( - *, - blocked_failures: int, - network_failures: int, - blocked_failure_threshold: int, - network_failure_threshold: int, - blocked_cooldown_seconds: int, - network_cooldown_seconds: int, -) -> tuple[str | None, int]: - if blocked_failures >= max(1, int(blocked_failure_threshold)): - return COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD, max(60, int(blocked_cooldown_seconds)) - if network_failures >= max(1, int(network_failure_threshold)): - return COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD, max(60, int(network_cooldown_seconds)) - return None, 0 - - -def _apply_cooldown_decision( - *, - settings: UserSetting, - counters: dict[str, Any], - now: datetime, - reason: str | None, - cooldown_seconds: int, -) -> None: - if reason is None: - clear_expired_cooldown(settings, now_utc=now) - return - settings.scrape_cooldown_reason = reason - settings.scrape_cooldown_until = now + timedelta(seconds=max(60, int(cooldown_seconds))) - counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1 - - -def apply_run_safety_outcome( - settings: UserSetting, - *, - run_id: int, - blocked_failure_count: int, - network_failure_count: int, - blocked_failure_threshold: int, - network_failure_threshold: int, - blocked_cooldown_seconds: int, - network_cooldown_seconds: int, - now_utc: datetime | None = None, -) -> tuple[dict[str, Any], str | None]: - now = now_utc or _utcnow() - counters = _counters_from_state(settings) - blocked_failures, network_failures = _update_run_counters( - counters=counters, - run_id=run_id, - blocked_failure_count=blocked_failure_count, - network_failure_count=network_failure_count, - ) - reason, cooldown_seconds = _resolve_cooldown_trigger( - blocked_failures=blocked_failures, - network_failures=network_failures, - blocked_failure_threshold=blocked_failure_threshold, - network_failure_threshold=network_failure_threshold, - blocked_cooldown_seconds=blocked_cooldown_seconds, - network_cooldown_seconds=network_cooldown_seconds, - ) - _apply_cooldown_decision( - settings=settings, - counters=counters, - now=now, - reason=reason, - cooldown_seconds=cooldown_seconds, - ) - settings.scrape_safety_state = counters - return get_safety_state_payload(settings, now_utc=now), reason - - -def get_safety_state_payload( - settings: UserSetting, - *, - now_utc: datetime | None = None, -) -> dict[str, Any]: - now = now_utc or _utcnow() - cooldown_until = _normalize_datetime(settings.scrape_cooldown_until) - cooldown_active = bool(cooldown_until is not None and cooldown_until > now) - cooldown_remaining_seconds = 0 - if cooldown_active and cooldown_until is not None: - cooldown_remaining_seconds = max(0, int((cooldown_until - now).total_seconds())) - - reason = settings.scrape_cooldown_reason if cooldown_active else None - - return { - "cooldown_active": cooldown_active, - "cooldown_reason": reason, - "cooldown_reason_label": _cooldown_reason_label(reason), - "cooldown_until": cooldown_until, - "cooldown_remaining_seconds": cooldown_remaining_seconds, - "recommended_action": _recommended_action(reason), - "counters": _counters_from_state(settings), - } - - -def get_safety_event_context( - settings: UserSetting, - *, - now_utc: datetime | None = None, -) -> dict[str, Any]: - payload = get_safety_state_payload(settings, now_utc=now_utc) - return { - "cooldown_active": bool(payload.get("cooldown_active")), - "cooldown_reason": payload.get("cooldown_reason"), - "cooldown_until": payload.get("cooldown_until"), - "cooldown_remaining_seconds": int(payload.get("cooldown_remaining_seconds") or 0), - "safety_counters": payload.get("counters", {}), - } diff --git a/build/lib/app/services/domains/ingestion/scheduler.py b/build/lib/app/services/domains/ingestion/scheduler.py deleted file mode 100644 index b05e84d..0000000 --- a/build/lib/app/services/domains/ingestion/scheduler.py +++ /dev/null @@ -1,622 +0,0 @@ -from __future__ import annotations - -import asyncio -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone -import logging - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import ( - CrawlRun, - QueueItemStatus, - RunTriggerType, - ScholarProfile, - User, - UserSetting, -) -from app.db.session import get_session_factory -from app.services.domains.ingestion import queue as queue_service -from app.services.domains.ingestion.application import ( - RunAlreadyInProgressError, - RunBlockedBySafetyPolicyError, - ScholarIngestionService, -) -from app.services.domains.settings import application as user_settings_service -from app.services.domains.scholar.source import LiveScholarSource -from app.settings import settings - -logger = logging.getLogger(__name__) - - -def _request_delay_floor_seconds() -> int: - return user_settings_service.resolve_request_delay_minimum( - settings.ingestion_min_request_delay_seconds - ) - - -def _effective_request_delay_seconds(value: int | None) -> int: - floor = _request_delay_floor_seconds() - try: - parsed = int(value) if value is not None else floor - except (TypeError, ValueError): - parsed = floor - return max(floor, parsed) - - -@dataclass(frozen=True) -class _AutoRunCandidate: - user_id: int - run_interval_minutes: int - request_delay_seconds: int - cooldown_until: datetime | None - cooldown_reason: str | None - - -class SchedulerService: - def __init__( - self, - *, - enabled: bool, - tick_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - continuation_queue_enabled: bool, - continuation_base_delay_seconds: int, - continuation_max_delay_seconds: int, - continuation_max_attempts: int, - queue_batch_size: int, - ) -> None: - self._enabled = enabled - self._tick_seconds = max(5, int(tick_seconds)) - self._network_error_retries = max(0, int(network_error_retries)) - self._retry_backoff_seconds = max(0.0, float(retry_backoff_seconds)) - self._max_pages_per_scholar = max(1, int(max_pages_per_scholar)) - self._page_size = max(1, int(page_size)) - self._continuation_queue_enabled = bool(continuation_queue_enabled) - self._continuation_base_delay_seconds = max(1, int(continuation_base_delay_seconds)) - self._continuation_max_delay_seconds = max( - self._continuation_base_delay_seconds, - int(continuation_max_delay_seconds), - ) - self._continuation_max_attempts = max(1, int(continuation_max_attempts)) - self._queue_batch_size = max(1, int(queue_batch_size)) - self._task: asyncio.Task[None] | None = None - self._source = LiveScholarSource() - - async def start(self) -> None: - if not self._enabled: - logger.info( - "scheduler.disabled", - extra={ - "event": "scheduler.disabled", - }, - ) - return - if self._task is not None: - return - self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler") - logger.info( - "scheduler.started", - extra={ - "event": "scheduler.started", - "tick_seconds": self._tick_seconds, - "network_error_retries": self._network_error_retries, - "retry_backoff_seconds": self._retry_backoff_seconds, - "max_pages_per_scholar": self._max_pages_per_scholar, - "page_size": self._page_size, - "continuation_queue_enabled": self._continuation_queue_enabled, - "continuation_base_delay_seconds": self._continuation_base_delay_seconds, - "continuation_max_delay_seconds": self._continuation_max_delay_seconds, - "continuation_max_attempts": self._continuation_max_attempts, - "queue_batch_size": self._queue_batch_size, - }, - ) - - async def stop(self) -> None: - if self._task is None: - return - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - finally: - self._task = None - logger.info("scheduler.stopped", extra={"event": "scheduler.stopped"}) - - async def _run_loop(self) -> None: - while True: - try: - await self._tick_once() - except asyncio.CancelledError: - raise - except Exception: - logger.exception( - "scheduler.tick_failed", - extra={ - "event": "scheduler.tick_failed", - }, - ) - await asyncio.sleep(float(self._tick_seconds)) - - async def _tick_once(self) -> None: - if self._continuation_queue_enabled: - await self._drain_continuation_queue() - - await self._drain_pdf_queue() - - candidates = await self._load_candidates() - if not candidates: - return - now = datetime.now(timezone.utc) - for candidate in candidates: - if not await self._is_due(candidate, now=now): - continue - await self._run_candidate(candidate) - - async def _load_candidate_rows(self) -> list[tuple]: - session_factory = get_session_factory() - async with session_factory() as session: - result = await session.execute( - select( - UserSetting.user_id, - UserSetting.run_interval_minutes, - UserSetting.request_delay_seconds, - UserSetting.scrape_cooldown_until, - UserSetting.scrape_cooldown_reason, - ) - .join(User, User.id == UserSetting.user_id) - .where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True)) - .order_by(UserSetting.user_id.asc()) - ) - return result.all() - - @staticmethod - def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None: - user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row - if cooldown_until is not None and cooldown_until.tzinfo is None: - cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) - if cooldown_until is not None and cooldown_until > now_utc: - logger.info( - "scheduler.run_skipped_safety_cooldown_precheck", - extra={ - "event": "scheduler.run_skipped_safety_cooldown_precheck", - "user_id": int(user_id), - "reason": cooldown_reason, - "cooldown_until": cooldown_until, - "cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()), - "metric_name": "scheduler_run_skipped_safety_cooldown_total", - "metric_value": 1, - }, - ) - return None - return _AutoRunCandidate( - user_id=int(user_id), - run_interval_minutes=int(run_interval_minutes), - request_delay_seconds=_effective_request_delay_seconds(request_delay_seconds), - cooldown_until=cooldown_until, - cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None), - ) - - async def _load_candidates(self) -> list[_AutoRunCandidate]: - if not settings.ingestion_automation_allowed: - return [] - rows = await self._load_candidate_rows() - now_utc = datetime.now(timezone.utc) - candidates: list[_AutoRunCandidate] = [] - for row in rows: - candidate = self._candidate_from_row(row, now_utc=now_utc) - if candidate is not None: - candidates.append(candidate) - return candidates - - async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool: - session_factory = get_session_factory() - async with session_factory() as session: - result = await session.execute( - select(CrawlRun.start_dt) - .where( - CrawlRun.user_id == candidate.user_id, - ) - .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) - .limit(1) - ) - last_run = result.scalar_one_or_none() - - if last_run is None: - return True - - next_due_dt = last_run + timedelta( - minutes=candidate.run_interval_minutes - ) - return now >= next_due_dt - - async def _run_candidate_ingestion( - self, - *, - candidate: _AutoRunCandidate, - ): - session_factory = get_session_factory() - async with session_factory() as session: - ingestion = ScholarIngestionService(source=self._source) - try: - return await ingestion.run_for_user( - session, - user_id=candidate.user_id, - trigger_type=RunTriggerType.SCHEDULED, - request_delay_seconds=candidate.request_delay_seconds, - network_error_retries=self._network_error_retries, - retry_backoff_seconds=self._retry_backoff_seconds, - max_pages_per_scholar=self._max_pages_per_scholar, - page_size=self._page_size, - auto_queue_continuations=self._continuation_queue_enabled, - queue_delay_seconds=self._continuation_base_delay_seconds, - alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, - alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, - alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, - ) - except RunAlreadyInProgressError: - await session.rollback() - logger.info("scheduler.run_skipped_locked", extra={"event": "scheduler.run_skipped_locked", "user_id": candidate.user_id}) - return None - except RunBlockedBySafetyPolicyError as exc: - await session.rollback() - logger.info( - "scheduler.run_skipped_safety_cooldown", - extra={ - "event": "scheduler.run_skipped_safety_cooldown", - "user_id": candidate.user_id, - "reason": exc.safety_state.get("cooldown_reason"), - "cooldown_until": exc.safety_state.get("cooldown_until"), - "cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"), - "metric_name": "scheduler_run_skipped_safety_cooldown_total", - "metric_value": 1, - }, - ) - return None - except Exception: - await session.rollback() - logger.exception("scheduler.run_failed", extra={"event": "scheduler.run_failed", "user_id": candidate.user_id}) - return None - - async def _run_candidate(self, candidate: _AutoRunCandidate) -> None: - run_summary = await self._run_candidate_ingestion(candidate=candidate) - if run_summary is None: - return - logger.info( - "scheduler.run_completed", - extra={ - "event": "scheduler.run_completed", - "user_id": candidate.user_id, - "run_id": run_summary.crawl_run_id, - "status": run_summary.status.value, - "scholar_count": run_summary.scholar_count, - "new_publication_count": run_summary.new_publication_count, - }, - ) - - async def _drain_continuation_queue(self) -> None: - now = datetime.now(timezone.utc) - session_factory = get_session_factory() - async with session_factory() as session: - jobs = await queue_service.list_due_jobs( - session, - now=now, - limit=self._queue_batch_size, - ) - for job in jobs: - await self._run_queue_job(job) - - async def _drop_queue_job_if_max_attempts( - self, - job: queue_service.ContinuationQueueJob, - ) -> bool: - if job.attempt_count < self._continuation_max_attempts: - return False - session_factory = get_session_factory() - async with session_factory() as session: - dropped = await queue_service.mark_dropped( - session, - job_id=job.id, - reason="max_attempts_reached", - ) - await session.commit() - if dropped is not None: - logger.warning( - "scheduler.queue_item_dropped_max_attempts", - extra={ - "event": "scheduler.queue_item_dropped_max_attempts", - "queue_item_id": job.id, - "user_id": job.user_id, - "scholar_profile_id": job.scholar_profile_id, - "attempt_count": job.attempt_count, - "max_attempts": self._continuation_max_attempts, - }, - ) - return True - - async def _mark_queue_job_retrying( - self, - job: queue_service.ContinuationQueueJob, - ) -> bool: - session_factory = get_session_factory() - async with session_factory() as session: - queue_item = await queue_service.mark_retrying(session, job_id=job.id) - await session.commit() - if queue_item is None: - return False - if queue_item.status == QueueItemStatus.DROPPED.value: - return False - return True - - async def _queue_job_has_available_scholar( - self, - job: queue_service.ContinuationQueueJob, - ) -> bool: - session_factory = get_session_factory() - async with session_factory() as session: - scholar_result = await session.execute( - select(ScholarProfile.id).where( - ScholarProfile.user_id == job.user_id, - ScholarProfile.id == job.scholar_profile_id, - ScholarProfile.is_enabled.is_(True), - ) - ) - scholar_id = scholar_result.scalar_one_or_none() - if scholar_id is not None: - return True - dropped = await queue_service.mark_dropped( - session, - job_id=job.id, - reason="scholar_unavailable", - ) - await session.commit() - if dropped is not None: - logger.info( - "scheduler.queue_item_dropped_scholar_unavailable", - extra={ - "event": "scheduler.queue_item_dropped_scholar_unavailable", - "queue_item_id": job.id, - "user_id": job.user_id, - "scholar_profile_id": job.scholar_profile_id, - }, - ) - return False - - async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None: - session_factory = get_session_factory() - async with session_factory() as recovery_session: - await queue_service.reschedule_job( - recovery_session, - job_id=job.id, - delay_seconds=max(self._tick_seconds, 15), - reason="user_run_lock_active", - error="run_already_in_progress", - ) - await recovery_session.commit() - logger.info( - "scheduler.queue_item_deferred_lock", - extra={"event": "scheduler.queue_item_deferred_lock", "queue_item_id": job.id, "user_id": job.user_id}, - ) - - async def _reschedule_queue_job_safety_cooldown( - self, - job: queue_service.ContinuationQueueJob, - exc: RunBlockedBySafetyPolicyError, - ) -> None: - cooldown_remaining_seconds = max( - self._tick_seconds, - int(exc.safety_state.get("cooldown_remaining_seconds") or 0), - ) - session_factory = get_session_factory() - async with session_factory() as recovery_session: - await queue_service.reschedule_job( - recovery_session, - job_id=job.id, - delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds), - reason="scrape_safety_cooldown", - error=str(exc.message), - ) - await recovery_session.commit() - logger.info( - "scheduler.queue_item_deferred_safety_cooldown", - extra={ - "event": "scheduler.queue_item_deferred_safety_cooldown", - "queue_item_id": job.id, - "user_id": job.user_id, - "reason": exc.safety_state.get("cooldown_reason"), - "cooldown_remaining_seconds": cooldown_remaining_seconds, - "metric_name": "scheduler_queue_item_deferred_safety_cooldown_total", - "metric_value": 1, - }, - ) - - async def _reschedule_queue_job_after_exception( - self, - job: queue_service.ContinuationQueueJob, - *, - exc: Exception, - ) -> None: - session_factory = get_session_factory() - async with session_factory() as recovery_session: - queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id) - if queue_item is None: - await recovery_session.commit() - return - if int(queue_item.attempt_count) >= self._continuation_max_attempts: - await queue_service.mark_dropped( - recovery_session, - job_id=job.id, - reason="scheduler_exception_max_attempts", - error=str(exc), - ) - await recovery_session.commit() - logger.warning( - "scheduler.queue_item_dropped_after_exception", - extra={"event": "scheduler.queue_item_dropped_after_exception", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count}, - ) - return - delay_seconds = queue_service.compute_backoff_seconds( - base_seconds=self._continuation_base_delay_seconds, - attempt_count=int(queue_item.attempt_count), - max_seconds=self._continuation_max_delay_seconds, - ) - await queue_service.reschedule_job( - recovery_session, - job_id=job.id, - delay_seconds=delay_seconds, - reason="scheduler_exception", - error=str(exc), - ) - await recovery_session.commit() - logger.exception("scheduler.queue_item_run_failed", extra={"event": "scheduler.queue_item_run_failed", "queue_item_id": job.id, "user_id": job.user_id}) - - async def _run_ingestion_for_queue_job( - self, - job: queue_service.ContinuationQueueJob, - ): - session_factory = get_session_factory() - async with session_factory() as session: - request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id) - ingestion = ScholarIngestionService(source=self._source) - try: - return await ingestion.run_for_user( - session, - user_id=job.user_id, - trigger_type=RunTriggerType.SCHEDULED, - request_delay_seconds=request_delay_seconds, - network_error_retries=self._network_error_retries, - retry_backoff_seconds=self._retry_backoff_seconds, - rate_limit_retries=settings.ingestion_rate_limit_retries, - rate_limit_backoff_seconds=settings.ingestion_rate_limit_backoff_seconds, - max_pages_per_scholar=self._max_pages_per_scholar, - page_size=self._page_size, - scholar_profile_ids={job.scholar_profile_id}, - start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart}, - auto_queue_continuations=self._continuation_queue_enabled, - queue_delay_seconds=self._continuation_base_delay_seconds, - alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, - alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, - alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, - ) - except RunAlreadyInProgressError: - await session.rollback() - await self._reschedule_queue_job_lock_active(job) - except RunBlockedBySafetyPolicyError as exc: - await session.rollback() - await self._reschedule_queue_job_safety_cooldown(job, exc) - except Exception as exc: - await session.rollback() - await self._reschedule_queue_job_after_exception(job, exc=exc) - return None - - @staticmethod - def _log_queue_item_resolved( - *, - event_name: str, - job: queue_service.ContinuationQueueJob, - run_summary, - attempt_count: int | None = None, - delay_seconds: int | None = None, - ) -> None: - payload = { - "event": event_name, - "queue_item_id": job.id, - "user_id": job.user_id, - "run_id": run_summary.crawl_run_id, - "status": run_summary.status.value, - } - if attempt_count is not None: - payload["attempt_count"] = attempt_count - if delay_seconds is not None: - payload["delay_seconds"] = delay_seconds - logger.info(event_name, extra=payload) - - async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None: - session_factory = get_session_factory() - async with session_factory() as session: - if int(run_summary.failed_count) <= 0: - queue_item = await queue_service.reset_attempt_count(session, job_id=job.id) - await session.commit() - if queue_item is None: - self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary) - return - self._log_queue_item_resolved( - event_name="scheduler.queue_item_progressed", - job=job, - run_summary=run_summary, - attempt_count=int(queue_item.attempt_count), - ) - return - queue_item = await queue_service.increment_attempt_count(session, job_id=job.id) - if queue_item is None: - await session.commit() - self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary) - return - if int(queue_item.attempt_count) >= self._continuation_max_attempts: - await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run") - await session.commit() - logger.warning( - "scheduler.queue_item_dropped_max_attempts_after_run", - extra={"event": "scheduler.queue_item_dropped_max_attempts_after_run", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count, "run_id": run_summary.crawl_run_id, "status": run_summary.status.value}, - ) - return - delay_seconds = queue_service.compute_backoff_seconds(base_seconds=self._continuation_base_delay_seconds, attempt_count=int(queue_item.attempt_count), max_seconds=self._continuation_max_delay_seconds) - await queue_service.reschedule_job(session, job_id=job.id, delay_seconds=delay_seconds, reason=queue_item.reason, error=queue_item.last_error) - await session.commit() - self._log_queue_item_resolved( - event_name="scheduler.queue_item_rescheduled_failed", - job=job, - run_summary=run_summary, - attempt_count=int(queue_item.attempt_count), - delay_seconds=delay_seconds, - ) - - async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None: - if await self._drop_queue_job_if_max_attempts(job): - return - if not await self._mark_queue_job_retrying(job): - return - if not await self._queue_job_has_available_scholar(job): - return - run_summary = await self._run_ingestion_for_queue_job(job) - if run_summary is None: - return - await self._finalize_queue_job_after_run(job, run_summary) - - async def _drain_pdf_queue(self) -> None: - from app.services.domains.publications.pdf_queue import drain_ready_jobs - - session_factory = get_session_factory() - async with session_factory() as session: - try: - processed = await drain_ready_jobs( - session, - limit=settings.scheduler_pdf_queue_batch_size, - max_attempts=settings.pdf_auto_retry_max_attempts, - ) - if processed > 0: - logger.info("scheduler.pdf_queue_drain_completed", extra={ - "event": "scheduler.pdf_queue_drain_completed", - "processed_count": processed, - }) - except Exception: - logger.exception("scheduler.pdf_queue_drain_failed", extra={ - "event": "scheduler.pdf_queue_drain_failed", - }) - - async def _load_request_delay_for_user( - self, - db_session: AsyncSession, - *, - user_id: int, - ) -> int: - result = await db_session.execute( - select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id) - ) - delay = result.scalar_one_or_none() - return _effective_request_delay_seconds(delay) diff --git a/build/lib/app/services/domains/ingestion/types.py b/build/lib/app/services/domains/ingestion/types.py deleted file mode 100644 index 09d87f1..0000000 --- a/build/lib/app/services/domains/ingestion/types.py +++ /dev/null @@ -1,110 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any - -from app.db.models import RunStatus -from app.services.domains.scholar.parser import ParsedProfilePage, PublicationCandidate -from app.services.domains.scholar.source import FetchResult - - -@dataclass(frozen=True) -class RunExecutionSummary: - crawl_run_id: int - status: RunStatus - scholar_count: int - succeeded_count: int - failed_count: int - partial_count: int - new_publication_count: int - - -@dataclass(frozen=True) -class PagedParseResult: - fetch_result: FetchResult - parsed_page: ParsedProfilePage - first_page_fetch_result: FetchResult - first_page_parsed_page: ParsedProfilePage - first_page_fingerprint_sha256: str | None - publications: list[PublicationCandidate] - attempt_log: list[dict[str, Any]] - page_logs: list[dict[str, Any]] - pages_fetched: int - pages_attempted: int - has_more_remaining: bool - pagination_truncated_reason: str | None - continuation_cstart: int | None - skipped_no_change: bool - discovered_publication_count: int - - -@dataclass -class RunProgress: - succeeded_count: int = 0 - failed_count: int = 0 - partial_count: int = 0 - scholar_results: list[dict[str, Any]] = field(default_factory=list) - - -@dataclass(frozen=True) -class ScholarProcessingOutcome: - result_entry: dict[str, Any] - succeeded_count_delta: int - failed_count_delta: int - partial_count_delta: int - discovered_publication_count: int - - -@dataclass(frozen=True) -class RunFailureSummary: - failed_state_counts: dict[str, int] - failed_reason_counts: dict[str, int] - scrape_failure_counts: dict[str, int] - retries_scheduled_count: int - scholars_with_retries_count: int - retry_exhausted_count: int - - -@dataclass(frozen=True) -class RunAlertSummary: - blocked_failure_count: int - network_failure_count: int - blocked_failure_threshold: int - network_failure_threshold: int - retry_scheduled_threshold: int - alert_flags: dict[str, bool] - - -@dataclass -class PagedLoopState: - fetch_result: FetchResult - parsed_page: ParsedProfilePage - attempt_log: list[dict[str, Any]] - page_logs: list[dict[str, Any]] - publications: list[PublicationCandidate] - pages_fetched: int - pages_attempted: int - current_cstart: int - next_cstart: int - has_more_remaining: bool = False - pagination_truncated_reason: str | None = None - continuation_cstart: int | None = None - discovered_publication_count: int = 0 - - -class RunAlreadyInProgressError(RuntimeError): - """Raised when a run lock for a user is already held by another process.""" - - -class RunBlockedBySafetyPolicyError(RuntimeError): - def __init__( - self, - *, - code: str, - message: str, - safety_state: dict[str, Any], - ) -> None: - super().__init__(message) - self.code = code - self.message = message - self.safety_state = safety_state diff --git a/build/lib/app/services/domains/openalex/__init__.py b/build/lib/app/services/domains/openalex/__init__.py deleted file mode 100644 index a51555a..0000000 --- a/build/lib/app/services/domains/openalex/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import annotations - -import logging - -logger = logging.getLogger(__name__) diff --git a/build/lib/app/services/domains/openalex/client.py b/build/lib/app/services/domains/openalex/client.py deleted file mode 100644 index c62d876..0000000 --- a/build/lib/app/services/domains/openalex/client.py +++ /dev/null @@ -1,135 +0,0 @@ -import asyncio -import logging -from typing import Any, Mapping - -import httpx -from tenacity import ( - retry, - retry_if_exception_type, - stop_after_attempt, - wait_exponential, -) - -from app.services.domains.openalex.types import OpenAlexWork - -logger = logging.getLogger(__name__) - -OPENALEX_BASE_URL = "https://api.openalex.org" - - -class OpenAlexClientError(Exception): - pass - - -class OpenAlexRateLimitError(OpenAlexClientError): - pass - - -class OpenAlexClient: - def __init__( - self, - api_key: str | None = None, - mailto: str | None = None, - timeout: float = 10.0, - ) -> None: - self.api_key = api_key - self.mailto = mailto - self.timeout = timeout - - @property - def _base_params(self) -> dict[str, str]: - params = {} - if self.mailto: - params["mailto"] = self.mailto - if self.api_key: - params["api_key"] = self.api_key - return params - - @retry( - retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException, OpenAlexRateLimitError)), - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=2, max=10), - reraise=True, - ) - async def get_work_by_doi(self, doi: str) -> OpenAlexWork | None: - """Fetch a single work by DOI directly.""" - clean_doi = doi.replace("https://doi.org/", "") - if not clean_doi: - return None - - url = f"{OPENALEX_BASE_URL}/works/{clean_doi}" - - headers = {} - if self.mailto: - headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})" - else: - headers["User-Agent"] = "scholar-scraper/1.0" - - async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client: - response = await client.get(url, params=self._base_params) - - if response.status_code == 404: - return None - if response.status_code == 429: - raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex work by DOI") - if response.status_code >= 400: - logger.warning("OpenAlex API error: %s %s", response.status_code, response.text) - raise OpenAlexClientError(f"API Error {response.status_code}") - - data = response.json() - return OpenAlexWork.from_api_dict(data) - - @retry( - retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException, OpenAlexRateLimitError)), - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=2, max=10), - reraise=True, - ) - async def get_works_by_filter( - self, - filters: dict[str, str], - limit: int = 50, - ) -> list[OpenAlexWork]: - """ - Fetch works using the ?filter= query parameter. - Supports fetching multiple records by joining filters with | (OR logic). - """ - if not filters: - return [] - - # Example: {"doi": "10.foo|10.bar", "title.search": "query"} - filter_str = ",".join(f"{k}:{v}" for k, v in filters.items()) - - params = self._base_params.copy() - params["filter"] = filter_str - params["per-page"] = str(limit) - - url = f"{OPENALEX_BASE_URL}/works" - - headers = {} - if self.mailto: - headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})" - else: - headers["User-Agent"] = "scholar-scraper/1.0" - - async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client: - response = await client.get(url, params=params) - - if response.status_code == 429: - raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex works list") - if response.status_code >= 400: - logger.warning("OpenAlex API error (filters=%s): %s %s", filters, response.status_code, response.text) - raise OpenAlexClientError(f"API Error {response.status_code}") - - data = response.json() - results = data.get("results") or [] - - parsed_works = [] - for raw_work in results: - try: - parsed_works.append(OpenAlexWork.from_api_dict(raw_work)) - except Exception as e: - logger.warning("Failed to parse OpenAlex raw dict: %s", e) - continue - - return parsed_works diff --git a/build/lib/app/services/domains/openalex/matching.py b/build/lib/app/services/domains/openalex/matching.py deleted file mode 100644 index cfa511c..0000000 --- a/build/lib/app/services/domains/openalex/matching.py +++ /dev/null @@ -1,108 +0,0 @@ -import logging -import re - -from rapidfuzz import fuzz - -from app.services.domains.openalex.types import OpenAlexWork - -logger = logging.getLogger(__name__) - -# A minimum similarity score out of 100 for a title to be considered a match candidate. -TITLE_MATCH_THRESHOLD = 90.0 -# The margin within the top score where a secondary tiebreaker (author/year) is necessary. -TIEBREAKER_MARGIN = 5.0 - - -def _clean_string(s: str | None) -> str: - if not s: - return "" - # Strip non-alphanumeric (keep spaces), lowercase, and collapse whitespace - cleaned = re.sub(r"[^a-z0-9\s]", " ", s.lower()) - return " ".join(cleaned.split()) - - -def _author_overlap_score(target_authors: str | None, candidate_authors: list[str]) -> bool: - if not target_authors or not candidate_authors: - return False - - target_clean = _clean_string(target_authors) - if not target_clean: - return False - - for candidate in candidate_authors: - cand_clean = _clean_string(candidate) - if cand_clean and (cand_clean in target_clean or target_clean in cand_clean): - return True - # Alternatively check rapidfuzz token_set_ratio - if cand_clean and fuzz.token_set_ratio(target_clean, cand_clean) > 80: - return True - - return False - - -def find_best_match( - target_title: str, - target_year: int | None, - target_authors: str | None, - candidates: list[OpenAlexWork], -) -> OpenAlexWork | None: - """ - Finds the best matching OpenAlexWork from a list of candidates, prioritizing title similarity (>90%) - with year and author overlap as tiebreakers for close candidates. - """ - if not target_title or not candidates: - return None - - clean_target = _clean_string(target_title) - if not clean_target: - return None - - scored_candidates: list[tuple[float, OpenAlexWork]] = [] - - for cand in candidates: - if not cand.title: - continue - - clean_cand = _clean_string(cand.title) - - # Primary sort: string similarity ratio - score = fuzz.ratio(clean_target, clean_cand) - - if score >= TITLE_MATCH_THRESHOLD: - scored_candidates.append((score, cand)) - - if not scored_candidates: - return None - - # Sort descending by score - scored_candidates.sort(key=lambda x: x[0], reverse=True) - - best_score = scored_candidates[0][0] - - # Extract all candidates within the tiebreaker margin - top_scored_candidates = [ - (score, cand) for score, cand in scored_candidates - if best_score - score <= TIEBREAKER_MARGIN - ] - - if len(top_scored_candidates) == 1: - return top_scored_candidates[0][1] - - # We have a tie or near-tie. Use year and author overlap to break the tie. - # Score candidates: +1 for year match (within 1 year), +1 for author overlap - tiebreaker_scores: list[tuple[int, float, OpenAlexWork]] = [] - - for original_score, cand in top_scored_candidates: - tb_score = 0 - if target_year is not None and cand.publication_year is not None: - if abs(target_year - cand.publication_year) <= 1: - tb_score += 1 - - candidate_author_names = [a.display_name for a in cand.authors if a.display_name] - if _author_overlap_score(target_authors, candidate_author_names): - tb_score += 1 - - tiebreaker_scores.append((tb_score, original_score, cand)) - - tiebreaker_scores.sort(key=lambda x: (x[0], x[1]), reverse=True) - return tiebreaker_scores[0][2] diff --git a/build/lib/app/services/domains/openalex/types.py b/build/lib/app/services/domains/openalex/types.py deleted file mode 100644 index 0d86754..0000000 --- a/build/lib/app/services/domains/openalex/types.py +++ /dev/null @@ -1,71 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass, field -from datetime import datetime -from typing import Any, Mapping - - -@dataclass(frozen=True) -class OpenAlexAuthor: - openalex_id: str | None - display_name: str | None - - -@dataclass(frozen=True) -class OpenAlexWork: - openalex_id: str - doi: str | None - pmid: str | None - pmcid: str | None - title: str | None - publication_year: int | None - cited_by_count: int - is_oa: bool - oa_url: str | None - authors: list[OpenAlexAuthor] = field(default_factory=list) - raw_data: Mapping[str, Any] = field(default_factory=dict, repr=False) - - @classmethod - def from_api_dict(cls, data: Mapping[str, Any]) -> OpenAlexWork: - ids = data.get("ids") or {} - - # Extract DOI without the https://doi.org/ prefix - doi = ids.get("doi") - if doi and doi.startswith("https://doi.org/"): - doi = doi[16:] - - # Extract PMID without the url prefix - pmid = ids.get("pmid") - if pmid and pmid.startswith("https://pubmed.ncbi.nlm.nih.gov/"): - pmid = pmid[32:] - - # Extract PMCID without the url prefix - pmcid = ids.get("pmcid") - if pmcid and pmcid.startswith("https://www.ncbi.nlm.nih.gov/pmc/articles/"): - pmcid = pmcid[42:] - - open_access = data.get("open_access") or {} - - authors = [] - for authorship in data.get("authorships") or []: - author_data = authorship.get("author") or {} - authors.append( - OpenAlexAuthor( - openalex_id=author_data.get("id"), - display_name=author_data.get("display_name"), - ) - ) - - return cls( - openalex_id=data.get("id", ""), - doi=doi, - pmid=pmid, - pmcid=pmcid, - title=data.get("title"), - publication_year=data.get("publication_year"), - cited_by_count=data.get("cited_by_count", 0), - is_oa=bool(open_access.get("is_oa")), - oa_url=open_access.get("oa_url"), - authors=authors, - raw_data=dict(data), - ) diff --git a/build/lib/app/services/domains/portability/__init__.py b/build/lib/app/services/domains/portability/__init__.py deleted file mode 100644 index 28aae8b..0000000 --- a/build/lib/app/services/domains/portability/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from app.services.domains.portability.application import * diff --git a/build/lib/app/services/domains/portability/application.py b/build/lib/app/services/domains/portability/application.py deleted file mode 100644 index 7e0b56e..0000000 --- a/build/lib/app/services/domains/portability/application.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import Publication -from app.services.domains.portability.constants import ( - EXPORT_SCHEMA_VERSION, - MAX_IMPORT_PUBLICATIONS, - MAX_IMPORT_SCHOLARS, -) -from app.services.domains.portability.exporting import export_user_data -from app.services.domains.portability.normalize import _validate_import_sizes -from app.services.domains.portability.publication_import import ( - _build_imported_publication_input, - _initialize_import_counters, - _upsert_imported_publication, -) -from app.services.domains.portability.scholar_import import _upsert_imported_scholars -from app.services.domains.portability.types import ImportExportError, ImportedPublicationInput - - -async def import_user_data( - db_session: AsyncSession, - *, - user_id: int, - scholars: list[dict[str, Any]], - publications: list[dict[str, Any]], -) -> dict[str, int]: - _validate_import_sizes(scholars=scholars, publications=publications) - scholar_map, counters = await _upsert_imported_scholars( - db_session, - user_id=user_id, - scholars=scholars, - ) - cluster_cache: dict[str, Publication | None] = {} - fingerprint_cache: dict[str, Publication | None] = {} - _initialize_import_counters(counters) - for item in publications: - parsed_item = _build_imported_publication_input( - item=item, - scholar_map=scholar_map, - ) - if parsed_item is None: - counters["skipped_records"] += 1 - continue - await _upsert_imported_publication( - db_session, - payload=parsed_item, - cluster_cache=cluster_cache, - fingerprint_cache=fingerprint_cache, - counters=counters, - ) - await db_session.commit() - return counters - - -__all__ = [ - "EXPORT_SCHEMA_VERSION", - "MAX_IMPORT_SCHOLARS", - "MAX_IMPORT_PUBLICATIONS", - "ImportExportError", - "ImportedPublicationInput", - "export_user_data", - "import_user_data", -] diff --git a/build/lib/app/services/domains/portability/constants.py b/build/lib/app/services/domains/portability/constants.py deleted file mode 100644 index 6815347..0000000 --- a/build/lib/app/services/domains/portability/constants.py +++ /dev/null @@ -1,9 +0,0 @@ -from __future__ import annotations - -import re - -EXPORT_SCHEMA_VERSION = 1 -MAX_IMPORT_SCHOLARS = 10_000 -MAX_IMPORT_PUBLICATIONS = 100_000 -WORD_RE = re.compile(r"[a-z0-9]+") -SHA256_RE = re.compile(r"^[0-9a-f]{64}$") diff --git a/build/lib/app/services/domains/portability/exporting.py b/build/lib/app/services/domains/portability/exporting.py deleted file mode 100644 index 5c89006..0000000 --- a/build/lib/app/services/domains/portability/exporting.py +++ /dev/null @@ -1,91 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timezone -from typing import Any - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import Publication, ScholarProfile, ScholarPublication -from app.services.domains.portability.constants import EXPORT_SCHEMA_VERSION - - -def _exported_at_iso() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat() - - -def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]: - return { - "scholar_id": profile.scholar_id, - "display_name": profile.display_name, - "is_enabled": bool(profile.is_enabled), - "profile_image_override_url": profile.profile_image_override_url, - } - - -def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]: - ( - scholar_id, - cluster_id, - fingerprint_sha256, - title_raw, - year, - citation_count, - author_text, - venue_text, - pub_url, - pdf_url, - is_read, - ) = row - return { - "scholar_id": scholar_id, - "cluster_id": cluster_id, - "fingerprint_sha256": fingerprint_sha256, - "title": title_raw, - "year": year, - "citation_count": int(citation_count or 0), - "author_text": author_text, - "venue_text": venue_text, - "pub_url": pub_url, - "pdf_url": pdf_url, - "is_read": bool(is_read), - } - - -async def export_user_data( - db_session: AsyncSession, - *, - user_id: int, -) -> dict[str, Any]: - scholars_result = await db_session.execute( - select(ScholarProfile) - .where(ScholarProfile.user_id == user_id) - .order_by(ScholarProfile.id.asc()) - ) - publication_result = await db_session.execute( - select( - ScholarProfile.scholar_id, - Publication.cluster_id, - Publication.fingerprint_sha256, - Publication.title_raw, - Publication.year, - Publication.citation_count, - Publication.author_text, - Publication.venue_text, - Publication.pub_url, - Publication.pdf_url, - ScholarPublication.is_read, - ) - .join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id) - .join(Publication, Publication.id == ScholarPublication.publication_id) - .where(ScholarProfile.user_id == user_id) - .order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) - ) - scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()] - publications = [_serialize_export_publication(row) for row in publication_result.all()] - return { - "schema_version": EXPORT_SCHEMA_VERSION, - "exported_at": _exported_at_iso(), - "scholars": scholars, - "publications": publications, - } diff --git a/build/lib/app/services/domains/portability/normalize.py b/build/lib/app/services/domains/portability/normalize.py deleted file mode 100644 index 60dd619..0000000 --- a/build/lib/app/services/domains/portability/normalize.py +++ /dev/null @@ -1,109 +0,0 @@ -from __future__ import annotations - -import hashlib -from typing import Any - -from app.services.domains.ingestion.application import normalize_title -from app.services.domains.portability.constants import ( - MAX_IMPORT_PUBLICATIONS, - MAX_IMPORT_SCHOLARS, - SHA256_RE, - WORD_RE, -) -from app.services.domains.portability.types import ImportExportError - - -def _normalize_optional_text(value: Any) -> str | None: - if value is None: - return None - normalized = str(value).strip() - return normalized or None - - -def _normalize_optional_year(value: Any) -> int | None: - if value is None: - return None - try: - year = int(value) - except (TypeError, ValueError): - return None - if year < 1500 or year > 3000: - return None - return year - - -def _normalize_citation_count(value: Any) -> int: - try: - parsed = int(value) - except (TypeError, ValueError): - return 0 - return max(0, parsed) - - -def _first_author_last_name(authors_text: str | None) -> str: - if not authors_text: - return "" - first_author = authors_text.split(",", maxsplit=1)[0].strip().lower() - words = WORD_RE.findall(first_author) - if not words: - return "" - return words[-1] - - -def _first_venue_word(venue_text: str | None) -> str: - if not venue_text: - return "" - words = WORD_RE.findall(venue_text.lower()) - if not words: - return "" - return words[0] - - -def _build_fingerprint( - *, - title: str, - year: int | None, - author_text: str | None, - venue_text: str | None, -) -> str: - canonical = "|".join( - [ - normalize_title(title), - str(year) if year is not None else "", - _first_author_last_name(author_text), - _first_venue_word(venue_text), - ] - ) - return hashlib.sha256(canonical.encode("utf-8")).hexdigest() - - -def _resolve_fingerprint( - *, - title: str, - year: int | None, - author_text: str | None, - venue_text: str | None, - provided_fingerprint: Any, -) -> str: - normalized = _normalize_optional_text(provided_fingerprint) - if normalized and SHA256_RE.fullmatch(normalized.lower()): - return normalized.lower() - return _build_fingerprint( - title=title, - year=year, - author_text=author_text, - venue_text=venue_text, - ) - - -def _validate_import_sizes( - *, - scholars: list[dict[str, Any]], - publications: list[dict[str, Any]], -) -> None: - if len(scholars) > MAX_IMPORT_SCHOLARS: - raise ImportExportError(f"Import exceeds max scholars ({MAX_IMPORT_SCHOLARS}).") - if len(publications) > MAX_IMPORT_PUBLICATIONS: - raise ImportExportError( - f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS})." - ) diff --git a/build/lib/app/services/domains/portability/publication_import.py b/build/lib/app/services/domains/portability/publication_import.py deleted file mode 100644 index 7654586..0000000 --- a/build/lib/app/services/domains/portability/publication_import.py +++ /dev/null @@ -1,370 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import Publication, ScholarProfile, ScholarPublication -from app.services.domains.ingestion.application import build_publication_url, normalize_title -from app.services.domains.doi.normalize import normalize_doi -from app.services.domains.portability.normalize import ( - _normalize_citation_count, - _normalize_optional_text, - _normalize_optional_year, - _resolve_fingerprint, -) -from app.services.domains.portability.types import ImportedPublicationInput - - -async def _find_publication_by_cluster( - db_session: AsyncSession, - *, - cluster_id: str, -) -> Publication | None: - result = await db_session.execute( - select(Publication).where(Publication.cluster_id == cluster_id) - ) - return result.scalar_one_or_none() - - -async def _find_publication_by_fingerprint( - db_session: AsyncSession, - *, - fingerprint_sha256: str, -) -> Publication | None: - result = await db_session.execute( - select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256) - ) - return result.scalar_one_or_none() - - -async def _find_linked_publication_by_title( - db_session: AsyncSession, - *, - scholar_profile_id: int, - title: str, -) -> Publication | None: - normalized_title = normalize_title(title) - result = await db_session.execute( - select(Publication) - .join( - ScholarPublication, - ScholarPublication.publication_id == Publication.id, - ) - .where( - ScholarPublication.scholar_profile_id == scholar_profile_id, - Publication.title_normalized == normalized_title, - ) - .order_by(Publication.id.asc()) - .limit(1) - ) - return result.scalar_one_or_none() - - -def _apply_imported_publication_values( - *, - publication: Publication, - title: str, - year: int | None, - citation_count: int, - author_text: str | None, - venue_text: str | None, - pub_url: str | None, - pdf_url: str | None, - cluster_id: str | None, -) -> bool: - updated = False - if cluster_id and publication.cluster_id != cluster_id: - publication.cluster_id = cluster_id - updated = True - if publication.title_raw != title: - publication.title_raw = title - publication.title_normalized = normalize_title(title) - updated = True - if publication.year != year: - publication.year = year - updated = True - if int(publication.citation_count or 0) != citation_count: - publication.citation_count = citation_count - updated = True - if publication.author_text != author_text: - publication.author_text = author_text - updated = True - if publication.venue_text != venue_text: - publication.venue_text = venue_text - updated = True - if pub_url and publication.pub_url != pub_url: - publication.pub_url = pub_url - updated = True - if pdf_url and publication.pdf_url != pdf_url: - publication.pdf_url = pdf_url - updated = True - return updated - - -def _new_publication( - *, - cluster_id: str | None, - fingerprint_sha256: str, - title: str, - year: int | None, - citation_count: int, - author_text: str | None, - venue_text: str | None, - pub_url: str | None, - pdf_url: str | None, -) -> Publication: - return Publication( - cluster_id=cluster_id, - fingerprint_sha256=fingerprint_sha256, - title_raw=title, - title_normalized=normalize_title(title), - year=year, - citation_count=citation_count, - author_text=author_text, - venue_text=venue_text, - pub_url=pub_url, - pdf_url=pdf_url, - ) - - -async def _resolve_publication_for_import( - db_session: AsyncSession, - *, - scholar_profile_id: int, - title: str, - cluster_id: str | None, - fingerprint_sha256: str, - cluster_cache: dict[str, Publication | None], - fingerprint_cache: dict[str, Publication | None], -) -> Publication | None: - if cluster_id: - if cluster_id not in cluster_cache: - cluster_cache[cluster_id] = await _find_publication_by_cluster( - db_session, - cluster_id=cluster_id, - ) - if cluster_cache[cluster_id] is not None: - return cluster_cache[cluster_id] - if fingerprint_sha256 not in fingerprint_cache: - fingerprint_cache[fingerprint_sha256] = await _find_publication_by_fingerprint( - db_session, - fingerprint_sha256=fingerprint_sha256, - ) - if fingerprint_cache[fingerprint_sha256] is not None: - return fingerprint_cache[fingerprint_sha256] - return await _find_linked_publication_by_title( - db_session, - scholar_profile_id=scholar_profile_id, - title=title, - ) - - -async def _upsert_scholar_publication_link( - db_session: AsyncSession, - *, - scholar_profile_id: int, - publication_id: int, - is_read: bool, -) -> tuple[bool, bool]: - result = await db_session.execute( - select(ScholarPublication).where( - ScholarPublication.scholar_profile_id == scholar_profile_id, - ScholarPublication.publication_id == publication_id, - ) - ) - link = result.scalar_one_or_none() - if link is None: - db_session.add( - ScholarPublication( - scholar_profile_id=scholar_profile_id, - publication_id=publication_id, - is_read=bool(is_read), - ) - ) - return True, False - if bool(link.is_read) == bool(is_read): - return False, False - link.is_read = bool(is_read) - return False, True - - -def _initialize_import_counters(counters: dict[str, int]) -> None: - counters.update( - { - "publications_created": 0, - "publications_updated": 0, - "links_created": 0, - "links_updated": 0, - } - ) - - -def _build_imported_publication_input( - *, - item: dict[str, Any], - scholar_map: dict[str, ScholarProfile], -) -> ImportedPublicationInput | None: - scholar_id = _normalize_optional_text(item.get("scholar_id")) - title = _normalize_optional_text(item.get("title")) - if not scholar_id or not title: - return None - - profile = scholar_map.get(scholar_id) - if profile is None: - return None - - year = _normalize_optional_year(item.get("year")) - author_text = _normalize_optional_text(item.get("author_text")) - venue_text = _normalize_optional_text(item.get("venue_text")) - return ImportedPublicationInput( - profile=profile, - title=title, - year=year, - citation_count=_normalize_citation_count(item.get("citation_count")), - author_text=author_text, - venue_text=venue_text, - cluster_id=_normalize_optional_text(item.get("cluster_id")), - pub_url=build_publication_url(_normalize_optional_text(item.get("pub_url"))), - doi=normalize_doi(_normalize_optional_text(item.get("doi"))), - pdf_url=build_publication_url(_normalize_optional_text(item.get("pdf_url"))), - fingerprint=_resolve_fingerprint( - title=title, - year=year, - author_text=author_text, - venue_text=venue_text, - provided_fingerprint=item.get("fingerprint_sha256"), - ), - is_read=bool(item.get("is_read", False)), - ) - - -def _update_link_counters( - *, - counters: dict[str, int], - link_created: bool, - link_updated: bool, -) -> None: - if link_created: - counters["links_created"] += 1 - if link_updated: - counters["links_updated"] += 1 - - -def _cache_resolved_publication( - *, - publication: Publication, - cluster_id: str | None, - fingerprint_sha256: str, - cluster_cache: dict[str, Publication | None], - fingerprint_cache: dict[str, Publication | None], -) -> None: - if cluster_id: - cluster_cache[cluster_id] = publication - fingerprint_cache[fingerprint_sha256] = publication - - -async def _create_import_publication( - db_session: AsyncSession, - *, - payload: ImportedPublicationInput, -) -> Publication: - publication = _new_publication( - cluster_id=payload.cluster_id, - fingerprint_sha256=payload.fingerprint, - title=payload.title, - year=payload.year, - citation_count=payload.citation_count, - author_text=payload.author_text, - venue_text=payload.venue_text, - pub_url=payload.pub_url, - pdf_url=payload.pdf_url, - ) - db_session.add(publication) - await db_session.flush() - return publication - - -def _update_import_publication( - *, - publication: Publication, - payload: ImportedPublicationInput, -) -> bool: - return _apply_imported_publication_values( - publication=publication, - title=payload.title, - year=payload.year, - citation_count=payload.citation_count, - author_text=payload.author_text, - venue_text=payload.venue_text, - pub_url=payload.pub_url, - pdf_url=payload.pdf_url, - cluster_id=payload.cluster_id, - ) - - -async def _upsert_publication_entity( - db_session: AsyncSession, - *, - payload: ImportedPublicationInput, - cluster_cache: dict[str, Publication | None], - fingerprint_cache: dict[str, Publication | None], -) -> tuple[Publication, bool, bool]: - publication = await _resolve_publication_for_import( - db_session, - scholar_profile_id=int(payload.profile.id), - title=payload.title, - cluster_id=payload.cluster_id, - fingerprint_sha256=payload.fingerprint, - cluster_cache=cluster_cache, - fingerprint_cache=fingerprint_cache, - ) - created = False - updated = False - if publication is None: - publication = await _create_import_publication(db_session, payload=payload) - created = True - else: - updated = _update_import_publication(publication=publication, payload=payload) - - _cache_resolved_publication( - publication=publication, - cluster_id=payload.cluster_id, - fingerprint_sha256=payload.fingerprint, - cluster_cache=cluster_cache, - fingerprint_cache=fingerprint_cache, - ) - return publication, created, updated - - -async def _upsert_imported_publication( - db_session: AsyncSession, - *, - payload: ImportedPublicationInput, - cluster_cache: dict[str, Publication | None], - fingerprint_cache: dict[str, Publication | None], - counters: dict[str, int], -) -> None: - publication, created, updated = await _upsert_publication_entity( - db_session, - payload=payload, - cluster_cache=cluster_cache, - fingerprint_cache=fingerprint_cache, - ) - if created: - counters["publications_created"] += 1 - if updated: - counters["publications_updated"] += 1 - - link_created, link_updated = await _upsert_scholar_publication_link( - db_session, - scholar_profile_id=int(payload.profile.id), - publication_id=int(publication.id), - is_read=payload.is_read, - ) - _update_link_counters( - counters=counters, - link_created=link_created, - link_updated=link_updated, - ) diff --git a/build/lib/app/services/domains/portability/scholar_import.py b/build/lib/app/services/domains/portability/scholar_import.py deleted file mode 100644 index 74e6ec1..0000000 --- a/build/lib/app/services/domains/portability/scholar_import.py +++ /dev/null @@ -1,107 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import ScholarProfile -from app.services.domains.portability.normalize import _normalize_optional_text -from app.services.domains.scholars import application as scholar_service - - -async def _load_user_scholar_map( - db_session: AsyncSession, - *, - user_id: int, -) -> dict[str, ScholarProfile]: - result = await db_session.execute( - select(ScholarProfile).where(ScholarProfile.user_id == user_id) - ) - profiles = list(result.scalars().all()) - return {profile.scholar_id: profile for profile in profiles} - - -def _apply_imported_scholar_values( - *, - profile: ScholarProfile, - display_name: str | None, - profile_image_override_url: str | None, - is_enabled: bool, -) -> bool: - updated = False - if display_name and profile.display_name != display_name: - profile.display_name = display_name - updated = True - if profile.profile_image_override_url != profile_image_override_url: - profile.profile_image_override_url = profile_image_override_url - updated = True - if bool(profile.is_enabled) != bool(is_enabled): - profile.is_enabled = bool(is_enabled) - updated = True - return updated - - -def _new_scholar_profile( - *, - user_id: int, - scholar_id: str, - display_name: str | None, - profile_image_override_url: str | None, - is_enabled: bool, -) -> ScholarProfile: - return ScholarProfile( - user_id=user_id, - scholar_id=scholar_id, - display_name=display_name, - profile_image_override_url=profile_image_override_url, - is_enabled=bool(is_enabled), - ) - - -async def _upsert_imported_scholars( - db_session: AsyncSession, - *, - user_id: int, - scholars: list[dict[str, Any]], -) -> tuple[dict[str, ScholarProfile], dict[str, int]]: - scholar_map = await _load_user_scholar_map(db_session, user_id=user_id) - counters = {"scholars_created": 0, "scholars_updated": 0, "skipped_records": 0} - for item in scholars: - try: - scholar_id = scholar_service.validate_scholar_id(str(item["scholar_id"])) - display_name = scholar_service.normalize_display_name( - str(item.get("display_name") or "") - ) - override_url = scholar_service.normalize_profile_image_url( - _normalize_optional_text(item.get("profile_image_override_url")) - ) - except (KeyError, scholar_service.ScholarServiceError): - counters["skipped_records"] += 1 - continue - - is_enabled = bool(item.get("is_enabled", True)) - existing = scholar_map.get(scholar_id) - if existing is None: - profile = _new_scholar_profile( - user_id=user_id, - scholar_id=scholar_id, - display_name=display_name, - profile_image_override_url=override_url, - is_enabled=is_enabled, - ) - db_session.add(profile) - scholar_map[scholar_id] = profile - counters["scholars_created"] += 1 - continue - - if _apply_imported_scholar_values( - profile=existing, - display_name=display_name, - profile_image_override_url=override_url, - is_enabled=is_enabled, - ): - counters["scholars_updated"] += 1 - - await db_session.flush() - return scholar_map, counters diff --git a/build/lib/app/services/domains/portability/types.py b/build/lib/app/services/domains/portability/types.py deleted file mode 100644 index 2a7f6e8..0000000 --- a/build/lib/app/services/domains/portability/types.py +++ /dev/null @@ -1,25 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from app.db.models import ScholarProfile - - -class ImportExportError(ValueError): - """Raised when import/export payload constraints are violated.""" - - -@dataclass(frozen=True) -class ImportedPublicationInput: - profile: ScholarProfile - title: str - year: int | None - citation_count: int - author_text: str | None - venue_text: str | None - cluster_id: str | None - pub_url: str | None - doi: str | None - pdf_url: str | None - fingerprint: str - is_read: bool diff --git a/build/lib/app/services/domains/publication_identifiers/__init__.py b/build/lib/app/services/domains/publication_identifiers/__init__.py deleted file mode 100644 index 3a35c66..0000000 --- a/build/lib/app/services/domains/publication_identifiers/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -from app.services.domains.publication_identifiers.application import ( - DisplayIdentifier, - derive_display_identifier_from_values, - overlay_pdf_queue_items_with_display_identifiers, - overlay_publication_items_with_display_identifiers, - sync_identifiers_for_publication_fields, - sync_identifiers_for_publication_resolution, -) - -__all__ = [ - "DisplayIdentifier", - "derive_display_identifier_from_values", - "overlay_pdf_queue_items_with_display_identifiers", - "overlay_publication_items_with_display_identifiers", - "sync_identifiers_for_publication_fields", - "sync_identifiers_for_publication_resolution", -] diff --git a/build/lib/app/services/domains/publication_identifiers/application.py b/build/lib/app/services/domains/publication_identifiers/application.py deleted file mode 100644 index ca57d57..0000000 --- a/build/lib/app/services/domains/publication_identifiers/application.py +++ /dev/null @@ -1,433 +0,0 @@ -from __future__ import annotations - -from dataclasses import replace -from typing import TYPE_CHECKING - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import Publication, PublicationIdentifier -from app.services.domains.doi.normalize import normalize_doi -from app.services.domains.publication_identifiers.normalize import ( - normalize_arxiv_id, - normalize_pmcid, - normalize_pmid, -) -from app.services.domains.publication_identifiers.types import ( - DisplayIdentifier, - IdentifierCandidate, - IdentifierKind, -) - -if TYPE_CHECKING: - from app.services.domains.publications.pdf_queue import PdfQueueListItem - from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem - -CONFIDENCE_HIGH = 0.98 -CONFIDENCE_MEDIUM = 0.9 -CONFIDENCE_LOW = 0.6 -CONFIDENCE_FALLBACK = 0.4 -PRIORITY_DOI = 400 -PRIORITY_ARXIV = 300 -PRIORITY_PMCID = 200 -PRIORITY_PMID = 100 - - -def derive_display_identifier_from_values( - *, - doi: str | None, - pub_url: str | None = None, - pdf_url: str | None = None, -) -> DisplayIdentifier | None: - candidates = _fallback_candidates_from_values(doi=doi, pub_url=pub_url, pdf_url=pdf_url) - return _best_display_identifier(candidates) - - -def _fallback_candidates_from_values( - *, - doi: str | None, - pub_url: str | None, - pdf_url: str | None, -) -> list[IdentifierCandidate]: - values = [value for value in [pub_url, pdf_url] if value] - candidates = [] - if doi: - normalized_doi = normalize_doi(doi) - if normalized_doi: - candidates.append(_candidate(IdentifierKind.DOI, doi, normalized_doi, "legacy_doi", CONFIDENCE_HIGH, pub_url)) - candidates.extend(_url_identifier_candidates(values=values, source="legacy_urls")) - return _dedup_candidates(candidates) - - -def _url_identifier_candidates(*, values: list[str], source: str) -> list[IdentifierCandidate]: - candidates: list[IdentifierCandidate] = [] - for value in values: - candidates.extend(_url_candidates_for_value(value=value, source=source)) - return candidates - - -def _url_candidates_for_value(*, value: str, source: str) -> list[IdentifierCandidate]: - candidates: list[IdentifierCandidate] = [] - arxiv = normalize_arxiv_id(value) - if arxiv: - candidates.append(_candidate(IdentifierKind.ARXIV, value, arxiv, source, CONFIDENCE_MEDIUM, value)) - pmcid = normalize_pmcid(value) - if pmcid: - candidates.append(_candidate(IdentifierKind.PMCID, value, pmcid, source, CONFIDENCE_LOW, value)) - pmid = normalize_pmid(value) - if pmid: - candidates.append(_candidate(IdentifierKind.PMID, value, pmid, source, CONFIDENCE_FALLBACK, value)) - return candidates - - -def _candidate( - kind: IdentifierKind, - value_raw: str, - value_normalized: str, - source: str, - confidence_score: float, - evidence_url: str | None, -) -> IdentifierCandidate: - return IdentifierCandidate( - kind=kind, - value_raw=value_raw, - value_normalized=value_normalized, - source=source, - confidence_score=float(confidence_score), - evidence_url=evidence_url, - ) - - -def _dedup_candidates(candidates: list[IdentifierCandidate]) -> list[IdentifierCandidate]: - deduped: dict[tuple[str, str], IdentifierCandidate] = {} - for candidate in candidates: - key = (candidate.kind.value, candidate.value_normalized) - current = deduped.get(key) - if current is None or candidate.confidence_score > current.confidence_score: - deduped[key] = candidate - return list(deduped.values()) - - -async def sync_identifiers_for_publication_fields( - db_session: AsyncSession, - *, - publication: Publication, -) -> None: - candidates = _publication_field_candidates(publication) - await _upsert_publication_candidates(db_session, publication_id=int(publication.id), candidates=candidates) - - -async def discover_and_sync_identifiers_for_publication( - db_session: AsyncSession, - *, - publication: Publication, - scholar_label: str, -) -> None: - await sync_identifiers_for_publication_fields(db_session, publication=publication) - - existing_doi = await _existing_identifier_by_kind( - db_session, - publication_id=int(publication.id), - kind=IdentifierKind.DOI.value, - ) - if existing_doi is not None: - return - - from app.services.domains.crossref import application as crossref_service - from app.services.domains.publications.types import UnreadPublicationItem - - item = UnreadPublicationItem( - publication_id=int(publication.id), - scholar_profile_id=0, - scholar_label=scholar_label, - title=str(publication.title_raw or ""), - year=publication.year, - citation_count=publication.citation_count, - venue_text=publication.venue_text, - pub_url=publication.pub_url, - pdf_url=publication.pdf_url, - ) - - discovered_doi = await crossref_service.discover_doi_for_publication(item=item) - if discovered_doi: - normalized_doi = normalize_doi(discovered_doi) - if normalized_doi: - candidate = _candidate( - IdentifierKind.DOI, - discovered_doi, - normalized_doi, - "crossref_api", - CONFIDENCE_MEDIUM, - None, - ) - await _upsert_publication_candidate(db_session, publication_id=int(publication.id), candidate=candidate) - - existing_arxiv = await _existing_identifier_by_kind( - db_session, - publication_id=int(publication.id), - kind=IdentifierKind.ARXIV.value, - ) - if existing_arxiv is None: - from app.services.domains.arxiv import application as arxiv_service - discovered_arxiv = await arxiv_service.discover_arxiv_id_for_publication(item=item) - if discovered_arxiv: - normalized_arxiv = normalize_arxiv_id(discovered_arxiv) - if normalized_arxiv: - candidate = _candidate( - IdentifierKind.ARXIV, - discovered_arxiv, - normalized_arxiv, - "arxiv_api", - CONFIDENCE_MEDIUM, - None, - ) - await _upsert_publication_candidate(db_session, publication_id=int(publication.id), candidate=candidate) - - -def _publication_field_candidates(publication: Publication) -> list[IdentifierCandidate]: - return _fallback_candidates_from_values( - doi=None, - pub_url=publication.pub_url, - pdf_url=publication.pdf_url, - ) - - -async def sync_identifiers_for_publication_resolution( - db_session: AsyncSession, - *, - publication: Publication, - source: str | None, -) -> None: - candidates = _publication_field_candidates(publication) - rewritten = [_candidate_with_source(candidate, source=source) for candidate in candidates] - await _upsert_publication_candidates(db_session, publication_id=int(publication.id), candidates=rewritten) - - -def _candidate_with_source(candidate: IdentifierCandidate, *, source: str | None) -> IdentifierCandidate: - if not source: - return candidate - return IdentifierCandidate( - kind=candidate.kind, - value_raw=candidate.value_raw, - value_normalized=candidate.value_normalized, - source=source, - confidence_score=candidate.confidence_score, - evidence_url=candidate.evidence_url, - ) - - -async def _upsert_publication_candidates( - db_session: AsyncSession, - *, - publication_id: int, - candidates: list[IdentifierCandidate], -) -> None: - for candidate in _dedup_candidates(candidates): - await _upsert_publication_candidate(db_session, publication_id=publication_id, candidate=candidate) - - -async def _upsert_publication_candidate( - db_session: AsyncSession, - *, - publication_id: int, - candidate: IdentifierCandidate, -) -> None: - existing = await _existing_identifier( - db_session, - publication_id=publication_id, - kind=candidate.kind.value, - value_normalized=candidate.value_normalized, - ) - if existing is None: - db_session.add(_new_identifier_row(publication_id=publication_id, candidate=candidate)) - return - _merge_identifier_row(existing, candidate=candidate) - - -async def _existing_identifier( - db_session: AsyncSession, - *, - publication_id: int, - kind: str, - value_normalized: str, -) -> PublicationIdentifier | None: - result = await db_session.execute( - select(PublicationIdentifier).where( - PublicationIdentifier.publication_id == publication_id, - PublicationIdentifier.kind == kind, - PublicationIdentifier.value_normalized == value_normalized, - ) - ) - return result.scalar_one_or_none() - - -async def _existing_identifier_by_kind( - db_session: AsyncSession, - *, - publication_id: int, - kind: str, -) -> PublicationIdentifier | None: - result = await db_session.execute( - select(PublicationIdentifier).where( - PublicationIdentifier.publication_id == publication_id, - PublicationIdentifier.kind == kind, - ).order_by(PublicationIdentifier.confidence_score.desc()).limit(1) - ) - return result.scalar_one_or_none() - - -def _new_identifier_row( - *, - publication_id: int, - candidate: IdentifierCandidate, -) -> PublicationIdentifier: - return PublicationIdentifier( - publication_id=publication_id, - kind=candidate.kind.value, - value_raw=candidate.value_raw, - value_normalized=candidate.value_normalized, - source=candidate.source, - confidence_score=candidate.confidence_score, - evidence_url=candidate.evidence_url, - ) - - -def _merge_identifier_row(existing: PublicationIdentifier, *, candidate: IdentifierCandidate) -> None: - if candidate.confidence_score >= float(existing.confidence_score): - existing.value_raw = candidate.value_raw - existing.source = candidate.source - existing.confidence_score = candidate.confidence_score - if candidate.evidence_url: - existing.evidence_url = candidate.evidence_url - - -async def overlay_publication_items_with_display_identifiers( - db_session: AsyncSession, - *, - items: list[PublicationListItem], -) -> list[PublicationListItem]: - if not items: - return [] - mapping = await _display_identifier_map(db_session, publication_ids=[item.publication_id for item in items]) - return [_overlay_publication_item(item, mapping.get(item.publication_id)) for item in items] - - -def _overlay_publication_item( - item: PublicationListItem, - display_identifier: DisplayIdentifier | None, -) -> PublicationListItem: - fallback = display_identifier or derive_display_identifier_from_values(doi=None, pub_url=item.pub_url, pdf_url=item.pdf_url) - return replace(item, display_identifier=fallback) - - -async def overlay_pdf_queue_items_with_display_identifiers( - db_session: AsyncSession, - *, - items: list[PdfQueueListItem], -) -> list[PdfQueueListItem]: - if not items: - return [] - mapping = await _display_identifier_map(db_session, publication_ids=[item.publication_id for item in items]) - return [_overlay_queue_item(item, mapping.get(item.publication_id)) for item in items] - - -def _overlay_queue_item( - item: PdfQueueListItem, - display_identifier: DisplayIdentifier | None, -) -> PdfQueueListItem: - fallback = display_identifier or derive_display_identifier_from_values(doi=None, pdf_url=item.pdf_url) - return replace(item, display_identifier=fallback) - - -async def _display_identifier_map( - db_session: AsyncSession, - *, - publication_ids: list[int], -) -> dict[int, DisplayIdentifier]: - normalized_ids = sorted({int(value) for value in publication_ids if int(value) > 0}) - if not normalized_ids: - return {} - result = await db_session.execute( - select(PublicationIdentifier).where(PublicationIdentifier.publication_id.in_(normalized_ids)) - ) - rows = list(result.scalars().all()) - return _best_display_identifier_map(rows) - - -def _best_display_identifier_map(rows: list[PublicationIdentifier]) -> dict[int, DisplayIdentifier]: - grouped: dict[int, list[IdentifierCandidate]] = {} - for row in rows: - grouped.setdefault(int(row.publication_id), []).append(_candidate_from_row(row)) - return { - publication_id: display - for publication_id, display in ( - (publication_id, _best_display_identifier(candidates)) - for publication_id, candidates in grouped.items() - ) - if display is not None - } - - -def _candidate_from_row(row: PublicationIdentifier) -> IdentifierCandidate: - return IdentifierCandidate( - kind=IdentifierKind(str(row.kind)), - value_raw=str(row.value_raw), - value_normalized=str(row.value_normalized), - source=str(row.source), - confidence_score=float(row.confidence_score), - evidence_url=row.evidence_url, - ) - - -def _best_display_identifier(candidates: list[IdentifierCandidate]) -> DisplayIdentifier | None: - if not candidates: - return None - ordered = sorted(candidates, key=_display_sort_key, reverse=True) - return _display_identifier_from_candidate(ordered[0]) - - -def _display_sort_key(candidate: IdentifierCandidate) -> tuple[int, float]: - return (_kind_priority(candidate.kind), float(candidate.confidence_score)) - - -def _kind_priority(kind: IdentifierKind) -> int: - if kind == IdentifierKind.DOI: - return PRIORITY_DOI - if kind == IdentifierKind.ARXIV: - return PRIORITY_ARXIV - if kind == IdentifierKind.PMCID: - return PRIORITY_PMCID - return PRIORITY_PMID - - -def _display_identifier_from_candidate(candidate: IdentifierCandidate) -> DisplayIdentifier: - value = candidate.value_normalized - return DisplayIdentifier( - kind=candidate.kind.value, - value=value, - label=_display_label(candidate.kind, value), - url=_identifier_url(candidate.kind, value), - confidence_score=float(candidate.confidence_score), - ) - - -def _display_label(kind: IdentifierKind, value: str) -> str: - if kind == IdentifierKind.DOI: - return f"DOI: {value}" - if kind == IdentifierKind.ARXIV: - return f"arXiv: {value}" - if kind == IdentifierKind.PMCID: - return f"PMCID: {value}" - return f"PMID: {value}" - - -def _identifier_url(kind: IdentifierKind, value: str) -> str | None: - if kind == IdentifierKind.DOI: - return f"https://doi.org/{value}" - if kind == IdentifierKind.ARXIV: - return f"https://arxiv.org/abs/{value}" - if kind == IdentifierKind.PMCID: - return f"https://pmc.ncbi.nlm.nih.gov/articles/{value}/" - if kind == IdentifierKind.PMID: - return f"https://pubmed.ncbi.nlm.nih.gov/{value}/" - return None diff --git a/build/lib/app/services/domains/publication_identifiers/normalize.py b/build/lib/app/services/domains/publication_identifiers/normalize.py deleted file mode 100644 index 9f7484a..0000000 --- a/build/lib/app/services/domains/publication_identifiers/normalize.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -import re -from urllib.parse import urlparse - -from app.services.domains.doi.normalize import normalize_doi -from app.services.domains.publication_identifiers.types import IdentifierKind - -ARXIV_ABS_RE = re.compile(r"\barxiv:\s*([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?\b", re.I) -ARXIV_PATH_RE = re.compile(r"^/(?:abs|pdf|html|ps|format)/([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?(?:\.pdf)?/?$", re.I) -PMCID_RE = re.compile(r"\b(PMC\d+)\b", re.I) -PUBMED_PATH_RE = re.compile(r"^/(\d+)/?$") - - -def normalize_identifier(kind: IdentifierKind, value: str | None) -> str | None: - if kind == IdentifierKind.DOI: - return normalize_doi(value) - if kind == IdentifierKind.ARXIV: - return normalize_arxiv_id(value) - if kind == IdentifierKind.PMCID: - return normalize_pmcid(value) - if kind == IdentifierKind.PMID: - return normalize_pmid(value) - return None - - -def normalize_arxiv_id(value: str | None) -> str | None: - text = (value or "").strip() - if not text: - return None - parsed = urlparse(text) - if parsed.scheme in {"http", "https"} and "arxiv.org" in parsed.netloc.lower(): - return _arxiv_from_path(parsed.path) - match = ARXIV_ABS_RE.search(text) - if not match: - return None - version = (match.group(2) or "").lower() - return f"{match.group(1).lower()}{version}" - - -def _arxiv_from_path(path: str) -> str | None: - match = ARXIV_PATH_RE.match(path or "") - if not match: - return None - version = (match.group(2) or "").lower() - return f"{match.group(1).lower()}{version}" - - -def normalize_pmcid(value: str | None) -> str | None: - text = (value or "").strip() - if not text: - return None - parsed = urlparse(text) - if parsed.scheme in {"http", "https"} and "ncbi.nlm.nih.gov" in parsed.netloc.lower(): - return _first_match(PMCID_RE, parsed.path) - return _first_match(PMCID_RE, text) - - -def normalize_pmid(value: str | None) -> str | None: - text = (value or "").strip() - if not text: - return None - parsed = urlparse(text) - if parsed.scheme in {"http", "https"} and "pubmed.ncbi.nlm.nih.gov" in parsed.netloc.lower(): - match = PUBMED_PATH_RE.match(parsed.path or "") - if not match: - return None - return match.group(1) - return None - - -def _first_match(pattern: re.Pattern[str], value: str) -> str | None: - match = pattern.search(value) - if not match: - return None - return match.group(1).upper() diff --git a/build/lib/app/services/domains/publication_identifiers/types.py b/build/lib/app/services/domains/publication_identifiers/types.py deleted file mode 100644 index 34b20d0..0000000 --- a/build/lib/app/services/domains/publication_identifiers/types.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from enum import StrEnum - - -class IdentifierKind(StrEnum): - DOI = "doi" - ARXIV = "arxiv" - PMCID = "pmcid" - PMID = "pmid" - - -@dataclass(frozen=True) -class DisplayIdentifier: - kind: str - value: str - label: str - url: str | None - confidence_score: float - - -@dataclass(frozen=True) -class IdentifierCandidate: - kind: IdentifierKind - value_raw: str - value_normalized: str - source: str - confidence_score: float - evidence_url: str | None diff --git a/build/lib/app/services/domains/publications/__init__.py b/build/lib/app/services/domains/publications/__init__.py deleted file mode 100644 index 371b02c..0000000 --- a/build/lib/app/services/domains/publications/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from app.services.domains.publications.application import * diff --git a/build/lib/app/services/domains/publications/application.py b/build/lib/app/services/domains/publications/application.py deleted file mode 100644 index b1161fb..0000000 --- a/build/lib/app/services/domains/publications/application.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -from app.services.domains.publications.counts import ( - count_for_user, - count_favorite_for_user, - count_latest_for_user, - count_unread_for_user, -) -from app.services.domains.publications.listing import ( - list_for_user, - retry_pdf_for_user, - list_unread_for_user, -) -from app.services.domains.publications.enrichment import ( - hydrate_pdf_enrichment_state, - schedule_missing_pdf_enrichment_for_user, - schedule_retry_pdf_enrichment_for_row, -) -from app.services.domains.publications.modes import ( - MODE_ALL, - MODE_LATEST, - MODE_NEW, - MODE_UNREAD, - resolve_publication_view_mode, -) -from app.services.domains.publications.queries import ( - get_latest_completed_run_id_for_user, - get_publication_item_for_user, - publications_query, -) -from app.services.domains.publications.read_state import ( - mark_all_unread_as_read_for_user, - mark_selected_as_read_for_user, - set_publication_favorite_for_user, -) -from app.services.domains.publications.pdf_queue import ( - count_pdf_queue_items, - enqueue_all_missing_pdf_jobs, - enqueue_retry_pdf_job_for_publication_id, - list_pdf_queue_page, - list_pdf_queue_items, -) -from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem - -__all__ = [ - "MODE_ALL", - "MODE_UNREAD", - "MODE_LATEST", - "MODE_NEW", - "PublicationListItem", - "UnreadPublicationItem", - "resolve_publication_view_mode", - "get_latest_completed_run_id_for_user", - "publications_query", - "get_publication_item_for_user", - "list_for_user", - "list_unread_for_user", - "retry_pdf_for_user", - "hydrate_pdf_enrichment_state", - "schedule_retry_pdf_enrichment_for_row", - "list_pdf_queue_items", - "list_pdf_queue_page", - "count_pdf_queue_items", - "enqueue_all_missing_pdf_jobs", - "enqueue_retry_pdf_job_for_publication_id", - "schedule_missing_pdf_enrichment_for_user", - "count_for_user", - "count_favorite_for_user", - "count_unread_for_user", - "count_latest_for_user", - "mark_all_unread_as_read_for_user", - "mark_selected_as_read_for_user", - "set_publication_favorite_for_user", -] diff --git a/build/lib/app/services/domains/publications/counts.py b/build/lib/app/services/domains/publications/counts.py deleted file mode 100644 index 8f65172..0000000 --- a/build/lib/app/services/domains/publications/counts.py +++ /dev/null @@ -1,90 +0,0 @@ -from __future__ import annotations - -from sqlalchemy import func, select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import ScholarProfile, ScholarPublication -from app.services.domains.publications.modes import ( - MODE_ALL, - MODE_LATEST, - MODE_UNREAD, - resolve_publication_view_mode, -) -from app.services.domains.publications.queries import get_latest_completed_run_id_for_user - - -async def count_for_user( - db_session: AsyncSession, - *, - user_id: int, - mode: str = MODE_ALL, - scholar_profile_id: int | None = None, - favorite_only: bool = False, -) -> int: - resolved_mode = resolve_publication_view_mode(mode) - latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) - stmt = ( - select(func.count()) - .select_from(ScholarPublication) - .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) - .where(ScholarProfile.user_id == user_id) - ) - if scholar_profile_id is not None: - stmt = stmt.where(ScholarProfile.id == scholar_profile_id) - if favorite_only: - stmt = stmt.where(ScholarPublication.is_favorite.is_(True)) - if resolved_mode == MODE_UNREAD: - stmt = stmt.where(ScholarPublication.is_read.is_(False)) - if resolved_mode == MODE_LATEST: - if latest_run_id is None: - return 0 - stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id) - result = await db_session.execute(stmt) - return int(result.scalar_one() or 0) - - -async def count_unread_for_user( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int | None = None, - favorite_only: bool = False, -) -> int: - return await count_for_user( - db_session, - user_id=user_id, - mode=MODE_UNREAD, - scholar_profile_id=scholar_profile_id, - favorite_only=favorite_only, - ) - - -async def count_latest_for_user( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int | None = None, - favorite_only: bool = False, -) -> int: - return await count_for_user( - db_session, - user_id=user_id, - mode=MODE_LATEST, - scholar_profile_id=scholar_profile_id, - favorite_only=favorite_only, - ) - - -async def count_favorite_for_user( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int | None = None, -) -> int: - return await count_for_user( - db_session, - user_id=user_id, - mode=MODE_ALL, - scholar_profile_id=scholar_profile_id, - favorite_only=True, - ) diff --git a/build/lib/app/services/domains/publications/enrichment.py b/build/lib/app/services/domains/publications/enrichment.py deleted file mode 100644 index 381107a..0000000 --- a/build/lib/app/services/domains/publications/enrichment.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import annotations - -import logging - -from sqlalchemy.ext.asyncio import AsyncSession - -from app.services.domains.publications.pdf_queue import ( - enqueue_missing_pdf_jobs, - enqueue_retry_pdf_job, - overlay_pdf_job_state, -) -from app.services.domains.publications.types import PublicationListItem - -logger = logging.getLogger(__name__) - - -async def schedule_missing_pdf_enrichment_for_user( - db_session: AsyncSession, - *, - user_id: int, - request_email: str | None, - items: list[PublicationListItem], - max_items: int, -) -> int: - queued_ids = await enqueue_missing_pdf_jobs( - db_session, - user_id=user_id, - request_email=request_email, - rows=items, - max_items=max_items, - ) - logger.info( - "publications.enrichment.scheduled", - extra={ - "event": "publications.enrichment.scheduled", - "user_id": user_id, - "publication_count": len(queued_ids), - }, - ) - return len(queued_ids) - - -async def schedule_retry_pdf_enrichment_for_row( - db_session: AsyncSession, - *, - user_id: int, - request_email: str | None, - item: PublicationListItem, -) -> bool: - queued = await enqueue_retry_pdf_job( - db_session, - user_id=user_id, - request_email=request_email, - row=item, - ) - logger.info( - "publications.enrichment.retry_scheduled", - extra={ - "event": "publications.enrichment.retry_scheduled", - "user_id": user_id, - "publication_id": item.publication_id, - "queued": queued, - }, - ) - return queued - - -async def hydrate_pdf_enrichment_state( - db_session: AsyncSession, - *, - items: list[PublicationListItem], -) -> list[PublicationListItem]: - return await overlay_pdf_job_state(db_session, rows=items) diff --git a/build/lib/app/services/domains/publications/listing.py b/build/lib/app/services/domains/publications/listing.py deleted file mode 100644 index 0d09d4c..0000000 --- a/build/lib/app/services/domains/publications/listing.py +++ /dev/null @@ -1,93 +0,0 @@ -from __future__ import annotations - -from sqlalchemy.ext.asyncio import AsyncSession - -from app.services.domains.publication_identifiers import application as identifier_service -from app.services.domains.publications.modes import ( - MODE_ALL, - MODE_UNREAD, - resolve_publication_view_mode, -) -from app.services.domains.publications.queries import ( - get_latest_completed_run_id_for_user, - get_publication_item_for_user, - publication_list_item_from_row, - publications_query, - unread_item_from_row, -) -from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem - - -async def list_for_user( - db_session: AsyncSession, - *, - user_id: int, - mode: str = MODE_ALL, - scholar_profile_id: int | None = None, - favorite_only: bool = False, - limit: int = 300, - offset: int = 0, -) -> list[PublicationListItem]: - resolved_mode = resolve_publication_view_mode(mode) - latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) - result = await db_session.execute( - publications_query( - user_id=user_id, - mode=resolved_mode, - latest_run_id=latest_run_id, - scholar_profile_id=scholar_profile_id, - favorite_only=favorite_only, - limit=limit, - offset=offset, - ) - ) - rows = [ - publication_list_item_from_row(row, latest_run_id=latest_run_id) - for row in result.all() - ] - return await identifier_service.overlay_publication_items_with_display_identifiers( - db_session, - items=rows, - ) - - -async def retry_pdf_for_user( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int, - publication_id: int, -) -> PublicationListItem | None: - item = await get_publication_item_for_user( - db_session, - user_id=user_id, - scholar_profile_id=scholar_profile_id, - publication_id=publication_id, - ) - if item is None: - return None - hydrated = await identifier_service.overlay_publication_items_with_display_identifiers( - db_session, - items=[item], - ) - return hydrated[0] if hydrated else item - - -async def list_unread_for_user( - db_session: AsyncSession, - *, - user_id: int, - limit: int = 100, -) -> list[UnreadPublicationItem]: - result = await db_session.execute( - publications_query( - user_id=user_id, - mode=MODE_UNREAD, - latest_run_id=None, - scholar_profile_id=None, - favorite_only=False, - limit=limit, - offset=0, - ) - ) - return [unread_item_from_row(row) for row in result.all()] diff --git a/build/lib/app/services/domains/publications/modes.py b/build/lib/app/services/domains/publications/modes.py deleted file mode 100644 index 2975169..0000000 --- a/build/lib/app/services/domains/publications/modes.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import annotations - -MODE_ALL = "all" -MODE_UNREAD = "unread" -MODE_LATEST = "latest" -MODE_NEW = "new" # compatibility alias for MODE_LATEST - - -def resolve_publication_view_mode(value: str | None) -> str: - if value == MODE_UNREAD: - return MODE_UNREAD - if value in {MODE_LATEST, MODE_NEW}: - return MODE_LATEST - return MODE_ALL diff --git a/build/lib/app/services/domains/publications/pdf_queue.py b/build/lib/app/services/domains/publications/pdf_queue.py deleted file mode 100644 index 18ac8d9..0000000 --- a/build/lib/app/services/domains/publications/pdf_queue.py +++ /dev/null @@ -1,915 +0,0 @@ -from __future__ import annotations - -import asyncio -from dataclasses import dataclass -from datetime import datetime, timezone -import logging - -from sqlalchemy import Select, func, literal, or_, select, union_all -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import ( - Publication, - PublicationPdfJob, - PublicationPdfJobEvent, - ScholarProfile, - ScholarPublication, - User, -) -from app.db.session import get_session_factory -from app.services.domains.publication_identifiers import application as identifier_service -from app.services.domains.publication_identifiers.types import DisplayIdentifier -from app.services.domains.publications.pdf_resolution_pipeline import ( - resolve_publication_pdf_outcome_for_row, -) -from app.services.domains.publications.types import PublicationListItem -from app.services.domains.unpaywall.application import ( - FAILURE_RESOLUTION_EXCEPTION, - OaResolutionOutcome, -) -from app.settings import settings - -PDF_STATUS_UNTRACKED = "untracked" -PDF_STATUS_QUEUED = "queued" -PDF_STATUS_RUNNING = "running" -PDF_STATUS_RESOLVED = "resolved" -PDF_STATUS_FAILED = "failed" - -PDF_EVENT_QUEUED = "queued" -PDF_EVENT_ATTEMPT_STARTED = "attempt_started" -PDF_EVENT_RESOLVED = "resolved" -PDF_EVENT_FAILED = "failed" - -logger = logging.getLogger(__name__) -_scheduled_tasks: set[asyncio.Task[None]] = set() - - -@dataclass(frozen=True) -class PdfQueueListItem: - publication_id: int - title: str - pdf_url: str | None - status: str - attempt_count: int - last_failure_reason: str | None - last_failure_detail: str | None - last_source: str | None - requested_by_user_id: int | None - requested_by_email: str | None - queued_at: datetime | None - last_attempt_at: datetime | None - resolved_at: datetime | None - updated_at: datetime - display_identifier: DisplayIdentifier | None = None - - -@dataclass(frozen=True) -class PdfRequeueResult: - publication_exists: bool - queued: bool - - -@dataclass(frozen=True) -class PdfBulkQueueResult: - requested_count: int - queued_count: int - - -@dataclass(frozen=True) -class PdfQueuePage: - items: list[PdfQueueListItem] - total_count: int - limit: int - offset: int - - -def _utcnow() -> datetime: - return datetime.now(timezone.utc) - - -def _publication_ids(rows: list[PublicationListItem]) -> list[int]: - return sorted({row.publication_id for row in rows}) - - -def _status_from_job(row: PublicationListItem, job: PublicationPdfJob | None) -> str: - if row.pdf_url: - return PDF_STATUS_RESOLVED - if job is None: - return PDF_STATUS_UNTRACKED - return job.status - - -def _item_from_row_and_job( - row: PublicationListItem, - job: PublicationPdfJob | None, -) -> PublicationListItem: - return PublicationListItem( - publication_id=row.publication_id, - scholar_profile_id=row.scholar_profile_id, - scholar_label=row.scholar_label, - title=row.title, - year=row.year, - citation_count=row.citation_count, - venue_text=row.venue_text, - pub_url=row.pub_url, - pdf_url=row.pdf_url, - is_read=row.is_read, - is_favorite=row.is_favorite, - first_seen_at=row.first_seen_at, - is_new_in_latest_run=row.is_new_in_latest_run, - pdf_status=_status_from_job(row, job), - pdf_attempt_count=int(job.attempt_count) if job is not None else 0, - pdf_failure_reason=job.last_failure_reason if job is not None else None, - pdf_failure_detail=job.last_failure_detail if job is not None else None, - display_identifier=row.display_identifier, - ) - - -def _queueable_rows( - rows: list[PublicationListItem], - *, - max_items: int, -) -> list[PublicationListItem]: - bounded = max(0, int(max_items)) - if bounded == 0: - return [] - candidates = [row for row in rows if not row.pdf_url] - return candidates[:bounded] - - -def _bounded_limit(limit: int, *, max_value: int = 500) -> int: - return max(1, min(int(limit), max_value)) - - -def _bounded_offset(offset: int) -> int: - return max(int(offset), 0) - - -def _auto_retry_interval_seconds() -> int: - return max(int(settings.pdf_auto_retry_interval_seconds), 1) - - -def _auto_retry_first_interval_seconds() -> int: - return max(int(settings.pdf_auto_retry_first_interval_seconds), 1) - - -def _auto_retry_max_attempts() -> int: - return max(int(settings.pdf_auto_retry_max_attempts), 1) - - -def _retry_interval_seconds_for_attempt_count(attempt_count: int) -> int: - if int(attempt_count) <= 1: - return _auto_retry_first_interval_seconds() - return _auto_retry_interval_seconds() - - -def _cooldown_active( - *, - last_attempt_at: datetime | None, - attempt_count: int, -) -> bool: - if last_attempt_at is None: - return False - elapsed = (_utcnow() - last_attempt_at).total_seconds() - return elapsed < _retry_interval_seconds_for_attempt_count(int(attempt_count)) - - -def _can_enqueue_job( - job: PublicationPdfJob | None, - *, - force_retry: bool, -) -> bool: - if job is None: - return True - if job.status in {PDF_STATUS_QUEUED, PDF_STATUS_RUNNING}: - return False - if force_retry: - return job.status in {PDF_STATUS_FAILED, PDF_STATUS_RESOLVED, PDF_STATUS_UNTRACKED} - if job.status == PDF_STATUS_RESOLVED: - return False - if int(job.attempt_count) >= _auto_retry_max_attempts(): - return False - if _cooldown_active( - last_attempt_at=job.last_attempt_at, - attempt_count=int(job.attempt_count), - ): - return False - return True - - -def _event_row( - *, - publication_id: int, - user_id: int | None, - event_type: str, - status: str | None, - source: str | None = None, - failure_reason: str | None = None, - message: str | None = None, -) -> PublicationPdfJobEvent: - return PublicationPdfJobEvent( - publication_id=publication_id, - user_id=user_id, - event_type=event_type, - status=status, - source=source, - failure_reason=failure_reason, - message=message, - ) - - -def _queued_job( - *, - publication_id: int, - user_id: int, -) -> PublicationPdfJob: - now = _utcnow() - return PublicationPdfJob( - publication_id=publication_id, - status=PDF_STATUS_QUEUED, - queued_at=now, - last_requested_by_user_id=user_id, - ) - - -def _mark_job_queued(job: PublicationPdfJob, *, user_id: int) -> None: - now = _utcnow() - job.status = PDF_STATUS_QUEUED - job.queued_at = now - job.last_requested_by_user_id = user_id - job.last_failure_reason = None - job.last_failure_detail = None - job.last_source = None - - -def _state_map(jobs: list[PublicationPdfJob]) -> dict[int, PublicationPdfJob]: - return {int(job.publication_id): job for job in jobs} - - -async def _jobs_for_publication_ids( - db_session: AsyncSession, - *, - publication_ids: list[int], -) -> dict[int, PublicationPdfJob]: - if not publication_ids: - return {} - result = await db_session.execute( - select(PublicationPdfJob).where(PublicationPdfJob.publication_id.in_(publication_ids)) - ) - return _state_map(list(result.scalars())) - - -async def overlay_pdf_job_state( - db_session: AsyncSession, - *, - rows: list[PublicationListItem], -) -> list[PublicationListItem]: - if not rows: - return [] - jobs = await _jobs_for_publication_ids( - db_session, - publication_ids=_publication_ids(rows), - ) - return [_item_from_row_and_job(row, jobs.get(row.publication_id)) for row in rows] - - -async def _enqueue_rows( - db_session: AsyncSession, - *, - user_id: int, - rows: list[PublicationListItem], - force_retry: bool, -) -> list[PublicationListItem]: - if not rows: - return [] - queued: list[PublicationListItem] = [] - jobs = await _jobs_for_publication_ids( - db_session, - publication_ids=_publication_ids(rows), - ) - for row in rows: - job = jobs.get(row.publication_id) - if not _can_enqueue_job(job, force_retry=force_retry): - continue - if job is None: - job = _queued_job(publication_id=row.publication_id, user_id=user_id) - jobs[row.publication_id] = job - db_session.add(job) - else: - _mark_job_queued(job, user_id=user_id) - db_session.add( - _event_row( - publication_id=row.publication_id, - user_id=user_id, - event_type=PDF_EVENT_QUEUED, - status=PDF_STATUS_QUEUED, - ) - ) - queued.append(row) - if queued: - await db_session.commit() - return queued - - -def _register_task(task: asyncio.Task[None]) -> None: - _scheduled_tasks.add(task) - - -def _drop_finished_task(task: asyncio.Task[None]) -> None: - _scheduled_tasks.discard(task) - try: - task.result() - except Exception: - logger.exception( - "publications.pdf_queue.task_failed", - extra={"event": "publications.pdf_queue.task_failed"}, - ) - - -async def _mark_attempt_started( - *, - publication_id: int, - user_id: int, -) -> None: - session_factory = get_session_factory() - async with session_factory() as db_session: - job = await db_session.get(PublicationPdfJob, publication_id) - if job is None: - job = _queued_job(publication_id=publication_id, user_id=user_id) - db_session.add(job) - job.status = PDF_STATUS_RUNNING - job.last_attempt_at = _utcnow() - job.attempt_count = int(job.attempt_count) + 1 - db_session.add( - _event_row( - publication_id=publication_id, - user_id=user_id, - event_type=PDF_EVENT_ATTEMPT_STARTED, - status=PDF_STATUS_RUNNING, - ) - ) - await db_session.commit() - - -def _failed_outcome( - *, - row: PublicationListItem, -) -> OaResolutionOutcome: - return OaResolutionOutcome( - publication_id=row.publication_id, - doi=None, - pdf_url=None, - failure_reason=FAILURE_RESOLUTION_EXCEPTION, - source=None, - used_crossref=False, - ) - - -async def _fetch_outcome_for_row( - *, - row: PublicationListItem, - request_email: str | None, -) -> OaResolutionOutcome: - pipeline_result = await resolve_publication_pdf_outcome_for_row( - row=row, - request_email=request_email, - ) - outcome = pipeline_result.outcome - if outcome is not None: - return outcome - return _failed_outcome(row=row) - - -def _apply_publication_update( - publication: Publication, - *, - pdf_url: str | None, -) -> None: - if pdf_url and publication.pdf_url != pdf_url: - publication.pdf_url = pdf_url - - -def _apply_job_outcome(job: PublicationPdfJob, *, outcome: OaResolutionOutcome) -> None: - job.last_source = outcome.source - if outcome.pdf_url: - job.status = PDF_STATUS_RESOLVED - job.resolved_at = _utcnow() - job.last_failure_reason = None - job.last_failure_detail = None - return - job.status = PDF_STATUS_FAILED - job.last_failure_reason = outcome.failure_reason - job.last_failure_detail = outcome.failure_reason - - -def _result_event(outcome: OaResolutionOutcome) -> tuple[str, str]: - if outcome.pdf_url: - return PDF_EVENT_RESOLVED, PDF_STATUS_RESOLVED - return PDF_EVENT_FAILED, PDF_STATUS_FAILED - - -async def _persist_outcome( - *, - publication_id: int, - user_id: int, - outcome: OaResolutionOutcome, -) -> None: - session_factory = get_session_factory() - async with session_factory() as db_session: - publication = await db_session.get(Publication, publication_id) - job = await db_session.get(PublicationPdfJob, publication_id) - if publication is None or job is None: - return - _apply_publication_update(publication, pdf_url=outcome.pdf_url) - await identifier_service.sync_identifiers_for_publication_resolution( - db_session, - publication=publication, - source=outcome.source, - ) - _apply_job_outcome(job, outcome=outcome) - event_type, status = _result_event(outcome) - db_session.add( - _event_row( - publication_id=publication_id, - user_id=user_id, - event_type=event_type, - status=status, - source=outcome.source, - failure_reason=outcome.failure_reason, - message=outcome.failure_reason, - ) - ) - await db_session.commit() - - -async def _resolve_publication_row( - *, - user_id: int, - request_email: str | None, - row: PublicationListItem, -) -> None: - await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id) - try: - outcome = await _fetch_outcome_for_row(row=row, request_email=request_email) - except Exception as exc: # pragma: no cover - defensive network boundary - logger.warning( - "publications.pdf_queue.resolve_failed", - extra={ - "event": "publications.pdf_queue.resolve_failed", - "publication_id": row.publication_id, - "error": str(exc), - }, - ) - outcome = _failed_outcome(row=row) - await _persist_outcome( - publication_id=row.publication_id, - user_id=user_id, - outcome=outcome, - ) - - -async def _run_resolution_task( - *, - user_id: int, - request_email: str | None, - rows: list[PublicationListItem], -) -> None: - for row in rows: - await _resolve_publication_row( - user_id=user_id, - request_email=request_email, - row=row, - ) - - -def _schedule_rows( - *, - user_id: int, - request_email: str | None, - rows: list[PublicationListItem], -) -> None: - if not rows: - return - task = asyncio.create_task( - _run_resolution_task( - user_id=user_id, - request_email=request_email, - rows=rows, - ) - ) - _register_task(task) - task.add_done_callback(_drop_finished_task) - - -async def enqueue_missing_pdf_jobs( - db_session: AsyncSession, - *, - user_id: int, - request_email: str | None, - rows: list[PublicationListItem], - max_items: int, -) -> list[int]: - queueable = _queueable_rows(rows, max_items=max_items) - queued_rows = await _enqueue_rows( - db_session, - user_id=user_id, - rows=queueable, - force_retry=False, - ) - _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows) - return [row.publication_id for row in queued_rows] - - -async def enqueue_retry_pdf_job( - db_session: AsyncSession, - *, - user_id: int, - request_email: str | None, - row: PublicationListItem, -) -> bool: - queued_rows = await _enqueue_rows( - db_session, - user_id=user_id, - rows=[row], - force_retry=True, - ) - _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows) - return bool(queued_rows) - - -def _retry_item_label(display_name: str | None, scholar_id: str | None) -> str: - return str(display_name or scholar_id or "unknown") - - -def _retry_item_from_publication( - publication: Publication, - *, - link_row: tuple | None, -) -> PublicationListItem: - if link_row is None: - scholar_profile_id = 0 - scholar_label = "unknown" - is_read = True - first_seen_at = publication.created_at or _utcnow() - else: - scholar_profile_id = int(link_row[0]) - scholar_label = _retry_item_label(link_row[1], link_row[2]) - is_read = bool(link_row[3]) - first_seen_at = link_row[4] or publication.created_at or _utcnow() - return PublicationListItem( - publication_id=int(publication.id), - scholar_profile_id=scholar_profile_id, - scholar_label=scholar_label, - title=publication.title_raw, - year=publication.year, - citation_count=int(publication.citation_count or 0), - venue_text=publication.venue_text, - pub_url=publication.pub_url, - pdf_url=publication.pdf_url, - is_read=is_read, - first_seen_at=first_seen_at, - is_new_in_latest_run=False, - ) - - -async def _retry_item_for_publication_id( - db_session: AsyncSession, - *, - publication_id: int, -) -> PublicationListItem | None: - publication = await db_session.get(Publication, publication_id) - if publication is None: - return None - result = await db_session.execute( - select( - ScholarProfile.id, - ScholarProfile.display_name, - ScholarProfile.scholar_id, - ScholarPublication.is_read, - ScholarPublication.created_at, - ) - .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) - .where(ScholarPublication.publication_id == publication_id) - .order_by(ScholarPublication.created_at.asc()) - .limit(1) - ) - return _retry_item_from_publication(publication, link_row=result.one_or_none()) - - -async def enqueue_retry_pdf_job_for_publication_id( - db_session: AsyncSession, - *, - user_id: int, - request_email: str | None, - publication_id: int, -) -> PdfRequeueResult: - row = await _retry_item_for_publication_id( - db_session, - publication_id=publication_id, - ) - if row is None: - return PdfRequeueResult(publication_exists=False, queued=False) - queued = await enqueue_retry_pdf_job( - db_session, - user_id=user_id, - request_email=request_email, - row=row, - ) - return PdfRequeueResult(publication_exists=True, queued=queued) - - -def _queue_candidate_from_publication(publication: Publication) -> PublicationListItem: - return PublicationListItem( - publication_id=int(publication.id), - scholar_profile_id=0, - scholar_label="admin", - title=publication.title_raw, - year=publication.year, - citation_count=int(publication.citation_count or 0), - venue_text=publication.venue_text, - pub_url=publication.pub_url, - pdf_url=publication.pdf_url, - is_read=True, - first_seen_at=publication.created_at or _utcnow(), - is_new_in_latest_run=False, - ) - - -async def _missing_pdf_candidates( - db_session: AsyncSession, - *, - limit: int, -) -> list[PublicationListItem]: - bounded_limit = max(1, min(int(limit), 5000)) - result = await db_session.execute( - select(Publication) - .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id) - .where(Publication.pdf_url.is_(None)) - .where( - or_( - PublicationPdfJob.publication_id.is_(None), - PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]), - ) - ) - .order_by(Publication.updated_at.desc(), Publication.id.desc()) - .limit(bounded_limit) - ) - return [ - _queue_candidate_from_publication(publication) - for publication in result.scalars() - ] - - -async def enqueue_all_missing_pdf_jobs( - db_session: AsyncSession, - *, - user_id: int, - request_email: str | None, - limit: int = 1000, -) -> PdfBulkQueueResult: - candidates = await _missing_pdf_candidates(db_session, limit=limit) - queued_rows = await _enqueue_rows( - db_session, - user_id=user_id, - rows=candidates, - force_retry=True, - ) - _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows) - return PdfBulkQueueResult( - requested_count=len(candidates), - queued_count=len(queued_rows), - ) - - -def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]: - stmt = ( - select( - PublicationPdfJob.publication_id, - Publication.title_raw, - Publication.pdf_url, - PublicationPdfJob.status, - PublicationPdfJob.attempt_count, - PublicationPdfJob.last_failure_reason, - PublicationPdfJob.last_failure_detail, - PublicationPdfJob.last_source, - PublicationPdfJob.last_requested_by_user_id, - User.email, - PublicationPdfJob.queued_at, - PublicationPdfJob.last_attempt_at, - PublicationPdfJob.resolved_at, - PublicationPdfJob.updated_at, - ) - .join(Publication, Publication.id == PublicationPdfJob.publication_id) - .outerjoin(User, User.id == PublicationPdfJob.last_requested_by_user_id) - ) - if status: - stmt = stmt.where(PublicationPdfJob.status == status) - return stmt - - -def _tracked_queue_select(*, limit: int, offset: int, status: str | None) -> Select[tuple]: - return ( - _tracked_queue_select_base(status=status) - .order_by(PublicationPdfJob.updated_at.desc()) - .limit(_bounded_limit(limit)) - .offset(_bounded_offset(offset)) - ) - - -def _untracked_queue_select_base() -> Select[tuple]: - return ( - select( - Publication.id, - Publication.title_raw, - Publication.pdf_url, - literal(PDF_STATUS_UNTRACKED), - literal(0), - literal(None), - literal(None), - literal(None), - literal(None), - literal(None), - literal(None), - literal(None), - literal(None), - Publication.updated_at, - ) - .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id) - .where(Publication.pdf_url.is_(None)) - .where(PublicationPdfJob.publication_id.is_(None)) - ) - - -def _untracked_queue_select(*, limit: int, offset: int) -> Select[tuple]: - return ( - _untracked_queue_select_base() - .order_by(Publication.updated_at.desc(), Publication.id.desc()) - .limit(_bounded_limit(limit)) - .offset(_bounded_offset(offset)) - ) - - -def _all_queue_select(*, limit: int, offset: int) -> Select[tuple]: - union_stmt = union_all( - _tracked_queue_select_base(status=None), - _untracked_queue_select_base(), - ).subquery() - return ( - select(union_stmt) - .order_by(union_stmt.c.updated_at.desc()) - .limit(_bounded_limit(limit)) - .offset(_bounded_offset(offset)) - ) - - -def _tracked_queue_count_select(*, status: str | None) -> Select[tuple]: - stmt = select(func.count()).select_from(PublicationPdfJob) - if status: - stmt = stmt.where(PublicationPdfJob.status == status) - return stmt - - -def _untracked_queue_count_select() -> Select[tuple]: - return ( - select(func.count()) - .select_from(Publication) - .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id) - .where(Publication.pdf_url.is_(None)) - .where(PublicationPdfJob.publication_id.is_(None)) - ) - - -def _queue_item_from_row(row: tuple) -> PdfQueueListItem: - return PdfQueueListItem( - publication_id=int(row[0]), - title=str(row[1] or ""), - pdf_url=row[2], - status=str(row[3] or PDF_STATUS_UNTRACKED), - attempt_count=int(row[4] or 0), - last_failure_reason=row[5], - last_failure_detail=row[6], - last_source=row[7], - requested_by_user_id=int(row[8]) if row[8] is not None else None, - requested_by_email=row[9], - queued_at=row[10], - last_attempt_at=row[11], - resolved_at=row[12], - updated_at=row[13], - ) - - -async def _hydrated_queue_items( - db_session: AsyncSession, - *, - rows: list[tuple], -) -> list[PdfQueueListItem]: - items = [_queue_item_from_row(row) for row in rows] - return await identifier_service.overlay_pdf_queue_items_with_display_identifiers( - db_session, - items=items, - ) - - -async def list_pdf_queue_items( - db_session: AsyncSession, - *, - limit: int = 100, - offset: int = 0, - status: str | None = None, -) -> list[PdfQueueListItem]: - bounded_limit = _bounded_limit(limit) - bounded_offset = _bounded_offset(offset) - normalized_status = (status or "").strip().lower() or None - if normalized_status == PDF_STATUS_UNTRACKED: - result = await db_session.execute( - _untracked_queue_select( - limit=bounded_limit, - offset=bounded_offset, - ) - ) - return await _hydrated_queue_items(db_session, rows=list(result.all())) - if normalized_status is None: - result = await db_session.execute( - _all_queue_select( - limit=bounded_limit, - offset=bounded_offset, - ) - ) - return await _hydrated_queue_items(db_session, rows=list(result.all())) - result = await db_session.execute( - _tracked_queue_select( - limit=bounded_limit, - offset=bounded_offset, - status=normalized_status, - ) - ) - return await _hydrated_queue_items(db_session, rows=list(result.all())) - - -async def count_pdf_queue_items( - db_session: AsyncSession, - *, - status: str | None = None, -) -> int: - normalized_status = (status or "").strip().lower() or None - if normalized_status == PDF_STATUS_UNTRACKED: - result = await db_session.execute(_untracked_queue_count_select()) - return int(result.scalar_one() or 0) - tracked_result = await db_session.execute( - _tracked_queue_count_select(status=normalized_status) - ) - tracked_count = int(tracked_result.scalar_one() or 0) - if normalized_status is not None: - return tracked_count - untracked_result = await db_session.execute(_untracked_queue_count_select()) - untracked_count = int(untracked_result.scalar_one() or 0) - return tracked_count + untracked_count - - -async def list_pdf_queue_page( - db_session: AsyncSession, - *, - limit: int = 100, - offset: int = 0, - status: str | None = None, -) -> PdfQueuePage: - bounded_limit = _bounded_limit(limit) - bounded_offset = _bounded_offset(offset) - items = await list_pdf_queue_items( - db_session, - limit=bounded_limit, - offset=bounded_offset, - status=status, - ) - total_count = await count_pdf_queue_items( - db_session, - status=status, - ) - return PdfQueuePage( - items=items, - total_count=total_count, - limit=bounded_limit, - offset=bounded_offset, - ) - - -async def drain_ready_jobs( - db_session: AsyncSession, - *, - limit: int, - max_attempts: int, -) -> int: - result = await db_session.execute( - select(User.id).where(User.is_active.is_(True)).order_by(User.id.asc()).limit(1) - ) - system_user_id = result.scalar_one_or_none() - if system_user_id is None: - return 0 - - bulk_result = await enqueue_all_missing_pdf_jobs( - db_session, - user_id=system_user_id, - request_email=settings.unpaywall_email, - limit=limit, - ) - return bulk_result.queued_count diff --git a/build/lib/app/services/domains/publications/pdf_resolution_pipeline.py b/build/lib/app/services/domains/publications/pdf_resolution_pipeline.py deleted file mode 100644 index 9e07c0b..0000000 --- a/build/lib/app/services/domains/publications/pdf_resolution_pipeline.py +++ /dev/null @@ -1,108 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -import logging -from typing import Any - -from app.services.domains.publications.types import PublicationListItem -from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes -from app.settings import settings - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class PipelineOutcome: - outcome: OaResolutionOutcome | None - scholar_candidates: Any | None # Kept for backward compatibility with calling signatures - - -async def resolve_publication_pdf_outcome_for_row( - *, - row: PublicationListItem, - request_email: str | None, -) -> PipelineOutcome: - # 1. OpenAlex OA - openalex_outcome = await _openalex_outcome(row, request_email=request_email) - if openalex_outcome and openalex_outcome.pdf_url: - return PipelineOutcome(openalex_outcome, None) - - # 2. arXiv - arxiv_outcome = await _arxiv_outcome(row, request_email=request_email) - if arxiv_outcome and arxiv_outcome.pdf_url: - return PipelineOutcome(arxiv_outcome, None) - - # 3. Unpaywall (which falls back to Crossref) - oa_outcome = await _oa_outcome(row=row, request_email=request_email) - return PipelineOutcome(oa_outcome, None) - - -async def _openalex_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None: - from app.services.domains.openalex.client import OpenAlexClient - from app.services.domains.openalex.matching import find_best_match - - if not row.title: - return None - - import re - safe_title = re.sub(r"[^\w\s]", " ", row.title) - safe_title = " ".join(safe_title.split()) - if not safe_title: - return None - - client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=request_email or settings.crossref_api_mailto) - try: - openalex_works = await client.get_works_by_filter({"title.search": safe_title}, limit=5) - match = find_best_match( - target_title=row.title, - target_year=row.year, - target_authors=row.scholar_label, - candidates=openalex_works, - ) - if match and match.oa_url: - return OaResolutionOutcome( - publication_id=row.publication_id, - doi=match.doi, - pdf_url=match.oa_url, - failure_reason=None, - source="openalex", - used_crossref=False, - ) - except Exception as exc: - logger.warning( - "publications.pdf_resolution.openalex_failed", - extra={"event": "publications.pdf_resolution.openalex_failed", "error": str(exc)}, - ) - return None - - -async def _arxiv_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None: - from app.services.domains.arxiv.application import discover_arxiv_id_for_publication - - try: - arxiv_id = await discover_arxiv_id_for_publication(item=row, request_email=request_email) - if arxiv_id: - pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf" - return OaResolutionOutcome( - publication_id=row.publication_id, - doi=None, - pdf_url=pdf_url, - failure_reason=None, - source="arxiv", - used_crossref=False, - ) - except Exception as exc: - logger.warning( - "publications.pdf_resolution.arxiv_failed", - extra={"event": "publications.pdf_resolution.arxiv_failed", "error": str(exc)}, - ) - return None - - -async def _oa_outcome( - *, - row: PublicationListItem, - request_email: str | None, -) -> OaResolutionOutcome | None: - outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email) - return outcomes.get(row.publication_id) diff --git a/build/lib/app/services/domains/publications/queries.py b/build/lib/app/services/domains/publications/queries.py deleted file mode 100644 index e7356f4..0000000 --- a/build/lib/app/services/domains/publications/queries.py +++ /dev/null @@ -1,203 +0,0 @@ -from __future__ import annotations - -from sqlalchemy import Select, func, select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import CrawlRun, Publication, RunStatus, ScholarProfile, ScholarPublication -from app.services.domains.publications.modes import MODE_LATEST, MODE_UNREAD -from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem - - -def _normalized_citation_count(value: object) -> int: - try: - return int(value or 0) - except (TypeError, ValueError): - return 0 - - -async def get_latest_completed_run_id_for_user( - db_session: AsyncSession, - *, - user_id: int, -) -> int | None: - result = await db_session.execute( - select(func.max(CrawlRun.id)).where( - CrawlRun.user_id == user_id, - CrawlRun.status != RunStatus.RUNNING, - ) - ) - latest_run_id = result.scalar_one_or_none() - return int(latest_run_id) if latest_run_id is not None else None - - -def publications_query( - *, - user_id: int, - mode: str, - latest_run_id: int | None, - scholar_profile_id: int | None, - favorite_only: bool, - limit: int, - offset: int = 0, -) -> Select[tuple]: - scholar_label = ScholarProfile.display_name - stmt = ( - select( - Publication.id, - ScholarProfile.id, - scholar_label, - ScholarProfile.scholar_id, - Publication.title_raw, - Publication.year, - Publication.citation_count, - Publication.venue_text, - Publication.pub_url, - Publication.pdf_url, - ScholarPublication.is_read, - ScholarPublication.is_favorite, - ScholarPublication.first_seen_run_id, - ScholarPublication.created_at, - ) - .join(ScholarPublication, ScholarPublication.publication_id == Publication.id) - .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) - .where(ScholarProfile.user_id == user_id) - .order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) - .offset(max(int(offset), 0)) - .limit(limit) - ) - if scholar_profile_id is not None: - stmt = stmt.where(ScholarProfile.id == scholar_profile_id) - if favorite_only: - stmt = stmt.where(ScholarPublication.is_favorite.is_(True)) - if mode == MODE_UNREAD: - stmt = stmt.where(ScholarPublication.is_read.is_(False)) - if mode == MODE_LATEST: - if latest_run_id is None: - return stmt.where(False) - stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id) - return stmt - - -def publication_query_for_user( - *, - user_id: int, - scholar_profile_id: int, - publication_id: int, -) -> Select[tuple]: - return ( - select( - Publication.id, - ScholarProfile.id, - ScholarProfile.display_name, - ScholarProfile.scholar_id, - Publication.title_raw, - Publication.year, - Publication.citation_count, - Publication.venue_text, - Publication.pub_url, - Publication.pdf_url, - ScholarPublication.is_read, - ScholarPublication.is_favorite, - ScholarPublication.first_seen_run_id, - ScholarPublication.created_at, - ) - .join(ScholarPublication, ScholarPublication.publication_id == Publication.id) - .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) - .where( - ScholarProfile.user_id == user_id, - ScholarProfile.id == scholar_profile_id, - Publication.id == publication_id, - ) - .limit(1) - ) - - -async def get_publication_item_for_user( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int, - publication_id: int, -) -> PublicationListItem | None: - latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) - result = await db_session.execute( - publication_query_for_user( - user_id=user_id, - scholar_profile_id=scholar_profile_id, - publication_id=publication_id, - ) - ) - row = result.one_or_none() - if row is None: - return None - return publication_list_item_from_row(row, latest_run_id=latest_run_id) - - -def publication_list_item_from_row( - row: tuple, - *, - latest_run_id: int | None, -) -> PublicationListItem: - ( - publication_id, - scholar_profile_id, - display_name, - scholar_id, - title_raw, - year, - citation_count, - venue_text, - pub_url, - pdf_url, - is_read, - is_favorite, - first_seen_run_id, - created_at, - ) = row - return PublicationListItem( - publication_id=int(publication_id), - scholar_profile_id=int(scholar_profile_id), - scholar_label=(display_name or scholar_id), - title=title_raw, - year=year, - citation_count=_normalized_citation_count(citation_count), - venue_text=venue_text, - pub_url=pub_url, - pdf_url=pdf_url, - is_read=bool(is_read), - is_favorite=bool(is_favorite), - first_seen_at=created_at, - is_new_in_latest_run=( - latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id - ), - ) - - -def unread_item_from_row(row: tuple) -> UnreadPublicationItem: - ( - publication_id, - scholar_profile_id, - display_name, - scholar_id, - title_raw, - year, - citation_count, - venue_text, - pub_url, - pdf_url, - _is_read, - _is_favorite, - _first_seen_run_id, - _created_at, - ) = row - return UnreadPublicationItem( - publication_id=int(publication_id), - scholar_profile_id=int(scholar_profile_id), - scholar_label=(display_name or scholar_id), - title=title_raw, - year=year, - citation_count=_normalized_citation_count(citation_count), - venue_text=venue_text, - pub_url=pub_url, - pdf_url=pdf_url, - ) diff --git a/build/lib/app/services/domains/publications/read_state.py b/build/lib/app/services/domains/publications/read_state.py deleted file mode 100644 index 41fc31a..0000000 --- a/build/lib/app/services/domains/publications/read_state.py +++ /dev/null @@ -1,94 +0,0 @@ -from __future__ import annotations - -from sqlalchemy import select, tuple_, update -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import ScholarProfile, ScholarPublication - - -def _normalized_selection_pairs(selections: list[tuple[int, int]]) -> set[tuple[int, int]]: - pairs: set[tuple[int, int]] = set() - for scholar_profile_id, publication_id in selections: - normalized = (int(scholar_profile_id), int(publication_id)) - if normalized[0] <= 0 or normalized[1] <= 0: - continue - pairs.add(normalized) - return pairs - - -def _scoped_scholar_ids_query(*, user_id: int): - return ( - select(ScholarProfile.id) - .where(ScholarProfile.user_id == user_id) - .scalar_subquery() - ) - - -async def mark_all_unread_as_read_for_user( - db_session: AsyncSession, - *, - user_id: int, -) -> int: - scholar_ids = _scoped_scholar_ids_query(user_id=user_id) - stmt = ( - update(ScholarPublication) - .where( - ScholarPublication.scholar_profile_id.in_(scholar_ids), - ScholarPublication.is_read.is_(False), - ) - .values(is_read=True) - ) - result = await db_session.execute(stmt) - await db_session.commit() - return int(result.rowcount or 0) - - -async def mark_selected_as_read_for_user( - db_session: AsyncSession, - *, - user_id: int, - selections: list[tuple[int, int]], -) -> int: - normalized_pairs = _normalized_selection_pairs(selections) - if not normalized_pairs: - return 0 - - scholar_ids = _scoped_scholar_ids_query(user_id=user_id) - stmt = ( - update(ScholarPublication) - .where( - ScholarPublication.scholar_profile_id.in_(scholar_ids), - tuple_( - ScholarPublication.scholar_profile_id, - ScholarPublication.publication_id, - ).in_(list(normalized_pairs)), - ScholarPublication.is_read.is_(False), - ) - .values(is_read=True) - ) - result = await db_session.execute(stmt) - await db_session.commit() - return int(result.rowcount or 0) - - -async def set_publication_favorite_for_user( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int, - publication_id: int, - is_favorite: bool, -) -> int: - scholar_ids = _scoped_scholar_ids_query(user_id=user_id) - stmt = ( - update(ScholarPublication) - .where( - ScholarPublication.scholar_profile_id.in_(scholar_ids), - ScholarPublication.scholar_profile_id == int(scholar_profile_id), - ScholarPublication.publication_id == int(publication_id), - ) - .values(is_favorite=bool(is_favorite)) - ) - result = await db_session.execute(stmt) - await db_session.commit() - return int(result.rowcount or 0) diff --git a/build/lib/app/services/domains/publications/types.py b/build/lib/app/services/domains/publications/types.py deleted file mode 100644 index 62d038b..0000000 --- a/build/lib/app/services/domains/publications/types.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime - -from app.services.domains.publication_identifiers.types import DisplayIdentifier - - -@dataclass(frozen=True) -class PublicationListItem: - publication_id: int - scholar_profile_id: int - scholar_label: str - title: str - year: int | None - citation_count: int - venue_text: str | None - pub_url: str | None - pdf_url: str | None - is_read: bool - first_seen_at: datetime - is_new_in_latest_run: bool - is_favorite: bool = False - pdf_status: str = "untracked" - pdf_attempt_count: int = 0 - pdf_failure_reason: str | None = None - pdf_failure_detail: str | None = None - display_identifier: DisplayIdentifier | None = None - - -@dataclass(frozen=True) -class UnreadPublicationItem: - publication_id: int - scholar_profile_id: int - scholar_label: str - title: str - year: int | None - citation_count: int - venue_text: str | None - pub_url: str | None - pdf_url: str | None diff --git a/build/lib/app/services/domains/runs/__init__.py b/build/lib/app/services/domains/runs/__init__.py deleted file mode 100644 index 848f6e0..0000000 --- a/build/lib/app/services/domains/runs/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from app.services.domains.runs.application import * diff --git a/build/lib/app/services/domains/runs/application.py b/build/lib/app/services/domains/runs/application.py deleted file mode 100644 index 0019446..0000000 --- a/build/lib/app/services/domains/runs/application.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import annotations - -from app.services.domains.runs.queue_service import ( - clear_queue_item_for_user, - drop_queue_item_for_user, - get_queue_item_for_user, - list_queue_items_for_user, - queue_status_counts_for_user, - retry_queue_item_for_user, -) -from app.services.domains.runs.runs_service import ( - get_manual_run_by_idempotency_key, - get_run_for_user, - list_recent_runs_for_user, - list_runs_for_user, -) -from app.services.domains.runs.summary import extract_run_summary -from app.services.domains.runs.types import ( - QUEUE_STATUS_DROPPED, - QUEUE_STATUS_QUEUED, - QUEUE_STATUS_RETRYING, - QueueClearResult, - QueueListItem, - QueueTransitionError, -) - -__all__ = [ - "QUEUE_STATUS_QUEUED", - "QUEUE_STATUS_RETRYING", - "QUEUE_STATUS_DROPPED", - "QueueListItem", - "QueueClearResult", - "QueueTransitionError", - "extract_run_summary", - "list_recent_runs_for_user", - "list_runs_for_user", - "get_run_for_user", - "get_manual_run_by_idempotency_key", - "list_queue_items_for_user", - "get_queue_item_for_user", - "retry_queue_item_for_user", - "drop_queue_item_for_user", - "clear_queue_item_for_user", - "queue_status_counts_for_user", -] diff --git a/build/lib/app/services/domains/runs/events.py b/build/lib/app/services/domains/runs/events.py deleted file mode 100644 index ef28069..0000000 --- a/build/lib/app/services/domains/runs/events.py +++ /dev/null @@ -1,59 +0,0 @@ -import asyncio -import json -import logging -from typing import Any, AsyncGenerator, Dict, Set - -logger = logging.getLogger(__name__) - -class RunEventPublisher: - def __init__(self) -> None: - # Maps run_id to a set of subscriber queues - self._subscribers: Dict[int, Set[asyncio.Queue]] = {} - - def subscribe(self, run_id: int) -> asyncio.Queue: - if run_id not in self._subscribers: - self._subscribers[run_id] = set() - queue: asyncio.Queue[Any] = asyncio.Queue() - self._subscribers[run_id].add(queue) - logger.debug(f"New subscriber for run {run_id}. Total: {len(self._subscribers[run_id])}") - return queue - - def unsubscribe(self, run_id: int, queue: asyncio.Queue) -> None: - if run_id in self._subscribers: - self._subscribers[run_id].discard(queue) - if not self._subscribers[run_id]: - self._subscribers.pop(run_id, None) - - async def publish(self, run_id: int, event_type: str, data: dict[str, Any]) -> None: - if run_id not in self._subscribers: - return - - message = { - "type": event_type, - "data": data - } - - # Fan-out to all active subscribers for this run - for queue in list(self._subscribers[run_id]): - try: - queue.put_nowait(message) - except asyncio.QueueFull: - logger.warning(f"Subscriber queue full for run {run_id}, dropping message") - -run_events = RunEventPublisher() - -async def event_generator(run_id: int) -> AsyncGenerator[str, None]: - queue = run_events.subscribe(run_id) - try: - while True: - # Wait for a new event - message = await queue.get() - # Server-Sent Events format: "event: \ndata: \n\n" - event_type = message["type"] - data_str = json.dumps(message["data"]) - yield f"event: {event_type}\ndata: {data_str}\n\n" - except asyncio.CancelledError: - logger.debug(f"Client disconnected from SSE stream for run {run_id}") - raise - finally: - run_events.unsubscribe(run_id, queue) diff --git a/build/lib/app/services/domains/runs/queue_queries.py b/build/lib/app/services/domains/runs/queue_queries.py deleted file mode 100644 index 7d2e501..0000000 --- a/build/lib/app/services/domains/runs/queue_queries.py +++ /dev/null @@ -1,70 +0,0 @@ -from __future__ import annotations - -from sqlalchemy import and_, select - -from app.db.models import IngestionQueueItem, ScholarProfile -from app.services.domains.runs.types import QueueListItem - - -def queue_item_columns() -> tuple: - return ( - IngestionQueueItem.id, - IngestionQueueItem.scholar_profile_id, - ScholarProfile.display_name, - ScholarProfile.scholar_id, - IngestionQueueItem.status, - IngestionQueueItem.reason, - IngestionQueueItem.dropped_reason, - IngestionQueueItem.attempt_count, - IngestionQueueItem.resume_cstart, - IngestionQueueItem.next_attempt_dt, - IngestionQueueItem.updated_at, - IngestionQueueItem.last_error, - IngestionQueueItem.last_run_id, - ) - - -def queue_item_select(*, user_id: int): - return ( - select(*queue_item_columns()) - .join( - ScholarProfile, - and_( - ScholarProfile.id == IngestionQueueItem.scholar_profile_id, - ScholarProfile.user_id == IngestionQueueItem.user_id, - ), - ) - .where(IngestionQueueItem.user_id == user_id) - ) - - -def queue_list_item_from_row(row: tuple) -> QueueListItem: - ( - item_id, - scholar_profile_id, - display_name, - scholar_id, - status, - reason, - dropped_reason, - attempt_count, - resume_cstart, - next_attempt_dt, - updated_at, - last_error, - last_run_id, - ) = row - return QueueListItem( - id=int(item_id), - scholar_profile_id=int(scholar_profile_id), - scholar_label=(display_name or scholar_id), - status=str(status), - reason=str(reason), - dropped_reason=dropped_reason, - attempt_count=int(attempt_count or 0), - resume_cstart=int(resume_cstart or 0), - next_attempt_dt=next_attempt_dt, - updated_at=updated_at, - last_error=last_error, - last_run_id=int(last_run_id) if last_run_id is not None else None, - ) diff --git a/build/lib/app/services/domains/runs/queue_service.py b/build/lib/app/services/domains/runs/queue_service.py deleted file mode 100644 index d69ce21..0000000 --- a/build/lib/app/services/domains/runs/queue_service.py +++ /dev/null @@ -1,182 +0,0 @@ -from __future__ import annotations - -from sqlalchemy import case, func, select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import IngestionQueueItem -from app.services.domains.ingestion import queue as queue_mutations -from app.services.domains.runs.queue_queries import queue_item_select, queue_list_item_from_row -from app.services.domains.runs.types import ( - QUEUE_STATUS_DROPPED, - QUEUE_STATUS_QUEUED, - QUEUE_STATUS_RETRYING, - QueueClearResult, - QueueListItem, - QueueTransitionError, -) - - -async def list_queue_items_for_user( - db_session: AsyncSession, - *, - user_id: int, - limit: int = 200, -) -> list[QueueListItem]: - result = await db_session.execute( - queue_item_select(user_id=user_id) - .order_by( - case((IngestionQueueItem.status == QUEUE_STATUS_DROPPED, 1), else_=0).asc(), - IngestionQueueItem.next_attempt_dt.asc(), - IngestionQueueItem.id.asc(), - ) - .limit(limit) - ) - return [queue_list_item_from_row(row) for row in result.all()] - - -async def get_queue_item_for_user( - db_session: AsyncSession, - *, - user_id: int, - queue_item_id: int, -) -> QueueListItem | None: - result = await db_session.execute( - queue_item_select(user_id=user_id) - .where(IngestionQueueItem.id == queue_item_id) - .limit(1) - ) - row = result.one_or_none() - if row is None: - return None - return queue_list_item_from_row(row) - - -async def retry_queue_item_for_user( - db_session: AsyncSession, - *, - user_id: int, - queue_item_id: int, -) -> QueueListItem | None: - result = await db_session.execute( - select(IngestionQueueItem).where( - IngestionQueueItem.id == queue_item_id, - IngestionQueueItem.user_id == user_id, - ) - ) - item = result.scalar_one_or_none() - if item is None: - return None - if item.status == QUEUE_STATUS_QUEUED: - raise QueueTransitionError( - code="queue_item_already_queued", - message="Queue item is already queued.", - current_status=item.status, - ) - if item.status == QUEUE_STATUS_RETRYING: - raise QueueTransitionError( - code="queue_item_retrying", - message="Queue item is currently retrying.", - current_status=item.status, - ) - - await queue_mutations.mark_queued_now( - db_session, - job_id=item.id, - reason="manual_retry", - reset_attempt_count=(item.status == QUEUE_STATUS_DROPPED), - ) - await db_session.commit() - return await get_queue_item_for_user( - db_session, - user_id=user_id, - queue_item_id=queue_item_id, - ) - - -async def drop_queue_item_for_user( - db_session: AsyncSession, - *, - user_id: int, - queue_item_id: int, -) -> QueueListItem | None: - result = await db_session.execute( - select(IngestionQueueItem).where( - IngestionQueueItem.id == queue_item_id, - IngestionQueueItem.user_id == user_id, - ) - ) - item = result.scalar_one_or_none() - if item is None: - return None - if item.status == QUEUE_STATUS_DROPPED: - raise QueueTransitionError( - code="queue_item_already_dropped", - message="Queue item is already dropped.", - current_status=item.status, - ) - - await queue_mutations.mark_dropped( - db_session, - job_id=int(item.id), - reason="manual_drop", - ) - await db_session.commit() - return await get_queue_item_for_user( - db_session, - user_id=user_id, - queue_item_id=queue_item_id, - ) - - -async def clear_queue_item_for_user( - db_session: AsyncSession, - *, - user_id: int, - queue_item_id: int, -) -> QueueClearResult | None: - result = await db_session.execute( - select(IngestionQueueItem).where( - IngestionQueueItem.id == queue_item_id, - IngestionQueueItem.user_id == user_id, - ) - ) - item = result.scalar_one_or_none() - if item is None: - return None - if item.status != QUEUE_STATUS_DROPPED: - raise QueueTransitionError( - code="queue_item_not_dropped", - message="Queue item can only be cleared after it is dropped.", - current_status=item.status, - ) - - item_id = int(item.id) - previous_status = str(item.status) - deleted = await queue_mutations.delete_job_by_id(db_session, job_id=item_id) - await db_session.commit() - if not deleted: - return None - return QueueClearResult(queue_item_id=item_id, previous_status=previous_status) - - -async def queue_status_counts_for_user( - db_session: AsyncSession, - *, - user_id: int, -) -> dict[str, int]: - result = await db_session.execute( - select( - IngestionQueueItem.status, - func.count(IngestionQueueItem.id), - ) - .where(IngestionQueueItem.user_id == user_id) - .group_by(IngestionQueueItem.status) - ) - counts: dict[str, int] = { - QUEUE_STATUS_QUEUED: 0, - QUEUE_STATUS_RETRYING: 0, - QUEUE_STATUS_DROPPED: 0, - } - for status, count in result.all(): - counts[str(status)] = int(count or 0) - return counts diff --git a/build/lib/app/services/domains/runs/runs_service.py b/build/lib/app/services/domains/runs/runs_service.py deleted file mode 100644 index 1cb9aa2..0000000 --- a/build/lib/app/services/domains/runs/runs_service.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -from sqlalchemy import or_, select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import CrawlRun, RunStatus, RunTriggerType - - -async def list_recent_runs_for_user( - db_session: AsyncSession, - *, - user_id: int, - limit: int = 20, -) -> list[CrawlRun]: - result = await db_session.execute( - select(CrawlRun) - .where(CrawlRun.user_id == user_id) - .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) - .limit(limit) - ) - return list(result.scalars().all()) - - -async def list_runs_for_user( - db_session: AsyncSession, - *, - user_id: int, - limit: int = 100, - failed_only: bool = False, -) -> list[CrawlRun]: - stmt = ( - select(CrawlRun) - .where(CrawlRun.user_id == user_id) - .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) - .limit(limit) - ) - if failed_only: - stmt = stmt.where( - CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE]) - ) - result = await db_session.execute(stmt) - return list(result.scalars().all()) - - -async def get_run_for_user( - db_session: AsyncSession, - *, - user_id: int, - run_id: int, -) -> CrawlRun | None: - result = await db_session.execute( - select(CrawlRun).where( - CrawlRun.user_id == user_id, - CrawlRun.id == run_id, - ) - ) - return result.scalar_one_or_none() - - -async def get_manual_run_by_idempotency_key( - db_session: AsyncSession, - *, - user_id: int, - idempotency_key: str, -) -> CrawlRun | None: - result = await db_session.execute( - select(CrawlRun) - .where( - CrawlRun.user_id == user_id, - CrawlRun.trigger_type == RunTriggerType.MANUAL, - or_( - CrawlRun.idempotency_key == idempotency_key, - CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key, - ), - ) - .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) - .limit(1) - ) - return result.scalar_one_or_none() diff --git a/build/lib/app/services/domains/runs/summary.py b/build/lib/app/services/domains/runs/summary.py deleted file mode 100644 index 9555558..0000000 --- a/build/lib/app/services/domains/runs/summary.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -from typing import Any - - -def _safe_int(value: object, default: int = 0) -> int: - try: - return int(value) - except (TypeError, ValueError): - return default - - -def _summary_dict(error_log: object) -> dict[str, Any]: - if not isinstance(error_log, dict): - return {} - summary = error_log.get("summary") - if not isinstance(summary, dict): - return {} - return summary - - -def _summary_int_dict(summary: dict[str, Any], key: str) -> dict[str, int]: - value = summary.get(key) - if not isinstance(value, dict): - return {} - return { - str(item_key): _safe_int(item_value, 0) - for item_key, item_value in value.items() - if isinstance(item_key, str) - } - - -def _summary_bool_dict(summary: dict[str, Any], key: str) -> dict[str, bool]: - value = summary.get(key) - if not isinstance(value, dict): - return {} - return { - str(item_key): bool(item_value) - for item_key, item_value in value.items() - if isinstance(item_key, str) - } - - -def _retry_counts(summary: dict[str, Any]) -> dict[str, int]: - retry_counts = summary.get("retry_counts") - if not isinstance(retry_counts, dict): - retry_counts = {} - return { - "retries_scheduled_count": _safe_int( - retry_counts.get("retries_scheduled_count", 0), - 0, - ), - "scholars_with_retries_count": _safe_int( - retry_counts.get("scholars_with_retries_count", 0), - 0, - ), - "retry_exhausted_count": _safe_int( - retry_counts.get("retry_exhausted_count", 0), - 0, - ), - } - - -def extract_run_summary(error_log: object) -> dict[str, Any]: - summary = _summary_dict(error_log) - return { - "succeeded_count": _safe_int(summary.get("succeeded_count", 0)), - "failed_count": _safe_int(summary.get("failed_count", 0)), - "partial_count": _safe_int(summary.get("partial_count", 0)), - "failed_state_counts": _summary_int_dict(summary, "failed_state_counts"), - "failed_reason_counts": _summary_int_dict(summary, "failed_reason_counts"), - "scrape_failure_counts": _summary_int_dict(summary, "scrape_failure_counts"), - "retry_counts": _retry_counts(summary), - "alert_thresholds": _summary_int_dict(summary, "alert_thresholds"), - "alert_flags": _summary_bool_dict(summary, "alert_flags"), - } diff --git a/build/lib/app/services/domains/runs/types.py b/build/lib/app/services/domains/runs/types.py deleted file mode 100644 index 2524269..0000000 --- a/build/lib/app/services/domains/runs/types.py +++ /dev/null @@ -1,38 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime - -QUEUE_STATUS_QUEUED = "queued" -QUEUE_STATUS_RETRYING = "retrying" -QUEUE_STATUS_DROPPED = "dropped" - - -@dataclass(frozen=True) -class QueueListItem: - id: int - scholar_profile_id: int - scholar_label: str - status: str - reason: str - dropped_reason: str | None - attempt_count: int - resume_cstart: int - next_attempt_dt: datetime | None - updated_at: datetime - last_error: str | None - last_run_id: int | None - - -@dataclass(frozen=True) -class QueueClearResult: - queue_item_id: int - previous_status: str - - -class QueueTransitionError(RuntimeError): - def __init__(self, *, code: str, message: str, current_status: str) -> None: - super().__init__(message) - self.code = code - self.message = message - self.current_status = current_status diff --git a/build/lib/app/services/domains/scholar/__init__.py b/build/lib/app/services/domains/scholar/__init__.py deleted file mode 100644 index 9d48db4..0000000 --- a/build/lib/app/services/domains/scholar/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from __future__ import annotations diff --git a/build/lib/app/services/domains/scholar/author_rows.py b/build/lib/app/services/domains/scholar/author_rows.py deleted file mode 100644 index 8acd783..0000000 --- a/build/lib/app/services/domains/scholar/author_rows.py +++ /dev/null @@ -1,202 +0,0 @@ -from __future__ import annotations - -import re -from html.parser import HTMLParser -from typing import Any -from urllib.parse import parse_qs, urlparse - -from app.services.domains.scholar.parser_constants import AUTHOR_SEARCH_MARKER_KEYS -from app.services.domains.scholar.parser_types import ScholarSearchCandidate -from app.services.domains.scholar.parser_utils import ( - attr_class, - attr_href, - attr_src, - build_absolute_scholar_url, - normalize_space, -) - - -def parse_scholar_id_from_href(href: str | None) -> str | None: - if not href: - return None - parsed = urlparse(href) - query = parse_qs(parsed.query) - user_values = query.get("user") - if not user_values: - return None - candidate = user_values[0].strip() - return candidate or None - - -def _extract_verified_email_domain(value: str | None) -> str | None: - if not value: - return None - match = re.search(r"verified email at\s+(.+)$", value.strip(), re.I) - if not match: - return None - domain = normalize_space(match.group(1)) - return domain or None - - -class ScholarAuthorSearchParser(HTMLParser): - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.candidates: list[ScholarSearchCandidate] = [] - self._candidate: dict[str, Any] | None = None - - def _begin_candidate(self) -> None: - self._candidate = { - "depth": 1, - "name_href": None, - "name_parts": [], - "aff_depth": 0, - "aff_parts": [], - "name_depth": 0, - "eml_depth": 0, - "eml_parts": [], - "cby_depth": 0, - "cby_parts": [], - "interest_depth": 0, - "interest_parts": [], - "interests": [], - "image_src": None, - } - - def _increment_capture_depths(self) -> None: - if self._candidate is None: - return - for key in ("name_depth", "aff_depth", "eml_depth", "cby_depth", "interest_depth"): - if self._candidate[key] > 0: - self._candidate[key] += 1 - - def _finalize_candidate(self) -> None: - if self._candidate is None: - return - - name = normalize_space("".join(self._candidate["name_parts"])) - scholar_id = parse_scholar_id_from_href(self._candidate["name_href"]) - if not name or not scholar_id: - return - - affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None - email_domain = _extract_verified_email_domain( - normalize_space("".join(self._candidate["eml_parts"])) or None - ) - cited_by_text = normalize_space("".join(self._candidate["cby_parts"])) - cited_by_match = re.search(r"\d+", cited_by_text) - cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None - - seen_interests: set[str] = set() - interests: list[str] = [] - for interest in self._candidate["interests"]: - normalized = normalize_space(interest) - if not normalized or normalized in seen_interests: - continue - seen_interests.add(normalized) - interests.append(normalized) - - profile_url = build_absolute_scholar_url(self._candidate["name_href"]) - if not profile_url: - profile_url = ( - "https://scholar.google.com/citations" - f"?hl=en&user={scholar_id}" - ) - - self.candidates.append( - ScholarSearchCandidate( - scholar_id=scholar_id, - display_name=name, - affiliation=affiliation, - email_domain=email_domain, - cited_by_count=cited_by_count, - interests=interests, - profile_url=profile_url, - profile_image_url=build_absolute_scholar_url(self._candidate["image_src"]), - ) - ) - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - classes = attr_class(attrs) - - if self._candidate is None: - if tag == "div" and "gsc_1usr" in classes: - self._begin_candidate() - return - - self._candidate["depth"] += 1 - self._increment_capture_depths() - - if tag == "a" and "gs_ai_name" in classes: - self._candidate["name_depth"] = 1 - self._candidate["name_href"] = attr_href(attrs) - return - - if tag == "div" and "gs_ai_aff" in classes: - self._candidate["aff_depth"] = 1 - return - - if tag == "div" and "gs_ai_eml" in classes: - self._candidate["eml_depth"] = 1 - return - - if tag == "div" and "gs_ai_cby" in classes: - self._candidate["cby_depth"] = 1 - return - - if tag == "a" and "gs_ai_one_int" in classes: - self._candidate["interest_depth"] = 1 - self._candidate["interest_parts"] = [] - return - - if tag == "img" and self._candidate["image_src"] is None: - self._candidate["image_src"] = attr_src(attrs) - - def handle_data(self, data: str) -> None: - if self._candidate is None: - return - if self._candidate["name_depth"] > 0: - self._candidate["name_parts"].append(data) - if self._candidate["aff_depth"] > 0: - self._candidate["aff_parts"].append(data) - if self._candidate["eml_depth"] > 0: - self._candidate["eml_parts"].append(data) - if self._candidate["cby_depth"] > 0: - self._candidate["cby_parts"].append(data) - if self._candidate["interest_depth"] > 0: - self._candidate["interest_parts"].append(data) - - def _decrement_capture_depth(self, key: str) -> bool: - if self._candidate is None: - return False - if self._candidate[key] <= 0: - return False - self._candidate[key] -= 1 - return self._candidate[key] == 0 - - def handle_endtag(self, _tag: str) -> None: - if self._candidate is None: - return - - interest_closed = self._decrement_capture_depth("interest_depth") - self._decrement_capture_depth("name_depth") - self._decrement_capture_depth("aff_depth") - self._decrement_capture_depth("eml_depth") - self._decrement_capture_depth("cby_depth") - - if interest_closed: - interest_text = normalize_space("".join(self._candidate["interest_parts"])) - if interest_text: - self._candidate["interests"].append(interest_text) - self._candidate["interest_parts"] = [] - - self._candidate["depth"] -= 1 - if self._candidate["depth"] > 0: - return - - self._finalize_candidate() - self._candidate = None - - -def count_author_search_markers(html: str) -> dict[str, int]: - lowered = html.lower() - return {key: lowered.count(key.lower()) for key in AUTHOR_SEARCH_MARKER_KEYS} diff --git a/build/lib/app/services/domains/scholar/parser.py b/build/lib/app/services/domains/scholar/parser.py deleted file mode 100644 index 060e825..0000000 --- a/build/lib/app/services/domains/scholar/parser.py +++ /dev/null @@ -1,165 +0,0 @@ -from __future__ import annotations - -from app.services.domains.scholar.author_rows import ( - ScholarAuthorSearchParser, - count_author_search_markers, - parse_scholar_id_from_href, -) -from app.services.domains.scholar.parser_constants import SCRIPT_STYLE_RE -from app.services.domains.scholar.parser_types import ( - ParseState, - ParsedAuthorSearchPage, - ParsedProfilePage, - PublicationCandidate, - ScholarDomInvariantError, - ScholarMalformedDataError, - ScholarParserError, - ScholarSearchCandidate, -) -from app.services.domains.scholar.parser_utils import ( - strip_tags, -) -from app.services.domains.scholar.profile_rows import ( - ScholarRowParser, - count_markers, - extract_articles_range, - extract_profile_image_url, - extract_profile_name, - extract_rows, - has_operation_error_banner, - has_show_more_button, - parse_citation_count, - parse_cluster_id_from_href, - parse_publications, - parse_year, -) -from app.services.domains.scholar.source import FetchResult -from app.services.domains.scholar.state_detection import ( - classify_block_or_captcha_reason, - classify_network_error_reason, - detect_author_search_state, - detect_state, -) - - -def _raise_dom_error(code: str, message: str) -> None: - raise ScholarDomInvariantError(code=code, message=message) - - -def _assert_profile_dom_invariants( - *, - fetch_result: FetchResult, - marker_counts: dict[str, int], - publications: list[PublicationCandidate], - warnings: list[str], - has_show_more_button_flag: bool, - articles_range: str | None, -) -> None: - if fetch_result.status_code is None: - return - final_url = (fetch_result.final_url or "").lower() - if "accounts.google.com" in final_url or "sorry/index" in final_url: - return - if any(code.startswith("layout_") for code in warnings): - reason = next(code for code in warnings if code.startswith("layout_")) - _raise_dom_error(reason, f"Detected layout warning: {reason}") - if has_show_more_button_flag and not articles_range: - _raise_dom_error( - "layout_show_more_without_articles_range", - "Show-more control exists without an articles range marker.", - ) - if marker_counts.get("gsc_a_tr", 0) > 0 and marker_counts.get("gsc_a_at", 0) <= 0: - _raise_dom_error( - "layout_missing_publication_title_anchor", - "Publication rows were present but title anchors were absent.", - ) - if not publications: - has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0 - has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0 - if not has_profile_markers and not has_table_markers: - _raise_dom_error("layout_markers_missing", "Expected scholar profile markers were absent.") - for publication in publications: - if not publication.title.strip(): - raise ScholarMalformedDataError( - code="malformed_publication_title", - message="Encountered a publication candidate with an empty title.", - ) - if publication.citation_count is not None and int(publication.citation_count) < 0: - raise ScholarMalformedDataError( - code="malformed_publication_negative_citations", - message="Encountered a publication candidate with negative citations.", - ) - - -def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage: - publications, warnings = parse_publications(fetch_result.body) - marker_counts = count_markers(fetch_result.body) - visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower() - - show_more = has_show_more_button(fetch_result.body) - operation_error_banner = has_operation_error_banner(fetch_result.body) - articles_range = extract_articles_range(fetch_result.body) - - if show_more: - warnings.append("possible_partial_page_show_more_present") - if operation_error_banner: - warnings.append("operation_error_banner_present") - - warnings = sorted(set(warnings)) - _assert_profile_dom_invariants( - fetch_result=fetch_result, - marker_counts=marker_counts, - publications=publications, - warnings=warnings, - has_show_more_button_flag=show_more, - articles_range=articles_range, - ) - - state, state_reason = detect_state( - fetch_result, - publications, - marker_counts, - warnings=warnings, - has_show_more_button_flag=show_more, - articles_range=articles_range, - visible_text=visible_text, - ) - - return ParsedProfilePage( - state=state, - state_reason=state_reason, - profile_name=extract_profile_name(fetch_result.body), - profile_image_url=extract_profile_image_url(fetch_result.body), - publications=publications, - marker_counts=marker_counts, - warnings=warnings, - has_show_more_button=show_more, - has_operation_error_banner=operation_error_banner, - articles_range=articles_range, - ) - - -def parse_author_search_page(fetch_result: FetchResult) -> ParsedAuthorSearchPage: - parser = ScholarAuthorSearchParser() - parser.feed(fetch_result.body) - - marker_counts = count_author_search_markers(fetch_result.body) - visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower() - warnings: list[str] = [] - if not parser.candidates: - warnings.append("no_author_candidates_detected") - - state, state_reason = detect_author_search_state( - fetch_result, - parser.candidates, - marker_counts, - visible_text=visible_text, - ) - - return ParsedAuthorSearchPage( - state=state, - state_reason=state_reason, - candidates=parser.candidates, - marker_counts=marker_counts, - warnings=warnings, - ) diff --git a/build/lib/app/services/domains/scholar/parser_constants.py b/build/lib/app/services/domains/scholar/parser_constants.py deleted file mode 100644 index c3c7a15..0000000 --- a/build/lib/app/services/domains/scholar/parser_constants.py +++ /dev/null @@ -1,87 +0,0 @@ -from __future__ import annotations - -import re - -BLOCKED_KEYWORDS = [ - "unusual traffic", - "sorry/index", - "not a robot", - "our systems have detected", - "automated queries", - "recaptcha", - "captcha", -] - -NO_RESULTS_KEYWORDS = [ - "didn't match any articles", - "did not match any articles", - "no articles", - "no documents", -] - -NO_AUTHOR_RESULTS_KEYWORDS = [ - "didn't match any user profiles", - "did not match any user profiles", - "didn't match any scholars", - "did not match any scholars", - "no user profiles", -] - -MARKER_KEYS = [ - "gsc_a_tr", - "gsc_a_at", - "gsc_a_ac", - "gsc_a_h", - "gsc_a_y", - "gs_gray", - "gsc_prf_in", - "gsc_rsb_st", -] - -AUTHOR_SEARCH_MARKER_KEYS = [ - "gsc_1usr", - "gs_ai_name", - "gs_ai_aff", - "gs_ai_eml", - "gs_ai_cby", - "gs_ai_one_int", -] - -NETWORK_DNS_ERROR_KEYWORDS = [ - "temporary failure in name resolution", - "name or service not known", - "nodename nor servname provided", - "getaddrinfo failed", -] - -NETWORK_TIMEOUT_KEYWORDS = [ - "timed out", - "timeout", -] - -NETWORK_TLS_ERROR_KEYWORDS = [ - "ssl", - "tls", - "certificate verify failed", -] - -TAG_RE = re.compile(r"<[^>]+>", re.S) -SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?", re.I | re.S) -SHOW_MORE_BUTTON_RE = re.compile( - r"]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>", - re.I | re.S, -) - -PROFILE_ROW_PARSER_DIRECT_MARKERS = ( - "gs_ggs", - "gs_ggsd", - "gs_ggsa", - "gs_or_ggsm", -) - -PROFILE_ROW_DIRECT_LABEL_TOKENS = ( - "pdf", - "[pdf]", - "full text", - "download", -) diff --git a/build/lib/app/services/domains/scholar/parser_types.py b/build/lib/app/services/domains/scholar/parser_types.py deleted file mode 100644 index 7b657d8..0000000 --- a/build/lib/app/services/domains/scholar/parser_types.py +++ /dev/null @@ -1,75 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from enum import StrEnum - - -class ParseState(StrEnum): - OK = "ok" - NO_RESULTS = "no_results" - BLOCKED_OR_CAPTCHA = "blocked_or_captcha" - LAYOUT_CHANGED = "layout_changed" - NETWORK_ERROR = "network_error" - - -class ScholarParserError(RuntimeError): - code: str - - def __init__(self, *, code: str, message: str) -> None: - super().__init__(message) - self.code = code - - -class ScholarDomInvariantError(ScholarParserError): - pass - - -class ScholarMalformedDataError(ScholarParserError): - pass - - -@dataclass(frozen=True) -class PublicationCandidate: - title: str - title_url: str | None - cluster_id: str | None - year: int | None - citation_count: int | None - authors_text: str | None - venue_text: str | None - pdf_url: str | None - - -@dataclass(frozen=True) -class ScholarSearchCandidate: - scholar_id: str - display_name: str - affiliation: str | None - email_domain: str | None - cited_by_count: int | None - interests: list[str] - profile_url: str - profile_image_url: str | None - - -@dataclass(frozen=True) -class ParsedProfilePage: - state: ParseState - state_reason: str - profile_name: str | None - profile_image_url: str | None - publications: list[PublicationCandidate] - marker_counts: dict[str, int] - warnings: list[str] - has_show_more_button: bool - has_operation_error_banner: bool - articles_range: str | None - - -@dataclass(frozen=True) -class ParsedAuthorSearchPage: - state: ParseState - state_reason: str - candidates: list[ScholarSearchCandidate] - marker_counts: dict[str, int] - warnings: list[str] diff --git a/build/lib/app/services/domains/scholar/parser_utils.py b/build/lib/app/services/domains/scholar/parser_utils.py deleted file mode 100644 index 1bf8687..0000000 --- a/build/lib/app/services/domains/scholar/parser_utils.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import annotations - -from html import unescape - -from app.services.domains.scholar.parser_constants import TAG_RE - - -def normalize_space(value: str) -> str: - return " ".join(unescape(value).split()) - - -def strip_tags(value: str) -> str: - return normalize_space(TAG_RE.sub(" ", value)) - - -def attr_class(attrs: list[tuple[str, str | None]]) -> str: - for name, raw_value in attrs: - if name.lower() == "class": - return raw_value or "" - return "" - - -def attr_href(attrs: list[tuple[str, str | None]]) -> str | None: - for name, raw_value in attrs: - if name.lower() == "href": - return raw_value - return None - - -def attr_src(attrs: list[tuple[str, str | None]]) -> str | None: - for name, raw_value in attrs: - if name.lower() == "src": - return raw_value - return None - - -def build_absolute_scholar_url(path_or_url: str | None) -> str | None: - if not path_or_url: - return None - from urllib.parse import urljoin - - return urljoin("https://scholar.google.com", path_or_url) diff --git a/build/lib/app/services/domains/scholar/profile_rows.py b/build/lib/app/services/domains/scholar/profile_rows.py deleted file mode 100644 index 6a63274..0000000 --- a/build/lib/app/services/domains/scholar/profile_rows.py +++ /dev/null @@ -1,270 +0,0 @@ -from __future__ import annotations - -import re -from html.parser import HTMLParser -from urllib.parse import parse_qs, urlparse - -from app.services.domains.scholar.parser_constants import ( - MARKER_KEYS, - SHOW_MORE_BUTTON_RE, -) -from app.services.domains.scholar.parser_types import PublicationCandidate -from app.services.domains.scholar.parser_utils import ( - attr_class, - attr_href, - attr_src, - build_absolute_scholar_url, - normalize_space, - strip_tags, -) - - -class ScholarRowParser(HTMLParser): - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.title_href: str | None = None - self.title_parts: list[str] = [] - self.citation_parts: list[str] = [] - self.year_parts: list[str] = [] - self.gray_texts: list[str] = [] - - self._title_depth = 0 - self._citation_depth = 0 - self._year_depth = 0 - self._gray_stack: list[dict[str, object]] = [] - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - if self._title_depth > 0: - self._title_depth += 1 - if self._citation_depth > 0: - self._citation_depth += 1 - if self._year_depth > 0: - self._year_depth += 1 - if self._gray_stack: - self._gray_stack[-1]["depth"] += 1 - - classes = attr_class(attrs) - - if tag == "a" and "gsc_a_at" in classes: - self._title_depth = 1 - self.title_href = attr_href(attrs) - return - - if tag == "a" and "gsc_a_ac" in classes: - self._citation_depth = 1 - return - - if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes): - self._year_depth = 1 - return - - if tag == "div" and "gs_gray" in classes: - self._gray_stack.append({"depth": 1, "parts": []}) - return - - def handle_data(self, data: str) -> None: - if self._title_depth > 0: - self.title_parts.append(data) - if self._citation_depth > 0: - self.citation_parts.append(data) - if self._year_depth > 0: - self.year_parts.append(data) - if self._gray_stack: - self._gray_stack[-1]["parts"].append(data) - - def handle_endtag(self, tag: str) -> None: - if self._title_depth > 0: - self._title_depth -= 1 - if self._citation_depth > 0: - self._citation_depth -= 1 - if self._year_depth > 0: - self._year_depth -= 1 - if self._gray_stack: - self._gray_stack[-1]["depth"] -= 1 - if self._gray_stack[-1]["depth"] == 0: - text_value = normalize_space("".join(self._gray_stack[-1]["parts"])) - if text_value: - self.gray_texts.append(text_value) - self._gray_stack.pop() - - -def extract_rows(html: str) -> list[str]: - pattern = re.compile( - r"]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)", - re.I | re.S, - ) - return [match.group(1) for match in pattern.finditer(html)] - - -def parse_cluster_id_from_href(href: str | None) -> str | None: - if not href: - return None - parsed = urlparse(href) - query = parse_qs(parsed.query) - - citation_for_view = query.get("citation_for_view") - if citation_for_view: - token = citation_for_view[0].strip() - if token: - return f"cfv:{token}" - - cluster = query.get("cluster") - if cluster: - token = cluster[0].strip() - if token: - return f"cluster:{token}" - return None - - -def parse_year(parts: list[str]) -> int | None: - text = normalize_space(" ".join(parts)) - match = re.search(r"\b(19|20)\d{2}\b", text) - if not match: - return None - try: - return int(match.group(0)) - except ValueError: - return None - - -def parse_citation_count(parts: list[str]) -> int | None: - text = normalize_space(" ".join(parts)) - if not text: - return 0 - digits = re.sub(r"\D+", "", text) - if not digits: - return None - return int(digits) - - -def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None, list[str]]: - parser = ScholarRowParser() - parser.feed(row_html) - warnings: list[str] = [] - title = normalize_space("".join(parser.title_parts)) - if not title: - warnings.append("row_missing_title") - return None, warnings - if not parser.title_href: - warnings.append("row_missing_title_href") - - citation_text = normalize_space(" ".join(parser.citation_parts)) - citation_count = parse_citation_count(parser.citation_parts) - if citation_text and citation_count is None: - warnings.append("layout_row_citation_unparseable") - - year_text = normalize_space(" ".join(parser.year_parts)) - year = parse_year(parser.year_parts) - if year_text and year is None: - warnings.append("layout_row_year_unparseable") - - authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None - venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None - return ( - PublicationCandidate( - title=title, - title_url=parser.title_href, - cluster_id=parse_cluster_id_from_href(parser.title_href), - year=year, - citation_count=citation_count, - authors_text=authors_text, - venue_text=venue_text, - pdf_url=None, - ), - warnings, - ) - - -def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]: - rows = extract_rows(html) - warnings: list[str] = [] - publications: list[PublicationCandidate] = [] - - for row_html in rows: - publication, row_warnings = _parse_publication_row(row_html) - warnings.extend(row_warnings) - if publication is None: - continue - publications.append(publication) - - if not rows: - warnings.append("no_rows_detected") - if rows and not publications: - warnings.append("layout_all_rows_unparseable") - - return publications, sorted(set(warnings)) - - -def extract_profile_name(html: str) -> str | None: - pattern = re.compile( - r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)]+>", - re.I | re.S, - ) - match = pattern.search(html) - if not match: - return None - value = strip_tags(match.group(1)) - return value or None - - -def extract_profile_image_url(html: str) -> str | None: - og_image_pattern = re.compile( - r"]+property=['\"]og:image['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>", - re.I | re.S, - ) - og_match = og_image_pattern.search(html) - if og_match: - value = normalize_space(og_match.group(1)) - absolute = build_absolute_scholar_url(value) - if absolute: - return absolute - - image_pattern = re.compile( - r"]*\bid=['\"]gsc_prf_pup-img['\"][^>]*\bsrc=['\"]([^'\"]+)['\"][^>]*>", - re.I | re.S, - ) - image_match = image_pattern.search(html) - if not image_match: - return None - - value = normalize_space(image_match.group(1)) - return build_absolute_scholar_url(value) - - -def extract_articles_range(html: str) -> str | None: - pattern = re.compile( - r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)]+>", - re.I | re.S, - ) - match = pattern.search(html) - if not match: - return None - value = strip_tags(match.group(1)) - return value or None - - -def has_show_more_button(html: str) -> bool: - match = SHOW_MORE_BUTTON_RE.search(html) - if match is None: - return False - - button_tag = match.group(0).lower() - if "disabled" in button_tag: - return False - if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag: - return False - if "gs_dis" in button_tag: - return False - return True - - -def has_operation_error_banner(html: str) -> bool: - lowered = html.lower() - if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered: - return False - return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered - - -def count_markers(html: str) -> dict[str, int]: - lowered = html.lower() - return {key: lowered.count(key.lower()) for key in MARKER_KEYS} diff --git a/build/lib/app/services/domains/scholar/rate_limit.py b/build/lib/app/services/domains/scholar/rate_limit.py deleted file mode 100644 index 947c91f..0000000 --- a/build/lib/app/services/domains/scholar/rate_limit.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import annotations - -import asyncio -import time - -_REQUEST_LOCK = asyncio.Lock() -_LAST_REQUEST_AT = 0.0 - - -async def wait_for_scholar_slot(*, min_interval_seconds: float) -> None: - global _LAST_REQUEST_AT - interval = max(float(min_interval_seconds), 0.0) - async with _REQUEST_LOCK: - elapsed = time.monotonic() - _LAST_REQUEST_AT - remaining = interval - elapsed - if remaining > 0: - await asyncio.sleep(remaining) - _LAST_REQUEST_AT = time.monotonic() diff --git a/build/lib/app/services/domains/scholar/source.py b/build/lib/app/services/domains/scholar/source.py deleted file mode 100644 index 92c6702..0000000 --- a/build/lib/app/services/domains/scholar/source.py +++ /dev/null @@ -1,235 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging -import random -from dataclasses import dataclass -from typing import Protocol -from urllib.error import HTTPError, URLError -from urllib.parse import urlencode -from urllib.request import Request, urlopen - -SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations" -DEFAULT_PAGE_SIZE = 100 - -DEFAULT_USER_AGENTS = [ - ( - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" - ), - ( - "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 " - "(KHTML, like Gecko) Version/18.1 Safari/605.1.15" - ), - ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) " - "Gecko/20100101 Firefox/131.0" - ), -] - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class FetchResult: - requested_url: str - status_code: int | None - final_url: str | None - body: str - error: str | None - - -class ScholarSource(Protocol): - async def fetch_profile_html(self, scholar_id: str) -> FetchResult: - ... - - async def fetch_profile_page_html( - self, - scholar_id: str, - *, - cstart: int, - pagesize: int, - ) -> FetchResult: - ... - - async def fetch_author_search_html( - self, - query: str, - *, - start: int, - ) -> FetchResult: - ... - - -class LiveScholarSource: - def __init__( - self, - *, - timeout_seconds: float = 25.0, - user_agents: list[str] | None = None, - ) -> None: - self._timeout_seconds = timeout_seconds - self._user_agents = user_agents or DEFAULT_USER_AGENTS - - async def fetch_profile_html(self, scholar_id: str) -> FetchResult: - return await self.fetch_profile_page_html( - scholar_id, - cstart=0, - pagesize=DEFAULT_PAGE_SIZE, - ) - - async def fetch_profile_page_html( - self, - scholar_id: str, - *, - cstart: int, - pagesize: int = DEFAULT_PAGE_SIZE, - ) -> FetchResult: - requested_url = _build_profile_url( - scholar_id=scholar_id, - cstart=cstart, - pagesize=pagesize, - ) - logger.debug( - "scholar_source.fetch_started", - extra={ - "event": "scholar_source.fetch_started", - "scholar_id": scholar_id, - "requested_url": requested_url, - "cstart": cstart, - "pagesize": pagesize, - }, - ) - return await asyncio.to_thread(self._fetch_sync, requested_url) - - async def fetch_author_search_html( - self, - query: str, - *, - start: int = 0, - ) -> FetchResult: - requested_url = _build_author_search_url( - query=query, - start=start, - ) - logger.debug( - "scholar_source.search_fetch_started", - extra={ - "event": "scholar_source.search_fetch_started", - "query": query, - "requested_url": requested_url, - "start": start, - }, - ) - return await asyncio.to_thread(self._fetch_sync, requested_url) - - async def fetch_publication_html(self, publication_url: str) -> FetchResult: - logger.debug( - "scholar_source.publication_fetch_started", - extra={ - "event": "scholar_source.publication_fetch_started", - "requested_url": publication_url, - }, - ) - return await asyncio.to_thread(self._fetch_sync, publication_url) - - def _build_request(self, requested_url: str) -> Request: - return Request( - requested_url, - headers={ - "User-Agent": random.choice(self._user_agents), - "Accept": "text/html,application/xhtml+xml", - "Accept-Language": "en-US,en;q=0.9", - "Connection": "close", - }, - ) - - @staticmethod - def _http_error_body(exc: HTTPError) -> str: - try: - return exc.read().decode("utf-8", errors="replace") - except Exception: - return "" - - @staticmethod - def _network_error_result(requested_url: str, exc: URLError) -> FetchResult: - logger.warning( - "scholar_source.fetch_network_error", - extra={"event": "scholar_source.fetch_network_error", "requested_url": requested_url}, - ) - return FetchResult( - requested_url=requested_url, - status_code=None, - final_url=None, - body="", - error=str(exc), - ) - - @staticmethod - def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult: - logger.warning( - "scholar_source.fetch_http_error", - extra={ - "event": "scholar_source.fetch_http_error", - "requested_url": requested_url, - "status_code": exc.code, - }, - ) - return FetchResult( - requested_url=requested_url, - status_code=exc.code, - final_url=exc.geturl(), - body=LiveScholarSource._http_error_body(exc), - error=str(exc), - ) - - @staticmethod - def _success_result(requested_url: str, response) -> FetchResult: - body = response.read().decode("utf-8", errors="replace") - status_code = getattr(response, "status", 200) - logger.debug( - "scholar_source.fetch_succeeded", - extra={ - "event": "scholar_source.fetch_succeeded", - "requested_url": requested_url, - "status_code": status_code, - }, - ) - return FetchResult( - requested_url=requested_url, - status_code=status_code, - final_url=response.geturl(), - body=body, - error=None, - ) - - def _fetch_sync(self, requested_url: str) -> FetchResult: - request = self._build_request(requested_url) - - try: - with urlopen(request, timeout=self._timeout_seconds) as response: - return self._success_result(requested_url, response) - except HTTPError as exc: - return self._http_error_result(requested_url, exc) - except URLError as exc: - return self._network_error_result(requested_url, exc) - - -def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str: - query: dict[str, int | str] = {"hl": "en", "user": scholar_id} - if cstart > 0: - query["cstart"] = int(cstart) - if pagesize > 0: - query["pagesize"] = int(pagesize) - return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}" - - -def _build_author_search_url(*, query: str, start: int) -> str: - params: dict[str, int | str] = { - "hl": "en", - "view_op": "search_authors", - "mauthors": query, - } - if start > 0: - params["astart"] = int(start) - return f"{SCHOLAR_PROFILE_URL}?{urlencode(params)}" diff --git a/build/lib/app/services/domains/scholar/state_detection.py b/build/lib/app/services/domains/scholar/state_detection.py deleted file mode 100644 index cdc8780..0000000 --- a/build/lib/app/services/domains/scholar/state_detection.py +++ /dev/null @@ -1,164 +0,0 @@ -from __future__ import annotations - -from app.services.domains.scholar.parser_constants import ( - BLOCKED_KEYWORDS, - NETWORK_DNS_ERROR_KEYWORDS, - NETWORK_TIMEOUT_KEYWORDS, - NETWORK_TLS_ERROR_KEYWORDS, - NO_AUTHOR_RESULTS_KEYWORDS, - NO_RESULTS_KEYWORDS, -) -from app.services.domains.scholar.parser_types import ParseState, PublicationCandidate, ScholarSearchCandidate -from app.services.domains.scholar.source import FetchResult - - -def classify_network_error_reason(fetch_error: str | None) -> str: - lowered = (fetch_error or "").lower() - if lowered: - if any(keyword in lowered for keyword in NETWORK_DNS_ERROR_KEYWORDS): - return "network_dns_resolution_failed" - if any(keyword in lowered for keyword in NETWORK_TIMEOUT_KEYWORDS): - return "network_timeout" - if any(keyword in lowered for keyword in NETWORK_TLS_ERROR_KEYWORDS): - return "network_tls_error" - if "connection reset" in lowered: - return "network_connection_reset" - if "connection refused" in lowered: - return "network_connection_refused" - if "network is unreachable" in lowered: - return "network_unreachable" - return "network_error_missing_status_code" - - -def classify_block_or_captcha_reason( - *, - status_code: int, - final_url: str, - body_lowered: str, -) -> str | None: - if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url): - return "blocked_accounts_redirect" - if status_code == 429: - return "blocked_http_429_rate_limited" - if status_code == 403: - if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url: - return "blocked_http_403_captcha_challenge" - return "blocked_http_403_forbidden" - if "sorry/index" in final_url or "sorry/index" in body_lowered: - return "blocked_google_sorry_challenge" - if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered: - return "blocked_unusual_traffic_detected" - if "automated queries" in body_lowered: - return "blocked_automated_queries_detected" - if "not a robot" in body_lowered: - return "blocked_not_a_robot_challenge" - if "recaptcha" in body_lowered: - return "blocked_recaptcha_challenge" - if "captcha" in body_lowered: - return "blocked_captcha_challenge" - if any(keyword in body_lowered for keyword in BLOCKED_KEYWORDS): - return "blocked_keyword_detected" - return None - - -def _warnings_contain(warnings: list[str], code: str) -> bool: - return any(item == code for item in warnings) - - -def _has_layout_row_failure(marker_counts: dict[str, int], warnings: list[str]) -> bool: - if _warnings_contain(warnings, "layout_all_rows_unparseable"): - return True - if marker_counts.get("gsc_a_tr", 0) <= 0: - return False - if _warnings_contain(warnings, "row_missing_title"): - return True - return marker_counts.get("gsc_a_at", 0) <= 0 - - -def _first_layout_warning(warnings: list[str]) -> str | None: - for warning in warnings: - if warning.startswith("layout_"): - return warning - return None - - -def detect_state( - fetch_result: FetchResult, - publications: list[PublicationCandidate], - marker_counts: dict[str, int], - *, - warnings: list[str], - has_show_more_button_flag: bool, - articles_range: str | None, - visible_text: str, -) -> tuple[ParseState, str]: - if fetch_result.status_code is None: - return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error) - - lowered = fetch_result.body.lower() - final = (fetch_result.final_url or "").lower() - status_code = int(fetch_result.status_code) - - block_reason = classify_block_or_captcha_reason( - status_code=status_code, - final_url=final, - body_lowered=lowered, - ) - if block_reason is not None: - return ParseState.BLOCKED_OR_CAPTCHA, block_reason - - if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS): - return ParseState.NO_RESULTS, "no_results_keyword_detected" - - layout_warning = _first_layout_warning(warnings) - if layout_warning is not None: - return ParseState.LAYOUT_CHANGED, layout_warning - - if _has_layout_row_failure(marker_counts, warnings): - return ParseState.LAYOUT_CHANGED, "layout_publication_rows_unparseable" - - if has_show_more_button_flag and not articles_range: - return ParseState.LAYOUT_CHANGED, "layout_show_more_without_articles_range" - - if not publications: - has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0 - has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0 - if not has_profile_markers and not has_table_markers: - return ParseState.LAYOUT_CHANGED, "layout_markers_missing" - return ParseState.OK, "no_rows_with_known_markers" - - return ParseState.OK, "publications_extracted" - - -def detect_author_search_state( - fetch_result: FetchResult, - candidates: list[ScholarSearchCandidate], - marker_counts: dict[str, int], - *, - visible_text: str, -) -> tuple[ParseState, str]: - if fetch_result.status_code is None: - return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error) - - lowered = fetch_result.body.lower() - final = (fetch_result.final_url or "").lower() - status_code = int(fetch_result.status_code) - - block_reason = classify_block_or_captcha_reason( - status_code=status_code, - final_url=final, - body_lowered=lowered, - ) - if block_reason is not None: - return ParseState.BLOCKED_OR_CAPTCHA, block_reason - - if not candidates and any(keyword in visible_text for keyword in NO_AUTHOR_RESULTS_KEYWORDS): - return ParseState.NO_RESULTS, "no_results_keyword_detected" - - if not candidates: - has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0 - if not has_search_markers: - return ParseState.NO_RESULTS, "no_search_candidates_detected" - return ParseState.LAYOUT_CHANGED, "layout_author_candidates_unparseable" - - return ParseState.OK, "author_candidates_extracted" diff --git a/build/lib/app/services/domains/scholars/__init__.py b/build/lib/app/services/domains/scholars/__init__.py deleted file mode 100644 index 9d48db4..0000000 --- a/build/lib/app/services/domains/scholars/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from __future__ import annotations diff --git a/build/lib/app/services/domains/scholars/application.py b/build/lib/app/services/domains/scholars/application.py deleted file mode 100644 index 8da041f..0000000 --- a/build/lib/app/services/domains/scholars/application.py +++ /dev/null @@ -1,974 +0,0 @@ -from __future__ import annotations - -import asyncio -from dataclasses import replace -from datetime import datetime -from datetime import timedelta -from datetime import timezone -import logging -import os -import random -from uuid import uuid4 - -from sqlalchemy import delete, func, select, text -from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import AuthorSearchCacheEntry, AuthorSearchRuntimeState, ScholarProfile -from app.services.domains.scholar.parser import ( - ParseState, - ParsedAuthorSearchPage, - ScholarParserError, - parse_author_search_page, - parse_profile_page, -) -from app.services.domains.scholar.source import ScholarSource -from app.services.domains.scholars.constants import ( - ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES, - AUTHOR_SEARCH_LOCK_KEY, - AUTHOR_SEARCH_LOCK_NAMESPACE, - AUTHOR_SEARCH_RUNTIME_STATE_KEY, - DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS, - DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES, - DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD, - DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD, - DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS, - DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS, - MAX_AUTHOR_SEARCH_LIMIT, - DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS, - DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, - SEARCH_CACHED_BLOCK_REASON, - SEARCH_COOLDOWN_REASON, - SEARCH_DISABLED_REASON, -) -from app.services.domains.scholars.exceptions import ScholarServiceError -from app.services.domains.scholars.search_hints import ( - _merge_warnings, - _policy_blocked_author_search_result, - _trim_author_search_result, - resolve_profile_image, - scrape_state_hint, -) -from app.services.domains.scholars.uploads import ( - _ensure_upload_root, - _resolve_upload_path, - _safe_remove_upload, - resolve_upload_file_path, -) -from app.services.domains.scholars.validators import ( - normalize_display_name, - normalize_profile_image_url, - validate_scholar_id, -) -logger = logging.getLogger(__name__) - - -async def _acquire_author_search_lock(db_session: AsyncSession) -> None: - await db_session.execute( - text("SELECT pg_advisory_xact_lock(:namespace, :lock_key)"), - { - "namespace": AUTHOR_SEARCH_LOCK_NAMESPACE, - "lock_key": AUTHOR_SEARCH_LOCK_KEY, - }, - ) - - -async def _load_runtime_state_for_update( - db_session: AsyncSession, -) -> AuthorSearchRuntimeState: - result = await db_session.execute( - select(AuthorSearchRuntimeState) - .where(AuthorSearchRuntimeState.state_key == AUTHOR_SEARCH_RUNTIME_STATE_KEY) - .with_for_update() - ) - state = result.scalar_one_or_none() - if state is not None: - return state - - state = AuthorSearchRuntimeState(state_key=AUTHOR_SEARCH_RUNTIME_STATE_KEY) - db_session.add(state) - await db_session.flush() - return state - - -def _serialize_parsed_author_search_page(parsed: ParsedAuthorSearchPage) -> dict: - return { - "state": parsed.state.value, - "state_reason": parsed.state_reason, - "marker_counts": {str(key): int(value) for key, value in parsed.marker_counts.items()}, - "warnings": [str(value) for value in parsed.warnings], - "candidates": [ - { - "scholar_id": candidate.scholar_id, - "display_name": candidate.display_name, - "affiliation": candidate.affiliation, - "email_domain": candidate.email_domain, - "cited_by_count": candidate.cited_by_count, - "interests": [str(interest) for interest in candidate.interests], - "profile_url": candidate.profile_url, - "profile_image_url": candidate.profile_image_url, - } - for candidate in parsed.candidates - ], - } - - -def _payload_state(payload: dict[str, object]) -> ParseState | None: - state_raw = str(payload.get("state", "")).strip() - try: - return ParseState(state_raw) - except ValueError: - return None - - -def _payload_marker_counts(payload: dict[str, object]) -> dict[str, int]: - marker_counts_payload = payload.get("marker_counts") - if not isinstance(marker_counts_payload, dict): - return {} - parsed: dict[str, int] = {} - for key, value in marker_counts_payload.items(): - try: - parsed[str(key)] = int(value) - except (TypeError, ValueError): - continue - return parsed - - -def _payload_warnings(payload: dict[str, object]) -> list[str]: - warnings_payload = payload.get("warnings") - if not isinstance(warnings_payload, list): - return [] - return [str(value) for value in warnings_payload if isinstance(value, str)] - - -def _parse_optional_string(value: object) -> str | None: - if value is None: - return None - normalized = str(value).strip() - return normalized or None - - -def _parse_optional_int(value: object) -> int | None: - if isinstance(value, int): - return value - if isinstance(value, str) and value.strip(): - try: - return int(value) - except ValueError: - return None - return None - - -def _normalize_interests(value: object) -> list[str]: - if not isinstance(value, list): - return [] - return [str(item) for item in value if isinstance(item, str)] - - -def _deserialize_candidate_payload(value: object) -> dict[str, object] | None: - if not isinstance(value, dict): - return None - scholar_id = str(value.get("scholar_id", "")).strip() - display_name = str(value.get("display_name", "")).strip() - profile_url = str(value.get("profile_url", "")).strip() - if not scholar_id or not display_name or not profile_url: - return None - return { - "scholar_id": scholar_id, - "display_name": display_name, - "affiliation": _parse_optional_string(value.get("affiliation")), - "email_domain": _parse_optional_string(value.get("email_domain")), - "cited_by_count": _parse_optional_int(value.get("cited_by_count")), - "interests": _normalize_interests(value.get("interests")), - "profile_url": profile_url, - "profile_image_url": _parse_optional_string(value.get("profile_image_url")), - } - - -def _deserialize_candidates(payload: dict[str, object]) -> list[dict[str, object]]: - candidates_payload = payload.get("candidates") - if not isinstance(candidates_payload, list): - return [] - normalized: list[dict[str, object]] = [] - for value in candidates_payload: - candidate = _deserialize_candidate_payload(value) - if candidate is not None: - normalized.append(candidate) - return normalized - - -def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearchPage | None: - if not isinstance(payload, dict): - return None - - state = _payload_state(payload) - if state is None: - return None - - marker_counts = _payload_marker_counts(payload) - warnings = _payload_warnings(payload) - from app.services.domains.scholar.parser import ScholarSearchCandidate - - normalized_candidates = _deserialize_candidates(payload) - - return ParsedAuthorSearchPage( - state=state, - state_reason=str(payload.get("state_reason", "")).strip() or "unknown", - candidates=[ - ScholarSearchCandidate( - scholar_id=item["scholar_id"], - display_name=item["display_name"], - affiliation=item["affiliation"], - email_domain=item["email_domain"], - cited_by_count=item["cited_by_count"], - interests=item["interests"], - profile_url=item["profile_url"], - profile_image_url=item["profile_image_url"], - ) - for item in normalized_candidates - ], - marker_counts=marker_counts, - warnings=warnings, - ) - - -async def _cache_get_author_search_result( - db_session: AsyncSession, - *, - query_key: str, - now_utc: datetime, -) -> ParsedAuthorSearchPage | None: - result = await db_session.execute( - select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key) - ) - entry = result.scalar_one_or_none() - if entry is None: - return None - expires_at = entry.expires_at - if expires_at.tzinfo is None: - expires_at = expires_at.replace(tzinfo=timezone.utc) - if expires_at <= now_utc: - await db_session.delete(entry) - return None - parsed = _deserialize_parsed_author_search_page(entry.payload) - if parsed is None: - await db_session.delete(entry) - return None - return parsed - - -async def _cache_set_author_search_result( - db_session: AsyncSession, - *, - query_key: str, - parsed: ParsedAuthorSearchPage, - ttl_seconds: float, - max_entries: int, - now_utc: datetime, -) -> None: - ttl = max(float(ttl_seconds), 0.0) - existing_result = await db_session.execute( - select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key) - ) - existing = existing_result.scalar_one_or_none() - - if ttl <= 0.0: - if existing is not None: - await db_session.delete(existing) - return - - expires_at = now_utc + timedelta(seconds=ttl) - payload = _serialize_parsed_author_search_page(parsed) - if existing is None: - db_session.add( - AuthorSearchCacheEntry( - query_key=query_key, - payload=payload, - expires_at=expires_at, - cached_at=now_utc, - updated_at=now_utc, - ) - ) - else: - existing.payload = payload - existing.expires_at = expires_at - existing.cached_at = now_utc - existing.updated_at = now_utc - - await _prune_author_search_cache(db_session, now_utc=now_utc, max_entries=max_entries) - - -async def _prune_author_search_cache( - db_session: AsyncSession, - *, - now_utc: datetime, - max_entries: int, -) -> None: - await db_session.execute( - delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.expires_at <= now_utc) - ) - bounded_max_entries = max(1, int(max_entries)) - count_result = await db_session.execute( - select(func.count()).select_from(AuthorSearchCacheEntry) - ) - entry_count = int(count_result.scalar_one() or 0) - overflow = max(0, entry_count - bounded_max_entries) - if overflow <= 0: - return - stale_keys_result = await db_session.execute( - select(AuthorSearchCacheEntry.query_key) - .order_by(AuthorSearchCacheEntry.cached_at.asc()) - .limit(overflow) - ) - stale_keys = [str(row[0]) for row in stale_keys_result.all()] - if stale_keys: - await db_session.execute( - delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key.in_(stale_keys)) - ) - - -def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool: - return parsed.state == ParseState.BLOCKED_OR_CAPTCHA - - -def _author_search_cooldown_remaining_seconds( - runtime_state: AuthorSearchRuntimeState, - now_utc: datetime, -) -> int: - cooldown_until = runtime_state.cooldown_until - if cooldown_until is None: - return 0 - if cooldown_until.tzinfo is None: - cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) - remaining_seconds = int((cooldown_until - now_utc).total_seconds()) - return max(0, remaining_seconds) - - -async def list_scholars_for_user( - db_session: AsyncSession, - *, - user_id: int, -) -> list[ScholarProfile]: - result = await db_session.execute( - select(ScholarProfile) - .where(ScholarProfile.user_id == user_id) - .order_by(ScholarProfile.created_at.desc(), ScholarProfile.id.desc()) - ) - return list(result.scalars().all()) - - -async def create_scholar_for_user( - db_session: AsyncSession, - *, - user_id: int, - scholar_id: str, - display_name: str, - profile_image_url: str | None = None, -) -> ScholarProfile: - profile = ScholarProfile( - user_id=user_id, - scholar_id=validate_scholar_id(scholar_id), - display_name=normalize_display_name(display_name), - profile_image_url=normalize_profile_image_url(profile_image_url), - ) - db_session.add(profile) - try: - await db_session.commit() - except IntegrityError as exc: - await db_session.rollback() - raise ScholarServiceError("That scholar is already tracked for this account.") from exc - await db_session.refresh(profile) - return profile - - -async def get_user_scholar_by_id( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int, -) -> ScholarProfile | None: - result = await db_session.execute( - select(ScholarProfile).where( - ScholarProfile.id == scholar_profile_id, - ScholarProfile.user_id == user_id, - ) - ) - return result.scalar_one_or_none() - - -async def toggle_scholar_enabled( - db_session: AsyncSession, - *, - profile: ScholarProfile, -) -> ScholarProfile: - profile.is_enabled = not profile.is_enabled - await db_session.commit() - await db_session.refresh(profile) - return profile - - -async def delete_scholar( - db_session: AsyncSession, - *, - profile: ScholarProfile, - upload_dir: str | None = None, -) -> None: - if upload_dir: - upload_root = _ensure_upload_root(upload_dir, create=True) - _safe_remove_upload(upload_root, profile.profile_image_upload_path) - - await db_session.delete(profile) - await db_session.commit() - - -def _normalize_author_search_inputs(query: str, limit: int) -> tuple[str, int, str]: - normalized_query = query.strip() - if len(normalized_query) < 2: - raise ScholarServiceError("Search query must be at least 2 characters.") - bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT)) - return normalized_query, bounded_limit, normalized_query.casefold() - - -def _disabled_search_result(*, normalized_query: str, bounded_limit: int) -> ParsedAuthorSearchPage: - logger.warning( - "scholar_search.disabled_by_configuration", - extra={"event": "scholar_search.disabled_by_configuration", "query": normalized_query}, - ) - return _policy_blocked_author_search_result( - reason=SEARCH_DISABLED_REASON, - warning_codes=["author_search_disabled_by_configuration"], - limit=bounded_limit, - ) - - -def _normalize_runtime_cooldown_state( - runtime_state: AuthorSearchRuntimeState, - *, - now_utc: datetime, -) -> bool: - if runtime_state.cooldown_until is None: - return False - cooldown_until = runtime_state.cooldown_until - updated = False - if cooldown_until.tzinfo is None: - cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) - runtime_state.cooldown_until = cooldown_until - updated = True - if now_utc < cooldown_until: - return updated - logger.info( - "scholar_search.cooldown_expired", - extra={"event": "scholar_search.cooldown_expired", "cooldown_until_utc": cooldown_until.isoformat()}, - ) - runtime_state.cooldown_until = None - runtime_state.cooldown_rejection_count = 0 - runtime_state.cooldown_alert_emitted = False - return True - - -def _cooldown_warning_codes( - *, - runtime_state: AuthorSearchRuntimeState, - cooldown_remaining_seconds: int, -) -> list[str]: - warning_codes = [ - "author_search_cooldown_active", - f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s", - ] - if bool(runtime_state.cooldown_alert_emitted): - warning_codes.append("author_search_cooldown_alert_threshold_exceeded") - return warning_codes - - -def _emit_cooldown_threshold_alert( - *, - runtime_state: AuthorSearchRuntimeState, - normalized_query: str, - cooldown_rejection_alert_threshold: int, -) -> bool: - runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1 - threshold = max(1, int(cooldown_rejection_alert_threshold)) - if int(runtime_state.cooldown_rejection_count) < threshold: - return True - if bool(runtime_state.cooldown_alert_emitted): - return True - logger.error( - "scholar_search.cooldown_rejection_threshold_exceeded", - extra={ - "event": "scholar_search.cooldown_rejection_threshold_exceeded", - "query": normalized_query, - "cooldown_rejection_count": int(runtime_state.cooldown_rejection_count), - "threshold": threshold, - "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, - }, - ) - runtime_state.cooldown_alert_emitted = True - return True - - -def _cooldown_block_result( - *, - runtime_state: AuthorSearchRuntimeState, - normalized_query: str, - bounded_limit: int, - cooldown_rejection_alert_threshold: int, - cooldown_remaining_seconds: int, -) -> ParsedAuthorSearchPage: - _emit_cooldown_threshold_alert( - runtime_state=runtime_state, - normalized_query=normalized_query, - cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, - ) - logger.warning( - "scholar_search.cooldown_active", - extra={ - "event": "scholar_search.cooldown_active", - "query": normalized_query, - "cooldown_remaining_seconds": cooldown_remaining_seconds, - "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, - }, - ) - return _policy_blocked_author_search_result( - reason=SEARCH_COOLDOWN_REASON, - warning_codes=_cooldown_warning_codes( - runtime_state=runtime_state, - cooldown_remaining_seconds=cooldown_remaining_seconds, - ), - limit=bounded_limit, - ) - - -async def _cache_hit_result( - db_session: AsyncSession, - *, - query_key: str, - now_utc: datetime, - normalized_query: str, - bounded_limit: int, -) -> ParsedAuthorSearchPage | None: - cached = await _cache_get_author_search_result( - db_session, - query_key=query_key, - now_utc=now_utc, - ) - if cached is None: - return None - logger.info( - "scholar_search.cache_hit", - extra={ - "event": "scholar_search.cache_hit", - "query": normalized_query, - "state": cached.state.value, - "state_reason": cached.state_reason, - }, - ) - state_reason_override = SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None - return _trim_author_search_result( - cached, - limit=bounded_limit, - extra_warnings=["author_search_served_from_cache"], - state_reason_override=state_reason_override, - ) - - -def _throttle_sleep_seconds( - *, - runtime_state: AuthorSearchRuntimeState, - now_utc: datetime, - min_interval_seconds: float, - interval_jitter_seconds: float, -) -> tuple[float, bool]: - updated = False - if runtime_state.last_live_request_at is None: - enforced_wait_seconds = 0.0 - else: - last_live_request_at = runtime_state.last_live_request_at - if last_live_request_at.tzinfo is None: - last_live_request_at = last_live_request_at.replace(tzinfo=timezone.utc) - runtime_state.last_live_request_at = last_live_request_at - updated = True - enforced_wait_seconds = ( - last_live_request_at + timedelta(seconds=max(float(min_interval_seconds), 0.0)) - now_utc - ).total_seconds() - jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0)) - return max(0.0, float(enforced_wait_seconds)) + jitter_seconds, updated - - -async def _wait_for_author_search_throttle( - *, - runtime_state: AuthorSearchRuntimeState, - normalized_query: str, - now_utc: datetime, - min_interval_seconds: float, - interval_jitter_seconds: float, -) -> bool: - sleep_seconds, updated = _throttle_sleep_seconds( - runtime_state=runtime_state, - now_utc=now_utc, - min_interval_seconds=min_interval_seconds, - interval_jitter_seconds=interval_jitter_seconds, - ) - if sleep_seconds <= 0.0: - return updated - logger.info( - "scholar_search.throttle_wait", - extra={"event": "scholar_search.throttle_wait", "query": normalized_query, "sleep_seconds": round(sleep_seconds, 3)}, - ) - await asyncio.sleep(sleep_seconds) - return True - - -async def _fetch_author_search_with_retries( - *, - source: ScholarSource, - normalized_query: str, - network_error_retries: int, - retry_backoff_seconds: float, -) -> tuple[ParsedAuthorSearchPage, int, list[str]]: - max_attempts = max(1, int(network_error_retries) + 1) - parsed: ParsedAuthorSearchPage | None = None - retry_warnings: list[str] = [] - retry_scheduled_count = 0 - for attempt_index in range(max_attempts): - fetch_result = await source.fetch_author_search_html(normalized_query, start=0) - try: - parsed = parse_author_search_page(fetch_result) - except ScholarParserError as exc: - parsed = ParsedAuthorSearchPage( - state=ParseState.LAYOUT_CHANGED, - state_reason=exc.code, - candidates=[], - marker_counts={}, - warnings=[exc.code], - ) - if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1: - break - retry_warnings.append("network_retry_scheduled_for_author_search") - retry_scheduled_count += 1 - retry_sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index) - if retry_sleep_seconds > 0: - await asyncio.sleep(retry_sleep_seconds) - if parsed is None: - raise ScholarServiceError("Unable to complete scholar author search.") - return parsed, retry_scheduled_count, retry_warnings - - -def _with_retry_warnings( - parsed: ParsedAuthorSearchPage, - *, - retry_warnings: list[str], - retry_scheduled_count: int, - retry_alert_threshold: int, - normalized_query: str, -) -> ParsedAuthorSearchPage: - merged = replace(parsed, warnings=_merge_warnings(parsed.warnings, retry_warnings)) - threshold = max(1, int(retry_alert_threshold)) - if retry_scheduled_count < threshold: - return merged - logger.warning( - "scholar_search.retry_threshold_exceeded", - extra={ - "event": "scholar_search.retry_threshold_exceeded", - "query": normalized_query, - "retry_scheduled_count": retry_scheduled_count, - "threshold": threshold, - "final_state": merged.state.value, - "final_state_reason": merged.state_reason, - }, - ) - return replace( - merged, - warnings=_merge_warnings( - merged.warnings, - [f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"], - ), - ) - - -def _apply_block_circuit_breaker( - *, - runtime_state: AuthorSearchRuntimeState, - merged_parsed: ParsedAuthorSearchPage, - cooldown_block_threshold: int, - cooldown_seconds: int, - normalized_query: str, -) -> ParsedAuthorSearchPage: - if not _is_author_search_block_state(merged_parsed): - runtime_state.consecutive_blocked_count = 0 - return merged_parsed - runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1 - logger.warning( - "scholar_search.block_detected", - extra={ - "event": "scholar_search.block_detected", - "query": normalized_query, - "state_reason": merged_parsed.state_reason, - "consecutive_blocked_count": int(runtime_state.consecutive_blocked_count), - }, - ) - if int(runtime_state.consecutive_blocked_count) < max(1, int(cooldown_block_threshold)): - return merged_parsed - runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta(seconds=max(60, int(cooldown_seconds))) - runtime_state.consecutive_blocked_count = 0 - runtime_state.cooldown_rejection_count = 0 - runtime_state.cooldown_alert_emitted = False - logger.error( - "scholar_search.cooldown_activated", - extra={ - "event": "scholar_search.cooldown_activated", - "query": normalized_query, - "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, - }, - ) - return replace( - merged_parsed, - warnings=_merge_warnings(merged_parsed.warnings, ["author_search_circuit_breaker_armed"]), - ) - - -def _resolve_author_search_cache_ttl_seconds( - *, - merged_parsed: ParsedAuthorSearchPage, - blocked_cache_ttl_seconds: int, - cache_ttl_seconds: int, -) -> int: - if _is_author_search_block_state(merged_parsed): - return min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds))) - return max(1, int(cache_ttl_seconds)) - - -async def _load_locked_runtime_state( - db_session: AsyncSession, -) -> AuthorSearchRuntimeState: - await _acquire_author_search_lock(db_session) - return await _load_runtime_state_for_update(db_session) - - -async def _cooldown_or_cache_result( - db_session: AsyncSession, - *, - runtime_state: AuthorSearchRuntimeState, - query_key: str, - normalized_query: str, - bounded_limit: int, - cooldown_rejection_alert_threshold: int, -) -> tuple[ParsedAuthorSearchPage | None, bool]: - runtime_state_updated = _normalize_runtime_cooldown_state( - runtime_state, - now_utc=datetime.now(timezone.utc), - ) - cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds( - runtime_state, - datetime.now(timezone.utc), - ) - if cooldown_remaining_seconds > 0: - return ( - _cooldown_block_result( - runtime_state=runtime_state, - normalized_query=normalized_query, - bounded_limit=bounded_limit, - cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, - cooldown_remaining_seconds=cooldown_remaining_seconds, - ), - True, - ) - cached_result = await _cache_hit_result( - db_session, - query_key=query_key, - now_utc=datetime.now(timezone.utc), - normalized_query=normalized_query, - bounded_limit=bounded_limit, - ) - return cached_result, runtime_state_updated - - -async def _perform_live_author_search(db_session: AsyncSession, *, source: ScholarSource, runtime_state: AuthorSearchRuntimeState, normalized_query: str, query_key: str, network_error_retries: int, retry_backoff_seconds: float, min_interval_seconds: float, interval_jitter_seconds: float, retry_alert_threshold: int, cooldown_block_threshold: int, cooldown_seconds: int, blocked_cache_ttl_seconds: int, cache_ttl_seconds: int, cache_max_entries: int) -> tuple[ParsedAuthorSearchPage, bool]: - runtime_state_updated = await _wait_for_author_search_throttle( - runtime_state=runtime_state, - normalized_query=normalized_query, - now_utc=datetime.now(timezone.utc), - min_interval_seconds=min_interval_seconds, - interval_jitter_seconds=interval_jitter_seconds, - ) - parsed, retry_count, retry_warnings = await _fetch_author_search_with_retries( - source=source, - normalized_query=normalized_query, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - runtime_state.last_live_request_at = datetime.now(timezone.utc) - merged = _with_retry_warnings( - parsed, - retry_warnings=retry_warnings, - retry_scheduled_count=retry_count, - retry_alert_threshold=retry_alert_threshold, - normalized_query=normalized_query, - ) - merged = _apply_block_circuit_breaker( - runtime_state=runtime_state, - merged_parsed=merged, - cooldown_block_threshold=cooldown_block_threshold, - cooldown_seconds=cooldown_seconds, - normalized_query=normalized_query, - ) - ttl_seconds = _resolve_author_search_cache_ttl_seconds( - merged_parsed=merged, - blocked_cache_ttl_seconds=blocked_cache_ttl_seconds, - cache_ttl_seconds=cache_ttl_seconds, - ) - await _cache_set_author_search_result( - db_session, - query_key=query_key, - parsed=merged, - ttl_seconds=float(ttl_seconds), - max_entries=cache_max_entries, - now_utc=datetime.now(timezone.utc), - ) - return merged, True - - -async def search_author_candidates(*, source: ScholarSource, db_session: AsyncSession, query: str, limit: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, search_enabled: bool = True, cache_ttl_seconds: int = 21_600, blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS, cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES, min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS, interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS, cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD, cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS, retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD) -> ParsedAuthorSearchPage: - normalized_query, bounded_limit, query_key = _normalize_author_search_inputs(query, limit) - if not search_enabled: - return _disabled_search_result( - normalized_query=normalized_query, - bounded_limit=bounded_limit, - ) - - runtime_state = await _load_locked_runtime_state(db_session) - early_result, runtime_state_updated = await _cooldown_or_cache_result( - db_session, - runtime_state=runtime_state, - query_key=query_key, - normalized_query=normalized_query, - bounded_limit=bounded_limit, - cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, - ) - if early_result is not None: - await db_session.commit() - return early_result - - merged_parsed, live_runtime_updated = await _perform_live_author_search( - db_session, - source=source, - runtime_state=runtime_state, - normalized_query=normalized_query, - query_key=query_key, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - min_interval_seconds=min_interval_seconds, - interval_jitter_seconds=interval_jitter_seconds, - retry_alert_threshold=retry_alert_threshold, - cooldown_block_threshold=cooldown_block_threshold, - cooldown_seconds=cooldown_seconds, - blocked_cache_ttl_seconds=blocked_cache_ttl_seconds, - cache_ttl_seconds=cache_ttl_seconds, - cache_max_entries=cache_max_entries, - ) - runtime_state_updated = runtime_state_updated or live_runtime_updated - if runtime_state_updated: - await db_session.commit() - return _trim_author_search_result(merged_parsed, limit=bounded_limit) - - -async def hydrate_profile_metadata( - db_session: AsyncSession, - *, - profile: ScholarProfile, - source: ScholarSource, -) -> ScholarProfile: - fetch_result = await source.fetch_profile_html(profile.scholar_id) - try: - parsed_page = parse_profile_page(fetch_result) - except ScholarParserError: - return profile - - if parsed_page.profile_name and not (profile.display_name or "").strip(): - profile.display_name = parsed_page.profile_name - if parsed_page.profile_image_url and not profile.profile_image_url: - profile.profile_image_url = parsed_page.profile_image_url - - await db_session.commit() - await db_session.refresh(profile) - return profile - - -async def set_profile_image_override_url( - db_session: AsyncSession, - *, - profile: ScholarProfile, - image_url: str | None, - upload_dir: str, -) -> ScholarProfile: - upload_root = _ensure_upload_root(upload_dir, create=True) - _safe_remove_upload(upload_root, profile.profile_image_upload_path) - - profile.profile_image_upload_path = None - profile.profile_image_override_url = normalize_profile_image_url(image_url) - - await db_session.commit() - await db_session.refresh(profile) - return profile - - -async def clear_profile_image_customization( - db_session: AsyncSession, - *, - profile: ScholarProfile, - upload_dir: str, -) -> ScholarProfile: - upload_root = _ensure_upload_root(upload_dir, create=True) - _safe_remove_upload(upload_root, profile.profile_image_upload_path) - - profile.profile_image_upload_path = None - profile.profile_image_override_url = None - - await db_session.commit() - await db_session.refresh(profile) - return profile - - -async def set_profile_image_upload( - db_session: AsyncSession, - *, - profile: ScholarProfile, - content_type: str | None, - image_bytes: bytes, - upload_dir: str, - max_upload_bytes: int, -) -> ScholarProfile: - normalized_content_type = (content_type or "").strip().lower() - extension = ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES.get(normalized_content_type) - if extension is None: - raise ScholarServiceError( - "Unsupported image type. Use JPEG, PNG, WEBP, or GIF." - ) - - if not image_bytes: - raise ScholarServiceError("Uploaded image file is empty.") - - if len(image_bytes) > max_upload_bytes: - raise ScholarServiceError( - f"Uploaded image exceeds {max_upload_bytes} bytes." - ) - - upload_root = _ensure_upload_root(upload_dir, create=True) - user_dir = upload_root / str(profile.user_id) - user_dir.mkdir(parents=True, exist_ok=True) - - filename = f"{profile.id}_{uuid4().hex}{extension}" - relative_path = os.path.join(str(profile.user_id), filename) - absolute_path = _resolve_upload_path(upload_root, relative_path) - absolute_path.write_bytes(image_bytes) - - old_path = profile.profile_image_upload_path - profile.profile_image_upload_path = relative_path - profile.profile_image_override_url = None - - await db_session.commit() - await db_session.refresh(profile) - - if old_path and old_path != relative_path: - _safe_remove_upload(upload_root, old_path) - - return profile diff --git a/build/lib/app/services/domains/scholars/constants.py b/build/lib/app/services/domains/scholars/constants.py deleted file mode 100644 index 7ee18ad..0000000 --- a/build/lib/app/services/domains/scholars/constants.py +++ /dev/null @@ -1,70 +0,0 @@ -from __future__ import annotations - -import re - -SCHOLAR_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]{12}$") -MAX_IMAGE_URL_LENGTH = 2048 -MAX_AUTHOR_SEARCH_LIMIT = 25 - -DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES = 512 -DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS = 300 -DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD = 1 -DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS = 1800 -DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS = 3.0 -DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS = 1.0 -DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD = 2 -DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD = 3 - -AUTHOR_SEARCH_RUNTIME_STATE_KEY = "global" -AUTHOR_SEARCH_LOCK_NAMESPACE = 3901 -AUTHOR_SEARCH_LOCK_KEY = 1 - -ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES = { - "image/jpeg": ".jpg", - "image/png": ".png", - "image/webp": ".webp", - "image/gif": ".gif", -} - -SEARCH_DISABLED_REASON = "search_disabled_by_configuration" -SEARCH_COOLDOWN_REASON = "search_temporarily_disabled_due_to_repeated_blocks" -SEARCH_CACHED_BLOCK_REASON = "search_temporarily_disabled_from_cached_blocked_response" - -STATE_REASON_HINTS: dict[str, str] = { - SEARCH_DISABLED_REASON: ( - "Scholar name search is currently disabled by service policy. " - "Add scholars by profile URL or Scholar ID." - ), - SEARCH_COOLDOWN_REASON: ( - "Scholar name search is temporarily paused after repeated block responses. " - "Use Scholar URL/ID adds until cooldown expires." - ), - SEARCH_CACHED_BLOCK_REASON: ( - "A recent blocked response was cached to reduce traffic. " - "Retry later or add by Scholar URL/ID." - ), - "network_dns_resolution_failed": ( - "DNS resolution failed while reaching scholar.google.com. " - "Verify container DNS/network and retry." - ), - "network_timeout": ( - "Request timed out before Google Scholar responded. " - "Increase delay/backoff and retry." - ), - "network_tls_error": ( - "TLS handshake/certificate validation failed. " - "Verify outbound TLS/network configuration." - ), - "blocked_http_429_rate_limited": ( - "Google Scholar rate-limited the request. " - "Slow request cadence and retry later." - ), - "blocked_unusual_traffic_detected": ( - "Google Scholar flagged traffic as unusual. " - "Increase delay/jitter and reduce concurrent scraping." - ), - "blocked_accounts_redirect": ( - "Request was redirected to Google Account sign-in. " - "Treat as access block and retry later." - ), -} diff --git a/build/lib/app/services/domains/scholars/exceptions.py b/build/lib/app/services/domains/scholars/exceptions.py deleted file mode 100644 index f1383e3..0000000 --- a/build/lib/app/services/domains/scholars/exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import annotations - - -class ScholarServiceError(ValueError): - """Raised for expected scholar-management validation failures.""" diff --git a/build/lib/app/services/domains/scholars/search_hints.py b/build/lib/app/services/domains/scholars/search_hints.py deleted file mode 100644 index f905c14..0000000 --- a/build/lib/app/services/domains/scholars/search_hints.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import annotations - -from app.db.models import ScholarProfile -from app.services.domains.scholar.parser import ParseState, ParsedAuthorSearchPage -from app.services.domains.scholars.constants import ( - MAX_AUTHOR_SEARCH_LIMIT, - STATE_REASON_HINTS, -) - - -def resolve_profile_image( - profile: ScholarProfile, - *, - uploaded_image_url: str | None, -) -> tuple[str | None, str]: - if profile.profile_image_upload_path and uploaded_image_url: - return uploaded_image_url, "upload" - if profile.profile_image_override_url: - return profile.profile_image_override_url, "override" - if profile.profile_image_url: - return profile.profile_image_url, "scraped" - return None, "none" - - -def scrape_state_hint(*, state: ParseState, state_reason: str) -> str | None: - if state not in {ParseState.NETWORK_ERROR, ParseState.BLOCKED_OR_CAPTCHA}: - return None - return STATE_REASON_HINTS.get(state_reason) - - -def _merge_warnings(base: list[str], extra: list[str]) -> list[str]: - if not extra: - return sorted(set(base)) - return sorted(set(base + extra)) - - -def _trim_author_search_result( - parsed: ParsedAuthorSearchPage, - *, - limit: int, - extra_warnings: list[str] | None = None, - state_reason_override: str | None = None, -) -> ParsedAuthorSearchPage: - bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT)) - return ParsedAuthorSearchPage( - state=parsed.state, - state_reason=state_reason_override or parsed.state_reason, - candidates=parsed.candidates[:bounded_limit], - marker_counts=parsed.marker_counts, - warnings=_merge_warnings(parsed.warnings, extra_warnings or []), - ) - - -def _policy_blocked_author_search_result( - *, - reason: str, - warning_codes: list[str], - limit: int, -) -> ParsedAuthorSearchPage: - _ = limit - return ParsedAuthorSearchPage( - state=ParseState.BLOCKED_OR_CAPTCHA, - state_reason=reason, - candidates=[], - marker_counts={}, - warnings=_merge_warnings([], warning_codes), - ) diff --git a/build/lib/app/services/domains/scholars/uploads.py b/build/lib/app/services/domains/scholars/uploads.py deleted file mode 100644 index c33fd47..0000000 --- a/build/lib/app/services/domains/scholars/uploads.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -from app.services.domains.scholars.exceptions import ScholarServiceError - - -def _ensure_upload_root(upload_dir: str, *, create: bool) -> Path: - root = Path(upload_dir).expanduser().resolve() - if create: - root.mkdir(parents=True, exist_ok=True) - return root - - -def _resolve_upload_path(upload_root: Path, relative_path: str) -> Path: - candidate = (upload_root / relative_path).resolve() - if upload_root != candidate and upload_root not in candidate.parents: - raise ScholarServiceError("Invalid scholar image path.") - return candidate - - -def _safe_remove_upload(upload_root: Path, relative_path: str | None) -> None: - if not relative_path: - return - try: - file_path = _resolve_upload_path(upload_root, relative_path) - except ScholarServiceError: - return - - try: - if file_path.exists() and file_path.is_file(): - file_path.unlink() - except OSError: - return - - -def resolve_upload_file_path(*, upload_dir: str, relative_path: str) -> Path: - root = _ensure_upload_root(upload_dir, create=False) - return _resolve_upload_path(root, relative_path) diff --git a/build/lib/app/services/domains/scholars/validators.py b/build/lib/app/services/domains/scholars/validators.py deleted file mode 100644 index 1c79baa..0000000 --- a/build/lib/app/services/domains/scholars/validators.py +++ /dev/null @@ -1,38 +0,0 @@ -from __future__ import annotations - -from urllib.parse import urlparse - -from app.services.domains.scholars.constants import MAX_IMAGE_URL_LENGTH, SCHOLAR_ID_PATTERN -from app.services.domains.scholars.exceptions import ScholarServiceError - - -def validate_scholar_id(value: str) -> str: - scholar_id = value.strip() - if not SCHOLAR_ID_PATTERN.fullmatch(scholar_id): - raise ScholarServiceError("Scholar ID must match [a-zA-Z0-9_-]{12}.") - return scholar_id - - -def normalize_display_name(value: str) -> str | None: - normalized = value.strip() - return normalized if normalized else None - - -def normalize_profile_image_url(value: str | None) -> str | None: - if value is None: - return None - - candidate = value.strip() - if not candidate: - return None - - if len(candidate) > MAX_IMAGE_URL_LENGTH: - raise ScholarServiceError( - f"Image URL must be {MAX_IMAGE_URL_LENGTH} characters or fewer." - ) - - parsed = urlparse(candidate) - if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc: - raise ScholarServiceError("Image URL must be an absolute http(s) URL.") - - return candidate diff --git a/build/lib/app/services/domains/settings/__init__.py b/build/lib/app/services/domains/settings/__init__.py deleted file mode 100644 index 9d48db4..0000000 --- a/build/lib/app/services/domains/settings/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from __future__ import annotations diff --git a/build/lib/app/services/domains/settings/application.py b/build/lib/app/services/domains/settings/application.py deleted file mode 100644 index 7f435e8..0000000 --- a/build/lib/app/services/domains/settings/application.py +++ /dev/null @@ -1,159 +0,0 @@ -from __future__ import annotations - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import UserSetting - - -class UserSettingsServiceError(ValueError): - """Raised for expected settings-validation failures.""" - - -NAV_PAGE_DASHBOARD = "dashboard" -NAV_PAGE_SCHOLARS = "scholars" -NAV_PAGE_PUBLICATIONS = "publications" -NAV_PAGE_SETTINGS = "settings" -NAV_PAGE_STYLE_GUIDE = "style-guide" -NAV_PAGE_RUNS = "runs" -NAV_PAGE_USERS = "users" - -ALLOWED_NAV_PAGES = ( - NAV_PAGE_DASHBOARD, - NAV_PAGE_SCHOLARS, - NAV_PAGE_PUBLICATIONS, - NAV_PAGE_SETTINGS, - NAV_PAGE_STYLE_GUIDE, - NAV_PAGE_RUNS, - NAV_PAGE_USERS, -) -REQUIRED_NAV_PAGES = ( - NAV_PAGE_DASHBOARD, - NAV_PAGE_SCHOLARS, - NAV_PAGE_SETTINGS, -) -DEFAULT_NAV_VISIBLE_PAGES = list(ALLOWED_NAV_PAGES) -HARD_MIN_RUN_INTERVAL_MINUTES = 15 -HARD_MIN_REQUEST_DELAY_SECONDS = 2 - - -def resolve_run_interval_minimum(configured_minimum: int | None) -> int: - try: - parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_RUN_INTERVAL_MINUTES - except (TypeError, ValueError): - parsed = HARD_MIN_RUN_INTERVAL_MINUTES - return max(HARD_MIN_RUN_INTERVAL_MINUTES, parsed) - - -def resolve_request_delay_minimum(configured_minimum: int | None) -> int: - try: - parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_REQUEST_DELAY_SECONDS - except (TypeError, ValueError): - parsed = HARD_MIN_REQUEST_DELAY_SECONDS - return max(HARD_MIN_REQUEST_DELAY_SECONDS, parsed) - - -def parse_run_interval_minutes(value: str, *, minimum: int = HARD_MIN_RUN_INTERVAL_MINUTES) -> int: - try: - parsed = int(value) - except ValueError as exc: - raise UserSettingsServiceError("Check interval must be a whole number.") from exc - effective_minimum = resolve_run_interval_minimum(minimum) - if parsed < effective_minimum: - raise UserSettingsServiceError( - f"Check interval must be at least {effective_minimum} minutes." - ) - return parsed - - -def parse_request_delay_seconds(value: str, *, minimum: int = HARD_MIN_REQUEST_DELAY_SECONDS) -> int: - try: - parsed = int(value) - except ValueError as exc: - raise UserSettingsServiceError("Request delay must be a whole number.") from exc - effective_minimum = resolve_request_delay_minimum(minimum) - if parsed < effective_minimum: - raise UserSettingsServiceError( - f"Request delay must be at least {effective_minimum} seconds." - ) - return parsed - - -def parse_nav_visible_pages(value: object) -> list[str]: - if not isinstance(value, list): - raise UserSettingsServiceError("Navigation visibility must be a list of page ids.") - - deduped: list[str] = [] - seen: set[str] = set() - - for raw_page in value: - if not isinstance(raw_page, str): - raise UserSettingsServiceError("Navigation visibility entries must be strings.") - - page_id = raw_page.strip() - if page_id not in ALLOWED_NAV_PAGES: - raise UserSettingsServiceError(f"Unsupported navigation page id: {page_id}") - - if page_id in seen: - continue - - seen.add(page_id) - deduped.append(page_id) - - missing_required = [page for page in REQUIRED_NAV_PAGES if page not in seen] - if missing_required: - raise UserSettingsServiceError( - "Dashboard, Scholars, and Settings must remain visible." - ) - - return deduped - - -from app.settings import settings as app_settings - -async def get_or_create_settings( - db_session: AsyncSession, - *, - user_id: int, -) -> UserSetting: - result = await db_session.execute( - select(UserSetting).where(UserSetting.user_id == user_id) - ) - settings = result.scalar_one_or_none() - if settings is not None: - return settings - - settings = UserSetting( - user_id=user_id, - openalex_api_key=app_settings.openalex_api_key, - crossref_api_token=app_settings.crossref_api_token, - crossref_api_mailto=app_settings.crossref_api_mailto, - ) - db_session.add(settings) - await db_session.commit() - await db_session.refresh(settings) - return settings - - -async def update_settings( - db_session: AsyncSession, - *, - settings: UserSetting, - auto_run_enabled: bool, - run_interval_minutes: int, - request_delay_seconds: int, - nav_visible_pages: list[str], - openalex_api_key: str | None, - crossref_api_token: str | None, - crossref_api_mailto: str | None, -) -> UserSetting: - settings.auto_run_enabled = auto_run_enabled - settings.run_interval_minutes = run_interval_minutes - settings.request_delay_seconds = request_delay_seconds - settings.nav_visible_pages = nav_visible_pages - settings.openalex_api_key = openalex_api_key - settings.crossref_api_token = crossref_api_token - settings.crossref_api_mailto = crossref_api_mailto - await db_session.commit() - await db_session.refresh(settings) - return settings diff --git a/build/lib/app/services/domains/unpaywall/__init__.py b/build/lib/app/services/domains/unpaywall/__init__.py deleted file mode 100644 index a1baa12..0000000 --- a/build/lib/app/services/domains/unpaywall/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import annotations - - -async def resolve_publication_pdf_urls(*args, **kwargs): - from app.services.domains.unpaywall.application import resolve_publication_pdf_urls as _impl - - return await _impl(*args, **kwargs) - - -__all__ = ["resolve_publication_pdf_urls"] diff --git a/build/lib/app/services/domains/unpaywall/application.py b/build/lib/app/services/domains/unpaywall/application.py deleted file mode 100644 index 13104e9..0000000 --- a/build/lib/app/services/domains/unpaywall/application.py +++ /dev/null @@ -1,458 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -import logging -import re -from typing import TYPE_CHECKING -from urllib.parse import unquote - -from app.services.domains.crossref.application import discover_doi_for_publication -from app.services.domains.doi.normalize import normalize_doi -from app.services.domains.unpaywall.pdf_discovery import ( - looks_like_pdf_url, - resolve_pdf_from_landing_page, -) -from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot -from app.settings import settings - -if TYPE_CHECKING: - from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem - -DOI_PATTERN = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I) -DOI_PREFIX_RE = re.compile(r"\bdoi\s*[:=]\s*(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I) -DOI_URL_RE = re.compile(r"(?:https?://)?(?:dx\.)?doi\.org/(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", re.I) -UNPAYWALL_URL_TEMPLATE = "https://api.unpaywall.org/v2/{doi}" -FAILURE_MISSING_DOI = "missing_doi" -FAILURE_NO_RECORD = "no_unpaywall_record" -FAILURE_NO_PDF = "no_pdf_found" -FAILURE_RESOLUTION_EXCEPTION = "resolution_exception" -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class OaResolutionOutcome: - publication_id: int - doi: str | None - pdf_url: str | None - failure_reason: str | None - source: str | None - used_crossref: bool - - -def _extract_doi_candidate(text: str | None) -> str | None: - if not text: - return None - decoded = unquote(text) - match = DOI_PATTERN.search(decoded) - if not match: - return None - return match.group(0).rstrip(" .;,)") - - -def _extract_explicit_doi(text: str | None) -> str | None: - if not text: - return None - decoded = unquote(text) - url_match = DOI_URL_RE.search(decoded) - if url_match: - return normalize_doi(url_match.group(1)) - prefix_match = DOI_PREFIX_RE.search(decoded) - if prefix_match: - return normalize_doi(prefix_match.group(1)) - return None - - -def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str | None: - stored = None - if getattr(item, "display_identifier", None) and item.display_identifier.kind == "doi": - stored = normalize_doi(item.display_identifier.value) - - explicit_doi = ( - _extract_explicit_doi(item.pub_url) - or _extract_explicit_doi(item.venue_text) - ) - if explicit_doi: - return explicit_doi - pub_url_doi = _extract_doi_candidate(item.pub_url) - if pub_url_doi: - return normalize_doi(pub_url_doi) - return stored - - -def _payload_locations(payload: dict) -> list[dict]: - locations: list[dict] = [] - best = payload.get("best_oa_location") - if isinstance(best, dict): - locations.append(best) - oa_locations = payload.get("oa_locations") - if isinstance(oa_locations, list): - locations.extend(location for location in oa_locations if isinstance(location, dict)) - return locations - - -def _location_value(location: dict, key: str) -> str | None: - value = location.get(key) - if not isinstance(value, str): - return None - normalized = value.strip() - return normalized or None - - -def _payload_pdf_candidates(payload: dict) -> list[str]: - candidates: list[str] = [] - seen: set[str] = set() - for location in _payload_locations(payload): - candidate = _location_value(location, "url_for_pdf") - if candidate is None or candidate in seen: - continue - seen.add(candidate) - candidates.append(candidate) - return candidates - - -def _payload_landing_candidates(payload: dict) -> list[str]: - candidates: list[str] = [] - seen: set[str] = set() - for location in _payload_locations(payload): - candidate = _location_value(location, "url") - if candidate is None or candidate in seen: - continue - seen.add(candidate) - candidates.append(candidate) - return candidates - - -def _crawl_targets( - *, - payload: dict, - pdf_candidates: list[str], -) -> list[str]: - targets = _payload_landing_candidates(payload) - seen = set(targets) - for candidate in pdf_candidates: - if candidate in seen: - continue - targets.append(candidate) - seen.add(candidate) - doi = normalize_doi(str(payload.get("doi") or "")) - doi_landing_url = f"https://doi.org/{doi}" if doi else None - if doi_landing_url and doi_landing_url not in seen: - targets.append(doi_landing_url) - return targets - - -def _has_direct_payload_pdf(payload: dict) -> bool: - return any(looks_like_pdf_url(candidate) for candidate in _payload_pdf_candidates(payload)) - - -async def _resolved_pdf_url_from_payload( - payload: dict, - *, - client, -) -> str | None: - pdf_candidates = _payload_pdf_candidates(payload) - for candidate in pdf_candidates: - if looks_like_pdf_url(candidate): - return candidate - for page_url in _crawl_targets(payload=payload, pdf_candidates=pdf_candidates)[:3]: - discovered = await resolve_pdf_from_landing_page(client, page_url=page_url) - if discovered: - logger.info( - "unpaywall.pdf_discovered_from_landing", - extra={"event": "unpaywall.pdf_discovered_from_landing", "landing_url": page_url}, - ) - return discovered - return None - - -async def _fetch_unpaywall_payload_by_doi( - *, - client, - doi: str, - email: str, -) -> dict | None: - await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds) - headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{email})"} - response = await client.get( - UNPAYWALL_URL_TEMPLATE.format(doi=doi), - params={"email": email}, - headers=headers, - ) - if response.status_code != 200: - return None - payload = response.json() - if not isinstance(payload, dict): - return None - return payload - - -def _email_for_request(request_email: str | None) -> str | None: - email = (request_email or "").strip() or settings.unpaywall_email.strip() - return email or None - - -def _log_resolution_summary( - *, - publication_count: int, - doi_input_count: int, - search_attempt_count: int, - resolved_pdf_count: int, - email: str, -) -> None: - logger.info( - "unpaywall.resolve_completed", - extra={ - "event": "unpaywall.resolve_completed", - "publication_count": publication_count, - "doi_input_count": doi_input_count, - "search_attempt_count": search_attempt_count, - "resolved_pdf_count": resolved_pdf_count, - "email_domain": email.split("@", 1)[-1] if "@" in email else None, - }, - ) - - -async def _resolve_item_payload( - *, - client, - item: PublicationListItem, - email: str, - allow_crossref: bool, -) -> tuple[dict | None, bool, str | None]: - doi = _publication_doi(item) - payload: dict | None = None - if doi: - payload = await _fetch_unpaywall_payload_by_doi(client=client, doi=doi, email=email) - if payload is not None and _has_direct_payload_pdf(payload): - return payload, False, doi - if not allow_crossref or not settings.crossref_enabled: - return payload, False, doi - crossref_doi = await discover_doi_for_publication( - item=item, - max_rows=settings.crossref_max_rows, - email=email, - ) - if crossref_doi is None or crossref_doi == doi: - return payload, crossref_doi is not None, doi or crossref_doi - crossref_payload = await _fetch_unpaywall_payload_by_doi( - client=client, - doi=crossref_doi, - email=email, - ) - if crossref_payload is not None: - return crossref_payload, True, crossref_doi - return payload, True, crossref_doi - - -async def _doi_and_pdf_from_payload( - payload: dict, - *, - client, -) -> tuple[str | None, str | None]: - doi = normalize_doi(str(payload.get("doi") or "")) - return doi, await _resolved_pdf_url_from_payload(payload, client=client) - - -def _resolution_targets(items: list[PublicationListItem]) -> list[PublicationListItem]: - return [item for item in items if not item.pdf_url] - - -def _crossref_budget_value() -> int: - return max(int(settings.crossref_max_lookups_per_request), 0) - - -def _outcome_with_failure( - *, - item: PublicationListItem, - failure_reason: str, - used_crossref: bool, - doi_override: str | None = None, -) -> OaResolutionOutcome: - return OaResolutionOutcome( - publication_id=item.publication_id, - doi=normalize_doi(doi_override) if doi_override is not None else _publication_doi(item), - pdf_url=None, - failure_reason=failure_reason, - source=None, - used_crossref=used_crossref, - ) - - -def _missing_payload_failure_reason(item: PublicationListItem, *, used_crossref: bool) -> str: - if _publication_doi(item): - return FAILURE_NO_RECORD - if used_crossref: - return FAILURE_NO_RECORD - return FAILURE_MISSING_DOI - - -def _source_name(*, used_crossref: bool) -> str: - return "crossref+unpaywall" if used_crossref else "unpaywall" - - -def _outcome_from_payload( - *, - item: PublicationListItem, - doi: str | None, - pdf_url: str | None, - used_crossref: bool, -) -> OaResolutionOutcome: - return OaResolutionOutcome( - publication_id=item.publication_id, - doi=doi, - pdf_url=pdf_url, - failure_reason=None if pdf_url else FAILURE_NO_PDF, - source=_source_name(used_crossref=used_crossref), - used_crossref=used_crossref, - ) - - -def _resolved_pdf_count(outcomes: dict[int, OaResolutionOutcome]) -> int: - return sum(1 for outcome in outcomes.values() if outcome.pdf_url) - - -def _outcome_metadata(outcomes: dict[int, OaResolutionOutcome]) -> dict[int, tuple[str | None, str | None]]: - return { - publication_id: (outcome.doi, outcome.pdf_url) - for publication_id, outcome in outcomes.items() - } - - -async def _resolve_outcome_for_item( - *, - client, - item: PublicationListItem, - email: str, - allow_crossref: bool, -) -> OaResolutionOutcome: - payload, used_crossref, resolved_doi = await _resolve_item_payload( - client=client, - item=item, - email=email, - allow_crossref=allow_crossref, - ) - if not isinstance(payload, dict): - return _outcome_with_failure( - item=item, - failure_reason=_missing_payload_failure_reason(item, used_crossref=used_crossref), - used_crossref=used_crossref, - doi_override=resolved_doi, - ) - doi, pdf_url = await _doi_and_pdf_from_payload(payload, client=client) - return _outcome_from_payload( - item=item, - doi=doi, - pdf_url=pdf_url, - used_crossref=used_crossref, - ) - - -def _doi_input_count(items: list[PublicationListItem]) -> int: - return len([item for item in items if _publication_doi(item)]) - - -def _search_attempt_count(*, targets: list[PublicationListItem]) -> int: - return len([item for item in targets if not _publication_doi(item)]) - - -async def _safe_outcome_for_item( - *, - client, - item: PublicationListItem, - email: str, - allow_crossref: bool, -) -> OaResolutionOutcome: - try: - return await _resolve_outcome_for_item( - client=client, - item=item, - email=email, - allow_crossref=allow_crossref, - ) - except Exception as exc: # pragma: no cover - defensive network boundary - logger.warning( - "unpaywall.resolve_item_failed", - extra={ - "event": "unpaywall.resolve_item_failed", - "publication_id": item.publication_id, - "error": str(exc), - }, - ) - return _outcome_with_failure( - item=item, - failure_reason=FAILURE_RESOLUTION_EXCEPTION, - used_crossref=allow_crossref and settings.crossref_enabled, - ) - - -async def _resolve_outcomes_with_client( - *, - client, - targets: list[PublicationListItem], - email: str, -) -> dict[int, OaResolutionOutcome]: - outcomes: dict[int, OaResolutionOutcome] = {} - crossref_budget = _crossref_budget_value() - crossref_lookups = 0 - for item in targets: - allow_crossref = crossref_budget > 0 and crossref_lookups < crossref_budget - outcome = await _safe_outcome_for_item( - client=client, - item=item, - email=email, - allow_crossref=allow_crossref, - ) - if outcome.used_crossref: - crossref_lookups += 1 - outcomes[item.publication_id] = outcome - return outcomes - - -async def resolve_publication_oa_metadata( - items: list[PublicationListItem], - *, - request_email: str | None = None, -) -> dict[int, tuple[str | None, str | None]]: - outcomes = await resolve_publication_oa_outcomes(items, request_email=request_email) - return _outcome_metadata(outcomes) - - -async def resolve_publication_oa_outcomes( - items: list[PublicationListItem], - *, - request_email: str | None = None, -) -> dict[int, OaResolutionOutcome]: - if not settings.unpaywall_enabled: - return {} - email = _email_for_request(request_email) - if email is None: - logger.debug("unpaywall.resolve_skipped_missing_email") - return {} - import httpx - - timeout_seconds = max(float(settings.unpaywall_timeout_seconds), 0.5) - targets = _resolution_targets(items)[: max(int(settings.unpaywall_max_items_per_request), 0)] - headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{email})"} - async with httpx.AsyncClient(timeout=timeout_seconds, follow_redirects=True, headers=headers) as client: - outcomes = await _resolve_outcomes_with_client( - client=client, - targets=targets, - email=email, - ) - _log_resolution_summary( - publication_count=len(items), - doi_input_count=_doi_input_count(items), - search_attempt_count=_search_attempt_count(targets=targets), - resolved_pdf_count=_resolved_pdf_count(outcomes), - email=email, - ) - return outcomes - - -async def resolve_publication_pdf_urls( - items: list[PublicationListItem], - *, - request_email: str | None = None, -) -> dict[int, str | None]: - resolved = await resolve_publication_oa_metadata(items, request_email=request_email) - return {publication_id: pdf for publication_id, (_doi, pdf) in resolved.items()} diff --git a/build/lib/app/services/domains/unpaywall/pdf_discovery.py b/build/lib/app/services/domains/unpaywall/pdf_discovery.py deleted file mode 100644 index 5f82a58..0000000 --- a/build/lib/app/services/domains/unpaywall/pdf_discovery.py +++ /dev/null @@ -1,156 +0,0 @@ -from __future__ import annotations - -from html.parser import HTMLParser -import re -from urllib.parse import urljoin, urlparse - -from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot -from app.settings import settings - -PDF_MIME = "application/pdf" -URL_RE = re.compile(r"https?://[^\s\"'<>]+", re.I) - - -class _LandingPdfParser(HTMLParser): - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.base_href: str | None = None - self.candidates: list[str] = [] - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - attrs_map = {key.lower(): (value or "").strip() for key, value in attrs} - if tag == "base": - self.base_href = attrs_map.get("href") or self.base_href - return - if tag == "meta": - self._append_meta_candidate(attrs_map) - return - if tag == "link": - self._append_link_candidate(attrs_map) - return - if tag == "a": - self._append_anchor_candidate(attrs_map) - - def _append_meta_candidate(self, attrs_map: dict[str, str]) -> None: - meta_name = (attrs_map.get("name") or attrs_map.get("property") or "").lower() - if meta_name != "citation_pdf_url": - return - content = attrs_map.get("content") - if content: - self.candidates.append(content) - - def _append_link_candidate(self, attrs_map: dict[str, str]) -> None: - href = attrs_map.get("href") - link_type = (attrs_map.get("type") or "").lower() - if href and "pdf" in link_type: - self.candidates.append(href) - - def _append_anchor_candidate(self, attrs_map: dict[str, str]) -> None: - href = attrs_map.get("href") - if href: - self.candidates.append(href) - - -def looks_like_pdf_url(url: str | None) -> bool: - if not isinstance(url, str): - return False - value = url.strip() - if not value: - return False - parsed = urlparse(value) - path = (parsed.path or "").lower() - query = (parsed.query or "").lower() - return path.endswith(".pdf") or ".pdf" in query - - -def _normalized_candidate_urls(*, page_url: str, html: str) -> list[str]: - parser = _LandingPdfParser() - parser.feed(html) - parser.close() - base_url = urljoin(page_url, parser.base_href) if parser.base_href else page_url - raw_candidates = [*parser.candidates, *_text_url_candidates(html)] - deduped: list[str] = [] - seen: set[str] = set() - for raw in raw_candidates: - absolute = urljoin(base_url, raw.strip()) - parsed = urlparse(absolute) - if parsed.scheme not in {"http", "https"}: - continue - if absolute in seen: - continue - seen.add(absolute) - deduped.append(absolute) - return sorted(deduped, key=_candidate_sort_key) - - -def _text_url_candidates(html: str) -> list[str]: - candidates: list[str] = [] - for match in URL_RE.findall(html): - cleaned = match.rstrip(".,);]}>") - if "pdf" not in cleaned.lower(): - continue - candidates.append(cleaned) - return candidates - - -def _candidate_sort_key(candidate: str) -> tuple[int, str]: - lowered = candidate.lower() - if looks_like_pdf_url(candidate): - return (0, lowered) - if any(token in lowered for token in ("doi", "full", "article", "download")): - return (1, lowered) - return (2, lowered) - - -def _is_html_response(response) -> bool: - content_type = str(response.headers.get("content-type") or "").lower() - return "text/html" in content_type or "application/xhtml+xml" in content_type - - -async def _fetch_page_html(client, *, page_url: str) -> str | None: - await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds) - response = await client.get(page_url, follow_redirects=True) - if response.status_code != 200 or not _is_html_response(response): - return None - text = response.text or "" - return text[: max(int(settings.unpaywall_pdf_discovery_max_html_bytes), 0)] - - -async def _candidate_is_pdf(client, *, candidate_url: str) -> bool: - if looks_like_pdf_url(candidate_url): - return True - await wait_for_unpaywall_slot(min_interval_seconds=settings.unpaywall_min_interval_seconds) - response = await client.get(candidate_url, follow_redirects=True) - content_type = str(response.headers.get("content-type") or "").lower() - return response.status_code == 200 and PDF_MIME in content_type - - -def _candidate_limit() -> int: - return max(int(settings.unpaywall_pdf_discovery_max_candidates), 1) - - -async def _resolve_pdf_from_candidate_page(client, *, candidate_url: str) -> str | None: - html = await _fetch_page_html(client, page_url=candidate_url) - if not html: - return None - nested_candidates = _normalized_candidate_urls(page_url=candidate_url, html=html) - for nested in nested_candidates[: _candidate_limit()]: - if await _candidate_is_pdf(client, candidate_url=nested): - return nested - return None - - -async def resolve_pdf_from_landing_page(client, *, page_url: str) -> str | None: - if not settings.unpaywall_pdf_discovery_enabled: - return None - html = await _fetch_page_html(client, page_url=page_url) - if not html: - return None - candidates = _normalized_candidate_urls(page_url=page_url, html=html) - for candidate in candidates[: _candidate_limit()]: - if await _candidate_is_pdf(client, candidate_url=candidate): - return candidate - nested_pdf = await _resolve_pdf_from_candidate_page(client, candidate_url=candidate) - if nested_pdf: - return nested_pdf - return None diff --git a/build/lib/app/services/domains/unpaywall/rate_limit.py b/build/lib/app/services/domains/unpaywall/rate_limit.py deleted file mode 100644 index 76d8a3d..0000000 --- a/build/lib/app/services/domains/unpaywall/rate_limit.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import annotations - -import asyncio -import time - -_REQUEST_LOCK = asyncio.Lock() -_LAST_REQUEST_AT = 0.0 - - -async def wait_for_unpaywall_slot(*, min_interval_seconds: float) -> None: - global _LAST_REQUEST_AT - interval = max(float(min_interval_seconds), 0.0) - async with _REQUEST_LOCK: - elapsed = time.monotonic() - _LAST_REQUEST_AT - remaining = interval - elapsed - if remaining > 0: - await asyncio.sleep(remaining) - _LAST_REQUEST_AT = time.monotonic() diff --git a/build/lib/app/services/domains/users/__init__.py b/build/lib/app/services/domains/users/__init__.py deleted file mode 100644 index 9d48db4..0000000 --- a/build/lib/app/services/domains/users/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from __future__ import annotations diff --git a/build/lib/app/services/domains/users/application.py b/build/lib/app/services/domains/users/application.py deleted file mode 100644 index 7f019c4..0000000 --- a/build/lib/app/services/domains/users/application.py +++ /dev/null @@ -1,98 +0,0 @@ -from __future__ import annotations - -import re - -from sqlalchemy import select -from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import User - -EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") - - -class UserServiceError(ValueError): - """Raised for expected user-management validation failures.""" - - -def normalize_email(value: str) -> str: - return value.strip().lower() - - -def validate_email(value: str) -> str: - email = normalize_email(value) - if not EMAIL_PATTERN.fullmatch(email): - raise UserServiceError("Enter a valid email address.") - return email - - -def validate_password(value: str) -> str: - password = value.strip() - if len(password) < 8: - raise UserServiceError("Password must be at least 8 characters.") - return password - - -async def get_user_by_id(db_session: AsyncSession, user_id: int) -> User | None: - result = await db_session.execute(select(User).where(User.id == user_id)) - return result.scalar_one_or_none() - - -async def get_user_by_email(db_session: AsyncSession, email: str) -> User | None: - result = await db_session.execute( - select(User).where(User.email == normalize_email(email)) - ) - return result.scalar_one_or_none() - - -async def list_users(db_session: AsyncSession) -> list[User]: - result = await db_session.execute(select(User).order_by(User.email.asc())) - return list(result.scalars().all()) - - -async def create_user( - db_session: AsyncSession, - *, - email: str, - password_hash: str, - is_admin: bool, -) -> User: - user = User( - email=validate_email(email), - password_hash=password_hash, - is_admin=is_admin, - is_active=True, - ) - db_session.add(user) - try: - await db_session.commit() - except IntegrityError as exc: - await db_session.rollback() - raise UserServiceError("A user with that email already exists.") from exc - await db_session.refresh(user) - return user - - -async def set_user_active( - db_session: AsyncSession, - *, - user: User, - is_active: bool, -) -> User: - user.is_active = is_active - await db_session.commit() - await db_session.refresh(user) - return user - - -async def set_user_password_hash( - db_session: AsyncSession, - *, - user: User, - password_hash: str, -) -> User: - user.password_hash = password_hash - await db_session.commit() - await db_session.refresh(user) - return user - diff --git a/build/lib/app/settings.py b/build/lib/app/settings.py deleted file mode 100644 index f56ff51..0000000 --- a/build/lib/app/settings.py +++ /dev/null @@ -1,267 +0,0 @@ -from dataclasses import dataclass -import os - -DEFAULT_SECURITY_PERMISSIONS_POLICY = ( - "accelerometer=(), autoplay=(), camera=(), display-capture=(), " - "geolocation=(), gyroscope=(), microphone=(), payment=(), usb=()" -) -DEFAULT_SECURITY_CSP_POLICY = ( - "default-src 'self'; " - "base-uri 'self'; " - "form-action 'self'; " - "frame-ancestors 'none'; " - "img-src 'self' data: https:; " - "script-src 'self'; " - "style-src 'self' 'unsafe-inline'; " - "font-src 'self' data:; " - "connect-src 'self'; " - "object-src 'none'" -) -DEFAULT_SECURITY_CSP_DOCS_POLICY = ( - "default-src 'self'; " - "img-src 'self' data: https:; " - "script-src 'self' 'unsafe-inline'; " - "style-src 'self' 'unsafe-inline'; " - "font-src 'self' data:; " - "connect-src 'self' https:; " - "object-src 'none'; " - "frame-ancestors 'none'" -) - - -def _env_bool(name: str, default: bool) -> bool: - value = os.getenv(name) - if value is None: - return default - return value.strip().lower() in {"1", "true", "yes", "on"} - - -def _env_int(name: str, default: int) -> int: - value = os.getenv(name) - if value is None: - return default - return int(value) - - -def _env_float(name: str, default: float) -> float: - value = os.getenv(name) - if value is None: - return default - return float(value) - - -def _env_str(name: str, default: str) -> str: - value = os.getenv(name) - if value is None: - return default - return value.strip() or default - - -@dataclass(frozen=True) -class Settings: - app_name: str = os.getenv("APP_NAME", "scholarr") - database_url: str = os.getenv( - "DATABASE_URL", - "postgresql+asyncpg://scholar:scholar@db:5432/scholar", - ) - database_pool_mode: str = _env_str("DATABASE_POOL_MODE", "auto") - database_pool_size: int = _env_int("DATABASE_POOL_SIZE", 5) - database_pool_max_overflow: int = _env_int("DATABASE_POOL_MAX_OVERFLOW", 10) - database_pool_timeout_seconds: int = _env_int("DATABASE_POOL_TIMEOUT_SECONDS", 30) - session_secret_key: str = os.getenv("SESSION_SECRET_KEY", "dev-insecure-session-key") - session_cookie_secure: bool = _env_bool("SESSION_COOKIE_SECURE", False) - security_headers_enabled: bool = _env_bool("SECURITY_HEADERS_ENABLED", True) - security_x_content_type_options: str = _env_str("SECURITY_X_CONTENT_TYPE_OPTIONS", "nosniff") - security_x_frame_options: str = _env_str("SECURITY_X_FRAME_OPTIONS", "DENY") - security_referrer_policy: str = _env_str("SECURITY_REFERRER_POLICY", "strict-origin-when-cross-origin") - security_permissions_policy: str = _env_str( - "SECURITY_PERMISSIONS_POLICY", - DEFAULT_SECURITY_PERMISSIONS_POLICY, - ) - security_cross_origin_opener_policy: str = _env_str( - "SECURITY_CROSS_ORIGIN_OPENER_POLICY", - "same-origin", - ) - security_cross_origin_resource_policy: str = _env_str( - "SECURITY_CROSS_ORIGIN_RESOURCE_POLICY", - "same-origin", - ) - security_csp_enabled: bool = _env_bool("SECURITY_CSP_ENABLED", True) - security_csp_policy: str = _env_str("SECURITY_CSP_POLICY", DEFAULT_SECURITY_CSP_POLICY) - security_csp_docs_policy: str = _env_str("SECURITY_CSP_DOCS_POLICY", DEFAULT_SECURITY_CSP_DOCS_POLICY) - security_csp_report_only: bool = _env_bool("SECURITY_CSP_REPORT_ONLY", False) - security_strict_transport_security_enabled: bool = _env_bool( - "SECURITY_STRICT_TRANSPORT_SECURITY_ENABLED", - False, - ) - security_strict_transport_security_max_age: int = _env_int( - "SECURITY_STRICT_TRANSPORT_SECURITY_MAX_AGE", - 31_536_000, - ) - security_strict_transport_security_include_subdomains: bool = _env_bool( - "SECURITY_STRICT_TRANSPORT_SECURITY_INCLUDE_SUBDOMAINS", - True, - ) - security_strict_transport_security_preload: bool = _env_bool( - "SECURITY_STRICT_TRANSPORT_SECURITY_PRELOAD", - False, - ) - login_rate_limit_attempts: int = _env_int("LOGIN_RATE_LIMIT_ATTEMPTS", 5) - login_rate_limit_window_seconds: int = _env_int( - "LOGIN_RATE_LIMIT_WINDOW_SECONDS", - 60, - ) - log_level: str = _env_str("LOG_LEVEL", "INFO") - log_format: str = _env_str("LOG_FORMAT", "console") - log_requests: bool = _env_bool("LOG_REQUESTS", True) - log_uvicorn_access: bool = _env_bool("LOG_UVICORN_ACCESS", False) - log_request_skip_paths: str = _env_str("LOG_REQUEST_SKIP_PATHS", "/healthz") - log_redact_fields: str = os.getenv("LOG_REDACT_FIELDS", "") - scheduler_enabled: bool = _env_bool("SCHEDULER_ENABLED", True) - scheduler_tick_seconds: int = _env_int("SCHEDULER_TICK_SECONDS", 60) - ingestion_automation_allowed: bool = _env_bool( - "INGESTION_AUTOMATION_ALLOWED", - True, - ) - ingestion_manual_run_allowed: bool = _env_bool( - "INGESTION_MANUAL_RUN_ALLOWED", - True, - ) - ingestion_min_run_interval_minutes: int = _env_int( - "INGESTION_MIN_RUN_INTERVAL_MINUTES", - 15, - ) - ingestion_min_request_delay_seconds: int = _env_int( - "INGESTION_MIN_REQUEST_DELAY_SECONDS", - 2, - ) - ingestion_network_error_retries: int = _env_int("INGESTION_NETWORK_ERROR_RETRIES", 1) - ingestion_retry_backoff_seconds: float = _env_float( - "INGESTION_RETRY_BACKOFF_SECONDS", - 1.0, - ) - ingestion_rate_limit_retries: int = _env_int("INGESTION_RATE_LIMIT_RETRIES", 3) - ingestion_rate_limit_backoff_seconds: float = _env_float( - "INGESTION_RATE_LIMIT_BACKOFF_SECONDS", - 30.0, - ) - ingestion_max_pages_per_scholar: int = _env_int( - "INGESTION_MAX_PAGES_PER_SCHOLAR", - 30, - ) - ingestion_page_size: int = _env_int("INGESTION_PAGE_SIZE", 100) - ingestion_alert_blocked_failure_threshold: int = _env_int( - "INGESTION_ALERT_BLOCKED_FAILURE_THRESHOLD", - 1, - ) - ingestion_alert_network_failure_threshold: int = _env_int( - "INGESTION_ALERT_NETWORK_FAILURE_THRESHOLD", - 2, - ) - ingestion_alert_retry_scheduled_threshold: int = _env_int( - "INGESTION_ALERT_RETRY_SCHEDULED_THRESHOLD", - 3, - ) - ingestion_safety_cooldown_blocked_seconds: int = _env_int( - "INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS", - 1800, - ) - ingestion_safety_cooldown_network_seconds: int = _env_int( - "INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS", - 900, - ) - ingestion_continuation_queue_enabled: bool = _env_bool( - "INGESTION_CONTINUATION_QUEUE_ENABLED", - True, - ) - ingestion_continuation_base_delay_seconds: int = _env_int( - "INGESTION_CONTINUATION_BASE_DELAY_SECONDS", - 120, - ) - ingestion_continuation_max_delay_seconds: int = _env_int( - "INGESTION_CONTINUATION_MAX_DELAY_SECONDS", - 3600, - ) - ingestion_continuation_max_attempts: int = _env_int( - "INGESTION_CONTINUATION_MAX_ATTEMPTS", - 6, - ) - scheduler_queue_batch_size: int = _env_int("SCHEDULER_QUEUE_BATCH_SIZE", 10) - scheduler_pdf_queue_batch_size: int = _env_int("SCHEDULER_PDF_QUEUE_BATCH_SIZE", 15) - frontend_enabled: bool = _env_bool("FRONTEND_ENABLED", True) - frontend_dist_dir: str = _env_str("FRONTEND_DIST_DIR", "/app/frontend/dist") - scholar_image_upload_dir: str = _env_str( - "SCHOLAR_IMAGE_UPLOAD_DIR", - "/tmp/scholarr_uploads/scholar_images", - ) - scholar_image_upload_max_bytes: int = _env_int( - "SCHOLAR_IMAGE_UPLOAD_MAX_BYTES", - 2_000_000, - ) - scholar_name_search_enabled: bool = _env_bool("SCHOLAR_NAME_SEARCH_ENABLED", True) - scholar_name_search_cache_ttl_seconds: int = _env_int( - "SCHOLAR_NAME_SEARCH_CACHE_TTL_SECONDS", - 21_600, - ) - scholar_name_search_blocked_cache_ttl_seconds: int = _env_int( - "SCHOLAR_NAME_SEARCH_BLOCKED_CACHE_TTL_SECONDS", - 300, - ) - scholar_name_search_cache_max_entries: int = _env_int( - "SCHOLAR_NAME_SEARCH_CACHE_MAX_ENTRIES", - 512, - ) - scholar_name_search_min_interval_seconds: float = _env_float( - "SCHOLAR_NAME_SEARCH_MIN_INTERVAL_SECONDS", - 8.0, - ) - scholar_name_search_interval_jitter_seconds: float = _env_float( - "SCHOLAR_NAME_SEARCH_INTERVAL_JITTER_SECONDS", - 2.0, - ) - scholar_name_search_cooldown_block_threshold: int = _env_int( - "SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD", - 1, - ) - scholar_name_search_cooldown_seconds: int = _env_int( - "SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS", - 1800, - ) - scholar_name_search_alert_retry_count_threshold: int = _env_int( - "SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD", - 2, - ) - scholar_name_search_alert_cooldown_rejections_threshold: int = _env_int( - "SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD", - 3, - ) - unpaywall_enabled: bool = _env_bool("UNPAYWALL_ENABLED", True) - unpaywall_email: str = _env_str("UNPAYWALL_EMAIL", "") - unpaywall_timeout_seconds: float = _env_float("UNPAYWALL_TIMEOUT_SECONDS", 4.0) - unpaywall_min_interval_seconds: float = _env_float("UNPAYWALL_MIN_INTERVAL_SECONDS", 0.6) - unpaywall_max_items_per_request: int = _env_int("UNPAYWALL_MAX_ITEMS_PER_REQUEST", 20) - unpaywall_retry_cooldown_seconds: int = _env_int("UNPAYWALL_RETRY_COOLDOWN_SECONDS", 1800) - pdf_auto_retry_interval_seconds: int = _env_int( - "PDF_AUTO_RETRY_INTERVAL_SECONDS", - 86_400, - ) - pdf_auto_retry_first_interval_seconds: int = _env_int( - "PDF_AUTO_RETRY_FIRST_INTERVAL_SECONDS", - 3_600, - ) - pdf_auto_retry_max_attempts: int = _env_int("PDF_AUTO_RETRY_MAX_ATTEMPTS", 3) - unpaywall_pdf_discovery_enabled: bool = _env_bool("UNPAYWALL_PDF_DISCOVERY_ENABLED", True) - unpaywall_pdf_discovery_max_candidates: int = _env_int("UNPAYWALL_PDF_DISCOVERY_MAX_CANDIDATES", 5) - unpaywall_pdf_discovery_max_html_bytes: int = _env_int("UNPAYWALL_PDF_DISCOVERY_MAX_HTML_BYTES", 500_000) - crossref_enabled: bool = _env_bool("CROSSREF_ENABLED", True) - crossref_max_rows: int = _env_int("CROSSREF_MAX_ROWS", 10) - crossref_timeout_seconds: float = _env_float("CROSSREF_TIMEOUT_SECONDS", 8.0) - crossref_min_interval_seconds: float = _env_float("CROSSREF_MIN_INTERVAL_SECONDS", 0.6) - crossref_max_lookups_per_request: int = _env_int("CROSSREF_MAX_LOOKUPS_PER_REQUEST", 8) - - openalex_api_key: str | None = os.getenv("OPENALEX_API_KEY") - crossref_api_token: str | None = os.getenv("CROSSREF_API_TOKEN") - crossref_api_mailto: str | None = os.getenv("CROSSREF_API_MAILTO") - - -settings = Settings() diff --git a/diag_pubs.py b/diag_pubs.py deleted file mode 100644 index 2417001..0000000 --- a/diag_pubs.py +++ /dev/null @@ -1,39 +0,0 @@ - -import asyncio -from sqlalchemy import select, func -from app.db.session import get_session_factory -from app.db.models import Publication, ScholarPublication - -async def check_publications(): - session_factory = get_session_factory() - async with session_factory() as session: - # Total publications - total_pubs = await session.execute(select(func.count()).select_from(Publication)) - print(f"Total Publications: {total_pubs.scalar()}") - - # Total scholar links - total_links = await session.execute(select(func.count()).select_from(ScholarPublication)) - print(f"Total Scholar-Publication Links: {total_links.scalar()}") - - # Check for orphans - orphans = await session.execute( - select(func.count()) - .select_from(Publication) - .outerjoin(ScholarPublication, Publication.id == ScholarPublication.publication_id) - .where(ScholarPublication.publication_id == None) - ) - print(f"Orphaned Publications (no link): {orphans.scalar()}") - - # Check sorting behavior - latest_links = await session.execute( - select(Publication.title_raw, ScholarPublication.created_at) - .join(ScholarPublication, Publication.id == ScholarPublication.publication_id) - .order_by(ScholarPublication.created_at.desc()) - .limit(10) - ) - print("\nLatest 10 links by created_at:") - for row in latest_links: - print(f"- {row.created_at}: {row.title_raw[:50]}") - -if __name__ == "__main__": - asyncio.run(check_publications()) diff --git a/frontend/docs/website/package-lock.json b/frontend/docs/website/package-lock.json deleted file mode 100644 index 4ff32e6..0000000 --- a/frontend/docs/website/package-lock.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "website", - "lockfileVersion": 3, - "requires": true, - "packages": {} -} diff --git a/test_enrich.py b/test_enrich.py deleted file mode 100644 index 9a76001..0000000 --- a/test_enrich.py +++ /dev/null @@ -1,42 +0,0 @@ -import asyncio -from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker -from app.settings import settings -from app.db.models import ScholarProfile -from app.services.domains.ingestion.application import ScholarIngestionService -from app.services.domains.scholar.parser_types import PublicationCandidate -from app.services.domains.openalex.client import OpenAlexClient - -async def main(): - service = ScholarIngestionService() - scholar = ScholarProfile( - id=1, - scholar_id="SaiiI5MAAAAJ", - name="Test Scholar" - ) - pubs = [ - PublicationCandidate( - title="A fast quantum mechanical algorithm for database search", - year=1996, - citation_count=1000, - authors_text="LK Grover", - venue_text="Proceedings of the 28th Annual ACM Symposium on the Theory of Computing", - cluster_id=None, - title_url=None, - pdf_url=None - ) - ] - - print("Enriching...") - try: - enriched = await service._enrich_publications_with_openalex(scholar, pubs) - for p in enriched: - print(f"Title: {p.title}") - print(f"Year: {p.year}") - print(f"Citations: {p.citation_count}") - print(f"PDF URL: {p.pdf_url}") - except Exception as e: - import traceback - traceback.print_exc() - -if __name__ == "__main__": - asyncio.run(main())