chore: remove stale build artifacts, fix gitignore, use npm ci in CI

- Remove 114 tracked build/lib/ files (stale setuptools artifact that
  broke the repo-hygiene CI guard)
- Add build/ to .gitignore
- Remove AGENTS.MD from .gitignore (track agents.md instead)
- Delete ad-hoc scripts: diag_pubs.py, test_enrich.py
- Delete orphaned frontend/docs/website/package-lock.json
- Replace npm install with npm ci in ci.yml and docs-pages.yml for
  deterministic lockfile installs

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.github/workflows/ci.yml

@@ -91,7 +91,7 @@ jobs:
 
       - name: Install frontend dependencies
         working-directory: frontend
-        run: npm install
+        run: npm ci
 
       - name: Frontend theme token policy
         working-directory: frontend
@@ -124,7 +124,7 @@ jobs:
           node-version: "20"
 
       - name: Install docs dependencies
-        run: npm install --prefix docs/website
+        run: npm ci --prefix docs/website
 
       - name: Docs build
         run: npm --prefix docs/website run build

.github/workflows/docs-pages.yml

@@ -32,7 +32,7 @@ jobs:
           node-version: "20"
 
       - name: Install docs dependencies
-        run: npm install --prefix docs/website
+        run: npm ci --prefix docs/website
 
       - name: Build docs site
         run: npm --prefix docs/website run build

.gitignore

@@ -12,9 +12,9 @@ htmlcov/
 *.log
 .DS_Store
 planning/
+build/
 frontend/node_modules/
 frontend/dist/
 frontend/.vite/
 docs/website/node_modules/
 docs/website/build/
-AGENTS.MD

build/lib/app/__init__.py

@@ -1 +0,0 @@
-

build/lib/app/api/__init__.py

@@ -1,2 +0,0 @@
-from __future__ import annotations
-

build/lib/app/api/deps.py

@@ -1,35 +0,0 @@
-from __future__ import annotations
-
-from fastapi import Depends, Request
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.api.errors import ApiException
-from app.auth import runtime as auth_runtime
-from app.db.models import User
-from app.db.session import get_db_session
-
-
-async def get_api_current_user(
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-) -> User:
-    current_user = await auth_runtime.get_authenticated_user(request, db_session)
-    if current_user is None:
-        raise ApiException(
-            status_code=401,
-            code="auth_required",
-            message="Authentication required.",
-        )
-    return current_user
-
-
-async def get_api_admin_user(
-    current_user: User = Depends(get_api_current_user),
-) -> User:
-    if not current_user.is_admin:
-        raise ApiException(
-            status_code=403,
-            code="forbidden",
-            message="Admin access required.",
-        )
-    return current_user

build/lib/app/api/errors.py

@@ -1,85 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-from fastapi import FastAPI, HTTPException, Request
-from fastapi.exception_handlers import (
-    http_exception_handler as fastapi_http_exception_handler,
-)
-from fastapi.exception_handlers import (
-    request_validation_exception_handler as fastapi_validation_exception_handler,
-)
-from fastapi.exceptions import RequestValidationError
-
-from app.api.responses import error_response
-
-
-def _is_api_path(path: str) -> bool:
-    return path.startswith("/api/")
-
-
-def _status_code_to_error_code(status_code: int) -> str:
-    mapping = {
-        400: "bad_request",
-        401: "auth_required",
-        403: "forbidden",
-        404: "not_found",
-        409: "conflict",
-        422: "validation_error",
-        429: "rate_limited",
-        500: "internal_error",
-    }
-    return mapping.get(status_code, "error")
-
-
-class ApiException(Exception):
-    def __init__(
-        self,
-        *,
-        status_code: int,
-        code: str,
-        message: str,
-        details: Any | None = None,
-    ) -> None:
-        super().__init__(message)
-        self.status_code = status_code
-        self.code = code
-        self.message = message
-        self.details = details
-
-
-def register_api_exception_handlers(app: FastAPI) -> None:
-    @app.exception_handler(ApiException)
-    async def _handle_api_exception(request: Request, exc: ApiException):
-        return error_response(
-            request,
-            status_code=exc.status_code,
-            code=exc.code,
-            message=exc.message,
-            details=exc.details,
-        )
-
-    @app.exception_handler(HTTPException)
-    async def _handle_http_exception(request: Request, exc: HTTPException):
-        if not _is_api_path(request.url.path):
-            return await fastapi_http_exception_handler(request, exc)
-        return error_response(
-            request,
-            status_code=exc.status_code,
-            code=_status_code_to_error_code(exc.status_code),
-            message=str(exc.detail) if exc.detail is not None else "Request failed.",
-            details=exc.detail if isinstance(exc.detail, (dict, list)) else None,
-        )
-
-    @app.exception_handler(RequestValidationError)
-    async def _handle_validation_exception(request: Request, exc: RequestValidationError):
-        if not _is_api_path(request.url.path):
-            return await fastapi_validation_exception_handler(request, exc)
-        return error_response(
-            request,
-            status_code=422,
-            code="validation_error",
-            message="Request validation failed.",
-            details=exc.errors(),
-        )
-

build/lib/app/api/media.py

@@ -1,63 +0,0 @@
-from __future__ import annotations
-
-import mimetypes
-
-from fastapi import APIRouter, Depends
-from fastapi.responses import FileResponse
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.api.deps import get_api_current_user
-from app.api.errors import ApiException
-from app.db.models import User
-from app.db.session import get_db_session
-from app.services.domains.scholars import application as scholar_service
-from app.settings import settings
-
-router = APIRouter(tags=["media"])
-
-
-@router.get("/scholar-images/{scholar_profile_id}/upload")
-async def get_uploaded_scholar_image(
-    scholar_profile_id: int,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    profile = await scholar_service.get_user_scholar_by_id(
-        db_session,
-        user_id=current_user.id,
-        scholar_profile_id=scholar_profile_id,
-    )
-    if profile is None:
-        raise ApiException(
-            status_code=404,
-            code="scholar_not_found",
-            message="Scholar not found.",
-        )
-    if not profile.profile_image_upload_path:
-        raise ApiException(
-            status_code=404,
-            code="scholar_image_not_found",
-            message="Scholar image not found.",
-        )
-
-    try:
-        image_path = scholar_service.resolve_upload_file_path(
-            upload_dir=settings.scholar_image_upload_dir,
-            relative_path=profile.profile_image_upload_path,
-        )
-    except scholar_service.ScholarServiceError as exc:
-        raise ApiException(
-            status_code=404,
-            code="scholar_image_not_found",
-            message="Scholar image not found.",
-        ) from exc
-
-    if not image_path.exists() or not image_path.is_file():
-        raise ApiException(
-            status_code=404,
-            code="scholar_image_not_found",
-            message="Scholar image not found.",
-        )
-
-    media_type = mimetypes.guess_type(str(image_path))[0] or "application/octet-stream"
-    return FileResponse(path=image_path, media_type=media_type)

build/lib/app/api/responses.py

@@ -1,64 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-from fastapi.encoders import jsonable_encoder
-from starlette.requests import Request
-from starlette.responses import JSONResponse
-
-
-def _request_id(request: Request) -> str | None:
-    request_state = getattr(request, "state", None)
-    if request_state is None:
-        return None
-    return getattr(request_state, "request_id", None)
-
-
-def success_payload(
-    request: Request,
-    *,
-    data: Any,
-) -> dict[str, Any]:
-    return {
-        "data": data,
-        "meta": {
-            "request_id": _request_id(request),
-        },
-    }
-
-
-def success_response(
-    request: Request,
-    *,
-    data: Any,
-    status_code: int = 200,
-) -> JSONResponse:
-    payload = success_payload(request, data=data)
-    return JSONResponse(
-        status_code=status_code,
-        content=jsonable_encoder(payload),
-    )
-
-
-def error_response(
-    request: Request,
-    *,
-    status_code: int,
-    code: str,
-    message: str,
-    details: Any | None = None,
-) -> JSONResponse:
-    payload = {
-        "error": {
-            "code": code,
-            "message": message,
-            "details": details,
-        },
-        "meta": {
-            "request_id": _request_id(request),
-        },
-    }
-    return JSONResponse(
-        status_code=status_code,
-        content=jsonable_encoder(payload),
-    )

build/lib/app/api/router.py

@@ -1,14 +0,0 @@
-from __future__ import annotations
-
-from fastapi import APIRouter
-
-from app.api.routers import admin, admin_dbops, auth, publications, runs, scholars, settings
-
-router = APIRouter(prefix="/api/v1")
-router.include_router(auth.router)
-router.include_router(admin.router)
-router.include_router(admin_dbops.router)
-router.include_router(scholars.router)
-router.include_router(settings.router)
-router.include_router(runs.router)
-router.include_router(publications.router)

build/lib/app/api/routers/__init__.py

@@ -1,2 +0,0 @@
-from __future__ import annotations
-

build/lib/app/api/routers/admin.py

@@ -1,202 +0,0 @@
-from __future__ import annotations
-
-import logging
-
-from fastapi import APIRouter, Depends, Request
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.api.deps import get_api_admin_user
-from app.api.errors import ApiException
-from app.api.responses import success_payload
-from app.api.schemas import (
-    AdminResetPasswordRequest,
-    AdminUserActiveUpdateRequest,
-    AdminUserCreateRequest,
-    AdminUserEnvelope,
-    AdminUsersListEnvelope,
-    MessageEnvelope,
-)
-from app.auth.deps import get_auth_service
-from app.auth.service import AuthService
-from app.db.models import User
-from app.db.session import get_db_session
-from app.services.domains.users import application as user_service
-
-logger = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/admin/users", tags=["api-admin-users"])
-
-
-def _serialize_user(user: User) -> dict[str, object]:
-    return {
-        "id": int(user.id),
-        "email": user.email,
-        "is_active": bool(user.is_active),
-        "is_admin": bool(user.is_admin),
-        "created_at": user.created_at,
-        "updated_at": user.updated_at,
-    }
-
-
-@router.get(
-    "",
-    response_model=AdminUsersListEnvelope,
-)
-async def list_users(
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    admin_user: User = Depends(get_api_admin_user),
-):
-    users = await user_service.list_users(db_session)
-    logger.info(
-        "api.admin.users_listed",
-        extra={
-            "event": "api.admin.users_listed",
-            "admin_user_id": int(admin_user.id),
-            "user_count": len(users),
-        },
-    )
-    return success_payload(
-        request,
-        data={
-            "users": [_serialize_user(user) for user in users],
-        },
-    )
-
-
-@router.post(
-    "",
-    response_model=AdminUserEnvelope,
-    status_code=201,
-)
-async def create_user(
-    payload: AdminUserCreateRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    auth_service: AuthService = Depends(get_auth_service),
-    admin_user: User = Depends(get_api_admin_user),
-):
-    try:
-        validated_email = user_service.validate_email(payload.email)
-        validated_password = user_service.validate_password(payload.password)
-        created_user = await user_service.create_user(
-            db_session,
-            email=validated_email,
-            password_hash=auth_service.hash_password(validated_password),
-            is_admin=bool(payload.is_admin),
-        )
-    except user_service.UserServiceError as exc:
-        raise ApiException(
-            status_code=400,
-            code="invalid_user_input",
-            message=str(exc),
-        ) from exc
-
-    logger.info(
-        "api.admin.user_created",
-        extra={
-            "event": "api.admin.user_created",
-            "admin_user_id": int(admin_user.id),
-            "target_user_id": int(created_user.id),
-            "target_is_admin": bool(created_user.is_admin),
-        },
-    )
-    return success_payload(
-        request,
-        data=_serialize_user(created_user),
-    )
-
-
-@router.patch(
-    "/{user_id}/active",
-    response_model=AdminUserEnvelope,
-)
-async def set_user_active(
-    user_id: int,
-    payload: AdminUserActiveUpdateRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    admin_user: User = Depends(get_api_admin_user),
-):
-    target_user = await user_service.get_user_by_id(db_session, user_id)
-    if target_user is None:
-        raise ApiException(
-            status_code=404,
-            code="user_not_found",
-            message="User not found.",
-        )
-    if (
-        int(target_user.id) == int(admin_user.id)
-        and bool(target_user.is_active)
-        and not bool(payload.is_active)
-    ):
-        raise ApiException(
-            status_code=400,
-            code="cannot_deactivate_self",
-            message="You cannot deactivate your own account.",
-        )
-    updated_user = await user_service.set_user_active(
-        db_session,
-        user=target_user,
-        is_active=bool(payload.is_active),
-    )
-    logger.info(
-        "api.admin.user_active_updated",
-        extra={
-            "event": "api.admin.user_active_updated",
-            "admin_user_id": int(admin_user.id),
-            "target_user_id": int(updated_user.id),
-            "is_active": bool(updated_user.is_active),
-        },
-    )
-    return success_payload(
-        request,
-        data=_serialize_user(updated_user),
-    )
-
-
-@router.post(
-    "/{user_id}/reset-password",
-    response_model=MessageEnvelope,
-)
-async def reset_user_password(
-    user_id: int,
-    payload: AdminResetPasswordRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    auth_service: AuthService = Depends(get_auth_service),
-    admin_user: User = Depends(get_api_admin_user),
-):
-    target_user = await user_service.get_user_by_id(db_session, user_id)
-    if target_user is None:
-        raise ApiException(
-            status_code=404,
-            code="user_not_found",
-            message="User not found.",
-        )
-    try:
-        validated_password = user_service.validate_password(payload.new_password)
-    except user_service.UserServiceError as exc:
-        raise ApiException(
-            status_code=400,
-            code="invalid_password",
-            message=str(exc),
-        ) from exc
-
-    await user_service.set_user_password_hash(
-        db_session,
-        user=target_user,
-        password_hash=auth_service.hash_password(validated_password),
-    )
-    logger.info(
-        "api.admin.user_password_reset",
-        extra={
-            "event": "api.admin.user_password_reset",
-            "admin_user_id": int(admin_user.id),
-            "target_user_id": int(target_user.id),
-        },
-    )
-    return success_payload(
-        request,
-        data={"message": f"Password reset: {target_user.email}"},
-    )

build/lib/app/api/routers/admin_dbops.py

@@ -1,349 +0,0 @@
-from __future__ import annotations
-
-import logging
-
-from fastapi import APIRouter, Depends, Path, Query, Request
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.api.deps import get_api_admin_user
-from app.api.errors import ApiException
-from app.api.responses import success_payload
-from app.api.schemas import (
-    AdminDbIntegrityEnvelope,
-    AdminPdfQueueBulkEnqueueEnvelope,
-    AdminPdfQueueRequeueEnvelope,
-    AdminPdfQueueEnvelope,
-    AdminDbRepairJobsEnvelope,
-    AdminRepairPublicationLinksEnvelope,
-    AdminRepairPublicationLinksRequest,
-)
-from app.db.models import DataRepairJob, User
-from app.db.session import get_db_session
-from app.services.domains.dbops import (
-    collect_integrity_report,
-    list_repair_jobs,
-    run_publication_link_repair,
-)
-from app.services.domains.publications import application as publication_service
-
-logger = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/admin/db", tags=["api-admin-dbops"])
-
-
-def _serialize_repair_job(job: DataRepairJob) -> dict[str, object]:
-    return {
-        "id": int(job.id),
-        "job_name": job.job_name,
-        "requested_by": job.requested_by,
-        "dry_run": bool(job.dry_run),
-        "status": job.status,
-        "scope": dict(job.scope or {}),
-        "summary": dict(job.summary or {}),
-        "error_text": job.error_text,
-        "started_at": job.started_at,
-        "finished_at": job.finished_at,
-        "created_at": job.created_at,
-        "updated_at": job.updated_at,
-    }
-
-
-def _requested_by_value(*, payload: AdminRepairPublicationLinksRequest, admin_user: User) -> str:
-    from_payload = (payload.requested_by or "").strip()
-    return from_payload or admin_user.email
-
-
-def _serialize_pdf_queue_item(item) -> dict[str, object]:
-    return {
-        "publication_id": item.publication_id,
-        "title": item.title,
-        "display_identifier": _serialize_display_identifier(item.display_identifier),
-        "pdf_url": item.pdf_url,
-        "status": item.status,
-        "attempt_count": item.attempt_count,
-        "last_failure_reason": item.last_failure_reason,
-        "last_failure_detail": item.last_failure_detail,
-        "last_source": item.last_source,
-        "requested_by_user_id": item.requested_by_user_id,
-        "requested_by_email": item.requested_by_email,
-        "queued_at": item.queued_at,
-        "last_attempt_at": item.last_attempt_at,
-        "resolved_at": item.resolved_at,
-        "updated_at": item.updated_at,
-    }
-
-
-def _serialize_display_identifier(value) -> dict[str, object] | None:
-    if value is None:
-        return None
-    return {
-        "kind": value.kind,
-        "value": value.value,
-        "label": value.label,
-        "url": value.url,
-        "confidence_score": float(value.confidence_score),
-    }
-
-
-def _requeue_response_state(*, queued: bool) -> tuple[str, str]:
-    if queued:
-        return "queued", "PDF lookup queued."
-    return "blocked", "PDF lookup is already queued or currently running."
-
-
-def _bulk_enqueue_message(*, queued_count: int, requested_count: int) -> str:
-    if requested_count == 0:
-        return "No missing-PDF publications were found."
-    if queued_count == 0:
-        return "No publications were queued; all candidates were already in-flight."
-    return f"Queued {queued_count} publication(s) for PDF lookup."
-
-
-def _resolve_pdf_queue_paging(
-    *,
-    page: int,
-    page_size: int,
-    limit: int | None,
-    offset: int | None,
-) -> tuple[int, int, int]:
-    resolved_limit = int(limit) if limit is not None else int(page_size)
-    resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0)
-    resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1)
-    return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0)
-
-
-def _pdf_queue_page_data(*, total_count: int, page: int, page_size: int, offset: int) -> dict[str, object]:
-    return {
-        "total_count": int(total_count),
-        "page": int(page),
-        "page_size": int(page_size),
-        "has_prev": int(offset) > 0,
-        "has_next": int(offset) + int(page_size) < int(total_count),
-    }
-
-
-@router.get(
-    "/integrity",
-    response_model=AdminDbIntegrityEnvelope,
-)
-async def get_integrity_report(
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    admin_user: User = Depends(get_api_admin_user),
-):
-    report = await collect_integrity_report(db_session)
-    logger.info(
-        "api.admin.db.integrity_checked",
-        extra={
-            "event": "api.admin.db.integrity_checked",
-            "admin_user_id": int(admin_user.id),
-            "status": report.get("status"),
-            "failure_count": len(report.get("failures", [])),
-            "warning_count": len(report.get("warnings", [])),
-        },
-    )
-    return success_payload(request, data=report)
-
-
-@router.get(
-    "/repair-jobs",
-    response_model=AdminDbRepairJobsEnvelope,
-)
-async def get_repair_jobs(
-    request: Request,
-    limit: int = Query(default=50, ge=1, le=200),
-    db_session: AsyncSession = Depends(get_db_session),
-    admin_user: User = Depends(get_api_admin_user),
-):
-    jobs = await list_repair_jobs(db_session, limit=limit)
-    logger.info(
-        "api.admin.db.repair_jobs_listed",
-        extra={
-            "event": "api.admin.db.repair_jobs_listed",
-            "admin_user_id": int(admin_user.id),
-            "limit": int(limit),
-            "job_count": len(jobs),
-        },
-    )
-    return success_payload(
-        request,
-        data={"jobs": [_serialize_repair_job(job) for job in jobs]},
-    )
-
-
-@router.get(
-    "/pdf-queue",
-    response_model=AdminPdfQueueEnvelope,
-)
-async def get_pdf_queue(
-    request: Request,
-    page: int = Query(default=1, ge=1),
-    page_size: int = Query(default=100, ge=1, le=500),
-    limit: int | None = Query(default=None, ge=1, le=500),
-    offset: int | None = Query(default=None, ge=0),
-    status: str | None = Query(default=None),
-    db_session: AsyncSession = Depends(get_db_session),
-    admin_user: User = Depends(get_api_admin_user),
-):
-    normalized_status = (status or "").strip().lower() or None
-    resolved_page, resolved_limit, resolved_offset = _resolve_pdf_queue_paging(
-        page=page,
-        page_size=page_size,
-        limit=limit,
-        offset=offset,
-    )
-    queue_page = await publication_service.list_pdf_queue_page(
-        db_session,
-        limit=resolved_limit,
-        offset=resolved_offset,
-        status=normalized_status,
-    )
-    logger.info(
-        "api.admin.db.pdf_queue_listed",
-        extra={
-            "event": "api.admin.db.pdf_queue_listed",
-            "admin_user_id": int(admin_user.id),
-            "page": int(resolved_page),
-            "page_size": int(resolved_limit),
-            "offset": int(resolved_offset),
-            "status": normalized_status,
-            "item_count": len(queue_page.items),
-            "total_count": int(queue_page.total_count),
-        },
-    )
-    return success_payload(
-        request,
-        data={
-            "items": [_serialize_pdf_queue_item(item) for item in queue_page.items],
-            **_pdf_queue_page_data(
-                total_count=queue_page.total_count,
-                page=resolved_page,
-                page_size=resolved_limit,
-                offset=resolved_offset,
-            ),
-        },
-    )
-
-
-@router.post(
-    "/pdf-queue/{publication_id}/requeue",
-    response_model=AdminPdfQueueRequeueEnvelope,
-)
-async def requeue_pdf_lookup(
-    request: Request,
-    publication_id: int = Path(ge=1),
-    db_session: AsyncSession = Depends(get_db_session),
-    admin_user: User = Depends(get_api_admin_user),
-):
-    result = await publication_service.enqueue_retry_pdf_job_for_publication_id(
-        db_session,
-        user_id=int(admin_user.id),
-        request_email=admin_user.email,
-        publication_id=publication_id,
-    )
-    if not result.publication_exists:
-        raise ApiException(
-            status_code=404,
-            code="publication_not_found",
-            message="Publication not found.",
-        )
-    status, message = _requeue_response_state(queued=result.queued)
-    logger.info(
-        "api.admin.db.pdf_queue_requeue_requested",
-        extra={
-            "event": "api.admin.db.pdf_queue_requeue_requested",
-            "admin_user_id": int(admin_user.id),
-            "publication_id": int(publication_id),
-            "queued": bool(result.queued),
-        },
-    )
-    return success_payload(
-        request,
-        data={
-            "publication_id": int(publication_id),
-            "queued": bool(result.queued),
-            "status": status,
-            "message": message,
-        },
-    )
-
-
-@router.post(
-    "/pdf-queue/requeue-all",
-    response_model=AdminPdfQueueBulkEnqueueEnvelope,
-)
-async def requeue_all_missing_pdfs(
-    request: Request,
-    limit: int = Query(default=1000, ge=1, le=5000),
-    db_session: AsyncSession = Depends(get_db_session),
-    admin_user: User = Depends(get_api_admin_user),
-):
-    result = await publication_service.enqueue_all_missing_pdf_jobs(
-        db_session,
-        user_id=int(admin_user.id),
-        request_email=admin_user.email,
-        limit=limit,
-    )
-    logger.info(
-        "api.admin.db.pdf_queue_requeue_all",
-        extra={
-            "event": "api.admin.db.pdf_queue_requeue_all",
-            "admin_user_id": int(admin_user.id),
-            "limit": int(limit),
-            "requested_count": int(result.requested_count),
-            "queued_count": int(result.queued_count),
-        },
-    )
-    return success_payload(
-        request,
-        data={
-            "requested_count": int(result.requested_count),
-            "queued_count": int(result.queued_count),
-            "message": _bulk_enqueue_message(
-                queued_count=int(result.queued_count),
-                requested_count=int(result.requested_count),
-            ),
-        },
-    )
-
-
-@router.post(
-    "/repairs/publication-links",
-    response_model=AdminRepairPublicationLinksEnvelope,
-)
-async def trigger_publication_link_repair(
-    payload: AdminRepairPublicationLinksRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    admin_user: User = Depends(get_api_admin_user),
-):
-    try:
-        result = await run_publication_link_repair(
-            db_session,
-            scope_mode=payload.scope_mode,
-            user_id=payload.user_id,
-            scholar_profile_ids=payload.scholar_profile_ids,
-            dry_run=bool(payload.dry_run),
-            gc_orphan_publications=bool(payload.gc_orphan_publications),
-            requested_by=_requested_by_value(payload=payload, admin_user=admin_user),
-        )
-    except ValueError as exc:
-        raise ApiException(
-            status_code=400,
-            code="invalid_repair_scope",
-            message=str(exc),
-        ) from exc
-    logger.info(
-        "api.admin.db.publication_link_repair_triggered",
-        extra={
-            "event": "api.admin.db.publication_link_repair_triggered",
-            "admin_user_id": int(admin_user.id),
-            "scope_mode": payload.scope_mode,
-            "target_user_id": int(payload.user_id) if payload.user_id is not None else None,
-            "dry_run": bool(payload.dry_run),
-            "gc_orphan_publications": bool(payload.gc_orphan_publications),
-            "job_id": int(result["job_id"]),
-            "status": result["status"],
-        },
-    )
-    return success_payload(request, data=result)

build/lib/app/api/routers/auth.py

@@ -1,237 +0,0 @@
-from __future__ import annotations
-
-import logging
-
-from fastapi import APIRouter, Depends, Request
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.api.errors import ApiException
-from app.api.deps import get_api_current_user
-from app.api.responses import success_payload
-from app.api.schemas import (
-    AuthMeEnvelope,
-    ChangePasswordRequest,
-    CsrfBootstrapEnvelope,
-    LoginEnvelope,
-    LoginRequest,
-    MessageEnvelope,
-)
-from app.auth.deps import get_auth_service, get_login_rate_limiter
-from app.auth.rate_limit import SlidingWindowRateLimiter
-from app.auth import runtime as auth_runtime
-from app.auth.service import AuthService
-from app.auth.session import set_session_user
-from app.db.models import User
-from app.db.session import get_db_session
-from app.security.csrf import ensure_csrf_token
-from app.services.domains.users import application as user_service
-
-logger = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/auth", tags=["api-auth"])
-
-
-def _login_limiter_key_and_email(request: Request, payload: LoginRequest) -> tuple[str, str]:
-    return auth_runtime.login_rate_limit_key(request, payload.email), payload.email.strip().lower()
-
-
-def _raise_rate_limited(normalized_email: str, retry_after_seconds: int) -> None:
-    logger.warning(
-        "api.auth.login_rate_limited",
-        extra={
-            "event": "api.auth.login_rate_limited",
-            "email": normalized_email,
-            "retry_after_seconds": retry_after_seconds,
-        },
-    )
-    raise ApiException(
-        status_code=429,
-        code="rate_limited",
-        message="Too many login attempts. Please try again later.",
-        details={"retry_after_seconds": retry_after_seconds},
-    )
-
-
-def _serialize_user_payload(user: User) -> dict[str, object]:
-    return {
-        "id": int(user.id),
-        "email": user.email,
-        "is_admin": bool(user.is_admin),
-        "is_active": bool(user.is_active),
-    }
-
-
-def _raise_invalid_credentials(*, normalized_email: str) -> None:
-    logger.info(
-        "api.auth.login_failed",
-        extra={"event": "api.auth.login_failed", "email": normalized_email},
-    )
-    raise ApiException(
-        status_code=401,
-        code="invalid_credentials",
-        message="Invalid email or password.",
-    )
-
-
-@router.post(
-    "/login",
-    response_model=LoginEnvelope,
-)
-async def login(
-    payload: LoginRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    auth_service: AuthService = Depends(get_auth_service),
-    rate_limiter: SlidingWindowRateLimiter = Depends(get_login_rate_limiter),
-):
-    limiter_key, normalized_email = _login_limiter_key_and_email(request, payload)
-    decision = rate_limiter.check(limiter_key)
-    if not decision.allowed:
-        _raise_rate_limited(normalized_email, int(decision.retry_after_seconds))
-
-    user = await auth_service.authenticate_user(
-        db_session,
-        email=payload.email,
-        password=payload.password,
-    )
-    if user is None:
-        rate_limiter.record_failure(limiter_key)
-        _raise_invalid_credentials(normalized_email=normalized_email)
-
-    rate_limiter.reset(limiter_key)
-    set_session_user(
-        request,
-        user_id=user.id,
-        email=user.email,
-        is_admin=user.is_admin,
-    )
-    logger.info(
-        "api.auth.login_succeeded",
-        extra={
-            "event": "api.auth.login_succeeded",
-            "user_id": user.id,
-            "is_admin": user.is_admin,
-        },
-    )
-    return success_payload(
-        request,
-        data={
-            "authenticated": True,
-            "csrf_token": ensure_csrf_token(request),
-            "user": _serialize_user_payload(user),
-        },
-    )
-
-
-@router.get(
-    "/me",
-    response_model=AuthMeEnvelope,
-)
-async def get_current_session(
-    request: Request,
-    current_user: User = Depends(get_api_current_user),
-):
-    return success_payload(
-        request,
-        data={
-            "authenticated": True,
-            "csrf_token": ensure_csrf_token(request),
-            "user": _serialize_user_payload(current_user),
-        },
-    )
-
-
-@router.get(
-    "/csrf",
-    response_model=CsrfBootstrapEnvelope,
-)
-async def get_csrf_bootstrap(
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-):
-    current_user = await auth_runtime.get_authenticated_user(request, db_session)
-    return success_payload(
-        request,
-        data={
-            "csrf_token": ensure_csrf_token(request),
-            "authenticated": current_user is not None,
-        },
-    )
-
-
-@router.post(
-    "/change-password",
-    response_model=MessageEnvelope,
-)
-async def change_password(
-    payload: ChangePasswordRequest,
-    request: Request,
-    current_user: User = Depends(get_api_current_user),
-    db_session: AsyncSession = Depends(get_db_session),
-    auth_service: AuthService = Depends(get_auth_service),
-):
-    if not auth_service.verify_password(
-        password_hash=current_user.password_hash,
-        password=payload.current_password,
-    ):
-        raise ApiException(
-            status_code=400,
-            code="invalid_current_password",
-            message="Current password is incorrect.",
-        )
-    if payload.new_password != payload.confirm_password:
-        raise ApiException(
-            status_code=400,
-            code="password_confirmation_mismatch",
-            message="New password and confirmation do not match.",
-        )
-    try:
-        validated_password = user_service.validate_password(payload.new_password)
-    except user_service.UserServiceError as exc:
-        raise ApiException(
-            status_code=400,
-            code="invalid_password",
-            message=str(exc),
-        ) from exc
-
-    await user_service.set_user_password_hash(
-        db_session,
-        user=current_user,
-        password_hash=auth_service.hash_password(validated_password),
-    )
-    logger.info(
-        "api.auth.password_changed",
-        extra={
-            "event": "api.auth.password_changed",
-            "user_id": int(current_user.id),
-        },
-    )
-    return success_payload(
-        request,
-        data={"message": "Password updated successfully."},
-    )
-
-
-@router.post(
-    "/logout",
-    response_model=MessageEnvelope,
-)
-async def logout(
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-):
-    current_user = await auth_runtime.get_authenticated_user(request, db_session)
-    auth_runtime.invalidate_session(request)
-    logger.info(
-        "api.auth.logout",
-        extra={
-            "event": "api.auth.logout",
-            "user_id": int(current_user.id) if current_user is not None else None,
-        },
-    )
-    return success_payload(
-        request,
-        data={
-            "message": "Logged out.",
-        },
-    )

build/lib/app/api/routers/publications.py

@@ -1,528 +0,0 @@
-from __future__ import annotations
-
-import logging
-from typing import Literal
-
-from fastapi import APIRouter, Depends, Path, Query, Request
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.api.deps import get_api_current_user
-from app.api.errors import ApiException
-from app.api.responses import success_payload
-from app.api.schemas import (
-    MarkAllReadEnvelope,
-    MarkSelectedReadEnvelope,
-    MarkSelectedReadRequest,
-    PublicationsListEnvelope,
-    RetryPublicationPdfEnvelope,
-    RetryPublicationPdfRequest,
-    TogglePublicationFavoriteEnvelope,
-    TogglePublicationFavoriteRequest,
-)
-from app.db.models import User
-from app.db.session import get_db_session
-from app.services.domains.publication_identifiers import application as identifier_service
-from app.services.domains.publications import application as publication_service
-from app.services.domains.scholars import application as scholar_service
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/publications", tags=["api-publications"])
-
-
-async def _require_selected_profile(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    selected_scholar_id: int | None,
-) -> None:
-    if selected_scholar_id is None:
-        return
-    selected_profile = await scholar_service.get_user_scholar_by_id(
-        db_session,
-        user_id=user_id,
-        scholar_profile_id=selected_scholar_id,
-    )
-    if selected_profile is None:
-        raise ApiException(
-            status_code=404,
-            code="scholar_not_found",
-            message="Scholar filter not found.",
-        )
-
-
-def _serialize_publication_item(item) -> dict[str, object]:
-    return {
-        "publication_id": item.publication_id,
-        "scholar_profile_id": item.scholar_profile_id,
-        "scholar_label": item.scholar_label,
-        "title": item.title,
-        "year": item.year,
-        "citation_count": item.citation_count,
-        "venue_text": item.venue_text,
-        "pub_url": item.pub_url,
-        "display_identifier": _serialize_display_identifier(item.display_identifier),
-        "pdf_url": item.pdf_url,
-        "pdf_status": item.pdf_status,
-        "pdf_attempt_count": item.pdf_attempt_count,
-        "pdf_failure_reason": item.pdf_failure_reason,
-        "pdf_failure_detail": item.pdf_failure_detail,
-        "is_read": item.is_read,
-        "is_favorite": item.is_favorite,
-        "first_seen_at": item.first_seen_at,
-        "is_new_in_latest_run": item.is_new_in_latest_run,
-    }
-
-
-def _serialize_display_identifier(value) -> dict[str, object] | None:
-    if value is None:
-        return None
-    return {
-        "kind": value.kind,
-        "value": value.value,
-        "label": value.label,
-        "url": value.url,
-        "confidence_score": float(value.confidence_score),
-    }
-
-
-async def _publication_counts(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    selected_scholar_id: int | None,
-    favorite_only: bool,
-) -> tuple[int, int, int, int]:
-    unread_count = await publication_service.count_unread_for_user(
-        db_session,
-        user_id=user_id,
-        scholar_profile_id=selected_scholar_id,
-        favorite_only=favorite_only,
-    )
-    favorites_count = await publication_service.count_favorite_for_user(
-        db_session,
-        user_id=user_id,
-        scholar_profile_id=selected_scholar_id,
-    )
-    latest_count = await publication_service.count_latest_for_user(
-        db_session,
-        user_id=user_id,
-        scholar_profile_id=selected_scholar_id,
-        favorite_only=favorite_only,
-    )
-    total_count = await publication_service.count_for_user(
-        db_session,
-        user_id=user_id,
-        mode=publication_service.MODE_ALL,
-        scholar_profile_id=selected_scholar_id,
-        favorite_only=favorite_only,
-    )
-    return unread_count, favorites_count, latest_count, total_count
-
-
-async def _list_publications_for_request(
-    db_session: AsyncSession,
-    *,
-    current_user: User,
-    mode: Literal["all", "unread", "latest", "new"] | None,
-    favorite_only: bool,
-    scholar_profile_id: int | None,
-    limit: int,
-    offset: int,
-) -> tuple[str, int | None, list]:
-    resolved_mode = publication_service.resolve_publication_view_mode(mode)
-    selected_scholar_id = scholar_profile_id
-    await _require_selected_profile(
-        db_session,
-        user_id=current_user.id,
-        selected_scholar_id=selected_scholar_id,
-    )
-    publications = await publication_service.list_for_user(
-        db_session,
-        user_id=current_user.id,
-        mode=resolved_mode,
-        scholar_profile_id=selected_scholar_id,
-        favorite_only=favorite_only,
-        limit=limit,
-        offset=offset,
-    )
-    await publication_service.schedule_missing_pdf_enrichment_for_user(
-        db_session,
-        user_id=current_user.id,
-        request_email=current_user.email,
-        items=publications,
-        max_items=settings.unpaywall_max_items_per_request,
-    )
-    hydrated = await publication_service.hydrate_pdf_enrichment_state(
-        db_session,
-        items=publications,
-    )
-    return resolved_mode, selected_scholar_id, hydrated
-
-
-def _resolve_publications_paging(
-    *,
-    page: int,
-    page_size: int,
-    limit: int | None,
-    offset: int | None,
-) -> tuple[int, int, int]:
-    resolved_limit = int(limit) if limit is not None else int(page_size)
-    resolved_offset = int(offset) if offset is not None else max((int(page) - 1) * resolved_limit, 0)
-    resolved_page = max((resolved_offset // max(resolved_limit, 1)) + 1, 1)
-    return resolved_page, max(resolved_limit, 1), max(resolved_offset, 0)
-
-
-def _publications_list_data(
-    *,
-    mode: str,
-    favorite_only: bool,
-    selected_scholar_id: int | None,
-    unread_count: int,
-    favorites_count: int,
-    latest_count: int,
-    total_count: int,
-    publications: list,
-    page: int,
-    page_size: int,
-    offset: int,
-) -> dict[str, object]:
-    return {
-        "mode": mode,
-        "favorite_only": favorite_only,
-        "selected_scholar_profile_id": selected_scholar_id,
-        "unread_count": unread_count,
-        "favorites_count": favorites_count,
-        "latest_count": latest_count,
-        "new_count": latest_count,
-        "total_count": total_count,
-        "page": int(page),
-        "page_size": int(page_size),
-        "has_prev": int(offset) > 0,
-        "has_next": int(offset) + int(page_size) < int(total_count),
-        "publications": [_serialize_publication_item(item) for item in publications],
-    }
-
-
-def _retry_pdf_message(*, queued: bool, resolved_pdf: bool, pdf_status: str) -> str:
-    if resolved_pdf:
-        return "Open-access PDF link already resolved."
-    if queued:
-        return "PDF lookup queued."
-    if pdf_status in {"queued", "running"}:
-        return "PDF lookup is already queued."
-    return "No open-access PDF link found."
-
-
-def _favorite_message(*, is_favorite: bool) -> str:
-    if is_favorite:
-        return "Publication marked as favorite."
-    return "Publication removed from favorites."
-
-
-async def _retry_publication_state(
-    db_session: AsyncSession,
-    *,
-    current_user: User,
-    scholar_profile_id: int,
-    publication_id: int,
-):
-    publication = await publication_service.retry_pdf_for_user(
-        db_session,
-        user_id=current_user.id,
-        scholar_profile_id=scholar_profile_id,
-        publication_id=publication_id,
-    )
-    if publication is None:
-        raise ApiException(
-            status_code=404,
-            code="publication_not_found",
-            message="Publication not found.",
-        )
-    queued = False
-    if not publication.pdf_url:
-        queued = await publication_service.schedule_retry_pdf_enrichment_for_row(
-            db_session,
-            user_id=current_user.id,
-            request_email=current_user.email,
-            item=publication,
-        )
-    hydrated = await publication_service.hydrate_pdf_enrichment_state(
-        db_session,
-        items=[publication],
-    )
-    current = hydrated[0] if hydrated else publication
-    return current, queued
-
-
-async def _favorite_publication_state(
-    db_session: AsyncSession,
-    *,
-    current_user: User,
-    scholar_profile_id: int,
-    publication_id: int,
-    is_favorite: bool,
-):
-    updated_count = await publication_service.set_publication_favorite_for_user(
-        db_session,
-        user_id=current_user.id,
-        scholar_profile_id=scholar_profile_id,
-        publication_id=publication_id,
-        is_favorite=is_favorite,
-    )
-    if updated_count <= 0:
-        raise ApiException(
-            status_code=404,
-            code="publication_not_found",
-            message="Publication not found.",
-        )
-    publication = await publication_service.get_publication_item_for_user(
-        db_session,
-        user_id=current_user.id,
-        scholar_profile_id=scholar_profile_id,
-        publication_id=publication_id,
-    )
-    if publication is None:
-        raise ApiException(
-            status_code=404,
-            code="publication_not_found",
-            message="Publication not found.",
-        )
-    hydrated = await publication_service.hydrate_pdf_enrichment_state(
-        db_session,
-        items=[publication],
-    )
-    current = hydrated[0] if hydrated else publication
-    identifiers = await identifier_service.overlay_publication_items_with_display_identifiers(
-        db_session,
-        items=[current],
-    )
-    return identifiers[0] if identifiers else current
-
-
-def _log_retry_pdf_result(
-    *,
-    current_user: User,
-    scholar_profile_id: int,
-    publication_id: int,
-    queued: bool,
-    resolved_pdf: bool,
-    pdf_status: str,
-    doi: str | None,
-) -> None:
-    logger.info(
-        "api.publications.retry_pdf",
-        extra={
-            "event": "api.publications.retry_pdf",
-            "user_id": current_user.id,
-            "scholar_profile_id": scholar_profile_id,
-            "publication_id": publication_id,
-            "queued": queued,
-            "resolved_pdf": resolved_pdf,
-            "pdf_status": pdf_status,
-        },
-    )
-
-
-@router.get(
-    "",
-    response_model=PublicationsListEnvelope,
-)
-async def list_publications(
-    request: Request,
-    mode: Literal["all", "unread", "latest", "new"] | None = Query(default=None),
-    favorite_only: bool = Query(default=False),
-    scholar_profile_id: int | None = Query(default=None, ge=1),
-    page: int = Query(default=1, ge=1),
-    page_size: int = Query(default=100, ge=1, le=500),
-    limit: int | None = Query(default=None, ge=1, le=1000),
-    offset: int | None = Query(default=None, ge=0),
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    resolved_page, resolved_limit, resolved_offset = _resolve_publications_paging(
-        page=page,
-        page_size=page_size,
-        limit=limit,
-        offset=offset,
-    )
-    resolved_mode, selected_scholar_id, publications = await _list_publications_for_request(
-        db_session,
-        current_user=current_user,
-        mode=mode,
-        favorite_only=favorite_only,
-        scholar_profile_id=scholar_profile_id,
-        limit=resolved_limit,
-        offset=resolved_offset,
-    )
-    unread_count, favorites_count, latest_count, total_count = await _publication_counts(
-        db_session,
-        user_id=current_user.id,
-        selected_scholar_id=selected_scholar_id,
-        favorite_only=favorite_only,
-    )
-    data = _publications_list_data(
-        mode=resolved_mode,
-        favorite_only=favorite_only,
-        selected_scholar_id=selected_scholar_id,
-        unread_count=unread_count,
-        favorites_count=favorites_count,
-        latest_count=latest_count,
-        total_count=total_count,
-        publications=publications,
-        page=resolved_page,
-        page_size=resolved_limit,
-        offset=resolved_offset,
-    )
-    return success_payload(request, data=data)
-
-
-@router.post(
-    "/mark-all-read",
-    response_model=MarkAllReadEnvelope,
-)
-async def mark_all_publications_read(
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    updated_count = await publication_service.mark_all_unread_as_read_for_user(
-        db_session,
-        user_id=current_user.id,
-    )
-    logger.info(
-        "api.publications.mark_all_read",
-        extra={
-            "event": "api.publications.mark_all_read",
-            "user_id": current_user.id,
-            "updated_count": updated_count,
-        },
-    )
-    return success_payload(
-        request,
-        data={
-            "message": "Marked all unread publications as read.",
-            "updated_count": updated_count,
-        },
-    )
-
-
-@router.post(
-    "/mark-read",
-    response_model=MarkSelectedReadEnvelope,
-)
-async def mark_selected_publications_read(
-    payload: MarkSelectedReadRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    selection_pairs = sorted(
-        {
-            (int(item.scholar_profile_id), int(item.publication_id))
-            for item in payload.selections
-        }
-    )
-    updated_count = await publication_service.mark_selected_as_read_for_user(
-        db_session,
-        user_id=current_user.id,
-        selections=selection_pairs,
-    )
-    logger.info(
-        "api.publications.mark_selected_read",
-        extra={
-            "event": "api.publications.mark_selected_read",
-            "user_id": current_user.id,
-            "requested_count": len(selection_pairs),
-            "updated_count": updated_count,
-        },
-    )
-    return success_payload(
-        request,
-        data={
-            "message": "Marked selected publications as read.",
-            "requested_count": len(selection_pairs),
-            "updated_count": updated_count,
-        },
-    )
-
-
-@router.post(
-    "/{publication_id}/retry-pdf",
-    response_model=RetryPublicationPdfEnvelope,
-)
-async def retry_publication_pdf(
-    payload: RetryPublicationPdfRequest,
-    request: Request,
-    publication_id: int = Path(ge=1),
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    current, queued = await _retry_publication_state(
-        db_session,
-        current_user=current_user,
-        scholar_profile_id=payload.scholar_profile_id,
-        publication_id=publication_id,
-    )
-    resolved_pdf = bool(current.pdf_url)
-    message = _retry_pdf_message(
-        queued=queued,
-        resolved_pdf=resolved_pdf,
-        pdf_status=current.pdf_status,
-    )
-    _log_retry_pdf_result(
-        current_user=current_user,
-        scholar_profile_id=payload.scholar_profile_id,
-        publication_id=publication_id,
-        queued=queued,
-        resolved_pdf=resolved_pdf,
-        pdf_status=current.pdf_status,
-        doi=None,
-    )
-    return success_payload(
-        request,
-        data={
-            "message": message,
-            "queued": queued,
-            "resolved_pdf": resolved_pdf,
-            "publication": _serialize_publication_item(current),
-        },
-    )
-
-
-@router.post(
-    "/{publication_id}/favorite",
-    response_model=TogglePublicationFavoriteEnvelope,
-)
-async def toggle_publication_favorite(
-    payload: TogglePublicationFavoriteRequest,
-    request: Request,
-    publication_id: int = Path(ge=1),
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    current = await _favorite_publication_state(
-        db_session,
-        current_user=current_user,
-        scholar_profile_id=payload.scholar_profile_id,
-        publication_id=publication_id,
-        is_favorite=payload.is_favorite,
-    )
-    logger.info(
-        "api.publications.favorite",
-        extra={
-            "event": "api.publications.favorite",
-            "user_id": current_user.id,
-            "scholar_profile_id": payload.scholar_profile_id,
-            "publication_id": publication_id,
-            "is_favorite": bool(payload.is_favorite),
-        },
-    )
-    return success_payload(
-        request,
-        data={
-            "message": _favorite_message(is_favorite=bool(payload.is_favorite)),
-            "publication": _serialize_publication_item(current),
-        },
-    )

build/lib/app/api/routers/runs.py

@@ -1,845 +0,0 @@
-from __future__ import annotations
-
-import logging
-import re
-from typing import Any
-
-from fastapi import APIRouter, Depends, Query, Request
-from sqlalchemy.exc import IntegrityError
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.api.deps import get_api_current_user
-from app.api.errors import ApiException
-from app.api.responses import success_payload
-from fastapi.responses import StreamingResponse
-from app.api.schemas import (
-    ManualRunEnvelope,
-    QueueClearEnvelope,
-    QueueItemEnvelope,
-    QueueListEnvelope,
-    RunDetailEnvelope,
-    RunsListEnvelope,
-)
-from app.db.models import RunStatus, RunTriggerType, User
-from app.db.session import get_db_session
-from app.services.domains.ingestion import application as ingestion_service
-from app.services.domains.ingestion import safety as run_safety_service
-from app.services.domains.runs import application as run_service
-from app.services.domains.runs.events import event_generator
-from app.services.domains.settings import application as user_settings_service
-from app.settings import settings
-from app.api.runtime_deps import get_ingestion_service
-
-logger = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/runs", tags=["api-runs"])
-
-IDEMPOTENCY_HEADER = "Idempotency-Key"
-IDEMPOTENCY_MAX_LENGTH = 128
-IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
-
-
-def _int_value(value: Any, default: int = 0) -> int:
-    try:
-        return int(value)
-    except (TypeError, ValueError):
-        return default
-
-
-def _str_value(value: Any) -> str | None:
-    if value is None:
-        return None
-    text = str(value).strip()
-    return text if text else None
-
-
-def _bool_value(value: Any, default: bool = False) -> bool:
-    if isinstance(value, bool):
-        return value
-    if isinstance(value, str):
-        lowered = value.strip().lower()
-        if lowered in {"1", "true", "yes", "on"}:
-            return True
-        if lowered in {"0", "false", "no", "off"}:
-            return False
-    return default
-
-
-def _normalize_idempotency_key(raw_value: str | None) -> str | None:
-    if raw_value is None:
-        return None
-    candidate = raw_value.strip()
-    if not candidate:
-        return None
-    if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate):
-        raise ApiException(
-            status_code=400,
-            code="invalid_idempotency_key",
-            message=(
-                "Invalid Idempotency-Key. Use 8-128 characters from: "
-                "A-Z a-z 0-9 . _ : -"
-            ),
-        )
-    return candidate
-
-
-def _serialize_run(run) -> dict[str, Any]:
-    summary = run_service.extract_run_summary(run.error_log)
-    return {
-        "id": int(run.id),
-        "trigger_type": run.trigger_type.value,
-        "status": run.status.value,
-        "start_dt": run.start_dt,
-        "end_dt": run.end_dt,
-        "scholar_count": int(run.scholar_count or 0),
-        "new_publication_count": int(run.new_pub_count or 0),
-        "failed_count": int(summary["failed_count"]),
-        "partial_count": int(summary["partial_count"]),
-    }
-
-
-def _serialize_queue_item(item) -> dict[str, Any]:
-    return {
-        "id": int(item.id),
-        "scholar_profile_id": int(item.scholar_profile_id),
-        "scholar_label": item.scholar_label,
-        "status": item.status,
-        "reason": item.reason,
-        "dropped_reason": item.dropped_reason,
-        "attempt_count": int(item.attempt_count),
-        "resume_cstart": int(item.resume_cstart),
-        "next_attempt_dt": item.next_attempt_dt,
-        "updated_at": item.updated_at,
-        "last_error": item.last_error,
-        "last_run_id": item.last_run_id,
-    }
-
-
-def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]:
-    if not isinstance(value, list):
-        return []
-    normalized: list[dict[str, Any]] = []
-    for item in value:
-        if not isinstance(item, dict):
-            continue
-        normalized.append(
-            {
-                "attempt": _int_value(item.get("attempt"), 0),
-                "cstart": _int_value(item.get("cstart"), 0),
-                "state": _str_value(item.get("state")),
-                "state_reason": _str_value(item.get("state_reason")),
-                "status_code": (
-                    _int_value(item.get("status_code"))
-                    if item.get("status_code") is not None
-                    else None
-                ),
-                "fetch_error": _str_value(item.get("fetch_error")),
-            }
-        )
-    return normalized
-
-
-def _normalize_page_logs(value: Any) -> list[dict[str, Any]]:
-    if not isinstance(value, list):
-        return []
-    normalized: list[dict[str, Any]] = []
-    for item in value:
-        if not isinstance(item, dict):
-            continue
-        warning_codes = item.get("warning_codes")
-        normalized.append(
-            {
-                "page": _int_value(item.get("page"), 0),
-                "cstart": _int_value(item.get("cstart"), 0),
-                "state": _str_value(item.get("state")) or "unknown",
-                "state_reason": _str_value(item.get("state_reason")),
-                "status_code": (
-                    _int_value(item.get("status_code"))
-                    if item.get("status_code") is not None
-                    else None
-                ),
-                "publication_count": _int_value(item.get("publication_count"), 0),
-                "attempt_count": _int_value(item.get("attempt_count"), 0),
-                "has_show_more_button": _bool_value(item.get("has_show_more_button"), False),
-                "articles_range": _str_value(item.get("articles_range")),
-                "warning_codes": [
-                    str(code)
-                    for code in (warning_codes if isinstance(warning_codes, list) else [])
-                ],
-            }
-        )
-    return normalized
-
-
-def _normalize_debug(value: Any) -> dict[str, Any] | None:
-    if not isinstance(value, dict):
-        return None
-    marker_counts = value.get("marker_counts_nonzero")
-    warning_codes = value.get("warning_codes")
-    return {
-        "status_code": (
-            _int_value(value.get("status_code"))
-            if value.get("status_code") is not None
-            else None
-        ),
-        "final_url": _str_value(value.get("final_url")),
-        "fetch_error": _str_value(value.get("fetch_error")),
-        "body_sha256": _str_value(value.get("body_sha256")),
-        "body_length": (
-            _int_value(value.get("body_length"))
-            if value.get("body_length") is not None
-            else None
-        ),
-        "has_show_more_button": (
-            _bool_value(value.get("has_show_more_button"), False)
-            if value.get("has_show_more_button") is not None
-            else None
-        ),
-        "articles_range": _str_value(value.get("articles_range")),
-        "state_reason": _str_value(value.get("state_reason")),
-        "warning_codes": [
-            str(code)
-            for code in (warning_codes if isinstance(warning_codes, list) else [])
-        ],
-        "marker_counts_nonzero": {
-            str(key): _int_value(count, 0)
-            for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else [])
-        },
-        "page_logs": _normalize_page_logs(value.get("page_logs")),
-        "attempt_log": _normalize_attempt_log(value.get("attempt_log")),
-    }
-
-
-def _normalize_scholar_result(value: Any) -> dict[str, Any]:
-    if not isinstance(value, dict):
-        return {
-            "scholar_profile_id": 0,
-            "scholar_id": "unknown",
-            "state": "unknown",
-            "state_reason": None,
-            "outcome": "failed",
-            "attempt_count": 0,
-            "publication_count": 0,
-            "start_cstart": 0,
-            "continuation_cstart": None,
-            "continuation_enqueued": False,
-            "continuation_cleared": False,
-            "warnings": [],
-            "error": None,
-            "debug": None,
-        }
-    warnings = value.get("warnings")
-    return {
-        "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0),
-        "scholar_id": _str_value(value.get("scholar_id")) or "unknown",
-        "state": _str_value(value.get("state")) or "unknown",
-        "state_reason": _str_value(value.get("state_reason")),
-        "outcome": _str_value(value.get("outcome")) or "failed",
-        "attempt_count": _int_value(value.get("attempt_count"), 0),
-        "publication_count": _int_value(value.get("publication_count"), 0),
-        "start_cstart": _int_value(value.get("start_cstart"), 0),
-        "continuation_cstart": (
-            _int_value(value.get("continuation_cstart"))
-            if value.get("continuation_cstart") is not None
-            else None
-        ),
-        "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False),
-        "continuation_cleared": _bool_value(value.get("continuation_cleared"), False),
-        "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])],
-        "error": _str_value(value.get("error")),
-        "debug": _normalize_debug(value.get("debug")),
-    }
-
-
-def _manual_run_payload_from_run(
-    run,
-    *,
-    idempotency_key: str | None,
-    reused_existing_run: bool,
-    safety_state: dict[str, Any],
-) -> dict[str, Any]:
-    summary = run_service.extract_run_summary(run.error_log)
-    return {
-        "run_id": int(run.id),
-        "status": run.status.value,
-        "scholar_count": int(run.scholar_count or 0),
-        "succeeded_count": int(summary["succeeded_count"]),
-        "failed_count": int(summary["failed_count"]),
-        "partial_count": int(summary["partial_count"]),
-        "new_publication_count": int(run.new_pub_count or 0),
-        "reused_existing_run": reused_existing_run,
-        "idempotency_key": idempotency_key,
-        "safety_state": safety_state,
-    }
-
-
-async def _load_safety_state(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-) -> dict[str, Any]:
-    user_settings = await user_settings_service.get_or_create_settings(
-        db_session,
-        user_id=user_id,
-    )
-    previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
-    if run_safety_service.clear_expired_cooldown(user_settings):
-        await db_session.commit()
-        await db_session.refresh(user_settings)
-        logger.info(
-            "api.runs.safety_cooldown_cleared",
-            extra={
-                "event": "api.runs.safety_cooldown_cleared",
-                "user_id": user_id,
-                "reason": previous_safety_state.get("cooldown_reason"),
-                "cooldown_until": previous_safety_state.get("cooldown_until"),
-                "metric_name": "api_runs_safety_cooldown_cleared_total",
-                "metric_value": 1,
-            },
-    )
-    return run_safety_service.get_safety_state_payload(user_settings)
-
-
-def _raise_manual_runs_disabled(*, user_id: int, safety_state: dict[str, Any]) -> None:
-    logger.warning(
-        "api.runs.manual_blocked_policy",
-        extra={
-            "event": "api.runs.manual_blocked_policy",
-            "user_id": user_id,
-            "policy": {"manual_run_allowed": False},
-            "safety_state": safety_state,
-            "metric_name": "api_runs_manual_blocked_policy_total",
-            "metric_value": 1,
-        },
-    )
-    raise ApiException(
-        status_code=403,
-        code="manual_runs_disabled",
-        message="Manual checks are disabled by server policy.",
-        details={
-            "policy": {"manual_run_allowed": False},
-            "safety_state": safety_state,
-        },
-    )
-
-
-async def _reused_manual_run_payload(
-    db_session: AsyncSession,
-    *,
-    request: Request,
-    user_id: int,
-    idempotency_key: str | None,
-    safety_state: dict[str, Any],
-) -> dict[str, Any] | None:
-    if idempotency_key is None:
-        return None
-    previous_run = await run_service.get_manual_run_by_idempotency_key(
-        db_session,
-        user_id=user_id,
-        idempotency_key=idempotency_key,
-    )
-    if previous_run is None:
-        return None
-    if previous_run.status == RunStatus.RUNNING:
-        raise ApiException(
-            status_code=409,
-            code="run_in_progress",
-            message="A run with this idempotency key is still in progress.",
-            details={"run_id": int(previous_run.id), "idempotency_key": idempotency_key},
-        )
-    return success_payload(
-        request,
-        data=_manual_run_payload_from_run(
-            previous_run,
-            idempotency_key=idempotency_key,
-            reused_existing_run=True,
-            safety_state=safety_state,
-        ),
-    )
-
-
-async def _run_ingestion_for_manual(
-    db_session: AsyncSession,
-    *,
-    ingest_service: ingestion_service.ScholarIngestionService,
-    user_id: int,
-    idempotency_key: str | None,
-):
-    user_settings = await user_settings_service.get_or_create_settings(
-        db_session,
-        user_id=user_id,
-    )
-    return await ingest_service.run_for_user(
-        db_session,
-        user_id=user_id,
-        trigger_type=RunTriggerType.MANUAL,
-        request_delay_seconds=user_settings.request_delay_seconds,
-        network_error_retries=settings.ingestion_network_error_retries,
-        retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
-        max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
-        page_size=settings.ingestion_page_size,
-        auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
-        queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
-        idempotency_key=idempotency_key,
-        alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
-        alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
-        alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
-    )
-
-
-async def _recover_integrity_error(
-    db_session: AsyncSession,
-    *,
-    request: Request,
-    user_id: int,
-    idempotency_key: str | None,
-    original_exc: IntegrityError,
-) -> dict[str, Any]:
-    if idempotency_key is None:
-        logger.exception(
-            "api.runs.manual_integrity_error",
-            extra={"event": "api.runs.manual_integrity_error", "user_id": user_id},
-        )
-        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
-    existing_run = await run_service.get_manual_run_by_idempotency_key(
-        db_session,
-        user_id=user_id,
-        idempotency_key=idempotency_key,
-    )
-    if existing_run is None:
-        logger.exception(
-            "api.runs.manual_integrity_error",
-            extra={"event": "api.runs.manual_integrity_error", "user_id": user_id},
-        )
-        raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
-    if existing_run.status == RunStatus.RUNNING:
-        raise ApiException(
-            status_code=409,
-            code="run_in_progress",
-            message="A run with this idempotency key is still in progress.",
-            details={"run_id": int(existing_run.id), "idempotency_key": idempotency_key},
-        ) from original_exc
-    return success_payload(
-        request,
-        data=_manual_run_payload_from_run(
-            existing_run,
-            idempotency_key=idempotency_key,
-            reused_existing_run=True,
-            safety_state=await _load_safety_state(db_session, user_id=user_id),
-        ),
-    )
-
-
-async def _execute_manual_run(
-    db_session: AsyncSession,
-    *,
-    request: Request,
-    ingest_service: ingestion_service.ScholarIngestionService,
-    user_id: int,
-    idempotency_key: str | None,
-):
-    try:
-        return await _run_ingestion_for_manual(
-            db_session,
-            ingest_service=ingest_service,
-            user_id=user_id,
-            idempotency_key=idempotency_key,
-        )
-    except ingestion_service.RunAlreadyInProgressError as exc:
-        await db_session.rollback()
-        raise ApiException(
-            status_code=409,
-            code="run_in_progress",
-            message="A run is already in progress for this account.",
-        ) from exc
-    except ingestion_service.RunBlockedBySafetyPolicyError as exc:
-        await db_session.rollback()
-        _raise_manual_blocked_safety(exc=exc, user_id=user_id)
-    except IntegrityError as exc:
-        await db_session.rollback()
-        return await _recover_integrity_error(
-            db_session,
-            request=request,
-            user_id=user_id,
-            idempotency_key=idempotency_key,
-            original_exc=exc,
-        )
-    except Exception as exc:
-        await db_session.rollback()
-        _raise_manual_failed(exc=exc, user_id=user_id)
-
-
-def _raise_manual_blocked_safety(*, exc, user_id: int) -> None:
-    logger.info(
-        "api.runs.manual_blocked_safety",
-        extra={
-            "event": "api.runs.manual_blocked_safety",
-            "user_id": user_id,
-            "reason": exc.safety_state.get("cooldown_reason"),
-            "cooldown_until": exc.safety_state.get("cooldown_until"),
-            "cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"),
-            "metric_name": "api_runs_manual_blocked_safety_total",
-            "metric_value": 1,
-        },
-    )
-    raise ApiException(
-        status_code=429,
-        code=exc.code,
-        message=exc.message,
-        details={"safety_state": exc.safety_state},
-    ) from exc
-
-
-def _raise_manual_failed(*, exc: Exception, user_id: int) -> None:
-    logger.exception(
-        "api.runs.manual_failed",
-        extra={"event": "api.runs.manual_failed", "user_id": user_id},
-    )
-    raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from exc
-
-
-def _manual_run_success_payload(
-    *,
-    run_summary,
-    idempotency_key: str | None,
-    safety_state: dict[str, Any],
-) -> dict[str, Any]:
-    return {
-        "run_id": run_summary.crawl_run_id,
-        "status": run_summary.status.value,
-        "scholar_count": run_summary.scholar_count,
-        "succeeded_count": run_summary.succeeded_count,
-        "failed_count": run_summary.failed_count,
-        "partial_count": run_summary.partial_count,
-        "new_publication_count": run_summary.new_publication_count,
-        "reused_existing_run": False,
-        "idempotency_key": idempotency_key,
-        "safety_state": safety_state,
-    }
-
-
-@router.get(
-    "",
-    response_model=RunsListEnvelope,
-)
-async def list_runs(
-    request: Request,
-    failed_only: bool = Query(default=False),
-    limit: int = Query(default=100, ge=1, le=500),
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    runs = await run_service.list_runs_for_user(
-        db_session,
-        user_id=current_user.id,
-        limit=limit,
-        failed_only=failed_only,
-    )
-    safety_state = await _load_safety_state(
-        db_session,
-        user_id=current_user.id,
-    )
-    return success_payload(
-        request,
-        data={
-            "runs": [_serialize_run(run) for run in runs],
-            "safety_state": safety_state,
-        },
-    )
-
-
-@router.get(
-    "/{run_id}",
-    response_model=RunDetailEnvelope,
-)
-async def get_run(
-    run_id: int,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    run = await run_service.get_run_for_user(
-        db_session,
-        user_id=current_user.id,
-        run_id=run_id,
-    )
-    if run is None:
-        raise ApiException(
-            status_code=404,
-            code="run_not_found",
-            message="Run not found.",
-        )
-    error_log = run.error_log if isinstance(run.error_log, dict) else {}
-    scholar_results = error_log.get("scholar_results")
-    if not isinstance(scholar_results, list):
-        scholar_results = []
-    safety_state = await _load_safety_state(
-        db_session,
-        user_id=current_user.id,
-    )
-    return success_payload(
-        request,
-        data={
-            "run": _serialize_run(run),
-            "summary": run_service.extract_run_summary(error_log),
-            "scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
-            "safety_state": safety_state,
-        },
-    )
-
-
-@router.post(
-    "/{run_id}/cancel",
-    response_model=RunDetailEnvelope,
-)
-async def cancel_run(
-    run_id: int,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    run = await run_service.get_run_for_user(
-        db_session,
-        user_id=current_user.id,
-        run_id=run_id,
-    )
-    if run is None:
-        raise ApiException(
-            status_code=404,
-            code="run_not_found",
-            message="Run not found.",
-        )
-        
-    if run.status == RunStatus.RUNNING:
-        run.status = RunStatus.CANCELED
-        await db_session.commit()
-        await db_session.refresh(run)
-
-    error_log = run.error_log if isinstance(run.error_log, dict) else {}
-    scholar_results = error_log.get("scholar_results")
-    if not isinstance(scholar_results, list):
-        scholar_results = []
-        
-    safety_state = await _load_safety_state(
-        db_session,
-        user_id=current_user.id,
-    )
-    
-    return success_payload(
-        request,
-        data={
-            "run": _serialize_run(run),
-            "summary": run_service.extract_run_summary(error_log),
-            "scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
-            "safety_state": safety_state,
-        },
-    )
-
-
-@router.post(
-    "/manual",
-    response_model=ManualRunEnvelope,
-)
-async def run_manual(
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-    ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
-):
-    safety_state = await _load_safety_state(
-        db_session,
-        user_id=current_user.id,
-    )
-    if not settings.ingestion_manual_run_allowed:
-        _raise_manual_runs_disabled(user_id=current_user.id, safety_state=safety_state)
-
-    idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
-    reused_payload = await _reused_manual_run_payload(
-        db_session,
-        request=request,
-        user_id=current_user.id,
-        idempotency_key=idempotency_key,
-        safety_state=safety_state,
-    )
-    if reused_payload is not None:
-        return reused_payload
-
-    run_summary = await _execute_manual_run(
-        db_session,
-        request=request,
-        ingest_service=ingest_service,
-        user_id=current_user.id,
-        idempotency_key=idempotency_key,
-    )
-    if isinstance(run_summary, dict):
-        return run_summary
-
-    current_safety_state = await _load_safety_state(
-        db_session,
-        user_id=current_user.id,
-    )
-    return success_payload(
-        request,
-        data=_manual_run_success_payload(
-            run_summary=run_summary,
-            idempotency_key=idempotency_key,
-            safety_state=current_safety_state,
-        ),
-    )
-
-
-
-@router.get(
-    "/queue/items",
-    response_model=QueueListEnvelope,
-)
-async def list_queue_items(
-    request: Request,
-    limit: int = Query(default=200, ge=1, le=500),
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    items = await run_service.list_queue_items_for_user(
-        db_session,
-        user_id=current_user.id,
-        limit=limit,
-    )
-    return success_payload(
-        request,
-        data={
-            "queue_items": [_serialize_queue_item(item) for item in items],
-        },
-    )
-
-
-@router.post(
-    "/queue/{queue_item_id}/retry",
-    response_model=QueueItemEnvelope,
-)
-async def retry_queue_item(
-    queue_item_id: int,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    try:
-        queue_item = await run_service.retry_queue_item_for_user(
-            db_session,
-            user_id=current_user.id,
-            queue_item_id=queue_item_id,
-        )
-    except run_service.QueueTransitionError as exc:
-        raise ApiException(
-            status_code=409,
-            code=exc.code,
-            message=exc.message,
-            details={"current_status": exc.current_status},
-        ) from exc
-    if queue_item is None:
-        raise ApiException(
-            status_code=404,
-            code="queue_item_not_found",
-            message="Queue item not found.",
-        )
-    return success_payload(
-        request,
-        data=_serialize_queue_item(queue_item),
-    )
-
-
-@router.post(
-    "/queue/{queue_item_id}/drop",
-    response_model=QueueItemEnvelope,
-)
-async def drop_queue_item(
-    queue_item_id: int,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    try:
-        dropped = await run_service.drop_queue_item_for_user(
-            db_session,
-            user_id=current_user.id,
-            queue_item_id=queue_item_id,
-        )
-    except run_service.QueueTransitionError as exc:
-        raise ApiException(
-            status_code=409,
-            code=exc.code,
-            message=exc.message,
-            details={"current_status": exc.current_status},
-        ) from exc
-    if dropped is None:
-        raise ApiException(
-            status_code=404,
-            code="queue_item_not_found",
-            message="Queue item not found.",
-        )
-    return success_payload(
-        request,
-        data=_serialize_queue_item(dropped),
-    )
-
-
-@router.delete(
-    "/queue/{queue_item_id}",
-    response_model=QueueClearEnvelope,
-)
-async def clear_queue_item(
-    queue_item_id: int,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    try:
-        deleted = await run_service.clear_queue_item_for_user(
-            db_session,
-            user_id=current_user.id,
-            queue_item_id=queue_item_id,
-        )
-    except run_service.QueueTransitionError as exc:
-        raise ApiException(
-            status_code=409,
-            code=exc.code,
-            message=exc.message,
-            details={"current_status": exc.current_status},
-        ) from exc
-    if deleted is None:
-        raise ApiException(
-            status_code=404,
-            code="queue_item_not_found",
-            message="Queue item not found.",
-        )
-    return success_payload(
-        request,
-        data={
-            "queue_item_id": deleted.queue_item_id,
-            "previous_status": deleted.previous_status,
-            "status": "cleared",
-            "message": "Queue item cleared.",
-        },
-    )
-
-
-@router.get("/{run_id}/stream")
-async def stream_run_events(
-    run_id: int,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    run = await run_service.get_run_for_user(
-        db_session,
-        user_id=current_user.id,
-        run_id=run_id,
-    )
-    if run is None:
-        raise ApiException(
-            status_code=404,
-            code="run_not_found",
-            message="Run not found.",
-        )
-    return StreamingResponse(
-        event_generator(run_id),
-        media_type="text/event-stream"
-    )

build/lib/app/api/routers/scholars.py

@@ -1,557 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-
-from fastapi import APIRouter, Depends, File, Query, Request, UploadFile
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.api.deps import get_api_current_user
-from app.api.errors import ApiException
-from app.api.responses import success_payload
-from app.api.runtime_deps import get_scholar_source
-from app.api.schemas import (
-    DataExportEnvelope,
-    DataImportEnvelope,
-    DataImportRequest,
-    MessageEnvelope,
-    ScholarCreateRequest,
-    ScholarEnvelope,
-    ScholarImageUrlUpdateRequest,
-    ScholarSearchEnvelope,
-    ScholarsListEnvelope,
-)
-from app.db.models import User
-from app.db.session import get_db_session
-from app.services.domains.portability import application as import_export_service
-from app.services.domains.scholars import application as scholar_service
-from app.services.domains.scholar.source import ScholarSource
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/scholars", tags=["api-scholars"])
-
-
-def _uploaded_image_media_path(scholar_profile_id: int) -> str:
-    return f"/scholar-images/{scholar_profile_id}/upload"
-
-
-def _serialize_scholar(profile) -> dict[str, object]:
-    uploaded_image_url = None
-    if profile.profile_image_upload_path:
-        uploaded_image_url = _uploaded_image_media_path(int(profile.id))
-
-    profile_image_url, profile_image_source = scholar_service.resolve_profile_image(
-        profile,
-        uploaded_image_url=uploaded_image_url,
-    )
-
-    return {
-        "id": int(profile.id),
-        "scholar_id": profile.scholar_id,
-        "display_name": profile.display_name,
-        "profile_image_url": profile_image_url,
-        "profile_image_source": profile_image_source,
-        "is_enabled": bool(profile.is_enabled),
-        "baseline_completed": bool(profile.baseline_completed),
-        "last_run_dt": profile.last_run_dt,
-        "last_run_status": (
-            profile.last_run_status.value if profile.last_run_status is not None else None
-        ),
-    }
-
-
-async def _hydrate_scholar_metadata_if_needed(
-    db_session: AsyncSession,
-    *,
-    profile,
-    source: ScholarSource,
-    user_id: int,
-):
-    try:
-        if not profile.profile_image_url or not (profile.display_name or "").strip():
-            return await asyncio.wait_for(
-                scholar_service.hydrate_profile_metadata(
-                    db_session,
-                    profile=profile,
-                    source=source,
-                ),
-                timeout=5.0,
-            )
-    except Exception:
-        logger.warning(
-            "api.scholars.create_metadata_hydration_failed",
-            extra={
-                "event": "api.scholars.create_metadata_hydration_failed",
-                "user_id": user_id,
-                "scholar_profile_id": profile.id,
-            },
-        )
-    return profile
-
-
-def _search_kwargs() -> dict[str, object]:
-    return {
-        "network_error_retries": settings.ingestion_network_error_retries,
-        "retry_backoff_seconds": settings.ingestion_retry_backoff_seconds,
-        "search_enabled": settings.scholar_name_search_enabled,
-        "cache_ttl_seconds": settings.scholar_name_search_cache_ttl_seconds,
-        "blocked_cache_ttl_seconds": settings.scholar_name_search_blocked_cache_ttl_seconds,
-        "cache_max_entries": settings.scholar_name_search_cache_max_entries,
-        "min_interval_seconds": settings.scholar_name_search_min_interval_seconds,
-        "interval_jitter_seconds": settings.scholar_name_search_interval_jitter_seconds,
-        "cooldown_block_threshold": settings.scholar_name_search_cooldown_block_threshold,
-        "cooldown_seconds": settings.scholar_name_search_cooldown_seconds,
-        "retry_alert_threshold": settings.scholar_name_search_alert_retry_count_threshold,
-        "cooldown_rejection_alert_threshold": (
-            settings.scholar_name_search_alert_cooldown_rejections_threshold
-        ),
-    }
-
-
-def _search_response_data(query: str, parsed) -> dict[str, object]:
-    return {
-        "query": query.strip(),
-        "state": parsed.state.value,
-        "state_reason": parsed.state_reason,
-        "action_hint": scholar_service.scrape_state_hint(
-            state=parsed.state,
-            state_reason=parsed.state_reason,
-        ),
-        "candidates": [
-            {
-                "scholar_id": item.scholar_id,
-                "display_name": item.display_name,
-                "affiliation": item.affiliation,
-                "email_domain": item.email_domain,
-                "cited_by_count": item.cited_by_count,
-                "interests": item.interests,
-                "profile_url": item.profile_url,
-                "profile_image_url": item.profile_image_url,
-            }
-            for item in parsed.candidates
-        ],
-        "warnings": parsed.warnings,
-    }
-
-
-async def _read_uploaded_image(image: UploadFile) -> bytes:
-    try:
-        return await image.read()
-    finally:
-        await image.close()
-
-
-async def _require_user_profile(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholar_profile_id: int,
-):
-    profile = await scholar_service.get_user_scholar_by_id(
-        db_session,
-        user_id=user_id,
-        scholar_profile_id=scholar_profile_id,
-    )
-    if profile is None:
-        raise ApiException(
-            status_code=404,
-            code="scholar_not_found",
-            message="Scholar not found.",
-        )
-    return profile
-
-
-@router.get(
-    "",
-    response_model=ScholarsListEnvelope,
-)
-async def list_scholars(
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    scholars = await scholar_service.list_scholars_for_user(
-        db_session,
-        user_id=current_user.id,
-    )
-    return success_payload(
-        request,
-        data={
-            "scholars": [_serialize_scholar(profile) for profile in scholars],
-        },
-    )
-
-
-@router.get(
-    "/export",
-    response_model=DataExportEnvelope,
-)
-async def export_scholars_and_publications(
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    data = await import_export_service.export_user_data(
-        db_session,
-        user_id=current_user.id,
-    )
-    return success_payload(request, data=data)
-
-
-@router.post(
-    "/import",
-    response_model=DataImportEnvelope,
-)
-async def import_scholars_and_publications(
-    payload: DataImportRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    if (
-        payload.schema_version is not None
-        and int(payload.schema_version) != import_export_service.EXPORT_SCHEMA_VERSION
-    ):
-        raise ApiException(
-            status_code=400,
-            code="invalid_import_schema_version",
-            message=(
-                "Import schema version is not supported. "
-                f"Expected {import_export_service.EXPORT_SCHEMA_VERSION}."
-            ),
-        )
-    try:
-        result = await import_export_service.import_user_data(
-            db_session,
-            user_id=current_user.id,
-            scholars=[item.model_dump() for item in payload.scholars],
-            publications=[item.model_dump() for item in payload.publications],
-        )
-    except import_export_service.ImportExportError as exc:
-        raise ApiException(
-            status_code=400,
-            code="invalid_import_payload",
-            message=str(exc),
-        ) from exc
-    logger.info(
-        "api.scholars.imported",
-        extra={
-            "event": "api.scholars.imported",
-            "user_id": current_user.id,
-            **result,
-        },
-    )
-    return success_payload(request, data=result)
-
-
-@router.post(
-    "",
-    response_model=ScholarEnvelope,
-    status_code=201,
-)
-async def create_scholar(
-    payload: ScholarCreateRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    source: ScholarSource = Depends(get_scholar_source),
-    current_user: User = Depends(get_api_current_user),
-):
-    try:
-        created = await scholar_service.create_scholar_for_user(
-            db_session,
-            user_id=current_user.id,
-            scholar_id=payload.scholar_id,
-            display_name="",
-            profile_image_url=payload.profile_image_url,
-        )
-    except scholar_service.ScholarServiceError as exc:
-        raise ApiException(
-            status_code=400,
-            code="invalid_scholar",
-            message=str(exc),
-        ) from exc
-    logger.info(
-        "api.scholars.created",
-        extra={
-            "event": "api.scholars.created",
-            "user_id": current_user.id,
-            "scholar_profile_id": created.id,
-        },
-    )
-    created = await _hydrate_scholar_metadata_if_needed(
-        db_session,
-        profile=created,
-        source=source,
-        user_id=current_user.id,
-    )
-
-    return success_payload(
-        request,
-        data=_serialize_scholar(created),
-    )
-
-
-@router.get(
-    "/search",
-    response_model=ScholarSearchEnvelope,
-)
-async def search_scholars(
-    request: Request,
-    query: str = Query(..., min_length=2, max_length=120),
-    limit: int = Query(10, ge=1, le=25),
-    db_session: AsyncSession = Depends(get_db_session),
-    source: ScholarSource = Depends(get_scholar_source),
-    current_user: User = Depends(get_api_current_user),
-):
-    try:
-        parsed = await scholar_service.search_author_candidates(
-            source=source,
-            db_session=db_session,
-            query=query,
-            limit=limit,
-            **_search_kwargs(),
-        )
-    except scholar_service.ScholarServiceError as exc:
-        raise ApiException(
-            status_code=400,
-            code="invalid_scholar_search",
-            message=str(exc),
-        ) from exc
-
-    logger.info(
-        "api.scholars.search.completed",
-        extra={
-            "event": "api.scholars.search.completed",
-            "user_id": current_user.id,
-            "query": query.strip(),
-            "candidate_count": len(parsed.candidates),
-            "state": parsed.state.value,
-        },
-    )
-    return success_payload(
-        request,
-        data=_search_response_data(query, parsed),
-    )
-
-
-@router.patch(
-    "/{scholar_profile_id}/toggle",
-    response_model=ScholarEnvelope,
-)
-async def toggle_scholar(
-    scholar_profile_id: int,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    profile = await scholar_service.get_user_scholar_by_id(
-        db_session,
-        user_id=current_user.id,
-        scholar_profile_id=scholar_profile_id,
-    )
-    if profile is None:
-        raise ApiException(
-            status_code=404,
-            code="scholar_not_found",
-            message="Scholar not found.",
-        )
-    updated = await scholar_service.toggle_scholar_enabled(db_session, profile=profile)
-    logger.info(
-        "api.scholars.toggled",
-        extra={
-            "event": "api.scholars.toggled",
-            "user_id": current_user.id,
-            "scholar_profile_id": updated.id,
-            "is_enabled": updated.is_enabled,
-        },
-    )
-    return success_payload(
-        request,
-        data=_serialize_scholar(updated),
-    )
-
-
-@router.delete(
-    "/{scholar_profile_id}",
-    response_model=MessageEnvelope,
-)
-async def delete_scholar(
-    scholar_profile_id: int,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    profile = await scholar_service.get_user_scholar_by_id(
-        db_session,
-        user_id=current_user.id,
-        scholar_profile_id=scholar_profile_id,
-    )
-    if profile is None:
-        raise ApiException(
-            status_code=404,
-            code="scholar_not_found",
-            message="Scholar not found.",
-        )
-    await scholar_service.delete_scholar(
-        db_session,
-        profile=profile,
-        upload_dir=settings.scholar_image_upload_dir,
-    )
-    logger.info(
-        "api.scholars.deleted",
-        extra={
-            "event": "api.scholars.deleted",
-            "user_id": current_user.id,
-            "scholar_profile_id": scholar_profile_id,
-        },
-    )
-    return success_payload(
-        request,
-        data={"message": "Scholar deleted."},
-    )
-
-
-@router.put(
-    "/{scholar_profile_id}/image/url",
-    response_model=ScholarEnvelope,
-)
-async def update_scholar_image_url(
-    scholar_profile_id: int,
-    payload: ScholarImageUrlUpdateRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    profile = await scholar_service.get_user_scholar_by_id(
-        db_session,
-        user_id=current_user.id,
-        scholar_profile_id=scholar_profile_id,
-    )
-    if profile is None:
-        raise ApiException(
-            status_code=404,
-            code="scholar_not_found",
-            message="Scholar not found.",
-        )
-
-    try:
-        updated = await scholar_service.set_profile_image_override_url(
-            db_session,
-            profile=profile,
-            image_url=payload.image_url,
-            upload_dir=settings.scholar_image_upload_dir,
-        )
-    except scholar_service.ScholarServiceError as exc:
-        raise ApiException(
-            status_code=400,
-            code="invalid_scholar_image",
-            message=str(exc),
-        ) from exc
-
-    logger.info(
-        "api.scholars.image_url_updated",
-        extra={
-            "event": "api.scholars.image_url_updated",
-            "user_id": current_user.id,
-            "scholar_profile_id": updated.id,
-        },
-    )
-    return success_payload(
-        request,
-        data=_serialize_scholar(updated),
-    )
-
-
-@router.post(
-    "/{scholar_profile_id}/image/upload",
-    response_model=ScholarEnvelope,
-)
-async def upload_scholar_image(
-    scholar_profile_id: int,
-    request: Request,
-    image: UploadFile = File(...),
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    profile = await _require_user_profile(
-        db_session,
-        user_id=current_user.id,
-        scholar_profile_id=scholar_profile_id,
-    )
-
-    image_bytes = await _read_uploaded_image(image)
-    try:
-        updated = await scholar_service.set_profile_image_upload(
-            db_session,
-            profile=profile,
-            content_type=image.content_type,
-            image_bytes=image_bytes,
-            upload_dir=settings.scholar_image_upload_dir,
-            max_upload_bytes=settings.scholar_image_upload_max_bytes,
-        )
-    except scholar_service.ScholarServiceError as exc:
-        raise ApiException(
-            status_code=400,
-            code="invalid_scholar_image",
-            message=str(exc),
-        ) from exc
-
-    image_size = len(image_bytes)
-    logger.info(
-        "api.scholars.image_uploaded",
-        extra={
-            "event": "api.scholars.image_uploaded",
-            "user_id": current_user.id,
-            "scholar_profile_id": updated.id,
-            "content_type": image.content_type,
-            "size_bytes": image_size,
-        },
-    )
-    return success_payload(
-        request,
-        data=_serialize_scholar(updated),
-    )
-
-
-@router.delete(
-    "/{scholar_profile_id}/image",
-    response_model=ScholarEnvelope,
-)
-async def clear_scholar_image_customization(
-    scholar_profile_id: int,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    profile = await scholar_service.get_user_scholar_by_id(
-        db_session,
-        user_id=current_user.id,
-        scholar_profile_id=scholar_profile_id,
-    )
-    if profile is None:
-        raise ApiException(
-            status_code=404,
-            code="scholar_not_found",
-            message="Scholar not found.",
-        )
-
-    updated = await scholar_service.clear_profile_image_customization(
-        db_session,
-        profile=profile,
-        upload_dir=settings.scholar_image_upload_dir,
-    )
-    logger.info(
-        "api.scholars.image_customization_cleared",
-        extra={
-            "event": "api.scholars.image_customization_cleared",
-            "user_id": current_user.id,
-            "scholar_profile_id": updated.id,
-        },
-    )
-    return success_payload(
-        request,
-        data=_serialize_scholar(updated),
-    )
-

build/lib/app/api/routers/settings.py

@@ -1,189 +0,0 @@
-from __future__ import annotations
-
-import logging
-
-from fastapi import APIRouter, Depends, Request
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.api.deps import get_api_current_user
-from app.api.errors import ApiException
-from app.api.responses import success_payload
-from app.api.schemas import SettingsEnvelope, SettingsUpdateRequest
-from app.db.models import User
-from app.db.session import get_db_session
-from app.services.domains.ingestion import safety as run_safety_service
-from app.services.domains.settings import application as user_settings_service
-from app.settings import settings as settings_module
-
-logger = logging.getLogger(__name__)
-
-router = APIRouter(prefix="/settings", tags=["api-settings"])
-
-
-def _serialize_settings(user_settings) -> dict[str, object]:
-    min_run_interval_minutes, min_request_delay_seconds = _minimum_policy()
-    return {
-        "auto_run_enabled": bool(user_settings.auto_run_enabled) and settings_module.ingestion_automation_allowed,
-        "run_interval_minutes": int(user_settings.run_interval_minutes),
-        "request_delay_seconds": int(user_settings.request_delay_seconds),
-        "nav_visible_pages": list(user_settings.nav_visible_pages or []),
-        "policy": {
-            "min_run_interval_minutes": min_run_interval_minutes,
-            "min_request_delay_seconds": min_request_delay_seconds,
-            "automation_allowed": bool(settings_module.ingestion_automation_allowed),
-            "manual_run_allowed": bool(settings_module.ingestion_manual_run_allowed),
-            "blocked_failure_threshold": max(1, int(settings_module.ingestion_alert_blocked_failure_threshold)),
-            "network_failure_threshold": max(1, int(settings_module.ingestion_alert_network_failure_threshold)),
-            "cooldown_blocked_seconds": max(60, int(settings_module.ingestion_safety_cooldown_blocked_seconds)),
-            "cooldown_network_seconds": max(60, int(settings_module.ingestion_safety_cooldown_network_seconds)),
-        },
-        "safety_state": run_safety_service.get_safety_state_payload(user_settings),
-        "openalex_api_key": user_settings.openalex_api_key,
-        "crossref_api_token": user_settings.crossref_api_token,
-        "crossref_api_mailto": user_settings.crossref_api_mailto,
-    }
-
-
-def _minimum_policy() -> tuple[int, int]:
-    min_run_interval_minutes = user_settings_service.resolve_run_interval_minimum(
-        settings_module.ingestion_min_run_interval_minutes
-    )
-    min_request_delay_seconds = user_settings_service.resolve_request_delay_minimum(
-        settings_module.ingestion_min_request_delay_seconds
-    )
-    return min_run_interval_minutes, min_request_delay_seconds
-
-
-def _parse_settings_payload(payload: SettingsUpdateRequest, user_settings) -> tuple[int, int, list[str]]:
-    min_run_interval_minutes, min_request_delay_seconds = _minimum_policy()
-    parsed_interval = user_settings_service.parse_run_interval_minutes(
-        str(payload.run_interval_minutes),
-        minimum=min_run_interval_minutes,
-    )
-    parsed_delay = user_settings_service.parse_request_delay_seconds(
-        str(payload.request_delay_seconds),
-        minimum=min_request_delay_seconds,
-    )
-    parsed_nav_visible_pages = user_settings_service.parse_nav_visible_pages(
-        payload.nav_visible_pages
-        if payload.nav_visible_pages is not None
-        else list(user_settings.nav_visible_pages or user_settings_service.DEFAULT_NAV_VISIBLE_PAGES)
-    )
-    return parsed_interval, parsed_delay, parsed_nav_visible_pages
-
-
-async def _clear_expired_cooldown_with_log(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    user_settings,
-) -> None:
-    previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
-    if not run_safety_service.clear_expired_cooldown(user_settings):
-        return
-    await db_session.commit()
-    await db_session.refresh(user_settings)
-    logger.info(
-        "api.settings.safety_cooldown_cleared",
-        extra={
-            "event": "api.settings.safety_cooldown_cleared",
-            "user_id": user_id,
-            "reason": previous_safety_state.get("cooldown_reason"),
-            "cooldown_until": previous_safety_state.get("cooldown_until"),
-            "metric_name": "api_settings_safety_cooldown_cleared_total",
-            "metric_value": 1,
-        },
-    )
-
-
-def _effective_auto_run_enabled(payload: SettingsUpdateRequest) -> bool:
-    return bool(payload.auto_run_enabled) and settings_module.ingestion_automation_allowed
-
-
-def _log_settings_update(*, user_id: int, updated) -> None:
-    logger.info(
-        "api.settings.updated",
-        extra={
-            "event": "api.settings.updated",
-            "user_id": user_id,
-            "auto_run_enabled": updated.auto_run_enabled,
-            "run_interval_minutes": updated.run_interval_minutes,
-            "request_delay_seconds": updated.request_delay_seconds,
-            "nav_visible_pages": updated.nav_visible_pages,
-            "openalex_api_key": "SET" if updated.openalex_api_key else "UNSET",
-        },
-    )
-
-
-@router.get(
-    "",
-    response_model=SettingsEnvelope,
-)
-async def get_settings(
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    user_settings = await user_settings_service.get_or_create_settings(
-        db_session,
-        user_id=current_user.id,
-    )
-    await _clear_expired_cooldown_with_log(
-        db_session,
-        user_id=current_user.id,
-        user_settings=user_settings,
-    )
-    return success_payload(
-        request,
-        data=_serialize_settings(user_settings),
-    )
-
-
-@router.put(
-    "",
-    response_model=SettingsEnvelope,
-)
-async def update_settings(
-    payload: SettingsUpdateRequest,
-    request: Request,
-    db_session: AsyncSession = Depends(get_db_session),
-    current_user: User = Depends(get_api_current_user),
-):
-    user_settings = await user_settings_service.get_or_create_settings(
-        db_session,
-        user_id=current_user.id,
-    )
-
-    try:
-        parsed_interval, parsed_delay, parsed_nav_visible_pages = _parse_settings_payload(
-            payload,
-            user_settings,
-        )
-    except user_settings_service.UserSettingsServiceError as exc:
-        raise ApiException(
-            status_code=400,
-            code="invalid_settings",
-            message=str(exc),
-        ) from exc
-
-    updated = await user_settings_service.update_settings(
-        db_session,
-        settings=user_settings,
-        auto_run_enabled=_effective_auto_run_enabled(payload),
-        run_interval_minutes=parsed_interval,
-        request_delay_seconds=parsed_delay,
-        nav_visible_pages=parsed_nav_visible_pages,
-        openalex_api_key=payload.openalex_api_key,
-        crossref_api_token=payload.crossref_api_token,
-        crossref_api_mailto=payload.crossref_api_mailto,
-    )
-    await _clear_expired_cooldown_with_log(
-        db_session,
-        user_id=current_user.id,
-        user_settings=updated,
-    )
-    _log_settings_update(user_id=current_user.id, updated=updated)
-    return success_payload(
-        request,
-        data=_serialize_settings(updated),
-    )

build/lib/app/api/runtime_deps.py

@@ -1,16 +0,0 @@
-from __future__ import annotations
-
-from fastapi import Depends
-
-from app.services.domains.ingestion import application as ingestion_service
-from app.services.domains.scholar import source as scholar_source_service
-
-
-def get_scholar_source() -> scholar_source_service.ScholarSource:
-    return scholar_source_service.LiveScholarSource()
-
-
-def get_ingestion_service(
-    source: scholar_source_service.ScholarSource = Depends(get_scholar_source),
-) -> ingestion_service.ScholarIngestionService:
-    return ingestion_service.ScholarIngestionService(source=source)

build/lib/app/api/schemas.py

@@ -1,884 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Any, Literal
-
-from pydantic import BaseModel, ConfigDict, Field, model_validator
-
-
-class ApiMeta(BaseModel):
-    request_id: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ApiErrorData(BaseModel):
-    code: str
-    message: str
-    details: Any | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ApiErrorEnvelope(BaseModel):
-    error: ApiErrorData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessageData(BaseModel):
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MessageEnvelope(BaseModel):
-    data: MessageData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SessionUserData(BaseModel):
-    id: int
-    email: str
-    is_admin: bool
-    is_active: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AuthMeData(BaseModel):
-    authenticated: bool
-    csrf_token: str
-    user: SessionUserData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AuthMeEnvelope(BaseModel):
-    data: AuthMeData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class CsrfBootstrapData(BaseModel):
-    csrf_token: str
-    authenticated: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class CsrfBootstrapEnvelope(BaseModel):
-    data: CsrfBootstrapData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginRequest(BaseModel):
-    email: str
-    password: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginData(BaseModel):
-    authenticated: bool
-    csrf_token: str
-    user: SessionUserData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class LoginEnvelope(BaseModel):
-    data: LoginData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ChangePasswordRequest(BaseModel):
-    current_password: str
-    new_password: str
-    confirm_password: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarItemData(BaseModel):
-    id: int
-    scholar_id: str
-    display_name: str | None
-    profile_image_url: str | None
-    profile_image_source: str
-    is_enabled: bool
-    baseline_completed: bool
-    last_run_dt: datetime | None
-    last_run_status: str | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarsListData(BaseModel):
-    scholars: list[ScholarItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarsListEnvelope(BaseModel):
-    data: ScholarsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarCreateRequest(BaseModel):
-    scholar_id: str
-    profile_image_url: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarEnvelope(BaseModel):
-    data: ScholarItemData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchCandidateData(BaseModel):
-    scholar_id: str
-    display_name: str
-    affiliation: str | None
-    email_domain: str | None
-    cited_by_count: int | None
-    interests: list[str] = Field(default_factory=list)
-    profile_url: str
-    profile_image_url: str | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchData(BaseModel):
-    query: str
-    state: str
-    state_reason: str
-    action_hint: str | None = None
-    candidates: list[ScholarSearchCandidateData]
-    warnings: list[str] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarSearchEnvelope(BaseModel):
-    data: ScholarSearchData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarImageUrlUpdateRequest(BaseModel):
-    image_url: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScholarExportItemData(BaseModel):
-    scholar_id: str
-    display_name: str | None = None
-    is_enabled: bool = True
-    profile_image_override_url: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationExportItemData(BaseModel):
-    scholar_id: str
-    cluster_id: str | None = None
-    fingerprint_sha256: str | None = None
-    title: str
-    year: int | None = None
-    citation_count: int = 0
-    author_text: str | None = None
-    venue_text: str | None = None
-    pub_url: str | None = None
-    pdf_url: str | None = None
-    is_read: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataExportData(BaseModel):
-    schema_version: int
-    exported_at: str
-    scholars: list[ScholarExportItemData]
-    publications: list[PublicationExportItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataExportEnvelope(BaseModel):
-    data: DataExportData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataImportRequest(BaseModel):
-    schema_version: int | None = None
-    scholars: list[ScholarExportItemData] = Field(default_factory=list)
-    publications: list[PublicationExportItemData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataImportResultData(BaseModel):
-    scholars_created: int
-    scholars_updated: int
-    publications_created: int
-    publications_updated: int
-    links_created: int
-    links_updated: int
-    skipped_records: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DataImportEnvelope(BaseModel):
-    data: DataImportResultData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunListItemData(BaseModel):
-    id: int
-    trigger_type: str
-    status: str
-    start_dt: datetime
-    end_dt: datetime | None
-    scholar_count: int
-    new_publication_count: int
-    failed_count: int
-    partial_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunsListData(BaseModel):
-    runs: list[RunListItemData]
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunsListEnvelope(BaseModel):
-    data: RunsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunSummaryData(BaseModel):
-    succeeded_count: int
-    failed_count: int
-    partial_count: int
-    failed_state_counts: dict[str, int] = Field(default_factory=dict)
-    failed_reason_counts: dict[str, int] = Field(default_factory=dict)
-    scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
-    retry_counts: dict[str, int] = Field(default_factory=dict)
-    alert_thresholds: dict[str, int] = Field(default_factory=dict)
-    alert_flags: dict[str, bool] = Field(default_factory=dict)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScrapeSafetyCountersData(BaseModel):
-    consecutive_blocked_runs: int = 0
-    consecutive_network_runs: int = 0
-    cooldown_entry_count: int = 0
-    blocked_start_count: int = 0
-    last_blocked_failure_count: int = 0
-    last_network_failure_count: int = 0
-    last_evaluated_run_id: int | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ScrapeSafetyStateData(BaseModel):
-    cooldown_active: bool
-    cooldown_reason: str | None = None
-    cooldown_reason_label: str | None = None
-    cooldown_until: datetime | None = None
-    cooldown_remaining_seconds: int = 0
-    recommended_action: str | None = None
-    counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunAttemptLogData(BaseModel):
-    attempt: int
-    cstart: int
-    state: str | None = None
-    state_reason: str | None = None
-    status_code: int | None = None
-    fetch_error: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunPageLogData(BaseModel):
-    page: int
-    cstart: int
-    state: str
-    state_reason: str | None = None
-    status_code: int | None = None
-    publication_count: int = 0
-    attempt_count: int = 0
-    has_show_more_button: bool = False
-    articles_range: str | None = None
-    warning_codes: list[str] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDebugData(BaseModel):
-    status_code: int | None = None
-    final_url: str | None = None
-    fetch_error: str | None = None
-    body_sha256: str | None = None
-    body_length: int | None = None
-    has_show_more_button: bool | None = None
-    articles_range: str | None = None
-    state_reason: str | None = None
-    warning_codes: list[str] = Field(default_factory=list)
-    marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
-    page_logs: list[RunPageLogData] = Field(default_factory=list)
-    attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunScholarResultData(BaseModel):
-    scholar_profile_id: int
-    scholar_id: str
-    state: str
-    state_reason: str | None = None
-    outcome: str
-    attempt_count: int = 0
-    publication_count: int = 0
-    start_cstart: int = 0
-    continuation_cstart: int | None = None
-    continuation_enqueued: bool = False
-    continuation_cleared: bool = False
-    warnings: list[str] = Field(default_factory=list)
-    error: str | None = None
-    debug: RunDebugData | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDetailData(BaseModel):
-    run: RunListItemData
-    summary: RunSummaryData
-    scholar_results: list[RunScholarResultData]
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RunDetailEnvelope(BaseModel):
-    data: RunDetailData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ManualRunData(BaseModel):
-    run_id: int
-    status: str
-    scholar_count: int
-    succeeded_count: int
-    failed_count: int
-    partial_count: int
-    new_publication_count: int
-    reused_existing_run: bool
-    idempotency_key: str | None = None
-    safety_state: ScrapeSafetyStateData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class ManualRunEnvelope(BaseModel):
-    data: ManualRunData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueItemData(BaseModel):
-    id: int
-    scholar_profile_id: int
-    scholar_label: str
-    status: str
-    reason: str
-    dropped_reason: str | None
-    attempt_count: int
-    resume_cstart: int
-    next_attempt_dt: datetime | None
-    updated_at: datetime
-    last_error: str | None
-    last_run_id: int | None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueListData(BaseModel):
-    queue_items: list[QueueItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueListEnvelope(BaseModel):
-    data: QueueListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueItemEnvelope(BaseModel):
-    data: QueueItemData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueClearData(BaseModel):
-    queue_item_id: int
-    previous_status: str
-    status: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class QueueClearEnvelope(BaseModel):
-    data: QueueClearData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserData(BaseModel):
-    id: int
-    email: str
-    is_active: bool
-    is_admin: bool
-    created_at: datetime
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUsersListData(BaseModel):
-    users: list[AdminUserData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUsersListEnvelope(BaseModel):
-    data: AdminUsersListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserCreateRequest(BaseModel):
-    email: str
-    password: str
-    is_admin: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserActiveUpdateRequest(BaseModel):
-    is_active: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminUserEnvelope(BaseModel):
-    data: AdminUserData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminResetPasswordRequest(BaseModel):
-    new_password: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityCheckData(BaseModel):
-    name: str
-    count: int
-    severity: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityData(BaseModel):
-    status: str
-    checked_at: datetime
-    failures: list[str] = Field(default_factory=list)
-    warnings: list[str] = Field(default_factory=list)
-    checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbIntegrityEnvelope(BaseModel):
-    data: AdminDbIntegrityData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobData(BaseModel):
-    id: int
-    job_name: str
-    requested_by: str | None
-    dry_run: bool
-    status: str
-    scope: dict[str, Any] = Field(default_factory=dict)
-    summary: dict[str, Any] = Field(default_factory=dict)
-    error_text: str | None
-    started_at: datetime | None
-    finished_at: datetime | None
-    created_at: datetime
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobsData(BaseModel):
-    jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminDbRepairJobsEnvelope(BaseModel):
-    data: AdminDbRepairJobsData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class DisplayIdentifierData(BaseModel):
-    kind: str
-    value: str
-    label: str
-    url: str | None
-    confidence_score: float = Field(ge=0.0, le=1.0)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueItemData(BaseModel):
-    publication_id: int
-    title: str
-    display_identifier: DisplayIdentifierData | None = None
-    pdf_url: str | None
-    status: str
-    attempt_count: int
-    last_failure_reason: str | None
-    last_failure_detail: str | None
-    last_source: str | None
-    requested_by_user_id: int | None
-    requested_by_email: str | None
-    queued_at: datetime | None
-    last_attempt_at: datetime | None
-    resolved_at: datetime | None
-    updated_at: datetime
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueData(BaseModel):
-    items: list[AdminPdfQueueItemData] = Field(default_factory=list)
-    total_count: int = 0
-    page: int = 1
-    page_size: int = 100
-    has_next: bool = False
-    has_prev: bool = False
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueEnvelope(BaseModel):
-    data: AdminPdfQueueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueRequeueData(BaseModel):
-    publication_id: int
-    queued: bool
-    status: str
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueRequeueEnvelope(BaseModel):
-    data: AdminPdfQueueRequeueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueBulkEnqueueData(BaseModel):
-    requested_count: int
-    queued_count: int
-    message: str
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
-    data: AdminPdfQueueBulkEnqueueData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationLinksRequest(BaseModel):
-    scope_mode: Literal["single_user", "all_users"] = "single_user"
-    user_id: int | None = Field(default=None, ge=1)
-    scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
-    dry_run: bool = True
-    gc_orphan_publications: bool = False
-    requested_by: str | None = None
-    confirmation_text: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-    @model_validator(mode="after")
-    def validate_scope(self) -> "AdminRepairPublicationLinksRequest":
-        if self.scope_mode == "single_user" and self.user_id is None:
-            raise ValueError("user_id is required when scope_mode=single_user.")
-        if self.scope_mode == "all_users" and self.user_id is not None:
-            raise ValueError("user_id must be omitted when scope_mode=all_users.")
-        if not self.dry_run and self.scope_mode == "all_users":
-            expected = "REPAIR ALL USERS"
-            provided = (self.confirmation_text or "").strip()
-            if provided != expected:
-                raise ValueError(
-                    "confirmation_text must equal 'REPAIR ALL USERS' "
-                    "when applying a repair to all users."
-                )
-        return self
-
-
-class AdminRepairPublicationLinksResultData(BaseModel):
-    job_id: int
-    status: str
-    scope: dict[str, Any] = Field(default_factory=dict)
-    summary: dict[str, Any] = Field(default_factory=dict)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class AdminRepairPublicationLinksEnvelope(BaseModel):
-    data: AdminRepairPublicationLinksResultData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsPolicyData(BaseModel):
-    min_run_interval_minutes: int
-    min_request_delay_seconds: int
-    automation_allowed: bool
-    manual_run_allowed: bool
-    blocked_failure_threshold: int
-    network_failure_threshold: int
-    cooldown_blocked_seconds: int
-    cooldown_network_seconds: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsData(BaseModel):
-    auto_run_enabled: bool
-    run_interval_minutes: int
-    request_delay_seconds: int
-    nav_visible_pages: list[str]
-    policy: SettingsPolicyData
-    safety_state: ScrapeSafetyStateData
-    
-    openalex_api_key: str | None = None
-    crossref_api_token: str | None = None
-    crossref_api_mailto: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsEnvelope(BaseModel):
-    data: SettingsData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class SettingsUpdateRequest(BaseModel):
-    auto_run_enabled: bool
-    run_interval_minutes: int
-    request_delay_seconds: int
-    nav_visible_pages: list[str] | None = None
-
-    openalex_api_key: str | None = None
-    crossref_api_token: str | None = None
-    crossref_api_mailto: str | None = None
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationItemData(BaseModel):
-    publication_id: int
-    scholar_profile_id: int
-    scholar_label: str
-    title: str
-    year: int | None
-    citation_count: int
-    venue_text: str | None
-    pub_url: str | None
-    display_identifier: DisplayIdentifierData | None = None
-    pdf_url: str | None
-    pdf_status: str = "untracked"
-    pdf_attempt_count: int = 0
-    pdf_failure_reason: str | None = None
-    pdf_failure_detail: str | None = None
-    is_read: bool
-    is_favorite: bool = False
-    first_seen_at: datetime
-    is_new_in_latest_run: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationsListData(BaseModel):
-    mode: str
-    favorite_only: bool = False
-    selected_scholar_profile_id: int | None
-    unread_count: int
-    favorites_count: int
-    latest_count: int
-    new_count: int
-    total_count: int
-    page: int
-    page_size: int
-    has_next: bool = False
-    has_prev: bool = False
-    publications: list[PublicationItemData]
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationsListEnvelope(BaseModel):
-    data: PublicationsListData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkAllReadData(BaseModel):
-    message: str
-    updated_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkAllReadEnvelope(BaseModel):
-    data: MarkAllReadData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class PublicationSelectionItem(BaseModel):
-    scholar_profile_id: int
-    publication_id: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadRequest(BaseModel):
-    selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadData(BaseModel):
-    message: str
-    requested_count: int
-    updated_count: int
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class MarkSelectedReadEnvelope(BaseModel):
-    data: MarkSelectedReadData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfRequest(BaseModel):
-    scholar_profile_id: int = Field(ge=1)
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfData(BaseModel):
-    message: str
-    queued: bool
-    resolved_pdf: bool
-    publication: PublicationItemData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class RetryPublicationPdfEnvelope(BaseModel):
-    data: RetryPublicationPdfData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteRequest(BaseModel):
-    scholar_profile_id: int = Field(ge=1)
-    is_favorite: bool
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteData(BaseModel):
-    message: str
-    publication: PublicationItemData
-
-    model_config = ConfigDict(extra="forbid")
-
-
-class TogglePublicationFavoriteEnvelope(BaseModel):
-    data: TogglePublicationFavoriteData
-    meta: ApiMeta
-
-    model_config = ConfigDict(extra="forbid")

build/lib/app/auth/__init__.py

@@ -1,2 +0,0 @@
-"""Authentication package for scholarr."""
-

build/lib/app/auth/deps.py

@@ -1,27 +0,0 @@
-from __future__ import annotations
-
-from functools import lru_cache
-
-from app.auth.rate_limit import SlidingWindowRateLimiter
-from app.auth.security import PasswordService
-from app.auth.service import AuthService
-from app.settings import settings
-
-
-@lru_cache
-def get_password_service() -> PasswordService:
-    return PasswordService()
-
-
-@lru_cache
-def get_auth_service() -> AuthService:
-    return AuthService(password_service=get_password_service())
-
-
-@lru_cache
-def get_login_rate_limiter() -> SlidingWindowRateLimiter:
-    return SlidingWindowRateLimiter(
-        max_attempts=settings.login_rate_limit_attempts,
-        window_seconds=settings.login_rate_limit_window_seconds,
-    )
-

build/lib/app/auth/rate_limit.py

@@ -1,69 +0,0 @@
-from __future__ import annotations
-
-from collections import deque
-from collections.abc import Callable
-from dataclasses import dataclass
-from math import ceil
-from threading import Lock
-from time import monotonic
-
-
-@dataclass(frozen=True)
-class RateLimitDecision:
-    allowed: bool
-    retry_after_seconds: int = 0
-
-
-class SlidingWindowRateLimiter:
-    def __init__(
-        self,
-        *,
-        max_attempts: int,
-        window_seconds: int,
-        now: Callable[[], float] = monotonic,
-    ) -> None:
-        self._max_attempts = max_attempts
-        self._window_seconds = window_seconds
-        self._now = now
-        self._attempts: dict[str, deque[float]] = {}
-        self._lock = Lock()
-
-    def check(self, key: str) -> RateLimitDecision:
-        with self._lock:
-            now_value = self._now()
-            attempts = self._attempts.get(key)
-            if attempts is None:
-                return RateLimitDecision(allowed=True)
-            self._trim_expired(attempts, now_value)
-            if not attempts:
-                self._attempts.pop(key, None)
-                return RateLimitDecision(allowed=True)
-            if len(attempts) >= self._max_attempts:
-                retry_after = self._window_seconds - (now_value - attempts[0])
-                return RateLimitDecision(
-                    allowed=False,
-                    retry_after_seconds=max(1, ceil(retry_after)),
-                )
-            return RateLimitDecision(allowed=True)
-
-    def record_failure(self, key: str) -> None:
-        with self._lock:
-            now_value = self._now()
-            attempts = self._attempts.get(key)
-            if attempts is None:
-                attempts = deque()
-                self._attempts[key] = attempts
-            self._trim_expired(attempts, now_value)
-            attempts.append(now_value)
-
-    def reset(self, key: str) -> None:
-        with self._lock:
-            self._attempts.pop(key, None)
-
-    def clear_all(self) -> None:
-        with self._lock:
-            self._attempts.clear()
-
-    def _trim_expired(self, attempts: deque[float], now_value: float) -> None:
-        while attempts and now_value - attempts[0] >= self._window_seconds:
-            attempts.popleft()

build/lib/app/auth/runtime.py

@@ -1,55 +0,0 @@
-from __future__ import annotations
-
-import logging
-
-from fastapi import Request
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.auth.session import clear_session_user, get_session_user, set_session_user
-from app.db.models import User
-from app.security.csrf import CSRF_SESSION_KEY
-from app.services.domains.users import application as user_service
-
-logger = logging.getLogger(__name__)
-
-
-def invalidate_session(request: Request) -> None:
-    clear_session_user(request)
-    request.session.pop(CSRF_SESSION_KEY, None)
-
-
-async def get_authenticated_user(
-    request: Request,
-    db_session: AsyncSession,
-) -> User | None:
-    session_user = get_session_user(request)
-    if session_user is None:
-        return None
-
-    user = await user_service.get_user_by_id(db_session, session_user.id)
-    if user is None or not user.is_active:
-        logger.info(
-            "auth.session_invalidated",
-            extra={
-                "event": "auth.session_invalidated",
-                "session_user_id": session_user.id,
-            },
-        )
-        invalidate_session(request)
-        return None
-
-    if user.email != session_user.email or user.is_admin != session_user.is_admin:
-        set_session_user(
-            request,
-            user_id=user.id,
-            email=user.email,
-            is_admin=user.is_admin,
-        )
-
-    return user
-
-
-def login_rate_limit_key(request: Request, email: str) -> str:
-    client_host = request.client.host if request.client is not None else "unknown"
-    normalized_email = email.strip().lower()
-    return f"{client_host}:{normalized_email or '<empty>'}"

build/lib/app/auth/security.py

@@ -1,19 +0,0 @@
-from __future__ import annotations
-
-from argon2 import PasswordHasher
-from argon2.exceptions import InvalidHashError, VerificationError
-
-
-class PasswordService:
-    def __init__(self, hasher: PasswordHasher | None = None) -> None:
-        self._hasher = hasher or PasswordHasher()
-
-    def hash_password(self, password: str) -> str:
-        return self._hasher.hash(password)
-
-    def verify_password(self, password_hash: str, password: str) -> bool:
-        try:
-            return bool(self._hasher.verify(password_hash, password))
-        except (InvalidHashError, VerificationError):
-            return False
-

build/lib/app/auth/service.py

@@ -1,38 +0,0 @@
-from __future__ import annotations
-
-from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.auth.security import PasswordService
-from app.db.models import User
-
-
-class AuthService:
-    def __init__(self, password_service: PasswordService) -> None:
-        self._password_service = password_service
-
-    async def authenticate_user(
-        self,
-        db_session: AsyncSession,
-        *,
-        email: str,
-        password: str,
-    ) -> User | None:
-        normalized_email = email.strip().lower()
-        if not normalized_email or not password:
-            return None
-        result = await db_session.execute(
-            select(User).where(User.email == normalized_email)
-        )
-        user = result.scalar_one_or_none()
-        if user is None or not user.is_active:
-            return None
-        if not self._password_service.verify_password(user.password_hash, password):
-            return None
-        return user
-
-    def hash_password(self, password: str) -> str:
-        return self._password_service.hash_password(password)
-
-    def verify_password(self, *, password_hash: str, password: str) -> bool:
-        return self._password_service.verify_password(password_hash, password)

build/lib/app/auth/session.py

@@ -1,50 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-
-from starlette.requests import Request
-
-
-SESSION_USER_ID_KEY = "auth_user_id"
-SESSION_USER_EMAIL_KEY = "auth_user_email"
-SESSION_USER_IS_ADMIN_KEY = "auth_user_is_admin"
-
-
-@dataclass(frozen=True)
-class SessionUser:
-    id: int
-    email: str
-    is_admin: bool
-
-
-def get_session_user(request: Request) -> SessionUser | None:
-    user_id = request.session.get(SESSION_USER_ID_KEY)
-    email = request.session.get(SESSION_USER_EMAIL_KEY)
-    is_admin = request.session.get(SESSION_USER_IS_ADMIN_KEY)
-    if user_id is None or email is None or is_admin is None:
-        return None
-    try:
-        parsed_user_id = int(user_id)
-    except (TypeError, ValueError):
-        return None
-    if not isinstance(email, str):
-        return None
-    return SessionUser(id=parsed_user_id, email=email, is_admin=bool(is_admin))
-
-
-def set_session_user(
-    request: Request,
-    *,
-    user_id: int,
-    email: str,
-    is_admin: bool,
-) -> None:
-    request.session[SESSION_USER_ID_KEY] = int(user_id)
-    request.session[SESSION_USER_EMAIL_KEY] = email
-    request.session[SESSION_USER_IS_ADMIN_KEY] = bool(is_admin)
-
-
-def clear_session_user(request: Request) -> None:
-    request.session.pop(SESSION_USER_ID_KEY, None)
-    request.session.pop(SESSION_USER_EMAIL_KEY, None)
-    request.session.pop(SESSION_USER_IS_ADMIN_KEY, None)

build/lib/app/db/__init__.py

@@ -1 +0,0 @@
-

build/lib/app/db/base.py

@@ -1,27 +0,0 @@
-from datetime import datetime, timezone
-
-from sqlalchemy import MetaData, event
-from sqlalchemy.orm import DeclarativeBase
-
-
-NAMING_CONVENTION = {
-    "ix": "ix_%(column_0_label)s",
-    "uq": "uq_%(table_name)s_%(column_0_name)s",
-    "ck": "ck_%(table_name)s_%(constraint_name)s",
-    "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
-    "pk": "pk_%(table_name)s",
-}
-
-
-class Base(DeclarativeBase):
-    metadata = MetaData(naming_convention=NAMING_CONVENTION)
-
-
-@event.listens_for(Base, "before_update", propagate=True)
-def _set_updated_at_before_update(_mapper, _connection, target) -> None:
-    # Keep audit timestamps current for ORM-managed updates.
-    if hasattr(target, "updated_at"):
-        target.updated_at = datetime.now(timezone.utc)
-
-
-metadata = Base.metadata

build/lib/app/db/models.py

@@ -1,546 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime
-from enum import StrEnum
-
-from sqlalchemy import (
-    Boolean,
-    CheckConstraint,
-    DateTime,
-    Enum,
-    Float,
-    ForeignKey,
-    Index,
-    Integer,
-    String,
-    Text,
-    UniqueConstraint,
-    func,
-    text,
-)
-from sqlalchemy.dialects.postgresql import JSONB
-from sqlalchemy.orm import Mapped, mapped_column
-
-from app.db.base import Base
-
-
-class RunTriggerType(StrEnum):
-    MANUAL = "manual"
-    SCHEDULED = "scheduled"
-
-
-class RunStatus(StrEnum):
-    RUNNING = "running"
-    SUCCESS = "success"
-    PARTIAL_FAILURE = "partial_failure"
-    FAILED = "failed"
-    CANCELED = "canceled"
-
-
-class QueueItemStatus(StrEnum):
-    QUEUED = "queued"
-    RETRYING = "retrying"
-    DROPPED = "dropped"
-
-
-RUN_STATUS_DB_ENUM = Enum(
-    RunStatus,
-    name="run_status",
-    values_callable=lambda members: [member.value for member in members],
-)
-RUN_TRIGGER_TYPE_DB_ENUM = Enum(
-    RunTriggerType,
-    name="run_trigger_type",
-    values_callable=lambda members: [member.value for member in members],
-)
-
-
-class User(Base):
-    __tablename__ = "users"
-
-    id: Mapped[int] = mapped_column(primary_key=True)
-    email: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
-    password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
-    is_active: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, server_default=text("true")
-    )
-    is_admin: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, server_default=text("false")
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-
-
-class UserSetting(Base):
-    __tablename__ = "user_settings"
-    __table_args__ = (
-        CheckConstraint(
-            "run_interval_minutes >= 15",
-            name="run_interval_minutes_min_15",
-        ),
-        CheckConstraint(
-            "request_delay_seconds >= 2",
-            name="request_delay_seconds_min_2",
-        ),
-    )
-
-    user_id: Mapped[int] = mapped_column(
-        ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
-    )
-    auto_run_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, server_default=text("false")
-    )
-    run_interval_minutes: Mapped[int] = mapped_column(
-        Integer, nullable=False, server_default=text("1440")
-    )
-    request_delay_seconds: Mapped[int] = mapped_column(
-        Integer, nullable=False, server_default=text("10")
-    )
-    nav_visible_pages: Mapped[list[str]] = mapped_column(
-        JSONB,
-        nullable=False,
-        server_default=text(
-            '\'["dashboard","scholars","publications","settings","style-guide","runs","users"]\'::jsonb'
-        ),
-    )
-    scrape_safety_state: Mapped[dict] = mapped_column(
-        JSONB,
-        nullable=False,
-        server_default=text("'{}'::jsonb"),
-    )
-    scrape_cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    scrape_cooldown_reason: Mapped[str | None] = mapped_column(String(64))
-    
-    openalex_api_key: Mapped[str | None] = mapped_column(String(255))
-    crossref_api_token: Mapped[str | None] = mapped_column(String(255))
-    crossref_api_mailto: Mapped[str | None] = mapped_column(String(255))
-
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-
-
-class ScholarProfile(Base):
-    __tablename__ = "scholar_profiles"
-    __table_args__ = (
-        UniqueConstraint("user_id", "scholar_id", name="uq_scholar_profiles_user_scholar"),
-        Index("ix_scholar_profiles_user_enabled", "user_id", "is_enabled"),
-    )
-
-    id: Mapped[int] = mapped_column(primary_key=True)
-    user_id: Mapped[int] = mapped_column(
-        ForeignKey("users.id", ondelete="CASCADE"), nullable=False
-    )
-    scholar_id: Mapped[str] = mapped_column(String(64), nullable=False)
-    display_name: Mapped[str | None] = mapped_column(String(255))
-    profile_image_url: Mapped[str | None] = mapped_column(Text)
-    profile_image_override_url: Mapped[str | None] = mapped_column(Text)
-    profile_image_upload_path: Mapped[str | None] = mapped_column(Text)
-    last_initial_page_fingerprint_sha256: Mapped[str | None] = mapped_column(String(64))
-    last_initial_page_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    is_enabled: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, server_default=text("true")
-    )
-    baseline_completed: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, server_default=text("false")
-    )
-    last_run_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    last_run_status: Mapped[RunStatus | None] = mapped_column(
-        RUN_STATUS_DB_ENUM,
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-
-
-class CrawlRun(Base):
-    __tablename__ = "crawl_runs"
-    __table_args__ = (
-        Index("ix_crawl_runs_user_start", "user_id", "start_dt"),
-        Index(
-            "uq_crawl_runs_user_manual_idempotency_key",
-            "user_id",
-            "idempotency_key",
-            unique=True,
-            postgresql_where=text(
-                "idempotency_key IS NOT NULL AND trigger_type = 'manual'::run_trigger_type"
-            ),
-        ),
-    )
-
-    id: Mapped[int] = mapped_column(primary_key=True)
-    user_id: Mapped[int] = mapped_column(
-        ForeignKey("users.id", ondelete="CASCADE"), nullable=False
-    )
-    trigger_type: Mapped[RunTriggerType] = mapped_column(
-        RUN_TRIGGER_TYPE_DB_ENUM, nullable=False
-    )
-    status: Mapped[RunStatus] = mapped_column(
-        RUN_STATUS_DB_ENUM, nullable=False
-    )
-    start_dt: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    end_dt: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    scholar_count: Mapped[int] = mapped_column(
-        Integer, nullable=False, server_default=text("0")
-    )
-    new_pub_count: Mapped[int] = mapped_column(
-        Integer, nullable=False, server_default=text("0")
-    )
-    idempotency_key: Mapped[str | None] = mapped_column(String(128))
-    error_log: Mapped[dict] = mapped_column(
-        JSONB, nullable=False, server_default=text("'{}'::jsonb")
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-
-
-class Publication(Base):
-    __tablename__ = "publications"
-    __table_args__ = (
-        UniqueConstraint("fingerprint_sha256", name="uq_publications_fingerprint"),
-        Index(
-            "uq_publications_cluster_id_not_null",
-            "cluster_id",
-            unique=True,
-            postgresql_where=text("cluster_id IS NOT NULL"),
-        ),
-    )
-
-    id: Mapped[int] = mapped_column(primary_key=True)
-    cluster_id: Mapped[str | None] = mapped_column(String(64))
-    fingerprint_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
-    title_raw: Mapped[str] = mapped_column(Text, nullable=False)
-    title_normalized: Mapped[str] = mapped_column(Text, nullable=False)
-    year: Mapped[int | None] = mapped_column(Integer)
-    citation_count: Mapped[int] = mapped_column(
-        Integer, nullable=False, server_default=text("0")
-    )
-    author_text: Mapped[str | None] = mapped_column(Text)
-    venue_text: Mapped[str | None] = mapped_column(Text)
-    pub_url: Mapped[str | None] = mapped_column(Text)
-    pdf_url: Mapped[str | None] = mapped_column(Text)
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-
-
-class PublicationIdentifier(Base):
-    __tablename__ = "publication_identifiers"
-    __table_args__ = (
-        UniqueConstraint(
-            "publication_id",
-            "kind",
-            "value_normalized",
-            name="uq_publication_identifiers_publication_kind_value",
-        ),
-        CheckConstraint(
-            "confidence_score >= 0 AND confidence_score <= 1",
-            name="publication_identifiers_confidence_score_range",
-        ),
-        Index(
-            "ix_publication_identifiers_kind_value",
-            "kind",
-            "value_normalized",
-        ),
-        Index(
-            "ix_publication_identifiers_publication_id",
-            "publication_id",
-        ),
-    )
-
-    id: Mapped[int] = mapped_column(primary_key=True)
-    publication_id: Mapped[int] = mapped_column(
-        ForeignKey("publications.id", ondelete="CASCADE"),
-        nullable=False,
-    )
-    kind: Mapped[str] = mapped_column(String(32), nullable=False)
-    value_raw: Mapped[str] = mapped_column(Text, nullable=False)
-    value_normalized: Mapped[str] = mapped_column(String(255), nullable=False)
-    source: Mapped[str] = mapped_column(String(64), nullable=False)
-    confidence_score: Mapped[float] = mapped_column(
-        Float,
-        nullable=False,
-        server_default=text("0"),
-    )
-    evidence_url: Mapped[str | None] = mapped_column(Text)
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-
-
-class PublicationPdfJob(Base):
-    __tablename__ = "publication_pdf_jobs"
-    __table_args__ = (
-        CheckConstraint(
-            "status IN ('queued', 'running', 'resolved', 'failed')",
-            name="publication_pdf_jobs_status_valid",
-        ),
-        Index("ix_publication_pdf_jobs_status", "status"),
-        Index("ix_publication_pdf_jobs_updated_at", "updated_at"),
-        Index("ix_publication_pdf_jobs_queued_at", "queued_at"),
-    )
-
-    publication_id: Mapped[int] = mapped_column(
-        ForeignKey("publications.id", ondelete="CASCADE"),
-        primary_key=True,
-    )
-    status: Mapped[str] = mapped_column(
-        String(16),
-        nullable=False,
-        server_default=text("'queued'"),
-    )
-    attempt_count: Mapped[int] = mapped_column(
-        Integer,
-        nullable=False,
-        server_default=text("0"),
-    )
-    queued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    last_failure_reason: Mapped[str | None] = mapped_column(String(64))
-    last_failure_detail: Mapped[str | None] = mapped_column(Text)
-    last_source: Mapped[str | None] = mapped_column(String(32))
-    last_requested_by_user_id: Mapped[int | None] = mapped_column(
-        ForeignKey("users.id", ondelete="SET NULL"),
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-
-
-class PublicationPdfJobEvent(Base):
-    __tablename__ = "publication_pdf_job_events"
-    __table_args__ = (
-        Index("ix_publication_pdf_job_events_publication_created", "publication_id", "created_at"),
-        Index("ix_publication_pdf_job_events_created_at", "created_at"),
-        Index("ix_publication_pdf_job_events_event_type", "event_type"),
-    )
-
-    id: Mapped[int] = mapped_column(primary_key=True)
-    publication_id: Mapped[int] = mapped_column(
-        ForeignKey("publications.id", ondelete="CASCADE"),
-        nullable=False,
-    )
-    user_id: Mapped[int | None] = mapped_column(
-        ForeignKey("users.id", ondelete="SET NULL"),
-    )
-    event_type: Mapped[str] = mapped_column(String(32), nullable=False)
-    status: Mapped[str | None] = mapped_column(String(16))
-    source: Mapped[str | None] = mapped_column(String(32))
-    failure_reason: Mapped[str | None] = mapped_column(String(64))
-    message: Mapped[str | None] = mapped_column(Text)
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-
-
-class ScholarPublication(Base):
-    __tablename__ = "scholar_publications"
-    __table_args__ = (
-        Index("ix_scholar_publications_is_read", "is_read"),
-        Index("ix_scholar_publications_is_favorite", "is_favorite"),
-    )
-
-    scholar_profile_id: Mapped[int] = mapped_column(
-        ForeignKey("scholar_profiles.id", ondelete="CASCADE"),
-        primary_key=True,
-    )
-    publication_id: Mapped[int] = mapped_column(
-        ForeignKey("publications.id", ondelete="CASCADE"),
-        primary_key=True,
-    )
-    is_read: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, server_default=text("false")
-    )
-    is_favorite: Mapped[bool] = mapped_column(
-        Boolean, nullable=False, server_default=text("false")
-    )
-    first_seen_run_id: Mapped[int | None] = mapped_column(
-        ForeignKey("crawl_runs.id", ondelete="SET NULL")
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-
-
-class IngestionQueueItem(Base):
-    __tablename__ = "ingestion_queue_items"
-    __table_args__ = (
-        UniqueConstraint(
-            "user_id",
-            "scholar_profile_id",
-            name="uq_ingestion_queue_user_scholar",
-        ),
-        CheckConstraint(
-            "status IN ('queued', 'retrying', 'dropped')",
-            name="ingestion_queue_status_valid",
-        ),
-        Index("ix_ingestion_queue_next_attempt", "next_attempt_dt"),
-        Index("ix_ingestion_queue_status_next_attempt", "status", "next_attempt_dt"),
-    )
-
-    id: Mapped[int] = mapped_column(primary_key=True)
-    user_id: Mapped[int] = mapped_column(
-        ForeignKey("users.id", ondelete="CASCADE"),
-        nullable=False,
-    )
-    scholar_profile_id: Mapped[int] = mapped_column(
-        ForeignKey("scholar_profiles.id", ondelete="CASCADE"),
-        nullable=False,
-    )
-    resume_cstart: Mapped[int] = mapped_column(
-        Integer,
-        nullable=False,
-        server_default=text("0"),
-    )
-    reason: Mapped[str] = mapped_column(String(64), nullable=False)
-    status: Mapped[str] = mapped_column(
-        String(16),
-        nullable=False,
-        server_default=text("'queued'"),
-    )
-    attempt_count: Mapped[int] = mapped_column(
-        Integer,
-        nullable=False,
-        server_default=text("0"),
-    )
-    next_attempt_dt: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True),
-        nullable=False,
-        server_default=func.now(),
-    )
-    last_run_id: Mapped[int | None] = mapped_column(
-        ForeignKey("crawl_runs.id", ondelete="SET NULL"),
-    )
-    last_error: Mapped[str | None] = mapped_column(Text)
-    dropped_reason: Mapped[str | None] = mapped_column(String(128))
-    dropped_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-
-
-class DataRepairJob(Base):
-    __tablename__ = "data_repair_jobs"
-    __table_args__ = (
-        CheckConstraint(
-            "status IN ('planned', 'running', 'completed', 'failed')",
-            name="data_repair_jobs_status_valid",
-        ),
-        Index("ix_data_repair_jobs_created_at", "created_at"),
-        Index("ix_data_repair_jobs_job_name_created_at", "job_name", "created_at"),
-    )
-
-    id: Mapped[int] = mapped_column(primary_key=True)
-    job_name: Mapped[str] = mapped_column(String(64), nullable=False)
-    requested_by: Mapped[str | None] = mapped_column(String(255))
-    scope: Mapped[dict] = mapped_column(
-        JSONB,
-        nullable=False,
-        server_default=text("'{}'::jsonb"),
-    )
-    dry_run: Mapped[bool] = mapped_column(
-        Boolean,
-        nullable=False,
-        server_default=text("true"),
-    )
-    status: Mapped[str] = mapped_column(
-        String(16),
-        nullable=False,
-        server_default=text("'planned'"),
-    )
-    summary: Mapped[dict] = mapped_column(
-        JSONB,
-        nullable=False,
-        server_default=text("'{}'::jsonb"),
-    )
-    error_text: Mapped[str | None] = mapped_column(Text)
-    started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-
-
-class AuthorSearchRuntimeState(Base):
-    __tablename__ = "author_search_runtime_state"
-
-    state_key: Mapped[str] = mapped_column(String(64), primary_key=True)
-    last_live_request_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
-    consecutive_blocked_count: Mapped[int] = mapped_column(
-        Integer,
-        nullable=False,
-        server_default=text("0"),
-    )
-    cooldown_rejection_count: Mapped[int] = mapped_column(
-        Integer,
-        nullable=False,
-        server_default=text("0"),
-    )
-    cooldown_alert_emitted: Mapped[bool] = mapped_column(
-        Boolean,
-        nullable=False,
-        server_default=text("false"),
-    )
-    created_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-
-
-class AuthorSearchCacheEntry(Base):
-    __tablename__ = "author_search_cache_entries"
-    __table_args__ = (
-        Index("ix_author_search_cache_expires_at", "expires_at"),
-        Index("ix_author_search_cache_cached_at", "cached_at"),
-    )
-
-    query_key: Mapped[str] = mapped_column(String(256), primary_key=True)
-    payload: Mapped[dict] = mapped_column(
-        JSONB,
-        nullable=False,
-    )
-    expires_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True),
-        nullable=False,
-    )
-    cached_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )
-    updated_at: Mapped[datetime] = mapped_column(
-        DateTime(timezone=True), nullable=False, server_default=func.now()
-    )

build/lib/app/db/session.py

@@ -1,99 +0,0 @@
-from collections.abc import AsyncIterator
-import logging
-import os
-
-from sqlalchemy import text
-from sqlalchemy.ext.asyncio import (
-    AsyncEngine,
-    AsyncSession,
-    async_sessionmaker,
-    create_async_engine,
-)
-from sqlalchemy.pool import NullPool
-
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-_engine: AsyncEngine | None = None
-_session_factory: async_sessionmaker[AsyncSession] | None = None
-
-
-def _normalized_pool_mode(raw_mode: str) -> str:
-    mode = (raw_mode or "").strip().lower()
-    if mode == "auto":
-        if os.getenv("PYTEST_CURRENT_TEST"):
-            return "null"
-        app_env = (os.getenv("APP_ENV") or "").strip().lower()
-        if app_env in {"test", "development", "dev", "local"}:
-            return "null"
-        return "queue"
-    if mode in {"null", "queue"}:
-        return mode
-    logger.warning(
-        "db.invalid_pool_mode_fallback",
-        extra={
-            "event": "db.invalid_pool_mode_fallback",
-            "database_pool_mode": raw_mode,
-            "fallback_mode": "queue",
-        },
-    )
-    return "queue"
-
-
-def get_engine() -> AsyncEngine:
-    global _engine
-    if _engine is None:
-        pool_mode = _normalized_pool_mode(settings.database_pool_mode)
-        engine_kwargs: dict[str, object] = {
-            "pool_pre_ping": True,
-        }
-        if pool_mode == "null":
-            engine_kwargs["poolclass"] = NullPool
-        else:
-            engine_kwargs["pool_size"] = max(1, int(settings.database_pool_size))
-            engine_kwargs["max_overflow"] = max(0, int(settings.database_pool_max_overflow))
-            engine_kwargs["pool_timeout"] = max(1, int(settings.database_pool_timeout_seconds))
-
-        _engine = create_async_engine(settings.database_url, **engine_kwargs)
-        logger.info(
-            "db.engine_initialized",
-            extra={
-                "event": "db.engine_initialized",
-                "pool_mode": pool_mode,
-            },
-        )
-    return _engine
-
-
-def get_session_factory() -> async_sessionmaker[AsyncSession]:
-    global _session_factory
-    if _session_factory is None:
-        _session_factory = async_sessionmaker(get_engine(), expire_on_commit=False)
-    return _session_factory
-
-
-async def get_db_session() -> AsyncIterator[AsyncSession]:
-    session_factory = get_session_factory()
-    async with session_factory() as session:
-        yield session
-
-
-async def check_database() -> bool:
-    engine = get_engine()
-    try:
-        async with engine.connect() as conn:
-            result = await conn.execute(text("SELECT 1"))
-            return result.scalar_one() == 1
-    except Exception:
-        logger.exception("db.healthcheck_failed", extra={"event": "db.healthcheck_failed"})
-        return False
-
-
-async def close_engine() -> None:
-    global _engine, _session_factory
-    if _engine is not None:
-        await _engine.dispose()
-        logger.info("db.engine_disposed", extra={"event": "db.engine_disposed"})
-        _engine = None
-        _session_factory = None

build/lib/app/http/__init__.py

@@ -1 +0,0 @@
-from __future__ import annotations

build/lib/app/http/middleware.py

@@ -1,205 +0,0 @@
-from __future__ import annotations
-
-from secrets import token_urlsafe
-import logging
-import time
-
-from starlette.middleware.base import BaseHTTPMiddleware
-from starlette.requests import Request
-from starlette.responses import Response
-
-from app.logging_context import set_request_id
-
-REQUEST_ID_HEADER = "X-Request-ID"
-DEFAULT_PERMISSIONS_POLICY = (
-    "accelerometer=(), autoplay=(), camera=(), display-capture=(), "
-    "geolocation=(), gyroscope=(), microphone=(), payment=(), usb=()"
-)
-DEFAULT_CSP_POLICY = (
-    "default-src 'self'; "
-    "base-uri 'self'; "
-    "form-action 'self'; "
-    "frame-ancestors 'none'; "
-    "img-src 'self' data: https:; "
-    "script-src 'self'; "
-    "style-src 'self' 'unsafe-inline'; "
-    "font-src 'self' data:; "
-    "connect-src 'self'; "
-    "object-src 'none'"
-)
-DEFAULT_CSP_DOCS_POLICY = (
-    "default-src 'self'; "
-    "img-src 'self' data: https:; "
-    "script-src 'self' 'unsafe-inline'; "
-    "style-src 'self' 'unsafe-inline'; "
-    "font-src 'self' data:; "
-    "connect-src 'self' https:; "
-    "object-src 'none'; "
-    "frame-ancestors 'none'"
-)
-
-logger = logging.getLogger(__name__)
-
-
-class RequestLoggingMiddleware(BaseHTTPMiddleware):
-    def __init__(
-        self,
-        app,
-        *,
-        log_requests: bool = True,
-        skip_paths: tuple[str, ...] = (),
-    ) -> None:
-        super().__init__(app)
-        self._log_requests = log_requests
-        self._skip_paths = tuple(path for path in skip_paths if path)
-
-    async def dispatch(self, request: Request, call_next) -> Response:
-        request_id = request.headers.get(REQUEST_ID_HEADER) or token_urlsafe(12)
-        request.state.request_id = request_id
-        set_request_id(request_id)
-
-        start = time.perf_counter()
-        should_log = self._log_requests and not self._is_skipped_path(request.url.path)
-        if should_log:
-            logger.debug(
-                "request.started",
-                extra={
-                    "event": "request.started",
-                    "method": request.method,
-                    "path": request.url.path,
-                },
-            )
-
-        try:
-            response = await call_next(request)
-        except Exception:
-            duration_ms = int((time.perf_counter() - start) * 1000)
-            logger.exception(
-                "request.failed",
-                extra={
-                    "event": "request.failed",
-                    "method": request.method,
-                    "path": request.url.path,
-                    "duration_ms": duration_ms,
-                },
-            )
-            raise
-        else:
-            duration_ms = int((time.perf_counter() - start) * 1000)
-            response.headers[REQUEST_ID_HEADER] = request_id
-            if should_log:
-                logger.debug(
-                    "request.completed",
-                    extra={
-                        "event": "request.completed",
-                        "method": request.method,
-                        "path": request.url.path,
-                        "status_code": response.status_code,
-                        "duration_ms": duration_ms,
-                    },
-                )
-            return response
-        finally:
-            set_request_id(None)
-
-    def _is_skipped_path(self, path: str) -> bool:
-        return any(path.startswith(prefix) for prefix in self._skip_paths)
-
-
-class SecurityHeadersMiddleware(BaseHTTPMiddleware):
-    def __init__(
-        self,
-        app,
-        *,
-        enabled: bool = True,
-        x_content_type_options: str = "nosniff",
-        x_frame_options: str = "DENY",
-        referrer_policy: str = "strict-origin-when-cross-origin",
-        permissions_policy: str = DEFAULT_PERMISSIONS_POLICY,
-        cross_origin_opener_policy: str = "same-origin",
-        cross_origin_resource_policy: str = "same-origin",
-        content_security_policy_enabled: bool = True,
-        content_security_policy: str = DEFAULT_CSP_POLICY,
-        content_security_policy_docs: str = DEFAULT_CSP_DOCS_POLICY,
-        content_security_policy_report_only: bool = False,
-        strict_transport_security_enabled: bool = False,
-        strict_transport_security_max_age: int = 31_536_000,
-        strict_transport_security_include_subdomains: bool = True,
-        strict_transport_security_preload: bool = False,
-    ) -> None:
-        super().__init__(app)
-        self._enabled = enabled
-        self._x_content_type_options = x_content_type_options.strip()
-        self._x_frame_options = x_frame_options.strip()
-        self._referrer_policy = referrer_policy.strip()
-        self._permissions_policy = permissions_policy.strip()
-        self._cross_origin_opener_policy = cross_origin_opener_policy.strip()
-        self._cross_origin_resource_policy = cross_origin_resource_policy.strip()
-        self._csp_enabled = content_security_policy_enabled
-        self._csp_policy = content_security_policy.strip()
-        self._csp_docs_policy = content_security_policy_docs.strip()
-        self._csp_report_only = content_security_policy_report_only
-        self._hsts_enabled = strict_transport_security_enabled
-        self._hsts_max_age = max(0, strict_transport_security_max_age)
-        self._hsts_include_subdomains = strict_transport_security_include_subdomains
-        self._hsts_preload = strict_transport_security_preload
-
-    async def dispatch(self, request: Request, call_next) -> Response:
-        response = await call_next(request)
-        if not self._enabled:
-            return response
-
-        if self._x_content_type_options:
-            response.headers.setdefault("X-Content-Type-Options", self._x_content_type_options)
-        if self._x_frame_options:
-            response.headers.setdefault("X-Frame-Options", self._x_frame_options)
-        if self._referrer_policy:
-            response.headers.setdefault("Referrer-Policy", self._referrer_policy)
-        if self._permissions_policy:
-            response.headers.setdefault("Permissions-Policy", self._permissions_policy)
-        if self._cross_origin_opener_policy:
-            response.headers.setdefault(
-                "Cross-Origin-Opener-Policy",
-                self._cross_origin_opener_policy,
-            )
-        if self._cross_origin_resource_policy:
-            response.headers.setdefault(
-                "Cross-Origin-Resource-Policy",
-                self._cross_origin_resource_policy,
-            )
-
-        csp_policy = self._csp_policy_for_path(request.url.path)
-        if self._csp_enabled and csp_policy:
-            csp_header = (
-                "Content-Security-Policy-Report-Only"
-                if self._csp_report_only
-                else "Content-Security-Policy"
-            )
-            response.headers.setdefault(csp_header, csp_policy)
-
-        hsts = self._strict_transport_security_value()
-        if hsts:
-            response.headers.setdefault("Strict-Transport-Security", hsts)
-
-        return response
-
-    def _csp_policy_for_path(self, path: str) -> str:
-        if path.startswith("/docs") or path.startswith("/redoc"):
-            return self._csp_docs_policy or self._csp_policy
-        return self._csp_policy
-
-    def _strict_transport_security_value(self) -> str:
-        if not self._hsts_enabled:
-            return ""
-
-        directives = [f"max-age={self._hsts_max_age}"]
-        if self._hsts_include_subdomains:
-            directives.append("includeSubDomains")
-        if self._hsts_preload:
-            directives.append("preload")
-        return "; ".join(directives)
-
-
-def parse_skip_paths(raw_value: str) -> tuple[str, ...]:
-    parts = [part.strip() for part in raw_value.split(",")]
-    return tuple(part for part in parts if part)

build/lib/app/logging_config.py

@@ -1,183 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime, timezone
-import json
-import logging
-import sys
-from typing import Any
-
-from app.logging_context import get_request_id
-
-DEFAULT_REDACT_FIELDS = {
-    "authorization",
-    "cookie",
-    "csrf_token",
-    "new_password",
-    "password",
-    "password_hash",
-    "session",
-    "session_secret_key",
-}
-
-_BASE_RECORD = logging.makeLogRecord({})
-_STANDARD_RECORD_FIELDS = set(_BASE_RECORD.__dict__.keys()) | {"message", "asctime"}
-_NOISY_RECORD_FIELDS = {"color_message"}
-
-
-def parse_redact_fields(raw: str | None) -> set[str]:
-    fields = {field.strip().lower() for field in (raw or "").split(",") if field.strip()}
-    return DEFAULT_REDACT_FIELDS | fields
-
-
-def configure_logging(
-    *,
-    level: str,
-    log_format: str,
-    redact_fields: set[str],
-    include_uvicorn_access: bool,
-) -> None:
-    normalized_level = _normalize_level(level)
-    root_logger = logging.getLogger()
-    root_logger.handlers.clear()
-    root_logger.setLevel(normalized_level)
-
-    handler = logging.StreamHandler(stream=sys.stdout)
-    handler.setLevel(normalized_level)
-    handler.addFilter(RequestContextFilter())
-
-    normalized_format = log_format.strip().lower()
-    if normalized_format == "json":
-        handler.setFormatter(JsonLogFormatter(redact_fields=redact_fields))
-    else:
-        handler.setFormatter(ConsoleLogFormatter(redact_fields=redact_fields))
-
-    root_logger.addHandler(handler)
-
-    # Route server/framework logs through our single root handler.
-    for logger_name in ("uvicorn", "uvicorn.error"):
-        framework_logger = logging.getLogger(logger_name)
-        framework_logger.handlers.clear()
-        framework_logger.propagate = True
-        framework_logger.setLevel(normalized_level)
-
-    access_logger = logging.getLogger("uvicorn.access")
-    access_logger.handlers.clear()
-    access_logger.propagate = True
-    access_logger.setLevel(normalized_level if include_uvicorn_access else logging.WARNING)
-
-
-class RequestContextFilter(logging.Filter):
-    def filter(self, record: logging.LogRecord) -> bool:
-        if not getattr(record, "request_id", None):
-            request_id = get_request_id()
-            if request_id:
-                record.request_id = request_id
-        return True
-
-
-class JsonLogFormatter(logging.Formatter):
-    def __init__(self, *, redact_fields: set[str]) -> None:
-        super().__init__()
-        self._redact_fields = {field.lower() for field in redact_fields}
-
-    def format(self, record: logging.LogRecord) -> str:
-        payload: dict[str, Any] = {
-            "timestamp": _format_timestamp(record.created),
-            "level": record.levelname.lower(),
-            "logger": record.name,
-            "event": getattr(record, "event", record.getMessage()),
-        }
-        request_id = getattr(record, "request_id", None)
-        if request_id:
-            payload["request_id"] = request_id
-        payload.update(self._redact_mapping(_extra_fields(record)))
-        if record.exc_info:
-            payload["exception"] = self.formatException(record.exc_info)
-        return json.dumps(payload, ensure_ascii=True, default=str)
-
-    def _redact_mapping(self, value: dict[str, Any]) -> dict[str, Any]:
-        return {key: self._redact_value(key, item) for key, item in value.items()}
-
-    def _redact_value(self, key: str, value: Any) -> Any:
-        if key.lower() in self._redact_fields:
-            return "[REDACTED]"
-        if isinstance(value, dict):
-            return {nested_key: self._redact_value(nested_key, nested_value) for nested_key, nested_value in value.items()}
-        if isinstance(value, (list, tuple)):
-            return [self._redact_value(key, item) for item in value]
-        return value
-
-
-class ConsoleLogFormatter(logging.Formatter):
-    def __init__(self, *, redact_fields: set[str]) -> None:
-        super().__init__()
-        self._json_formatter = JsonLogFormatter(redact_fields=redact_fields)
-
-    def format(self, record: logging.LogRecord) -> str:
-        payload = json.loads(self._json_formatter.format(record))
-        timestamp = payload.get("timestamp", "")
-        level = _short_level(payload.get("level", "info"))
-        logger_name = str(payload.get("logger", "app"))
-        event = str(payload.get("event", ""))
-
-        parts = [timestamp, level, logger_name, event]
-
-        request_id = payload.pop("request_id", None)
-        method = payload.pop("method", None)
-        path = payload.pop("path", None)
-        status_code = payload.pop("status_code", None)
-        duration_ms = payload.pop("duration_ms", None)
-
-        if request_id:
-            parts.append(f"rid={request_id}")
-        if method and path:
-            parts.append(f"{method} {path}")
-        if status_code is not None:
-            parts.append(str(status_code))
-        if duration_ms is not None:
-            parts.append(f"{duration_ms}ms")
-
-        for key in sorted(payload.keys()):
-            if key in {"timestamp", "level", "logger", "event", "exception"}:
-                continue
-            parts.append(f"{key}={payload[key]}")
-
-        if "exception" in payload:
-            parts.append(f"exception={payload['exception']}")
-
-        return " | ".join(str(part) for part in parts if part)
-
-
-def _extra_fields(record: logging.LogRecord) -> dict[str, Any]:
-    extras: dict[str, Any] = {}
-    for key, value in record.__dict__.items():
-        if key in _STANDARD_RECORD_FIELDS or key.startswith("_"):
-            continue
-        if key in _NOISY_RECORD_FIELDS:
-            continue
-        extras[key] = value
-    return extras
-
-
-def _normalize_level(level: str) -> int:
-    normalized = level.strip().upper()
-    mapping = logging.getLevelNamesMapping()
-    if normalized not in mapping:
-        return logging.INFO
-    return mapping[normalized]
-
-
-def _format_timestamp(created_ts: float) -> str:
-    dt = datetime.fromtimestamp(created_ts, tz=timezone.utc)
-    return dt.strftime("%Y-%m-%d %H:%M:%SZ")
-
-
-def _short_level(level: str) -> str:
-    mapping = {
-        "debug": "DBG",
-        "info": "INF",
-        "warning": "WRN",
-        "error": "ERR",
-        "critical": "CRT",
-    }
-    return mapping.get(level.lower(), level[:3].upper())

build/lib/app/logging_context.py

@@ -1,15 +0,0 @@
-from __future__ import annotations
-
-from contextvars import ContextVar
-
-
-_request_id_ctx: ContextVar[str | None] = ContextVar("request_id", default=None)
-
-
-def get_request_id() -> str | None:
-    return _request_id_ctx.get()
-
-
-def set_request_id(value: str | None) -> None:
-    _request_id_ctx.set(value)
-

build/lib/app/main.py

@@ -1,168 +0,0 @@
-from __future__ import annotations
-
-from contextlib import asynccontextmanager
-import logging
-from pathlib import Path
-
-from fastapi import FastAPI, HTTPException
-from fastapi.responses import FileResponse
-from fastapi.staticfiles import StaticFiles
-from starlette.middleware.sessions import SessionMiddleware
-
-from app.api.errors import register_api_exception_handlers
-from app.api.media import router as media_router
-from app.api.router import router as api_router
-from app.api.runtime_deps import get_ingestion_service, get_scholar_source
-from app.db.session import check_database
-from app.db.session import close_engine
-from app.http.middleware import (
-    RequestLoggingMiddleware,
-    SecurityHeadersMiddleware,
-    parse_skip_paths,
-)
-from app.logging_config import configure_logging, parse_redact_fields
-from app.security.csrf import CSRFMiddleware
-from app.services.domains.ingestion.scheduler import SchedulerService
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-BUILD_MARKER = "2026-02-19.phase2.direct-pdf-import-export-dashboard-sync"
-
-configure_logging(
-    level=settings.log_level,
-    log_format=settings.log_format,
-    redact_fields=parse_redact_fields(settings.log_redact_fields),
-    include_uvicorn_access=settings.log_uvicorn_access,
-)
-
-scheduler_service = SchedulerService(
-    enabled=settings.scheduler_enabled,
-    tick_seconds=settings.scheduler_tick_seconds,
-    network_error_retries=settings.ingestion_network_error_retries,
-    retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
-    max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
-    page_size=settings.ingestion_page_size,
-    continuation_queue_enabled=settings.ingestion_continuation_queue_enabled,
-    continuation_base_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
-    continuation_max_delay_seconds=settings.ingestion_continuation_max_delay_seconds,
-    continuation_max_attempts=settings.ingestion_continuation_max_attempts,
-    queue_batch_size=settings.scheduler_queue_batch_size,
-)
-
-
-def _log_startup_build_marker() -> None:
-    logger.info(
-        "app.startup_build_marker",
-        extra={
-            "event": "app.startup_build_marker",
-            "build_marker": BUILD_MARKER,
-            "frontend_enabled": settings.frontend_enabled,
-            "scheduler_enabled": settings.scheduler_enabled,
-            "log_format": settings.log_format,
-        },
-    )
-
-
-@asynccontextmanager
-async def lifespan(_: FastAPI):
-    _log_startup_build_marker()
-    
-    from app.db.session import get_session_factory
-    from sqlalchemy import text
-    try:
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            await session.execute(text("UPDATE crawl_runs SET status = 'failed' WHERE status = 'running'"))
-            await session.commit()
-            logger.info("app.startup_orphaned_runs_cleaned", extra={"event": "app.startup_orphaned_runs_cleaned"})
-    except Exception as e:
-        logger.error(f"Failed to clean orphaned runs at startup: {e}")
-
-    await scheduler_service.start()
-    yield
-    await scheduler_service.stop()
-    await close_engine()
-
-
-app = FastAPI(title=settings.app_name, lifespan=lifespan)
-register_api_exception_handlers(app)
-app.add_middleware(CSRFMiddleware)
-app.add_middleware(
-    SessionMiddleware,
-    secret_key=settings.session_secret_key,
-    same_site="lax",
-    https_only=settings.session_cookie_secure,
-)
-app.add_middleware(
-    RequestLoggingMiddleware,
-    log_requests=settings.log_requests,
-    skip_paths=parse_skip_paths(settings.log_request_skip_paths),
-)
-app.add_middleware(
-    SecurityHeadersMiddleware,
-    enabled=settings.security_headers_enabled,
-    x_content_type_options=settings.security_x_content_type_options,
-    x_frame_options=settings.security_x_frame_options,
-    referrer_policy=settings.security_referrer_policy,
-    permissions_policy=settings.security_permissions_policy,
-    cross_origin_opener_policy=settings.security_cross_origin_opener_policy,
-    cross_origin_resource_policy=settings.security_cross_origin_resource_policy,
-    content_security_policy_enabled=settings.security_csp_enabled,
-    content_security_policy=settings.security_csp_policy,
-    content_security_policy_docs=settings.security_csp_docs_policy,
-    content_security_policy_report_only=settings.security_csp_report_only,
-    strict_transport_security_enabled=settings.security_strict_transport_security_enabled,
-    strict_transport_security_max_age=settings.security_strict_transport_security_max_age,
-    strict_transport_security_include_subdomains=(
-        settings.security_strict_transport_security_include_subdomains
-    ),
-    strict_transport_security_preload=settings.security_strict_transport_security_preload,
-)
-app.include_router(api_router)
-app.include_router(media_router)
-
-
-@app.get("/healthz")
-async def healthz() -> dict[str, str]:
-    if await check_database():
-        return {"status": "ok"}
-    raise HTTPException(status_code=500, detail="database unavailable")
-
-
-def _configure_frontend_routes(application: FastAPI) -> None:
-    if not settings.frontend_enabled:
-        return
-
-    dist_dir = Path(settings.frontend_dist_dir)
-    index_file = dist_dir / "index.html"
-
-    if not index_file.is_file():
-        return
-
-    assets_dir = dist_dir / "assets"
-    if assets_dir.is_dir():
-        application.mount("/assets", StaticFiles(directory=assets_dir), name="frontend-assets")
-
-    @application.get("/", include_in_schema=False)
-    async def frontend_index() -> FileResponse:
-        return FileResponse(index_file)
-
-    @application.get("/{full_path:path}", include_in_schema=False)
-    async def frontend_entry(full_path: str) -> FileResponse:
-        normalized = full_path.lstrip("/")
-        if normalized in {"healthz", "api"} or normalized.startswith("api/"):
-            raise HTTPException(status_code=404, detail="Not Found")
-
-        candidate = (dist_dir / normalized).resolve()
-        try:
-            candidate.relative_to(dist_dir.resolve())
-        except ValueError as error:
-            raise HTTPException(status_code=404, detail="Not Found") from error
-
-        if candidate.is_file():
-            return FileResponse(candidate)
-
-        return FileResponse(index_file)
-
-
-_configure_frontend_routes(app)

build/lib/app/security/__init__.py

@@ -1,2 +0,0 @@
-"""Security middleware and helpers for scholarr."""
-

build/lib/app/security/csrf.py

@@ -1,133 +0,0 @@
-from __future__ import annotations
-
-import logging
-from secrets import compare_digest, token_urlsafe
-from urllib.parse import parse_qs
-
-from starlette.middleware.base import BaseHTTPMiddleware
-from starlette.requests import Request
-from starlette.responses import JSONResponse, PlainTextResponse, Response
-from starlette.types import Message
-
-
-CSRF_SESSION_KEY = "csrf_token"
-CSRF_FORM_FIELD = "csrf_token"
-CSRF_HEADER_NAME = "x-csrf-token"
-SAFE_METHODS = {"GET", "HEAD", "OPTIONS", "TRACE"}
-logger = logging.getLogger(__name__)
-
-
-def ensure_csrf_token(request: Request) -> str:
-    token = request.session.get(CSRF_SESSION_KEY)
-    if token is None:
-        token = token_urlsafe(32)
-        request.session[CSRF_SESSION_KEY] = token
-    return str(token)
-
-
-class CSRFMiddleware(BaseHTTPMiddleware):
-    def __init__(self, app, *, exempt_paths: set[str] | None = None) -> None:
-        super().__init__(app)
-        self._exempt_paths = exempt_paths or set()
-
-    async def dispatch(self, request: Request, call_next) -> Response:
-        if self._should_skip(request):
-            return await call_next(request)
-
-        session_token = request.session.get(CSRF_SESSION_KEY)
-        if not session_token:
-            logger.warning(
-                "csrf.missing_session_token",
-                extra={
-                    "event": "csrf.missing_session_token",
-                    "method": request.method,
-                    "path": request.url.path,
-                },
-            )
-            return self._csrf_error_response(
-                request,
-                code="csrf_missing",
-                message="CSRF token missing.",
-            )
-
-        request_token = request.headers.get(CSRF_HEADER_NAME)
-        if request_token is None and self._is_form_payload(request):
-            body = await request.body()
-            request_token = self._token_from_form_body(request, body)
-            request._receive = self._build_receive(body)  # type: ignore[attr-defined]
-
-        if not request_token or not compare_digest(str(session_token), str(request_token)):
-            logger.warning(
-                "csrf.invalid_token",
-                extra={
-                    "event": "csrf.invalid_token",
-                    "method": request.method,
-                    "path": request.url.path,
-                },
-            )
-            return self._csrf_error_response(
-                request,
-                code="csrf_invalid",
-                message="CSRF token invalid.",
-            )
-
-        return await call_next(request)
-
-    def _should_skip(self, request: Request) -> bool:
-        if request.method in SAFE_METHODS:
-            return True
-        path = request.url.path
-        return path in self._exempt_paths
-
-    def _is_form_payload(self, request: Request) -> bool:
-        content_type = request.headers.get("content-type", "")
-        return content_type.startswith("application/x-www-form-urlencoded") or (
-            content_type.startswith("multipart/form-data")
-        )
-
-    def _token_from_form_body(self, request: Request, body: bytes) -> str | None:
-        content_type = request.headers.get("content-type", "")
-        if not content_type.startswith("application/x-www-form-urlencoded"):
-            return None
-        parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True)
-        values = parsed.get(CSRF_FORM_FIELD)
-        if not values:
-            return None
-        return values[0]
-
-    def _build_receive(self, body: bytes):
-        consumed = False
-
-        async def receive() -> Message:
-            nonlocal consumed
-            if consumed:
-                return {"type": "http.request", "body": b"", "more_body": False}
-            consumed = True
-            return {"type": "http.request", "body": body, "more_body": False}
-
-        return receive
-
-    def _csrf_error_response(
-        self,
-        request: Request,
-        *,
-        code: str,
-        message: str,
-    ) -> Response:
-        if request.url.path.startswith("/api/"):
-            request_state = getattr(request, "state", None)
-            request_id = getattr(request_state, "request_id", None) if request_state else None
-            return JSONResponse(
-                {
-                    "error": {
-                        "code": code,
-                        "message": message,
-                        "details": None,
-                    },
-                    "meta": {
-                        "request_id": request_id,
-                    },
-                },
-                status_code=403,
-            )
-        return PlainTextResponse(message, status_code=403)

build/lib/app/services/__init__.py

@@ -1,2 +0,0 @@
-"""Service layer for scholarr application workflows."""
-

build/lib/app/services/domains/__init__.py

@@ -1 +0,0 @@
-from __future__ import annotations

build/lib/app/services/domains/arxiv/application.py

@@ -1,84 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-from typing import TYPE_CHECKING
-import xml.etree.ElementTree as ET
-
-import httpx
-
-from app.services.domains.publication_identifiers.normalize import normalize_arxiv_id
-from app.settings import settings
-
-if TYPE_CHECKING:
-    from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
-
-logger = logging.getLogger(__name__)
-
-
-def _build_arxiv_query(title: str, author_surname: str | None) -> str | None:
-    parts = []
-    if title:
-        # arXiv api allows strict title searching using ti:
-        clean_title = title.replace('"', '').replace("'", "")
-        parts.append(f'ti:"{clean_title}"')
-    if author_surname:
-        parts.append(f'au:"{author_surname}"')
-    
-    if not parts:
-        return None
-    return " AND ".join(parts)
-
-
-async def discover_arxiv_id_for_publication(
-    *,
-    item: PublicationListItem | UnreadPublicationItem,
-    request_email: str | None = None,
-    timeout_seconds: float = 3.0,
-) -> str | None:
-    title = (item.title or "").strip()
-    if not title:
-        return None
-        
-    author_surname = None
-    if item.scholar_label:
-        tokens = [t for t in item.scholar_label.strip().split() if t]
-        if tokens:
-            author_surname = tokens[-1].lower()
-
-    query = _build_arxiv_query(title, author_surname)
-    if not query:
-        return None
-
-    url = "https://export.arxiv.org/api/query"
-    params = {
-        "search_query": query,
-        "start": 0,
-        "max_results": 3,
-    }
-
-    headers = {"User-Agent": f"scholar-scraper/1.0 (mailto:{request_email or settings.crossref_api_mailto or 'unknown@example.com'})"}
-    
-    try:
-        async with httpx.AsyncClient(timeout=timeout_seconds, follow_redirects=True, headers=headers) as client:
-            response = await client.get(url, params=params)
-            response.raise_for_status()
-            
-        root = ET.fromstring(response.text)
-        namespace = {'atom': 'http://www.w3.org/2005/Atom'}
-        
-        for entry in root.findall('atom:entry', namespace):
-            id_elem = entry.find('atom:id', namespace)
-            if id_elem is not None and id_elem.text:
-                candidate = str(id_elem.text)
-                if '/abs/' in candidate:
-                    candidate = candidate.split('/abs/')[-1]
-                normalized = normalize_arxiv_id(candidate)
-                if normalized:
-                    logger.debug("arxiv.id_discovered", extra={"event": "arxiv.id_discovered", "arxiv_id": normalized})
-                    return normalized
-                    
-    except Exception as exc:
-        logger.debug(f"Failed to query arXiv API: {exc}")
-        
-    return None

build/lib/app/services/domains/crossref/__init__.py

@@ -1,5 +0,0 @@
-from __future__ import annotations
-
-from app.services.domains.crossref.application import discover_doi_for_publication
-
-__all__ = ["discover_doi_for_publication"]

build/lib/app/services/domains/crossref/application.py

@@ -1,382 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-import re
-import threading
-import time
-from typing import TYPE_CHECKING
-
-from crossref.restful import Etiquette, Works
-
-from app.services.domains.doi.normalize import normalize_doi
-from app.settings import settings
-
-if TYPE_CHECKING:
-    from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
-
-TOKEN_RE = re.compile(r"[a-z0-9]+")
-NON_ALNUM_RE = re.compile(r"[^a-z0-9\\s]+")
-STOP_WORDS = {"the", "and", "for", "with", "from", "method", "study", "analysis"}
-_RATE_LOCK = threading.Lock()
-_LAST_REQUEST_AT = 0.0
-logger = logging.getLogger(__name__)
-STRICT_TITLE_MATCH_THRESHOLD = 0.75
-RELAXED_TITLE_MATCH_THRESHOLD = 0.85
-
-
-def _rate_limit_wait(min_interval_seconds: float) -> None:
-    global _LAST_REQUEST_AT
-    interval = max(float(min_interval_seconds), 0.0)
-    with _RATE_LOCK:
-        elapsed = time.monotonic() - _LAST_REQUEST_AT
-        remaining = interval - elapsed
-        if remaining > 0:
-            time.sleep(remaining)
-        _LAST_REQUEST_AT = time.monotonic()
-
-
-def _normalized_tokens(value: str) -> list[str]:
-    lowered = value.lower().replace("’", "'").replace("“", "\"").replace("”", "\"")
-    lowered = NON_ALNUM_RE.sub(" ", lowered)
-    return [token for token in TOKEN_RE.findall(lowered) if len(token) >= 3]
-
-
-def _normalized_query(value: str) -> str:
-    tokens = [token for token in _normalized_tokens(value) if token not in STOP_WORDS]
-    if len(tokens) < 3:
-        tokens = _normalized_tokens(value)
-    if len(tokens) < 3:
-        return ""
-    return " ".join(tokens[:12]).strip()
-
-
-def _query_author(value: str) -> str | None:
-    tokens = [token for token in value.strip().split() if token]
-    if len(tokens) < 2:
-        return None
-    return " ".join(tokens[:2])[:64]
-
-
-def _author_surname(value: str) -> str | None:
-    tokens = [token for token in value.strip().split() if token]
-    if not tokens:
-        return None
-    return NON_ALNUM_RE.sub("", tokens[-1].lower()) or None
-
-
-def _query_filters(year: int | None) -> list[tuple[str, str] | None]:
-    if year is None:
-        return [None]
-    return [
-        (f"{year - 1}-01-01", f"{year + 1}-12-31"),
-        (f"{year}-01-01", f"{year}-12-31"),
-        None,
-    ]
-
-
-def _candidate_title(item: dict) -> str:
-    titles = item.get("title")
-    if isinstance(titles, list) and titles:
-        return str(titles[0] or "")
-    return str(item.get("title") or "")
-
-
-def _title_match_score(source: str, candidate: str) -> float:
-    source_tokens = {token for token in _normalized_tokens(source) if len(token) >= 3}
-    candidate_tokens = {token for token in _normalized_tokens(candidate) if len(token) >= 3}
-    if not source_tokens or not candidate_tokens:
-        return 0.0
-    return len(source_tokens & candidate_tokens) / float(len(source_tokens))
-
-
-def _candidate_year(item: dict) -> int | None:
-    issued = item.get("issued")
-    if not isinstance(issued, dict):
-        return None
-    date_parts = issued.get("date-parts")
-    if not isinstance(date_parts, list) or not date_parts:
-        return None
-    first = date_parts[0]
-    if not isinstance(first, list) or not first:
-        return None
-    try:
-        return int(first[0])
-    except (TypeError, ValueError):
-        return None
-
-
-def _candidate_author_match(item: dict, surname: str | None) -> bool:
-    if not surname:
-        return True
-    authors = item.get("author")
-    if not isinstance(authors, list):
-        return False
-    for author in authors:
-        if not isinstance(author, dict):
-            continue
-        family = NON_ALNUM_RE.sub("", str(author.get("family") or "").lower())
-        if family and family == surname:
-            return True
-    return False
-
-
-def _candidate_rank(*, title: str, year: int | None, item: dict) -> tuple[float, str | None]:
-    doi = normalize_doi(str(item.get("DOI") or ""))
-    if doi is None:
-        return 0.0, None
-    score = _title_match_score(title, _candidate_title(item))
-    candidate_year = _candidate_year(item)
-    if year is not None and candidate_year is not None:
-        if abs(year - candidate_year) > 1:
-            return 0.0, None
-        score += 0.1
-    return score, doi
-
-
-def _year_delta(source_year: int | None, candidate_year: int | None) -> int | None:
-    if source_year is None or candidate_year is None:
-        return None
-    return abs(int(source_year) - int(candidate_year))
-
-
-def _candidate_rank_relaxed(
-    *,
-    title: str,
-    year: int | None,
-    item: dict,
-    author_surname: str | None,
-) -> tuple[float, str | None]:
-    doi = normalize_doi(str(item.get("DOI") or ""))
-    if doi is None:
-        return 0.0, None
-    score = _title_match_score(title, _candidate_title(item))
-    if score <= 0:
-        return 0.0, None
-    candidate_year = _candidate_year(item)
-    delta = _year_delta(year, candidate_year)
-    if delta is not None:
-        if delta <= 1:
-            score += 0.05
-        elif delta <= 3:
-            score += 0.0
-        elif delta <= 5:
-            score -= 0.03
-        else:
-            score -= 0.08
-    if _candidate_author_match(item, author_surname):
-        score += 0.03
-    return score, doi
-
-
-def _best_candidate_doi_strict(
-    *,
-    title: str,
-    year: int | None,
-    items: list[dict],
-    author_surname: str | None,
-) -> str | None:
-    best_score = 0.0
-    best_doi: str | None = None
-    best_year: int | None = None
-    for item in items:
-        if not isinstance(item, dict):
-            continue
-        if not _candidate_author_match(item, author_surname):
-            continue
-        score, doi = _candidate_rank(title=title, year=year, item=item)
-        candidate_year = _candidate_year(item)
-        if doi is None or score < STRICT_TITLE_MATCH_THRESHOLD:
-            continue
-        if score > best_score:
-            best_score = score
-            best_doi = doi
-            best_year = candidate_year
-            continue
-        if abs(score - best_score) > 0.02:
-            continue
-        if best_year is None or candidate_year is None:
-            continue
-        if candidate_year < best_year:
-            best_doi = doi
-            best_year = candidate_year
-    return best_doi
-
-
-def _best_candidate_doi_relaxed(
-    *,
-    title: str,
-    year: int | None,
-    items: list[dict],
-    author_surname: str | None,
-) -> str | None:
-    best_score = 0.0
-    best_doi: str | None = None
-    best_author_match = False
-    best_delta: int | None = None
-    best_year: int | None = None
-    for item in items:
-        if not isinstance(item, dict):
-            continue
-        score, doi = _candidate_rank_relaxed(
-            title=title,
-            year=year,
-            item=item,
-            author_surname=author_surname,
-        )
-        if doi is None or score < RELAXED_TITLE_MATCH_THRESHOLD:
-            continue
-        candidate_year = _candidate_year(item)
-        candidate_author_match = _candidate_author_match(item, author_surname)
-        candidate_delta = _year_delta(year, candidate_year)
-        if score > best_score:
-            best_score = score
-            best_doi = doi
-            best_author_match = candidate_author_match
-            best_delta = candidate_delta
-            best_year = candidate_year
-            continue
-        if abs(score - best_score) > 0.02:
-            continue
-        if candidate_author_match and not best_author_match:
-            best_doi = doi
-            best_author_match = True
-            best_delta = candidate_delta
-            best_year = candidate_year
-            continue
-        if best_delta is None and candidate_delta is not None:
-            best_doi = doi
-            best_author_match = candidate_author_match
-            best_delta = candidate_delta
-            best_year = candidate_year
-            continue
-        if best_delta is not None and candidate_delta is not None and candidate_delta < best_delta:
-            best_doi = doi
-            best_author_match = candidate_author_match
-            best_delta = candidate_delta
-            best_year = candidate_year
-            continue
-        if best_year is None or candidate_year is None:
-            continue
-        if candidate_year < best_year:
-            best_doi = doi
-            best_author_match = candidate_author_match
-            best_delta = candidate_delta
-            best_year = candidate_year
-    return best_doi
-
-
-def _best_candidate_doi(
-    *,
-    title: str,
-    year: int | None,
-    items: list[dict],
-    author_surname: str | None,
-) -> str | None:
-    strict_match = _best_candidate_doi_strict(
-        title=title,
-        year=year,
-        items=items,
-        author_surname=author_surname,
-    )
-    if strict_match:
-        return strict_match
-    return _best_candidate_doi_relaxed(
-        title=title,
-        year=year,
-        items=items,
-        author_surname=author_surname,
-    )
-
-
-def _works_client(email: str | None) -> Works:
-    if email:
-        etiquette = Etiquette(settings.app_name, "0.1.0", "https://scholarr.local", email)
-        return Works(etiquette=etiquette)
-    return Works()
-
-
-def _fetch_items_sync(
-    *,
-    query: str,
-    author: str | None,
-    date_range: tuple[str, str] | None,
-    max_rows: int,
-    email: str | None,
-    min_interval_seconds: float,
-) -> list[dict]:
-    _rate_limit_wait(min_interval_seconds)
-    works = _works_client(email)
-    params = {"bibliographic": query}
-    if author:
-        params["author"] = author
-    request = works.query(**params)
-    if date_range is not None:
-        from_date, until_date = date_range
-        request = request.filter(from_pub_date=from_date, until_pub_date=until_date)
-    request = request.select(["DOI", "title", "issued", "score", "author"])
-    items: list[dict] = []
-    for entry in request:
-        if isinstance(entry, dict):
-            items.append(entry)
-        if len(items) >= max(max_rows, 1):
-            break
-    return items
-
-
-async def _fetch_items(
-    *,
-    query: str,
-    author: str | None,
-    date_range: tuple[str, str] | None,
-    max_rows: int,
-    email: str | None,
-) -> list[dict]:
-    timeout = max(float(settings.crossref_timeout_seconds), 0.5)
-    try:
-        return await asyncio.wait_for(
-            asyncio.to_thread(
-                _fetch_items_sync,
-                query=query,
-                author=author,
-                date_range=date_range,
-                max_rows=max_rows,
-                email=email,
-                min_interval_seconds=settings.crossref_min_interval_seconds,
-            ),
-            timeout=timeout,
-        )
-    except Exception:
-        return []
-
-
-async def discover_doi_for_publication(
-    *,
-    item: PublicationListItem | UnreadPublicationItem,
-    max_rows: int = 10,
-    email: str | None = None,
-) -> str | None:
-    title = (item.title or "").strip()
-    query = _normalized_query(title)
-    if not query:
-        return None
-    author = _query_author(item.scholar_label)
-    author_surname = _author_surname(item.scholar_label)
-    for date_range in _query_filters(item.year):
-        items = await _fetch_items(
-            query=query,
-            author=author,
-            date_range=date_range,
-            max_rows=max_rows,
-            email=email,
-        )
-        doi = _best_candidate_doi(
-            title=title,
-            year=item.year,
-            items=items,
-            author_surname=author_surname,
-        )
-        if doi:
-            logger.debug("crossref.doi_discovered", extra={"event": "crossref.doi_discovered"})
-            return doi
-    return None

build/lib/app/services/domains/dbops/__init__.py

@@ -1,5 +0,0 @@
-from app.services.domains.dbops.application import run_publication_link_repair
-from app.services.domains.dbops.integrity import collect_integrity_report
-from app.services.domains.dbops.query import list_repair_jobs
-
-__all__ = ["collect_integrity_report", "list_repair_jobs", "run_publication_link_repair"]

build/lib/app/services/domains/dbops/application.py

@@ -1,364 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime, timezone
-from typing import Any
-
-from sqlalchemy import delete, exists, func, select, update
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import DataRepairJob, IngestionQueueItem, Publication, ScholarProfile, ScholarPublication
-
-
-REPAIR_STATUS_PLANNED = "planned"
-REPAIR_STATUS_RUNNING = "running"
-REPAIR_STATUS_COMPLETED = "completed"
-REPAIR_STATUS_FAILED = "failed"
-SCOPE_MODE_SINGLE_USER = "single_user"
-SCOPE_MODE_ALL_USERS = "all_users"
-
-
-def _utcnow() -> datetime:
-    return datetime.now(timezone.utc)
-
-
-def _normalize_scope_mode(scope_mode: str) -> str:
-    normalized = scope_mode.strip().lower()
-    if normalized in {SCOPE_MODE_SINGLE_USER, SCOPE_MODE_ALL_USERS}:
-        return normalized
-    raise ValueError("Unknown scope mode.")
-
-
-def _scope_user_id(*, scope_mode: str, user_id: int | None) -> int | None:
-    if scope_mode == SCOPE_MODE_SINGLE_USER:
-        if user_id is None:
-            raise ValueError("user_id is required when scope_mode=single_user.")
-        return int(user_id)
-    if user_id is not None:
-        raise ValueError("user_id must be omitted when scope_mode=all_users.")
-    return None
-
-
-def _scope_payload(
-    *,
-    scope_mode: str,
-    user_id: int | None,
-    target_scholar_profile_ids: list[int],
-    orphan_gc: bool,
-) -> dict[str, Any]:
-    payload: dict[str, Any] = {
-        "scope_mode": scope_mode,
-        "scholar_profile_ids": [int(value) for value in target_scholar_profile_ids],
-        "gc_orphan_publications": bool(orphan_gc),
-    }
-    if user_id is not None:
-        payload["user_id"] = int(user_id)
-    return payload
-
-
-async def _target_scholar_profile_ids(
-    db_session: AsyncSession,
-    *,
-    scope_mode: str,
-    user_id: int | None,
-    scholar_profile_ids: list[int] | None,
-) -> list[int]:
-    stmt = select(ScholarProfile.id)
-    if scope_mode == SCOPE_MODE_SINGLE_USER:
-        stmt = stmt.where(ScholarProfile.user_id == user_id)
-    if scholar_profile_ids:
-        normalized_ids = [int(value) for value in scholar_profile_ids]
-        stmt = stmt.where(ScholarProfile.id.in_(normalized_ids))
-    result = await db_session.execute(stmt.order_by(ScholarProfile.id.asc()))
-    ids = [int(row[0]) for row in result.all()]
-    if not ids:
-        raise ValueError("No target scholar profiles found for the requested scope.")
-    return ids
-
-
-async def _count_scope(
-    db_session: AsyncSession,
-    *,
-    user_id: int | None,
-    target_scholar_profile_ids: list[int],
-) -> dict[str, int]:
-    links_result = await db_session.execute(
-        select(func.count())
-        .select_from(ScholarPublication)
-        .where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids))
-    )
-    queue_stmt = (
-        select(func.count())
-        .select_from(IngestionQueueItem)
-        .where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids))
-    )
-    if user_id is not None:
-        queue_stmt = queue_stmt.where(IngestionQueueItem.user_id == user_id)
-    queue_result = await db_session.execute(queue_stmt)
-    return {
-        "target_scholar_count": len(target_scholar_profile_ids),
-        "links_in_scope": int(links_result.scalar_one() or 0),
-        "queue_items_in_scope": int(queue_result.scalar_one() or 0),
-    }
-
-
-async def _count_orphan_publications(db_session: AsyncSession) -> int:
-    stmt = (
-        select(func.count())
-        .select_from(Publication)
-        .where(
-            ~exists(
-                select(1).where(ScholarPublication.publication_id == Publication.id)
-            )
-        )
-    )
-    result = await db_session.execute(stmt)
-    return int(result.scalar_one() or 0)
-
-
-async def _create_job(
-    db_session: AsyncSession,
-    *,
-    requested_by: str | None,
-    scope: dict[str, Any],
-    dry_run: bool,
-) -> DataRepairJob:
-    job = DataRepairJob(
-        job_name="repair_publication_links",
-        requested_by=(requested_by or "").strip() or None,
-        scope=scope,
-        dry_run=dry_run,
-        status=REPAIR_STATUS_PLANNED,
-        summary={},
-    )
-    db_session.add(job)
-    await db_session.flush()
-    return job
-
-
-def _job_summary(
-    *,
-    counts: dict[str, int],
-    dry_run: bool,
-    links_deleted: int,
-    queue_items_deleted: int,
-    scholars_reset: int,
-    orphan_publications_before: int,
-    orphan_publications_deleted: int,
-) -> dict[str, Any]:
-    return {
-        **counts,
-        "dry_run": bool(dry_run),
-        "links_deleted": int(links_deleted),
-        "queue_items_deleted": int(queue_items_deleted),
-        "scholars_reset": int(scholars_reset),
-        "orphan_publications_before": int(orphan_publications_before),
-        "orphan_publications_deleted": int(orphan_publications_deleted),
-    }
-
-
-async def _delete_links_for_targets(db_session: AsyncSession, *, target_scholar_profile_ids: list[int]) -> int:
-    result = await db_session.execute(
-        delete(ScholarPublication).where(
-            ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids)
-        )
-    )
-    return int(result.rowcount or 0)
-
-
-async def _delete_queue_for_targets(
-    db_session: AsyncSession,
-    *,
-    user_id: int | None,
-    target_scholar_profile_ids: list[int],
-) -> int:
-    stmt = delete(IngestionQueueItem).where(
-        IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids)
-    )
-    if user_id is not None:
-        stmt = stmt.where(IngestionQueueItem.user_id == user_id)
-    result = await db_session.execute(stmt)
-    return int(result.rowcount or 0)
-
-
-async def _reset_scholar_tracking_state(
-    db_session: AsyncSession,
-    *,
-    user_id: int | None,
-    target_scholar_profile_ids: list[int],
-) -> int:
-    stmt = update(ScholarProfile).where(ScholarProfile.id.in_(target_scholar_profile_ids))
-    if user_id is not None:
-        stmt = stmt.where(ScholarProfile.user_id == user_id)
-    result = await db_session.execute(
-        stmt.values(
-            baseline_completed=False,
-            last_initial_page_fingerprint_sha256=None,
-            last_initial_page_checked_at=None,
-            last_run_dt=None,
-            last_run_status=None,
-        )
-    )
-    return int(result.rowcount or 0)
-
-
-async def _delete_orphan_publications(db_session: AsyncSession) -> int:
-    result = await db_session.execute(
-        delete(Publication).where(
-            ~exists(select(1).where(ScholarPublication.publication_id == Publication.id))
-        )
-    )
-    return int(result.rowcount or 0)
-
-
-async def _mutation_counts(
-    db_session: AsyncSession,
-    *,
-    user_id: int | None,
-    target_ids: list[int],
-    dry_run: bool,
-    gc_orphan_publications: bool,
-) -> tuple[int, int, int, int]:
-    if dry_run:
-        return 0, 0, 0, 0
-    links_deleted = await _delete_links_for_targets(
-        db_session,
-        target_scholar_profile_ids=target_ids,
-    )
-    queue_deleted = await _delete_queue_for_targets(
-        db_session,
-        user_id=user_id,
-        target_scholar_profile_ids=target_ids,
-    )
-    scholars_reset = await _reset_scholar_tracking_state(
-        db_session,
-        user_id=user_id,
-        target_scholar_profile_ids=target_ids,
-    )
-    orphan_deleted = 0
-    if gc_orphan_publications:
-        orphan_deleted = await _delete_orphan_publications(db_session)
-    return links_deleted, queue_deleted, scholars_reset, orphan_deleted
-
-
-def _result_payload(*, job: DataRepairJob, scope: dict[str, Any], summary: dict[str, Any]) -> dict[str, Any]:
-    return {
-        "job_id": int(job.id),
-        "status": job.status,
-        "scope": scope,
-        "summary": summary,
-    }
-
-
-async def _complete_job(
-    db_session: AsyncSession,
-    *,
-    job: DataRepairJob,
-    summary: dict[str, Any],
-    scope: dict[str, Any],
-) -> dict[str, Any]:
-    job.summary = summary
-    job.status = REPAIR_STATUS_COMPLETED
-    job.finished_at = _utcnow()
-    await db_session.commit()
-    return _result_payload(job=job, scope=scope, summary=summary)
-
-
-async def _fail_job(db_session: AsyncSession, *, job: DataRepairJob, error: Exception) -> None:
-    await db_session.rollback()
-    job.status = REPAIR_STATUS_FAILED
-    job.error_text = str(error)
-    job.finished_at = _utcnow()
-    db_session.add(job)
-    await db_session.commit()
-
-
-async def _prepare_repair_job(
-    db_session: AsyncSession,
-    *,
-    scope_mode: str,
-    user_id: int | None,
-    scholar_profile_ids: list[int] | None,
-    dry_run: bool,
-    gc_orphan_publications: bool,
-    requested_by: str | None,
-) -> tuple[int | None, list[int], dict[str, Any], DataRepairJob]:
-    normalized_scope = _normalize_scope_mode(scope_mode)
-    scope_user_id = _scope_user_id(scope_mode=normalized_scope, user_id=user_id)
-    target_ids = await _target_scholar_profile_ids(
-        db_session,
-        scope_mode=normalized_scope,
-        user_id=scope_user_id,
-        scholar_profile_ids=scholar_profile_ids,
-    )
-    scope = _scope_payload(
-        scope_mode=normalized_scope,
-        user_id=scope_user_id,
-        target_scholar_profile_ids=target_ids,
-        orphan_gc=gc_orphan_publications,
-    )
-    job = await _create_job(db_session, requested_by=requested_by, scope=scope, dry_run=dry_run)
-    job.status = REPAIR_STATUS_RUNNING
-    job.started_at = _utcnow()
-    return scope_user_id, target_ids, scope, job
-
-
-async def _build_repair_summary(
-    db_session: AsyncSession,
-    *,
-    scope_user_id: int | None,
-    target_ids: list[int],
-    dry_run: bool,
-    gc_orphan_publications: bool,
-) -> dict[str, Any]:
-    counts = await _count_scope(db_session, user_id=scope_user_id, target_scholar_profile_ids=target_ids)
-    orphan_before = await _count_orphan_publications(db_session)
-    links_deleted, queue_deleted, scholars_reset, orphan_deleted = await _mutation_counts(
-        db_session,
-        user_id=scope_user_id,
-        target_ids=target_ids,
-        dry_run=dry_run,
-        gc_orphan_publications=gc_orphan_publications,
-    )
-    return _job_summary(
-        counts=counts,
-        dry_run=dry_run,
-        links_deleted=links_deleted,
-        queue_items_deleted=queue_deleted,
-        scholars_reset=scholars_reset,
-        orphan_publications_before=orphan_before,
-        orphan_publications_deleted=orphan_deleted,
-    )
-
-
-async def run_publication_link_repair(
-    db_session: AsyncSession,
-    *,
-    scope_mode: str = SCOPE_MODE_SINGLE_USER,
-    user_id: int | None = None,
-    scholar_profile_ids: list[int] | None = None,
-    dry_run: bool = True,
-    gc_orphan_publications: bool = False,
-    requested_by: str | None = None,
-) -> dict[str, Any]:
-    scope_user_id, target_ids, scope, job = await _prepare_repair_job(
-        db_session,
-        scope_mode=scope_mode,
-        user_id=user_id,
-        scholar_profile_ids=scholar_profile_ids,
-        dry_run=dry_run,
-        gc_orphan_publications=gc_orphan_publications,
-        requested_by=requested_by,
-    )
-
-    try:
-        summary = await _build_repair_summary(
-            db_session,
-            scope_user_id=scope_user_id,
-            target_ids=target_ids,
-            dry_run=dry_run,
-            gc_orphan_publications=gc_orphan_publications,
-        )
-        return await _complete_job(db_session, job=job, summary=summary, scope=scope)
-    except Exception as exc:
-        await _fail_job(db_session, job=job, error=exc)
-        raise

build/lib/app/services/domains/dbops/integrity.py

@@ -1,175 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime, timezone
-from typing import Any
-
-from sqlalchemy import func, select, text
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import Publication
-
-INTEGRITY_CHECK_DEFS = (
-    (
-        "legacy_cluster_id_format",
-        "warning",
-        "Publications with non-namespaced cluster IDs.",
-    ),
-    (
-        "negative_citation_count",
-        "failure",
-        "Publications with negative citation counts.",
-    ),
-    (
-        "orphan_publications",
-        "warning",
-        "Publications with no scholar links.",
-    ),
-    (
-        "orphan_scholar_publication_links",
-        "failure",
-        "Link rows missing parent scholar/publication.",
-    ),
-    (
-        "duplicate_fingerprint_keys",
-        "failure",
-        "Duplicate publication fingerprint keys.",
-    ),
-    (
-        "duplicate_cluster_ids",
-        "failure",
-        "Duplicate non-null publication cluster IDs.",
-    ),
-    (
-        "missing_pdf_url",
-        "metric",
-        "Publications without a resolved PDF URL.",
-    ),
-)
-
-
-async def _legacy_cluster_id_count(db_session: AsyncSession) -> int:
-    stmt = (
-        select(func.count())
-        .select_from(Publication)
-        .where(Publication.cluster_id.is_not(None))
-        .where(~Publication.cluster_id.like("cfv:%"))
-        .where(~Publication.cluster_id.like("cluster:%"))
-    )
-    result = await db_session.execute(stmt)
-    return int(result.scalar_one() or 0)
-
-
-async def _negative_citation_count(db_session: AsyncSession) -> int:
-    result = await db_session.execute(
-        select(func.count()).select_from(Publication).where(Publication.citation_count < 0)
-    )
-    return int(result.scalar_one() or 0)
-
-
-async def _missing_pdf_url_count(db_session: AsyncSession) -> int:
-    result = await db_session.execute(
-        select(func.count()).select_from(Publication).where(Publication.pdf_url.is_(None))
-    )
-    return int(result.scalar_one() or 0)
-
-
-async def _count_from_sql(db_session: AsyncSession, *, sql: str) -> int:
-    result = await db_session.execute(text(sql))
-    return int(result.scalar_one() or 0)
-
-
-def _issues_for_severity(*, checks: list[dict[str, Any]], severity: str) -> list[str]:
-    return [row["name"] for row in checks if row["severity"] == severity and row["count"] > 0]
-
-
-def _check_row(*, name: str, count: int, severity: str, message: str) -> dict[str, Any]:
-    return {
-        "name": name,
-        "count": int(count),
-        "severity": severity,
-        "message": message,
-    }
-
-
-async def _collect_counts(db_session: AsyncSession) -> dict[str, int]:
-    orphan_publications = await _count_from_sql(
-        db_session,
-        sql=(
-            "SELECT count(*) FROM publications p "
-            "LEFT JOIN scholar_publications sp ON sp.publication_id = p.id "
-            "WHERE sp.publication_id IS NULL"
-        ),
-    )
-    orphan_links = await _count_from_sql(
-        db_session,
-        sql=(
-            "SELECT count(*) FROM scholar_publications sp "
-            "LEFT JOIN publications p ON p.id = sp.publication_id "
-            "LEFT JOIN scholar_profiles s ON s.id = sp.scholar_profile_id "
-            "WHERE p.id IS NULL OR s.id IS NULL"
-        ),
-    )
-    duplicate_fingerprints = await _count_from_sql(
-        db_session,
-        sql=(
-            "SELECT count(*) FROM ("
-            "SELECT fingerprint_sha256 FROM publications "
-            "GROUP BY fingerprint_sha256 HAVING count(*) > 1"
-            ") dup"
-        ),
-    )
-    duplicate_cluster_ids = await _count_from_sql(
-        db_session,
-        sql=(
-            "SELECT count(*) FROM ("
-            "SELECT cluster_id FROM publications "
-            "WHERE cluster_id IS NOT NULL "
-            "GROUP BY cluster_id HAVING count(*) > 1"
-            ") dup"
-        ),
-    )
-    return {
-        "legacy_cluster_id_format": await _legacy_cluster_id_count(db_session),
-        "negative_citation_count": await _negative_citation_count(db_session),
-        "orphan_publications": orphan_publications,
-        "orphan_scholar_publication_links": orphan_links,
-        "duplicate_fingerprint_keys": duplicate_fingerprints,
-        "duplicate_cluster_ids": duplicate_cluster_ids,
-        "missing_pdf_url": await _missing_pdf_url_count(db_session),
-    }
-
-
-def _build_checks(*, counts: dict[str, int]) -> list[dict[str, Any]]:
-    return [
-        _check_row(
-            name=name,
-            count=counts[name],
-            severity=severity,
-            message=message,
-        )
-        for name, severity, message in INTEGRITY_CHECK_DEFS
-    ]
-
-
-def _status_from_issues(*, failures: list[str], warnings: list[str]) -> str:
-    status = "ok"
-    if failures:
-        status = "failed"
-    elif warnings:
-        status = "warning"
-    return status
-
-
-async def collect_integrity_report(db_session: AsyncSession) -> dict[str, Any]:
-    counts = await _collect_counts(db_session)
-    checks = _build_checks(counts=counts)
-    failures = _issues_for_severity(checks=checks, severity="failure")
-    warnings = _issues_for_severity(checks=checks, severity="warning")
-
-    return {
-        "status": _status_from_issues(failures=failures, warnings=warnings),
-        "checked_at": datetime.now(timezone.utc).isoformat(),
-        "failures": failures,
-        "warnings": warnings,
-        "checks": checks,
-    }

build/lib/app/services/domains/dbops/query.py

@@ -1,22 +0,0 @@
-from __future__ import annotations
-
-from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import DataRepairJob
-
-
-def _bounded_limit(limit: int) -> int:
-    return max(1, min(int(limit), 200))
-
-
-async def list_repair_jobs(
-    db_session: AsyncSession,
-    *,
-    limit: int = 50,
-) -> list[DataRepairJob]:
-    bounded = _bounded_limit(limit)
-    result = await db_session.execute(
-        select(DataRepairJob).order_by(DataRepairJob.created_at.desc()).limit(bounded)
-    )
-    return list(result.scalars())

build/lib/app/services/domains/doi/normalize.py

@@ -1,23 +0,0 @@
-from __future__ import annotations
-
-import re
-from urllib.parse import unquote
-
-DOI_RE = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
-
-
-def normalize_doi(value: str | None) -> str | None:
-    if not value:
-        return None
-    match = DOI_RE.search(unquote(value))
-    if not match:
-        return None
-    return match.group(0).rstrip(" .;,)").lower()
-
-
-def first_doi_from_texts(*values: str | None) -> str | None:
-    for value in values:
-        doi = normalize_doi(value)
-        if doi:
-            return doi
-    return None

build/lib/app/services/domains/ingestion/__init__.py

@@ -1 +0,0 @@
-from __future__ import annotations

build/lib/app/services/domains/ingestion/constants.py

@@ -1,31 +0,0 @@
-from __future__ import annotations
-
-import re
-
-from app.services.domains.scholar.parser import ParseState
-
-TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+")
-WORD_RE = re.compile(r"[a-z0-9]+")
-HTML_TAG_RE = re.compile(r"<[^>]+>", re.S)
-SPACE_RE = re.compile(r"\s+")
-
-FAILED_STATES = {
-    ParseState.BLOCKED_OR_CAPTCHA.value,
-    ParseState.LAYOUT_CHANGED.value,
-    ParseState.NETWORK_ERROR.value,
-    "ingestion_error",
-}
-
-FAILURE_BUCKET_BLOCKED = "blocked_or_captcha"
-FAILURE_BUCKET_NETWORK = "network_error"
-FAILURE_BUCKET_LAYOUT = "layout_changed"
-FAILURE_BUCKET_INGESTION = "ingestion_error"
-FAILURE_BUCKET_OTHER = "other_failure"
-
-RUN_LOCK_NAMESPACE = 8217
-RESUMABLE_PARTIAL_REASONS = {
-    "max_pages_reached",
-    "pagination_cursor_stalled",
-}
-RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",)
-INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS = 30

build/lib/app/services/domains/ingestion/fingerprints.py

@@ -1,175 +0,0 @@
-from __future__ import annotations
-
-import hashlib
-import json
-import re
-from typing import Any
-from urllib.parse import urljoin
-
-from app.services.domains.ingestion.constants import (
-    HTML_TAG_RE,
-    INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS,
-    SPACE_RE,
-    TITLE_ALNUM_RE,
-    WORD_RE,
-)
-from app.services.domains.scholar.parser import ParseState, ParsedProfilePage, PublicationCandidate
-
-
-def normalize_title(value: str) -> str:
-    lowered = value.lower()
-    return TITLE_ALNUM_RE.sub("", lowered)
-
-
-def _first_author_last_name(authors_text: str | None) -> str:
-    if not authors_text:
-        return ""
-    first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
-    words = WORD_RE.findall(first_author)
-    if not words:
-        return ""
-    return words[-1]
-
-
-def _first_venue_word(venue_text: str | None) -> str:
-    if not venue_text:
-        return ""
-    words = WORD_RE.findall(venue_text.lower())
-    if not words:
-        return ""
-    return words[0]
-
-
-def build_publication_fingerprint(candidate: PublicationCandidate) -> str:
-    canonical = "|".join(
-        [
-            normalize_title(candidate.title),
-            str(candidate.year) if candidate.year is not None else "",
-            _first_author_last_name(candidate.authors_text),
-            _first_venue_word(candidate.venue_text),
-        ]
-    )
-    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
-
-
-def build_initial_page_fingerprint(parsed_page: ParsedProfilePage) -> str | None:
-    if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}:
-        return None
-
-    normalized_rows: list[dict[str, Any]] = []
-    for publication in parsed_page.publications[:INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS]:
-        normalized_rows.append(
-            {
-                "cluster_id": publication.cluster_id or "",
-                "title_normalized": normalize_title(publication.title),
-                "year": publication.year,
-                "citation_count": publication.citation_count,
-            }
-        )
-
-    payload = {
-        "state": parsed_page.state.value,
-        "articles_range": parsed_page.articles_range or "",
-        "has_show_more_button": parsed_page.has_show_more_button,
-        "profile_name": parsed_page.profile_name or "",
-        "publications": normalized_rows,
-    }
-    canonical = json.dumps(
-        payload,
-        sort_keys=True,
-        separators=(",", ":"),
-        ensure_ascii=True,
-    )
-    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
-
-
-def build_publication_url(path_or_url: str | None) -> str | None:
-    if not path_or_url:
-        return None
-    return urljoin("https://scholar.google.com", path_or_url)
-
-
-def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int:
-    if articles_range:
-        numbers = re.findall(r"\d+", articles_range)
-        if len(numbers) >= 2:
-            try:
-                return int(numbers[1])
-            except ValueError:
-                pass
-    return int(fallback)
-
-
-def _title_tokens(value: str) -> set[str]:
-    """Extract normalized word tokens for fuzzy title comparison."""
-    return set(WORD_RE.findall(value.lower()))
-
-
-def fuzzy_titles_match(
-    title_a: str,
-    title_b: str,
-    *,
-    threshold: float = 0.85,
-) -> bool:
-    """Return True if two titles are near-duplicates by token-level Jaccard similarity.
-
-    A threshold of 0.85 catches common academic duplicate patterns:
-    differences in punctuation, minor word variations, subtitle changes.
-    """
-    tokens_a = _title_tokens(title_a)
-    tokens_b = _title_tokens(title_b)
-    if not tokens_a or not tokens_b:
-        return False
-    intersection = tokens_a & tokens_b
-    union = tokens_a | tokens_b
-    return (len(intersection) / len(union)) >= threshold
-
-
-def _dedupe_publication_candidates(
-    publications: list[PublicationCandidate],
-) -> list[PublicationCandidate]:
-    deduped: list[PublicationCandidate] = []
-    seen: set[str] = set()
-    seen_titles: list[tuple[str, int]] = []  # (normalized_title, index into deduped)
-
-    for publication in publications:
-        if publication.cluster_id:
-            identity = f"cluster:{publication.cluster_id}"
-        else:
-            identity = "|".join(
-                [
-                    "fallback",
-                    normalize_title(publication.title),
-                    str(publication.year) if publication.year is not None else "",
-                    _first_author_last_name(publication.authors_text),
-                    _first_venue_word(publication.venue_text),
-                ]
-            )
-        if identity in seen:
-            continue
-
-        # Fuzzy title check — catch near-identical titles not caught by exact fingerprint
-        norm_title = normalize_title(publication.title)
-        is_fuzzy_dup = False
-        for existing_title, _idx in seen_titles:
-            if fuzzy_titles_match(norm_title, existing_title):
-                is_fuzzy_dup = True
-                break
-        if is_fuzzy_dup:
-            continue
-
-        seen.add(identity)
-        seen_titles.append((norm_title, len(deduped)))
-        deduped.append(publication)
-    return deduped
-
-
-def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None:
-    if not body:
-        return None
-    flattened = SPACE_RE.sub(" ", HTML_TAG_RE.sub(" ", body)).strip()
-    if not flattened:
-        return None
-    if len(flattened) <= max_chars:
-        return flattened
-    return f"{flattened[:max_chars - 1]}..."

build/lib/app/services/domains/ingestion/queue.py

@@ -1,348 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from datetime import datetime, timedelta, timezone
-
-from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import IngestionQueueItem, QueueItemStatus
-
-
-@dataclass(frozen=True)
-class ContinuationQueueJob:
-    id: int
-    user_id: int
-    scholar_profile_id: int
-    resume_cstart: int
-    reason: str
-    status: str
-    attempt_count: int
-    next_attempt_dt: datetime
-
-
-ACTIVE_QUEUE_STATUSES: tuple[str, ...] = (
-    QueueItemStatus.QUEUED.value,
-    QueueItemStatus.RETRYING.value,
-)
-
-
-def normalize_cstart(value: int | None) -> int:
-    if value is None:
-        return 0
-    return max(0, int(value))
-
-
-def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_seconds: int) -> int:
-    base = max(1, int(base_seconds))
-    attempts = max(1, int(attempt_count))
-    maximum = max(base, int(max_seconds))
-    seconds = base * (2 ** max(0, attempts - 1))
-    return min(seconds, maximum)
-
-
-async def _get_item_for_user_scholar(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholar_profile_id: int,
-) -> IngestionQueueItem | None:
-    result = await db_session.execute(
-        select(IngestionQueueItem).where(
-            IngestionQueueItem.user_id == user_id,
-            IngestionQueueItem.scholar_profile_id == scholar_profile_id,
-        )
-    )
-    return result.scalar_one_or_none()
-
-
-def _build_queue_item(
-    *,
-    now: datetime,
-    user_id: int,
-    scholar_profile_id: int,
-    normalized_cstart: int,
-    reason: str,
-    run_id: int | None,
-    next_attempt_dt: datetime,
-) -> IngestionQueueItem:
-    return IngestionQueueItem(
-        user_id=user_id,
-        scholar_profile_id=scholar_profile_id,
-        resume_cstart=normalized_cstart,
-        reason=reason,
-        status=QueueItemStatus.QUEUED.value,
-        attempt_count=0,
-        next_attempt_dt=next_attempt_dt,
-        last_run_id=run_id,
-        last_error=None,
-        dropped_reason=None,
-        dropped_at=None,
-        created_at=now,
-        updated_at=now,
-    )
-
-
-def _update_existing_queue_item(
-    *,
-    item: IngestionQueueItem,
-    now: datetime,
-    normalized_cstart: int,
-    reason: str,
-    run_id: int | None,
-    next_attempt_dt: datetime,
-) -> None:
-    item.resume_cstart = normalized_cstart
-    item.reason = reason
-    if item.status == QueueItemStatus.DROPPED.value:
-        item.attempt_count = 0
-    item.status = QueueItemStatus.QUEUED.value
-    item.next_attempt_dt = next_attempt_dt
-    item.last_run_id = run_id
-    item.last_error = None
-    item.dropped_reason = None
-    item.dropped_at = None
-    item.updated_at = now
-
-
-async def upsert_job(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholar_profile_id: int,
-    resume_cstart: int,
-    reason: str,
-    run_id: int | None,
-    delay_seconds: int,
-) -> IngestionQueueItem:
-    now = datetime.now(timezone.utc)
-    next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds)))
-    item = await _get_item_for_user_scholar(
-        db_session,
-        user_id=user_id,
-        scholar_profile_id=scholar_profile_id,
-    )
-    normalized_cstart = normalize_cstart(resume_cstart)
-    if item is None:
-        item = _build_queue_item(
-            now=now,
-            user_id=user_id,
-            scholar_profile_id=scholar_profile_id,
-            normalized_cstart=normalized_cstart,
-            reason=reason,
-            run_id=run_id,
-            next_attempt_dt=next_attempt_dt,
-        )
-        db_session.add(item)
-        return item
-
-    _update_existing_queue_item(
-        item=item,
-        now=now,
-        normalized_cstart=normalized_cstart,
-        reason=reason,
-        run_id=run_id,
-        next_attempt_dt=next_attempt_dt,
-    )
-    return item
-
-
-async def clear_job_for_scholar(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholar_profile_id: int,
-) -> bool:
-    result = await db_session.execute(
-        select(IngestionQueueItem).where(
-            IngestionQueueItem.user_id == user_id,
-            IngestionQueueItem.scholar_profile_id == scholar_profile_id,
-        )
-    )
-    item = result.scalar_one_or_none()
-    if item is None:
-        return False
-    await db_session.delete(item)
-    return True
-
-
-async def delete_job_by_id(
-    db_session: AsyncSession,
-    *,
-    job_id: int,
-) -> bool:
-    result = await db_session.execute(
-        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
-    )
-    item = result.scalar_one_or_none()
-    if item is None:
-        return False
-    await db_session.delete(item)
-    return True
-
-
-async def list_due_jobs(
-    db_session: AsyncSession,
-    *,
-    now: datetime,
-    limit: int,
-) -> list[ContinuationQueueJob]:
-    result = await db_session.execute(
-        select(IngestionQueueItem)
-        .where(
-            IngestionQueueItem.next_attempt_dt <= now,
-            IngestionQueueItem.status.in_(ACTIVE_QUEUE_STATUSES),
-        )
-        .order_by(
-            IngestionQueueItem.next_attempt_dt.asc(),
-            IngestionQueueItem.id.asc(),
-        )
-        .limit(limit)
-    )
-    rows = list(result.scalars().all())
-    jobs: list[ContinuationQueueJob] = []
-    for row in rows:
-        jobs.append(
-            ContinuationQueueJob(
-                id=int(row.id),
-                user_id=int(row.user_id),
-                scholar_profile_id=int(row.scholar_profile_id),
-                resume_cstart=normalize_cstart(row.resume_cstart),
-                reason=row.reason,
-                status=row.status,
-                attempt_count=int(row.attempt_count),
-                next_attempt_dt=row.next_attempt_dt,
-            )
-        )
-    return jobs
-
-
-async def increment_attempt_count(
-    db_session: AsyncSession,
-    *,
-    job_id: int,
-) -> IngestionQueueItem | None:
-    now = datetime.now(timezone.utc)
-    result = await db_session.execute(
-        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
-    )
-    item = result.scalar_one_or_none()
-    if item is None:
-        return None
-    item.attempt_count = int(item.attempt_count or 0) + 1
-    item.updated_at = now
-    return item
-
-
-async def reset_attempt_count(
-    db_session: AsyncSession,
-    *,
-    job_id: int,
-) -> IngestionQueueItem | None:
-    now = datetime.now(timezone.utc)
-    result = await db_session.execute(
-        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
-    )
-    item = result.scalar_one_or_none()
-    if item is None:
-        return None
-    item.attempt_count = 0
-    item.updated_at = now
-    return item
-
-
-async def reschedule_job(
-    db_session: AsyncSession,
-    *,
-    job_id: int,
-    delay_seconds: int,
-    reason: str,
-    error: str | None = None,
-) -> IngestionQueueItem | None:
-    now = datetime.now(timezone.utc)
-    result = await db_session.execute(
-        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
-    )
-    item = result.scalar_one_or_none()
-    if item is None:
-        return None
-    item.next_attempt_dt = now + timedelta(seconds=max(1, int(delay_seconds)))
-    item.status = QueueItemStatus.QUEUED.value
-    item.reason = reason
-    item.last_error = error
-    item.dropped_reason = None
-    item.dropped_at = None
-    item.updated_at = now
-    return item
-
-
-async def mark_retrying(
-    db_session: AsyncSession,
-    *,
-    job_id: int,
-    reason: str | None = None,
-) -> IngestionQueueItem | None:
-    now = datetime.now(timezone.utc)
-    result = await db_session.execute(
-        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
-    )
-    item = result.scalar_one_or_none()
-    if item is None:
-        return None
-    if item.status == QueueItemStatus.DROPPED.value:
-        return item
-    item.status = QueueItemStatus.RETRYING.value
-    if reason:
-        item.reason = reason
-    item.updated_at = now
-    return item
-
-
-async def mark_dropped(
-    db_session: AsyncSession,
-    *,
-    job_id: int,
-    reason: str,
-    error: str | None = None,
-) -> IngestionQueueItem | None:
-    now = datetime.now(timezone.utc)
-    result = await db_session.execute(
-        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
-    )
-    item = result.scalar_one_or_none()
-    if item is None:
-        return None
-    item.status = QueueItemStatus.DROPPED.value
-    item.reason = "dropped"
-    item.dropped_reason = reason
-    item.dropped_at = now
-    if error is not None:
-        item.last_error = error
-    item.updated_at = now
-    return item
-
-
-async def mark_queued_now(
-    db_session: AsyncSession,
-    *,
-    job_id: int,
-    reason: str,
-    reset_attempt_count: bool = False,
-) -> IngestionQueueItem | None:
-    now = datetime.now(timezone.utc)
-    result = await db_session.execute(
-        select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
-    )
-    item = result.scalar_one_or_none()
-    if item is None:
-        return None
-    item.status = QueueItemStatus.QUEUED.value
-    item.reason = reason
-    item.next_attempt_dt = now
-    if reset_attempt_count:
-        item.attempt_count = 0
-    item.last_error = None
-    item.dropped_reason = None
-    item.dropped_at = None
-    item.updated_at = now
-    return item

build/lib/app/services/domains/ingestion/safety.py

@@ -1,282 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime, timedelta, timezone
-from typing import Any
-
-from app.db.models import UserSetting
-
-COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD = "blocked_failure_threshold_exceeded"
-COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD = "network_failure_threshold_exceeded"
-
-_COUNTER_CONSECUTIVE_BLOCKED_RUNS = "consecutive_blocked_runs"
-_COUNTER_CONSECUTIVE_NETWORK_RUNS = "consecutive_network_runs"
-_COUNTER_COOLDOWN_ENTRY_COUNT = "cooldown_entry_count"
-_COUNTER_BLOCKED_START_COUNT = "blocked_start_count"
-_COUNTER_LAST_BLOCKED_FAILURE_COUNT = "last_blocked_failure_count"
-_COUNTER_LAST_NETWORK_FAILURE_COUNT = "last_network_failure_count"
-_COUNTER_LAST_EVALUATED_RUN_ID = "last_evaluated_run_id"
-
-
-def _utcnow() -> datetime:
-    return datetime.now(timezone.utc)
-
-
-def _safe_int(value: Any, default: int = 0) -> int:
-    try:
-        return int(value)
-    except (TypeError, ValueError):
-        return default
-
-
-def _safe_optional_int(value: Any) -> int | None:
-    if value is None:
-        return None
-    try:
-        return int(value)
-    except (TypeError, ValueError):
-        return None
-
-
-def _normalize_datetime(value: datetime | None) -> datetime | None:
-    if value is None:
-        return None
-    if value.tzinfo is None:
-        return value.replace(tzinfo=timezone.utc)
-    return value.astimezone(timezone.utc)
-
-
-def _state_dict(settings: UserSetting) -> dict[str, Any]:
-    state = settings.scrape_safety_state
-    if isinstance(state, dict):
-        return state
-    return {}
-
-
-def _counters_from_state(settings: UserSetting) -> dict[str, Any]:
-    state = _state_dict(settings)
-    return {
-        _COUNTER_CONSECUTIVE_BLOCKED_RUNS: max(
-            0,
-            _safe_int(state.get(_COUNTER_CONSECUTIVE_BLOCKED_RUNS), 0),
-        ),
-        _COUNTER_CONSECUTIVE_NETWORK_RUNS: max(
-            0,
-            _safe_int(state.get(_COUNTER_CONSECUTIVE_NETWORK_RUNS), 0),
-        ),
-        _COUNTER_COOLDOWN_ENTRY_COUNT: max(
-            0,
-            _safe_int(state.get(_COUNTER_COOLDOWN_ENTRY_COUNT), 0),
-        ),
-        _COUNTER_BLOCKED_START_COUNT: max(
-            0,
-            _safe_int(state.get(_COUNTER_BLOCKED_START_COUNT), 0),
-        ),
-        _COUNTER_LAST_BLOCKED_FAILURE_COUNT: max(
-            0,
-            _safe_int(state.get(_COUNTER_LAST_BLOCKED_FAILURE_COUNT), 0),
-        ),
-        _COUNTER_LAST_NETWORK_FAILURE_COUNT: max(
-            0,
-            _safe_int(state.get(_COUNTER_LAST_NETWORK_FAILURE_COUNT), 0),
-        ),
-        _COUNTER_LAST_EVALUATED_RUN_ID: _safe_optional_int(
-            state.get(_COUNTER_LAST_EVALUATED_RUN_ID),
-        ),
-    }
-
-
-def _cooldown_reason_label(reason: str | None) -> str | None:
-    if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
-        return "Blocked responses exceeded safety threshold"
-    if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
-        return "Network failures exceeded safety threshold"
-    return None
-
-
-def _recommended_action(reason: str | None) -> str | None:
-    if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
-        return (
-            "Google Scholar appears to be blocking requests. Wait for cooldown to expire, "
-            "increase request delay, and avoid repeated manual retries."
-        )
-    if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
-        return (
-            "Network failures crossed the threshold. Verify connectivity and retry after cooldown."
-        )
-    return None
-
-
-def is_cooldown_active(
-    settings: UserSetting,
-    *,
-    now_utc: datetime | None = None,
-) -> bool:
-    now = now_utc or _utcnow()
-    cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
-    if cooldown_until is None:
-        return False
-    return cooldown_until > now
-
-
-def clear_expired_cooldown(
-    settings: UserSetting,
-    *,
-    now_utc: datetime | None = None,
-) -> bool:
-    now = now_utc or _utcnow()
-    cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
-    if cooldown_until is None:
-        return False
-    if cooldown_until > now:
-        return False
-    settings.scrape_cooldown_until = None
-    settings.scrape_cooldown_reason = None
-    return True
-
-
-def register_cooldown_blocked_start(
-    settings: UserSetting,
-    *,
-    now_utc: datetime | None = None,
-) -> dict[str, Any]:
-    now = now_utc or _utcnow()
-    counters = _counters_from_state(settings)
-    counters[_COUNTER_BLOCKED_START_COUNT] = int(counters[_COUNTER_BLOCKED_START_COUNT]) + 1
-    settings.scrape_safety_state = counters
-    return get_safety_state_payload(settings, now_utc=now)
-
-
-def _update_run_counters(
-    *,
-    counters: dict[str, Any],
-    run_id: int,
-    blocked_failure_count: int,
-    network_failure_count: int,
-) -> tuple[int, int]:
-    bounded_blocked_failures = max(0, int(blocked_failure_count))
-    bounded_network_failures = max(0, int(network_failure_count))
-    counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures
-    counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures
-    counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id)
-    counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = (
-        int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1
-        if bounded_blocked_failures > 0
-        else 0
-    )
-    counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = (
-        int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1
-        if bounded_network_failures > 0
-        else 0
-    )
-    return bounded_blocked_failures, bounded_network_failures
-
-
-def _resolve_cooldown_trigger(
-    *,
-    blocked_failures: int,
-    network_failures: int,
-    blocked_failure_threshold: int,
-    network_failure_threshold: int,
-    blocked_cooldown_seconds: int,
-    network_cooldown_seconds: int,
-) -> tuple[str | None, int]:
-    if blocked_failures >= max(1, int(blocked_failure_threshold)):
-        return COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD, max(60, int(blocked_cooldown_seconds))
-    if network_failures >= max(1, int(network_failure_threshold)):
-        return COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD, max(60, int(network_cooldown_seconds))
-    return None, 0
-
-
-def _apply_cooldown_decision(
-    *,
-    settings: UserSetting,
-    counters: dict[str, Any],
-    now: datetime,
-    reason: str | None,
-    cooldown_seconds: int,
-) -> None:
-    if reason is None:
-        clear_expired_cooldown(settings, now_utc=now)
-        return
-    settings.scrape_cooldown_reason = reason
-    settings.scrape_cooldown_until = now + timedelta(seconds=max(60, int(cooldown_seconds)))
-    counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1
-
-
-def apply_run_safety_outcome(
-    settings: UserSetting,
-    *,
-    run_id: int,
-    blocked_failure_count: int,
-    network_failure_count: int,
-    blocked_failure_threshold: int,
-    network_failure_threshold: int,
-    blocked_cooldown_seconds: int,
-    network_cooldown_seconds: int,
-    now_utc: datetime | None = None,
-) -> tuple[dict[str, Any], str | None]:
-    now = now_utc or _utcnow()
-    counters = _counters_from_state(settings)
-    blocked_failures, network_failures = _update_run_counters(
-        counters=counters,
-        run_id=run_id,
-        blocked_failure_count=blocked_failure_count,
-        network_failure_count=network_failure_count,
-    )
-    reason, cooldown_seconds = _resolve_cooldown_trigger(
-        blocked_failures=blocked_failures,
-        network_failures=network_failures,
-        blocked_failure_threshold=blocked_failure_threshold,
-        network_failure_threshold=network_failure_threshold,
-        blocked_cooldown_seconds=blocked_cooldown_seconds,
-        network_cooldown_seconds=network_cooldown_seconds,
-    )
-    _apply_cooldown_decision(
-        settings=settings,
-        counters=counters,
-        now=now,
-        reason=reason,
-        cooldown_seconds=cooldown_seconds,
-    )
-    settings.scrape_safety_state = counters
-    return get_safety_state_payload(settings, now_utc=now), reason
-
-
-def get_safety_state_payload(
-    settings: UserSetting,
-    *,
-    now_utc: datetime | None = None,
-) -> dict[str, Any]:
-    now = now_utc or _utcnow()
-    cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
-    cooldown_active = bool(cooldown_until is not None and cooldown_until > now)
-    cooldown_remaining_seconds = 0
-    if cooldown_active and cooldown_until is not None:
-        cooldown_remaining_seconds = max(0, int((cooldown_until - now).total_seconds()))
-
-    reason = settings.scrape_cooldown_reason if cooldown_active else None
-
-    return {
-        "cooldown_active": cooldown_active,
-        "cooldown_reason": reason,
-        "cooldown_reason_label": _cooldown_reason_label(reason),
-        "cooldown_until": cooldown_until,
-        "cooldown_remaining_seconds": cooldown_remaining_seconds,
-        "recommended_action": _recommended_action(reason),
-        "counters": _counters_from_state(settings),
-    }
-
-
-def get_safety_event_context(
-    settings: UserSetting,
-    *,
-    now_utc: datetime | None = None,
-) -> dict[str, Any]:
-    payload = get_safety_state_payload(settings, now_utc=now_utc)
-    return {
-        "cooldown_active": bool(payload.get("cooldown_active")),
-        "cooldown_reason": payload.get("cooldown_reason"),
-        "cooldown_until": payload.get("cooldown_until"),
-        "cooldown_remaining_seconds": int(payload.get("cooldown_remaining_seconds") or 0),
-        "safety_counters": payload.get("counters", {}),
-    }

build/lib/app/services/domains/ingestion/scheduler.py

@@ -1,622 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-from dataclasses import dataclass
-from datetime import datetime, timedelta, timezone
-import logging
-
-from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import (
-    CrawlRun,
-    QueueItemStatus,
-    RunTriggerType,
-    ScholarProfile,
-    User,
-    UserSetting,
-)
-from app.db.session import get_session_factory
-from app.services.domains.ingestion import queue as queue_service
-from app.services.domains.ingestion.application import (
-    RunAlreadyInProgressError,
-    RunBlockedBySafetyPolicyError,
-    ScholarIngestionService,
-)
-from app.services.domains.settings import application as user_settings_service
-from app.services.domains.scholar.source import LiveScholarSource
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-
-def _request_delay_floor_seconds() -> int:
-    return user_settings_service.resolve_request_delay_minimum(
-        settings.ingestion_min_request_delay_seconds
-    )
-
-
-def _effective_request_delay_seconds(value: int | None) -> int:
-    floor = _request_delay_floor_seconds()
-    try:
-        parsed = int(value) if value is not None else floor
-    except (TypeError, ValueError):
-        parsed = floor
-    return max(floor, parsed)
-
-
-@dataclass(frozen=True)
-class _AutoRunCandidate:
-    user_id: int
-    run_interval_minutes: int
-    request_delay_seconds: int
-    cooldown_until: datetime | None
-    cooldown_reason: str | None
-
-
-class SchedulerService:
-    def __init__(
-        self,
-        *,
-        enabled: bool,
-        tick_seconds: int,
-        network_error_retries: int,
-        retry_backoff_seconds: float,
-        max_pages_per_scholar: int,
-        page_size: int,
-        continuation_queue_enabled: bool,
-        continuation_base_delay_seconds: int,
-        continuation_max_delay_seconds: int,
-        continuation_max_attempts: int,
-        queue_batch_size: int,
-    ) -> None:
-        self._enabled = enabled
-        self._tick_seconds = max(5, int(tick_seconds))
-        self._network_error_retries = max(0, int(network_error_retries))
-        self._retry_backoff_seconds = max(0.0, float(retry_backoff_seconds))
-        self._max_pages_per_scholar = max(1, int(max_pages_per_scholar))
-        self._page_size = max(1, int(page_size))
-        self._continuation_queue_enabled = bool(continuation_queue_enabled)
-        self._continuation_base_delay_seconds = max(1, int(continuation_base_delay_seconds))
-        self._continuation_max_delay_seconds = max(
-            self._continuation_base_delay_seconds,
-            int(continuation_max_delay_seconds),
-        )
-        self._continuation_max_attempts = max(1, int(continuation_max_attempts))
-        self._queue_batch_size = max(1, int(queue_batch_size))
-        self._task: asyncio.Task[None] | None = None
-        self._source = LiveScholarSource()
-
-    async def start(self) -> None:
-        if not self._enabled:
-            logger.info(
-                "scheduler.disabled",
-                extra={
-                    "event": "scheduler.disabled",
-                },
-            )
-            return
-        if self._task is not None:
-            return
-        self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler")
-        logger.info(
-            "scheduler.started",
-            extra={
-                "event": "scheduler.started",
-                "tick_seconds": self._tick_seconds,
-                "network_error_retries": self._network_error_retries,
-                "retry_backoff_seconds": self._retry_backoff_seconds,
-                "max_pages_per_scholar": self._max_pages_per_scholar,
-                "page_size": self._page_size,
-                "continuation_queue_enabled": self._continuation_queue_enabled,
-                "continuation_base_delay_seconds": self._continuation_base_delay_seconds,
-                "continuation_max_delay_seconds": self._continuation_max_delay_seconds,
-                "continuation_max_attempts": self._continuation_max_attempts,
-                "queue_batch_size": self._queue_batch_size,
-            },
-        )
-
-    async def stop(self) -> None:
-        if self._task is None:
-            return
-        self._task.cancel()
-        try:
-            await self._task
-        except asyncio.CancelledError:
-            pass
-        finally:
-            self._task = None
-        logger.info("scheduler.stopped", extra={"event": "scheduler.stopped"})
-
-    async def _run_loop(self) -> None:
-        while True:
-            try:
-                await self._tick_once()
-            except asyncio.CancelledError:
-                raise
-            except Exception:
-                logger.exception(
-                    "scheduler.tick_failed",
-                    extra={
-                        "event": "scheduler.tick_failed",
-                    },
-                )
-            await asyncio.sleep(float(self._tick_seconds))
-
-    async def _tick_once(self) -> None:
-        if self._continuation_queue_enabled:
-            await self._drain_continuation_queue()
-        
-        await self._drain_pdf_queue()
-
-        candidates = await self._load_candidates()
-        if not candidates:
-            return
-        now = datetime.now(timezone.utc)
-        for candidate in candidates:
-            if not await self._is_due(candidate, now=now):
-                continue
-            await self._run_candidate(candidate)
-
-    async def _load_candidate_rows(self) -> list[tuple]:
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            result = await session.execute(
-                select(
-                    UserSetting.user_id,
-                    UserSetting.run_interval_minutes,
-                    UserSetting.request_delay_seconds,
-                    UserSetting.scrape_cooldown_until,
-                    UserSetting.scrape_cooldown_reason,
-                )
-                .join(User, User.id == UserSetting.user_id)
-                .where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True))
-                .order_by(UserSetting.user_id.asc())
-            )
-            return result.all()
-
-    @staticmethod
-    def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None:
-        user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row
-        if cooldown_until is not None and cooldown_until.tzinfo is None:
-            cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
-        if cooldown_until is not None and cooldown_until > now_utc:
-            logger.info(
-                "scheduler.run_skipped_safety_cooldown_precheck",
-                extra={
-                    "event": "scheduler.run_skipped_safety_cooldown_precheck",
-                    "user_id": int(user_id),
-                    "reason": cooldown_reason,
-                    "cooldown_until": cooldown_until,
-                    "cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()),
-                    "metric_name": "scheduler_run_skipped_safety_cooldown_total",
-                    "metric_value": 1,
-                },
-            )
-            return None
-        return _AutoRunCandidate(
-            user_id=int(user_id),
-            run_interval_minutes=int(run_interval_minutes),
-            request_delay_seconds=_effective_request_delay_seconds(request_delay_seconds),
-            cooldown_until=cooldown_until,
-            cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
-        )
-
-    async def _load_candidates(self) -> list[_AutoRunCandidate]:
-        if not settings.ingestion_automation_allowed:
-            return []
-        rows = await self._load_candidate_rows()
-        now_utc = datetime.now(timezone.utc)
-        candidates: list[_AutoRunCandidate] = []
-        for row in rows:
-            candidate = self._candidate_from_row(row, now_utc=now_utc)
-            if candidate is not None:
-                candidates.append(candidate)
-        return candidates
-
-    async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            result = await session.execute(
-                select(CrawlRun.start_dt)
-                .where(
-                    CrawlRun.user_id == candidate.user_id,
-                )
-                .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
-                .limit(1)
-            )
-            last_run = result.scalar_one_or_none()
-
-        if last_run is None:
-            return True
-
-        next_due_dt = last_run + timedelta(
-            minutes=candidate.run_interval_minutes
-        )
-        return now >= next_due_dt
-
-    async def _run_candidate_ingestion(
-        self,
-        *,
-        candidate: _AutoRunCandidate,
-    ):
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            ingestion = ScholarIngestionService(source=self._source)
-            try:
-                return await ingestion.run_for_user(
-                    session,
-                    user_id=candidate.user_id,
-                    trigger_type=RunTriggerType.SCHEDULED,
-                    request_delay_seconds=candidate.request_delay_seconds,
-                    network_error_retries=self._network_error_retries,
-                    retry_backoff_seconds=self._retry_backoff_seconds,
-                    max_pages_per_scholar=self._max_pages_per_scholar,
-                    page_size=self._page_size,
-                    auto_queue_continuations=self._continuation_queue_enabled,
-                    queue_delay_seconds=self._continuation_base_delay_seconds,
-                    alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
-                    alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
-                    alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
-                )
-            except RunAlreadyInProgressError:
-                await session.rollback()
-                logger.info("scheduler.run_skipped_locked", extra={"event": "scheduler.run_skipped_locked", "user_id": candidate.user_id})
-                return None
-            except RunBlockedBySafetyPolicyError as exc:
-                await session.rollback()
-                logger.info(
-                    "scheduler.run_skipped_safety_cooldown",
-                    extra={
-                        "event": "scheduler.run_skipped_safety_cooldown",
-                        "user_id": candidate.user_id,
-                        "reason": exc.safety_state.get("cooldown_reason"),
-                        "cooldown_until": exc.safety_state.get("cooldown_until"),
-                        "cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"),
-                        "metric_name": "scheduler_run_skipped_safety_cooldown_total",
-                        "metric_value": 1,
-                    },
-                )
-                return None
-            except Exception:
-                await session.rollback()
-                logger.exception("scheduler.run_failed", extra={"event": "scheduler.run_failed", "user_id": candidate.user_id})
-                return None
-
-    async def _run_candidate(self, candidate: _AutoRunCandidate) -> None:
-        run_summary = await self._run_candidate_ingestion(candidate=candidate)
-        if run_summary is None:
-            return
-        logger.info(
-            "scheduler.run_completed",
-            extra={
-                "event": "scheduler.run_completed",
-                "user_id": candidate.user_id,
-                "run_id": run_summary.crawl_run_id,
-                "status": run_summary.status.value,
-                "scholar_count": run_summary.scholar_count,
-                "new_publication_count": run_summary.new_publication_count,
-            },
-        )
-
-    async def _drain_continuation_queue(self) -> None:
-        now = datetime.now(timezone.utc)
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            jobs = await queue_service.list_due_jobs(
-                session,
-                now=now,
-                limit=self._queue_batch_size,
-            )
-        for job in jobs:
-            await self._run_queue_job(job)
-
-    async def _drop_queue_job_if_max_attempts(
-        self,
-        job: queue_service.ContinuationQueueJob,
-    ) -> bool:
-        if job.attempt_count < self._continuation_max_attempts:
-            return False
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            dropped = await queue_service.mark_dropped(
-                session,
-                job_id=job.id,
-                reason="max_attempts_reached",
-            )
-            await session.commit()
-        if dropped is not None:
-            logger.warning(
-                "scheduler.queue_item_dropped_max_attempts",
-                extra={
-                    "event": "scheduler.queue_item_dropped_max_attempts",
-                    "queue_item_id": job.id,
-                    "user_id": job.user_id,
-                    "scholar_profile_id": job.scholar_profile_id,
-                    "attempt_count": job.attempt_count,
-                    "max_attempts": self._continuation_max_attempts,
-                },
-            )
-        return True
-
-    async def _mark_queue_job_retrying(
-        self,
-        job: queue_service.ContinuationQueueJob,
-    ) -> bool:
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            queue_item = await queue_service.mark_retrying(session, job_id=job.id)
-            await session.commit()
-        if queue_item is None:
-            return False
-        if queue_item.status == QueueItemStatus.DROPPED.value:
-            return False
-        return True
-
-    async def _queue_job_has_available_scholar(
-        self,
-        job: queue_service.ContinuationQueueJob,
-    ) -> bool:
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            scholar_result = await session.execute(
-                select(ScholarProfile.id).where(
-                    ScholarProfile.user_id == job.user_id,
-                    ScholarProfile.id == job.scholar_profile_id,
-                    ScholarProfile.is_enabled.is_(True),
-                )
-            )
-            scholar_id = scholar_result.scalar_one_or_none()
-            if scholar_id is not None:
-                return True
-            dropped = await queue_service.mark_dropped(
-                session,
-                job_id=job.id,
-                reason="scholar_unavailable",
-            )
-            await session.commit()
-        if dropped is not None:
-            logger.info(
-                "scheduler.queue_item_dropped_scholar_unavailable",
-                extra={
-                    "event": "scheduler.queue_item_dropped_scholar_unavailable",
-                    "queue_item_id": job.id,
-                    "user_id": job.user_id,
-                    "scholar_profile_id": job.scholar_profile_id,
-                },
-            )
-        return False
-
-    async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
-        session_factory = get_session_factory()
-        async with session_factory() as recovery_session:
-            await queue_service.reschedule_job(
-                recovery_session,
-                job_id=job.id,
-                delay_seconds=max(self._tick_seconds, 15),
-                reason="user_run_lock_active",
-                error="run_already_in_progress",
-            )
-            await recovery_session.commit()
-        logger.info(
-            "scheduler.queue_item_deferred_lock",
-            extra={"event": "scheduler.queue_item_deferred_lock", "queue_item_id": job.id, "user_id": job.user_id},
-        )
-
-    async def _reschedule_queue_job_safety_cooldown(
-        self,
-        job: queue_service.ContinuationQueueJob,
-        exc: RunBlockedBySafetyPolicyError,
-    ) -> None:
-        cooldown_remaining_seconds = max(
-            self._tick_seconds,
-            int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
-        )
-        session_factory = get_session_factory()
-        async with session_factory() as recovery_session:
-            await queue_service.reschedule_job(
-                recovery_session,
-                job_id=job.id,
-                delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds),
-                reason="scrape_safety_cooldown",
-                error=str(exc.message),
-            )
-            await recovery_session.commit()
-        logger.info(
-            "scheduler.queue_item_deferred_safety_cooldown",
-            extra={
-                "event": "scheduler.queue_item_deferred_safety_cooldown",
-                "queue_item_id": job.id,
-                "user_id": job.user_id,
-                "reason": exc.safety_state.get("cooldown_reason"),
-                "cooldown_remaining_seconds": cooldown_remaining_seconds,
-                "metric_name": "scheduler_queue_item_deferred_safety_cooldown_total",
-                "metric_value": 1,
-            },
-        )
-
-    async def _reschedule_queue_job_after_exception(
-        self,
-        job: queue_service.ContinuationQueueJob,
-        *,
-        exc: Exception,
-    ) -> None:
-        session_factory = get_session_factory()
-        async with session_factory() as recovery_session:
-            queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
-            if queue_item is None:
-                await recovery_session.commit()
-                return
-            if int(queue_item.attempt_count) >= self._continuation_max_attempts:
-                await queue_service.mark_dropped(
-                    recovery_session,
-                    job_id=job.id,
-                    reason="scheduler_exception_max_attempts",
-                    error=str(exc),
-                )
-                await recovery_session.commit()
-                logger.warning(
-                    "scheduler.queue_item_dropped_after_exception",
-                    extra={"event": "scheduler.queue_item_dropped_after_exception", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count},
-                )
-                return
-            delay_seconds = queue_service.compute_backoff_seconds(
-                base_seconds=self._continuation_base_delay_seconds,
-                attempt_count=int(queue_item.attempt_count),
-                max_seconds=self._continuation_max_delay_seconds,
-            )
-            await queue_service.reschedule_job(
-                recovery_session,
-                job_id=job.id,
-                delay_seconds=delay_seconds,
-                reason="scheduler_exception",
-                error=str(exc),
-            )
-            await recovery_session.commit()
-        logger.exception("scheduler.queue_item_run_failed", extra={"event": "scheduler.queue_item_run_failed", "queue_item_id": job.id, "user_id": job.user_id})
-
-    async def _run_ingestion_for_queue_job(
-        self,
-        job: queue_service.ContinuationQueueJob,
-    ):
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id)
-            ingestion = ScholarIngestionService(source=self._source)
-            try:
-                return await ingestion.run_for_user(
-                    session,
-                    user_id=job.user_id,
-                    trigger_type=RunTriggerType.SCHEDULED,
-                    request_delay_seconds=request_delay_seconds,
-                    network_error_retries=self._network_error_retries,
-                    retry_backoff_seconds=self._retry_backoff_seconds,
-                    rate_limit_retries=settings.ingestion_rate_limit_retries,
-                    rate_limit_backoff_seconds=settings.ingestion_rate_limit_backoff_seconds,
-                    max_pages_per_scholar=self._max_pages_per_scholar,
-                    page_size=self._page_size,
-                    scholar_profile_ids={job.scholar_profile_id},
-                    start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart},
-                    auto_queue_continuations=self._continuation_queue_enabled,
-                    queue_delay_seconds=self._continuation_base_delay_seconds,
-                    alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
-                    alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
-                    alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
-                )
-            except RunAlreadyInProgressError:
-                await session.rollback()
-                await self._reschedule_queue_job_lock_active(job)
-            except RunBlockedBySafetyPolicyError as exc:
-                await session.rollback()
-                await self._reschedule_queue_job_safety_cooldown(job, exc)
-            except Exception as exc:
-                await session.rollback()
-                await self._reschedule_queue_job_after_exception(job, exc=exc)
-        return None
-
-    @staticmethod
-    def _log_queue_item_resolved(
-        *,
-        event_name: str,
-        job: queue_service.ContinuationQueueJob,
-        run_summary,
-        attempt_count: int | None = None,
-        delay_seconds: int | None = None,
-    ) -> None:
-        payload = {
-            "event": event_name,
-            "queue_item_id": job.id,
-            "user_id": job.user_id,
-            "run_id": run_summary.crawl_run_id,
-            "status": run_summary.status.value,
-        }
-        if attempt_count is not None:
-            payload["attempt_count"] = attempt_count
-        if delay_seconds is not None:
-            payload["delay_seconds"] = delay_seconds
-        logger.info(event_name, extra=payload)
-
-    async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            if int(run_summary.failed_count) <= 0:
-                queue_item = await queue_service.reset_attempt_count(session, job_id=job.id)
-                await session.commit()
-                if queue_item is None:
-                    self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
-                    return
-                self._log_queue_item_resolved(
-                    event_name="scheduler.queue_item_progressed",
-                    job=job,
-                    run_summary=run_summary,
-                    attempt_count=int(queue_item.attempt_count),
-                )
-                return
-            queue_item = await queue_service.increment_attempt_count(session, job_id=job.id)
-            if queue_item is None:
-                await session.commit()
-                self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
-                return
-            if int(queue_item.attempt_count) >= self._continuation_max_attempts:
-                await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
-                await session.commit()
-                logger.warning(
-                    "scheduler.queue_item_dropped_max_attempts_after_run",
-                    extra={"event": "scheduler.queue_item_dropped_max_attempts_after_run", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count, "run_id": run_summary.crawl_run_id, "status": run_summary.status.value},
-                )
-                return
-            delay_seconds = queue_service.compute_backoff_seconds(base_seconds=self._continuation_base_delay_seconds, attempt_count=int(queue_item.attempt_count), max_seconds=self._continuation_max_delay_seconds)
-            await queue_service.reschedule_job(session, job_id=job.id, delay_seconds=delay_seconds, reason=queue_item.reason, error=queue_item.last_error)
-            await session.commit()
-        self._log_queue_item_resolved(
-            event_name="scheduler.queue_item_rescheduled_failed",
-            job=job,
-            run_summary=run_summary,
-            attempt_count=int(queue_item.attempt_count),
-            delay_seconds=delay_seconds,
-        )
-
-    async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
-        if await self._drop_queue_job_if_max_attempts(job):
-            return
-        if not await self._mark_queue_job_retrying(job):
-            return
-        if not await self._queue_job_has_available_scholar(job):
-            return
-        run_summary = await self._run_ingestion_for_queue_job(job)
-        if run_summary is None:
-            return
-        await self._finalize_queue_job_after_run(job, run_summary)
-
-    async def _drain_pdf_queue(self) -> None:
-        from app.services.domains.publications.pdf_queue import drain_ready_jobs
-        
-        session_factory = get_session_factory()
-        async with session_factory() as session:
-            try:
-                processed = await drain_ready_jobs(
-                    session,
-                    limit=settings.scheduler_pdf_queue_batch_size,
-                    max_attempts=settings.pdf_auto_retry_max_attempts,
-                )
-                if processed > 0:
-                    logger.info("scheduler.pdf_queue_drain_completed", extra={
-                        "event": "scheduler.pdf_queue_drain_completed",
-                        "processed_count": processed,
-                    })
-            except Exception:
-                logger.exception("scheduler.pdf_queue_drain_failed", extra={
-                    "event": "scheduler.pdf_queue_drain_failed",
-                })
-
-    async def _load_request_delay_for_user(
-        self,
-        db_session: AsyncSession,
-        *,
-        user_id: int,
-    ) -> int:
-        result = await db_session.execute(
-            select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id)
-        )
-        delay = result.scalar_one_or_none()
-        return _effective_request_delay_seconds(delay)

build/lib/app/services/domains/ingestion/types.py

@@ -1,110 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import Any
-
-from app.db.models import RunStatus
-from app.services.domains.scholar.parser import ParsedProfilePage, PublicationCandidate
-from app.services.domains.scholar.source import FetchResult
-
-
-@dataclass(frozen=True)
-class RunExecutionSummary:
-    crawl_run_id: int
-    status: RunStatus
-    scholar_count: int
-    succeeded_count: int
-    failed_count: int
-    partial_count: int
-    new_publication_count: int
-
-
-@dataclass(frozen=True)
-class PagedParseResult:
-    fetch_result: FetchResult
-    parsed_page: ParsedProfilePage
-    first_page_fetch_result: FetchResult
-    first_page_parsed_page: ParsedProfilePage
-    first_page_fingerprint_sha256: str | None
-    publications: list[PublicationCandidate]
-    attempt_log: list[dict[str, Any]]
-    page_logs: list[dict[str, Any]]
-    pages_fetched: int
-    pages_attempted: int
-    has_more_remaining: bool
-    pagination_truncated_reason: str | None
-    continuation_cstart: int | None
-    skipped_no_change: bool
-    discovered_publication_count: int
-
-
-@dataclass
-class RunProgress:
-    succeeded_count: int = 0
-    failed_count: int = 0
-    partial_count: int = 0
-    scholar_results: list[dict[str, Any]] = field(default_factory=list)
-
-
-@dataclass(frozen=True)
-class ScholarProcessingOutcome:
-    result_entry: dict[str, Any]
-    succeeded_count_delta: int
-    failed_count_delta: int
-    partial_count_delta: int
-    discovered_publication_count: int
-
-
-@dataclass(frozen=True)
-class RunFailureSummary:
-    failed_state_counts: dict[str, int]
-    failed_reason_counts: dict[str, int]
-    scrape_failure_counts: dict[str, int]
-    retries_scheduled_count: int
-    scholars_with_retries_count: int
-    retry_exhausted_count: int
-
-
-@dataclass(frozen=True)
-class RunAlertSummary:
-    blocked_failure_count: int
-    network_failure_count: int
-    blocked_failure_threshold: int
-    network_failure_threshold: int
-    retry_scheduled_threshold: int
-    alert_flags: dict[str, bool]
-
-
-@dataclass
-class PagedLoopState:
-    fetch_result: FetchResult
-    parsed_page: ParsedProfilePage
-    attempt_log: list[dict[str, Any]]
-    page_logs: list[dict[str, Any]]
-    publications: list[PublicationCandidate]
-    pages_fetched: int
-    pages_attempted: int
-    current_cstart: int
-    next_cstart: int
-    has_more_remaining: bool = False
-    pagination_truncated_reason: str | None = None
-    continuation_cstart: int | None = None
-    discovered_publication_count: int = 0
-
-
-class RunAlreadyInProgressError(RuntimeError):
-    """Raised when a run lock for a user is already held by another process."""
-
-
-class RunBlockedBySafetyPolicyError(RuntimeError):
-    def __init__(
-        self,
-        *,
-        code: str,
-        message: str,
-        safety_state: dict[str, Any],
-    ) -> None:
-        super().__init__(message)
-        self.code = code
-        self.message = message
-        self.safety_state = safety_state

build/lib/app/services/domains/openalex/__init__.py

@@ -1,5 +0,0 @@
-from __future__ import annotations
-
-import logging
-
-logger = logging.getLogger(__name__)

build/lib/app/services/domains/openalex/client.py

@@ -1,135 +0,0 @@
-import asyncio
-import logging
-from typing import Any, Mapping
-
-import httpx
-from tenacity import (
-    retry,
-    retry_if_exception_type,
-    stop_after_attempt,
-    wait_exponential,
-)
-
-from app.services.domains.openalex.types import OpenAlexWork
-
-logger = logging.getLogger(__name__)
-
-OPENALEX_BASE_URL = "https://api.openalex.org"
-
-
-class OpenAlexClientError(Exception):
-    pass
-
-
-class OpenAlexRateLimitError(OpenAlexClientError):
-    pass
-
-
-class OpenAlexClient:
-    def __init__(
-        self,
-        api_key: str | None = None,
-        mailto: str | None = None,
-        timeout: float = 10.0,
-    ) -> None:
-        self.api_key = api_key
-        self.mailto = mailto
-        self.timeout = timeout
-
-    @property
-    def _base_params(self) -> dict[str, str]:
-        params = {}
-        if self.mailto:
-            params["mailto"] = self.mailto
-        if self.api_key:
-            params["api_key"] = self.api_key
-        return params
-
-    @retry(
-        retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException, OpenAlexRateLimitError)),
-        stop=stop_after_attempt(3),
-        wait=wait_exponential(multiplier=1, min=2, max=10),
-        reraise=True,
-    )
-    async def get_work_by_doi(self, doi: str) -> OpenAlexWork | None:
-        """Fetch a single work by DOI directly."""
-        clean_doi = doi.replace("https://doi.org/", "")
-        if not clean_doi:
-            return None
-
-        url = f"{OPENALEX_BASE_URL}/works/{clean_doi}"
-        
-        headers = {}
-        if self.mailto:
-            headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
-        else:
-            headers["User-Agent"] = "scholar-scraper/1.0"
-
-        async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client:
-            response = await client.get(url, params=self._base_params)
-
-        if response.status_code == 404:
-            return None
-        if response.status_code == 429:
-            raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex work by DOI")
-        if response.status_code >= 400:
-            logger.warning("OpenAlex API error: %s %s", response.status_code, response.text)
-            raise OpenAlexClientError(f"API Error {response.status_code}")
-
-        data = response.json()
-        return OpenAlexWork.from_api_dict(data)
-
-    @retry(
-        retry=retry_if_exception_type((httpx.NetworkError, httpx.TimeoutException, OpenAlexRateLimitError)),
-        stop=stop_after_attempt(3),
-        wait=wait_exponential(multiplier=1, min=2, max=10),
-        reraise=True,
-    )
-    async def get_works_by_filter(
-        self,
-        filters: dict[str, str],
-        limit: int = 50,
-    ) -> list[OpenAlexWork]:
-        """
-        Fetch works using the ?filter= query parameter.
-        Supports fetching multiple records by joining filters with | (OR logic).
-        """
-        if not filters:
-            return []
-
-        # Example: {"doi": "10.foo|10.bar", "title.search": "query"}
-        filter_str = ",".join(f"{k}:{v}" for k, v in filters.items())
-        
-        params = self._base_params.copy()
-        params["filter"] = filter_str
-        params["per-page"] = str(limit)
-
-        url = f"{OPENALEX_BASE_URL}/works"
-        
-        headers = {}
-        if self.mailto:
-            headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
-        else:
-            headers["User-Agent"] = "scholar-scraper/1.0"
-
-        async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True, headers=headers) as client:
-            response = await client.get(url, params=params)
-
-        if response.status_code == 429:
-            raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex works list")
-        if response.status_code >= 400:
-            logger.warning("OpenAlex API error (filters=%s): %s %s", filters, response.status_code, response.text)
-            raise OpenAlexClientError(f"API Error {response.status_code}")
-
-        data = response.json()
-        results = data.get("results") or []
-        
-        parsed_works = []
-        for raw_work in results:
-            try:
-                parsed_works.append(OpenAlexWork.from_api_dict(raw_work))
-            except Exception as e:
-                logger.warning("Failed to parse OpenAlex raw dict: %s", e)
-                continue
-
-        return parsed_works

build/lib/app/services/domains/openalex/matching.py

@@ -1,108 +0,0 @@
-import logging
-import re
-
-from rapidfuzz import fuzz
-
-from app.services.domains.openalex.types import OpenAlexWork
-
-logger = logging.getLogger(__name__)
-
-# A minimum similarity score out of 100 for a title to be considered a match candidate.
-TITLE_MATCH_THRESHOLD = 90.0
-# The margin within the top score where a secondary tiebreaker (author/year) is necessary.
-TIEBREAKER_MARGIN = 5.0
-
-
-def _clean_string(s: str | None) -> str:
-    if not s:
-        return ""
-    # Strip non-alphanumeric (keep spaces), lowercase, and collapse whitespace
-    cleaned = re.sub(r"[^a-z0-9\s]", " ", s.lower())
-    return " ".join(cleaned.split())
-
-
-def _author_overlap_score(target_authors: str | None, candidate_authors: list[str]) -> bool:
-    if not target_authors or not candidate_authors:
-        return False
-    
-    target_clean = _clean_string(target_authors)
-    if not target_clean:
-        return False
-        
-    for candidate in candidate_authors:
-        cand_clean = _clean_string(candidate)
-        if cand_clean and (cand_clean in target_clean or target_clean in cand_clean):
-            return True
-        # Alternatively check rapidfuzz token_set_ratio
-        if cand_clean and fuzz.token_set_ratio(target_clean, cand_clean) > 80:
-            return True
-            
-    return False
-
-
-def find_best_match(
-    target_title: str,
-    target_year: int | None,
-    target_authors: str | None,
-    candidates: list[OpenAlexWork],
-) -> OpenAlexWork | None:
-    """
-    Finds the best matching OpenAlexWork from a list of candidates, prioritizing title similarity (>90%)
-    with year and author overlap as tiebreakers for close candidates.
-    """
-    if not target_title or not candidates:
-        return None
-
-    clean_target = _clean_string(target_title)
-    if not clean_target:
-        return None
-
-    scored_candidates: list[tuple[float, OpenAlexWork]] = []
-
-    for cand in candidates:
-        if not cand.title:
-            continue
-            
-        clean_cand = _clean_string(cand.title)
-        
-        # Primary sort: string similarity ratio
-        score = fuzz.ratio(clean_target, clean_cand)
-        
-        if score >= TITLE_MATCH_THRESHOLD:
-            scored_candidates.append((score, cand))
-
-    if not scored_candidates:
-        return None
-
-    # Sort descending by score
-    scored_candidates.sort(key=lambda x: x[0], reverse=True)
-    
-    best_score = scored_candidates[0][0]
-    
-    # Extract all candidates within the tiebreaker margin
-    top_scored_candidates = [
-        (score, cand) for score, cand in scored_candidates 
-        if best_score - score <= TIEBREAKER_MARGIN
-    ]
-
-    if len(top_scored_candidates) == 1:
-        return top_scored_candidates[0][1]
-
-    # We have a tie or near-tie. Use year and author overlap to break the tie.
-    # Score candidates: +1 for year match (within 1 year), +1 for author overlap
-    tiebreaker_scores: list[tuple[int, float, OpenAlexWork]] = []
-    
-    for original_score, cand in top_scored_candidates:
-        tb_score = 0
-        if target_year is not None and cand.publication_year is not None:
-            if abs(target_year - cand.publication_year) <= 1:
-                tb_score += 1
-                
-        candidate_author_names = [a.display_name for a in cand.authors if a.display_name]
-        if _author_overlap_score(target_authors, candidate_author_names):
-            tb_score += 1
-            
-        tiebreaker_scores.append((tb_score, original_score, cand))
-        
-    tiebreaker_scores.sort(key=lambda x: (x[0], x[1]), reverse=True)
-    return tiebreaker_scores[0][2]

build/lib/app/services/domains/openalex/types.py

@@ -1,71 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from datetime import datetime
-from typing import Any, Mapping
-
-
-@dataclass(frozen=True)
-class OpenAlexAuthor:
-    openalex_id: str | None
-    display_name: str | None
-
-
-@dataclass(frozen=True)
-class OpenAlexWork:
-    openalex_id: str
-    doi: str | None
-    pmid: str | None
-    pmcid: str | None
-    title: str | None
-    publication_year: int | None
-    cited_by_count: int
-    is_oa: bool
-    oa_url: str | None
-    authors: list[OpenAlexAuthor] = field(default_factory=list)
-    raw_data: Mapping[str, Any] = field(default_factory=dict, repr=False)
-
-    @classmethod
-    def from_api_dict(cls, data: Mapping[str, Any]) -> OpenAlexWork:
-        ids = data.get("ids") or {}
-        
-        # Extract DOI without the https://doi.org/ prefix
-        doi = ids.get("doi")
-        if doi and doi.startswith("https://doi.org/"):
-            doi = doi[16:]
-            
-        # Extract PMID without the url prefix
-        pmid = ids.get("pmid")
-        if pmid and pmid.startswith("https://pubmed.ncbi.nlm.nih.gov/"):
-            pmid = pmid[32:]
-            
-        # Extract PMCID without the url prefix
-        pmcid = ids.get("pmcid")
-        if pmcid and pmcid.startswith("https://www.ncbi.nlm.nih.gov/pmc/articles/"):
-            pmcid = pmcid[42:]
-
-        open_access = data.get("open_access") or {}
-        
-        authors = []
-        for authorship in data.get("authorships") or []:
-            author_data = authorship.get("author") or {}
-            authors.append(
-                OpenAlexAuthor(
-                    openalex_id=author_data.get("id"),
-                    display_name=author_data.get("display_name"),
-                )
-            )
-
-        return cls(
-            openalex_id=data.get("id", ""),
-            doi=doi,
-            pmid=pmid,
-            pmcid=pmcid,
-            title=data.get("title"),
-            publication_year=data.get("publication_year"),
-            cited_by_count=data.get("cited_by_count", 0),
-            is_oa=bool(open_access.get("is_oa")),
-            oa_url=open_access.get("oa_url"),
-            authors=authors,
-            raw_data=dict(data),
-        )

build/lib/app/services/domains/portability/__init__.py

@@ -1 +0,0 @@
-from app.services.domains.portability.application import *

build/lib/app/services/domains/portability/application.py

@@ -1,67 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import Publication
-from app.services.domains.portability.constants import (
-    EXPORT_SCHEMA_VERSION,
-    MAX_IMPORT_PUBLICATIONS,
-    MAX_IMPORT_SCHOLARS,
-)
-from app.services.domains.portability.exporting import export_user_data
-from app.services.domains.portability.normalize import _validate_import_sizes
-from app.services.domains.portability.publication_import import (
-    _build_imported_publication_input,
-    _initialize_import_counters,
-    _upsert_imported_publication,
-)
-from app.services.domains.portability.scholar_import import _upsert_imported_scholars
-from app.services.domains.portability.types import ImportExportError, ImportedPublicationInput
-
-
-async def import_user_data(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholars: list[dict[str, Any]],
-    publications: list[dict[str, Any]],
-) -> dict[str, int]:
-    _validate_import_sizes(scholars=scholars, publications=publications)
-    scholar_map, counters = await _upsert_imported_scholars(
-        db_session,
-        user_id=user_id,
-        scholars=scholars,
-    )
-    cluster_cache: dict[str, Publication | None] = {}
-    fingerprint_cache: dict[str, Publication | None] = {}
-    _initialize_import_counters(counters)
-    for item in publications:
-        parsed_item = _build_imported_publication_input(
-            item=item,
-            scholar_map=scholar_map,
-        )
-        if parsed_item is None:
-            counters["skipped_records"] += 1
-            continue
-        await _upsert_imported_publication(
-            db_session,
-            payload=parsed_item,
-            cluster_cache=cluster_cache,
-            fingerprint_cache=fingerprint_cache,
-            counters=counters,
-        )
-    await db_session.commit()
-    return counters
-
-
-__all__ = [
-    "EXPORT_SCHEMA_VERSION",
-    "MAX_IMPORT_SCHOLARS",
-    "MAX_IMPORT_PUBLICATIONS",
-    "ImportExportError",
-    "ImportedPublicationInput",
-    "export_user_data",
-    "import_user_data",
-]

build/lib/app/services/domains/portability/constants.py

@@ -1,9 +0,0 @@
-from __future__ import annotations
-
-import re
-
-EXPORT_SCHEMA_VERSION = 1
-MAX_IMPORT_SCHOLARS = 10_000
-MAX_IMPORT_PUBLICATIONS = 100_000
-WORD_RE = re.compile(r"[a-z0-9]+")
-SHA256_RE = re.compile(r"^[0-9a-f]{64}$")

build/lib/app/services/domains/portability/exporting.py

@@ -1,91 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime, timezone
-from typing import Any
-
-from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import Publication, ScholarProfile, ScholarPublication
-from app.services.domains.portability.constants import EXPORT_SCHEMA_VERSION
-
-
-def _exported_at_iso() -> str:
-    return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
-
-
-def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]:
-    return {
-        "scholar_id": profile.scholar_id,
-        "display_name": profile.display_name,
-        "is_enabled": bool(profile.is_enabled),
-        "profile_image_override_url": profile.profile_image_override_url,
-    }
-
-
-def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]:
-    (
-        scholar_id,
-        cluster_id,
-        fingerprint_sha256,
-        title_raw,
-        year,
-        citation_count,
-        author_text,
-        venue_text,
-        pub_url,
-        pdf_url,
-        is_read,
-    ) = row
-    return {
-        "scholar_id": scholar_id,
-        "cluster_id": cluster_id,
-        "fingerprint_sha256": fingerprint_sha256,
-        "title": title_raw,
-        "year": year,
-        "citation_count": int(citation_count or 0),
-        "author_text": author_text,
-        "venue_text": venue_text,
-        "pub_url": pub_url,
-        "pdf_url": pdf_url,
-        "is_read": bool(is_read),
-    }
-
-
-async def export_user_data(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-) -> dict[str, Any]:
-    scholars_result = await db_session.execute(
-        select(ScholarProfile)
-        .where(ScholarProfile.user_id == user_id)
-        .order_by(ScholarProfile.id.asc())
-    )
-    publication_result = await db_session.execute(
-        select(
-            ScholarProfile.scholar_id,
-            Publication.cluster_id,
-            Publication.fingerprint_sha256,
-            Publication.title_raw,
-            Publication.year,
-            Publication.citation_count,
-            Publication.author_text,
-            Publication.venue_text,
-            Publication.pub_url,
-            Publication.pdf_url,
-            ScholarPublication.is_read,
-        )
-        .join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id)
-        .join(Publication, Publication.id == ScholarPublication.publication_id)
-        .where(ScholarProfile.user_id == user_id)
-        .order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
-    )
-    scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()]
-    publications = [_serialize_export_publication(row) for row in publication_result.all()]
-    return {
-        "schema_version": EXPORT_SCHEMA_VERSION,
-        "exported_at": _exported_at_iso(),
-        "scholars": scholars,
-        "publications": publications,
-    }

build/lib/app/services/domains/portability/normalize.py

@@ -1,109 +0,0 @@
-from __future__ import annotations
-
-import hashlib
-from typing import Any
-
-from app.services.domains.ingestion.application import normalize_title
-from app.services.domains.portability.constants import (
-    MAX_IMPORT_PUBLICATIONS,
-    MAX_IMPORT_SCHOLARS,
-    SHA256_RE,
-    WORD_RE,
-)
-from app.services.domains.portability.types import ImportExportError
-
-
-def _normalize_optional_text(value: Any) -> str | None:
-    if value is None:
-        return None
-    normalized = str(value).strip()
-    return normalized or None
-
-
-def _normalize_optional_year(value: Any) -> int | None:
-    if value is None:
-        return None
-    try:
-        year = int(value)
-    except (TypeError, ValueError):
-        return None
-    if year < 1500 or year > 3000:
-        return None
-    return year
-
-
-def _normalize_citation_count(value: Any) -> int:
-    try:
-        parsed = int(value)
-    except (TypeError, ValueError):
-        return 0
-    return max(0, parsed)
-
-
-def _first_author_last_name(authors_text: str | None) -> str:
-    if not authors_text:
-        return ""
-    first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
-    words = WORD_RE.findall(first_author)
-    if not words:
-        return ""
-    return words[-1]
-
-
-def _first_venue_word(venue_text: str | None) -> str:
-    if not venue_text:
-        return ""
-    words = WORD_RE.findall(venue_text.lower())
-    if not words:
-        return ""
-    return words[0]
-
-
-def _build_fingerprint(
-    *,
-    title: str,
-    year: int | None,
-    author_text: str | None,
-    venue_text: str | None,
-) -> str:
-    canonical = "|".join(
-        [
-            normalize_title(title),
-            str(year) if year is not None else "",
-            _first_author_last_name(author_text),
-            _first_venue_word(venue_text),
-        ]
-    )
-    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
-
-
-def _resolve_fingerprint(
-    *,
-    title: str,
-    year: int | None,
-    author_text: str | None,
-    venue_text: str | None,
-    provided_fingerprint: Any,
-) -> str:
-    normalized = _normalize_optional_text(provided_fingerprint)
-    if normalized and SHA256_RE.fullmatch(normalized.lower()):
-        return normalized.lower()
-    return _build_fingerprint(
-        title=title,
-        year=year,
-        author_text=author_text,
-        venue_text=venue_text,
-    )
-
-
-def _validate_import_sizes(
-    *,
-    scholars: list[dict[str, Any]],
-    publications: list[dict[str, Any]],
-) -> None:
-    if len(scholars) > MAX_IMPORT_SCHOLARS:
-        raise ImportExportError(f"Import exceeds max scholars ({MAX_IMPORT_SCHOLARS}).")
-    if len(publications) > MAX_IMPORT_PUBLICATIONS:
-        raise ImportExportError(
-            f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS})."
-        )

build/lib/app/services/domains/portability/publication_import.py

@@ -1,370 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import Publication, ScholarProfile, ScholarPublication
-from app.services.domains.ingestion.application import build_publication_url, normalize_title
-from app.services.domains.doi.normalize import normalize_doi
-from app.services.domains.portability.normalize import (
-    _normalize_citation_count,
-    _normalize_optional_text,
-    _normalize_optional_year,
-    _resolve_fingerprint,
-)
-from app.services.domains.portability.types import ImportedPublicationInput
-
-
-async def _find_publication_by_cluster(
-    db_session: AsyncSession,
-    *,
-    cluster_id: str,
-) -> Publication | None:
-    result = await db_session.execute(
-        select(Publication).where(Publication.cluster_id == cluster_id)
-    )
-    return result.scalar_one_or_none()
-
-
-async def _find_publication_by_fingerprint(
-    db_session: AsyncSession,
-    *,
-    fingerprint_sha256: str,
-) -> Publication | None:
-    result = await db_session.execute(
-        select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256)
-    )
-    return result.scalar_one_or_none()
-
-
-async def _find_linked_publication_by_title(
-    db_session: AsyncSession,
-    *,
-    scholar_profile_id: int,
-    title: str,
-) -> Publication | None:
-    normalized_title = normalize_title(title)
-    result = await db_session.execute(
-        select(Publication)
-        .join(
-            ScholarPublication,
-            ScholarPublication.publication_id == Publication.id,
-        )
-        .where(
-            ScholarPublication.scholar_profile_id == scholar_profile_id,
-            Publication.title_normalized == normalized_title,
-        )
-        .order_by(Publication.id.asc())
-        .limit(1)
-    )
-    return result.scalar_one_or_none()
-
-
-def _apply_imported_publication_values(
-    *,
-    publication: Publication,
-    title: str,
-    year: int | None,
-    citation_count: int,
-    author_text: str | None,
-    venue_text: str | None,
-    pub_url: str | None,
-    pdf_url: str | None,
-    cluster_id: str | None,
-) -> bool:
-    updated = False
-    if cluster_id and publication.cluster_id != cluster_id:
-        publication.cluster_id = cluster_id
-        updated = True
-    if publication.title_raw != title:
-        publication.title_raw = title
-        publication.title_normalized = normalize_title(title)
-        updated = True
-    if publication.year != year:
-        publication.year = year
-        updated = True
-    if int(publication.citation_count or 0) != citation_count:
-        publication.citation_count = citation_count
-        updated = True
-    if publication.author_text != author_text:
-        publication.author_text = author_text
-        updated = True
-    if publication.venue_text != venue_text:
-        publication.venue_text = venue_text
-        updated = True
-    if pub_url and publication.pub_url != pub_url:
-        publication.pub_url = pub_url
-        updated = True
-    if pdf_url and publication.pdf_url != pdf_url:
-        publication.pdf_url = pdf_url
-        updated = True
-    return updated
-
-
-def _new_publication(
-    *,
-    cluster_id: str | None,
-    fingerprint_sha256: str,
-    title: str,
-    year: int | None,
-    citation_count: int,
-    author_text: str | None,
-    venue_text: str | None,
-    pub_url: str | None,
-    pdf_url: str | None,
-) -> Publication:
-    return Publication(
-        cluster_id=cluster_id,
-        fingerprint_sha256=fingerprint_sha256,
-        title_raw=title,
-        title_normalized=normalize_title(title),
-        year=year,
-        citation_count=citation_count,
-        author_text=author_text,
-        venue_text=venue_text,
-        pub_url=pub_url,
-        pdf_url=pdf_url,
-    )
-
-
-async def _resolve_publication_for_import(
-    db_session: AsyncSession,
-    *,
-    scholar_profile_id: int,
-    title: str,
-    cluster_id: str | None,
-    fingerprint_sha256: str,
-    cluster_cache: dict[str, Publication | None],
-    fingerprint_cache: dict[str, Publication | None],
-) -> Publication | None:
-    if cluster_id:
-        if cluster_id not in cluster_cache:
-            cluster_cache[cluster_id] = await _find_publication_by_cluster(
-                db_session,
-                cluster_id=cluster_id,
-            )
-        if cluster_cache[cluster_id] is not None:
-            return cluster_cache[cluster_id]
-    if fingerprint_sha256 not in fingerprint_cache:
-        fingerprint_cache[fingerprint_sha256] = await _find_publication_by_fingerprint(
-            db_session,
-            fingerprint_sha256=fingerprint_sha256,
-        )
-    if fingerprint_cache[fingerprint_sha256] is not None:
-        return fingerprint_cache[fingerprint_sha256]
-    return await _find_linked_publication_by_title(
-        db_session,
-        scholar_profile_id=scholar_profile_id,
-        title=title,
-    )
-
-
-async def _upsert_scholar_publication_link(
-    db_session: AsyncSession,
-    *,
-    scholar_profile_id: int,
-    publication_id: int,
-    is_read: bool,
-) -> tuple[bool, bool]:
-    result = await db_session.execute(
-        select(ScholarPublication).where(
-            ScholarPublication.scholar_profile_id == scholar_profile_id,
-            ScholarPublication.publication_id == publication_id,
-        )
-    )
-    link = result.scalar_one_or_none()
-    if link is None:
-        db_session.add(
-            ScholarPublication(
-                scholar_profile_id=scholar_profile_id,
-                publication_id=publication_id,
-                is_read=bool(is_read),
-            )
-        )
-        return True, False
-    if bool(link.is_read) == bool(is_read):
-        return False, False
-    link.is_read = bool(is_read)
-    return False, True
-
-
-def _initialize_import_counters(counters: dict[str, int]) -> None:
-    counters.update(
-        {
-            "publications_created": 0,
-            "publications_updated": 0,
-            "links_created": 0,
-            "links_updated": 0,
-        }
-    )
-
-
-def _build_imported_publication_input(
-    *,
-    item: dict[str, Any],
-    scholar_map: dict[str, ScholarProfile],
-) -> ImportedPublicationInput | None:
-    scholar_id = _normalize_optional_text(item.get("scholar_id"))
-    title = _normalize_optional_text(item.get("title"))
-    if not scholar_id or not title:
-        return None
-
-    profile = scholar_map.get(scholar_id)
-    if profile is None:
-        return None
-
-    year = _normalize_optional_year(item.get("year"))
-    author_text = _normalize_optional_text(item.get("author_text"))
-    venue_text = _normalize_optional_text(item.get("venue_text"))
-    return ImportedPublicationInput(
-        profile=profile,
-        title=title,
-        year=year,
-        citation_count=_normalize_citation_count(item.get("citation_count")),
-        author_text=author_text,
-        venue_text=venue_text,
-        cluster_id=_normalize_optional_text(item.get("cluster_id")),
-        pub_url=build_publication_url(_normalize_optional_text(item.get("pub_url"))),
-        doi=normalize_doi(_normalize_optional_text(item.get("doi"))),
-        pdf_url=build_publication_url(_normalize_optional_text(item.get("pdf_url"))),
-        fingerprint=_resolve_fingerprint(
-            title=title,
-            year=year,
-            author_text=author_text,
-            venue_text=venue_text,
-            provided_fingerprint=item.get("fingerprint_sha256"),
-        ),
-        is_read=bool(item.get("is_read", False)),
-    )
-
-
-def _update_link_counters(
-    *,
-    counters: dict[str, int],
-    link_created: bool,
-    link_updated: bool,
-) -> None:
-    if link_created:
-        counters["links_created"] += 1
-    if link_updated:
-        counters["links_updated"] += 1
-
-
-def _cache_resolved_publication(
-    *,
-    publication: Publication,
-    cluster_id: str | None,
-    fingerprint_sha256: str,
-    cluster_cache: dict[str, Publication | None],
-    fingerprint_cache: dict[str, Publication | None],
-) -> None:
-    if cluster_id:
-        cluster_cache[cluster_id] = publication
-    fingerprint_cache[fingerprint_sha256] = publication
-
-
-async def _create_import_publication(
-    db_session: AsyncSession,
-    *,
-    payload: ImportedPublicationInput,
-) -> Publication:
-    publication = _new_publication(
-        cluster_id=payload.cluster_id,
-        fingerprint_sha256=payload.fingerprint,
-        title=payload.title,
-        year=payload.year,
-        citation_count=payload.citation_count,
-        author_text=payload.author_text,
-        venue_text=payload.venue_text,
-        pub_url=payload.pub_url,
-        pdf_url=payload.pdf_url,
-    )
-    db_session.add(publication)
-    await db_session.flush()
-    return publication
-
-
-def _update_import_publication(
-    *,
-    publication: Publication,
-    payload: ImportedPublicationInput,
-) -> bool:
-    return _apply_imported_publication_values(
-        publication=publication,
-        title=payload.title,
-        year=payload.year,
-        citation_count=payload.citation_count,
-        author_text=payload.author_text,
-        venue_text=payload.venue_text,
-        pub_url=payload.pub_url,
-        pdf_url=payload.pdf_url,
-        cluster_id=payload.cluster_id,
-    )
-
-
-async def _upsert_publication_entity(
-    db_session: AsyncSession,
-    *,
-    payload: ImportedPublicationInput,
-    cluster_cache: dict[str, Publication | None],
-    fingerprint_cache: dict[str, Publication | None],
-) -> tuple[Publication, bool, bool]:
-    publication = await _resolve_publication_for_import(
-        db_session,
-        scholar_profile_id=int(payload.profile.id),
-        title=payload.title,
-        cluster_id=payload.cluster_id,
-        fingerprint_sha256=payload.fingerprint,
-        cluster_cache=cluster_cache,
-        fingerprint_cache=fingerprint_cache,
-    )
-    created = False
-    updated = False
-    if publication is None:
-        publication = await _create_import_publication(db_session, payload=payload)
-        created = True
-    else:
-        updated = _update_import_publication(publication=publication, payload=payload)
-
-    _cache_resolved_publication(
-        publication=publication,
-        cluster_id=payload.cluster_id,
-        fingerprint_sha256=payload.fingerprint,
-        cluster_cache=cluster_cache,
-        fingerprint_cache=fingerprint_cache,
-    )
-    return publication, created, updated
-
-
-async def _upsert_imported_publication(
-    db_session: AsyncSession,
-    *,
-    payload: ImportedPublicationInput,
-    cluster_cache: dict[str, Publication | None],
-    fingerprint_cache: dict[str, Publication | None],
-    counters: dict[str, int],
-) -> None:
-    publication, created, updated = await _upsert_publication_entity(
-        db_session,
-        payload=payload,
-        cluster_cache=cluster_cache,
-        fingerprint_cache=fingerprint_cache,
-    )
-    if created:
-        counters["publications_created"] += 1
-    if updated:
-        counters["publications_updated"] += 1
-
-    link_created, link_updated = await _upsert_scholar_publication_link(
-        db_session,
-        scholar_profile_id=int(payload.profile.id),
-        publication_id=int(publication.id),
-        is_read=payload.is_read,
-    )
-    _update_link_counters(
-        counters=counters,
-        link_created=link_created,
-        link_updated=link_updated,
-    )

build/lib/app/services/domains/portability/scholar_import.py

@@ -1,107 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import ScholarProfile
-from app.services.domains.portability.normalize import _normalize_optional_text
-from app.services.domains.scholars import application as scholar_service
-
-
-async def _load_user_scholar_map(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-) -> dict[str, ScholarProfile]:
-    result = await db_session.execute(
-        select(ScholarProfile).where(ScholarProfile.user_id == user_id)
-    )
-    profiles = list(result.scalars().all())
-    return {profile.scholar_id: profile for profile in profiles}
-
-
-def _apply_imported_scholar_values(
-    *,
-    profile: ScholarProfile,
-    display_name: str | None,
-    profile_image_override_url: str | None,
-    is_enabled: bool,
-) -> bool:
-    updated = False
-    if display_name and profile.display_name != display_name:
-        profile.display_name = display_name
-        updated = True
-    if profile.profile_image_override_url != profile_image_override_url:
-        profile.profile_image_override_url = profile_image_override_url
-        updated = True
-    if bool(profile.is_enabled) != bool(is_enabled):
-        profile.is_enabled = bool(is_enabled)
-        updated = True
-    return updated
-
-
-def _new_scholar_profile(
-    *,
-    user_id: int,
-    scholar_id: str,
-    display_name: str | None,
-    profile_image_override_url: str | None,
-    is_enabled: bool,
-) -> ScholarProfile:
-    return ScholarProfile(
-        user_id=user_id,
-        scholar_id=scholar_id,
-        display_name=display_name,
-        profile_image_override_url=profile_image_override_url,
-        is_enabled=bool(is_enabled),
-    )
-
-
-async def _upsert_imported_scholars(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholars: list[dict[str, Any]],
-) -> tuple[dict[str, ScholarProfile], dict[str, int]]:
-    scholar_map = await _load_user_scholar_map(db_session, user_id=user_id)
-    counters = {"scholars_created": 0, "scholars_updated": 0, "skipped_records": 0}
-    for item in scholars:
-        try:
-            scholar_id = scholar_service.validate_scholar_id(str(item["scholar_id"]))
-            display_name = scholar_service.normalize_display_name(
-                str(item.get("display_name") or "")
-            )
-            override_url = scholar_service.normalize_profile_image_url(
-                _normalize_optional_text(item.get("profile_image_override_url"))
-            )
-        except (KeyError, scholar_service.ScholarServiceError):
-            counters["skipped_records"] += 1
-            continue
-
-        is_enabled = bool(item.get("is_enabled", True))
-        existing = scholar_map.get(scholar_id)
-        if existing is None:
-            profile = _new_scholar_profile(
-                user_id=user_id,
-                scholar_id=scholar_id,
-                display_name=display_name,
-                profile_image_override_url=override_url,
-                is_enabled=is_enabled,
-            )
-            db_session.add(profile)
-            scholar_map[scholar_id] = profile
-            counters["scholars_created"] += 1
-            continue
-
-        if _apply_imported_scholar_values(
-            profile=existing,
-            display_name=display_name,
-            profile_image_override_url=override_url,
-            is_enabled=is_enabled,
-        ):
-            counters["scholars_updated"] += 1
-
-    await db_session.flush()
-    return scholar_map, counters

build/lib/app/services/domains/portability/types.py

@@ -1,25 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-
-from app.db.models import ScholarProfile
-
-
-class ImportExportError(ValueError):
-    """Raised when import/export payload constraints are violated."""
-
-
-@dataclass(frozen=True)
-class ImportedPublicationInput:
-    profile: ScholarProfile
-    title: str
-    year: int | None
-    citation_count: int
-    author_text: str | None
-    venue_text: str | None
-    cluster_id: str | None
-    pub_url: str | None
-    doi: str | None
-    pdf_url: str | None
-    fingerprint: str
-    is_read: bool

build/lib/app/services/domains/publication_identifiers/__init__.py

@@ -1,17 +0,0 @@
-from app.services.domains.publication_identifiers.application import (
-    DisplayIdentifier,
-    derive_display_identifier_from_values,
-    overlay_pdf_queue_items_with_display_identifiers,
-    overlay_publication_items_with_display_identifiers,
-    sync_identifiers_for_publication_fields,
-    sync_identifiers_for_publication_resolution,
-)
-
-__all__ = [
-    "DisplayIdentifier",
-    "derive_display_identifier_from_values",
-    "overlay_pdf_queue_items_with_display_identifiers",
-    "overlay_publication_items_with_display_identifiers",
-    "sync_identifiers_for_publication_fields",
-    "sync_identifiers_for_publication_resolution",
-]

build/lib/app/services/domains/publication_identifiers/application.py

@@ -1,433 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import replace
-from typing import TYPE_CHECKING
-
-from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import Publication, PublicationIdentifier
-from app.services.domains.doi.normalize import normalize_doi
-from app.services.domains.publication_identifiers.normalize import (
-    normalize_arxiv_id,
-    normalize_pmcid,
-    normalize_pmid,
-)
-from app.services.domains.publication_identifiers.types import (
-    DisplayIdentifier,
-    IdentifierCandidate,
-    IdentifierKind,
-)
-
-if TYPE_CHECKING:
-    from app.services.domains.publications.pdf_queue import PdfQueueListItem
-    from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
-
-CONFIDENCE_HIGH = 0.98
-CONFIDENCE_MEDIUM = 0.9
-CONFIDENCE_LOW = 0.6
-CONFIDENCE_FALLBACK = 0.4
-PRIORITY_DOI = 400
-PRIORITY_ARXIV = 300
-PRIORITY_PMCID = 200
-PRIORITY_PMID = 100
-
-
-def derive_display_identifier_from_values(
-    *,
-    doi: str | None,
-    pub_url: str | None = None,
-    pdf_url: str | None = None,
-) -> DisplayIdentifier | None:
-    candidates = _fallback_candidates_from_values(doi=doi, pub_url=pub_url, pdf_url=pdf_url)
-    return _best_display_identifier(candidates)
-
-
-def _fallback_candidates_from_values(
-    *,
-    doi: str | None,
-    pub_url: str | None,
-    pdf_url: str | None,
-) -> list[IdentifierCandidate]:
-    values = [value for value in [pub_url, pdf_url] if value]
-    candidates = []
-    if doi:
-        normalized_doi = normalize_doi(doi)
-        if normalized_doi:
-            candidates.append(_candidate(IdentifierKind.DOI, doi, normalized_doi, "legacy_doi", CONFIDENCE_HIGH, pub_url))
-    candidates.extend(_url_identifier_candidates(values=values, source="legacy_urls"))
-    return _dedup_candidates(candidates)
-
-
-def _url_identifier_candidates(*, values: list[str], source: str) -> list[IdentifierCandidate]:
-    candidates: list[IdentifierCandidate] = []
-    for value in values:
-        candidates.extend(_url_candidates_for_value(value=value, source=source))
-    return candidates
-
-
-def _url_candidates_for_value(*, value: str, source: str) -> list[IdentifierCandidate]:
-    candidates: list[IdentifierCandidate] = []
-    arxiv = normalize_arxiv_id(value)
-    if arxiv:
-        candidates.append(_candidate(IdentifierKind.ARXIV, value, arxiv, source, CONFIDENCE_MEDIUM, value))
-    pmcid = normalize_pmcid(value)
-    if pmcid:
-        candidates.append(_candidate(IdentifierKind.PMCID, value, pmcid, source, CONFIDENCE_LOW, value))
-    pmid = normalize_pmid(value)
-    if pmid:
-        candidates.append(_candidate(IdentifierKind.PMID, value, pmid, source, CONFIDENCE_FALLBACK, value))
-    return candidates
-
-
-def _candidate(
-    kind: IdentifierKind,
-    value_raw: str,
-    value_normalized: str,
-    source: str,
-    confidence_score: float,
-    evidence_url: str | None,
-) -> IdentifierCandidate:
-    return IdentifierCandidate(
-        kind=kind,
-        value_raw=value_raw,
-        value_normalized=value_normalized,
-        source=source,
-        confidence_score=float(confidence_score),
-        evidence_url=evidence_url,
-    )
-
-
-def _dedup_candidates(candidates: list[IdentifierCandidate]) -> list[IdentifierCandidate]:
-    deduped: dict[tuple[str, str], IdentifierCandidate] = {}
-    for candidate in candidates:
-        key = (candidate.kind.value, candidate.value_normalized)
-        current = deduped.get(key)
-        if current is None or candidate.confidence_score > current.confidence_score:
-            deduped[key] = candidate
-    return list(deduped.values())
-
-
-async def sync_identifiers_for_publication_fields(
-    db_session: AsyncSession,
-    *,
-    publication: Publication,
-) -> None:
-    candidates = _publication_field_candidates(publication)
-    await _upsert_publication_candidates(db_session, publication_id=int(publication.id), candidates=candidates)
-
-
-async def discover_and_sync_identifiers_for_publication(
-    db_session: AsyncSession,
-    *,
-    publication: Publication,
-    scholar_label: str,
-) -> None:
-    await sync_identifiers_for_publication_fields(db_session, publication=publication)
-    
-    existing_doi = await _existing_identifier_by_kind(
-        db_session,
-        publication_id=int(publication.id),
-        kind=IdentifierKind.DOI.value,
-    )
-    if existing_doi is not None:
-        return
-
-    from app.services.domains.crossref import application as crossref_service
-    from app.services.domains.publications.types import UnreadPublicationItem
-    
-    item = UnreadPublicationItem(
-        publication_id=int(publication.id),
-        scholar_profile_id=0,
-        scholar_label=scholar_label,
-        title=str(publication.title_raw or ""),
-        year=publication.year,
-        citation_count=publication.citation_count,
-        venue_text=publication.venue_text,
-        pub_url=publication.pub_url,
-        pdf_url=publication.pdf_url,
-    )
-    
-    discovered_doi = await crossref_service.discover_doi_for_publication(item=item)
-    if discovered_doi:
-        normalized_doi = normalize_doi(discovered_doi)
-        if normalized_doi:
-            candidate = _candidate(
-                IdentifierKind.DOI,
-                discovered_doi,
-                normalized_doi,
-                "crossref_api",
-                CONFIDENCE_MEDIUM,
-                None,
-            )
-            await _upsert_publication_candidate(db_session, publication_id=int(publication.id), candidate=candidate)
-
-    existing_arxiv = await _existing_identifier_by_kind(
-        db_session,
-        publication_id=int(publication.id),
-        kind=IdentifierKind.ARXIV.value,
-    )
-    if existing_arxiv is None:
-        from app.services.domains.arxiv import application as arxiv_service
-        discovered_arxiv = await arxiv_service.discover_arxiv_id_for_publication(item=item)
-        if discovered_arxiv:
-            normalized_arxiv = normalize_arxiv_id(discovered_arxiv)
-            if normalized_arxiv:
-                candidate = _candidate(
-                    IdentifierKind.ARXIV,
-                    discovered_arxiv,
-                    normalized_arxiv,
-                    "arxiv_api",
-                    CONFIDENCE_MEDIUM,
-                    None,
-                )
-                await _upsert_publication_candidate(db_session, publication_id=int(publication.id), candidate=candidate)
-
-
-def _publication_field_candidates(publication: Publication) -> list[IdentifierCandidate]:
-    return _fallback_candidates_from_values(
-        doi=None,
-        pub_url=publication.pub_url,
-        pdf_url=publication.pdf_url,
-    )
-
-
-async def sync_identifiers_for_publication_resolution(
-    db_session: AsyncSession,
-    *,
-    publication: Publication,
-    source: str | None,
-) -> None:
-    candidates = _publication_field_candidates(publication)
-    rewritten = [_candidate_with_source(candidate, source=source) for candidate in candidates]
-    await _upsert_publication_candidates(db_session, publication_id=int(publication.id), candidates=rewritten)
-
-
-def _candidate_with_source(candidate: IdentifierCandidate, *, source: str | None) -> IdentifierCandidate:
-    if not source:
-        return candidate
-    return IdentifierCandidate(
-        kind=candidate.kind,
-        value_raw=candidate.value_raw,
-        value_normalized=candidate.value_normalized,
-        source=source,
-        confidence_score=candidate.confidence_score,
-        evidence_url=candidate.evidence_url,
-    )
-
-
-async def _upsert_publication_candidates(
-    db_session: AsyncSession,
-    *,
-    publication_id: int,
-    candidates: list[IdentifierCandidate],
-) -> None:
-    for candidate in _dedup_candidates(candidates):
-        await _upsert_publication_candidate(db_session, publication_id=publication_id, candidate=candidate)
-
-
-async def _upsert_publication_candidate(
-    db_session: AsyncSession,
-    *,
-    publication_id: int,
-    candidate: IdentifierCandidate,
-) -> None:
-    existing = await _existing_identifier(
-        db_session,
-        publication_id=publication_id,
-        kind=candidate.kind.value,
-        value_normalized=candidate.value_normalized,
-    )
-    if existing is None:
-        db_session.add(_new_identifier_row(publication_id=publication_id, candidate=candidate))
-        return
-    _merge_identifier_row(existing, candidate=candidate)
-
-
-async def _existing_identifier(
-    db_session: AsyncSession,
-    *,
-    publication_id: int,
-    kind: str,
-    value_normalized: str,
-) -> PublicationIdentifier | None:
-    result = await db_session.execute(
-        select(PublicationIdentifier).where(
-            PublicationIdentifier.publication_id == publication_id,
-            PublicationIdentifier.kind == kind,
-            PublicationIdentifier.value_normalized == value_normalized,
-        )
-    )
-    return result.scalar_one_or_none()
-
-
-async def _existing_identifier_by_kind(
-    db_session: AsyncSession,
-    *,
-    publication_id: int,
-    kind: str,
-) -> PublicationIdentifier | None:
-    result = await db_session.execute(
-        select(PublicationIdentifier).where(
-            PublicationIdentifier.publication_id == publication_id,
-            PublicationIdentifier.kind == kind,
-        ).order_by(PublicationIdentifier.confidence_score.desc()).limit(1)
-    )
-    return result.scalar_one_or_none()
-
-
-def _new_identifier_row(
-    *,
-    publication_id: int,
-    candidate: IdentifierCandidate,
-) -> PublicationIdentifier:
-    return PublicationIdentifier(
-        publication_id=publication_id,
-        kind=candidate.kind.value,
-        value_raw=candidate.value_raw,
-        value_normalized=candidate.value_normalized,
-        source=candidate.source,
-        confidence_score=candidate.confidence_score,
-        evidence_url=candidate.evidence_url,
-    )
-
-
-def _merge_identifier_row(existing: PublicationIdentifier, *, candidate: IdentifierCandidate) -> None:
-    if candidate.confidence_score >= float(existing.confidence_score):
-        existing.value_raw = candidate.value_raw
-        existing.source = candidate.source
-        existing.confidence_score = candidate.confidence_score
-        if candidate.evidence_url:
-            existing.evidence_url = candidate.evidence_url
-
-
-async def overlay_publication_items_with_display_identifiers(
-    db_session: AsyncSession,
-    *,
-    items: list[PublicationListItem],
-) -> list[PublicationListItem]:
-    if not items:
-        return []
-    mapping = await _display_identifier_map(db_session, publication_ids=[item.publication_id for item in items])
-    return [_overlay_publication_item(item, mapping.get(item.publication_id)) for item in items]
-
-
-def _overlay_publication_item(
-    item: PublicationListItem,
-    display_identifier: DisplayIdentifier | None,
-) -> PublicationListItem:
-    fallback = display_identifier or derive_display_identifier_from_values(doi=None, pub_url=item.pub_url, pdf_url=item.pdf_url)
-    return replace(item, display_identifier=fallback)
-
-
-async def overlay_pdf_queue_items_with_display_identifiers(
-    db_session: AsyncSession,
-    *,
-    items: list[PdfQueueListItem],
-) -> list[PdfQueueListItem]:
-    if not items:
-        return []
-    mapping = await _display_identifier_map(db_session, publication_ids=[item.publication_id for item in items])
-    return [_overlay_queue_item(item, mapping.get(item.publication_id)) for item in items]
-
-
-def _overlay_queue_item(
-    item: PdfQueueListItem,
-    display_identifier: DisplayIdentifier | None,
-) -> PdfQueueListItem:
-    fallback = display_identifier or derive_display_identifier_from_values(doi=None, pdf_url=item.pdf_url)
-    return replace(item, display_identifier=fallback)
-
-
-async def _display_identifier_map(
-    db_session: AsyncSession,
-    *,
-    publication_ids: list[int],
-) -> dict[int, DisplayIdentifier]:
-    normalized_ids = sorted({int(value) for value in publication_ids if int(value) > 0})
-    if not normalized_ids:
-        return {}
-    result = await db_session.execute(
-        select(PublicationIdentifier).where(PublicationIdentifier.publication_id.in_(normalized_ids))
-    )
-    rows = list(result.scalars().all())
-    return _best_display_identifier_map(rows)
-
-
-def _best_display_identifier_map(rows: list[PublicationIdentifier]) -> dict[int, DisplayIdentifier]:
-    grouped: dict[int, list[IdentifierCandidate]] = {}
-    for row in rows:
-        grouped.setdefault(int(row.publication_id), []).append(_candidate_from_row(row))
-    return {
-        publication_id: display
-        for publication_id, display in (
-            (publication_id, _best_display_identifier(candidates))
-            for publication_id, candidates in grouped.items()
-        )
-        if display is not None
-    }
-
-
-def _candidate_from_row(row: PublicationIdentifier) -> IdentifierCandidate:
-    return IdentifierCandidate(
-        kind=IdentifierKind(str(row.kind)),
-        value_raw=str(row.value_raw),
-        value_normalized=str(row.value_normalized),
-        source=str(row.source),
-        confidence_score=float(row.confidence_score),
-        evidence_url=row.evidence_url,
-    )
-
-
-def _best_display_identifier(candidates: list[IdentifierCandidate]) -> DisplayIdentifier | None:
-    if not candidates:
-        return None
-    ordered = sorted(candidates, key=_display_sort_key, reverse=True)
-    return _display_identifier_from_candidate(ordered[0])
-
-
-def _display_sort_key(candidate: IdentifierCandidate) -> tuple[int, float]:
-    return (_kind_priority(candidate.kind), float(candidate.confidence_score))
-
-
-def _kind_priority(kind: IdentifierKind) -> int:
-    if kind == IdentifierKind.DOI:
-        return PRIORITY_DOI
-    if kind == IdentifierKind.ARXIV:
-        return PRIORITY_ARXIV
-    if kind == IdentifierKind.PMCID:
-        return PRIORITY_PMCID
-    return PRIORITY_PMID
-
-
-def _display_identifier_from_candidate(candidate: IdentifierCandidate) -> DisplayIdentifier:
-    value = candidate.value_normalized
-    return DisplayIdentifier(
-        kind=candidate.kind.value,
-        value=value,
-        label=_display_label(candidate.kind, value),
-        url=_identifier_url(candidate.kind, value),
-        confidence_score=float(candidate.confidence_score),
-    )
-
-
-def _display_label(kind: IdentifierKind, value: str) -> str:
-    if kind == IdentifierKind.DOI:
-        return f"DOI: {value}"
-    if kind == IdentifierKind.ARXIV:
-        return f"arXiv: {value}"
-    if kind == IdentifierKind.PMCID:
-        return f"PMCID: {value}"
-    return f"PMID: {value}"
-
-
-def _identifier_url(kind: IdentifierKind, value: str) -> str | None:
-    if kind == IdentifierKind.DOI:
-        return f"https://doi.org/{value}"
-    if kind == IdentifierKind.ARXIV:
-        return f"https://arxiv.org/abs/{value}"
-    if kind == IdentifierKind.PMCID:
-        return f"https://pmc.ncbi.nlm.nih.gov/articles/{value}/"
-    if kind == IdentifierKind.PMID:
-        return f"https://pubmed.ncbi.nlm.nih.gov/{value}/"
-    return None

build/lib/app/services/domains/publication_identifiers/normalize.py

@@ -1,76 +0,0 @@
-from __future__ import annotations
-
-import re
-from urllib.parse import urlparse
-
-from app.services.domains.doi.normalize import normalize_doi
-from app.services.domains.publication_identifiers.types import IdentifierKind
-
-ARXIV_ABS_RE = re.compile(r"\barxiv:\s*([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?\b", re.I)
-ARXIV_PATH_RE = re.compile(r"^/(?:abs|pdf|html|ps|format)/([a-z-]+/\d{7}|\d{4}\.\d{4,5})(v\d+)?(?:\.pdf)?/?$", re.I)
-PMCID_RE = re.compile(r"\b(PMC\d+)\b", re.I)
-PUBMED_PATH_RE = re.compile(r"^/(\d+)/?$")
-
-
-def normalize_identifier(kind: IdentifierKind, value: str | None) -> str | None:
-    if kind == IdentifierKind.DOI:
-        return normalize_doi(value)
-    if kind == IdentifierKind.ARXIV:
-        return normalize_arxiv_id(value)
-    if kind == IdentifierKind.PMCID:
-        return normalize_pmcid(value)
-    if kind == IdentifierKind.PMID:
-        return normalize_pmid(value)
-    return None
-
-
-def normalize_arxiv_id(value: str | None) -> str | None:
-    text = (value or "").strip()
-    if not text:
-        return None
-    parsed = urlparse(text)
-    if parsed.scheme in {"http", "https"} and "arxiv.org" in parsed.netloc.lower():
-        return _arxiv_from_path(parsed.path)
-    match = ARXIV_ABS_RE.search(text)
-    if not match:
-        return None
-    version = (match.group(2) or "").lower()
-    return f"{match.group(1).lower()}{version}"
-
-
-def _arxiv_from_path(path: str) -> str | None:
-    match = ARXIV_PATH_RE.match(path or "")
-    if not match:
-        return None
-    version = (match.group(2) or "").lower()
-    return f"{match.group(1).lower()}{version}"
-
-
-def normalize_pmcid(value: str | None) -> str | None:
-    text = (value or "").strip()
-    if not text:
-        return None
-    parsed = urlparse(text)
-    if parsed.scheme in {"http", "https"} and "ncbi.nlm.nih.gov" in parsed.netloc.lower():
-        return _first_match(PMCID_RE, parsed.path)
-    return _first_match(PMCID_RE, text)
-
-
-def normalize_pmid(value: str | None) -> str | None:
-    text = (value or "").strip()
-    if not text:
-        return None
-    parsed = urlparse(text)
-    if parsed.scheme in {"http", "https"} and "pubmed.ncbi.nlm.nih.gov" in parsed.netloc.lower():
-        match = PUBMED_PATH_RE.match(parsed.path or "")
-        if not match:
-            return None
-        return match.group(1)
-    return None
-
-
-def _first_match(pattern: re.Pattern[str], value: str) -> str | None:
-    match = pattern.search(value)
-    if not match:
-        return None
-    return match.group(1).upper()

build/lib/app/services/domains/publication_identifiers/types.py

@@ -1,30 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from enum import StrEnum
-
-
-class IdentifierKind(StrEnum):
-    DOI = "doi"
-    ARXIV = "arxiv"
-    PMCID = "pmcid"
-    PMID = "pmid"
-
-
-@dataclass(frozen=True)
-class DisplayIdentifier:
-    kind: str
-    value: str
-    label: str
-    url: str | None
-    confidence_score: float
-
-
-@dataclass(frozen=True)
-class IdentifierCandidate:
-    kind: IdentifierKind
-    value_raw: str
-    value_normalized: str
-    source: str
-    confidence_score: float
-    evidence_url: str | None

build/lib/app/services/domains/publications/__init__.py

@@ -1 +0,0 @@
-from app.services.domains.publications.application import *

build/lib/app/services/domains/publications/application.py

@@ -1,74 +0,0 @@
-from __future__ import annotations
-
-from app.services.domains.publications.counts import (
-    count_for_user,
-    count_favorite_for_user,
-    count_latest_for_user,
-    count_unread_for_user,
-)
-from app.services.domains.publications.listing import (
-    list_for_user,
-    retry_pdf_for_user,
-    list_unread_for_user,
-)
-from app.services.domains.publications.enrichment import (
-    hydrate_pdf_enrichment_state,
-    schedule_missing_pdf_enrichment_for_user,
-    schedule_retry_pdf_enrichment_for_row,
-)
-from app.services.domains.publications.modes import (
-    MODE_ALL,
-    MODE_LATEST,
-    MODE_NEW,
-    MODE_UNREAD,
-    resolve_publication_view_mode,
-)
-from app.services.domains.publications.queries import (
-    get_latest_completed_run_id_for_user,
-    get_publication_item_for_user,
-    publications_query,
-)
-from app.services.domains.publications.read_state import (
-    mark_all_unread_as_read_for_user,
-    mark_selected_as_read_for_user,
-    set_publication_favorite_for_user,
-)
-from app.services.domains.publications.pdf_queue import (
-    count_pdf_queue_items,
-    enqueue_all_missing_pdf_jobs,
-    enqueue_retry_pdf_job_for_publication_id,
-    list_pdf_queue_page,
-    list_pdf_queue_items,
-)
-from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
-
-__all__ = [
-    "MODE_ALL",
-    "MODE_UNREAD",
-    "MODE_LATEST",
-    "MODE_NEW",
-    "PublicationListItem",
-    "UnreadPublicationItem",
-    "resolve_publication_view_mode",
-    "get_latest_completed_run_id_for_user",
-    "publications_query",
-    "get_publication_item_for_user",
-    "list_for_user",
-    "list_unread_for_user",
-    "retry_pdf_for_user",
-    "hydrate_pdf_enrichment_state",
-    "schedule_retry_pdf_enrichment_for_row",
-    "list_pdf_queue_items",
-    "list_pdf_queue_page",
-    "count_pdf_queue_items",
-    "enqueue_all_missing_pdf_jobs",
-    "enqueue_retry_pdf_job_for_publication_id",
-    "schedule_missing_pdf_enrichment_for_user",
-    "count_for_user",
-    "count_favorite_for_user",
-    "count_unread_for_user",
-    "count_latest_for_user",
-    "mark_all_unread_as_read_for_user",
-    "mark_selected_as_read_for_user",
-    "set_publication_favorite_for_user",
-]

build/lib/app/services/domains/publications/counts.py

@@ -1,90 +0,0 @@
-from __future__ import annotations
-
-from sqlalchemy import func, select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import ScholarProfile, ScholarPublication
-from app.services.domains.publications.modes import (
-    MODE_ALL,
-    MODE_LATEST,
-    MODE_UNREAD,
-    resolve_publication_view_mode,
-)
-from app.services.domains.publications.queries import get_latest_completed_run_id_for_user
-
-
-async def count_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    mode: str = MODE_ALL,
-    scholar_profile_id: int | None = None,
-    favorite_only: bool = False,
-) -> int:
-    resolved_mode = resolve_publication_view_mode(mode)
-    latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
-    stmt = (
-        select(func.count())
-        .select_from(ScholarPublication)
-        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
-        .where(ScholarProfile.user_id == user_id)
-    )
-    if scholar_profile_id is not None:
-        stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
-    if favorite_only:
-        stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
-    if resolved_mode == MODE_UNREAD:
-        stmt = stmt.where(ScholarPublication.is_read.is_(False))
-    if resolved_mode == MODE_LATEST:
-        if latest_run_id is None:
-            return 0
-        stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
-    result = await db_session.execute(stmt)
-    return int(result.scalar_one() or 0)
-
-
-async def count_unread_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholar_profile_id: int | None = None,
-    favorite_only: bool = False,
-) -> int:
-    return await count_for_user(
-        db_session,
-        user_id=user_id,
-        mode=MODE_UNREAD,
-        scholar_profile_id=scholar_profile_id,
-        favorite_only=favorite_only,
-    )
-
-
-async def count_latest_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholar_profile_id: int | None = None,
-    favorite_only: bool = False,
-) -> int:
-    return await count_for_user(
-        db_session,
-        user_id=user_id,
-        mode=MODE_LATEST,
-        scholar_profile_id=scholar_profile_id,
-        favorite_only=favorite_only,
-    )
-
-
-async def count_favorite_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholar_profile_id: int | None = None,
-) -> int:
-    return await count_for_user(
-        db_session,
-        user_id=user_id,
-        mode=MODE_ALL,
-        scholar_profile_id=scholar_profile_id,
-        favorite_only=True,
-    )

build/lib/app/services/domains/publications/enrichment.py

@@ -1,73 +0,0 @@
-from __future__ import annotations
-
-import logging
-
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.services.domains.publications.pdf_queue import (
-    enqueue_missing_pdf_jobs,
-    enqueue_retry_pdf_job,
-    overlay_pdf_job_state,
-)
-from app.services.domains.publications.types import PublicationListItem
-
-logger = logging.getLogger(__name__)
-
-
-async def schedule_missing_pdf_enrichment_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    request_email: str | None,
-    items: list[PublicationListItem],
-    max_items: int,
-) -> int:
-    queued_ids = await enqueue_missing_pdf_jobs(
-        db_session,
-        user_id=user_id,
-        request_email=request_email,
-        rows=items,
-        max_items=max_items,
-    )
-    logger.info(
-        "publications.enrichment.scheduled",
-        extra={
-            "event": "publications.enrichment.scheduled",
-            "user_id": user_id,
-            "publication_count": len(queued_ids),
-        },
-    )
-    return len(queued_ids)
-
-
-async def schedule_retry_pdf_enrichment_for_row(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    request_email: str | None,
-    item: PublicationListItem,
-) -> bool:
-    queued = await enqueue_retry_pdf_job(
-        db_session,
-        user_id=user_id,
-        request_email=request_email,
-        row=item,
-    )
-    logger.info(
-        "publications.enrichment.retry_scheduled",
-        extra={
-            "event": "publications.enrichment.retry_scheduled",
-            "user_id": user_id,
-            "publication_id": item.publication_id,
-            "queued": queued,
-        },
-    )
-    return queued
-
-
-async def hydrate_pdf_enrichment_state(
-    db_session: AsyncSession,
-    *,
-    items: list[PublicationListItem],
-) -> list[PublicationListItem]:
-    return await overlay_pdf_job_state(db_session, rows=items)

build/lib/app/services/domains/publications/listing.py

@@ -1,93 +0,0 @@
-from __future__ import annotations
-
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.services.domains.publication_identifiers import application as identifier_service
-from app.services.domains.publications.modes import (
-    MODE_ALL,
-    MODE_UNREAD,
-    resolve_publication_view_mode,
-)
-from app.services.domains.publications.queries import (
-    get_latest_completed_run_id_for_user,
-    get_publication_item_for_user,
-    publication_list_item_from_row,
-    publications_query,
-    unread_item_from_row,
-)
-from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
-
-
-async def list_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    mode: str = MODE_ALL,
-    scholar_profile_id: int | None = None,
-    favorite_only: bool = False,
-    limit: int = 300,
-    offset: int = 0,
-) -> list[PublicationListItem]:
-    resolved_mode = resolve_publication_view_mode(mode)
-    latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
-    result = await db_session.execute(
-        publications_query(
-            user_id=user_id,
-            mode=resolved_mode,
-            latest_run_id=latest_run_id,
-            scholar_profile_id=scholar_profile_id,
-            favorite_only=favorite_only,
-            limit=limit,
-            offset=offset,
-        )
-    )
-    rows = [
-        publication_list_item_from_row(row, latest_run_id=latest_run_id)
-        for row in result.all()
-    ]
-    return await identifier_service.overlay_publication_items_with_display_identifiers(
-        db_session,
-        items=rows,
-    )
-
-
-async def retry_pdf_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholar_profile_id: int,
-    publication_id: int,
-) -> PublicationListItem | None:
-    item = await get_publication_item_for_user(
-        db_session,
-        user_id=user_id,
-        scholar_profile_id=scholar_profile_id,
-        publication_id=publication_id,
-    )
-    if item is None:
-        return None
-    hydrated = await identifier_service.overlay_publication_items_with_display_identifiers(
-        db_session,
-        items=[item],
-    )
-    return hydrated[0] if hydrated else item
-
-
-async def list_unread_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    limit: int = 100,
-) -> list[UnreadPublicationItem]:
-    result = await db_session.execute(
-        publications_query(
-            user_id=user_id,
-            mode=MODE_UNREAD,
-            latest_run_id=None,
-            scholar_profile_id=None,
-            favorite_only=False,
-            limit=limit,
-            offset=0,
-        )
-    )
-    return [unread_item_from_row(row) for row in result.all()]

build/lib/app/services/domains/publications/modes.py

@@ -1,14 +0,0 @@
-from __future__ import annotations
-
-MODE_ALL = "all"
-MODE_UNREAD = "unread"
-MODE_LATEST = "latest"
-MODE_NEW = "new"  # compatibility alias for MODE_LATEST
-
-
-def resolve_publication_view_mode(value: str | None) -> str:
-    if value == MODE_UNREAD:
-        return MODE_UNREAD
-    if value in {MODE_LATEST, MODE_NEW}:
-        return MODE_LATEST
-    return MODE_ALL

build/lib/app/services/domains/publications/pdf_queue.py

@@ -1,915 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-from dataclasses import dataclass
-from datetime import datetime, timezone
-import logging
-
-from sqlalchemy import Select, func, literal, or_, select, union_all
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import (
-    Publication,
-    PublicationPdfJob,
-    PublicationPdfJobEvent,
-    ScholarProfile,
-    ScholarPublication,
-    User,
-)
-from app.db.session import get_session_factory
-from app.services.domains.publication_identifiers import application as identifier_service
-from app.services.domains.publication_identifiers.types import DisplayIdentifier
-from app.services.domains.publications.pdf_resolution_pipeline import (
-    resolve_publication_pdf_outcome_for_row,
-)
-from app.services.domains.publications.types import PublicationListItem
-from app.services.domains.unpaywall.application import (
-    FAILURE_RESOLUTION_EXCEPTION,
-    OaResolutionOutcome,
-)
-from app.settings import settings
-
-PDF_STATUS_UNTRACKED = "untracked"
-PDF_STATUS_QUEUED = "queued"
-PDF_STATUS_RUNNING = "running"
-PDF_STATUS_RESOLVED = "resolved"
-PDF_STATUS_FAILED = "failed"
-
-PDF_EVENT_QUEUED = "queued"
-PDF_EVENT_ATTEMPT_STARTED = "attempt_started"
-PDF_EVENT_RESOLVED = "resolved"
-PDF_EVENT_FAILED = "failed"
-
-logger = logging.getLogger(__name__)
-_scheduled_tasks: set[asyncio.Task[None]] = set()
-
-
-@dataclass(frozen=True)
-class PdfQueueListItem:
-    publication_id: int
-    title: str
-    pdf_url: str | None
-    status: str
-    attempt_count: int
-    last_failure_reason: str | None
-    last_failure_detail: str | None
-    last_source: str | None
-    requested_by_user_id: int | None
-    requested_by_email: str | None
-    queued_at: datetime | None
-    last_attempt_at: datetime | None
-    resolved_at: datetime | None
-    updated_at: datetime
-    display_identifier: DisplayIdentifier | None = None
-
-
-@dataclass(frozen=True)
-class PdfRequeueResult:
-    publication_exists: bool
-    queued: bool
-
-
-@dataclass(frozen=True)
-class PdfBulkQueueResult:
-    requested_count: int
-    queued_count: int
-
-
-@dataclass(frozen=True)
-class PdfQueuePage:
-    items: list[PdfQueueListItem]
-    total_count: int
-    limit: int
-    offset: int
-
-
-def _utcnow() -> datetime:
-    return datetime.now(timezone.utc)
-
-
-def _publication_ids(rows: list[PublicationListItem]) -> list[int]:
-    return sorted({row.publication_id for row in rows})
-
-
-def _status_from_job(row: PublicationListItem, job: PublicationPdfJob | None) -> str:
-    if row.pdf_url:
-        return PDF_STATUS_RESOLVED
-    if job is None:
-        return PDF_STATUS_UNTRACKED
-    return job.status
-
-
-def _item_from_row_and_job(
-    row: PublicationListItem,
-    job: PublicationPdfJob | None,
-) -> PublicationListItem:
-    return PublicationListItem(
-        publication_id=row.publication_id,
-        scholar_profile_id=row.scholar_profile_id,
-        scholar_label=row.scholar_label,
-        title=row.title,
-        year=row.year,
-        citation_count=row.citation_count,
-        venue_text=row.venue_text,
-        pub_url=row.pub_url,
-        pdf_url=row.pdf_url,
-        is_read=row.is_read,
-        is_favorite=row.is_favorite,
-        first_seen_at=row.first_seen_at,
-        is_new_in_latest_run=row.is_new_in_latest_run,
-        pdf_status=_status_from_job(row, job),
-        pdf_attempt_count=int(job.attempt_count) if job is not None else 0,
-        pdf_failure_reason=job.last_failure_reason if job is not None else None,
-        pdf_failure_detail=job.last_failure_detail if job is not None else None,
-        display_identifier=row.display_identifier,
-    )
-
-
-def _queueable_rows(
-    rows: list[PublicationListItem],
-    *,
-    max_items: int,
-) -> list[PublicationListItem]:
-    bounded = max(0, int(max_items))
-    if bounded == 0:
-        return []
-    candidates = [row for row in rows if not row.pdf_url]
-    return candidates[:bounded]
-
-
-def _bounded_limit(limit: int, *, max_value: int = 500) -> int:
-    return max(1, min(int(limit), max_value))
-
-
-def _bounded_offset(offset: int) -> int:
-    return max(int(offset), 0)
-
-
-def _auto_retry_interval_seconds() -> int:
-    return max(int(settings.pdf_auto_retry_interval_seconds), 1)
-
-
-def _auto_retry_first_interval_seconds() -> int:
-    return max(int(settings.pdf_auto_retry_first_interval_seconds), 1)
-
-
-def _auto_retry_max_attempts() -> int:
-    return max(int(settings.pdf_auto_retry_max_attempts), 1)
-
-
-def _retry_interval_seconds_for_attempt_count(attempt_count: int) -> int:
-    if int(attempt_count) <= 1:
-        return _auto_retry_first_interval_seconds()
-    return _auto_retry_interval_seconds()
-
-
-def _cooldown_active(
-    *,
-    last_attempt_at: datetime | None,
-    attempt_count: int,
-) -> bool:
-    if last_attempt_at is None:
-        return False
-    elapsed = (_utcnow() - last_attempt_at).total_seconds()
-    return elapsed < _retry_interval_seconds_for_attempt_count(int(attempt_count))
-
-
-def _can_enqueue_job(
-    job: PublicationPdfJob | None,
-    *,
-    force_retry: bool,
-) -> bool:
-    if job is None:
-        return True
-    if job.status in {PDF_STATUS_QUEUED, PDF_STATUS_RUNNING}:
-        return False
-    if force_retry:
-        return job.status in {PDF_STATUS_FAILED, PDF_STATUS_RESOLVED, PDF_STATUS_UNTRACKED}
-    if job.status == PDF_STATUS_RESOLVED:
-        return False
-    if int(job.attempt_count) >= _auto_retry_max_attempts():
-        return False
-    if _cooldown_active(
-        last_attempt_at=job.last_attempt_at,
-        attempt_count=int(job.attempt_count),
-    ):
-        return False
-    return True
-
-
-def _event_row(
-    *,
-    publication_id: int,
-    user_id: int | None,
-    event_type: str,
-    status: str | None,
-    source: str | None = None,
-    failure_reason: str | None = None,
-    message: str | None = None,
-) -> PublicationPdfJobEvent:
-    return PublicationPdfJobEvent(
-        publication_id=publication_id,
-        user_id=user_id,
-        event_type=event_type,
-        status=status,
-        source=source,
-        failure_reason=failure_reason,
-        message=message,
-    )
-
-
-def _queued_job(
-    *,
-    publication_id: int,
-    user_id: int,
-) -> PublicationPdfJob:
-    now = _utcnow()
-    return PublicationPdfJob(
-        publication_id=publication_id,
-        status=PDF_STATUS_QUEUED,
-        queued_at=now,
-        last_requested_by_user_id=user_id,
-    )
-
-
-def _mark_job_queued(job: PublicationPdfJob, *, user_id: int) -> None:
-    now = _utcnow()
-    job.status = PDF_STATUS_QUEUED
-    job.queued_at = now
-    job.last_requested_by_user_id = user_id
-    job.last_failure_reason = None
-    job.last_failure_detail = None
-    job.last_source = None
-
-
-def _state_map(jobs: list[PublicationPdfJob]) -> dict[int, PublicationPdfJob]:
-    return {int(job.publication_id): job for job in jobs}
-
-
-async def _jobs_for_publication_ids(
-    db_session: AsyncSession,
-    *,
-    publication_ids: list[int],
-) -> dict[int, PublicationPdfJob]:
-    if not publication_ids:
-        return {}
-    result = await db_session.execute(
-        select(PublicationPdfJob).where(PublicationPdfJob.publication_id.in_(publication_ids))
-    )
-    return _state_map(list(result.scalars()))
-
-
-async def overlay_pdf_job_state(
-    db_session: AsyncSession,
-    *,
-    rows: list[PublicationListItem],
-) -> list[PublicationListItem]:
-    if not rows:
-        return []
-    jobs = await _jobs_for_publication_ids(
-        db_session,
-        publication_ids=_publication_ids(rows),
-    )
-    return [_item_from_row_and_job(row, jobs.get(row.publication_id)) for row in rows]
-
-
-async def _enqueue_rows(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    rows: list[PublicationListItem],
-    force_retry: bool,
-) -> list[PublicationListItem]:
-    if not rows:
-        return []
-    queued: list[PublicationListItem] = []
-    jobs = await _jobs_for_publication_ids(
-        db_session,
-        publication_ids=_publication_ids(rows),
-    )
-    for row in rows:
-        job = jobs.get(row.publication_id)
-        if not _can_enqueue_job(job, force_retry=force_retry):
-            continue
-        if job is None:
-            job = _queued_job(publication_id=row.publication_id, user_id=user_id)
-            jobs[row.publication_id] = job
-            db_session.add(job)
-        else:
-            _mark_job_queued(job, user_id=user_id)
-        db_session.add(
-            _event_row(
-                publication_id=row.publication_id,
-                user_id=user_id,
-                event_type=PDF_EVENT_QUEUED,
-                status=PDF_STATUS_QUEUED,
-            )
-        )
-        queued.append(row)
-    if queued:
-        await db_session.commit()
-    return queued
-
-
-def _register_task(task: asyncio.Task[None]) -> None:
-    _scheduled_tasks.add(task)
-
-
-def _drop_finished_task(task: asyncio.Task[None]) -> None:
-    _scheduled_tasks.discard(task)
-    try:
-        task.result()
-    except Exception:
-        logger.exception(
-            "publications.pdf_queue.task_failed",
-            extra={"event": "publications.pdf_queue.task_failed"},
-        )
-
-
-async def _mark_attempt_started(
-    *,
-    publication_id: int,
-    user_id: int,
-) -> None:
-    session_factory = get_session_factory()
-    async with session_factory() as db_session:
-        job = await db_session.get(PublicationPdfJob, publication_id)
-        if job is None:
-            job = _queued_job(publication_id=publication_id, user_id=user_id)
-            db_session.add(job)
-        job.status = PDF_STATUS_RUNNING
-        job.last_attempt_at = _utcnow()
-        job.attempt_count = int(job.attempt_count) + 1
-        db_session.add(
-            _event_row(
-                publication_id=publication_id,
-                user_id=user_id,
-                event_type=PDF_EVENT_ATTEMPT_STARTED,
-                status=PDF_STATUS_RUNNING,
-            )
-        )
-        await db_session.commit()
-
-
-def _failed_outcome(
-    *,
-    row: PublicationListItem,
-) -> OaResolutionOutcome:
-    return OaResolutionOutcome(
-        publication_id=row.publication_id,
-        doi=None,
-        pdf_url=None,
-        failure_reason=FAILURE_RESOLUTION_EXCEPTION,
-        source=None,
-        used_crossref=False,
-    )
-
-
-async def _fetch_outcome_for_row(
-    *,
-    row: PublicationListItem,
-    request_email: str | None,
-) -> OaResolutionOutcome:
-    pipeline_result = await resolve_publication_pdf_outcome_for_row(
-        row=row,
-        request_email=request_email,
-    )
-    outcome = pipeline_result.outcome
-    if outcome is not None:
-        return outcome
-    return _failed_outcome(row=row)
-
-
-def _apply_publication_update(
-    publication: Publication,
-    *,
-    pdf_url: str | None,
-) -> None:
-    if pdf_url and publication.pdf_url != pdf_url:
-        publication.pdf_url = pdf_url
-
-
-def _apply_job_outcome(job: PublicationPdfJob, *, outcome: OaResolutionOutcome) -> None:
-    job.last_source = outcome.source
-    if outcome.pdf_url:
-        job.status = PDF_STATUS_RESOLVED
-        job.resolved_at = _utcnow()
-        job.last_failure_reason = None
-        job.last_failure_detail = None
-        return
-    job.status = PDF_STATUS_FAILED
-    job.last_failure_reason = outcome.failure_reason
-    job.last_failure_detail = outcome.failure_reason
-
-
-def _result_event(outcome: OaResolutionOutcome) -> tuple[str, str]:
-    if outcome.pdf_url:
-        return PDF_EVENT_RESOLVED, PDF_STATUS_RESOLVED
-    return PDF_EVENT_FAILED, PDF_STATUS_FAILED
-
-
-async def _persist_outcome(
-    *,
-    publication_id: int,
-    user_id: int,
-    outcome: OaResolutionOutcome,
-) -> None:
-    session_factory = get_session_factory()
-    async with session_factory() as db_session:
-        publication = await db_session.get(Publication, publication_id)
-        job = await db_session.get(PublicationPdfJob, publication_id)
-        if publication is None or job is None:
-            return
-        _apply_publication_update(publication, pdf_url=outcome.pdf_url)
-        await identifier_service.sync_identifiers_for_publication_resolution(
-            db_session,
-            publication=publication,
-            source=outcome.source,
-        )
-        _apply_job_outcome(job, outcome=outcome)
-        event_type, status = _result_event(outcome)
-        db_session.add(
-            _event_row(
-                publication_id=publication_id,
-                user_id=user_id,
-                event_type=event_type,
-                status=status,
-                source=outcome.source,
-                failure_reason=outcome.failure_reason,
-                message=outcome.failure_reason,
-            )
-        )
-        await db_session.commit()
-
-
-async def _resolve_publication_row(
-    *,
-    user_id: int,
-    request_email: str | None,
-    row: PublicationListItem,
-) -> None:
-    await _mark_attempt_started(publication_id=row.publication_id, user_id=user_id)
-    try:
-        outcome = await _fetch_outcome_for_row(row=row, request_email=request_email)
-    except Exception as exc:  # pragma: no cover - defensive network boundary
-        logger.warning(
-            "publications.pdf_queue.resolve_failed",
-            extra={
-                "event": "publications.pdf_queue.resolve_failed",
-                "publication_id": row.publication_id,
-                "error": str(exc),
-            },
-        )
-        outcome = _failed_outcome(row=row)
-    await _persist_outcome(
-        publication_id=row.publication_id,
-        user_id=user_id,
-        outcome=outcome,
-    )
-
-
-async def _run_resolution_task(
-    *,
-    user_id: int,
-    request_email: str | None,
-    rows: list[PublicationListItem],
-) -> None:
-    for row in rows:
-        await _resolve_publication_row(
-            user_id=user_id,
-            request_email=request_email,
-            row=row,
-        )
-
-
-def _schedule_rows(
-    *,
-    user_id: int,
-    request_email: str | None,
-    rows: list[PublicationListItem],
-) -> None:
-    if not rows:
-        return
-    task = asyncio.create_task(
-        _run_resolution_task(
-            user_id=user_id,
-            request_email=request_email,
-            rows=rows,
-        )
-    )
-    _register_task(task)
-    task.add_done_callback(_drop_finished_task)
-
-
-async def enqueue_missing_pdf_jobs(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    request_email: str | None,
-    rows: list[PublicationListItem],
-    max_items: int,
-) -> list[int]:
-    queueable = _queueable_rows(rows, max_items=max_items)
-    queued_rows = await _enqueue_rows(
-        db_session,
-        user_id=user_id,
-        rows=queueable,
-        force_retry=False,
-    )
-    _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
-    return [row.publication_id for row in queued_rows]
-
-
-async def enqueue_retry_pdf_job(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    request_email: str | None,
-    row: PublicationListItem,
-) -> bool:
-    queued_rows = await _enqueue_rows(
-        db_session,
-        user_id=user_id,
-        rows=[row],
-        force_retry=True,
-    )
-    _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
-    return bool(queued_rows)
-
-
-def _retry_item_label(display_name: str | None, scholar_id: str | None) -> str:
-    return str(display_name or scholar_id or "unknown")
-
-
-def _retry_item_from_publication(
-    publication: Publication,
-    *,
-    link_row: tuple | None,
-) -> PublicationListItem:
-    if link_row is None:
-        scholar_profile_id = 0
-        scholar_label = "unknown"
-        is_read = True
-        first_seen_at = publication.created_at or _utcnow()
-    else:
-        scholar_profile_id = int(link_row[0])
-        scholar_label = _retry_item_label(link_row[1], link_row[2])
-        is_read = bool(link_row[3])
-        first_seen_at = link_row[4] or publication.created_at or _utcnow()
-    return PublicationListItem(
-        publication_id=int(publication.id),
-        scholar_profile_id=scholar_profile_id,
-        scholar_label=scholar_label,
-        title=publication.title_raw,
-        year=publication.year,
-        citation_count=int(publication.citation_count or 0),
-        venue_text=publication.venue_text,
-        pub_url=publication.pub_url,
-        pdf_url=publication.pdf_url,
-        is_read=is_read,
-        first_seen_at=first_seen_at,
-        is_new_in_latest_run=False,
-    )
-
-
-async def _retry_item_for_publication_id(
-    db_session: AsyncSession,
-    *,
-    publication_id: int,
-) -> PublicationListItem | None:
-    publication = await db_session.get(Publication, publication_id)
-    if publication is None:
-        return None
-    result = await db_session.execute(
-        select(
-            ScholarProfile.id,
-            ScholarProfile.display_name,
-            ScholarProfile.scholar_id,
-            ScholarPublication.is_read,
-            ScholarPublication.created_at,
-        )
-        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
-        .where(ScholarPublication.publication_id == publication_id)
-        .order_by(ScholarPublication.created_at.asc())
-        .limit(1)
-    )
-    return _retry_item_from_publication(publication, link_row=result.one_or_none())
-
-
-async def enqueue_retry_pdf_job_for_publication_id(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    request_email: str | None,
-    publication_id: int,
-) -> PdfRequeueResult:
-    row = await _retry_item_for_publication_id(
-        db_session,
-        publication_id=publication_id,
-    )
-    if row is None:
-        return PdfRequeueResult(publication_exists=False, queued=False)
-    queued = await enqueue_retry_pdf_job(
-        db_session,
-        user_id=user_id,
-        request_email=request_email,
-        row=row,
-    )
-    return PdfRequeueResult(publication_exists=True, queued=queued)
-
-
-def _queue_candidate_from_publication(publication: Publication) -> PublicationListItem:
-    return PublicationListItem(
-        publication_id=int(publication.id),
-        scholar_profile_id=0,
-        scholar_label="admin",
-        title=publication.title_raw,
-        year=publication.year,
-        citation_count=int(publication.citation_count or 0),
-        venue_text=publication.venue_text,
-        pub_url=publication.pub_url,
-        pdf_url=publication.pdf_url,
-        is_read=True,
-        first_seen_at=publication.created_at or _utcnow(),
-        is_new_in_latest_run=False,
-    )
-
-
-async def _missing_pdf_candidates(
-    db_session: AsyncSession,
-    *,
-    limit: int,
-) -> list[PublicationListItem]:
-    bounded_limit = max(1, min(int(limit), 5000))
-    result = await db_session.execute(
-        select(Publication)
-        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
-        .where(Publication.pdf_url.is_(None))
-        .where(
-            or_(
-                PublicationPdfJob.publication_id.is_(None),
-                PublicationPdfJob.status.notin_([PDF_STATUS_QUEUED, PDF_STATUS_RUNNING]),
-            )
-        )
-        .order_by(Publication.updated_at.desc(), Publication.id.desc())
-        .limit(bounded_limit)
-    )
-    return [
-        _queue_candidate_from_publication(publication)
-        for publication in result.scalars()
-    ]
-
-
-async def enqueue_all_missing_pdf_jobs(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    request_email: str | None,
-    limit: int = 1000,
-) -> PdfBulkQueueResult:
-    candidates = await _missing_pdf_candidates(db_session, limit=limit)
-    queued_rows = await _enqueue_rows(
-        db_session,
-        user_id=user_id,
-        rows=candidates,
-        force_retry=True,
-    )
-    _schedule_rows(user_id=user_id, request_email=request_email, rows=queued_rows)
-    return PdfBulkQueueResult(
-        requested_count=len(candidates),
-        queued_count=len(queued_rows),
-    )
-
-
-def _tracked_queue_select_base(*, status: str | None) -> Select[tuple]:
-    stmt = (
-        select(
-            PublicationPdfJob.publication_id,
-            Publication.title_raw,
-            Publication.pdf_url,
-            PublicationPdfJob.status,
-            PublicationPdfJob.attempt_count,
-            PublicationPdfJob.last_failure_reason,
-            PublicationPdfJob.last_failure_detail,
-            PublicationPdfJob.last_source,
-            PublicationPdfJob.last_requested_by_user_id,
-            User.email,
-            PublicationPdfJob.queued_at,
-            PublicationPdfJob.last_attempt_at,
-            PublicationPdfJob.resolved_at,
-            PublicationPdfJob.updated_at,
-        )
-        .join(Publication, Publication.id == PublicationPdfJob.publication_id)
-        .outerjoin(User, User.id == PublicationPdfJob.last_requested_by_user_id)
-    )
-    if status:
-        stmt = stmt.where(PublicationPdfJob.status == status)
-    return stmt
-
-
-def _tracked_queue_select(*, limit: int, offset: int, status: str | None) -> Select[tuple]:
-    return (
-        _tracked_queue_select_base(status=status)
-        .order_by(PublicationPdfJob.updated_at.desc())
-        .limit(_bounded_limit(limit))
-        .offset(_bounded_offset(offset))
-    )
-
-
-def _untracked_queue_select_base() -> Select[tuple]:
-    return (
-        select(
-            Publication.id,
-            Publication.title_raw,
-            Publication.pdf_url,
-            literal(PDF_STATUS_UNTRACKED),
-            literal(0),
-            literal(None),
-            literal(None),
-            literal(None),
-            literal(None),
-            literal(None),
-            literal(None),
-            literal(None),
-            literal(None),
-            Publication.updated_at,
-        )
-        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
-        .where(Publication.pdf_url.is_(None))
-        .where(PublicationPdfJob.publication_id.is_(None))
-    )
-
-
-def _untracked_queue_select(*, limit: int, offset: int) -> Select[tuple]:
-    return (
-        _untracked_queue_select_base()
-        .order_by(Publication.updated_at.desc(), Publication.id.desc())
-        .limit(_bounded_limit(limit))
-        .offset(_bounded_offset(offset))
-    )
-
-
-def _all_queue_select(*, limit: int, offset: int) -> Select[tuple]:
-    union_stmt = union_all(
-        _tracked_queue_select_base(status=None),
-        _untracked_queue_select_base(),
-    ).subquery()
-    return (
-        select(union_stmt)
-        .order_by(union_stmt.c.updated_at.desc())
-        .limit(_bounded_limit(limit))
-        .offset(_bounded_offset(offset))
-    )
-
-
-def _tracked_queue_count_select(*, status: str | None) -> Select[tuple]:
-    stmt = select(func.count()).select_from(PublicationPdfJob)
-    if status:
-        stmt = stmt.where(PublicationPdfJob.status == status)
-    return stmt
-
-
-def _untracked_queue_count_select() -> Select[tuple]:
-    return (
-        select(func.count())
-        .select_from(Publication)
-        .outerjoin(PublicationPdfJob, PublicationPdfJob.publication_id == Publication.id)
-        .where(Publication.pdf_url.is_(None))
-        .where(PublicationPdfJob.publication_id.is_(None))
-    )
-
-
-def _queue_item_from_row(row: tuple) -> PdfQueueListItem:
-    return PdfQueueListItem(
-        publication_id=int(row[0]),
-        title=str(row[1] or ""),
-        pdf_url=row[2],
-        status=str(row[3] or PDF_STATUS_UNTRACKED),
-        attempt_count=int(row[4] or 0),
-        last_failure_reason=row[5],
-        last_failure_detail=row[6],
-        last_source=row[7],
-        requested_by_user_id=int(row[8]) if row[8] is not None else None,
-        requested_by_email=row[9],
-        queued_at=row[10],
-        last_attempt_at=row[11],
-        resolved_at=row[12],
-        updated_at=row[13],
-    )
-
-
-async def _hydrated_queue_items(
-    db_session: AsyncSession,
-    *,
-    rows: list[tuple],
-) -> list[PdfQueueListItem]:
-    items = [_queue_item_from_row(row) for row in rows]
-    return await identifier_service.overlay_pdf_queue_items_with_display_identifiers(
-        db_session,
-        items=items,
-    )
-
-
-async def list_pdf_queue_items(
-    db_session: AsyncSession,
-    *,
-    limit: int = 100,
-    offset: int = 0,
-    status: str | None = None,
-) -> list[PdfQueueListItem]:
-    bounded_limit = _bounded_limit(limit)
-    bounded_offset = _bounded_offset(offset)
-    normalized_status = (status or "").strip().lower() or None
-    if normalized_status == PDF_STATUS_UNTRACKED:
-        result = await db_session.execute(
-            _untracked_queue_select(
-                limit=bounded_limit,
-                offset=bounded_offset,
-            )
-        )
-        return await _hydrated_queue_items(db_session, rows=list(result.all()))
-    if normalized_status is None:
-        result = await db_session.execute(
-            _all_queue_select(
-                limit=bounded_limit,
-                offset=bounded_offset,
-            )
-        )
-        return await _hydrated_queue_items(db_session, rows=list(result.all()))
-    result = await db_session.execute(
-        _tracked_queue_select(
-            limit=bounded_limit,
-            offset=bounded_offset,
-            status=normalized_status,
-        )
-    )
-    return await _hydrated_queue_items(db_session, rows=list(result.all()))
-
-
-async def count_pdf_queue_items(
-    db_session: AsyncSession,
-    *,
-    status: str | None = None,
-) -> int:
-    normalized_status = (status or "").strip().lower() or None
-    if normalized_status == PDF_STATUS_UNTRACKED:
-        result = await db_session.execute(_untracked_queue_count_select())
-        return int(result.scalar_one() or 0)
-    tracked_result = await db_session.execute(
-        _tracked_queue_count_select(status=normalized_status)
-    )
-    tracked_count = int(tracked_result.scalar_one() or 0)
-    if normalized_status is not None:
-        return tracked_count
-    untracked_result = await db_session.execute(_untracked_queue_count_select())
-    untracked_count = int(untracked_result.scalar_one() or 0)
-    return tracked_count + untracked_count
-
-
-async def list_pdf_queue_page(
-    db_session: AsyncSession,
-    *,
-    limit: int = 100,
-    offset: int = 0,
-    status: str | None = None,
-) -> PdfQueuePage:
-    bounded_limit = _bounded_limit(limit)
-    bounded_offset = _bounded_offset(offset)
-    items = await list_pdf_queue_items(
-        db_session,
-        limit=bounded_limit,
-        offset=bounded_offset,
-        status=status,
-    )
-    total_count = await count_pdf_queue_items(
-        db_session,
-        status=status,
-    )
-    return PdfQueuePage(
-        items=items,
-        total_count=total_count,
-        limit=bounded_limit,
-        offset=bounded_offset,
-    )
-
-
-async def drain_ready_jobs(
-    db_session: AsyncSession,
-    *,
-    limit: int,
-    max_attempts: int,
-) -> int:
-    result = await db_session.execute(
-        select(User.id).where(User.is_active.is_(True)).order_by(User.id.asc()).limit(1)
-    )
-    system_user_id = result.scalar_one_or_none()
-    if system_user_id is None:
-        return 0
-
-    bulk_result = await enqueue_all_missing_pdf_jobs(
-        db_session,
-        user_id=system_user_id,
-        request_email=settings.unpaywall_email,
-        limit=limit,
-    )
-    return bulk_result.queued_count

build/lib/app/services/domains/publications/pdf_resolution_pipeline.py

@@ -1,108 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-import logging
-from typing import Any
-
-from app.services.domains.publications.types import PublicationListItem
-from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
-from app.settings import settings
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass(frozen=True)
-class PipelineOutcome:
-    outcome: OaResolutionOutcome | None
-    scholar_candidates: Any | None  # Kept for backward compatibility with calling signatures
-
-
-async def resolve_publication_pdf_outcome_for_row(
-    *,
-    row: PublicationListItem,
-    request_email: str | None,
-) -> PipelineOutcome:
-    # 1. OpenAlex OA
-    openalex_outcome = await _openalex_outcome(row, request_email=request_email)
-    if openalex_outcome and openalex_outcome.pdf_url:
-        return PipelineOutcome(openalex_outcome, None)
-
-    # 2. arXiv
-    arxiv_outcome = await _arxiv_outcome(row, request_email=request_email)
-    if arxiv_outcome and arxiv_outcome.pdf_url:
-        return PipelineOutcome(arxiv_outcome, None)
-
-    # 3. Unpaywall (which falls back to Crossref)
-    oa_outcome = await _oa_outcome(row=row, request_email=request_email)
-    return PipelineOutcome(oa_outcome, None)
-
-
-async def _openalex_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None:
-    from app.services.domains.openalex.client import OpenAlexClient
-    from app.services.domains.openalex.matching import find_best_match
-
-    if not row.title:
-        return None
-
-    import re
-    safe_title = re.sub(r"[^\w\s]", " ", row.title)
-    safe_title = " ".join(safe_title.split())
-    if not safe_title:
-        return None
-
-    client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=request_email or settings.crossref_api_mailto)
-    try:
-        openalex_works = await client.get_works_by_filter({"title.search": safe_title}, limit=5)
-        match = find_best_match(
-            target_title=row.title,
-            target_year=row.year,
-            target_authors=row.scholar_label,
-            candidates=openalex_works,
-        )
-        if match and match.oa_url:
-            return OaResolutionOutcome(
-                publication_id=row.publication_id,
-                doi=match.doi,
-                pdf_url=match.oa_url,
-                failure_reason=None,
-                source="openalex",
-                used_crossref=False,
-            )
-    except Exception as exc:
-        logger.warning(
-            "publications.pdf_resolution.openalex_failed",
-            extra={"event": "publications.pdf_resolution.openalex_failed", "error": str(exc)},
-        )
-    return None
-
-
-async def _arxiv_outcome(row: PublicationListItem, request_email: str | None) -> OaResolutionOutcome | None:
-    from app.services.domains.arxiv.application import discover_arxiv_id_for_publication
-
-    try:
-        arxiv_id = await discover_arxiv_id_for_publication(item=row, request_email=request_email)
-        if arxiv_id:
-            pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
-            return OaResolutionOutcome(
-                publication_id=row.publication_id,
-                doi=None,
-                pdf_url=pdf_url,
-                failure_reason=None,
-                source="arxiv",
-                used_crossref=False,
-            )
-    except Exception as exc:
-        logger.warning(
-            "publications.pdf_resolution.arxiv_failed",
-            extra={"event": "publications.pdf_resolution.arxiv_failed", "error": str(exc)},
-        )
-    return None
-
-
-async def _oa_outcome(
-    *,
-    row: PublicationListItem,
-    request_email: str | None,
-) -> OaResolutionOutcome | None:
-    outcomes = await resolve_publication_oa_outcomes([row], request_email=request_email)
-    return outcomes.get(row.publication_id)

build/lib/app/services/domains/publications/queries.py

@@ -1,203 +0,0 @@
-from __future__ import annotations
-
-from sqlalchemy import Select, func, select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import CrawlRun, Publication, RunStatus, ScholarProfile, ScholarPublication
-from app.services.domains.publications.modes import MODE_LATEST, MODE_UNREAD
-from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
-
-
-def _normalized_citation_count(value: object) -> int:
-    try:
-        return int(value or 0)
-    except (TypeError, ValueError):
-        return 0
-
-
-async def get_latest_completed_run_id_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-) -> int | None:
-    result = await db_session.execute(
-        select(func.max(CrawlRun.id)).where(
-            CrawlRun.user_id == user_id,
-            CrawlRun.status != RunStatus.RUNNING,
-        )
-    )
-    latest_run_id = result.scalar_one_or_none()
-    return int(latest_run_id) if latest_run_id is not None else None
-
-
-def publications_query(
-    *,
-    user_id: int,
-    mode: str,
-    latest_run_id: int | None,
-    scholar_profile_id: int | None,
-    favorite_only: bool,
-    limit: int,
-    offset: int = 0,
-) -> Select[tuple]:
-    scholar_label = ScholarProfile.display_name
-    stmt = (
-        select(
-            Publication.id,
-            ScholarProfile.id,
-            scholar_label,
-            ScholarProfile.scholar_id,
-            Publication.title_raw,
-            Publication.year,
-            Publication.citation_count,
-            Publication.venue_text,
-            Publication.pub_url,
-            Publication.pdf_url,
-            ScholarPublication.is_read,
-            ScholarPublication.is_favorite,
-            ScholarPublication.first_seen_run_id,
-            ScholarPublication.created_at,
-        )
-        .join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
-        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
-        .where(ScholarProfile.user_id == user_id)
-        .order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
-        .offset(max(int(offset), 0))
-        .limit(limit)
-    )
-    if scholar_profile_id is not None:
-        stmt = stmt.where(ScholarProfile.id == scholar_profile_id)
-    if favorite_only:
-        stmt = stmt.where(ScholarPublication.is_favorite.is_(True))
-    if mode == MODE_UNREAD:
-        stmt = stmt.where(ScholarPublication.is_read.is_(False))
-    if mode == MODE_LATEST:
-        if latest_run_id is None:
-            return stmt.where(False)
-        stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id)
-    return stmt
-
-
-def publication_query_for_user(
-    *,
-    user_id: int,
-    scholar_profile_id: int,
-    publication_id: int,
-) -> Select[tuple]:
-    return (
-        select(
-            Publication.id,
-            ScholarProfile.id,
-            ScholarProfile.display_name,
-            ScholarProfile.scholar_id,
-            Publication.title_raw,
-            Publication.year,
-            Publication.citation_count,
-            Publication.venue_text,
-            Publication.pub_url,
-            Publication.pdf_url,
-            ScholarPublication.is_read,
-            ScholarPublication.is_favorite,
-            ScholarPublication.first_seen_run_id,
-            ScholarPublication.created_at,
-        )
-        .join(ScholarPublication, ScholarPublication.publication_id == Publication.id)
-        .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id)
-        .where(
-            ScholarProfile.user_id == user_id,
-            ScholarProfile.id == scholar_profile_id,
-            Publication.id == publication_id,
-        )
-        .limit(1)
-    )
-
-
-async def get_publication_item_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholar_profile_id: int,
-    publication_id: int,
-) -> PublicationListItem | None:
-    latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id)
-    result = await db_session.execute(
-        publication_query_for_user(
-            user_id=user_id,
-            scholar_profile_id=scholar_profile_id,
-            publication_id=publication_id,
-        )
-    )
-    row = result.one_or_none()
-    if row is None:
-        return None
-    return publication_list_item_from_row(row, latest_run_id=latest_run_id)
-
-
-def publication_list_item_from_row(
-    row: tuple,
-    *,
-    latest_run_id: int | None,
-) -> PublicationListItem:
-    (
-        publication_id,
-        scholar_profile_id,
-        display_name,
-        scholar_id,
-        title_raw,
-        year,
-        citation_count,
-        venue_text,
-        pub_url,
-        pdf_url,
-        is_read,
-        is_favorite,
-        first_seen_run_id,
-        created_at,
-    ) = row
-    return PublicationListItem(
-        publication_id=int(publication_id),
-        scholar_profile_id=int(scholar_profile_id),
-        scholar_label=(display_name or scholar_id),
-        title=title_raw,
-        year=year,
-        citation_count=_normalized_citation_count(citation_count),
-        venue_text=venue_text,
-        pub_url=pub_url,
-        pdf_url=pdf_url,
-        is_read=bool(is_read),
-        is_favorite=bool(is_favorite),
-        first_seen_at=created_at,
-        is_new_in_latest_run=(
-            latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
-        ),
-    )
-
-
-def unread_item_from_row(row: tuple) -> UnreadPublicationItem:
-    (
-        publication_id,
-        scholar_profile_id,
-        display_name,
-        scholar_id,
-        title_raw,
-        year,
-        citation_count,
-        venue_text,
-        pub_url,
-        pdf_url,
-        _is_read,
-        _is_favorite,
-        _first_seen_run_id,
-        _created_at,
-    ) = row
-    return UnreadPublicationItem(
-        publication_id=int(publication_id),
-        scholar_profile_id=int(scholar_profile_id),
-        scholar_label=(display_name or scholar_id),
-        title=title_raw,
-        year=year,
-        citation_count=_normalized_citation_count(citation_count),
-        venue_text=venue_text,
-        pub_url=pub_url,
-        pdf_url=pdf_url,
-    )

build/lib/app/services/domains/publications/read_state.py

@@ -1,94 +0,0 @@
-from __future__ import annotations
-
-from sqlalchemy import select, tuple_, update
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import ScholarProfile, ScholarPublication
-
-
-def _normalized_selection_pairs(selections: list[tuple[int, int]]) -> set[tuple[int, int]]:
-    pairs: set[tuple[int, int]] = set()
-    for scholar_profile_id, publication_id in selections:
-        normalized = (int(scholar_profile_id), int(publication_id))
-        if normalized[0] <= 0 or normalized[1] <= 0:
-            continue
-        pairs.add(normalized)
-    return pairs
-
-
-def _scoped_scholar_ids_query(*, user_id: int):
-    return (
-        select(ScholarProfile.id)
-        .where(ScholarProfile.user_id == user_id)
-        .scalar_subquery()
-    )
-
-
-async def mark_all_unread_as_read_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-) -> int:
-    scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
-    stmt = (
-        update(ScholarPublication)
-        .where(
-            ScholarPublication.scholar_profile_id.in_(scholar_ids),
-            ScholarPublication.is_read.is_(False),
-        )
-        .values(is_read=True)
-    )
-    result = await db_session.execute(stmt)
-    await db_session.commit()
-    return int(result.rowcount or 0)
-
-
-async def mark_selected_as_read_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    selections: list[tuple[int, int]],
-) -> int:
-    normalized_pairs = _normalized_selection_pairs(selections)
-    if not normalized_pairs:
-        return 0
-
-    scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
-    stmt = (
-        update(ScholarPublication)
-        .where(
-            ScholarPublication.scholar_profile_id.in_(scholar_ids),
-            tuple_(
-                ScholarPublication.scholar_profile_id,
-                ScholarPublication.publication_id,
-            ).in_(list(normalized_pairs)),
-            ScholarPublication.is_read.is_(False),
-        )
-        .values(is_read=True)
-    )
-    result = await db_session.execute(stmt)
-    await db_session.commit()
-    return int(result.rowcount or 0)
-
-
-async def set_publication_favorite_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    scholar_profile_id: int,
-    publication_id: int,
-    is_favorite: bool,
-) -> int:
-    scholar_ids = _scoped_scholar_ids_query(user_id=user_id)
-    stmt = (
-        update(ScholarPublication)
-        .where(
-            ScholarPublication.scholar_profile_id.in_(scholar_ids),
-            ScholarPublication.scholar_profile_id == int(scholar_profile_id),
-            ScholarPublication.publication_id == int(publication_id),
-        )
-        .values(is_favorite=bool(is_favorite))
-    )
-    result = await db_session.execute(stmt)
-    await db_session.commit()
-    return int(result.rowcount or 0)

build/lib/app/services/domains/publications/types.py

@@ -1,41 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from datetime import datetime
-
-from app.services.domains.publication_identifiers.types import DisplayIdentifier
-
-
-@dataclass(frozen=True)
-class PublicationListItem:
-    publication_id: int
-    scholar_profile_id: int
-    scholar_label: str
-    title: str
-    year: int | None
-    citation_count: int
-    venue_text: str | None
-    pub_url: str | None
-    pdf_url: str | None
-    is_read: bool
-    first_seen_at: datetime
-    is_new_in_latest_run: bool
-    is_favorite: bool = False
-    pdf_status: str = "untracked"
-    pdf_attempt_count: int = 0
-    pdf_failure_reason: str | None = None
-    pdf_failure_detail: str | None = None
-    display_identifier: DisplayIdentifier | None = None
-
-
-@dataclass(frozen=True)
-class UnreadPublicationItem:
-    publication_id: int
-    scholar_profile_id: int
-    scholar_label: str
-    title: str
-    year: int | None
-    citation_count: int
-    venue_text: str | None
-    pub_url: str | None
-    pdf_url: str | None

build/lib/app/services/domains/runs/__init__.py

@@ -1 +0,0 @@
-from app.services.domains.runs.application import *

build/lib/app/services/domains/runs/application.py

@@ -1,45 +0,0 @@
-from __future__ import annotations
-
-from app.services.domains.runs.queue_service import (
-    clear_queue_item_for_user,
-    drop_queue_item_for_user,
-    get_queue_item_for_user,
-    list_queue_items_for_user,
-    queue_status_counts_for_user,
-    retry_queue_item_for_user,
-)
-from app.services.domains.runs.runs_service import (
-    get_manual_run_by_idempotency_key,
-    get_run_for_user,
-    list_recent_runs_for_user,
-    list_runs_for_user,
-)
-from app.services.domains.runs.summary import extract_run_summary
-from app.services.domains.runs.types import (
-    QUEUE_STATUS_DROPPED,
-    QUEUE_STATUS_QUEUED,
-    QUEUE_STATUS_RETRYING,
-    QueueClearResult,
-    QueueListItem,
-    QueueTransitionError,
-)
-
-__all__ = [
-    "QUEUE_STATUS_QUEUED",
-    "QUEUE_STATUS_RETRYING",
-    "QUEUE_STATUS_DROPPED",
-    "QueueListItem",
-    "QueueClearResult",
-    "QueueTransitionError",
-    "extract_run_summary",
-    "list_recent_runs_for_user",
-    "list_runs_for_user",
-    "get_run_for_user",
-    "get_manual_run_by_idempotency_key",
-    "list_queue_items_for_user",
-    "get_queue_item_for_user",
-    "retry_queue_item_for_user",
-    "drop_queue_item_for_user",
-    "clear_queue_item_for_user",
-    "queue_status_counts_for_user",
-]

build/lib/app/services/domains/runs/events.py

@@ -1,59 +0,0 @@
-import asyncio
-import json
-import logging
-from typing import Any, AsyncGenerator, Dict, Set
-
-logger = logging.getLogger(__name__)
-
-class RunEventPublisher:
-    def __init__(self) -> None:
-        # Maps run_id to a set of subscriber queues
-        self._subscribers: Dict[int, Set[asyncio.Queue]] = {}
-
-    def subscribe(self, run_id: int) -> asyncio.Queue:
-        if run_id not in self._subscribers:
-            self._subscribers[run_id] = set()
-        queue: asyncio.Queue[Any] = asyncio.Queue()
-        self._subscribers[run_id].add(queue)
-        logger.debug(f"New subscriber for run {run_id}. Total: {len(self._subscribers[run_id])}")
-        return queue
-
-    def unsubscribe(self, run_id: int, queue: asyncio.Queue) -> None:
-        if run_id in self._subscribers:
-            self._subscribers[run_id].discard(queue)
-            if not self._subscribers[run_id]:
-                self._subscribers.pop(run_id, None)
-
-    async def publish(self, run_id: int, event_type: str, data: dict[str, Any]) -> None:
-        if run_id not in self._subscribers:
-            return
-        
-        message = {
-            "type": event_type,
-            "data": data
-        }
-        
-        # Fan-out to all active subscribers for this run
-        for queue in list(self._subscribers[run_id]):
-            try:
-                queue.put_nowait(message)
-            except asyncio.QueueFull:
-                logger.warning(f"Subscriber queue full for run {run_id}, dropping message")
-
-run_events = RunEventPublisher()
-
-async def event_generator(run_id: int) -> AsyncGenerator[str, None]:
-    queue = run_events.subscribe(run_id)
-    try:
-        while True:
-            # Wait for a new event
-            message = await queue.get()
-            # Server-Sent Events format: "event: <type>\ndata: <json>\n\n"
-            event_type = message["type"]
-            data_str = json.dumps(message["data"])
-            yield f"event: {event_type}\ndata: {data_str}\n\n"
-    except asyncio.CancelledError:
-        logger.debug(f"Client disconnected from SSE stream for run {run_id}")
-        raise
-    finally:
-        run_events.unsubscribe(run_id, queue)

build/lib/app/services/domains/runs/queue_queries.py

@@ -1,70 +0,0 @@
-from __future__ import annotations
-
-from sqlalchemy import and_, select
-
-from app.db.models import IngestionQueueItem, ScholarProfile
-from app.services.domains.runs.types import QueueListItem
-
-
-def queue_item_columns() -> tuple:
-    return (
-        IngestionQueueItem.id,
-        IngestionQueueItem.scholar_profile_id,
-        ScholarProfile.display_name,
-        ScholarProfile.scholar_id,
-        IngestionQueueItem.status,
-        IngestionQueueItem.reason,
-        IngestionQueueItem.dropped_reason,
-        IngestionQueueItem.attempt_count,
-        IngestionQueueItem.resume_cstart,
-        IngestionQueueItem.next_attempt_dt,
-        IngestionQueueItem.updated_at,
-        IngestionQueueItem.last_error,
-        IngestionQueueItem.last_run_id,
-    )
-
-
-def queue_item_select(*, user_id: int):
-    return (
-        select(*queue_item_columns())
-        .join(
-            ScholarProfile,
-            and_(
-                ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
-                ScholarProfile.user_id == IngestionQueueItem.user_id,
-            ),
-        )
-        .where(IngestionQueueItem.user_id == user_id)
-    )
-
-
-def queue_list_item_from_row(row: tuple) -> QueueListItem:
-    (
-        item_id,
-        scholar_profile_id,
-        display_name,
-        scholar_id,
-        status,
-        reason,
-        dropped_reason,
-        attempt_count,
-        resume_cstart,
-        next_attempt_dt,
-        updated_at,
-        last_error,
-        last_run_id,
-    ) = row
-    return QueueListItem(
-        id=int(item_id),
-        scholar_profile_id=int(scholar_profile_id),
-        scholar_label=(display_name or scholar_id),
-        status=str(status),
-        reason=str(reason),
-        dropped_reason=dropped_reason,
-        attempt_count=int(attempt_count or 0),
-        resume_cstart=int(resume_cstart or 0),
-        next_attempt_dt=next_attempt_dt,
-        updated_at=updated_at,
-        last_error=last_error,
-        last_run_id=int(last_run_id) if last_run_id is not None else None,
-    )

build/lib/app/services/domains/runs/queue_service.py

@@ -1,182 +0,0 @@
-from __future__ import annotations
-
-from sqlalchemy import case, func, select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import IngestionQueueItem
-from app.services.domains.ingestion import queue as queue_mutations
-from app.services.domains.runs.queue_queries import queue_item_select, queue_list_item_from_row
-from app.services.domains.runs.types import (
-    QUEUE_STATUS_DROPPED,
-    QUEUE_STATUS_QUEUED,
-    QUEUE_STATUS_RETRYING,
-    QueueClearResult,
-    QueueListItem,
-    QueueTransitionError,
-)
-
-
-async def list_queue_items_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    limit: int = 200,
-) -> list[QueueListItem]:
-    result = await db_session.execute(
-        queue_item_select(user_id=user_id)
-        .order_by(
-            case((IngestionQueueItem.status == QUEUE_STATUS_DROPPED, 1), else_=0).asc(),
-            IngestionQueueItem.next_attempt_dt.asc(),
-            IngestionQueueItem.id.asc(),
-        )
-        .limit(limit)
-    )
-    return [queue_list_item_from_row(row) for row in result.all()]
-
-
-async def get_queue_item_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    queue_item_id: int,
-) -> QueueListItem | None:
-    result = await db_session.execute(
-        queue_item_select(user_id=user_id)
-        .where(IngestionQueueItem.id == queue_item_id)
-        .limit(1)
-    )
-    row = result.one_or_none()
-    if row is None:
-        return None
-    return queue_list_item_from_row(row)
-
-
-async def retry_queue_item_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    queue_item_id: int,
-) -> QueueListItem | None:
-    result = await db_session.execute(
-        select(IngestionQueueItem).where(
-            IngestionQueueItem.id == queue_item_id,
-            IngestionQueueItem.user_id == user_id,
-        )
-    )
-    item = result.scalar_one_or_none()
-    if item is None:
-        return None
-    if item.status == QUEUE_STATUS_QUEUED:
-        raise QueueTransitionError(
-            code="queue_item_already_queued",
-            message="Queue item is already queued.",
-            current_status=item.status,
-        )
-    if item.status == QUEUE_STATUS_RETRYING:
-        raise QueueTransitionError(
-            code="queue_item_retrying",
-            message="Queue item is currently retrying.",
-            current_status=item.status,
-        )
-
-    await queue_mutations.mark_queued_now(
-        db_session,
-        job_id=item.id,
-        reason="manual_retry",
-        reset_attempt_count=(item.status == QUEUE_STATUS_DROPPED),
-    )
-    await db_session.commit()
-    return await get_queue_item_for_user(
-        db_session,
-        user_id=user_id,
-        queue_item_id=queue_item_id,
-    )
-
-
-async def drop_queue_item_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    queue_item_id: int,
-) -> QueueListItem | None:
-    result = await db_session.execute(
-        select(IngestionQueueItem).where(
-            IngestionQueueItem.id == queue_item_id,
-            IngestionQueueItem.user_id == user_id,
-        )
-    )
-    item = result.scalar_one_or_none()
-    if item is None:
-        return None
-    if item.status == QUEUE_STATUS_DROPPED:
-        raise QueueTransitionError(
-            code="queue_item_already_dropped",
-            message="Queue item is already dropped.",
-            current_status=item.status,
-        )
-
-    await queue_mutations.mark_dropped(
-        db_session,
-        job_id=int(item.id),
-        reason="manual_drop",
-    )
-    await db_session.commit()
-    return await get_queue_item_for_user(
-        db_session,
-        user_id=user_id,
-        queue_item_id=queue_item_id,
-    )
-
-
-async def clear_queue_item_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    queue_item_id: int,
-) -> QueueClearResult | None:
-    result = await db_session.execute(
-        select(IngestionQueueItem).where(
-            IngestionQueueItem.id == queue_item_id,
-            IngestionQueueItem.user_id == user_id,
-        )
-    )
-    item = result.scalar_one_or_none()
-    if item is None:
-        return None
-    if item.status != QUEUE_STATUS_DROPPED:
-        raise QueueTransitionError(
-            code="queue_item_not_dropped",
-            message="Queue item can only be cleared after it is dropped.",
-            current_status=item.status,
-        )
-
-    item_id = int(item.id)
-    previous_status = str(item.status)
-    deleted = await queue_mutations.delete_job_by_id(db_session, job_id=item_id)
-    await db_session.commit()
-    if not deleted:
-        return None
-    return QueueClearResult(queue_item_id=item_id, previous_status=previous_status)
-
-
-async def queue_status_counts_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-) -> dict[str, int]:
-    result = await db_session.execute(
-        select(
-            IngestionQueueItem.status,
-            func.count(IngestionQueueItem.id),
-        )
-        .where(IngestionQueueItem.user_id == user_id)
-        .group_by(IngestionQueueItem.status)
-    )
-    counts: dict[str, int] = {
-        QUEUE_STATUS_QUEUED: 0,
-        QUEUE_STATUS_RETRYING: 0,
-        QUEUE_STATUS_DROPPED: 0,
-    }
-    for status, count in result.all():
-        counts[str(status)] = int(count or 0)
-    return counts

build/lib/app/services/domains/runs/runs_service.py

@@ -1,79 +0,0 @@
-from __future__ import annotations
-
-from sqlalchemy import or_, select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db.models import CrawlRun, RunStatus, RunTriggerType
-
-
-async def list_recent_runs_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    limit: int = 20,
-) -> list[CrawlRun]:
-    result = await db_session.execute(
-        select(CrawlRun)
-        .where(CrawlRun.user_id == user_id)
-        .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
-        .limit(limit)
-    )
-    return list(result.scalars().all())
-
-
-async def list_runs_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    limit: int = 100,
-    failed_only: bool = False,
-) -> list[CrawlRun]:
-    stmt = (
-        select(CrawlRun)
-        .where(CrawlRun.user_id == user_id)
-        .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
-        .limit(limit)
-    )
-    if failed_only:
-        stmt = stmt.where(
-            CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE])
-        )
-    result = await db_session.execute(stmt)
-    return list(result.scalars().all())
-
-
-async def get_run_for_user(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    run_id: int,
-) -> CrawlRun | None:
-    result = await db_session.execute(
-        select(CrawlRun).where(
-            CrawlRun.user_id == user_id,
-            CrawlRun.id == run_id,
-        )
-    )
-    return result.scalar_one_or_none()
-
-
-async def get_manual_run_by_idempotency_key(
-    db_session: AsyncSession,
-    *,
-    user_id: int,
-    idempotency_key: str,
-) -> CrawlRun | None:
-    result = await db_session.execute(
-        select(CrawlRun)
-        .where(
-            CrawlRun.user_id == user_id,
-            CrawlRun.trigger_type == RunTriggerType.MANUAL,
-            or_(
-                CrawlRun.idempotency_key == idempotency_key,
-                CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key,
-            ),
-        )
-        .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc())
-        .limit(1)
-    )
-    return result.scalar_one_or_none()

build/lib/app/services/domains/runs/summary.py

@@ -1,76 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-
-def _safe_int(value: object, default: int = 0) -> int:
-    try:
-        return int(value)
-    except (TypeError, ValueError):
-        return default
-
-
-def _summary_dict(error_log: object) -> dict[str, Any]:
-    if not isinstance(error_log, dict):
-        return {}
-    summary = error_log.get("summary")
-    if not isinstance(summary, dict):
-        return {}
-    return summary
-
-
-def _summary_int_dict(summary: dict[str, Any], key: str) -> dict[str, int]:
-    value = summary.get(key)
-    if not isinstance(value, dict):
-        return {}
-    return {
-        str(item_key): _safe_int(item_value, 0)
-        for item_key, item_value in value.items()
-        if isinstance(item_key, str)
-    }
-
-
-def _summary_bool_dict(summary: dict[str, Any], key: str) -> dict[str, bool]:
-    value = summary.get(key)
-    if not isinstance(value, dict):
-        return {}
-    return {
-        str(item_key): bool(item_value)
-        for item_key, item_value in value.items()
-        if isinstance(item_key, str)
-    }
-
-
-def _retry_counts(summary: dict[str, Any]) -> dict[str, int]:
-    retry_counts = summary.get("retry_counts")
-    if not isinstance(retry_counts, dict):
-        retry_counts = {}
-    return {
-        "retries_scheduled_count": _safe_int(
-            retry_counts.get("retries_scheduled_count", 0),
-            0,
-        ),
-        "scholars_with_retries_count": _safe_int(
-            retry_counts.get("scholars_with_retries_count", 0),
-            0,
-        ),
-        "retry_exhausted_count": _safe_int(
-            retry_counts.get("retry_exhausted_count", 0),
-            0,
-        ),
-    }
-
-
-def extract_run_summary(error_log: object) -> dict[str, Any]:
-    summary = _summary_dict(error_log)
-    return {
-        "succeeded_count": _safe_int(summary.get("succeeded_count", 0)),
-        "failed_count": _safe_int(summary.get("failed_count", 0)),
-        "partial_count": _safe_int(summary.get("partial_count", 0)),
-        "failed_state_counts": _summary_int_dict(summary, "failed_state_counts"),
-        "failed_reason_counts": _summary_int_dict(summary, "failed_reason_counts"),
-        "scrape_failure_counts": _summary_int_dict(summary, "scrape_failure_counts"),
-        "retry_counts": _retry_counts(summary),
-        "alert_thresholds": _summary_int_dict(summary, "alert_thresholds"),
-        "alert_flags": _summary_bool_dict(summary, "alert_flags"),
-    }

build/lib/app/services/domains/runs/types.py

@@ -1,38 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from datetime import datetime
-
-QUEUE_STATUS_QUEUED = "queued"
-QUEUE_STATUS_RETRYING = "retrying"
-QUEUE_STATUS_DROPPED = "dropped"
-
-
-@dataclass(frozen=True)
-class QueueListItem:
-    id: int
-    scholar_profile_id: int
-    scholar_label: str
-    status: str
-    reason: str
-    dropped_reason: str | None
-    attempt_count: int
-    resume_cstart: int
-    next_attempt_dt: datetime | None
-    updated_at: datetime
-    last_error: str | None
-    last_run_id: int | None
-
-
-@dataclass(frozen=True)
-class QueueClearResult:
-    queue_item_id: int
-    previous_status: str
-
-
-class QueueTransitionError(RuntimeError):
-    def __init__(self, *, code: str, message: str, current_status: str) -> None:
-        super().__init__(message)
-        self.code = code
-        self.message = message
-        self.current_status = current_status

build/lib/app/services/domains/scholar/__init__.py

@@ -1 +0,0 @@
-from __future__ import annotations

build/lib/app/services/domains/scholar/author_rows.py

@@ -1,202 +0,0 @@
-from __future__ import annotations
-
-import re
-from html.parser import HTMLParser
-from typing import Any
-from urllib.parse import parse_qs, urlparse
-
-from app.services.domains.scholar.parser_constants import AUTHOR_SEARCH_MARKER_KEYS
-from app.services.domains.scholar.parser_types import ScholarSearchCandidate
-from app.services.domains.scholar.parser_utils import (
-    attr_class,
-    attr_href,
-    attr_src,
-    build_absolute_scholar_url,
-    normalize_space,
-)
-
-
-def parse_scholar_id_from_href(href: str | None) -> str | None:
-    if not href:
-        return None
-    parsed = urlparse(href)
-    query = parse_qs(parsed.query)
-    user_values = query.get("user")
-    if not user_values:
-        return None
-    candidate = user_values[0].strip()
-    return candidate or None
-
-
-def _extract_verified_email_domain(value: str | None) -> str | None:
-    if not value:
-        return None
-    match = re.search(r"verified email at\s+(.+)$", value.strip(), re.I)
-    if not match:
-        return None
-    domain = normalize_space(match.group(1))
-    return domain or None
-
-
-class ScholarAuthorSearchParser(HTMLParser):
-    def __init__(self) -> None:
-        super().__init__(convert_charrefs=True)
-        self.candidates: list[ScholarSearchCandidate] = []
-        self._candidate: dict[str, Any] | None = None
-
-    def _begin_candidate(self) -> None:
-        self._candidate = {
-            "depth": 1,
-            "name_href": None,
-            "name_parts": [],
-            "aff_depth": 0,
-            "aff_parts": [],
-            "name_depth": 0,
-            "eml_depth": 0,
-            "eml_parts": [],
-            "cby_depth": 0,
-            "cby_parts": [],
-            "interest_depth": 0,
-            "interest_parts": [],
-            "interests": [],
-            "image_src": None,
-        }
-
-    def _increment_capture_depths(self) -> None:
-        if self._candidate is None:
-            return
-        for key in ("name_depth", "aff_depth", "eml_depth", "cby_depth", "interest_depth"):
-            if self._candidate[key] > 0:
-                self._candidate[key] += 1
-
-    def _finalize_candidate(self) -> None:
-        if self._candidate is None:
-            return
-
-        name = normalize_space("".join(self._candidate["name_parts"]))
-        scholar_id = parse_scholar_id_from_href(self._candidate["name_href"])
-        if not name or not scholar_id:
-            return
-
-        affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None
-        email_domain = _extract_verified_email_domain(
-            normalize_space("".join(self._candidate["eml_parts"])) or None
-        )
-        cited_by_text = normalize_space("".join(self._candidate["cby_parts"]))
-        cited_by_match = re.search(r"\d+", cited_by_text)
-        cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None
-
-        seen_interests: set[str] = set()
-        interests: list[str] = []
-        for interest in self._candidate["interests"]:
-            normalized = normalize_space(interest)
-            if not normalized or normalized in seen_interests:
-                continue
-            seen_interests.add(normalized)
-            interests.append(normalized)
-
-        profile_url = build_absolute_scholar_url(self._candidate["name_href"])
-        if not profile_url:
-            profile_url = (
-                "https://scholar.google.com/citations"
-                f"?hl=en&user={scholar_id}"
-            )
-
-        self.candidates.append(
-            ScholarSearchCandidate(
-                scholar_id=scholar_id,
-                display_name=name,
-                affiliation=affiliation,
-                email_domain=email_domain,
-                cited_by_count=cited_by_count,
-                interests=interests,
-                profile_url=profile_url,
-                profile_image_url=build_absolute_scholar_url(self._candidate["image_src"]),
-            )
-        )
-
-    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
-        classes = attr_class(attrs)
-
-        if self._candidate is None:
-            if tag == "div" and "gsc_1usr" in classes:
-                self._begin_candidate()
-            return
-
-        self._candidate["depth"] += 1
-        self._increment_capture_depths()
-
-        if tag == "a" and "gs_ai_name" in classes:
-            self._candidate["name_depth"] = 1
-            self._candidate["name_href"] = attr_href(attrs)
-            return
-
-        if tag == "div" and "gs_ai_aff" in classes:
-            self._candidate["aff_depth"] = 1
-            return
-
-        if tag == "div" and "gs_ai_eml" in classes:
-            self._candidate["eml_depth"] = 1
-            return
-
-        if tag == "div" and "gs_ai_cby" in classes:
-            self._candidate["cby_depth"] = 1
-            return
-
-        if tag == "a" and "gs_ai_one_int" in classes:
-            self._candidate["interest_depth"] = 1
-            self._candidate["interest_parts"] = []
-            return
-
-        if tag == "img" and self._candidate["image_src"] is None:
-            self._candidate["image_src"] = attr_src(attrs)
-
-    def handle_data(self, data: str) -> None:
-        if self._candidate is None:
-            return
-        if self._candidate["name_depth"] > 0:
-            self._candidate["name_parts"].append(data)
-        if self._candidate["aff_depth"] > 0:
-            self._candidate["aff_parts"].append(data)
-        if self._candidate["eml_depth"] > 0:
-            self._candidate["eml_parts"].append(data)
-        if self._candidate["cby_depth"] > 0:
-            self._candidate["cby_parts"].append(data)
-        if self._candidate["interest_depth"] > 0:
-            self._candidate["interest_parts"].append(data)
-
-    def _decrement_capture_depth(self, key: str) -> bool:
-        if self._candidate is None:
-            return False
-        if self._candidate[key] <= 0:
-            return False
-        self._candidate[key] -= 1
-        return self._candidate[key] == 0
-
-    def handle_endtag(self, _tag: str) -> None:
-        if self._candidate is None:
-            return
-
-        interest_closed = self._decrement_capture_depth("interest_depth")
-        self._decrement_capture_depth("name_depth")
-        self._decrement_capture_depth("aff_depth")
-        self._decrement_capture_depth("eml_depth")
-        self._decrement_capture_depth("cby_depth")
-
-        if interest_closed:
-            interest_text = normalize_space("".join(self._candidate["interest_parts"]))
-            if interest_text:
-                self._candidate["interests"].append(interest_text)
-            self._candidate["interest_parts"] = []
-
-        self._candidate["depth"] -= 1
-        if self._candidate["depth"] > 0:
-            return
-
-        self._finalize_candidate()
-        self._candidate = None
-
-
-def count_author_search_markers(html: str) -> dict[str, int]:
-    lowered = html.lower()
-    return {key: lowered.count(key.lower()) for key in AUTHOR_SEARCH_MARKER_KEYS}

build/lib/app/services/domains/scholar/parser.py

@@ -1,165 +0,0 @@
-from __future__ import annotations
-
-from app.services.domains.scholar.author_rows import (
-    ScholarAuthorSearchParser,
-    count_author_search_markers,
-    parse_scholar_id_from_href,
-)
-from app.services.domains.scholar.parser_constants import SCRIPT_STYLE_RE
-from app.services.domains.scholar.parser_types import (
-    ParseState,
-    ParsedAuthorSearchPage,
-    ParsedProfilePage,
-    PublicationCandidate,
-    ScholarDomInvariantError,
-    ScholarMalformedDataError,
-    ScholarParserError,
-    ScholarSearchCandidate,
-)
-from app.services.domains.scholar.parser_utils import (
-    strip_tags,
-)
-from app.services.domains.scholar.profile_rows import (
-    ScholarRowParser,
-    count_markers,
-    extract_articles_range,
-    extract_profile_image_url,
-    extract_profile_name,
-    extract_rows,
-    has_operation_error_banner,
-    has_show_more_button,
-    parse_citation_count,
-    parse_cluster_id_from_href,
-    parse_publications,
-    parse_year,
-)
-from app.services.domains.scholar.source import FetchResult
-from app.services.domains.scholar.state_detection import (
-    classify_block_or_captcha_reason,
-    classify_network_error_reason,
-    detect_author_search_state,
-    detect_state,
-)
-
-
-def _raise_dom_error(code: str, message: str) -> None:
-    raise ScholarDomInvariantError(code=code, message=message)
-
-
-def _assert_profile_dom_invariants(
-    *,
-    fetch_result: FetchResult,
-    marker_counts: dict[str, int],
-    publications: list[PublicationCandidate],
-    warnings: list[str],
-    has_show_more_button_flag: bool,
-    articles_range: str | None,
-) -> None:
-    if fetch_result.status_code is None:
-        return
-    final_url = (fetch_result.final_url or "").lower()
-    if "accounts.google.com" in final_url or "sorry/index" in final_url:
-        return
-    if any(code.startswith("layout_") for code in warnings):
-        reason = next(code for code in warnings if code.startswith("layout_"))
-        _raise_dom_error(reason, f"Detected layout warning: {reason}")
-    if has_show_more_button_flag and not articles_range:
-        _raise_dom_error(
-            "layout_show_more_without_articles_range",
-            "Show-more control exists without an articles range marker.",
-        )
-    if marker_counts.get("gsc_a_tr", 0) > 0 and marker_counts.get("gsc_a_at", 0) <= 0:
-        _raise_dom_error(
-            "layout_missing_publication_title_anchor",
-            "Publication rows were present but title anchors were absent.",
-        )
-    if not publications:
-        has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
-        has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
-        if not has_profile_markers and not has_table_markers:
-            _raise_dom_error("layout_markers_missing", "Expected scholar profile markers were absent.")
-    for publication in publications:
-        if not publication.title.strip():
-            raise ScholarMalformedDataError(
-                code="malformed_publication_title",
-                message="Encountered a publication candidate with an empty title.",
-            )
-        if publication.citation_count is not None and int(publication.citation_count) < 0:
-            raise ScholarMalformedDataError(
-                code="malformed_publication_negative_citations",
-                message="Encountered a publication candidate with negative citations.",
-            )
-
-
-def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
-    publications, warnings = parse_publications(fetch_result.body)
-    marker_counts = count_markers(fetch_result.body)
-    visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
-
-    show_more = has_show_more_button(fetch_result.body)
-    operation_error_banner = has_operation_error_banner(fetch_result.body)
-    articles_range = extract_articles_range(fetch_result.body)
-
-    if show_more:
-        warnings.append("possible_partial_page_show_more_present")
-    if operation_error_banner:
-        warnings.append("operation_error_banner_present")
-
-    warnings = sorted(set(warnings))
-    _assert_profile_dom_invariants(
-        fetch_result=fetch_result,
-        marker_counts=marker_counts,
-        publications=publications,
-        warnings=warnings,
-        has_show_more_button_flag=show_more,
-        articles_range=articles_range,
-    )
-
-    state, state_reason = detect_state(
-        fetch_result,
-        publications,
-        marker_counts,
-        warnings=warnings,
-        has_show_more_button_flag=show_more,
-        articles_range=articles_range,
-        visible_text=visible_text,
-    )
-
-    return ParsedProfilePage(
-        state=state,
-        state_reason=state_reason,
-        profile_name=extract_profile_name(fetch_result.body),
-        profile_image_url=extract_profile_image_url(fetch_result.body),
-        publications=publications,
-        marker_counts=marker_counts,
-        warnings=warnings,
-        has_show_more_button=show_more,
-        has_operation_error_banner=operation_error_banner,
-        articles_range=articles_range,
-    )
-
-
-def parse_author_search_page(fetch_result: FetchResult) -> ParsedAuthorSearchPage:
-    parser = ScholarAuthorSearchParser()
-    parser.feed(fetch_result.body)
-
-    marker_counts = count_author_search_markers(fetch_result.body)
-    visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower()
-    warnings: list[str] = []
-    if not parser.candidates:
-        warnings.append("no_author_candidates_detected")
-
-    state, state_reason = detect_author_search_state(
-        fetch_result,
-        parser.candidates,
-        marker_counts,
-        visible_text=visible_text,
-    )
-
-    return ParsedAuthorSearchPage(
-        state=state,
-        state_reason=state_reason,
-        candidates=parser.candidates,
-        marker_counts=marker_counts,
-        warnings=warnings,
-    )

build/lib/app/services/domains/scholar/parser_constants.py

@@ -1,87 +0,0 @@
-from __future__ import annotations
-
-import re
-
-BLOCKED_KEYWORDS = [
-    "unusual traffic",
-    "sorry/index",
-    "not a robot",
-    "our systems have detected",
-    "automated queries",
-    "recaptcha",
-    "captcha",
-]
-
-NO_RESULTS_KEYWORDS = [
-    "didn't match any articles",
-    "did not match any articles",
-    "no articles",
-    "no documents",
-]
-
-NO_AUTHOR_RESULTS_KEYWORDS = [
-    "didn't match any user profiles",
-    "did not match any user profiles",
-    "didn't match any scholars",
-    "did not match any scholars",
-    "no user profiles",
-]
-
-MARKER_KEYS = [
-    "gsc_a_tr",
-    "gsc_a_at",
-    "gsc_a_ac",
-    "gsc_a_h",
-    "gsc_a_y",
-    "gs_gray",
-    "gsc_prf_in",
-    "gsc_rsb_st",
-]
-
-AUTHOR_SEARCH_MARKER_KEYS = [
-    "gsc_1usr",
-    "gs_ai_name",
-    "gs_ai_aff",
-    "gs_ai_eml",
-    "gs_ai_cby",
-    "gs_ai_one_int",
-]
-
-NETWORK_DNS_ERROR_KEYWORDS = [
-    "temporary failure in name resolution",
-    "name or service not known",
-    "nodename nor servname provided",
-    "getaddrinfo failed",
-]
-
-NETWORK_TIMEOUT_KEYWORDS = [
-    "timed out",
-    "timeout",
-]
-
-NETWORK_TLS_ERROR_KEYWORDS = [
-    "ssl",
-    "tls",
-    "certificate verify failed",
-]
-
-TAG_RE = re.compile(r"<[^>]+>", re.S)
-SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?</\1>", re.I | re.S)
-SHOW_MORE_BUTTON_RE = re.compile(
-    r"<button\b[^>]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>",
-    re.I | re.S,
-)
-
-PROFILE_ROW_PARSER_DIRECT_MARKERS = (
-    "gs_ggs",
-    "gs_ggsd",
-    "gs_ggsa",
-    "gs_or_ggsm",
-)
-
-PROFILE_ROW_DIRECT_LABEL_TOKENS = (
-    "pdf",
-    "[pdf]",
-    "full text",
-    "download",
-)

build/lib/app/services/domains/scholar/parser_types.py

@@ -1,75 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from enum import StrEnum
-
-
-class ParseState(StrEnum):
-    OK = "ok"
-    NO_RESULTS = "no_results"
-    BLOCKED_OR_CAPTCHA = "blocked_or_captcha"
-    LAYOUT_CHANGED = "layout_changed"
-    NETWORK_ERROR = "network_error"
-
-
-class ScholarParserError(RuntimeError):
-    code: str
-
-    def __init__(self, *, code: str, message: str) -> None:
-        super().__init__(message)
-        self.code = code
-
-
-class ScholarDomInvariantError(ScholarParserError):
-    pass
-
-
-class ScholarMalformedDataError(ScholarParserError):
-    pass
-
-
-@dataclass(frozen=True)
-class PublicationCandidate:
-    title: str
-    title_url: str | None
-    cluster_id: str | None
-    year: int | None
-    citation_count: int | None
-    authors_text: str | None
-    venue_text: str | None
-    pdf_url: str | None
-
-
-@dataclass(frozen=True)
-class ScholarSearchCandidate:
-    scholar_id: str
-    display_name: str
-    affiliation: str | None
-    email_domain: str | None
-    cited_by_count: int | None
-    interests: list[str]
-    profile_url: str
-    profile_image_url: str | None
-
-
-@dataclass(frozen=True)
-class ParsedProfilePage:
-    state: ParseState
-    state_reason: str
-    profile_name: str | None
-    profile_image_url: str | None
-    publications: list[PublicationCandidate]
-    marker_counts: dict[str, int]
-    warnings: list[str]
-    has_show_more_button: bool
-    has_operation_error_banner: bool
-    articles_range: str | None
-
-
-@dataclass(frozen=True)
-class ParsedAuthorSearchPage:
-    state: ParseState
-    state_reason: str
-    candidates: list[ScholarSearchCandidate]
-    marker_counts: dict[str, int]
-    warnings: list[str]

build/lib/app/services/domains/scholar/parser_utils.py

@@ -1,42 +0,0 @@
-from __future__ import annotations
-
-from html import unescape
-
-from app.services.domains.scholar.parser_constants import TAG_RE
-
-
-def normalize_space(value: str) -> str:
-    return " ".join(unescape(value).split())
-
-
-def strip_tags(value: str) -> str:
-    return normalize_space(TAG_RE.sub(" ", value))
-
-
-def attr_class(attrs: list[tuple[str, str | None]]) -> str:
-    for name, raw_value in attrs:
-        if name.lower() == "class":
-            return raw_value or ""
-    return ""
-
-
-def attr_href(attrs: list[tuple[str, str | None]]) -> str | None:
-    for name, raw_value in attrs:
-        if name.lower() == "href":
-            return raw_value
-    return None
-
-
-def attr_src(attrs: list[tuple[str, str | None]]) -> str | None:
-    for name, raw_value in attrs:
-        if name.lower() == "src":
-            return raw_value
-    return None
-
-
-def build_absolute_scholar_url(path_or_url: str | None) -> str | None:
-    if not path_or_url:
-        return None
-    from urllib.parse import urljoin
-
-    return urljoin("https://scholar.google.com", path_or_url)

build/lib/app/services/domains/scholar/profile_rows.py

@@ -1,270 +0,0 @@
-from __future__ import annotations
-
-import re
-from html.parser import HTMLParser
-from urllib.parse import parse_qs, urlparse
-
-from app.services.domains.scholar.parser_constants import (
-    MARKER_KEYS,
-    SHOW_MORE_BUTTON_RE,
-)
-from app.services.domains.scholar.parser_types import PublicationCandidate
-from app.services.domains.scholar.parser_utils import (
-    attr_class,
-    attr_href,
-    attr_src,
-    build_absolute_scholar_url,
-    normalize_space,
-    strip_tags,
-)
-
-
-class ScholarRowParser(HTMLParser):
-    def __init__(self) -> None:
-        super().__init__(convert_charrefs=True)
-        self.title_href: str | None = None
-        self.title_parts: list[str] = []
-        self.citation_parts: list[str] = []
-        self.year_parts: list[str] = []
-        self.gray_texts: list[str] = []
-
-        self._title_depth = 0
-        self._citation_depth = 0
-        self._year_depth = 0
-        self._gray_stack: list[dict[str, object]] = []
-
-    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
-        if self._title_depth > 0:
-            self._title_depth += 1
-        if self._citation_depth > 0:
-            self._citation_depth += 1
-        if self._year_depth > 0:
-            self._year_depth += 1
-        if self._gray_stack:
-            self._gray_stack[-1]["depth"] += 1
-
-        classes = attr_class(attrs)
-
-        if tag == "a" and "gsc_a_at" in classes:
-            self._title_depth = 1
-            self.title_href = attr_href(attrs)
-            return
-
-        if tag == "a" and "gsc_a_ac" in classes:
-            self._citation_depth = 1
-            return
-
-        if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes):
-            self._year_depth = 1
-            return
-
-        if tag == "div" and "gs_gray" in classes:
-            self._gray_stack.append({"depth": 1, "parts": []})
-            return
-
-    def handle_data(self, data: str) -> None:
-        if self._title_depth > 0:
-            self.title_parts.append(data)
-        if self._citation_depth > 0:
-            self.citation_parts.append(data)
-        if self._year_depth > 0:
-            self.year_parts.append(data)
-        if self._gray_stack:
-            self._gray_stack[-1]["parts"].append(data)
-
-    def handle_endtag(self, tag: str) -> None:
-        if self._title_depth > 0:
-            self._title_depth -= 1
-        if self._citation_depth > 0:
-            self._citation_depth -= 1
-        if self._year_depth > 0:
-            self._year_depth -= 1
-        if self._gray_stack:
-            self._gray_stack[-1]["depth"] -= 1
-            if self._gray_stack[-1]["depth"] == 0:
-                text_value = normalize_space("".join(self._gray_stack[-1]["parts"]))
-                if text_value:
-                    self.gray_texts.append(text_value)
-                self._gray_stack.pop()
-
-
-def extract_rows(html: str) -> list[str]:
-    pattern = re.compile(
-        r"<tr\b(?=[^>]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)</tr>",
-        re.I | re.S,
-    )
-    return [match.group(1) for match in pattern.finditer(html)]
-
-
-def parse_cluster_id_from_href(href: str | None) -> str | None:
-    if not href:
-        return None
-    parsed = urlparse(href)
-    query = parse_qs(parsed.query)
-
-    citation_for_view = query.get("citation_for_view")
-    if citation_for_view:
-        token = citation_for_view[0].strip()
-        if token:
-            return f"cfv:{token}"
-
-    cluster = query.get("cluster")
-    if cluster:
-        token = cluster[0].strip()
-        if token:
-            return f"cluster:{token}"
-    return None
-
-
-def parse_year(parts: list[str]) -> int | None:
-    text = normalize_space(" ".join(parts))
-    match = re.search(r"\b(19|20)\d{2}\b", text)
-    if not match:
-        return None
-    try:
-        return int(match.group(0))
-    except ValueError:
-        return None
-
-
-def parse_citation_count(parts: list[str]) -> int | None:
-    text = normalize_space(" ".join(parts))
-    if not text:
-        return 0
-    digits = re.sub(r"\D+", "", text)
-    if not digits:
-        return None
-    return int(digits)
-
-
-def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None, list[str]]:
-    parser = ScholarRowParser()
-    parser.feed(row_html)
-    warnings: list[str] = []
-    title = normalize_space("".join(parser.title_parts))
-    if not title:
-        warnings.append("row_missing_title")
-        return None, warnings
-    if not parser.title_href:
-        warnings.append("row_missing_title_href")
-
-    citation_text = normalize_space(" ".join(parser.citation_parts))
-    citation_count = parse_citation_count(parser.citation_parts)
-    if citation_text and citation_count is None:
-        warnings.append("layout_row_citation_unparseable")
-
-    year_text = normalize_space(" ".join(parser.year_parts))
-    year = parse_year(parser.year_parts)
-    if year_text and year is None:
-        warnings.append("layout_row_year_unparseable")
-
-    authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
-    venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
-    return (
-        PublicationCandidate(
-            title=title,
-            title_url=parser.title_href,
-            cluster_id=parse_cluster_id_from_href(parser.title_href),
-            year=year,
-            citation_count=citation_count,
-            authors_text=authors_text,
-            venue_text=venue_text,
-            pdf_url=None,
-        ),
-        warnings,
-    )
-
-
-def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]:
-    rows = extract_rows(html)
-    warnings: list[str] = []
-    publications: list[PublicationCandidate] = []
-
-    for row_html in rows:
-        publication, row_warnings = _parse_publication_row(row_html)
-        warnings.extend(row_warnings)
-        if publication is None:
-            continue
-        publications.append(publication)
-
-    if not rows:
-        warnings.append("no_rows_detected")
-    if rows and not publications:
-        warnings.append("layout_all_rows_unparseable")
-
-    return publications, sorted(set(warnings))
-
-
-def extract_profile_name(html: str) -> str | None:
-    pattern = re.compile(
-        r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)</[^>]+>",
-        re.I | re.S,
-    )
-    match = pattern.search(html)
-    if not match:
-        return None
-    value = strip_tags(match.group(1))
-    return value or None
-
-
-def extract_profile_image_url(html: str) -> str | None:
-    og_image_pattern = re.compile(
-        r"<meta[^>]+property=['\"]og:image['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>",
-        re.I | re.S,
-    )
-    og_match = og_image_pattern.search(html)
-    if og_match:
-        value = normalize_space(og_match.group(1))
-        absolute = build_absolute_scholar_url(value)
-        if absolute:
-            return absolute
-
-    image_pattern = re.compile(
-        r"<img[^>]*\bid=['\"]gsc_prf_pup-img['\"][^>]*\bsrc=['\"]([^'\"]+)['\"][^>]*>",
-        re.I | re.S,
-    )
-    image_match = image_pattern.search(html)
-    if not image_match:
-        return None
-
-    value = normalize_space(image_match.group(1))
-    return build_absolute_scholar_url(value)
-
-
-def extract_articles_range(html: str) -> str | None:
-    pattern = re.compile(
-        r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)</[^>]+>",
-        re.I | re.S,
-    )
-    match = pattern.search(html)
-    if not match:
-        return None
-    value = strip_tags(match.group(1))
-    return value or None
-
-
-def has_show_more_button(html: str) -> bool:
-    match = SHOW_MORE_BUTTON_RE.search(html)
-    if match is None:
-        return False
-
-    button_tag = match.group(0).lower()
-    if "disabled" in button_tag:
-        return False
-    if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag:
-        return False
-    if "gs_dis" in button_tag:
-        return False
-    return True
-
-
-def has_operation_error_banner(html: str) -> bool:
-    lowered = html.lower()
-    if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered:
-        return False
-    return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered
-
-
-def count_markers(html: str) -> dict[str, int]:
-    lowered = html.lower()
-    return {key: lowered.count(key.lower()) for key in MARKER_KEYS}

build/lib/app/services/domains/scholar/rate_limit.py

@@ -1,18 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import time
-
-_REQUEST_LOCK = asyncio.Lock()
-_LAST_REQUEST_AT = 0.0
-
-
-async def wait_for_scholar_slot(*, min_interval_seconds: float) -> None:
-    global _LAST_REQUEST_AT
-    interval = max(float(min_interval_seconds), 0.0)
-    async with _REQUEST_LOCK:
-        elapsed = time.monotonic() - _LAST_REQUEST_AT
-        remaining = interval - elapsed
-        if remaining > 0:
-            await asyncio.sleep(remaining)
-        _LAST_REQUEST_AT = time.monotonic()

build/lib/app/services/domains/scholar/source.py

@@ -1,235 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-import random
-from dataclasses import dataclass
-from typing import Protocol
-from urllib.error import HTTPError, URLError
-from urllib.parse import urlencode
-from urllib.request import Request, urlopen
-
-SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations"
-DEFAULT_PAGE_SIZE = 100
-
-DEFAULT_USER_AGENTS = [
-    (
-        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
-        "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
-    ),
-    (
-        "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 "
-        "(KHTML, like Gecko) Version/18.1 Safari/605.1.15"
-    ),
-    (
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) "
-        "Gecko/20100101 Firefox/131.0"
-    ),
-]
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass(frozen=True)
-class FetchResult:
-    requested_url: str
-    status_code: int | None
-    final_url: str | None
-    body: str
-    error: str | None
-
-
-class ScholarSource(Protocol):
-    async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
-        ...
-
-    async def fetch_profile_page_html(
-        self,
-        scholar_id: str,
-        *,
-        cstart: int,
-        pagesize: int,
-    ) -> FetchResult:
-        ...
-
-    async def fetch_author_search_html(
-        self,
-        query: str,
-        *,
-        start: int,
-    ) -> FetchResult:
-        ...
-
-
-class LiveScholarSource:
-    def __init__(
-        self,
-        *,
-        timeout_seconds: float = 25.0,
-        user_agents: list[str] | None = None,
-    ) -> None:
-        self._timeout_seconds = timeout_seconds
-        self._user_agents = user_agents or DEFAULT_USER_AGENTS
-
-    async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
-        return await self.fetch_profile_page_html(
-            scholar_id,
-            cstart=0,
-            pagesize=DEFAULT_PAGE_SIZE,
-        )
-
-    async def fetch_profile_page_html(
-        self,
-        scholar_id: str,
-        *,
-        cstart: int,
-        pagesize: int = DEFAULT_PAGE_SIZE,
-    ) -> FetchResult:
-        requested_url = _build_profile_url(
-            scholar_id=scholar_id,
-            cstart=cstart,
-            pagesize=pagesize,
-        )
-        logger.debug(
-            "scholar_source.fetch_started",
-            extra={
-                "event": "scholar_source.fetch_started",
-                "scholar_id": scholar_id,
-                "requested_url": requested_url,
-                "cstart": cstart,
-                "pagesize": pagesize,
-            },
-        )
-        return await asyncio.to_thread(self._fetch_sync, requested_url)
-
-    async def fetch_author_search_html(
-        self,
-        query: str,
-        *,
-        start: int = 0,
-    ) -> FetchResult:
-        requested_url = _build_author_search_url(
-            query=query,
-            start=start,
-        )
-        logger.debug(
-            "scholar_source.search_fetch_started",
-            extra={
-                "event": "scholar_source.search_fetch_started",
-                "query": query,
-                "requested_url": requested_url,
-                "start": start,
-            },
-        )
-        return await asyncio.to_thread(self._fetch_sync, requested_url)
-
-    async def fetch_publication_html(self, publication_url: str) -> FetchResult:
-        logger.debug(
-            "scholar_source.publication_fetch_started",
-            extra={
-                "event": "scholar_source.publication_fetch_started",
-                "requested_url": publication_url,
-            },
-        )
-        return await asyncio.to_thread(self._fetch_sync, publication_url)
-
-    def _build_request(self, requested_url: str) -> Request:
-        return Request(
-            requested_url,
-            headers={
-                "User-Agent": random.choice(self._user_agents),
-                "Accept": "text/html,application/xhtml+xml",
-                "Accept-Language": "en-US,en;q=0.9",
-                "Connection": "close",
-            },
-        )
-
-    @staticmethod
-    def _http_error_body(exc: HTTPError) -> str:
-        try:
-            return exc.read().decode("utf-8", errors="replace")
-        except Exception:
-            return ""
-
-    @staticmethod
-    def _network_error_result(requested_url: str, exc: URLError) -> FetchResult:
-        logger.warning(
-            "scholar_source.fetch_network_error",
-            extra={"event": "scholar_source.fetch_network_error", "requested_url": requested_url},
-        )
-        return FetchResult(
-            requested_url=requested_url,
-            status_code=None,
-            final_url=None,
-            body="",
-            error=str(exc),
-        )
-
-    @staticmethod
-    def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult:
-        logger.warning(
-            "scholar_source.fetch_http_error",
-            extra={
-                "event": "scholar_source.fetch_http_error",
-                "requested_url": requested_url,
-                "status_code": exc.code,
-            },
-        )
-        return FetchResult(
-            requested_url=requested_url,
-            status_code=exc.code,
-            final_url=exc.geturl(),
-            body=LiveScholarSource._http_error_body(exc),
-            error=str(exc),
-        )
-
-    @staticmethod
-    def _success_result(requested_url: str, response) -> FetchResult:
-        body = response.read().decode("utf-8", errors="replace")
-        status_code = getattr(response, "status", 200)
-        logger.debug(
-            "scholar_source.fetch_succeeded",
-            extra={
-                "event": "scholar_source.fetch_succeeded",
-                "requested_url": requested_url,
-                "status_code": status_code,
-            },
-        )
-        return FetchResult(
-            requested_url=requested_url,
-            status_code=status_code,
-            final_url=response.geturl(),
-            body=body,
-            error=None,
-        )
-
-    def _fetch_sync(self, requested_url: str) -> FetchResult:
-        request = self._build_request(requested_url)
-
-        try:
-            with urlopen(request, timeout=self._timeout_seconds) as response:
-                return self._success_result(requested_url, response)
-        except HTTPError as exc:
-            return self._http_error_result(requested_url, exc)
-        except URLError as exc:
-            return self._network_error_result(requested_url, exc)
-
-
-def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:
-    query: dict[str, int | str] = {"hl": "en", "user": scholar_id}
-    if cstart > 0:
-        query["cstart"] = int(cstart)
-    if pagesize > 0:
-        query["pagesize"] = int(pagesize)
-    return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}"
-
-
-def _build_author_search_url(*, query: str, start: int) -> str:
-    params: dict[str, int | str] = {
-        "hl": "en",
-        "view_op": "search_authors",
-        "mauthors": query,
-    }
-    if start > 0:
-        params["astart"] = int(start)
-    return f"{SCHOLAR_PROFILE_URL}?{urlencode(params)}"