First product

app/api/routers/runs.py

@@ -0,0 +1,560 @@
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+
+from fastapi import APIRouter, Depends, Query, Request
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.deps import get_api_current_user
+from app.api.errors import ApiException
+from app.api.responses import success_payload
+from app.api.schemas import (
+    ManualRunEnvelope,
+    QueueClearEnvelope,
+    QueueItemEnvelope,
+    QueueListEnvelope,
+    RunDetailEnvelope,
+    RunsListEnvelope,
+)
+from app.db.models import RunStatus, RunTriggerType, User
+from app.db.session import get_db_session
+from app.services import ingestion as ingestion_service
+from app.services import runs as run_service
+from app.services import user_settings as user_settings_service
+from app.settings import settings
+from app.web.deps import get_ingestion_service
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/runs", tags=["api-runs"])
+
+IDEMPOTENCY_HEADER = "Idempotency-Key"
+IDEMPOTENCY_MAX_LENGTH = 128
+IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
+
+
+def _int_value(value: Any, default: int = 0) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _str_value(value: Any) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text if text else None
+
+
+def _bool_value(value: Any, default: bool = False) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        lowered = value.strip().lower()
+        if lowered in {"1", "true", "yes", "on"}:
+            return True
+        if lowered in {"0", "false", "no", "off"}:
+            return False
+    return default
+
+
+def _normalize_idempotency_key(raw_value: str | None) -> str | None:
+    if raw_value is None:
+        return None
+    candidate = raw_value.strip()
+    if not candidate:
+        return None
+    if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate):
+        raise ApiException(
+            status_code=400,
+            code="invalid_idempotency_key",
+            message=(
+                "Invalid Idempotency-Key. Use 8-128 characters from: "
+                "A-Z a-z 0-9 . _ : -"
+            ),
+        )
+    return candidate
+
+
+def _serialize_run(run) -> dict[str, Any]:
+    summary = run_service.extract_run_summary(run.error_log)
+    return {
+        "id": int(run.id),
+        "trigger_type": run.trigger_type.value,
+        "status": run.status.value,
+        "start_dt": run.start_dt,
+        "end_dt": run.end_dt,
+        "scholar_count": int(run.scholar_count or 0),
+        "new_publication_count": int(run.new_pub_count or 0),
+        "failed_count": int(summary["failed_count"]),
+        "partial_count": int(summary["partial_count"]),
+    }
+
+
+def _serialize_queue_item(item) -> dict[str, Any]:
+    return {
+        "id": int(item.id),
+        "scholar_profile_id": int(item.scholar_profile_id),
+        "scholar_label": item.scholar_label,
+        "status": item.status,
+        "reason": item.reason,
+        "dropped_reason": item.dropped_reason,
+        "attempt_count": int(item.attempt_count),
+        "resume_cstart": int(item.resume_cstart),
+        "next_attempt_dt": item.next_attempt_dt,
+        "updated_at": item.updated_at,
+        "last_error": item.last_error,
+        "last_run_id": item.last_run_id,
+    }
+
+
+def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]:
+    if not isinstance(value, list):
+        return []
+    normalized: list[dict[str, Any]] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        normalized.append(
+            {
+                "attempt": _int_value(item.get("attempt"), 0),
+                "cstart": _int_value(item.get("cstart"), 0),
+                "state": _str_value(item.get("state")),
+                "state_reason": _str_value(item.get("state_reason")),
+                "status_code": (
+                    _int_value(item.get("status_code"))
+                    if item.get("status_code") is not None
+                    else None
+                ),
+                "fetch_error": _str_value(item.get("fetch_error")),
+            }
+        )
+    return normalized
+
+
+def _normalize_page_logs(value: Any) -> list[dict[str, Any]]:
+    if not isinstance(value, list):
+        return []
+    normalized: list[dict[str, Any]] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        warning_codes = item.get("warning_codes")
+        normalized.append(
+            {
+                "page": _int_value(item.get("page"), 0),
+                "cstart": _int_value(item.get("cstart"), 0),
+                "state": _str_value(item.get("state")) or "unknown",
+                "state_reason": _str_value(item.get("state_reason")),
+                "status_code": (
+                    _int_value(item.get("status_code"))
+                    if item.get("status_code") is not None
+                    else None
+                ),
+                "publication_count": _int_value(item.get("publication_count"), 0),
+                "attempt_count": _int_value(item.get("attempt_count"), 0),
+                "has_show_more_button": _bool_value(item.get("has_show_more_button"), False),
+                "articles_range": _str_value(item.get("articles_range")),
+                "warning_codes": [
+                    str(code)
+                    for code in (warning_codes if isinstance(warning_codes, list) else [])
+                ],
+            }
+        )
+    return normalized
+
+
+def _normalize_debug(value: Any) -> dict[str, Any] | None:
+    if not isinstance(value, dict):
+        return None
+    marker_counts = value.get("marker_counts_nonzero")
+    warning_codes = value.get("warning_codes")
+    return {
+        "status_code": (
+            _int_value(value.get("status_code"))
+            if value.get("status_code") is not None
+            else None
+        ),
+        "final_url": _str_value(value.get("final_url")),
+        "fetch_error": _str_value(value.get("fetch_error")),
+        "body_sha256": _str_value(value.get("body_sha256")),
+        "body_length": (
+            _int_value(value.get("body_length"))
+            if value.get("body_length") is not None
+            else None
+        ),
+        "has_show_more_button": (
+            _bool_value(value.get("has_show_more_button"), False)
+            if value.get("has_show_more_button") is not None
+            else None
+        ),
+        "articles_range": _str_value(value.get("articles_range")),
+        "state_reason": _str_value(value.get("state_reason")),
+        "warning_codes": [
+            str(code)
+            for code in (warning_codes if isinstance(warning_codes, list) else [])
+        ],
+        "marker_counts_nonzero": {
+            str(key): _int_value(count, 0)
+            for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else [])
+        },
+        "page_logs": _normalize_page_logs(value.get("page_logs")),
+        "attempt_log": _normalize_attempt_log(value.get("attempt_log")),
+    }
+
+
+def _normalize_scholar_result(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {
+            "scholar_profile_id": 0,
+            "scholar_id": "unknown",
+            "state": "unknown",
+            "state_reason": None,
+            "outcome": "failed",
+            "attempt_count": 0,
+            "publication_count": 0,
+            "start_cstart": 0,
+            "continuation_cstart": None,
+            "continuation_enqueued": False,
+            "continuation_cleared": False,
+            "warnings": [],
+            "error": None,
+            "debug": None,
+        }
+    warnings = value.get("warnings")
+    return {
+        "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0),
+        "scholar_id": _str_value(value.get("scholar_id")) or "unknown",
+        "state": _str_value(value.get("state")) or "unknown",
+        "state_reason": _str_value(value.get("state_reason")),
+        "outcome": _str_value(value.get("outcome")) or "failed",
+        "attempt_count": _int_value(value.get("attempt_count"), 0),
+        "publication_count": _int_value(value.get("publication_count"), 0),
+        "start_cstart": _int_value(value.get("start_cstart"), 0),
+        "continuation_cstart": (
+            _int_value(value.get("continuation_cstart"))
+            if value.get("continuation_cstart") is not None
+            else None
+        ),
+        "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False),
+        "continuation_cleared": _bool_value(value.get("continuation_cleared"), False),
+        "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])],
+        "error": _str_value(value.get("error")),
+        "debug": _normalize_debug(value.get("debug")),
+    }
+
+
+def _manual_run_payload_from_run(
+    run,
+    *,
+    idempotency_key: str | None,
+    reused_existing_run: bool,
+) -> dict[str, Any]:
+    summary = run_service.extract_run_summary(run.error_log)
+    return {
+        "run_id": int(run.id),
+        "status": run.status.value,
+        "scholar_count": int(run.scholar_count or 0),
+        "succeeded_count": int(summary["succeeded_count"]),
+        "failed_count": int(summary["failed_count"]),
+        "partial_count": int(summary["partial_count"]),
+        "new_publication_count": int(run.new_pub_count or 0),
+        "reused_existing_run": reused_existing_run,
+        "idempotency_key": idempotency_key,
+    }
+
+
+@router.get(
+    "",
+    response_model=RunsListEnvelope,
+)
+async def list_runs(
+    request: Request,
+    failed_only: bool = Query(default=False),
+    limit: int = Query(default=100, ge=1, le=500),
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    runs = await run_service.list_runs_for_user(
+        db_session,
+        user_id=current_user.id,
+        limit=limit,
+        failed_only=failed_only,
+    )
+    return success_payload(
+        request,
+        data={
+            "runs": [_serialize_run(run) for run in runs],
+        },
+    )
+
+
+@router.get(
+    "/{run_id}",
+    response_model=RunDetailEnvelope,
+)
+async def get_run(
+    run_id: int,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    run = await run_service.get_run_for_user(
+        db_session,
+        user_id=current_user.id,
+        run_id=run_id,
+    )
+    if run is None:
+        raise ApiException(
+            status_code=404,
+            code="run_not_found",
+            message="Run not found.",
+        )
+    error_log = run.error_log if isinstance(run.error_log, dict) else {}
+    scholar_results = error_log.get("scholar_results")
+    if not isinstance(scholar_results, list):
+        scholar_results = []
+    return success_payload(
+        request,
+        data={
+            "run": _serialize_run(run),
+            "summary": run_service.extract_run_summary(error_log),
+            "scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
+        },
+    )
+
+
+@router.post(
+    "/manual",
+    response_model=ManualRunEnvelope,
+)
+async def run_manual(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+    ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
+):
+    idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
+    if idempotency_key is not None:
+        previous_run = await run_service.get_manual_run_by_idempotency_key(
+            db_session,
+            user_id=current_user.id,
+            idempotency_key=idempotency_key,
+        )
+        if previous_run is not None:
+            if previous_run.status == RunStatus.RUNNING:
+                raise ApiException(
+                    status_code=409,
+                    code="run_in_progress",
+                    message="A run with this idempotency key is still in progress.",
+                    details={
+                        "run_id": int(previous_run.id),
+                        "idempotency_key": idempotency_key,
+                    },
+                )
+            return success_payload(
+                request,
+                data=_manual_run_payload_from_run(
+                    previous_run,
+                    idempotency_key=idempotency_key,
+                    reused_existing_run=True,
+                ),
+            )
+
+    user_settings = await user_settings_service.get_or_create_settings(
+        db_session,
+        user_id=current_user.id,
+    )
+    try:
+        run_summary = await ingest_service.run_for_user(
+            db_session,
+            user_id=current_user.id,
+            trigger_type=RunTriggerType.MANUAL,
+            request_delay_seconds=user_settings.request_delay_seconds,
+            network_error_retries=settings.ingestion_network_error_retries,
+            retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
+            max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
+            page_size=settings.ingestion_page_size,
+            auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
+            queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
+        )
+    except ingestion_service.RunAlreadyInProgressError as exc:
+        await db_session.rollback()
+        raise ApiException(
+            status_code=409,
+            code="run_in_progress",
+            message="A run is already in progress for this account.",
+        ) from exc
+    except Exception as exc:
+        await db_session.rollback()
+        logger.exception(
+            "api.runs.manual_failed",
+            extra={
+                "event": "api.runs.manual_failed",
+                "user_id": current_user.id,
+            },
+        )
+        raise ApiException(
+            status_code=500,
+            code="manual_run_failed",
+            message="Manual run failed.",
+        ) from exc
+
+    if idempotency_key is not None:
+        await run_service.set_manual_run_idempotency_key(
+            db_session,
+            user_id=current_user.id,
+            run_id=run_summary.crawl_run_id,
+            idempotency_key=idempotency_key,
+        )
+
+    return success_payload(
+        request,
+        data={
+            "run_id": run_summary.crawl_run_id,
+            "status": run_summary.status.value,
+            "scholar_count": run_summary.scholar_count,
+            "succeeded_count": run_summary.succeeded_count,
+            "failed_count": run_summary.failed_count,
+            "partial_count": run_summary.partial_count,
+            "new_publication_count": run_summary.new_publication_count,
+            "reused_existing_run": False,
+            "idempotency_key": idempotency_key,
+        },
+    )
+
+
+@router.get(
+    "/queue/items",
+    response_model=QueueListEnvelope,
+)
+async def list_queue_items(
+    request: Request,
+    limit: int = Query(default=200, ge=1, le=500),
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    items = await run_service.list_queue_items_for_user(
+        db_session,
+        user_id=current_user.id,
+        limit=limit,
+    )
+    return success_payload(
+        request,
+        data={
+            "queue_items": [_serialize_queue_item(item) for item in items],
+        },
+    )
+
+
+@router.post(
+    "/queue/{queue_item_id}/retry",
+    response_model=QueueItemEnvelope,
+)
+async def retry_queue_item(
+    queue_item_id: int,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    try:
+        queue_item = await run_service.retry_queue_item_for_user(
+            db_session,
+            user_id=current_user.id,
+            queue_item_id=queue_item_id,
+        )
+    except run_service.QueueTransitionError as exc:
+        raise ApiException(
+            status_code=409,
+            code=exc.code,
+            message=exc.message,
+            details={"current_status": exc.current_status},
+        ) from exc
+    if queue_item is None:
+        raise ApiException(
+            status_code=404,
+            code="queue_item_not_found",
+            message="Queue item not found.",
+        )
+    return success_payload(
+        request,
+        data=_serialize_queue_item(queue_item),
+    )
+
+
+@router.post(
+    "/queue/{queue_item_id}/drop",
+    response_model=QueueItemEnvelope,
+)
+async def drop_queue_item(
+    queue_item_id: int,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    try:
+        dropped = await run_service.drop_queue_item_for_user(
+            db_session,
+            user_id=current_user.id,
+            queue_item_id=queue_item_id,
+        )
+    except run_service.QueueTransitionError as exc:
+        raise ApiException(
+            status_code=409,
+            code=exc.code,
+            message=exc.message,
+            details={"current_status": exc.current_status},
+        ) from exc
+    if dropped is None:
+        raise ApiException(
+            status_code=404,
+            code="queue_item_not_found",
+            message="Queue item not found.",
+        )
+    return success_payload(
+        request,
+        data=_serialize_queue_item(dropped),
+    )
+
+
+@router.delete(
+    "/queue/{queue_item_id}",
+    response_model=QueueClearEnvelope,
+)
+async def clear_queue_item(
+    queue_item_id: int,
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    current_user: User = Depends(get_api_current_user),
+):
+    try:
+        deleted = await run_service.clear_queue_item_for_user(
+            db_session,
+            user_id=current_user.id,
+            queue_item_id=queue_item_id,
+        )
+    except run_service.QueueTransitionError as exc:
+        raise ApiException(
+            status_code=409,
+            code=exc.code,
+            message=exc.message,
+            details={"current_status": exc.current_status},
+        ) from exc
+    if deleted is None:
+        raise ApiException(
+            status_code=404,
+            code="queue_item_not_found",
+            message="Queue item not found.",
+        )
+    return success_payload(
+        request,
+        data={
+            "queue_item_id": deleted.queue_item_id,
+            "previous_status": deleted.previous_status,
+            "status": "cleared",
+            "message": "Queue item cleared.",
+        },
+    )