from __future__ import annotations import logging import re from typing import Any from fastapi import APIRouter, Depends, Query, Request from fastapi.responses import StreamingResponse from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps import get_api_current_user from app.api.errors import ApiException from app.api.responses import success_payload from app.api.runtime_deps import get_ingestion_service from app.api.schemas import ( ManualRunEnvelope, QueueClearEnvelope, QueueItemEnvelope, QueueListEnvelope, RunDetailEnvelope, RunsListEnvelope, ) from app.db.models import RunStatus, RunTriggerType, User from app.db.session import get_db_session from app.logging_utils import structured_log from app.services.domains.ingestion import application as ingestion_service from app.services.domains.ingestion import safety as run_safety_service from app.services.domains.runs import application as run_service from app.services.domains.runs.events import event_generator from app.services.domains.settings import application as user_settings_service from app.settings import settings logger = logging.getLogger(__name__) router = APIRouter(prefix="/runs", tags=["api-runs"]) ACTIVE_RUN_STATUSES = {RunStatus.RUNNING, RunStatus.RESOLVING} IDEMPOTENCY_HEADER = "Idempotency-Key" IDEMPOTENCY_MAX_LENGTH = 128 IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$") def _int_value(value: Any, default: int = 0) -> int: try: return int(value) except (TypeError, ValueError): return default def _str_value(value: Any) -> str | None: if value is None: return None text = str(value).strip() return text if text else None def _bool_value(value: Any, default: bool = False) -> bool: if isinstance(value, bool): return value if isinstance(value, str): lowered = value.strip().lower() if lowered in {"1", "true", "yes", "on"}: return True if lowered in {"0", "false", "no", "off"}: return False return default def _normalize_idempotency_key(raw_value: str | None) -> str | None: if raw_value is None: return None candidate = raw_value.strip() if not candidate: return None if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate): raise ApiException( status_code=400, code="invalid_idempotency_key", message=("Invalid Idempotency-Key. Use 8-128 characters from: A-Z a-z 0-9 . _ : -"), ) return candidate def _serialize_run(run) -> dict[str, Any]: summary = run_service.extract_run_summary(run.error_log) return { "id": int(run.id), "trigger_type": run.trigger_type.value, "status": run.status.value, "start_dt": run.start_dt, "end_dt": run.end_dt, "scholar_count": int(run.scholar_count or 0), "new_publication_count": int(run.new_pub_count or 0), "failed_count": int(summary["failed_count"]), "partial_count": int(summary["partial_count"]), } def _serialize_queue_item(item) -> dict[str, Any]: return { "id": int(item.id), "scholar_profile_id": int(item.scholar_profile_id), "scholar_label": item.scholar_label, "status": item.status, "reason": item.reason, "dropped_reason": item.dropped_reason, "attempt_count": int(item.attempt_count), "resume_cstart": int(item.resume_cstart), "next_attempt_dt": item.next_attempt_dt, "updated_at": item.updated_at, "last_error": item.last_error, "last_run_id": item.last_run_id, } def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]: if not isinstance(value, list): return [] normalized: list[dict[str, Any]] = [] for item in value: if not isinstance(item, dict): continue normalized.append( { "attempt": _int_value(item.get("attempt"), 0), "cstart": _int_value(item.get("cstart"), 0), "state": _str_value(item.get("state")), "state_reason": _str_value(item.get("state_reason")), "status_code": (_int_value(item.get("status_code")) if item.get("status_code") is not None else None), "fetch_error": _str_value(item.get("fetch_error")), } ) return normalized def _normalize_page_logs(value: Any) -> list[dict[str, Any]]: if not isinstance(value, list): return [] normalized: list[dict[str, Any]] = [] for item in value: if not isinstance(item, dict): continue warning_codes = item.get("warning_codes") normalized.append( { "page": _int_value(item.get("page"), 0), "cstart": _int_value(item.get("cstart"), 0), "state": _str_value(item.get("state")) or "unknown", "state_reason": _str_value(item.get("state_reason")), "status_code": (_int_value(item.get("status_code")) if item.get("status_code") is not None else None), "publication_count": _int_value(item.get("publication_count"), 0), "attempt_count": _int_value(item.get("attempt_count"), 0), "has_show_more_button": _bool_value(item.get("has_show_more_button"), False), "articles_range": _str_value(item.get("articles_range")), "warning_codes": [str(code) for code in (warning_codes if isinstance(warning_codes, list) else [])], } ) return normalized def _normalize_debug(value: Any) -> dict[str, Any] | None: if not isinstance(value, dict): return None marker_counts = value.get("marker_counts_nonzero") warning_codes = value.get("warning_codes") return { "status_code": (_int_value(value.get("status_code")) if value.get("status_code") is not None else None), "final_url": _str_value(value.get("final_url")), "fetch_error": _str_value(value.get("fetch_error")), "body_sha256": _str_value(value.get("body_sha256")), "body_length": (_int_value(value.get("body_length")) if value.get("body_length") is not None else None), "has_show_more_button": ( _bool_value(value.get("has_show_more_button"), False) if value.get("has_show_more_button") is not None else None ), "articles_range": _str_value(value.get("articles_range")), "state_reason": _str_value(value.get("state_reason")), "warning_codes": [str(code) for code in (warning_codes if isinstance(warning_codes, list) else [])], "marker_counts_nonzero": { str(key): _int_value(count, 0) for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else []) }, "page_logs": _normalize_page_logs(value.get("page_logs")), "attempt_log": _normalize_attempt_log(value.get("attempt_log")), } def _normalize_scholar_result(value: Any) -> dict[str, Any]: if not isinstance(value, dict): return { "scholar_profile_id": 0, "scholar_id": "unknown", "state": "unknown", "state_reason": None, "outcome": "failed", "attempt_count": 0, "publication_count": 0, "start_cstart": 0, "continuation_cstart": None, "continuation_enqueued": False, "continuation_cleared": False, "warnings": [], "error": None, "debug": None, } warnings = value.get("warnings") return { "scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0), "scholar_id": _str_value(value.get("scholar_id")) or "unknown", "state": _str_value(value.get("state")) or "unknown", "state_reason": _str_value(value.get("state_reason")), "outcome": _str_value(value.get("outcome")) or "failed", "attempt_count": _int_value(value.get("attempt_count"), 0), "publication_count": _int_value(value.get("publication_count"), 0), "start_cstart": _int_value(value.get("start_cstart"), 0), "continuation_cstart": ( _int_value(value.get("continuation_cstart")) if value.get("continuation_cstart") is not None else None ), "continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False), "continuation_cleared": _bool_value(value.get("continuation_cleared"), False), "warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])], "error": _str_value(value.get("error")), "debug": _normalize_debug(value.get("debug")), } def _manual_run_payload_from_run( run, *, idempotency_key: str | None, reused_existing_run: bool, safety_state: dict[str, Any], ) -> dict[str, Any]: summary = run_service.extract_run_summary(run.error_log) return { "run_id": int(run.id), "status": run.status.value, "scholar_count": int(run.scholar_count or 0), "succeeded_count": int(summary["succeeded_count"]), "failed_count": int(summary["failed_count"]), "partial_count": int(summary["partial_count"]), "new_publication_count": int(run.new_pub_count or 0), "reused_existing_run": reused_existing_run, "idempotency_key": idempotency_key, "safety_state": safety_state, } async def _load_safety_state( db_session: AsyncSession, *, user_id: int, ) -> dict[str, Any]: user_settings = await user_settings_service.get_or_create_settings( db_session, user_id=user_id, ) previous_safety_state = run_safety_service.get_safety_event_context(user_settings) if run_safety_service.clear_expired_cooldown(user_settings): await db_session.commit() await db_session.refresh(user_settings) structured_log( logger, "info", "api.runs.safety_cooldown_cleared", user_id=user_id, reason=previous_safety_state.get("cooldown_reason"), cooldown_until=previous_safety_state.get("cooldown_until"), ) return run_safety_service.get_safety_state_payload(user_settings) def _raise_manual_runs_disabled(*, user_id: int, safety_state: dict[str, Any]) -> None: structured_log( logger, "warning", "api.runs.manual_blocked_policy", user_id=user_id, policy={"manual_run_allowed": False}, safety_state=safety_state, ) raise ApiException( status_code=403, code="manual_runs_disabled", message="Manual checks are disabled by server policy.", details={ "policy": {"manual_run_allowed": False}, "safety_state": safety_state, }, ) async def _reused_manual_run_payload( db_session: AsyncSession, *, request: Request, user_id: int, idempotency_key: str | None, safety_state: dict[str, Any], ) -> dict[str, Any] | None: if idempotency_key is None: return None previous_run = await run_service.get_manual_run_by_idempotency_key( db_session, user_id=user_id, idempotency_key=idempotency_key, ) if previous_run is None: return None if previous_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING): raise ApiException( status_code=409, code="run_in_progress", message="A run with this idempotency key is still in progress.", details={"run_id": int(previous_run.id), "idempotency_key": idempotency_key}, ) return success_payload( request, data=_manual_run_payload_from_run( previous_run, idempotency_key=idempotency_key, reused_existing_run=True, safety_state=safety_state, ), ) async def _run_ingestion_for_manual( db_session: AsyncSession, *, ingest_service: ingestion_service.ScholarIngestionService, user_id: int, idempotency_key: str | None, ): user_settings = await user_settings_service.get_or_create_settings( db_session, user_id=user_id, ) return await ingest_service.run_for_user( db_session, user_id=user_id, trigger_type=RunTriggerType.MANUAL, request_delay_seconds=user_settings.request_delay_seconds, network_error_retries=settings.ingestion_network_error_retries, retry_backoff_seconds=settings.ingestion_retry_backoff_seconds, max_pages_per_scholar=settings.ingestion_max_pages_per_scholar, page_size=settings.ingestion_page_size, auto_queue_continuations=settings.ingestion_continuation_queue_enabled, queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds, idempotency_key=idempotency_key, alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, ) async def _recover_integrity_error( db_session: AsyncSession, *, request: Request, user_id: int, idempotency_key: str | None, original_exc: IntegrityError, ) -> dict[str, Any]: if idempotency_key is None: logger.exception( "api.runs.manual_integrity_error", extra={"user_id": user_id}, ) raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc existing_run = await run_service.get_manual_run_by_idempotency_key( db_session, user_id=user_id, idempotency_key=idempotency_key, ) if existing_run is None: logger.exception( "api.runs.manual_integrity_error", extra={"user_id": user_id}, ) raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc if existing_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING): raise ApiException( status_code=409, code="run_in_progress", message="A run with this idempotency key is still in progress.", details={"run_id": int(existing_run.id), "idempotency_key": idempotency_key}, ) from original_exc return success_payload( request, data=_manual_run_payload_from_run( existing_run, idempotency_key=idempotency_key, reused_existing_run=True, safety_state=await _load_safety_state(db_session, user_id=user_id), ), ) async def _execute_manual_run( db_session: AsyncSession, *, request: Request, ingest_service: ingestion_service.ScholarIngestionService, user_id: int, idempotency_key: str | None, ): try: return await _run_ingestion_for_manual( db_session, ingest_service=ingest_service, user_id=user_id, idempotency_key=idempotency_key, ) except ingestion_service.RunAlreadyInProgressError as exc: await db_session.rollback() raise ApiException( status_code=409, code="run_in_progress", message="A run is already in progress for this account.", ) from exc except ingestion_service.RunBlockedBySafetyPolicyError as exc: await db_session.rollback() _raise_manual_blocked_safety(exc=exc, user_id=user_id) except IntegrityError as exc: await db_session.rollback() return await _recover_integrity_error( db_session, request=request, user_id=user_id, idempotency_key=idempotency_key, original_exc=exc, ) except Exception as exc: await db_session.rollback() _raise_manual_failed(exc=exc, user_id=user_id) def _raise_manual_blocked_safety(*, exc, user_id: int) -> None: structured_log( logger, "info", "api.runs.manual_blocked_safety", user_id=user_id, reason=exc.safety_state.get("cooldown_reason"), cooldown_until=exc.safety_state.get("cooldown_until"), cooldown_remaining_seconds=exc.safety_state.get("cooldown_remaining_seconds"), ) raise ApiException( status_code=429, code=exc.code, message=exc.message, details={"safety_state": exc.safety_state}, ) from exc def _raise_manual_failed(*, exc: Exception, user_id: int) -> None: logger.exception( "api.runs.manual_failed", extra={"user_id": user_id}, ) raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from exc def _manual_run_success_payload( *, run_summary, idempotency_key: str | None, safety_state: dict[str, Any], ) -> dict[str, Any]: return { "run_id": run_summary.crawl_run_id, "status": run_summary.status.value, "scholar_count": run_summary.scholar_count, "succeeded_count": run_summary.succeeded_count, "failed_count": run_summary.failed_count, "partial_count": run_summary.partial_count, "new_publication_count": run_summary.new_publication_count, "reused_existing_run": False, "idempotency_key": idempotency_key, "safety_state": safety_state, } @router.get( "", response_model=RunsListEnvelope, ) async def list_runs( request: Request, failed_only: bool = Query(default=False), limit: int = Query(default=100, ge=1, le=500), db_session: AsyncSession = Depends(get_db_session), current_user: User = Depends(get_api_current_user), ): runs = await run_service.list_runs_for_user( db_session, user_id=current_user.id, limit=limit, failed_only=failed_only, ) safety_state = await _load_safety_state( db_session, user_id=current_user.id, ) return success_payload( request, data={ "runs": [_serialize_run(run) for run in runs], "safety_state": safety_state, }, ) @router.get( "/{run_id}", response_model=RunDetailEnvelope, ) async def get_run( run_id: int, request: Request, db_session: AsyncSession = Depends(get_db_session), current_user: User = Depends(get_api_current_user), ): run = await run_service.get_run_for_user( db_session, user_id=current_user.id, run_id=run_id, ) if run is None: raise ApiException( status_code=404, code="run_not_found", message="Run not found.", ) error_log = run.error_log if isinstance(run.error_log, dict) else {} scholar_results = error_log.get("scholar_results") if not isinstance(scholar_results, list): scholar_results = [] safety_state = await _load_safety_state( db_session, user_id=current_user.id, ) return success_payload( request, data={ "run": _serialize_run(run), "summary": run_service.extract_run_summary(error_log), "scholar_results": [_normalize_scholar_result(item) for item in scholar_results], "safety_state": safety_state, }, ) @router.post( "/{run_id}/cancel", response_model=RunDetailEnvelope, ) async def cancel_run( run_id: int, request: Request, db_session: AsyncSession = Depends(get_db_session), current_user: User = Depends(get_api_current_user), ): run = await run_service.get_run_for_user( db_session, user_id=current_user.id, run_id=run_id, ) if run is None: raise ApiException( status_code=404, code="run_not_found", message="Run not found.", ) if run.status in ACTIVE_RUN_STATUSES: run.status = RunStatus.CANCELED await db_session.commit() await db_session.refresh(run) else: raise ApiException( status_code=409, code="run_not_cancelable", message="Run is already terminal and cannot be canceled.", details={"run_id": int(run.id), "status": run.status.value}, ) error_log = run.error_log if isinstance(run.error_log, dict) else {} scholar_results = error_log.get("scholar_results") if not isinstance(scholar_results, list): scholar_results = [] safety_state = await _load_safety_state( db_session, user_id=current_user.id, ) return success_payload( request, data={ "run": _serialize_run(run), "summary": run_service.extract_run_summary(error_log), "scholar_results": [_normalize_scholar_result(item) for item in scholar_results], "safety_state": safety_state, }, ) @router.post( "/manual", response_model=ManualRunEnvelope, ) async def run_manual( request: Request, db_session: AsyncSession = Depends(get_db_session), current_user: User = Depends(get_api_current_user), ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service), ): safety_state = await _load_safety_state(db_session, user_id=current_user.id) if not settings.ingestion_manual_run_allowed: _raise_manual_runs_disabled(user_id=current_user.id, safety_state=safety_state) idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER)) reused_payload = await _reused_manual_run_payload( db_session, request=request, user_id=current_user.id, idempotency_key=idempotency_key, safety_state=safety_state, ) if reused_payload is not None: return reused_payload try: user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=current_user.id) # Initialize run (creates the record and performs safety checks) run, scholars, start_cstart_map = await ingest_service.initialize_run( db_session, user_id=current_user.id, trigger_type=RunTriggerType.MANUAL, request_delay_seconds=user_settings.request_delay_seconds, network_error_retries=settings.ingestion_network_error_retries, retry_backoff_seconds=settings.ingestion_retry_backoff_seconds, max_pages_per_scholar=settings.ingestion_max_pages_per_scholar, page_size=settings.ingestion_page_size, idempotency_key=idempotency_key, alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, ) await db_session.commit() # Kick off background execution import asyncio from app.db.session import get_session_factory asyncio.create_task( ingest_service.execute_run( session_factory=get_session_factory(), run_id=run.id, user_id=current_user.id, scholars=scholars, start_cstart_map=start_cstart_map, request_delay_seconds=user_settings.request_delay_seconds, network_error_retries=settings.ingestion_network_error_retries, retry_backoff_seconds=settings.ingestion_retry_backoff_seconds, max_pages_per_scholar=settings.ingestion_max_pages_per_scholar, page_size=settings.ingestion_page_size, auto_queue_continuations=settings.ingestion_continuation_queue_enabled, queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds, alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, idempotency_key=idempotency_key, ) ) return success_payload( request, data={ "run_id": int(run.id), "status": run.status.value, "scholar_count": int(run.scholar_count or 0), "succeeded_count": 0, "failed_count": 0, "partial_count": 0, "new_publication_count": 0, "reused_existing_run": False, "idempotency_key": idempotency_key, "safety_state": await _load_safety_state(db_session, user_id=current_user.id), }, ) except ingestion_service.RunBlockedBySafetyPolicyError as exc: await db_session.rollback() _raise_manual_blocked_safety(exc=exc, user_id=current_user.id) except ingestion_service.RunAlreadyInProgressError as exc: await db_session.rollback() raise ApiException( status_code=409, code="run_in_progress", message="A run is already in progress for this account.", ) from exc except IntegrityError as exc: await db_session.rollback() return await _recover_integrity_error( db_session, request=request, user_id=current_user.id, idempotency_key=idempotency_key, original_exc=exc, ) except Exception as exc: await db_session.rollback() _raise_manual_failed(exc=exc, user_id=current_user.id) @router.get( "/queue/items", response_model=QueueListEnvelope, ) async def list_queue_items( request: Request, limit: int = Query(default=200, ge=1, le=500), db_session: AsyncSession = Depends(get_db_session), current_user: User = Depends(get_api_current_user), ): items = await run_service.list_queue_items_for_user( db_session, user_id=current_user.id, limit=limit, ) return success_payload( request, data={ "queue_items": [_serialize_queue_item(item) for item in items], }, ) @router.post( "/queue/{queue_item_id}/retry", response_model=QueueItemEnvelope, ) async def retry_queue_item( queue_item_id: int, request: Request, db_session: AsyncSession = Depends(get_db_session), current_user: User = Depends(get_api_current_user), ): try: queue_item = await run_service.retry_queue_item_for_user( db_session, user_id=current_user.id, queue_item_id=queue_item_id, ) except run_service.QueueTransitionError as exc: raise ApiException( status_code=409, code=exc.code, message=exc.message, details={"current_status": exc.current_status}, ) from exc if queue_item is None: raise ApiException( status_code=404, code="queue_item_not_found", message="Queue item not found.", ) return success_payload( request, data=_serialize_queue_item(queue_item), ) @router.post( "/queue/{queue_item_id}/drop", response_model=QueueItemEnvelope, ) async def drop_queue_item( queue_item_id: int, request: Request, db_session: AsyncSession = Depends(get_db_session), current_user: User = Depends(get_api_current_user), ): try: dropped = await run_service.drop_queue_item_for_user( db_session, user_id=current_user.id, queue_item_id=queue_item_id, ) except run_service.QueueTransitionError as exc: raise ApiException( status_code=409, code=exc.code, message=exc.message, details={"current_status": exc.current_status}, ) from exc if dropped is None: raise ApiException( status_code=404, code="queue_item_not_found", message="Queue item not found.", ) return success_payload( request, data=_serialize_queue_item(dropped), ) @router.delete( "/queue/{queue_item_id}", response_model=QueueClearEnvelope, ) async def clear_queue_item( queue_item_id: int, request: Request, db_session: AsyncSession = Depends(get_db_session), current_user: User = Depends(get_api_current_user), ): try: deleted = await run_service.clear_queue_item_for_user( db_session, user_id=current_user.id, queue_item_id=queue_item_id, ) except run_service.QueueTransitionError as exc: raise ApiException( status_code=409, code=exc.code, message=exc.message, details={"current_status": exc.current_status}, ) from exc if deleted is None: raise ApiException( status_code=404, code="queue_item_not_found", message="Queue item not found.", ) return success_payload( request, data={ "queue_item_id": deleted.queue_item_id, "previous_status": deleted.previous_status, "status": "cleared", "message": "Queue item cleared.", }, ) @router.get("/{run_id}/stream") async def stream_run_events( run_id: int, db_session: AsyncSession = Depends(get_db_session), current_user: User = Depends(get_api_current_user), ): run = await run_service.get_run_for_user( db_session, user_id=current_user.id, run_id=run_id, ) if run is None: raise ApiException( status_code=404, code="run_not_found", message="Run not found.", ) return StreamingResponse(event_generator(run_id), media_type="text/event-stream")