justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

refactor: extract helpers from oversized router and scheduler files

Browse source 
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Justin Visser 2026-02-27 15:41:23 +01:00
parent 55315299c7
commit f11947aad2
9 changed files with 1187 additions and 1042 deletions
Download patch file Download diff file Expand all files Collapse all files

503
app/api/routers/runs.py
View file

@ -2,7 +2,6 @@ from __future__ import annotations
import asyncio
import logging
import re
from typing import Any
from fastapi import APIRouter, Depends, Query, Request
@ -13,6 +12,21 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_api_current_user
from app.api.errors import ApiException
from app.api.responses import success_payload
from app.api.routers.run_manual import (
load_safety_state,
raise_manual_blocked_safety,
raise_manual_failed,
raise_manual_runs_disabled,
recover_integrity_error,
reused_manual_run_payload,
)
from app.api.routers.run_serializers import (
IDEMPOTENCY_HEADER,
normalize_idempotency_key,
normalize_scholar_result,
serialize_queue_item,
serialize_run,
)
from app.api.runtime_deps import get_ingestion_service
from app.api.schemas import (
ManualRunEnvelope,
@ -24,9 +38,7 @@ from app.api.schemas import (
)
from app.db.models import RunStatus, RunTriggerType, User
from app.db.session import get_db_session
from app.logging_utils import structured_log
from app.services.ingestion import application as ingestion_service
from app.services.ingestion import safety as run_safety_service
from app.services.runs import application as run_service
from app.services.runs.events import event_generator
from app.services.settings import application as user_settings_service
@ -38,453 +50,6 @@ _background_tasks: set[asyncio.Task[Any]] = set()
router = APIRouter(prefix="/runs", tags=["api-runs"])
ACTIVE_RUN_STATUSES = {RunStatus.RUNNING, RunStatus.RESOLVING}
IDEMPOTENCY_HEADER = "Idempotency-Key"
IDEMPOTENCY_MAX_LENGTH = 128
IDEMPOTENCY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{8,128}$")
def _int_value(value: Any, default: int = 0) -> int:
try:
return int(value)
except (TypeError, ValueError):
return default
def _str_value(value: Any) -> str | None:
if value is None:
return None
text = str(value).strip()
return text if text else None
def _bool_value(value: Any, default: bool = False) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, str):
lowered = value.strip().lower()
if lowered in {"1", "true", "yes", "on"}:
return True
if lowered in {"0", "false", "no", "off"}:
return False
return default
def _normalize_idempotency_key(raw_value: str | None) -> str | None:
if raw_value is None:
return None
candidate = raw_value.strip()
if not candidate:
return None
if len(candidate) > IDEMPOTENCY_MAX_LENGTH or not IDEMPOTENCY_PATTERN.match(candidate):
raise ApiException(
status_code=400,
code="invalid_idempotency_key",
message=("Invalid Idempotency-Key. Use 8-128 characters from: A-Z a-z 0-9 . _ : -"),
)
return candidate
def _serialize_run(run) -> dict[str, Any]:
summary = run_service.extract_run_summary(run.error_log)
return {
"id": int(run.id),
"trigger_type": run.trigger_type.value,
"status": run.status.value,
"start_dt": run.start_dt,
"end_dt": run.end_dt,
"scholar_count": int(run.scholar_count or 0),
"new_publication_count": int(run.new_pub_count or 0),
"failed_count": int(summary["failed_count"]),
"partial_count": int(summary["partial_count"]),
}
def _serialize_queue_item(item) -> dict[str, Any]:
return {
"id": int(item.id),
"scholar_profile_id": int(item.scholar_profile_id),
"scholar_label": item.scholar_label,
"status": item.status,
"reason": item.reason,
"dropped_reason": item.dropped_reason,
"attempt_count": int(item.attempt_count),
"resume_cstart": int(item.resume_cstart),
"next_attempt_dt": item.next_attempt_dt,
"updated_at": item.updated_at,
"last_error": item.last_error,
"last_run_id": item.last_run_id,
}
def _normalize_attempt_log(value: Any) -> list[dict[str, Any]]:
if not isinstance(value, list):
return []
normalized: list[dict[str, Any]] = []
for item in value:
if not isinstance(item, dict):
continue
normalized.append(
{
"attempt": _int_value(item.get("attempt"), 0),
"cstart": _int_value(item.get("cstart"), 0),
"state": _str_value(item.get("state")),
"state_reason": _str_value(item.get("state_reason")),
"status_code": (_int_value(item.get("status_code")) if item.get("status_code") is not None else None),
"fetch_error": _str_value(item.get("fetch_error")),
}
)
return normalized
def _normalize_page_logs(value: Any) -> list[dict[str, Any]]:
if not isinstance(value, list):
return []
normalized: list[dict[str, Any]] = []
for item in value:
if not isinstance(item, dict):
continue
warning_codes = item.get("warning_codes")
normalized.append(
{
"page": _int_value(item.get("page"), 0),
"cstart": _int_value(item.get("cstart"), 0),
"state": _str_value(item.get("state")) or "unknown",
"state_reason": _str_value(item.get("state_reason")),
"status_code": (_int_value(item.get("status_code")) if item.get("status_code") is not None else None),
"publication_count": _int_value(item.get("publication_count"), 0),
"attempt_count": _int_value(item.get("attempt_count"), 0),
"has_show_more_button": _bool_value(item.get("has_show_more_button"), False),
"articles_range": _str_value(item.get("articles_range")),
"warning_codes": [str(code) for code in (warning_codes if isinstance(warning_codes, list) else [])],
}
)
return normalized
def _normalize_debug(value: Any) -> dict[str, Any] | None:
if not isinstance(value, dict):
return None
marker_counts = value.get("marker_counts_nonzero")
warning_codes = value.get("warning_codes")
return {
"status_code": (_int_value(value.get("status_code")) if value.get("status_code") is not None else None),
"final_url": _str_value(value.get("final_url")),
"fetch_error": _str_value(value.get("fetch_error")),
"body_sha256": _str_value(value.get("body_sha256")),
"body_length": (_int_value(value.get("body_length")) if value.get("body_length") is not None else None),
"has_show_more_button": (
_bool_value(value.get("has_show_more_button"), False)
if value.get("has_show_more_button") is not None
else None
),
"articles_range": _str_value(value.get("articles_range")),
"state_reason": _str_value(value.get("state_reason")),
"warning_codes": [str(code) for code in (warning_codes if isinstance(warning_codes, list) else [])],
"marker_counts_nonzero": {
str(key): _int_value(count, 0)
for key, count in (marker_counts.items() if isinstance(marker_counts, dict) else [])
},
"page_logs": _normalize_page_logs(value.get("page_logs")),
"attempt_log": _normalize_attempt_log(value.get("attempt_log")),
}
def _normalize_scholar_result(value: Any) -> dict[str, Any]:
if not isinstance(value, dict):
return {
"scholar_profile_id": 0,
"scholar_id": "unknown",
"state": "unknown",
"state_reason": None,
"outcome": "failed",
"attempt_count": 0,
"publication_count": 0,
"start_cstart": 0,
"continuation_cstart": None,
"continuation_enqueued": False,
"continuation_cleared": False,
"warnings": [],
"error": None,
"debug": None,
}
warnings = value.get("warnings")
return {
"scholar_profile_id": _int_value(value.get("scholar_profile_id"), 0),
"scholar_id": _str_value(value.get("scholar_id")) or "unknown",
"state": _str_value(value.get("state")) or "unknown",
"state_reason": _str_value(value.get("state_reason")),
"outcome": _str_value(value.get("outcome")) or "failed",
"attempt_count": _int_value(value.get("attempt_count"), 0),
"publication_count": _int_value(value.get("publication_count"), 0),
"start_cstart": _int_value(value.get("start_cstart"), 0),
"continuation_cstart": (
_int_value(value.get("continuation_cstart")) if value.get("continuation_cstart") is not None else None
),
"continuation_enqueued": _bool_value(value.get("continuation_enqueued"), False),
"continuation_cleared": _bool_value(value.get("continuation_cleared"), False),
"warnings": [str(item) for item in (warnings if isinstance(warnings, list) else [])],
"error": _str_value(value.get("error")),
"debug": _normalize_debug(value.get("debug")),
}
def _manual_run_payload_from_run(
run,
*,
idempotency_key: str | None,
reused_existing_run: bool,
safety_state: dict[str, Any],
) -> dict[str, Any]:
summary = run_service.extract_run_summary(run.error_log)
return {
"run_id": int(run.id),
"status": run.status.value,
"scholar_count": int(run.scholar_count or 0),
"succeeded_count": int(summary["succeeded_count"]),
"failed_count": int(summary["failed_count"]),
"partial_count": int(summary["partial_count"]),
"new_publication_count": int(run.new_pub_count or 0),
"reused_existing_run": reused_existing_run,
"idempotency_key": idempotency_key,
"safety_state": safety_state,
}
async def _load_safety_state(
db_session: AsyncSession,
*,
user_id: int,
) -> dict[str, Any]:
user_settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=user_id,
)
previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
if run_safety_service.clear_expired_cooldown(user_settings):
await db_session.commit()
await db_session.refresh(user_settings)
structured_log(
logger,
"info",
"api.runs.safety_cooldown_cleared",
user_id=user_id,
reason=previous_safety_state.get("cooldown_reason"),
cooldown_until=previous_safety_state.get("cooldown_until"),
)
return run_safety_service.get_safety_state_payload(user_settings)
def _raise_manual_runs_disabled(*, user_id: int, safety_state: dict[str, Any]) -> None:
structured_log(
logger,
"warning",
"api.runs.manual_blocked_policy",
user_id=user_id,
policy={"manual_run_allowed": False},
safety_state=safety_state,
)
raise ApiException(
status_code=403,
code="manual_runs_disabled",
message="Manual checks are disabled by server policy.",
details={
"policy": {"manual_run_allowed": False},
"safety_state": safety_state,
},
)
async def _reused_manual_run_payload(
db_session: AsyncSession,
*,
request: Request,
user_id: int,
idempotency_key: str | None,
safety_state: dict[str, Any],
) -> dict[str, Any] | None:
if idempotency_key is None:
return None
previous_run = await run_service.get_manual_run_by_idempotency_key(
db_session,
user_id=user_id,
idempotency_key=idempotency_key,
)
if previous_run is None:
return None
if previous_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING):
raise ApiException(
status_code=409,
code="run_in_progress",
message="A run with this idempotency key is still in progress.",
details={"run_id": int(previous_run.id), "idempotency_key": idempotency_key},
)
return success_payload(
request,
data=_manual_run_payload_from_run(
previous_run,
idempotency_key=idempotency_key,
reused_existing_run=True,
safety_state=safety_state,
),
)
async def _run_ingestion_for_manual(
db_session: AsyncSession,
*,
ingest_service: ingestion_service.ScholarIngestionService,
user_id: int,
idempotency_key: str | None,
):
user_settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=user_id,
)
return await ingest_service.run_for_user(
db_session,
user_id=user_id,
trigger_type=RunTriggerType.MANUAL,
request_delay_seconds=user_settings.request_delay_seconds,
network_error_retries=settings.ingestion_network_error_retries,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
max_pages_per_scholar=settings.ingestion_max_pages_per_scholar,
page_size=settings.ingestion_page_size,
auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
idempotency_key=idempotency_key,
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
)
async def _recover_integrity_error(
db_session: AsyncSession,
*,
request: Request,
user_id: int,
idempotency_key: str | None,
original_exc: IntegrityError,
) -> dict[str, Any]:
if idempotency_key is None:
logger.exception(
"api.runs.manual_integrity_error",
extra={"user_id": user_id},
)
raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
existing_run = await run_service.get_manual_run_by_idempotency_key(
db_session,
user_id=user_id,
idempotency_key=idempotency_key,
)
if existing_run is None:
logger.exception(
"api.runs.manual_integrity_error",
extra={"user_id": user_id},
)
raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from original_exc
if existing_run.status in (RunStatus.RUNNING, RunStatus.RESOLVING):
raise ApiException(
status_code=409,
code="run_in_progress",
message="A run with this idempotency key is still in progress.",
details={"run_id": int(existing_run.id), "idempotency_key": idempotency_key},
) from original_exc
return success_payload(
request,
data=_manual_run_payload_from_run(
existing_run,
idempotency_key=idempotency_key,
reused_existing_run=True,
safety_state=await _load_safety_state(db_session, user_id=user_id),
),
)
async def _execute_manual_run(
db_session: AsyncSession,
*,
request: Request,
ingest_service: ingestion_service.ScholarIngestionService,
user_id: int,
idempotency_key: str | None,
):
try:
return await _run_ingestion_for_manual(
db_session,
ingest_service=ingest_service,
user_id=user_id,
idempotency_key=idempotency_key,
)
except ingestion_service.RunAlreadyInProgressError as exc:
await db_session.rollback()
raise ApiException(
status_code=409,
code="run_in_progress",
message="A run is already in progress for this account.",
) from exc
except ingestion_service.RunBlockedBySafetyPolicyError as exc:
await db_session.rollback()
_raise_manual_blocked_safety(exc=exc, user_id=user_id)
except IntegrityError as exc:
await db_session.rollback()
return await _recover_integrity_error(
db_session,
request=request,
user_id=user_id,
idempotency_key=idempotency_key,
original_exc=exc,
)
except Exception as exc:
await db_session.rollback()
_raise_manual_failed(exc=exc, user_id=user_id)
def _raise_manual_blocked_safety(*, exc, user_id: int) -> None:
structured_log(
logger,
"info",
"api.runs.manual_blocked_safety",
user_id=user_id,
reason=exc.safety_state.get("cooldown_reason"),
cooldown_until=exc.safety_state.get("cooldown_until"),
cooldown_remaining_seconds=exc.safety_state.get("cooldown_remaining_seconds"),
)
raise ApiException(
status_code=429,
code=exc.code,
message=exc.message,
details={"safety_state": exc.safety_state},
) from exc
def _raise_manual_failed(*, exc: Exception, user_id: int) -> None:
logger.exception(
"api.runs.manual_failed",
extra={"user_id": user_id},
)
raise ApiException(status_code=500, code="manual_run_failed", message="Manual run failed.") from exc
def _manual_run_success_payload(
*,
run_summary,
idempotency_key: str | None,
safety_state: dict[str, Any],
) -> dict[str, Any]:
return {
"run_id": run_summary.crawl_run_id,
"status": run_summary.status.value,
"scholar_count": run_summary.scholar_count,
"succeeded_count": run_summary.succeeded_count,
"failed_count": run_summary.failed_count,
"partial_count": run_summary.partial_count,
"new_publication_count": run_summary.new_publication_count,
"reused_existing_run": False,
"idempotency_key": idempotency_key,
"safety_state": safety_state,
}
@router.get(
"",
@ -503,14 +68,14 @@ async def list_runs(
limit=limit,
failed_only=failed_only,
)
safety_state = await _load_safety_state(
safety_state = await load_safety_state(
db_session,
user_id=current_user.id,
)
return success_payload(
request,
data={
"runs": [_serialize_run(run) for run in runs],
"runs": [serialize_run(run) for run in runs],
"safety_state": safety_state,
},
)
@ -541,16 +106,16 @@ async def get_run(
scholar_results = error_log.get("scholar_results")
if not isinstance(scholar_results, list):
scholar_results = []
safety_state = await _load_safety_state(
safety_state = await load_safety_state(
db_session,
user_id=current_user.id,
)
return success_payload(
request,
data={
"run": _serialize_run(run),
"run": serialize_run(run),
"summary": run_service.extract_run_summary(error_log),
"scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
"scholar_results": [normalize_scholar_result(item) for item in scholar_results],
"safety_state": safety_state,
},
)
@ -595,7 +160,7 @@ async def cancel_run(
if not isinstance(scholar_results, list):
scholar_results = []
safety_state = await _load_safety_state(
safety_state = await load_safety_state(
db_session,
user_id=current_user.id,
)
@ -603,9 +168,9 @@ async def cancel_run(
return success_payload(
request,
data={
"run": _serialize_run(run),
"run": serialize_run(run),
"summary": run_service.extract_run_summary(error_log),
"scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
"scholar_results": [normalize_scholar_result(item) for item in scholar_results],
"safety_state": safety_state,
},
)
@ -621,12 +186,12 @@ async def run_manual(
current_user: User = Depends(get_api_current_user),
ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
):
safety_state = await _load_safety_state(db_session, user_id=current_user.id)
safety_state = await load_safety_state(db_session, user_id=current_user.id)
if not settings.ingestion_manual_run_allowed:
_raise_manual_runs_disabled(user_id=current_user.id, safety_state=safety_state)
raise_manual_runs_disabled(user_id=current_user.id, safety_state=safety_state)
idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
reused_payload = await _reused_manual_run_payload(
idempotency_key = normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
reused_payload = await reused_manual_run_payload(
db_session,
request=request,
user_id=current_user.id,
@ -695,12 +260,12 @@ async def run_manual(
"new_publication_count": 0,
"reused_existing_run": False,
"idempotency_key": idempotency_key,
"safety_state": await _load_safety_state(db_session, user_id=current_user.id),
"safety_state": await load_safety_state(db_session, user_id=current_user.id),
},
)
except ingestion_service.RunBlockedBySafetyPolicyError as exc:
await db_session.rollback()
_raise_manual_blocked_safety(exc=exc, user_id=current_user.id)
raise_manual_blocked_safety(exc=exc, user_id=current_user.id)
except ingestion_service.RunAlreadyInProgressError as exc:
await db_session.rollback()
raise ApiException(
@ -710,7 +275,7 @@ async def run_manual(
) from exc
except IntegrityError as exc:
await db_session.rollback()
return await _recover_integrity_error(
return await recover_integrity_error(
db_session,
request=request,
user_id=current_user.id,
@ -719,7 +284,7 @@ async def run_manual(
)
except Exception as exc:
await db_session.rollback()
_raise_manual_failed(exc=exc, user_id=current_user.id)
raise_manual_failed(exc=exc, user_id=current_user.id)
@router.get(
@ -740,7 +305,7 @@ async def list_queue_items(
return success_payload(
request,
data={
"queue_items": [_serialize_queue_item(item) for item in items],
"queue_items": [serialize_queue_item(item) for item in items],
},
)
@ -776,7 +341,7 @@ async def retry_queue_item(
)
return success_payload(
request,
data=_serialize_queue_item(queue_item),
data=serialize_queue_item(queue_item),
)
@ -811,7 +376,7 @@ async def drop_queue_item(
)
return success_payload(
request,
data=_serialize_queue_item(dropped),
data=serialize_queue_item(dropped),
)