justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

arden scrape safety telemetry and polish run state UX

Browse source
This commit is contained in:
Justin Visser 2026-02-19 21:59:36 +01:00
parent 110e246a1c
commit d7404abf18
33 changed files with 2074 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

100
app/api/routers/runs.py
View file

@ -22,6 +22,7 @@ from app.api.schemas import (
from app.db.models import RunStatus, RunTriggerType, User
from app.db.session import get_db_session
from app.services import ingestion as ingestion_service
from app.services import run_safety as run_safety_service
from app.services import runs as run_service
from app.services import user_settings as user_settings_service
from app.settings import settings
@ -253,6 +254,7 @@ def _manual_run_payload_from_run(
*,
idempotency_key: str | None,
reused_existing_run: bool,
safety_state: dict[str, Any],
) -> dict[str, Any]:
summary = run_service.extract_run_summary(run.error_log)
return {
@ -265,9 +267,37 @@ def _manual_run_payload_from_run(
"new_publication_count": int(run.new_pub_count or 0),
"reused_existing_run": reused_existing_run,
"idempotency_key": idempotency_key,
"safety_state": safety_state,
}
async def _load_safety_state(
db_session: AsyncSession,
*,
user_id: int,
) -> dict[str, Any]:
user_settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=user_id,
)
previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
if run_safety_service.clear_expired_cooldown(user_settings):
await db_session.commit()
await db_session.refresh(user_settings)
logger.info(
"api.runs.safety_cooldown_cleared",
extra={
"event": "api.runs.safety_cooldown_cleared",
"user_id": user_id,
"reason": previous_safety_state.get("cooldown_reason"),
"cooldown_until": previous_safety_state.get("cooldown_until"),
"metric_name": "api_runs_safety_cooldown_cleared_total",
"metric_value": 1,
},
)
return run_safety_service.get_safety_state_payload(user_settings)
@router.get(
"",
response_model=RunsListEnvelope,
@ -285,10 +315,15 @@ async def list_runs(
limit=limit,
failed_only=failed_only,
)
safety_state = await _load_safety_state(
db_session,
user_id=current_user.id,
)
return success_payload(
request,
data={
"runs": [_serialize_run(run) for run in runs],
"safety_state": safety_state,
},
)
@ -318,12 +353,17 @@ async def get_run(
scholar_results = error_log.get("scholar_results")
if not isinstance(scholar_results, list):
scholar_results = []
safety_state = await _load_safety_state(
db_session,
user_id=current_user.id,
)
return success_payload(
request,
data={
"run": _serialize_run(run),
"summary": run_service.extract_run_summary(error_log),
"scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
"safety_state": safety_state,
},
)
@ -338,6 +378,34 @@ async def run_manual(
current_user: User = Depends(get_api_current_user),
ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
):
safety_state = await _load_safety_state(
db_session,
user_id=current_user.id,
)
if not settings.ingestion_manual_run_allowed:
logger.warning(
"api.runs.manual_blocked_policy",
extra={
"event": "api.runs.manual_blocked_policy",
"user_id": current_user.id,
"policy": {"manual_run_allowed": False},
"safety_state": safety_state,
"metric_name": "api_runs_manual_blocked_policy_total",
"metric_value": 1,
},
)
raise ApiException(
status_code=403,
code="manual_runs_disabled",
message="Manual checks are disabled by server policy.",
details={
"policy": {
"manual_run_allowed": False,
},
"safety_state": safety_state,
},
)
idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
if idempotency_key is not None:
previous_run = await run_service.get_manual_run_by_idempotency_key(
@ -362,6 +430,7 @@ async def run_manual(
previous_run,
idempotency_key=idempotency_key,
reused_existing_run=True,
safety_state=safety_state,
),
)
@ -393,6 +462,28 @@ async def run_manual(
code="run_in_progress",
message="A run is already in progress for this account.",
) from exc
except ingestion_service.RunBlockedBySafetyPolicyError as exc:
await db_session.rollback()
logger.info(
"api.runs.manual_blocked_safety",
extra={
"event": "api.runs.manual_blocked_safety",
"user_id": current_user.id,
"reason": exc.safety_state.get("cooldown_reason"),
"cooldown_until": exc.safety_state.get("cooldown_until"),
"cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"),
"metric_name": "api_runs_manual_blocked_safety_total",
"metric_value": 1,
},
)
raise ApiException(
status_code=429,
code=exc.code,
message=exc.message,
details={
"safety_state": exc.safety_state,
},
) from exc
except IntegrityError as exc:
await db_session.rollback()
if idempotency_key is not None:
@ -418,6 +509,10 @@ async def run_manual(
existing_run,
idempotency_key=idempotency_key,
reused_existing_run=True,
safety_state=await _load_safety_state(
db_session,
user_id=current_user.id,
),
),
)
logger.exception(
@ -447,6 +542,10 @@ async def run_manual(
message="Manual run failed.",
) from exc
current_safety_state = await _load_safety_state(
db_session,
user_id=current_user.id,
)
return success_payload(
request,
data={
@ -459,6 +558,7 @@ async def run_manual(
"new_publication_count": run_summary.new_publication_count,
"reused_existing_run": False,
"idempotency_key": idempotency_key,
"safety_state": current_safety_state,
},
)

88
app/api/routers/settings.py
View file

@ -11,19 +11,38 @@ from app.api.responses import success_payload
from app.api.schemas import SettingsEnvelope, SettingsUpdateRequest
from app.db.models import User
from app.db.session import get_db_session
from app.services import run_safety as run_safety_service
from app.services import user_settings as user_settings_service
from app.settings import settings as settings_module
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/settings", tags=["api-settings"])
def _serialize_settings(settings) -> dict[str, object]:
def _serialize_settings(user_settings) -> dict[str, object]:
min_run_interval_minutes = user_settings_service.resolve_run_interval_minimum(
settings_module.ingestion_min_run_interval_minutes
)
min_request_delay_seconds = user_settings_service.resolve_request_delay_minimum(
settings_module.ingestion_min_request_delay_seconds
)
return {
"auto_run_enabled": bool(settings.auto_run_enabled),
"run_interval_minutes": int(settings.run_interval_minutes),
"request_delay_seconds": int(settings.request_delay_seconds),
"nav_visible_pages": list(settings.nav_visible_pages or []),
"auto_run_enabled": bool(user_settings.auto_run_enabled) and settings_module.ingestion_automation_allowed,
"run_interval_minutes": int(user_settings.run_interval_minutes),
"request_delay_seconds": int(user_settings.request_delay_seconds),
"nav_visible_pages": list(user_settings.nav_visible_pages or []),
"policy": {
"min_run_interval_minutes": min_run_interval_minutes,
"min_request_delay_seconds": min_request_delay_seconds,
"automation_allowed": bool(settings_module.ingestion_automation_allowed),
"manual_run_allowed": bool(settings_module.ingestion_manual_run_allowed),
"blocked_failure_threshold": max(1, int(settings_module.ingestion_alert_blocked_failure_threshold)),
"network_failure_threshold": max(1, int(settings_module.ingestion_alert_network_failure_threshold)),
"cooldown_blocked_seconds": max(60, int(settings_module.ingestion_safety_cooldown_blocked_seconds)),
"cooldown_network_seconds": max(60, int(settings_module.ingestion_safety_cooldown_network_seconds)),
},
"safety_state": run_safety_service.get_safety_state_payload(user_settings),
}
@ -36,13 +55,28 @@ async def get_settings(
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
settings = await user_settings_service.get_or_create_settings(
user_settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=current_user.id,
)
previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
if run_safety_service.clear_expired_cooldown(user_settings):
await db_session.commit()
await db_session.refresh(user_settings)
logger.info(
"api.settings.safety_cooldown_cleared",
extra={
"event": "api.settings.safety_cooldown_cleared",
"user_id": current_user.id,
"reason": previous_safety_state.get("cooldown_reason"),
"cooldown_until": previous_safety_state.get("cooldown_until"),
"metric_name": "api_settings_safety_cooldown_cleared_total",
"metric_value": 1,
},
)
return success_payload(
request,
data=_serialize_settings(settings),
data=_serialize_settings(user_settings),
)
@ -56,22 +90,31 @@ async def update_settings(
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
settings = await user_settings_service.get_or_create_settings(
user_settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=current_user.id,
)
min_run_interval_minutes = user_settings_service.resolve_run_interval_minimum(
settings_module.ingestion_min_run_interval_minutes
)
min_request_delay_seconds = user_settings_service.resolve_request_delay_minimum(
settings_module.ingestion_min_request_delay_seconds
)
try:
parsed_interval = user_settings_service.parse_run_interval_minutes(
str(payload.run_interval_minutes)
str(payload.run_interval_minutes),
minimum=min_run_interval_minutes,
)
parsed_delay = user_settings_service.parse_request_delay_seconds(
str(payload.request_delay_seconds)
str(payload.request_delay_seconds),
minimum=min_request_delay_seconds,
)
parsed_nav_visible_pages = user_settings_service.parse_nav_visible_pages(
payload.nav_visible_pages
if payload.nav_visible_pages is not None
else list(settings.nav_visible_pages or user_settings_service.DEFAULT_NAV_VISIBLE_PAGES)
else list(user_settings.nav_visible_pages or user_settings_service.DEFAULT_NAV_VISIBLE_PAGES)
)
except user_settings_service.UserSettingsServiceError as exc:
raise ApiException(
@ -80,14 +123,33 @@ async def update_settings(
message=str(exc),
) from exc
effective_auto_run_enabled = (
bool(payload.auto_run_enabled) and settings_module.ingestion_automation_allowed
)
updated = await user_settings_service.update_settings(
db_session,
settings=settings,
auto_run_enabled=bool(payload.auto_run_enabled),
settings=user_settings,
auto_run_enabled=effective_auto_run_enabled,
run_interval_minutes=parsed_interval,
request_delay_seconds=parsed_delay,
nav_visible_pages=parsed_nav_visible_pages,
)
previous_safety_state = run_safety_service.get_safety_event_context(updated)
if run_safety_service.clear_expired_cooldown(updated):
await db_session.commit()
await db_session.refresh(updated)
logger.info(
"api.settings.safety_cooldown_cleared",
extra={
"event": "api.settings.safety_cooldown_cleared",
"user_id": current_user.id,
"reason": previous_safety_state.get("cooldown_reason"),
"cooldown_until": previous_safety_state.get("cooldown_until"),
"metric_name": "api_settings_safety_cooldown_cleared_total",
"metric_value": 1,
},
)
logger.info(
"api.settings.updated",
extra={

42
app/api/schemas.py
View file

@ -202,6 +202,7 @@ class RunListItemData(BaseModel):
class RunsListData(BaseModel):
runs: list[RunListItemData]
safety_state: ScrapeSafetyStateData
model_config = ConfigDict(extra="forbid")
@ -227,6 +228,30 @@ class RunSummaryData(BaseModel):
model_config = ConfigDict(extra="forbid")
class ScrapeSafetyCountersData(BaseModel):
consecutive_blocked_runs: int = 0
consecutive_network_runs: int = 0
cooldown_entry_count: int = 0
blocked_start_count: int = 0
last_blocked_failure_count: int = 0
last_network_failure_count: int = 0
last_evaluated_run_id: int | None = None
model_config = ConfigDict(extra="forbid")
class ScrapeSafetyStateData(BaseModel):
cooldown_active: bool
cooldown_reason: str | None = None
cooldown_reason_label: str | None = None
cooldown_until: datetime | None = None
cooldown_remaining_seconds: int = 0
recommended_action: str | None = None
counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
model_config = ConfigDict(extra="forbid")
class RunAttemptLogData(BaseModel):
attempt: int
cstart: int
@ -293,6 +318,7 @@ class RunDetailData(BaseModel):
run: RunListItemData
summary: RunSummaryData
scholar_results: list[RunScholarResultData]
safety_state: ScrapeSafetyStateData
model_config = ConfigDict(extra="forbid")
@ -314,6 +340,7 @@ class ManualRunData(BaseModel):
new_publication_count: int
reused_existing_run: bool
idempotency_key: str | None = None
safety_state: ScrapeSafetyStateData
model_config = ConfigDict(extra="forbid")
@ -429,11 +456,26 @@ class AdminResetPasswordRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
class SettingsPolicyData(BaseModel):
min_run_interval_minutes: int
min_request_delay_seconds: int
automation_allowed: bool
manual_run_allowed: bool
blocked_failure_threshold: int
network_failure_threshold: int
cooldown_blocked_seconds: int
cooldown_network_seconds: int
model_config = ConfigDict(extra="forbid")
class SettingsData(BaseModel):
auto_run_enabled: bool
run_interval_minutes: int
request_delay_seconds: int
nav_visible_pages: list[str]
policy: SettingsPolicyData
safety_state: ScrapeSafetyStateData
model_config = ConfigDict(extra="forbid")