justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

arden scrape safety telemetry and polish run state UX

Browse source
This commit is contained in:
Justin Visser 2026-02-19 21:59:36 +01:00
parent 110e246a1c
commit d7404abf18
33 changed files with 2074 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

100
app/api/routers/runs.py
View file

@ -22,6 +22,7 @@ from app.api.schemas import (
from app.db.models import RunStatus, RunTriggerType, User
from app.db.session import get_db_session
from app.services import ingestion as ingestion_service
from app.services import run_safety as run_safety_service
from app.services import runs as run_service
from app.services import user_settings as user_settings_service
from app.settings import settings
@ -253,6 +254,7 @@ def _manual_run_payload_from_run(
*,
idempotency_key: str | None,
reused_existing_run: bool,
safety_state: dict[str, Any],
) -> dict[str, Any]:
summary = run_service.extract_run_summary(run.error_log)
return {
@ -265,9 +267,37 @@ def _manual_run_payload_from_run(
"new_publication_count": int(run.new_pub_count or 0),
"reused_existing_run": reused_existing_run,
"idempotency_key": idempotency_key,
"safety_state": safety_state,
}
async def _load_safety_state(
db_session: AsyncSession,
*,
user_id: int,
) -> dict[str, Any]:
user_settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=user_id,
)
previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
if run_safety_service.clear_expired_cooldown(user_settings):
await db_session.commit()
await db_session.refresh(user_settings)
logger.info(
"api.runs.safety_cooldown_cleared",
extra={
"event": "api.runs.safety_cooldown_cleared",
"user_id": user_id,
"reason": previous_safety_state.get("cooldown_reason"),
"cooldown_until": previous_safety_state.get("cooldown_until"),
"metric_name": "api_runs_safety_cooldown_cleared_total",
"metric_value": 1,
},
)
return run_safety_service.get_safety_state_payload(user_settings)
@router.get(
"",
response_model=RunsListEnvelope,
@ -285,10 +315,15 @@ async def list_runs(
limit=limit,
failed_only=failed_only,
)
safety_state = await _load_safety_state(
db_session,
user_id=current_user.id,
)
return success_payload(
request,
data={
"runs": [_serialize_run(run) for run in runs],
"safety_state": safety_state,
},
)
@ -318,12 +353,17 @@ async def get_run(
scholar_results = error_log.get("scholar_results")
if not isinstance(scholar_results, list):
scholar_results = []
safety_state = await _load_safety_state(
db_session,
user_id=current_user.id,
)
return success_payload(
request,
data={
"run": _serialize_run(run),
"summary": run_service.extract_run_summary(error_log),
"scholar_results": [_normalize_scholar_result(item) for item in scholar_results],
"safety_state": safety_state,
},
)
@ -338,6 +378,34 @@ async def run_manual(
current_user: User = Depends(get_api_current_user),
ingest_service: ingestion_service.ScholarIngestionService = Depends(get_ingestion_service),
):
safety_state = await _load_safety_state(
db_session,
user_id=current_user.id,
)
if not settings.ingestion_manual_run_allowed:
logger.warning(
"api.runs.manual_blocked_policy",
extra={
"event": "api.runs.manual_blocked_policy",
"user_id": current_user.id,
"policy": {"manual_run_allowed": False},
"safety_state": safety_state,
"metric_name": "api_runs_manual_blocked_policy_total",
"metric_value": 1,
},
)
raise ApiException(
status_code=403,
code="manual_runs_disabled",
message="Manual checks are disabled by server policy.",
details={
"policy": {
"manual_run_allowed": False,
},
"safety_state": safety_state,
},
)
idempotency_key = _normalize_idempotency_key(request.headers.get(IDEMPOTENCY_HEADER))
if idempotency_key is not None:
previous_run = await run_service.get_manual_run_by_idempotency_key(
@ -362,6 +430,7 @@ async def run_manual(
previous_run,
idempotency_key=idempotency_key,
reused_existing_run=True,
safety_state=safety_state,
),
)
@ -393,6 +462,28 @@ async def run_manual(
code="run_in_progress",
message="A run is already in progress for this account.",
) from exc
except ingestion_service.RunBlockedBySafetyPolicyError as exc:
await db_session.rollback()
logger.info(
"api.runs.manual_blocked_safety",
extra={
"event": "api.runs.manual_blocked_safety",
"user_id": current_user.id,
"reason": exc.safety_state.get("cooldown_reason"),
"cooldown_until": exc.safety_state.get("cooldown_until"),
"cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"),
"metric_name": "api_runs_manual_blocked_safety_total",
"metric_value": 1,
},
)
raise ApiException(
status_code=429,
code=exc.code,
message=exc.message,
details={
"safety_state": exc.safety_state,
},
) from exc
except IntegrityError as exc:
await db_session.rollback()
if idempotency_key is not None:
@ -418,6 +509,10 @@ async def run_manual(
existing_run,
idempotency_key=idempotency_key,
reused_existing_run=True,
safety_state=await _load_safety_state(
db_session,
user_id=current_user.id,
),
),
)
logger.exception(
@ -447,6 +542,10 @@ async def run_manual(
message="Manual run failed.",
) from exc
current_safety_state = await _load_safety_state(
db_session,
user_id=current_user.id,
)
return success_payload(
request,
data={
@ -459,6 +558,7 @@ async def run_manual(
"new_publication_count": run_summary.new_publication_count,
"reused_existing_run": False,
"idempotency_key": idempotency_key,
"safety_state": current_safety_state,
},
)

88
app/api/routers/settings.py
View file

@ -11,19 +11,38 @@ from app.api.responses import success_payload
from app.api.schemas import SettingsEnvelope, SettingsUpdateRequest
from app.db.models import User
from app.db.session import get_db_session
from app.services import run_safety as run_safety_service
from app.services import user_settings as user_settings_service
from app.settings import settings as settings_module
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/settings", tags=["api-settings"])
def _serialize_settings(settings) -> dict[str, object]:
def _serialize_settings(user_settings) -> dict[str, object]:
min_run_interval_minutes = user_settings_service.resolve_run_interval_minimum(
settings_module.ingestion_min_run_interval_minutes
)
min_request_delay_seconds = user_settings_service.resolve_request_delay_minimum(
settings_module.ingestion_min_request_delay_seconds
)
return {
"auto_run_enabled": bool(settings.auto_run_enabled),
"run_interval_minutes": int(settings.run_interval_minutes),
"request_delay_seconds": int(settings.request_delay_seconds),
"nav_visible_pages": list(settings.nav_visible_pages or []),
"auto_run_enabled": bool(user_settings.auto_run_enabled) and settings_module.ingestion_automation_allowed,
"run_interval_minutes": int(user_settings.run_interval_minutes),
"request_delay_seconds": int(user_settings.request_delay_seconds),
"nav_visible_pages": list(user_settings.nav_visible_pages or []),
"policy": {
"min_run_interval_minutes": min_run_interval_minutes,
"min_request_delay_seconds": min_request_delay_seconds,
"automation_allowed": bool(settings_module.ingestion_automation_allowed),
"manual_run_allowed": bool(settings_module.ingestion_manual_run_allowed),
"blocked_failure_threshold": max(1, int(settings_module.ingestion_alert_blocked_failure_threshold)),
"network_failure_threshold": max(1, int(settings_module.ingestion_alert_network_failure_threshold)),
"cooldown_blocked_seconds": max(60, int(settings_module.ingestion_safety_cooldown_blocked_seconds)),
"cooldown_network_seconds": max(60, int(settings_module.ingestion_safety_cooldown_network_seconds)),
},
"safety_state": run_safety_service.get_safety_state_payload(user_settings),
}
@ -36,13 +55,28 @@ async def get_settings(
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
settings = await user_settings_service.get_or_create_settings(
user_settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=current_user.id,
)
previous_safety_state = run_safety_service.get_safety_event_context(user_settings)
if run_safety_service.clear_expired_cooldown(user_settings):
await db_session.commit()
await db_session.refresh(user_settings)
logger.info(
"api.settings.safety_cooldown_cleared",
extra={
"event": "api.settings.safety_cooldown_cleared",
"user_id": current_user.id,
"reason": previous_safety_state.get("cooldown_reason"),
"cooldown_until": previous_safety_state.get("cooldown_until"),
"metric_name": "api_settings_safety_cooldown_cleared_total",
"metric_value": 1,
},
)
return success_payload(
request,
data=_serialize_settings(settings),
data=_serialize_settings(user_settings),
)
@ -56,22 +90,31 @@ async def update_settings(
db_session: AsyncSession = Depends(get_db_session),
current_user: User = Depends(get_api_current_user),
):
settings = await user_settings_service.get_or_create_settings(
user_settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=current_user.id,
)
min_run_interval_minutes = user_settings_service.resolve_run_interval_minimum(
settings_module.ingestion_min_run_interval_minutes
)
min_request_delay_seconds = user_settings_service.resolve_request_delay_minimum(
settings_module.ingestion_min_request_delay_seconds
)
try:
parsed_interval = user_settings_service.parse_run_interval_minutes(
str(payload.run_interval_minutes)
str(payload.run_interval_minutes),
minimum=min_run_interval_minutes,
)
parsed_delay = user_settings_service.parse_request_delay_seconds(
str(payload.request_delay_seconds)
str(payload.request_delay_seconds),
minimum=min_request_delay_seconds,
)
parsed_nav_visible_pages = user_settings_service.parse_nav_visible_pages(
payload.nav_visible_pages
if payload.nav_visible_pages is not None
else list(settings.nav_visible_pages or user_settings_service.DEFAULT_NAV_VISIBLE_PAGES)
else list(user_settings.nav_visible_pages or user_settings_service.DEFAULT_NAV_VISIBLE_PAGES)
)
except user_settings_service.UserSettingsServiceError as exc:
raise ApiException(
@ -80,14 +123,33 @@ async def update_settings(
message=str(exc),
) from exc
effective_auto_run_enabled = (
bool(payload.auto_run_enabled) and settings_module.ingestion_automation_allowed
)
updated = await user_settings_service.update_settings(
db_session,
settings=settings,
auto_run_enabled=bool(payload.auto_run_enabled),
settings=user_settings,
auto_run_enabled=effective_auto_run_enabled,
run_interval_minutes=parsed_interval,
request_delay_seconds=parsed_delay,
nav_visible_pages=parsed_nav_visible_pages,
)
previous_safety_state = run_safety_service.get_safety_event_context(updated)
if run_safety_service.clear_expired_cooldown(updated):
await db_session.commit()
await db_session.refresh(updated)
logger.info(
"api.settings.safety_cooldown_cleared",
extra={
"event": "api.settings.safety_cooldown_cleared",
"user_id": current_user.id,
"reason": previous_safety_state.get("cooldown_reason"),
"cooldown_until": previous_safety_state.get("cooldown_until"),
"metric_name": "api_settings_safety_cooldown_cleared_total",
"metric_value": 1,
},
)
logger.info(
"api.settings.updated",
extra={

42
app/api/schemas.py
View file

@ -202,6 +202,7 @@ class RunListItemData(BaseModel):
class RunsListData(BaseModel):
runs: list[RunListItemData]
safety_state: ScrapeSafetyStateData
model_config = ConfigDict(extra="forbid")
@ -227,6 +228,30 @@ class RunSummaryData(BaseModel):
model_config = ConfigDict(extra="forbid")
class ScrapeSafetyCountersData(BaseModel):
consecutive_blocked_runs: int = 0
consecutive_network_runs: int = 0
cooldown_entry_count: int = 0
blocked_start_count: int = 0
last_blocked_failure_count: int = 0
last_network_failure_count: int = 0
last_evaluated_run_id: int | None = None
model_config = ConfigDict(extra="forbid")
class ScrapeSafetyStateData(BaseModel):
cooldown_active: bool
cooldown_reason: str | None = None
cooldown_reason_label: str | None = None
cooldown_until: datetime | None = None
cooldown_remaining_seconds: int = 0
recommended_action: str | None = None
counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
model_config = ConfigDict(extra="forbid")
class RunAttemptLogData(BaseModel):
attempt: int
cstart: int
@ -293,6 +318,7 @@ class RunDetailData(BaseModel):
run: RunListItemData
summary: RunSummaryData
scholar_results: list[RunScholarResultData]
safety_state: ScrapeSafetyStateData
model_config = ConfigDict(extra="forbid")
@ -314,6 +340,7 @@ class ManualRunData(BaseModel):
new_publication_count: int
reused_existing_run: bool
idempotency_key: str | None = None
safety_state: ScrapeSafetyStateData
model_config = ConfigDict(extra="forbid")
@ -429,11 +456,26 @@ class AdminResetPasswordRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
class SettingsPolicyData(BaseModel):
min_run_interval_minutes: int
min_request_delay_seconds: int
automation_allowed: bool
manual_run_allowed: bool
blocked_failure_threshold: int
network_failure_threshold: int
cooldown_blocked_seconds: int
cooldown_network_seconds: int
model_config = ConfigDict(extra="forbid")
class SettingsData(BaseModel):
auto_run_enabled: bool
run_interval_minutes: int
request_delay_seconds: int
nav_visible_pages: list[str]
policy: SettingsPolicyData
safety_state: ScrapeSafetyStateData
model_config = ConfigDict(extra="forbid")

7
app/db/models.py
View file

@ -105,6 +105,13 @@ class UserSetting(Base):
'\'["dashboard","scholars","publications","settings","style-guide","runs","users"]\'::jsonb'
),
)
scrape_safety_state: Mapped[dict] = mapped_column(
JSONB,
nullable=False,
server_default=text("'{}'::jsonb"),
)
scrape_cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
scrape_cooldown_reason: Mapped[str | None] = mapped_column(String(64))
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)

126
app/services/ingestion.py
View file

@ -22,6 +22,8 @@ from app.db.models import (
ScholarPublication,
)
from app.services import continuation_queue as queue_service
from app.services import run_safety as run_safety_service
from app.services import user_settings as user_settings_service
from app.services.scholar_parser import (
ParseState,
ParsedProfilePage,
@ -29,6 +31,7 @@ from app.services.scholar_parser import (
parse_profile_page,
)
from app.services.scholar_source import FetchResult, ScholarSource
from app.settings import settings
TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+")
WORD_RE = re.compile(r"[a-z0-9]+")
@ -88,6 +91,20 @@ class RunAlreadyInProgressError(RuntimeError):
"""Raised when a run lock for a user is already held by another process."""
class RunBlockedBySafetyPolicyError(RuntimeError):
def __init__(
self,
*,
code: str,
message: str,
safety_state: dict[str, Any],
) -> None:
super().__init__(message)
self.code = code
self.message = message
self.safety_state = safety_state
def _int_or_default(value: Any, default: int = 0) -> int:
try:
return int(value)
@ -134,6 +151,61 @@ class ScholarIngestionService:
alert_network_failure_threshold: int = 2,
alert_retry_scheduled_threshold: int = 3,
) -> RunExecutionSummary:
user_settings = await user_settings_service.get_or_create_settings(
db_session,
user_id=user_id,
)
now_utc = datetime.now(timezone.utc)
previous_safety_state = run_safety_service.get_safety_event_context(
user_settings,
now_utc=now_utc,
)
if run_safety_service.clear_expired_cooldown(
user_settings,
now_utc=now_utc,
):
await db_session.commit()
await db_session.refresh(user_settings)
logger.info(
"ingestion.safety_cooldown_cleared",
extra={
"event": "ingestion.safety_cooldown_cleared",
"user_id": user_id,
"reason": previous_safety_state.get("cooldown_reason"),
"cooldown_until": previous_safety_state.get("cooldown_until"),
"metric_name": "ingestion_safety_cooldown_cleared_total",
"metric_value": 1,
},
)
now_utc = datetime.now(timezone.utc)
if run_safety_service.is_cooldown_active(user_settings, now_utc=now_utc):
safety_state = run_safety_service.register_cooldown_blocked_start(
user_settings,
now_utc=now_utc,
)
await db_session.commit()
logger.warning(
"ingestion.safety_policy_blocked_run_start",
extra={
"event": "ingestion.safety_policy_blocked_run_start",
"user_id": user_id,
"trigger_type": trigger_type.value,
"reason": safety_state.get("cooldown_reason"),
"cooldown_until": safety_state.get("cooldown_until"),
"cooldown_remaining_seconds": safety_state.get("cooldown_remaining_seconds"),
"blocked_start_count": (
(safety_state.get("counters") or {}).get("blocked_start_count")
),
"metric_name": "ingestion_safety_run_start_blocked_total",
"metric_value": 1,
},
)
raise RunBlockedBySafetyPolicyError(
code="scrape_cooldown_active",
message="Scrape safety cooldown is active; run start is temporarily blocked.",
safety_state=safety_state,
)
lock_acquired = await self._try_acquire_user_lock(
db_session,
user_id=user_id,
@ -505,6 +577,8 @@ class ScholarIngestionService:
"crawl_run_id": run.id,
"blocked_failure_count": blocked_failure_count,
"threshold": bounded_blocked_threshold,
"metric_name": "ingestion_blocked_failure_threshold_exceeded_total",
"metric_value": 1,
},
)
if alert_flags["network_failure_threshold_exceeded"]:
@ -516,6 +590,8 @@ class ScholarIngestionService:
"crawl_run_id": run.id,
"network_failure_count": network_failure_count,
"threshold": bounded_network_threshold,
"metric_name": "ingestion_network_failure_threshold_exceeded_total",
"metric_value": 1,
},
)
if alert_flags["retry_scheduled_threshold_exceeded"]:
@ -527,6 +603,56 @@ class ScholarIngestionService:
"crawl_run_id": run.id,
"retries_scheduled_count": retries_scheduled_count,
"threshold": bounded_retry_threshold,
"metric_name": "ingestion_retry_scheduled_threshold_exceeded_total",
"metric_value": 1,
},
)
pre_apply_safety_state = run_safety_service.get_safety_event_context(
user_settings,
now_utc=datetime.now(timezone.utc),
)
safety_state, cooldown_trigger_reason = run_safety_service.apply_run_safety_outcome(
user_settings,
run_id=int(run.id),
blocked_failure_count=blocked_failure_count,
network_failure_count=network_failure_count,
blocked_failure_threshold=bounded_blocked_threshold,
network_failure_threshold=bounded_network_threshold,
blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds,
network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds,
now_utc=datetime.now(timezone.utc),
)
if cooldown_trigger_reason is not None:
logger.warning(
"ingestion.safety_cooldown_entered",
extra={
"event": "ingestion.safety_cooldown_entered",
"user_id": user_id,
"crawl_run_id": run.id,
"reason": cooldown_trigger_reason,
"blocked_failure_count": blocked_failure_count,
"network_failure_count": network_failure_count,
"blocked_failure_threshold": bounded_blocked_threshold,
"network_failure_threshold": bounded_network_threshold,
"cooldown_until": safety_state.get("cooldown_until"),
"cooldown_remaining_seconds": safety_state.get("cooldown_remaining_seconds"),
"safety_counters": safety_state.get("counters", {}),
"metric_name": "ingestion_safety_cooldown_entered_total",
"metric_value": 1,
},
)
elif pre_apply_safety_state.get("cooldown_active") and not safety_state.get("cooldown_active"):
logger.info(
"ingestion.safety_cooldown_cleared",
extra={
"event": "ingestion.safety_cooldown_cleared",
"user_id": user_id,
"crawl_run_id": run.id,
"reason": pre_apply_safety_state.get("cooldown_reason"),
"cooldown_until": pre_apply_safety_state.get("cooldown_until"),
"metric_name": "ingestion_safety_cooldown_cleared_total",
"metric_value": 1,
},
)

239
app/services/run_safety.py Normal file
View file

@ -0,0 +1,239 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import Any
from app.db.models import UserSetting
COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD = "blocked_failure_threshold_exceeded"
COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD = "network_failure_threshold_exceeded"
_COUNTER_CONSECUTIVE_BLOCKED_RUNS = "consecutive_blocked_runs"
_COUNTER_CONSECUTIVE_NETWORK_RUNS = "consecutive_network_runs"
_COUNTER_COOLDOWN_ENTRY_COUNT = "cooldown_entry_count"
_COUNTER_BLOCKED_START_COUNT = "blocked_start_count"
_COUNTER_LAST_BLOCKED_FAILURE_COUNT = "last_blocked_failure_count"
_COUNTER_LAST_NETWORK_FAILURE_COUNT = "last_network_failure_count"
_COUNTER_LAST_EVALUATED_RUN_ID = "last_evaluated_run_id"
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
def _safe_int(value: Any, default: int = 0) -> int:
try:
return int(value)
except (TypeError, ValueError):
return default
def _safe_optional_int(value: Any) -> int | None:
if value is None:
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _normalize_datetime(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
def _state_dict(settings: UserSetting) -> dict[str, Any]:
state = settings.scrape_safety_state
if isinstance(state, dict):
return state
return {}
def _counters_from_state(settings: UserSetting) -> dict[str, Any]:
state = _state_dict(settings)
return {
_COUNTER_CONSECUTIVE_BLOCKED_RUNS: max(
0,
_safe_int(state.get(_COUNTER_CONSECUTIVE_BLOCKED_RUNS), 0),
),
_COUNTER_CONSECUTIVE_NETWORK_RUNS: max(
0,
_safe_int(state.get(_COUNTER_CONSECUTIVE_NETWORK_RUNS), 0),
),
_COUNTER_COOLDOWN_ENTRY_COUNT: max(
0,
_safe_int(state.get(_COUNTER_COOLDOWN_ENTRY_COUNT), 0),
),
_COUNTER_BLOCKED_START_COUNT: max(
0,
_safe_int(state.get(_COUNTER_BLOCKED_START_COUNT), 0),
),
_COUNTER_LAST_BLOCKED_FAILURE_COUNT: max(
0,
_safe_int(state.get(_COUNTER_LAST_BLOCKED_FAILURE_COUNT), 0),
),
_COUNTER_LAST_NETWORK_FAILURE_COUNT: max(
0,
_safe_int(state.get(_COUNTER_LAST_NETWORK_FAILURE_COUNT), 0),
),
_COUNTER_LAST_EVALUATED_RUN_ID: _safe_optional_int(
state.get(_COUNTER_LAST_EVALUATED_RUN_ID),
),
}
def _cooldown_reason_label(reason: str | None) -> str | None:
if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
return "Blocked responses exceeded safety threshold"
if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
return "Network failures exceeded safety threshold"
return None
def _recommended_action(reason: str | None) -> str | None:
if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD:
return (
"Google Scholar appears to be blocking requests. Wait for cooldown to expire, "
"increase request delay, and avoid repeated manual retries."
)
if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
return (
"Network failures crossed the threshold. Verify connectivity and retry after cooldown."
)
return None
def is_cooldown_active(
settings: UserSetting,
*,
now_utc: datetime | None = None,
) -> bool:
now = now_utc or _utcnow()
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
if cooldown_until is None:
return False
return cooldown_until > now
def clear_expired_cooldown(
settings: UserSetting,
*,
now_utc: datetime | None = None,
) -> bool:
now = now_utc or _utcnow()
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
if cooldown_until is None:
return False
if cooldown_until > now:
return False
settings.scrape_cooldown_until = None
settings.scrape_cooldown_reason = None
return True
def register_cooldown_blocked_start(
settings: UserSetting,
*,
now_utc: datetime | None = None,
) -> dict[str, Any]:
now = now_utc or _utcnow()
counters = _counters_from_state(settings)
counters[_COUNTER_BLOCKED_START_COUNT] = int(counters[_COUNTER_BLOCKED_START_COUNT]) + 1
settings.scrape_safety_state = counters
return get_safety_state_payload(settings, now_utc=now)
def apply_run_safety_outcome(
settings: UserSetting,
*,
run_id: int,
blocked_failure_count: int,
network_failure_count: int,
blocked_failure_threshold: int,
network_failure_threshold: int,
blocked_cooldown_seconds: int,
network_cooldown_seconds: int,
now_utc: datetime | None = None,
) -> tuple[dict[str, Any], str | None]:
now = now_utc or _utcnow()
counters = _counters_from_state(settings)
bounded_blocked_failures = max(0, int(blocked_failure_count))
bounded_network_failures = max(0, int(network_failure_count))
counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures
counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures
counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id)
counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = (
int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1
if bounded_blocked_failures > 0
else 0
)
counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = (
int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1
if bounded_network_failures > 0
else 0
)
reason: str | None = None
cooldown_seconds = 0
if bounded_blocked_failures >= max(1, int(blocked_failure_threshold)):
reason = COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD
cooldown_seconds = max(60, int(blocked_cooldown_seconds))
elif bounded_network_failures >= max(1, int(network_failure_threshold)):
reason = COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD
cooldown_seconds = max(60, int(network_cooldown_seconds))
if reason is not None:
settings.scrape_cooldown_reason = reason
settings.scrape_cooldown_until = now + timedelta(seconds=cooldown_seconds)
counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1
else:
clear_expired_cooldown(settings, now_utc=now)
settings.scrape_safety_state = counters
return get_safety_state_payload(settings, now_utc=now), reason
def get_safety_state_payload(
settings: UserSetting,
*,
now_utc: datetime | None = None,
) -> dict[str, Any]:
now = now_utc or _utcnow()
cooldown_until = _normalize_datetime(settings.scrape_cooldown_until)
cooldown_active = bool(cooldown_until is not None and cooldown_until > now)
cooldown_remaining_seconds = 0
if cooldown_active and cooldown_until is not None:
cooldown_remaining_seconds = max(0, int((cooldown_until - now).total_seconds()))
reason = settings.scrape_cooldown_reason if cooldown_active else None
return {
"cooldown_active": cooldown_active,
"cooldown_reason": reason,
"cooldown_reason_label": _cooldown_reason_label(reason),
"cooldown_until": cooldown_until,
"cooldown_remaining_seconds": cooldown_remaining_seconds,
"recommended_action": _recommended_action(reason),
"counters": _counters_from_state(settings),
}
def get_safety_event_context(
settings: UserSetting,
*,
now_utc: datetime | None = None,
) -> dict[str, Any]:
payload = get_safety_state_payload(settings, now_utc=now_utc)
return {
"cooldown_active": bool(payload.get("cooldown_active")),
"cooldown_reason": payload.get("cooldown_reason"),
"cooldown_until": payload.get("cooldown_until"),
"cooldown_remaining_seconds": int(payload.get("cooldown_remaining_seconds") or 0),
"safety_counters": payload.get("counters", {}),
}

91
app/services/scheduler.py
View file

@ -18,7 +18,11 @@ from app.db.models import (
)
from app.db.session import get_session_factory
from app.services import continuation_queue as queue_service
from app.services.ingestion import RunAlreadyInProgressError, ScholarIngestionService
from app.services.ingestion import (
RunAlreadyInProgressError,
RunBlockedBySafetyPolicyError,
ScholarIngestionService,
)
from app.services.scholar_source import LiveScholarSource
from app.settings import settings
@ -30,6 +34,8 @@ class _AutoRunCandidate:
user_id: int
run_interval_minutes: int
request_delay_seconds: int
cooldown_until: datetime | None
cooldown_reason: str | None
class SchedulerService:
@ -134,6 +140,9 @@ class SchedulerService:
await self._run_candidate(candidate)
async def _load_candidates(self) -> list[_AutoRunCandidate]:
if not settings.ingestion_automation_allowed:
return []
session_factory = get_session_factory()
async with session_factory() as session:
result = await session.execute(
@ -141,6 +150,8 @@ class SchedulerService:
UserSetting.user_id,
UserSetting.run_interval_minutes,
UserSetting.request_delay_seconds,
UserSetting.scrape_cooldown_until,
UserSetting.scrape_cooldown_reason,
)
.join(User, User.id == UserSetting.user_id)
.where(
@ -150,14 +161,35 @@ class SchedulerService:
.order_by(UserSetting.user_id.asc())
)
rows = result.all()
return [
_AutoRunCandidate(
user_id=int(user_id),
run_interval_minutes=int(run_interval_minutes),
request_delay_seconds=int(request_delay_seconds),
now_utc = datetime.now(timezone.utc)
candidates: list[_AutoRunCandidate] = []
for user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason in rows:
if cooldown_until is not None and cooldown_until.tzinfo is None:
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
if cooldown_until is not None and cooldown_until > now_utc:
logger.info(
"scheduler.run_skipped_safety_cooldown_precheck",
extra={
"event": "scheduler.run_skipped_safety_cooldown_precheck",
"user_id": int(user_id),
"reason": cooldown_reason,
"cooldown_until": cooldown_until,
"cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()),
"metric_name": "scheduler_run_skipped_safety_cooldown_total",
"metric_value": 1,
},
)
continue
candidates.append(
_AutoRunCandidate(
user_id=int(user_id),
run_interval_minutes=int(run_interval_minutes),
request_delay_seconds=int(request_delay_seconds),
cooldown_until=cooldown_until,
cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
)
)
for user_id, run_interval_minutes, request_delay_seconds in rows
]
return candidates
async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
session_factory = get_session_factory()
@ -210,6 +242,21 @@ class SchedulerService:
},
)
return
except RunBlockedBySafetyPolicyError as exc:
await session.rollback()
logger.info(
"scheduler.run_skipped_safety_cooldown",
extra={
"event": "scheduler.run_skipped_safety_cooldown",
"user_id": candidate.user_id,
"reason": exc.safety_state.get("cooldown_reason"),
"cooldown_until": exc.safety_state.get("cooldown_until"),
"cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"),
"metric_name": "scheduler_run_skipped_safety_cooldown_total",
"metric_value": 1,
},
)
return
except Exception:
await session.rollback()
logger.exception(
@ -354,6 +401,34 @@ class SchedulerService:
},
)
return
except RunBlockedBySafetyPolicyError as exc:
await session.rollback()
cooldown_remaining_seconds = max(
self._tick_seconds,
int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
)
async with session_factory() as recovery_session:
await queue_service.reschedule_job(
recovery_session,
job_id=job.id,
delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds),
reason="scrape_safety_cooldown",
error=str(exc.message),
)
await recovery_session.commit()
logger.info(
"scheduler.queue_item_deferred_safety_cooldown",
extra={
"event": "scheduler.queue_item_deferred_safety_cooldown",
"queue_item_id": job.id,
"user_id": job.user_id,
"reason": exc.safety_state.get("cooldown_reason"),
"cooldown_remaining_seconds": cooldown_remaining_seconds,
"metric_name": "scheduler_queue_item_deferred_safety_cooldown_total",
"metric_value": 1,
},
)
return
except Exception as exc:
await session.rollback()
async with session_factory() as recovery_session:

36
app/services/user_settings.py
View file

@ -33,25 +33,49 @@ REQUIRED_NAV_PAGES = (
NAV_PAGE_SETTINGS,
)
DEFAULT_NAV_VISIBLE_PAGES = list(ALLOWED_NAV_PAGES)
HARD_MIN_RUN_INTERVAL_MINUTES = 15
HARD_MIN_REQUEST_DELAY_SECONDS = 2
def parse_run_interval_minutes(value: str) -> int:
def resolve_run_interval_minimum(configured_minimum: int | None) -> int:
try:
parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_RUN_INTERVAL_MINUTES
except (TypeError, ValueError):
parsed = HARD_MIN_RUN_INTERVAL_MINUTES
return max(HARD_MIN_RUN_INTERVAL_MINUTES, parsed)
def resolve_request_delay_minimum(configured_minimum: int | None) -> int:
try:
parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_REQUEST_DELAY_SECONDS
except (TypeError, ValueError):
parsed = HARD_MIN_REQUEST_DELAY_SECONDS
return max(HARD_MIN_REQUEST_DELAY_SECONDS, parsed)
def parse_run_interval_minutes(value: str, *, minimum: int = HARD_MIN_RUN_INTERVAL_MINUTES) -> int:
try:
parsed = int(value)
except ValueError as exc:
raise UserSettingsServiceError("Check interval must be a whole number.") from exc
if parsed < 15:
raise UserSettingsServiceError("Check interval must be at least 15 minutes.")
effective_minimum = resolve_run_interval_minimum(minimum)
if parsed < effective_minimum:
raise UserSettingsServiceError(
f"Check interval must be at least {effective_minimum} minutes."
)
return parsed
def parse_request_delay_seconds(value: str) -> int:
def parse_request_delay_seconds(value: str, *, minimum: int = HARD_MIN_REQUEST_DELAY_SECONDS) -> int:
try:
parsed = int(value)
except ValueError as exc:
raise UserSettingsServiceError("Request delay must be a whole number.") from exc
if parsed < 2:
raise UserSettingsServiceError("Request delay must be at least 2 seconds.")
effective_minimum = resolve_request_delay_minimum(minimum)
if parsed < effective_minimum:
raise UserSettingsServiceError(
f"Request delay must be at least {effective_minimum} seconds."
)
return parsed

24
app/settings.py
View file

@ -115,6 +115,22 @@ class Settings:
log_redact_fields: str = os.getenv("LOG_REDACT_FIELDS", "")
scheduler_enabled: bool = _env_bool("SCHEDULER_ENABLED", True)
scheduler_tick_seconds: int = _env_int("SCHEDULER_TICK_SECONDS", 60)
ingestion_automation_allowed: bool = _env_bool(
"INGESTION_AUTOMATION_ALLOWED",
True,
)
ingestion_manual_run_allowed: bool = _env_bool(
"INGESTION_MANUAL_RUN_ALLOWED",
True,
)
ingestion_min_run_interval_minutes: int = _env_int(
"INGESTION_MIN_RUN_INTERVAL_MINUTES",
15,
)
ingestion_min_request_delay_seconds: int = _env_int(
"INGESTION_MIN_REQUEST_DELAY_SECONDS",
2,
)
ingestion_network_error_retries: int = _env_int("INGESTION_NETWORK_ERROR_RETRIES", 1)
ingestion_retry_backoff_seconds: float = _env_float(
"INGESTION_RETRY_BACKOFF_SECONDS",
@ -137,6 +153,14 @@ class Settings:
"INGESTION_ALERT_RETRY_SCHEDULED_THRESHOLD",
3,
)
ingestion_safety_cooldown_blocked_seconds: int = _env_int(
"INGESTION_SAFETY_COOLDOWN_BLOCKED_SECONDS",
1800,
)
ingestion_safety_cooldown_network_seconds: int = _env_int(
"INGESTION_SAFETY_COOLDOWN_NETWORK_SECONDS",
900,
)
ingestion_continuation_queue_enabled: bool = _env_bool(
"INGESTION_CONTINUATION_QUEUE_ENABLED",
True,