From e8e20637e6d7e6f7c32ca42dc454b386ab9f3fcc Mon Sep 17 00:00:00 2001 From: Justin Visser Date: Fri, 27 Feb 2026 16:44:54 +0100 Subject: [PATCH] Better crawling safety with preflight checks and cooldowns --- app/services/ingestion/application.py | 50 ++++++++++++++ app/services/ingestion/preflight.py | 70 ++++++++++++++++++++ app/services/ingestion/scholar_processing.py | 39 ++++++++++- app/services/scholar/state_detection.py | 5 ++ frontend/src/pages/DashboardPage.vue | 5 ++ frontend/src/pages/RunsPage.vue | 5 ++ 6 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 app/services/ingestion/preflight.py diff --git a/app/services/ingestion/application.py b/app/services/ingestion/application.py index 9bfc46a..6087925 100644 --- a/app/services/ingestion/application.py +++ b/app/services/ingestion/application.py @@ -21,6 +21,7 @@ from app.services.ingestion import safety as run_safety_service from app.services.ingestion.constants import RUN_LOCK_NAMESPACE from app.services.ingestion.enrichment import EnrichmentRunner from app.services.ingestion.pagination import PaginationEngine +from app.services.ingestion.preflight import check_scholar_reachable from app.services.ingestion.run_completion import ( complete_run_for_user, int_or_default, @@ -231,6 +232,49 @@ class ScholarIngestionService: await queue_service.clear_job_for_scholar(db_session, user_id=user_id, scholar_profile_id=sid) return scholars + async def _run_preflight_guard( + self, + db_session: AsyncSession, + *, + user_settings: Any, + user_id: int, + scholars: list[ScholarProfile], + ) -> None: + if not scholars: + return + result = await check_scholar_reachable( + self._source, + scholar_id=scholars[0].scholar_id, + ) + if result.passed: + return + now_utc = datetime.now(UTC) + safety_state, _ = run_safety_service.apply_run_safety_outcome( + user_settings, + run_id=0, + blocked_failure_count=1, + network_failure_count=0, + blocked_failure_threshold=1, + network_failure_threshold=1, + blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds, + network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds, + now_utc=now_utc, + ) + await db_session.commit() + structured_log( + logger, + "warning", + "ingestion.cooldown_entered_preflight", + user_id=user_id, + block_reason=result.block_reason, + cooldown_until=safety_state.get("cooldown_until"), + ) + raise RunBlockedBySafetyPolicyError( + code="scrape_cooldown_active", + message=f"Preflight detected Scholar block ({result.block_reason}). Cooldown activated.", + safety_state=safety_state, + ) + async def _initialize_run_for_user( self, db_session: AsyncSession, @@ -257,6 +301,12 @@ class ScholarIngestionService: user_id=user_id, filtered_scholar_ids=filtered_scholar_ids, ) + await self._run_preflight_guard( + db_session, + user_settings=user_settings, + user_id=user_id, + scholars=scholars, + ) run = await self._start_run_record( db_session, user_id=user_id, diff --git a/app/services/ingestion/preflight.py b/app/services/ingestion/preflight.py new file mode 100644 index 0000000..44441e3 --- /dev/null +++ b/app/services/ingestion/preflight.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass + +from app.logging_utils import structured_log +from app.services.scholar.source import FetchResult, ScholarSource +from app.services.scholar.state_detection import ( + classify_block_or_captcha_reason, + is_hard_challenge_reason, +) + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class PreflightResult: + passed: bool + block_reason: str | None + status_code: int | None + + +def _evaluate_fetch_result(fetch_result: FetchResult) -> PreflightResult: + if fetch_result.status_code is None: + return PreflightResult(passed=True, block_reason=None, status_code=None) + block_reason = classify_block_or_captcha_reason( + status_code=fetch_result.status_code, + final_url=(fetch_result.final_url or "").lower(), + body_lowered=fetch_result.body.lower(), + ) + if block_reason is not None and is_hard_challenge_reason(block_reason): + return PreflightResult( + passed=False, + block_reason=block_reason, + status_code=fetch_result.status_code, + ) + return PreflightResult( + passed=True, + block_reason=None, + status_code=fetch_result.status_code, + ) + + +async def check_scholar_reachable( + source: ScholarSource, + *, + scholar_id: str, +) -> PreflightResult: + """Single-request probe to detect active Scholar blocks before a full run.""" + structured_log(logger, "info", "ingestion.preflight_started", scholar_id=scholar_id) + fetch_result = await source.fetch_profile_html(scholar_id) + result = _evaluate_fetch_result(fetch_result) + if result.passed: + structured_log( + logger, + "info", + "ingestion.preflight_passed", + scholar_id=scholar_id, + status_code=result.status_code, + ) + else: + structured_log( + logger, + "warning", + "ingestion.preflight_failed", + scholar_id=scholar_id, + block_reason=result.block_reason, + status_code=result.status_code, + ) + return result diff --git a/app/services/ingestion/scholar_processing.py b/app/services/ingestion/scholar_processing.py index 2bbddfd..37b47d3 100644 --- a/app/services/ingestion/scholar_processing.py +++ b/app/services/ingestion/scholar_processing.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import logging +import random from datetime import UTC, datetime from typing import Any @@ -27,6 +28,7 @@ from app.services.ingestion.types import ( ScholarProcessingOutcome, ) from app.services.scholar.parser import ParseState +from app.services.scholar.state_detection import is_hard_challenge_reason logger = logging.getLogger(__name__) @@ -528,6 +530,13 @@ def unexpected_scholar_exception_outcome( ) +def _is_hard_challenge_outcome(outcome: ScholarProcessingOutcome) -> bool: + entry = outcome.result_entry + return str(entry.get("state", "")) == ParseState.BLOCKED_OR_CAPTCHA.value and is_hard_challenge_reason( + str(entry.get("state_reason", "")) + ) + + async def _run_first_pass( db_session: AsyncSession, *, @@ -548,7 +557,8 @@ async def _run_first_pass( structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id) return first_pass_cstarts if index > 0 and request_delay_seconds > 0: - await asyncio.sleep(float(request_delay_seconds)) + jitter = random.uniform(0.0, min(float(request_delay_seconds), 2.0)) + await asyncio.sleep(float(request_delay_seconds) + jitter) start_cstart = int(start_cstart_map.get(int(scholar.id), 0)) outcome = await process_scholar( db_session, @@ -563,6 +573,18 @@ async def _run_first_pass( **scholar_kwargs, ) apply_outcome_to_progress(progress=progress, outcome=outcome) + if _is_hard_challenge_outcome(outcome): + structured_log( + logger, + "warning", + "ingestion.run_aborted_hard_challenge", + run_id=run.id, + user_id=user_id, + scholar_id=scholar.scholar_id, + state_reason=outcome.result_entry.get("state_reason"), + scholars_remaining=len(scholars) - index - 1, + ) + return first_pass_cstarts resume_cstart = outcome.result_entry.get("continuation_cstart") if resume_cstart is not None and int(resume_cstart) > start_cstart: first_pass_cstarts[int(scholar.id)] = int(resume_cstart) @@ -593,7 +615,8 @@ async def _run_depth_pass( structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id) break if index > 0 and request_delay_seconds > 0: - await asyncio.sleep(float(request_delay_seconds)) + jitter = random.uniform(0.0, min(float(request_delay_seconds), 2.0)) + await asyncio.sleep(float(request_delay_seconds) + jitter) outcome = await process_scholar( db_session, pagination=pagination, @@ -607,6 +630,18 @@ async def _run_depth_pass( **scholar_kwargs, ) apply_outcome_to_progress(progress=progress, outcome=outcome) + if _is_hard_challenge_outcome(outcome): + structured_log( + logger, + "warning", + "ingestion.run_aborted_hard_challenge", + run_id=run.id, + user_id=user_id, + scholar_id=scholar.scholar_id, + state_reason=outcome.result_entry.get("state_reason"), + scholars_remaining=len(scholars) - index - 1, + ) + break async def run_scholar_iteration( diff --git a/app/services/scholar/state_detection.py b/app/services/scholar/state_detection.py index 7532afa..28a413d 100644 --- a/app/services/scholar/state_detection.py +++ b/app/services/scholar/state_detection.py @@ -30,6 +30,11 @@ def classify_network_error_reason(fetch_error: str | None) -> str: return "network_error_missing_status_code" +def is_hard_challenge_reason(state_reason: str) -> bool: + """True if the block reason indicates an active Scholar challenge (not retryable in short loops).""" + return state_reason.startswith("blocked_") and state_reason != "blocked_http_429_rate_limited" + + def classify_block_or_captcha_reason( *, status_code: int, diff --git a/frontend/src/pages/DashboardPage.vue b/frontend/src/pages/DashboardPage.vue index fce9e02..fbd0b4c 100644 --- a/frontend/src/pages/DashboardPage.vue +++ b/frontend/src/pages/DashboardPage.vue @@ -2,6 +2,7 @@ import { computed, onMounted, onUnmounted, ref, watch } from "vue"; import { fetchDashboardSnapshot, type DashboardSnapshot } from "@/features/dashboard"; +import { formatCooldownCountdown } from "@/features/safety"; import { ApiRequestError } from "@/lib/api/errors"; import AppPage from "@/components/layout/AppPage.vue"; import AsyncStateGate from "@/components/patterns/AsyncStateGate.vue"; @@ -53,6 +54,10 @@ const startCheckLabel = computed(() => { return "Manual checks disabled"; } if (runStatus.safetyState.cooldown_active) { + const remaining = runStatus.safetyState.cooldown_remaining_seconds; + if (remaining > 0) { + return `Cooldown (${formatCooldownCountdown(remaining)})`; + } return "Safety cooldown"; } if (runStatus.isLikelyRunning) { diff --git a/frontend/src/pages/RunsPage.vue b/frontend/src/pages/RunsPage.vue index d28a3dc..49894b7 100644 --- a/frontend/src/pages/RunsPage.vue +++ b/frontend/src/pages/RunsPage.vue @@ -22,6 +22,7 @@ import { type QueueItem, type RunListItem, } from "@/features/runs"; +import { formatCooldownCountdown } from "@/features/safety"; import { ApiRequestError } from "@/lib/api/errors"; import { useAuthStore } from "@/stores/auth"; import { useRunStatusStore } from "@/stores/run_status"; @@ -89,6 +90,10 @@ const runButtonLabel = computed(() => { return "Manual checks disabled"; } if (runStatus.safetyState.cooldown_active) { + const remaining = runStatus.safetyState.cooldown_remaining_seconds; + if (remaining > 0) { + return `Cooldown (${formatCooldownCountdown(remaining)})`; + } return "Safety cooldown"; } if (runStatus.isLikelyRunning) {