Better crawling safety with preflight checks and cooldowns

app/services/ingestion/application.py

@@ -21,6 +21,7 @@ from app.services.ingestion import safety as run_safety_service
 from app.services.ingestion.constants import RUN_LOCK_NAMESPACE
 from app.services.ingestion.enrichment import EnrichmentRunner
 from app.services.ingestion.pagination import PaginationEngine
+from app.services.ingestion.preflight import check_scholar_reachable
 from app.services.ingestion.run_completion import (
     complete_run_for_user,
     int_or_default,
@@ -231,6 +232,49 @@ class ScholarIngestionService:
                 await queue_service.clear_job_for_scholar(db_session, user_id=user_id, scholar_profile_id=sid)
         return scholars
 
+    async def _run_preflight_guard(
+        self,
+        db_session: AsyncSession,
+        *,
+        user_settings: Any,
+        user_id: int,
+        scholars: list[ScholarProfile],
+    ) -> None:
+        if not scholars:
+            return
+        result = await check_scholar_reachable(
+            self._source,
+            scholar_id=scholars[0].scholar_id,
+        )
+        if result.passed:
+            return
+        now_utc = datetime.now(UTC)
+        safety_state, _ = run_safety_service.apply_run_safety_outcome(
+            user_settings,
+            run_id=0,
+            blocked_failure_count=1,
+            network_failure_count=0,
+            blocked_failure_threshold=1,
+            network_failure_threshold=1,
+            blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds,
+            network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds,
+            now_utc=now_utc,
+        )
+        await db_session.commit()
+        structured_log(
+            logger,
+            "warning",
+            "ingestion.cooldown_entered_preflight",
+            user_id=user_id,
+            block_reason=result.block_reason,
+            cooldown_until=safety_state.get("cooldown_until"),
+        )
+        raise RunBlockedBySafetyPolicyError(
+            code="scrape_cooldown_active",
+            message=f"Preflight detected Scholar block ({result.block_reason}). Cooldown activated.",
+            safety_state=safety_state,
+        )
+
     async def _initialize_run_for_user(
         self,
         db_session: AsyncSession,
@@ -257,6 +301,12 @@ class ScholarIngestionService:
             user_id=user_id,
             filtered_scholar_ids=filtered_scholar_ids,
         )
+        await self._run_preflight_guard(
+            db_session,
+            user_settings=user_settings,
+            user_id=user_id,
+            scholars=scholars,
+        )
         run = await self._start_run_record(
             db_session,
             user_id=user_id,

app/services/ingestion/preflight.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+
+from app.logging_utils import structured_log
+from app.services.scholar.source import FetchResult, ScholarSource
+from app.services.scholar.state_detection import (
+    classify_block_or_captcha_reason,
+    is_hard_challenge_reason,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class PreflightResult:
+    passed: bool
+    block_reason: str | None
+    status_code: int | None
+
+
+def _evaluate_fetch_result(fetch_result: FetchResult) -> PreflightResult:
+    if fetch_result.status_code is None:
+        return PreflightResult(passed=True, block_reason=None, status_code=None)
+    block_reason = classify_block_or_captcha_reason(
+        status_code=fetch_result.status_code,
+        final_url=(fetch_result.final_url or "").lower(),
+        body_lowered=fetch_result.body.lower(),
+    )
+    if block_reason is not None and is_hard_challenge_reason(block_reason):
+        return PreflightResult(
+            passed=False,
+            block_reason=block_reason,
+            status_code=fetch_result.status_code,
+        )
+    return PreflightResult(
+        passed=True,
+        block_reason=None,
+        status_code=fetch_result.status_code,
+    )
+
+
+async def check_scholar_reachable(
+    source: ScholarSource,
+    *,
+    scholar_id: str,
+) -> PreflightResult:
+    """Single-request probe to detect active Scholar blocks before a full run."""
+    structured_log(logger, "info", "ingestion.preflight_started", scholar_id=scholar_id)
+    fetch_result = await source.fetch_profile_html(scholar_id)
+    result = _evaluate_fetch_result(fetch_result)
+    if result.passed:
+        structured_log(
+            logger,
+            "info",
+            "ingestion.preflight_passed",
+            scholar_id=scholar_id,
+            status_code=result.status_code,
+        )
+    else:
+        structured_log(
+            logger,
+            "warning",
+            "ingestion.preflight_failed",
+            scholar_id=scholar_id,
+            block_reason=result.block_reason,
+            status_code=result.status_code,
+        )
+    return result

app/services/ingestion/scholar_processing.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import random
 from datetime import UTC, datetime
 from typing import Any
 
@@ -27,6 +28,7 @@ from app.services.ingestion.types import (
     ScholarProcessingOutcome,
 )
 from app.services.scholar.parser import ParseState
+from app.services.scholar.state_detection import is_hard_challenge_reason
 
 logger = logging.getLogger(__name__)
 
@@ -528,6 +530,13 @@ def unexpected_scholar_exception_outcome(
     )
 
 
+def _is_hard_challenge_outcome(outcome: ScholarProcessingOutcome) -> bool:
+    entry = outcome.result_entry
+    return str(entry.get("state", "")) == ParseState.BLOCKED_OR_CAPTCHA.value and is_hard_challenge_reason(
+        str(entry.get("state_reason", ""))
+    )
+
+
 async def _run_first_pass(
     db_session: AsyncSession,
     *,
@@ -548,7 +557,8 @@ async def _run_first_pass(
             structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id)
             return first_pass_cstarts
         if index > 0 and request_delay_seconds > 0:
-            await asyncio.sleep(float(request_delay_seconds))
+            jitter = random.uniform(0.0, min(float(request_delay_seconds), 2.0))
+            await asyncio.sleep(float(request_delay_seconds) + jitter)
         start_cstart = int(start_cstart_map.get(int(scholar.id), 0))
         outcome = await process_scholar(
             db_session,
@@ -563,6 +573,18 @@ async def _run_first_pass(
             **scholar_kwargs,
         )
         apply_outcome_to_progress(progress=progress, outcome=outcome)
+        if _is_hard_challenge_outcome(outcome):
+            structured_log(
+                logger,
+                "warning",
+                "ingestion.run_aborted_hard_challenge",
+                run_id=run.id,
+                user_id=user_id,
+                scholar_id=scholar.scholar_id,
+                state_reason=outcome.result_entry.get("state_reason"),
+                scholars_remaining=len(scholars) - index - 1,
+            )
+            return first_pass_cstarts
         resume_cstart = outcome.result_entry.get("continuation_cstart")
         if resume_cstart is not None and int(resume_cstart) > start_cstart:
             first_pass_cstarts[int(scholar.id)] = int(resume_cstart)
@@ -593,7 +615,8 @@ async def _run_depth_pass(
             structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id)
             break
         if index > 0 and request_delay_seconds > 0:
-            await asyncio.sleep(float(request_delay_seconds))
+            jitter = random.uniform(0.0, min(float(request_delay_seconds), 2.0))
+            await asyncio.sleep(float(request_delay_seconds) + jitter)
         outcome = await process_scholar(
             db_session,
             pagination=pagination,
@@ -607,6 +630,18 @@ async def _run_depth_pass(
             **scholar_kwargs,
         )
         apply_outcome_to_progress(progress=progress, outcome=outcome)
+        if _is_hard_challenge_outcome(outcome):
+            structured_log(
+                logger,
+                "warning",
+                "ingestion.run_aborted_hard_challenge",
+                run_id=run.id,
+                user_id=user_id,
+                scholar_id=scholar.scholar_id,
+                state_reason=outcome.result_entry.get("state_reason"),
+                scholars_remaining=len(scholars) - index - 1,
+            )
+            break
 
 
 async def run_scholar_iteration(

app/services/scholar/state_detection.py

@@ -30,6 +30,11 @@ def classify_network_error_reason(fetch_error: str | None) -> str:
     return "network_error_missing_status_code"
 
 
+def is_hard_challenge_reason(state_reason: str) -> bool:
+    """True if the block reason indicates an active Scholar challenge (not retryable in short loops)."""
+    return state_reason.startswith("blocked_") and state_reason != "blocked_http_429_rate_limited"
+
+
 def classify_block_or_captcha_reason(
     *,
     status_code: int,