decompose application.py into enrichment, scholar processing, run completion
This commit is contained in:
parent
9a7fa5004e
commit
6379c97e90
28 changed files with 2982 additions and 2576 deletions
417
app/services/ingestion/run_completion.py
Normal file
417
app/services/ingestion/run_completion.py
Normal file
|
|
@ -0,0 +1,417 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
RunStatus,
|
||||
ScholarProfile,
|
||||
)
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.ingestion import safety as run_safety_service
|
||||
from app.services.ingestion.constants import (
|
||||
FAILED_STATES,
|
||||
FAILURE_BUCKET_BLOCKED,
|
||||
FAILURE_BUCKET_INGESTION,
|
||||
FAILURE_BUCKET_LAYOUT,
|
||||
FAILURE_BUCKET_NETWORK,
|
||||
FAILURE_BUCKET_OTHER,
|
||||
)
|
||||
from app.services.ingestion.fingerprints import _build_body_excerpt
|
||||
from app.services.ingestion.types import (
|
||||
RunAlertSummary,
|
||||
RunExecutionSummary,
|
||||
RunFailureSummary,
|
||||
RunProgress,
|
||||
ScholarProcessingOutcome,
|
||||
)
|
||||
from app.services.scholar.parser import ParsedProfilePage, ParseState
|
||||
from app.services.scholar.source import FetchResult
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _int_or_default(value: Any, default: int = 0) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def classify_failure_bucket(*, state: str, state_reason: str) -> str:
|
||||
reason = state_reason.strip().lower()
|
||||
normalized_state = state.strip().lower()
|
||||
|
||||
if normalized_state == ParseState.BLOCKED_OR_CAPTCHA.value or reason.startswith("blocked_"):
|
||||
return FAILURE_BUCKET_BLOCKED
|
||||
if normalized_state == ParseState.NETWORK_ERROR.value or reason.startswith("network_"):
|
||||
return FAILURE_BUCKET_NETWORK
|
||||
if normalized_state == ParseState.LAYOUT_CHANGED.value:
|
||||
return FAILURE_BUCKET_LAYOUT
|
||||
if normalized_state == "ingestion_error":
|
||||
return FAILURE_BUCKET_INGESTION
|
||||
return FAILURE_BUCKET_OTHER
|
||||
|
||||
|
||||
def summarize_failures(
|
||||
*,
|
||||
scholar_results: list[dict[str, Any]],
|
||||
) -> RunFailureSummary:
|
||||
failed_state_counts: dict[str, int] = {}
|
||||
failed_reason_counts: dict[str, int] = {}
|
||||
scrape_failure_counts: dict[str, int] = {}
|
||||
retries_scheduled_count = 0
|
||||
scholars_with_retries_count = 0
|
||||
retry_exhausted_count = 0
|
||||
for entry in scholar_results:
|
||||
retries_for_entry = max(0, _int_or_default(entry.get("attempt_count"), 0) - 1)
|
||||
if retries_for_entry > 0:
|
||||
retries_scheduled_count += retries_for_entry
|
||||
scholars_with_retries_count += 1
|
||||
if str(entry.get("outcome", "")) != "failed":
|
||||
continue
|
||||
state = str(entry.get("state", "")).strip()
|
||||
if state not in FAILED_STATES:
|
||||
continue
|
||||
failed_state_counts[state] = failed_state_counts.get(state, 0) + 1
|
||||
reason = str(entry.get("state_reason", "")).strip()
|
||||
if reason:
|
||||
failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1
|
||||
bucket = classify_failure_bucket(state=state, state_reason=reason)
|
||||
scrape_failure_counts[bucket] = scrape_failure_counts.get(bucket, 0) + 1
|
||||
if state == ParseState.NETWORK_ERROR.value and retries_for_entry > 0:
|
||||
retry_exhausted_count += 1
|
||||
return RunFailureSummary(
|
||||
failed_state_counts=failed_state_counts,
|
||||
failed_reason_counts=failed_reason_counts,
|
||||
scrape_failure_counts=scrape_failure_counts,
|
||||
retries_scheduled_count=retries_scheduled_count,
|
||||
scholars_with_retries_count=scholars_with_retries_count,
|
||||
retry_exhausted_count=retry_exhausted_count,
|
||||
)
|
||||
|
||||
|
||||
def build_alert_summary(
|
||||
*,
|
||||
failure_summary: RunFailureSummary,
|
||||
alert_blocked_failure_threshold: int,
|
||||
alert_network_failure_threshold: int,
|
||||
alert_retry_scheduled_threshold: int,
|
||||
) -> RunAlertSummary:
|
||||
blocked_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_BLOCKED, 0))
|
||||
network_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_NETWORK, 0))
|
||||
blocked_threshold = max(1, int(alert_blocked_failure_threshold))
|
||||
network_threshold = max(1, int(alert_network_failure_threshold))
|
||||
retry_threshold = max(1, int(alert_retry_scheduled_threshold))
|
||||
alert_flags = {
|
||||
"blocked_failure_threshold_exceeded": blocked_failure_count >= blocked_threshold,
|
||||
"network_failure_threshold_exceeded": network_failure_count >= network_threshold,
|
||||
"retry_scheduled_threshold_exceeded": failure_summary.retries_scheduled_count >= retry_threshold,
|
||||
}
|
||||
return RunAlertSummary(
|
||||
blocked_failure_count=blocked_failure_count,
|
||||
network_failure_count=network_failure_count,
|
||||
blocked_failure_threshold=blocked_threshold,
|
||||
network_failure_threshold=network_threshold,
|
||||
retry_scheduled_threshold=retry_threshold,
|
||||
alert_flags=alert_flags,
|
||||
)
|
||||
|
||||
|
||||
def apply_safety_outcome(
|
||||
*,
|
||||
user_settings: Any,
|
||||
run: CrawlRun,
|
||||
user_id: int,
|
||||
alert_summary: RunAlertSummary,
|
||||
) -> None:
|
||||
pre_apply_state = run_safety_service.get_safety_event_context(
|
||||
user_settings,
|
||||
now_utc=datetime.now(UTC),
|
||||
)
|
||||
safety_state, cooldown_trigger_reason = run_safety_service.apply_run_safety_outcome(
|
||||
user_settings,
|
||||
run_id=int(run.id),
|
||||
blocked_failure_count=alert_summary.blocked_failure_count,
|
||||
network_failure_count=alert_summary.network_failure_count,
|
||||
blocked_failure_threshold=alert_summary.blocked_failure_threshold,
|
||||
network_failure_threshold=alert_summary.network_failure_threshold,
|
||||
blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds,
|
||||
network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds,
|
||||
now_utc=datetime.now(UTC),
|
||||
)
|
||||
if cooldown_trigger_reason is not None:
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"ingestion.safety_cooldown_entered",
|
||||
user_id=user_id,
|
||||
crawl_run_id=int(run.id),
|
||||
reason=cooldown_trigger_reason,
|
||||
blocked_failure_count=alert_summary.blocked_failure_count,
|
||||
network_failure_count=alert_summary.network_failure_count,
|
||||
blocked_failure_threshold=alert_summary.blocked_failure_threshold,
|
||||
network_failure_threshold=alert_summary.network_failure_threshold,
|
||||
cooldown_until=safety_state.get("cooldown_until"),
|
||||
cooldown_remaining_seconds=safety_state.get("cooldown_remaining_seconds"),
|
||||
safety_counters=safety_state.get("counters", {}),
|
||||
)
|
||||
elif pre_apply_state.get("cooldown_active") and not safety_state.get("cooldown_active"):
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"ingestion.cooldown_cleared",
|
||||
user_id=user_id,
|
||||
crawl_run_id=int(run.id),
|
||||
reason=pre_apply_state.get("cooldown_reason"),
|
||||
cooldown_until=pre_apply_state.get("cooldown_until"),
|
||||
)
|
||||
|
||||
|
||||
def finalize_run_record(
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholars: list[ScholarProfile],
|
||||
progress: RunProgress,
|
||||
failure_summary: RunFailureSummary,
|
||||
alert_summary: RunAlertSummary,
|
||||
idempotency_key: str | None,
|
||||
run_status: RunStatus,
|
||||
) -> None:
|
||||
run.end_dt = datetime.now(UTC)
|
||||
if run.status != RunStatus.CANCELED:
|
||||
run.status = run_status
|
||||
run.error_log = {
|
||||
"scholar_results": progress.scholar_results,
|
||||
"summary": {
|
||||
"succeeded_count": progress.succeeded_count,
|
||||
"failed_count": progress.failed_count,
|
||||
"partial_count": progress.partial_count,
|
||||
"failed_state_counts": failure_summary.failed_state_counts,
|
||||
"failed_reason_counts": failure_summary.failed_reason_counts,
|
||||
"scrape_failure_counts": failure_summary.scrape_failure_counts,
|
||||
"retry_counts": {
|
||||
"retries_scheduled_count": failure_summary.retries_scheduled_count,
|
||||
"scholars_with_retries_count": failure_summary.scholars_with_retries_count,
|
||||
"retry_exhausted_count": failure_summary.retry_exhausted_count,
|
||||
},
|
||||
"alert_thresholds": {
|
||||
"blocked_failure_threshold": alert_summary.blocked_failure_threshold,
|
||||
"network_failure_threshold": alert_summary.network_failure_threshold,
|
||||
"retry_scheduled_threshold": alert_summary.retry_scheduled_threshold,
|
||||
},
|
||||
"alert_flags": alert_summary.alert_flags,
|
||||
},
|
||||
"meta": {"idempotency_key": idempotency_key} if idempotency_key else {},
|
||||
}
|
||||
|
||||
|
||||
def resolve_run_status(
|
||||
*,
|
||||
scholar_count: int,
|
||||
succeeded_count: int,
|
||||
failed_count: int,
|
||||
partial_count: int,
|
||||
) -> RunStatus:
|
||||
if scholar_count == 0:
|
||||
return RunStatus.SUCCESS
|
||||
if failed_count == scholar_count:
|
||||
return RunStatus.FAILED
|
||||
if failed_count > 0 or partial_count > 0:
|
||||
return RunStatus.PARTIAL_FAILURE
|
||||
if succeeded_count > 0:
|
||||
return RunStatus.SUCCESS
|
||||
return RunStatus.FAILED
|
||||
|
||||
|
||||
def build_failure_debug_context(
|
||||
*,
|
||||
fetch_result: FetchResult,
|
||||
parsed_page: ParsedProfilePage,
|
||||
attempt_log: list[dict[str, Any]],
|
||||
page_logs: list[dict[str, Any]] | None = None,
|
||||
exception: Exception | None = None,
|
||||
) -> dict[str, Any]:
|
||||
context: dict[str, Any] = {
|
||||
"requested_url": fetch_result.requested_url,
|
||||
"final_url": fetch_result.final_url,
|
||||
"status_code": fetch_result.status_code,
|
||||
"fetch_error": fetch_result.error,
|
||||
"state_reason": parsed_page.state_reason,
|
||||
"profile_name": parsed_page.profile_name,
|
||||
"profile_image_url": parsed_page.profile_image_url,
|
||||
"articles_range": parsed_page.articles_range,
|
||||
"has_show_more_button": parsed_page.has_show_more_button,
|
||||
"has_operation_error_banner": parsed_page.has_operation_error_banner,
|
||||
"warning_codes": parsed_page.warnings,
|
||||
"marker_counts_nonzero": {key: value for key, value in parsed_page.marker_counts.items() if value > 0},
|
||||
"body_length": len(fetch_result.body),
|
||||
"body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest() if fetch_result.body else None,
|
||||
"body_excerpt": _build_body_excerpt(fetch_result.body),
|
||||
"attempt_log": attempt_log,
|
||||
}
|
||||
if page_logs:
|
||||
context["page_logs"] = page_logs
|
||||
if exception is not None:
|
||||
context["exception_type"] = type(exception).__name__
|
||||
context["exception_message"] = str(exception)
|
||||
return context
|
||||
|
||||
|
||||
def complete_run_for_user(
|
||||
*,
|
||||
user_settings: Any,
|
||||
run: CrawlRun,
|
||||
scholars: list[ScholarProfile],
|
||||
user_id: int,
|
||||
progress: RunProgress,
|
||||
idempotency_key: str | None,
|
||||
alert_blocked_failure_threshold: int,
|
||||
alert_network_failure_threshold: int,
|
||||
alert_retry_scheduled_threshold: int,
|
||||
) -> tuple[RunFailureSummary, RunAlertSummary]:
|
||||
failure_summary = summarize_failures(scholar_results=progress.scholar_results)
|
||||
alert_summary = build_alert_summary(
|
||||
failure_summary=failure_summary,
|
||||
alert_blocked_failure_threshold=alert_blocked_failure_threshold,
|
||||
alert_network_failure_threshold=alert_network_failure_threshold,
|
||||
alert_retry_scheduled_threshold=alert_retry_scheduled_threshold,
|
||||
)
|
||||
if alert_summary.alert_flags["blocked_failure_threshold_exceeded"]:
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"ingestion.alert_blocked_failure_threshold_exceeded",
|
||||
user_id=user_id,
|
||||
crawl_run_id=int(run.id),
|
||||
blocked_failure_count=alert_summary.blocked_failure_count,
|
||||
threshold=alert_summary.blocked_failure_threshold,
|
||||
)
|
||||
if alert_summary.alert_flags["network_failure_threshold_exceeded"]:
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"ingestion.alert_network_failure_threshold_exceeded",
|
||||
user_id=user_id,
|
||||
crawl_run_id=int(run.id),
|
||||
network_failure_count=alert_summary.network_failure_count,
|
||||
threshold=alert_summary.network_failure_threshold,
|
||||
)
|
||||
if alert_summary.alert_flags["retry_scheduled_threshold_exceeded"]:
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"ingestion.alert_retry_scheduled_threshold_exceeded",
|
||||
user_id=user_id,
|
||||
crawl_run_id=int(run.id),
|
||||
retries_scheduled_count=failure_summary.retries_scheduled_count,
|
||||
threshold=alert_summary.retry_scheduled_threshold,
|
||||
)
|
||||
apply_safety_outcome(user_settings=user_settings, run=run, user_id=user_id, alert_summary=alert_summary)
|
||||
run_status = resolve_run_status(
|
||||
scholar_count=len(scholars),
|
||||
succeeded_count=progress.succeeded_count,
|
||||
failed_count=progress.failed_count,
|
||||
partial_count=progress.partial_count,
|
||||
)
|
||||
finalize_run_record(
|
||||
run=run,
|
||||
scholars=scholars,
|
||||
progress=progress,
|
||||
failure_summary=failure_summary,
|
||||
alert_summary=alert_summary,
|
||||
idempotency_key=idempotency_key,
|
||||
run_status=run_status,
|
||||
)
|
||||
return failure_summary, alert_summary
|
||||
|
||||
|
||||
def result_counters(result_entry: dict[str, Any]) -> tuple[int, int, int]:
|
||||
outcome = str(result_entry.get("outcome", "")).strip().lower()
|
||||
if outcome == "success":
|
||||
return 1, 0, 0
|
||||
if outcome == "partial":
|
||||
return 1, 0, 1
|
||||
if outcome == "failed":
|
||||
return 0, 1, 0
|
||||
raise RuntimeError(f"Unexpected scholar outcome label: {outcome!r}")
|
||||
|
||||
|
||||
def find_scholar_result_index(
|
||||
*,
|
||||
scholar_results: list[dict[str, Any]],
|
||||
scholar_profile_id: int,
|
||||
) -> int | None:
|
||||
for index, result_entry in enumerate(scholar_results):
|
||||
current_scholar_id = _int_or_default(result_entry.get("scholar_profile_id"), 0)
|
||||
if current_scholar_id == scholar_profile_id:
|
||||
return index
|
||||
return None
|
||||
|
||||
|
||||
def adjust_progress_counts(
|
||||
*,
|
||||
progress: RunProgress,
|
||||
succeeded_delta: int,
|
||||
failed_delta: int,
|
||||
partial_delta: int,
|
||||
) -> None:
|
||||
progress.succeeded_count += succeeded_delta
|
||||
progress.failed_count += failed_delta
|
||||
progress.partial_count += partial_delta
|
||||
if progress.succeeded_count < 0 or progress.failed_count < 0 or progress.partial_count < 0:
|
||||
raise RuntimeError("RunProgress counters entered invalid negative state.")
|
||||
|
||||
|
||||
def apply_outcome_to_progress(
|
||||
*,
|
||||
progress: RunProgress,
|
||||
outcome: ScholarProcessingOutcome,
|
||||
) -> None:
|
||||
scholar_profile_id = _int_or_default(outcome.result_entry.get("scholar_profile_id"), 0)
|
||||
if scholar_profile_id <= 0:
|
||||
raise RuntimeError("Scholar outcome missing valid scholar_profile_id.")
|
||||
prior_index = find_scholar_result_index(
|
||||
scholar_results=progress.scholar_results,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
next_succeeded, next_failed, next_partial = result_counters(outcome.result_entry)
|
||||
if prior_index is None:
|
||||
progress.scholar_results.append(outcome.result_entry)
|
||||
adjust_progress_counts(
|
||||
progress=progress,
|
||||
succeeded_delta=next_succeeded,
|
||||
failed_delta=next_failed,
|
||||
partial_delta=next_partial,
|
||||
)
|
||||
return
|
||||
previous_entry = progress.scholar_results[prior_index]
|
||||
prev_succeeded, prev_failed, prev_partial = result_counters(previous_entry)
|
||||
progress.scholar_results[prior_index] = outcome.result_entry
|
||||
adjust_progress_counts(
|
||||
progress=progress,
|
||||
succeeded_delta=next_succeeded - prev_succeeded,
|
||||
failed_delta=next_failed - prev_failed,
|
||||
partial_delta=next_partial - prev_partial,
|
||||
)
|
||||
|
||||
|
||||
def run_execution_summary(
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholars: list[ScholarProfile],
|
||||
progress: RunProgress,
|
||||
) -> RunExecutionSummary:
|
||||
return RunExecutionSummary(
|
||||
crawl_run_id=run.id,
|
||||
status=run.status,
|
||||
scholar_count=len(scholars),
|
||||
succeeded_count=progress.succeeded_count,
|
||||
failed_count=progress.failed_count,
|
||||
partial_count=progress.partial_count,
|
||||
new_publication_count=run.new_pub_count,
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue