justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

streamlined

Browse source
This commit is contained in:
Justin Visser 2026-02-17 21:59:41 +01:00
parent d1dd3213dc
commit ab1d29b203
23 changed files with 853 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

3
app/api/routers/runs.py
View file

@ -382,6 +382,9 @@ async def run_manual(
auto_queue_continuations=settings.ingestion_continuation_queue_enabled,
queue_delay_seconds=settings.ingestion_continuation_base_delay_seconds,
idempotency_key=idempotency_key,
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
)
except ingestion_service.RunAlreadyInProgressError as exc:
await db_session.rollback()

4
app/api/routers/scholars.py
View file

@ -167,6 +167,10 @@ async def search_scholars(
interval_jitter_seconds=settings.scholar_name_search_interval_jitter_seconds,
cooldown_block_threshold=settings.scholar_name_search_cooldown_block_threshold,
cooldown_seconds=settings.scholar_name_search_cooldown_seconds,
retry_alert_threshold=settings.scholar_name_search_alert_retry_count_threshold,
cooldown_rejection_alert_threshold=(
settings.scholar_name_search_alert_cooldown_rejections_threshold
),
)
except scholar_service.ScholarServiceError as exc:
raise ApiException(

4
app/api/schemas.py
View file

@ -219,6 +219,10 @@ class RunSummaryData(BaseModel):
partial_count: int
failed_state_counts: dict[str, int] = Field(default_factory=dict)
failed_reason_counts: dict[str, int] = Field(default_factory=dict)
scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
retry_counts: dict[str, int] = Field(default_factory=dict)
alert_thresholds: dict[str, int] = Field(default_factory=dict)
alert_flags: dict[str, bool] = Field(default_factory=dict)
model_config = ConfigDict(extra="forbid")

123
app/services/ingestion.py
View file

@ -40,6 +40,11 @@ FAILED_STATES = {
ParseState.NETWORK_ERROR.value,
"ingestion_error",
}
FAILURE_BUCKET_BLOCKED = "blocked_or_captcha"
FAILURE_BUCKET_NETWORK = "network_error"
FAILURE_BUCKET_LAYOUT = "layout_changed"
FAILURE_BUCKET_INGESTION = "ingestion_error"
FAILURE_BUCKET_OTHER = "other_failure"
RUN_LOCK_NAMESPACE = 8217
RESUMABLE_PARTIAL_REASONS = {
"max_pages_reached",
@ -83,6 +88,28 @@ class RunAlreadyInProgressError(RuntimeError):
"""Raised when a run lock for a user is already held by another process."""
def _int_or_default(value: Any, default: int = 0) -> int:
try:
return int(value)
except (TypeError, ValueError):
return default
def _classify_failure_bucket(*, state: str, state_reason: str) -> str:
reason = state_reason.strip().lower()
normalized_state = state.strip().lower()
if normalized_state == ParseState.BLOCKED_OR_CAPTCHA.value or reason.startswith("blocked_"):
return FAILURE_BUCKET_BLOCKED
if normalized_state == ParseState.NETWORK_ERROR.value or reason.startswith("network_"):
return FAILURE_BUCKET_NETWORK
if normalized_state == ParseState.LAYOUT_CHANGED.value:
return FAILURE_BUCKET_LAYOUT
if normalized_state == "ingestion_error":
return FAILURE_BUCKET_INGESTION
return FAILURE_BUCKET_OTHER
class ScholarIngestionService:
def __init__(self, *, source: ScholarSource) -> None:
self._source = source
@ -103,6 +130,9 @@ class ScholarIngestionService:
auto_queue_continuations: bool = True,
queue_delay_seconds: int = 60,
idempotency_key: str | None = None,
alert_blocked_failure_threshold: int = 1,
alert_network_failure_threshold: int = 2,
alert_retry_scheduled_threshold: int = 3,
) -> RunExecutionSummary:
lock_acquired = await self._try_acquire_user_lock(
db_session,
@ -161,6 +191,9 @@ class ScholarIngestionService:
"max_pages_per_scholar": max_pages_per_scholar,
"page_size": page_size,
"idempotency_key": idempotency_key,
"alert_blocked_failure_threshold": alert_blocked_failure_threshold,
"alert_network_failure_threshold": alert_network_failure_threshold,
"alert_retry_scheduled_threshold": alert_retry_scheduled_threshold,
},
)
@ -416,17 +449,87 @@ class ScholarIngestionService:
failed_state_counts: dict[str, int] = {}
failed_reason_counts: dict[str, int] = {}
scrape_failure_counts: dict[str, int] = {}
retries_scheduled_count = 0
scholars_with_retries_count = 0
retry_exhausted_count = 0
for entry in scholar_results:
if str(entry.get("outcome", "")) != "failed":
attempt_count = max(0, _int_or_default(entry.get("attempt_count"), 0))
retries_for_entry = max(0, attempt_count - 1)
if retries_for_entry > 0:
retries_scheduled_count += retries_for_entry
scholars_with_retries_count += 1
outcome = str(entry.get("outcome", ""))
if outcome != "failed":
continue
state = str(entry.get("state", ""))
state = str(entry.get("state", "")).strip()
if state not in FAILED_STATES:
continue
failed_state_counts[state] = failed_state_counts.get(state, 0) + 1
reason = str(entry.get("state_reason", "")).strip()
if reason:
failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1
failure_bucket = _classify_failure_bucket(state=state, state_reason=reason)
scrape_failure_counts[failure_bucket] = (
scrape_failure_counts.get(failure_bucket, 0) + 1
)
if (
state == ParseState.NETWORK_ERROR.value
and retries_for_entry > 0
):
retry_exhausted_count += 1
blocked_failure_count = int(scrape_failure_counts.get(FAILURE_BUCKET_BLOCKED, 0))
network_failure_count = int(scrape_failure_counts.get(FAILURE_BUCKET_NETWORK, 0))
bounded_blocked_threshold = max(1, int(alert_blocked_failure_threshold))
bounded_network_threshold = max(1, int(alert_network_failure_threshold))
bounded_retry_threshold = max(1, int(alert_retry_scheduled_threshold))
alert_flags = {
"blocked_failure_threshold_exceeded": blocked_failure_count >= bounded_blocked_threshold,
"network_failure_threshold_exceeded": network_failure_count >= bounded_network_threshold,
"retry_scheduled_threshold_exceeded": retries_scheduled_count >= bounded_retry_threshold,
}
if alert_flags["blocked_failure_threshold_exceeded"]:
logger.warning(
"ingestion.alert_blocked_failure_threshold_exceeded",
extra={
"event": "ingestion.alert_blocked_failure_threshold_exceeded",
"user_id": user_id,
"crawl_run_id": run.id,
"blocked_failure_count": blocked_failure_count,
"threshold": bounded_blocked_threshold,
},
)
if alert_flags["network_failure_threshold_exceeded"]:
logger.warning(
"ingestion.alert_network_failure_threshold_exceeded",
extra={
"event": "ingestion.alert_network_failure_threshold_exceeded",
"user_id": user_id,
"crawl_run_id": run.id,
"network_failure_count": network_failure_count,
"threshold": bounded_network_threshold,
},
)
if alert_flags["retry_scheduled_threshold_exceeded"]:
logger.warning(
"ingestion.alert_retry_scheduled_threshold_exceeded",
extra={
"event": "ingestion.alert_retry_scheduled_threshold_exceeded",
"user_id": user_id,
"crawl_run_id": run.id,
"retries_scheduled_count": retries_scheduled_count,
"threshold": bounded_retry_threshold,
},
)
run.end_dt = datetime.now(timezone.utc)
run.status = self._resolve_run_status(
scholar_count=len(scholars),
@ -442,6 +545,18 @@ class ScholarIngestionService:
"partial_count": partial_count,
"failed_state_counts": failed_state_counts,
"failed_reason_counts": failed_reason_counts,
"scrape_failure_counts": scrape_failure_counts,
"retry_counts": {
"retries_scheduled_count": retries_scheduled_count,
"scholars_with_retries_count": scholars_with_retries_count,
"retry_exhausted_count": retry_exhausted_count,
},
"alert_thresholds": {
"blocked_failure_threshold": bounded_blocked_threshold,
"network_failure_threshold": bounded_network_threshold,
"retry_scheduled_threshold": bounded_retry_threshold,
},
"alert_flags": alert_flags,
},
"meta": {
"idempotency_key": idempotency_key,
@ -463,6 +578,10 @@ class ScholarIngestionService:
"failed_count": failed_count,
"partial_count": partial_count,
"new_publication_count": run.new_pub_count,
"blocked_failure_count": blocked_failure_count,
"network_failure_count": network_failure_count,
"retries_scheduled_count": retries_scheduled_count,
"alert_flags": alert_flags,
},
)

67
app/services/runs.py
View file

@ -67,26 +67,65 @@ def _summary_dict(error_log: object) -> dict[str, Any]:
return summary
def _summary_int_dict(summary: dict[str, Any], key: str) -> dict[str, int]:
value = summary.get(key)
if not isinstance(value, dict):
return {}
return {
str(item_key): _safe_int(item_value, 0)
for item_key, item_value in value.items()
if isinstance(item_key, str)
}
def _summary_bool_dict(summary: dict[str, Any], key: str) -> dict[str, bool]:
value = summary.get(key)
if not isinstance(value, dict):
return {}
return {
str(item_key): bool(item_value)
for item_key, item_value in value.items()
if isinstance(item_key, str)
}
def extract_run_summary(error_log: object) -> dict[str, Any]:
summary = _summary_dict(error_log)
return {
"succeeded_count": _safe_int(summary.get("succeeded_count", 0)),
"failed_count": _safe_int(summary.get("failed_count", 0)),
"partial_count": _safe_int(summary.get("partial_count", 0)),
"failed_state_counts": {
str(key): _safe_int(value, 0)
for key, value in summary.get("failed_state_counts", {}).items()
if isinstance(key, str)
}
if isinstance(summary.get("failed_state_counts"), dict)
else {},
"failed_reason_counts": {
str(key): _safe_int(value, 0)
for key, value in summary.get("failed_reason_counts", {}).items()
if isinstance(key, str)
}
if isinstance(summary.get("failed_reason_counts"), dict)
else {},
"failed_state_counts": _summary_int_dict(summary, "failed_state_counts"),
"failed_reason_counts": _summary_int_dict(summary, "failed_reason_counts"),
"scrape_failure_counts": _summary_int_dict(summary, "scrape_failure_counts"),
"retry_counts": {
"retries_scheduled_count": _safe_int(
(
summary.get("retry_counts", {}).get("retries_scheduled_count")
if isinstance(summary.get("retry_counts"), dict)
else 0
),
0,
),
"scholars_with_retries_count": _safe_int(
(
summary.get("retry_counts", {}).get("scholars_with_retries_count")
if isinstance(summary.get("retry_counts"), dict)
else 0
),
0,
),
"retry_exhausted_count": _safe_int(
(
summary.get("retry_counts", {}).get("retry_exhausted_count")
if isinstance(summary.get("retry_counts"), dict)
else 0
),
0,
),
},
"alert_thresholds": _summary_int_dict(summary, "alert_thresholds"),
"alert_flags": _summary_bool_dict(summary, "alert_flags"),
}

7
app/services/scheduler.py
View file

@ -20,6 +20,7 @@ from app.db.session import get_session_factory
from app.services import continuation_queue as queue_service
from app.services.ingestion import RunAlreadyInProgressError, ScholarIngestionService
from app.services.scholar_source import LiveScholarSource
from app.settings import settings
logger = logging.getLogger(__name__)
@ -195,6 +196,9 @@ class SchedulerService:
page_size=self._page_size,
auto_queue_continuations=self._continuation_queue_enabled,
queue_delay_seconds=self._continuation_base_delay_seconds,
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
)
except RunAlreadyInProgressError:
await session.rollback()
@ -326,6 +330,9 @@ class SchedulerService:
},
auto_queue_continuations=self._continuation_queue_enabled,
queue_delay_seconds=self._continuation_base_delay_seconds,
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold,
alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold,
)
except RunAlreadyInProgressError:
await session.rollback()

87
app/services/scholars.py
View file

@ -38,6 +38,8 @@ DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD = 1
DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS = 1800
DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS = 3.0
DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS = 1.0
DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD = 2
DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD = 3
ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES = {
"image/jpeg": ".jpg",
"image/png": ".png",
@ -101,6 +103,8 @@ _AUTHOR_SEARCH_CACHE: OrderedDict[str, _AuthorSearchCacheEntry] = OrderedDict()
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = 0.0
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC: datetime | None = None
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = False
class ScholarServiceError(ValueError):
@ -281,10 +285,14 @@ def _reset_author_search_runtime_state_for_tests() -> None:
global _AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC
global _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
global _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT
global _AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT
global _AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED
_AUTHOR_SEARCH_CACHE.clear()
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = 0.0
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC = None
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = False
async def list_scholars_for_user(
@ -379,10 +387,14 @@ async def search_author_candidates(
interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS,
cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD,
cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS,
retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD,
cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD,
) -> ParsedAuthorSearchPage:
global _AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC
global _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
global _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT
global _AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT
global _AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED
normalized_query = query.strip()
if len(normalized_query) < 2:
@ -408,8 +420,47 @@ async def search_author_candidates(
now_utc = datetime.now(timezone.utc)
now_monotonic = time.monotonic()
if (
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC is not None
and now_utc >= _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
):
logger.info(
"scholar_search.cooldown_expired",
extra={
"event": "scholar_search.cooldown_expired",
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat(),
},
)
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC = None
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = False
cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(now_utc)
if cooldown_remaining_seconds > 0:
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT += 1
bounded_cooldown_rejection_alert_threshold = max(
1,
int(cooldown_rejection_alert_threshold),
)
if (
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT
>= bounded_cooldown_rejection_alert_threshold
and not _AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED
):
logger.error(
"scholar_search.cooldown_rejection_threshold_exceeded",
extra={
"event": "scholar_search.cooldown_rejection_threshold_exceeded",
"query": normalized_query,
"cooldown_rejection_count": _AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT,
"threshold": bounded_cooldown_rejection_alert_threshold,
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat()
if _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
else None,
},
)
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = True
logger.warning(
"scholar_search.cooldown_active",
extra={
@ -421,12 +472,15 @@ async def search_author_candidates(
else None,
},
)
warning_codes = [
"author_search_cooldown_active",
f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
]
if _AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED:
warning_codes.append("author_search_cooldown_alert_threshold_exceeded")
return _policy_blocked_author_search_result(
reason=SEARCH_COOLDOWN_REASON,
warning_codes=[
"author_search_cooldown_active",
f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
],
warning_codes=warning_codes,
limit=bounded_limit,
)
@ -472,6 +526,7 @@ async def search_author_candidates(
max_attempts = max(1, int(network_error_retries) + 1)
parsed: ParsedAuthorSearchPage | None = None
retry_warnings: list[str] = []
retry_scheduled_count = 0
for attempt_index in range(max_attempts):
fetch_result = await source.fetch_author_search_html(normalized_query, start=0)
@ -480,6 +535,7 @@ async def search_author_candidates(
break
retry_warnings.append("network_retry_scheduled_for_author_search")
retry_scheduled_count += 1
sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index)
if sleep_seconds > 0:
await asyncio.sleep(sleep_seconds)
@ -494,6 +550,27 @@ async def search_author_candidates(
warnings=_merge_warnings(parsed.warnings, retry_warnings),
)
bounded_retry_alert_threshold = max(1, int(retry_alert_threshold))
if retry_scheduled_count >= bounded_retry_alert_threshold:
logger.warning(
"scholar_search.retry_threshold_exceeded",
extra={
"event": "scholar_search.retry_threshold_exceeded",
"query": normalized_query,
"retry_scheduled_count": retry_scheduled_count,
"threshold": bounded_retry_alert_threshold,
"final_state": merged_parsed.state.value,
"final_state_reason": merged_parsed.state_reason,
},
)
merged_parsed = replace(
merged_parsed,
warnings=_merge_warnings(
merged_parsed.warnings,
[f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"],
),
)
if _is_author_search_block_state(merged_parsed):
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT += 1
logger.warning(
@ -510,6 +587,8 @@ async def search_author_candidates(
seconds=max(60, int(cooldown_seconds))
)
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = False
merged_parsed = replace(
merged_parsed,
warnings=_merge_warnings(

20
app/settings.py
View file

@ -62,6 +62,18 @@ class Settings:
30,
)
ingestion_page_size: int = _env_int("INGESTION_PAGE_SIZE", 100)
ingestion_alert_blocked_failure_threshold: int = _env_int(
"INGESTION_ALERT_BLOCKED_FAILURE_THRESHOLD",
1,
)
ingestion_alert_network_failure_threshold: int = _env_int(
"INGESTION_ALERT_NETWORK_FAILURE_THRESHOLD",
2,
)
ingestion_alert_retry_scheduled_threshold: int = _env_int(
"INGESTION_ALERT_RETRY_SCHEDULED_THRESHOLD",
3,
)
ingestion_continuation_queue_enabled: bool = _env_bool(
"INGESTION_CONTINUATION_QUEUE_ENABLED",
True,
@ -118,6 +130,14 @@ class Settings:
"SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS",
1800,
)
scholar_name_search_alert_retry_count_threshold: int = _env_int(
"SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD",
2,
)
scholar_name_search_alert_cooldown_rejections_threshold: int = _env_int(
"SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD",
3,
)
settings = Settings()