justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

feat: refactor backend services and add direct PDF links, import/export, and dashboard sync

Browse source
This commit is contained in:
Justin Visser 2026-02-19 23:45:52 +01:00
parent ba7976d935
commit 7f7a8ce2b0
26 changed files with 4170 additions and 2440 deletions
Download patch file Download diff file Expand all files Collapse all files

107
app/services/continuation_queue.py
View file

@ -41,6 +41,70 @@ def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_second
return min(seconds, maximum)
async def _get_item_for_user_scholar(
db_session: AsyncSession,
*,
user_id: int,
scholar_profile_id: int,
) -> IngestionQueueItem | None:
result = await db_session.execute(
select(IngestionQueueItem).where(
IngestionQueueItem.user_id == user_id,
IngestionQueueItem.scholar_profile_id == scholar_profile_id,
)
)
return result.scalar_one_or_none()
def _build_queue_item(
*,
now: datetime,
user_id: int,
scholar_profile_id: int,
normalized_cstart: int,
reason: str,
run_id: int | None,
next_attempt_dt: datetime,
) -> IngestionQueueItem:
return IngestionQueueItem(
user_id=user_id,
scholar_profile_id=scholar_profile_id,
resume_cstart=normalized_cstart,
reason=reason,
status=QueueItemStatus.QUEUED.value,
attempt_count=0,
next_attempt_dt=next_attempt_dt,
last_run_id=run_id,
last_error=None,
dropped_reason=None,
dropped_at=None,
created_at=now,
updated_at=now,
)
def _update_existing_queue_item(
*,
item: IngestionQueueItem,
now: datetime,
normalized_cstart: int,
reason: str,
run_id: int | None,
next_attempt_dt: datetime,
) -> None:
item.resume_cstart = normalized_cstart
item.reason = reason
if item.status == QueueItemStatus.DROPPED.value:
item.attempt_count = 0
item.status = QueueItemStatus.QUEUED.value
item.next_attempt_dt = next_attempt_dt
item.last_run_id = run_id
item.last_error = None
item.dropped_reason = None
item.dropped_at = None
item.updated_at = now
async def upsert_job(
db_session: AsyncSession,
*,
@ -53,44 +117,33 @@ async def upsert_job(
) -> IngestionQueueItem:
now = datetime.now(timezone.utc)
next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds)))
result = await db_session.execute(
select(IngestionQueueItem).where(
IngestionQueueItem.user_id == user_id,
IngestionQueueItem.scholar_profile_id == scholar_profile_id,
)
item = await _get_item_for_user_scholar(
db_session,
user_id=user_id,
scholar_profile_id=scholar_profile_id,
)
item = result.scalar_one_or_none()
normalized_cstart = normalize_cstart(resume_cstart)
if item is None:
item = IngestionQueueItem(
item = _build_queue_item(
now=now,
user_id=user_id,
scholar_profile_id=scholar_profile_id,
resume_cstart=normalized_cstart,
normalized_cstart=normalized_cstart,
reason=reason,
status=QueueItemStatus.QUEUED.value,
attempt_count=0,
run_id=run_id,
next_attempt_dt=next_attempt_dt,
last_run_id=run_id,
last_error=None,
dropped_reason=None,
dropped_at=None,
created_at=now,
updated_at=now,
)
db_session.add(item)
return item
item.resume_cstart = normalized_cstart
item.reason = reason
if item.status == QueueItemStatus.DROPPED.value:
item.attempt_count = 0
item.status = QueueItemStatus.QUEUED.value
item.next_attempt_dt = next_attempt_dt
item.last_run_id = run_id
item.last_error = None
item.dropped_reason = None
item.dropped_at = None
item.updated_at = now
_update_existing_queue_item(
item=item,
now=now,
normalized_cstart=normalized_cstart,
reason=reason,
run_id=run_id,
next_attempt_dt=next_attempt_dt,
)
return item

667
app/services/import_export.py Normal file
View file

@ -0,0 +1,667 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timezone
import hashlib
import re
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Publication, ScholarProfile, ScholarPublication
from app.services import scholars as scholar_service
from app.services.ingestion import build_publication_url, normalize_title
EXPORT_SCHEMA_VERSION = 1
MAX_IMPORT_SCHOLARS = 10_000
MAX_IMPORT_PUBLICATIONS = 100_000
WORD_RE = re.compile(r"[a-z0-9]+")
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
class ImportExportError(ValueError):
"""Raised when import/export payload constraints are violated."""
@dataclass(frozen=True)
class ImportedPublicationInput:
profile: ScholarProfile
title: str
year: int | None
citation_count: int
author_text: str | None
venue_text: str | None
cluster_id: str | None
pub_url: str | None
pdf_url: str | None
fingerprint: str
is_read: bool
def _normalize_optional_text(value: Any) -> str | None:
if value is None:
return None
normalized = str(value).strip()
return normalized or None
def _normalize_optional_year(value: Any) -> int | None:
if value is None:
return None
try:
year = int(value)
except (TypeError, ValueError):
return None
if year < 1500 or year > 3000:
return None
return year
def _normalize_citation_count(value: Any) -> int:
try:
parsed = int(value)
except (TypeError, ValueError):
return 0
return max(0, parsed)
def _first_author_last_name(authors_text: str | None) -> str:
if not authors_text:
return ""
first_author = authors_text.split(",", maxsplit=1)[0].strip().lower()
words = WORD_RE.findall(first_author)
if not words:
return ""
return words[-1]
def _first_venue_word(venue_text: str | None) -> str:
if not venue_text:
return ""
words = WORD_RE.findall(venue_text.lower())
if not words:
return ""
return words[0]
def _build_fingerprint(
*,
title: str,
year: int | None,
author_text: str | None,
venue_text: str | None,
) -> str:
canonical = "|".join(
[
normalize_title(title),
str(year) if year is not None else "",
_first_author_last_name(author_text),
_first_venue_word(venue_text),
]
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def _resolve_fingerprint(
*,
title: str,
year: int | None,
author_text: str | None,
venue_text: str | None,
provided_fingerprint: Any,
) -> str:
normalized = _normalize_optional_text(provided_fingerprint)
if normalized and SHA256_RE.fullmatch(normalized.lower()):
return normalized.lower()
return _build_fingerprint(
title=title,
year=year,
author_text=author_text,
venue_text=venue_text,
)
def _validate_import_sizes(
*,
scholars: list[dict[str, Any]],
publications: list[dict[str, Any]],
) -> None:
if len(scholars) > MAX_IMPORT_SCHOLARS:
raise ImportExportError(f"Import exceeds max scholars ({MAX_IMPORT_SCHOLARS}).")
if len(publications) > MAX_IMPORT_PUBLICATIONS:
raise ImportExportError(
f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS})."
)
async def _load_user_scholar_map(
db_session: AsyncSession,
*,
user_id: int,
) -> dict[str, ScholarProfile]:
result = await db_session.execute(
select(ScholarProfile).where(ScholarProfile.user_id == user_id)
)
profiles = list(result.scalars().all())
return {profile.scholar_id: profile for profile in profiles}
def _apply_imported_scholar_values(
*,
profile: ScholarProfile,
display_name: str | None,
profile_image_override_url: str | None,
is_enabled: bool,
) -> bool:
updated = False
if display_name and profile.display_name != display_name:
profile.display_name = display_name
updated = True
if profile.profile_image_override_url != profile_image_override_url:
profile.profile_image_override_url = profile_image_override_url
updated = True
if bool(profile.is_enabled) != bool(is_enabled):
profile.is_enabled = bool(is_enabled)
updated = True
return updated
def _new_scholar_profile(
*,
user_id: int,
scholar_id: str,
display_name: str | None,
profile_image_override_url: str | None,
is_enabled: bool,
) -> ScholarProfile:
return ScholarProfile(
user_id=user_id,
scholar_id=scholar_id,
display_name=display_name,
profile_image_override_url=profile_image_override_url,
is_enabled=bool(is_enabled),
)
async def _upsert_imported_scholars(
db_session: AsyncSession,
*,
user_id: int,
scholars: list[dict[str, Any]],
) -> tuple[dict[str, ScholarProfile], dict[str, int]]:
scholar_map = await _load_user_scholar_map(db_session, user_id=user_id)
counters = {"scholars_created": 0, "scholars_updated": 0, "skipped_records": 0}
for item in scholars:
try:
scholar_id = scholar_service.validate_scholar_id(str(item["scholar_id"]))
display_name = scholar_service.normalize_display_name(str(item.get("display_name") or ""))
override_url = scholar_service.normalize_profile_image_url(
_normalize_optional_text(item.get("profile_image_override_url"))
)
except (KeyError, scholar_service.ScholarServiceError):
counters["skipped_records"] += 1
continue
is_enabled = bool(item.get("is_enabled", True))
existing = scholar_map.get(scholar_id)
if existing is None:
profile = _new_scholar_profile(
user_id=user_id,
scholar_id=scholar_id,
display_name=display_name,
profile_image_override_url=override_url,
is_enabled=is_enabled,
)
db_session.add(profile)
scholar_map[scholar_id] = profile
counters["scholars_created"] += 1
continue
if _apply_imported_scholar_values(
profile=existing,
display_name=display_name,
profile_image_override_url=override_url,
is_enabled=is_enabled,
):
counters["scholars_updated"] += 1
await db_session.flush()
return scholar_map, counters
async def _find_publication_by_cluster(
db_session: AsyncSession,
*,
cluster_id: str,
) -> Publication | None:
result = await db_session.execute(
select(Publication).where(Publication.cluster_id == cluster_id)
)
return result.scalar_one_or_none()
async def _find_publication_by_fingerprint(
db_session: AsyncSession,
*,
fingerprint_sha256: str,
) -> Publication | None:
result = await db_session.execute(
select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256)
)
return result.scalar_one_or_none()
def _apply_imported_publication_values(
*,
publication: Publication,
title: str,
year: int | None,
citation_count: int,
author_text: str | None,
venue_text: str | None,
pub_url: str | None,
pdf_url: str | None,
cluster_id: str | None,
) -> bool:
updated = False
if cluster_id and publication.cluster_id != cluster_id:
publication.cluster_id = cluster_id
updated = True
if publication.title_raw != title:
publication.title_raw = title
publication.title_normalized = normalize_title(title)
updated = True
if publication.year != year:
publication.year = year
updated = True
if int(publication.citation_count or 0) != citation_count:
publication.citation_count = citation_count
updated = True
if publication.author_text != author_text:
publication.author_text = author_text
updated = True
if publication.venue_text != venue_text:
publication.venue_text = venue_text
updated = True
if pub_url and publication.pub_url != pub_url:
publication.pub_url = pub_url
updated = True
if pdf_url and publication.pdf_url != pdf_url:
publication.pdf_url = pdf_url
updated = True
return updated
def _new_publication(
*,
cluster_id: str | None,
fingerprint_sha256: str,
title: str,
year: int | None,
citation_count: int,
author_text: str | None,
venue_text: str | None,
pub_url: str | None,
pdf_url: str | None,
) -> Publication:
return Publication(
cluster_id=cluster_id,
fingerprint_sha256=fingerprint_sha256,
title_raw=title,
title_normalized=normalize_title(title),
year=year,
citation_count=citation_count,
author_text=author_text,
venue_text=venue_text,
pub_url=pub_url,
pdf_url=pdf_url,
)
async def _resolve_publication_for_import(
db_session: AsyncSession,
*,
cluster_id: str | None,
fingerprint_sha256: str,
cluster_cache: dict[str, Publication | None],
fingerprint_cache: dict[str, Publication | None],
) -> Publication | None:
if cluster_id:
if cluster_id not in cluster_cache:
cluster_cache[cluster_id] = await _find_publication_by_cluster(
db_session,
cluster_id=cluster_id,
)
if cluster_cache[cluster_id] is not None:
return cluster_cache[cluster_id]
if fingerprint_sha256 not in fingerprint_cache:
fingerprint_cache[fingerprint_sha256] = await _find_publication_by_fingerprint(
db_session,
fingerprint_sha256=fingerprint_sha256,
)
return fingerprint_cache[fingerprint_sha256]
async def _upsert_scholar_publication_link(
db_session: AsyncSession,
*,
scholar_profile_id: int,
publication_id: int,
is_read: bool,
) -> tuple[bool, bool]:
result = await db_session.execute(
select(ScholarPublication).where(
ScholarPublication.scholar_profile_id == scholar_profile_id,
ScholarPublication.publication_id == publication_id,
)
)
link = result.scalar_one_or_none()
if link is None:
db_session.add(
ScholarPublication(
scholar_profile_id=scholar_profile_id,
publication_id=publication_id,
is_read=bool(is_read),
)
)
return True, False
if bool(link.is_read) == bool(is_read):
return False, False
link.is_read = bool(is_read)
return False, True
def _initialize_import_counters(counters: dict[str, int]) -> None:
counters.update(
{
"publications_created": 0,
"publications_updated": 0,
"links_created": 0,
"links_updated": 0,
}
)
def _build_imported_publication_input(
*,
item: dict[str, Any],
scholar_map: dict[str, ScholarProfile],
) -> ImportedPublicationInput | None:
scholar_id = _normalize_optional_text(item.get("scholar_id"))
title = _normalize_optional_text(item.get("title"))
if not scholar_id or not title:
return None
profile = scholar_map.get(scholar_id)
if profile is None:
return None
year = _normalize_optional_year(item.get("year"))
author_text = _normalize_optional_text(item.get("author_text"))
venue_text = _normalize_optional_text(item.get("venue_text"))
return ImportedPublicationInput(
profile=profile,
title=title,
year=year,
citation_count=_normalize_citation_count(item.get("citation_count")),
author_text=author_text,
venue_text=venue_text,
cluster_id=_normalize_optional_text(item.get("cluster_id")),
pub_url=build_publication_url(_normalize_optional_text(item.get("pub_url"))),
pdf_url=build_publication_url(_normalize_optional_text(item.get("pdf_url"))),
fingerprint=_resolve_fingerprint(
title=title,
year=year,
author_text=author_text,
venue_text=venue_text,
provided_fingerprint=item.get("fingerprint_sha256"),
),
is_read=bool(item.get("is_read", False)),
)
def _update_link_counters(
*,
counters: dict[str, int],
link_created: bool,
link_updated: bool,
) -> None:
if link_created:
counters["links_created"] += 1
if link_updated:
counters["links_updated"] += 1
def _cache_resolved_publication(
*,
publication: Publication,
cluster_id: str | None,
fingerprint_sha256: str,
cluster_cache: dict[str, Publication | None],
fingerprint_cache: dict[str, Publication | None],
) -> None:
if cluster_id:
cluster_cache[cluster_id] = publication
fingerprint_cache[fingerprint_sha256] = publication
async def _create_import_publication(
db_session: AsyncSession,
*,
payload: ImportedPublicationInput,
) -> Publication:
publication = _new_publication(
cluster_id=payload.cluster_id,
fingerprint_sha256=payload.fingerprint,
title=payload.title,
year=payload.year,
citation_count=payload.citation_count,
author_text=payload.author_text,
venue_text=payload.venue_text,
pub_url=payload.pub_url,
pdf_url=payload.pdf_url,
)
db_session.add(publication)
await db_session.flush()
return publication
def _update_import_publication(
*,
publication: Publication,
payload: ImportedPublicationInput,
) -> bool:
return _apply_imported_publication_values(
publication=publication,
title=payload.title,
year=payload.year,
citation_count=payload.citation_count,
author_text=payload.author_text,
venue_text=payload.venue_text,
pub_url=payload.pub_url,
pdf_url=payload.pdf_url,
cluster_id=payload.cluster_id,
)
async def _upsert_publication_entity(
db_session: AsyncSession,
*,
payload: ImportedPublicationInput,
cluster_cache: dict[str, Publication | None],
fingerprint_cache: dict[str, Publication | None],
) -> tuple[Publication, bool, bool]:
publication = await _resolve_publication_for_import(
db_session,
cluster_id=payload.cluster_id,
fingerprint_sha256=payload.fingerprint,
cluster_cache=cluster_cache,
fingerprint_cache=fingerprint_cache,
)
created = False
updated = False
if publication is None:
publication = await _create_import_publication(
db_session,
payload=payload,
)
created = True
else:
updated = _update_import_publication(
publication=publication,
payload=payload,
)
_cache_resolved_publication(
publication=publication,
cluster_id=payload.cluster_id,
fingerprint_sha256=payload.fingerprint,
cluster_cache=cluster_cache,
fingerprint_cache=fingerprint_cache,
)
return publication, created, updated
async def _upsert_imported_publication(
db_session: AsyncSession,
*,
payload: ImportedPublicationInput,
cluster_cache: dict[str, Publication | None],
fingerprint_cache: dict[str, Publication | None],
counters: dict[str, int],
) -> None:
publication, created, updated = await _upsert_publication_entity(
db_session,
payload=payload,
cluster_cache=cluster_cache,
fingerprint_cache=fingerprint_cache,
)
if created:
counters["publications_created"] += 1
if updated:
counters["publications_updated"] += 1
link_created, link_updated = await _upsert_scholar_publication_link(
db_session,
scholar_profile_id=int(payload.profile.id),
publication_id=int(publication.id),
is_read=payload.is_read,
)
_update_link_counters(
counters=counters,
link_created=link_created,
link_updated=link_updated,
)
def _exported_at_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]:
return {
"scholar_id": profile.scholar_id,
"display_name": profile.display_name,
"is_enabled": bool(profile.is_enabled),
"profile_image_override_url": profile.profile_image_override_url,
}
def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]:
(
scholar_id,
cluster_id,
fingerprint_sha256,
title_raw,
year,
citation_count,
author_text,
venue_text,
pub_url,
pdf_url,
is_read,
) = row
return {
"scholar_id": scholar_id,
"cluster_id": cluster_id,
"fingerprint_sha256": fingerprint_sha256,
"title": title_raw,
"year": year,
"citation_count": int(citation_count or 0),
"author_text": author_text,
"venue_text": venue_text,
"pub_url": pub_url,
"pdf_url": pdf_url,
"is_read": bool(is_read),
}
async def export_user_data(
db_session: AsyncSession,
*,
user_id: int,
) -> dict[str, Any]:
scholars_result = await db_session.execute(
select(ScholarProfile)
.where(ScholarProfile.user_id == user_id)
.order_by(ScholarProfile.id.asc())
)
publication_result = await db_session.execute(
select(
ScholarProfile.scholar_id,
Publication.cluster_id,
Publication.fingerprint_sha256,
Publication.title_raw,
Publication.year,
Publication.citation_count,
Publication.author_text,
Publication.venue_text,
Publication.pub_url,
Publication.pdf_url,
ScholarPublication.is_read,
)
.join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id)
.join(Publication, Publication.id == ScholarPublication.publication_id)
.where(ScholarProfile.user_id == user_id)
.order_by(ScholarPublication.created_at.desc(), Publication.id.desc())
)
scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()]
publications = [
_serialize_export_publication(row)
for row in publication_result.all()
]
return {
"schema_version": EXPORT_SCHEMA_VERSION,
"exported_at": _exported_at_iso(),
"scholars": scholars,
"publications": publications,
}
async def import_user_data(
db_session: AsyncSession,
*,
user_id: int,
scholars: list[dict[str, Any]],
publications: list[dict[str, Any]],
) -> dict[str, int]:
_validate_import_sizes(scholars=scholars, publications=publications)
scholar_map, counters = await _upsert_imported_scholars(
db_session,
user_id=user_id,
scholars=scholars,
)
cluster_cache: dict[str, Publication | None] = {}
fingerprint_cache: dict[str, Publication | None] = {}
_initialize_import_counters(counters)
for item in publications:
parsed_item = _build_imported_publication_input(
item=item,
scholar_map=scholar_map,
)
if parsed_item is None:
counters["skipped_records"] += 1
continue
await _upsert_imported_publication(
db_session,
payload=parsed_item,
cluster_cache=cluster_cache,
fingerprint_cache=fingerprint_cache,
counters=counters,
)
await db_session.commit()
return counters

2240
app/services/ingestion.py
View file

File diff suppressed because it is too large Load diff

84
app/services/publications.py
View file

@ -30,6 +30,7 @@ class PublicationListItem:
citation_count: int
venue_text: str | None
pub_url: str | None
pdf_url: str | None
is_read: bool
first_seen_at: datetime
is_new_in_latest_run: bool
@ -45,6 +46,7 @@ class UnreadPublicationItem:
citation_count: int
venue_text: str | None
pub_url: str | None
pdf_url: str | None
def resolve_publication_view_mode(value: str | None) -> str:
@ -95,6 +97,7 @@ def publications_query(
Publication.citation_count,
Publication.venue_text,
Publication.pub_url,
Publication.pdf_url,
ScholarPublication.is_read,
ScholarPublication.first_seen_run_id,
ScholarPublication.created_at,
@ -118,6 +121,44 @@ def publications_query(
return stmt
def _publication_list_item_from_row(
row: tuple,
*,
latest_run_id: int | None,
) -> PublicationListItem:
(
publication_id,
scholar_profile_id,
display_name,
scholar_id,
title_raw,
year,
citation_count,
venue_text,
pub_url,
pdf_url,
is_read,
first_seen_run_id,
created_at,
) = row
return PublicationListItem(
publication_id=int(publication_id),
scholar_profile_id=int(scholar_profile_id),
scholar_label=(display_name or scholar_id),
title=title_raw,
year=year,
citation_count=int(citation_count or 0),
venue_text=venue_text,
pub_url=pub_url,
pdf_url=pdf_url,
is_read=bool(is_read),
first_seen_at=created_at,
is_new_in_latest_run=(
latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
),
)
async def list_for_user(
db_session: AsyncSession,
*,
@ -140,42 +181,10 @@ async def list_for_user(
limit=limit,
)
)
rows = result.all()
items: list[PublicationListItem] = []
for row in rows:
(
publication_id,
scholar_profile_id,
display_name,
scholar_id,
title_raw,
year,
citation_count,
venue_text,
pub_url,
is_read,
first_seen_run_id,
created_at,
) = row
items.append(
PublicationListItem(
publication_id=int(publication_id),
scholar_profile_id=int(scholar_profile_id),
scholar_label=(display_name or scholar_id),
title=title_raw,
year=year,
citation_count=int(citation_count or 0),
venue_text=venue_text,
pub_url=pub_url,
is_read=bool(is_read),
first_seen_at=created_at,
is_new_in_latest_run=(
latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
),
)
)
return items
return [
_publication_list_item_from_row(row, latest_run_id=latest_run_id)
for row in result.all()
]
async def list_unread_for_user(
@ -206,6 +215,7 @@ async def list_unread_for_user(
citation_count,
venue_text,
pub_url,
pdf_url,
_is_read,
_first_seen_run_id,
_created_at,
@ -220,6 +230,7 @@ async def list_unread_for_user(
citation_count=int(citation_count or 0),
venue_text=venue_text,
pub_url=pub_url,
pdf_url=pdf_url,
)
)
return items
@ -248,6 +259,7 @@ async def list_new_for_latest_run_for_user(
citation_count=row.citation_count,
venue_text=row.venue_text,
pub_url=row.pub_url,
pdf_url=row.pdf_url,
)
for row in rows
]

109
app/services/run_safety.py
View file

@ -146,6 +146,63 @@ def register_cooldown_blocked_start(
return get_safety_state_payload(settings, now_utc=now)
def _update_run_counters(
*,
counters: dict[str, Any],
run_id: int,
blocked_failure_count: int,
network_failure_count: int,
) -> tuple[int, int]:
bounded_blocked_failures = max(0, int(blocked_failure_count))
bounded_network_failures = max(0, int(network_failure_count))
counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures
counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures
counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id)
counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = (
int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1
if bounded_blocked_failures > 0
else 0
)
counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = (
int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1
if bounded_network_failures > 0
else 0
)
return bounded_blocked_failures, bounded_network_failures
def _resolve_cooldown_trigger(
*,
blocked_failures: int,
network_failures: int,
blocked_failure_threshold: int,
network_failure_threshold: int,
blocked_cooldown_seconds: int,
network_cooldown_seconds: int,
) -> tuple[str | None, int]:
if blocked_failures >= max(1, int(blocked_failure_threshold)):
return COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD, max(60, int(blocked_cooldown_seconds))
if network_failures >= max(1, int(network_failure_threshold)):
return COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD, max(60, int(network_cooldown_seconds))
return None, 0
def _apply_cooldown_decision(
*,
settings: UserSetting,
counters: dict[str, Any],
now: datetime,
reason: str | None,
cooldown_seconds: int,
) -> None:
if reason is None:
clear_expired_cooldown(settings, now_utc=now)
return
settings.scrape_cooldown_reason = reason
settings.scrape_cooldown_until = now + timedelta(seconds=max(60, int(cooldown_seconds)))
counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1
def apply_run_safety_outcome(
settings: UserSetting,
*,
@ -160,41 +217,27 @@ def apply_run_safety_outcome(
) -> tuple[dict[str, Any], str | None]:
now = now_utc or _utcnow()
counters = _counters_from_state(settings)
bounded_blocked_failures = max(0, int(blocked_failure_count))
bounded_network_failures = max(0, int(network_failure_count))
counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures
counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures
counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id)
counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = (
int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1
if bounded_blocked_failures > 0
else 0
blocked_failures, network_failures = _update_run_counters(
counters=counters,
run_id=run_id,
blocked_failure_count=blocked_failure_count,
network_failure_count=network_failure_count,
)
counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = (
int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1
if bounded_network_failures > 0
else 0
reason, cooldown_seconds = _resolve_cooldown_trigger(
blocked_failures=blocked_failures,
network_failures=network_failures,
blocked_failure_threshold=blocked_failure_threshold,
network_failure_threshold=network_failure_threshold,
blocked_cooldown_seconds=blocked_cooldown_seconds,
network_cooldown_seconds=network_cooldown_seconds,
)
_apply_cooldown_decision(
settings=settings,
counters=counters,
now=now,
reason=reason,
cooldown_seconds=cooldown_seconds,
)
reason: str | None = None
cooldown_seconds = 0
if bounded_blocked_failures >= max(1, int(blocked_failure_threshold)):
reason = COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD
cooldown_seconds = max(60, int(blocked_cooldown_seconds))
elif bounded_network_failures >= max(1, int(network_failure_threshold)):
reason = COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD
cooldown_seconds = max(60, int(network_cooldown_seconds))
if reason is not None:
settings.scrape_cooldown_reason = reason
settings.scrape_cooldown_until = now + timedelta(seconds=cooldown_seconds)
counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1
else:
clear_expired_cooldown(settings, now_utc=now)
settings.scrape_safety_state = counters
return get_safety_state_payload(settings, now_utc=now), reason

182
app/services/runs.py
View file

@ -129,6 +129,70 @@ def extract_run_summary(error_log: object) -> dict[str, Any]:
}
def _queue_item_columns() -> tuple:
return (
IngestionQueueItem.id,
IngestionQueueItem.scholar_profile_id,
ScholarProfile.display_name,
ScholarProfile.scholar_id,
IngestionQueueItem.status,
IngestionQueueItem.reason,
IngestionQueueItem.dropped_reason,
IngestionQueueItem.attempt_count,
IngestionQueueItem.resume_cstart,
IngestionQueueItem.next_attempt_dt,
IngestionQueueItem.updated_at,
IngestionQueueItem.last_error,
IngestionQueueItem.last_run_id,
)
def _queue_item_select(*, user_id: int):
return (
select(*_queue_item_columns())
.join(
ScholarProfile,
and_(
ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
ScholarProfile.user_id == IngestionQueueItem.user_id,
),
)
.where(IngestionQueueItem.user_id == user_id)
)
def _queue_list_item_from_row(row: tuple) -> QueueListItem:
(
item_id,
scholar_profile_id,
display_name,
scholar_id,
status,
reason,
dropped_reason,
attempt_count,
resume_cstart,
next_attempt_dt,
updated_at,
last_error,
last_run_id,
) = row
return QueueListItem(
id=int(item_id),
scholar_profile_id=int(scholar_profile_id),
scholar_label=(display_name or scholar_id),
status=str(status),
reason=str(reason),
dropped_reason=dropped_reason,
attempt_count=int(attempt_count or 0),
resume_cstart=int(resume_cstart or 0),
next_attempt_dt=next_attempt_dt,
updated_at=updated_at,
last_error=last_error,
last_run_id=int(last_run_id) if last_run_id is not None else None,
)
async def list_recent_runs_for_user(
db_session: AsyncSession,
*,
@ -209,29 +273,7 @@ async def list_queue_items_for_user(
limit: int = 200,
) -> list[QueueListItem]:
result = await db_session.execute(
select(
IngestionQueueItem.id,
IngestionQueueItem.scholar_profile_id,
ScholarProfile.display_name,
ScholarProfile.scholar_id,
IngestionQueueItem.status,
IngestionQueueItem.reason,
IngestionQueueItem.dropped_reason,
IngestionQueueItem.attempt_count,
IngestionQueueItem.resume_cstart,
IngestionQueueItem.next_attempt_dt,
IngestionQueueItem.updated_at,
IngestionQueueItem.last_error,
IngestionQueueItem.last_run_id,
)
.join(
ScholarProfile,
and_(
ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
ScholarProfile.user_id == IngestionQueueItem.user_id,
),
)
.where(IngestionQueueItem.user_id == user_id)
_queue_item_select(user_id=user_id)
.order_by(
case((IngestionQueueItem.status == "dropped", 1), else_=0).asc(),
IngestionQueueItem.next_attempt_dt.asc(),
@ -239,41 +281,7 @@ async def list_queue_items_for_user(
)
.limit(limit)
)
items: list[QueueListItem] = []
for row in result.all():
(
item_id,
scholar_profile_id,
display_name,
scholar_id,
status,
reason,
dropped_reason,
attempt_count,
resume_cstart,
next_attempt_dt,
updated_at,
last_error,
last_run_id,
) = row
items.append(
QueueListItem(
id=int(item_id),
scholar_profile_id=int(scholar_profile_id),
scholar_label=(display_name or scholar_id),
status=str(status),
reason=str(reason),
dropped_reason=dropped_reason,
attempt_count=int(attempt_count or 0),
resume_cstart=int(resume_cstart or 0),
next_attempt_dt=next_attempt_dt,
updated_at=updated_at,
last_error=last_error,
last_run_id=int(last_run_id) if last_run_id is not None else None,
)
)
return items
return [_queue_list_item_from_row(row) for row in result.all()]
async def get_queue_item_for_user(
@ -283,66 +291,14 @@ async def get_queue_item_for_user(
queue_item_id: int,
) -> QueueListItem | None:
result = await db_session.execute(
select(
IngestionQueueItem.id,
IngestionQueueItem.scholar_profile_id,
ScholarProfile.display_name,
ScholarProfile.scholar_id,
IngestionQueueItem.status,
IngestionQueueItem.reason,
IngestionQueueItem.dropped_reason,
IngestionQueueItem.attempt_count,
IngestionQueueItem.resume_cstart,
IngestionQueueItem.next_attempt_dt,
IngestionQueueItem.updated_at,
IngestionQueueItem.last_error,
IngestionQueueItem.last_run_id,
)
.join(
ScholarProfile,
and_(
ScholarProfile.id == IngestionQueueItem.scholar_profile_id,
ScholarProfile.user_id == IngestionQueueItem.user_id,
),
)
.where(
IngestionQueueItem.user_id == user_id,
IngestionQueueItem.id == queue_item_id,
)
_queue_item_select(user_id=user_id)
.where(IngestionQueueItem.id == queue_item_id)
.limit(1)
)
row = result.one_or_none()
if row is None:
return None
(
item_id,
scholar_profile_id,
display_name,
scholar_id,
status,
reason,
dropped_reason,
attempt_count,
resume_cstart,
next_attempt_dt,
updated_at,
last_error,
last_run_id,
) = row
return QueueListItem(
id=int(item_id),
scholar_profile_id=int(scholar_profile_id),
scholar_label=(display_name or scholar_id),
status=str(status),
reason=str(reason),
dropped_reason=dropped_reason,
attempt_count=int(attempt_count or 0),
resume_cstart=int(resume_cstart or 0),
next_attempt_dt=next_attempt_dt,
updated_at=updated_at,
last_error=last_error,
last_run_id=int(last_run_id) if last_run_id is not None else None,
)
return _queue_list_item_from_row(row)
async def retry_queue_item_for_user(

555
app/services/scheduler.py
View file

@ -139,10 +139,7 @@ class SchedulerService:
continue
await self._run_candidate(candidate)
async def _load_candidates(self) -> list[_AutoRunCandidate]:
if not settings.ingestion_automation_allowed:
return []
async def _load_candidate_rows(self) -> list[tuple]:
session_factory = get_session_factory()
async with session_factory() as session:
result = await session.execute(
@ -154,41 +151,48 @@ class SchedulerService:
UserSetting.scrape_cooldown_reason,
)
.join(User, User.id == UserSetting.user_id)
.where(
User.is_active.is_(True),
UserSetting.auto_run_enabled.is_(True),
)
.where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True))
.order_by(UserSetting.user_id.asc())
)
rows = result.all()
return result.all()
@staticmethod
def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None:
user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row
if cooldown_until is not None and cooldown_until.tzinfo is None:
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
if cooldown_until is not None and cooldown_until > now_utc:
logger.info(
"scheduler.run_skipped_safety_cooldown_precheck",
extra={
"event": "scheduler.run_skipped_safety_cooldown_precheck",
"user_id": int(user_id),
"reason": cooldown_reason,
"cooldown_until": cooldown_until,
"cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()),
"metric_name": "scheduler_run_skipped_safety_cooldown_total",
"metric_value": 1,
},
)
return None
return _AutoRunCandidate(
user_id=int(user_id),
run_interval_minutes=int(run_interval_minutes),
request_delay_seconds=int(request_delay_seconds),
cooldown_until=cooldown_until,
cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
)
async def _load_candidates(self) -> list[_AutoRunCandidate]:
if not settings.ingestion_automation_allowed:
return []
rows = await self._load_candidate_rows()
now_utc = datetime.now(timezone.utc)
candidates: list[_AutoRunCandidate] = []
for user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason in rows:
if cooldown_until is not None and cooldown_until.tzinfo is None:
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
if cooldown_until is not None and cooldown_until > now_utc:
logger.info(
"scheduler.run_skipped_safety_cooldown_precheck",
extra={
"event": "scheduler.run_skipped_safety_cooldown_precheck",
"user_id": int(user_id),
"reason": cooldown_reason,
"cooldown_until": cooldown_until,
"cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()),
"metric_name": "scheduler_run_skipped_safety_cooldown_total",
"metric_value": 1,
},
)
continue
candidates.append(
_AutoRunCandidate(
user_id=int(user_id),
run_interval_minutes=int(run_interval_minutes),
request_delay_seconds=int(request_delay_seconds),
cooldown_until=cooldown_until,
cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None),
)
)
for row in rows:
candidate = self._candidate_from_row(row, now_utc=now_utc)
if candidate is not None:
candidates.append(candidate)
return candidates
async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool:
@ -212,12 +216,16 @@ class SchedulerService:
)
return now >= next_due_dt
async def _run_candidate(self, candidate: _AutoRunCandidate) -> None:
async def _run_candidate_ingestion(
self,
*,
candidate: _AutoRunCandidate,
):
session_factory = get_session_factory()
async with session_factory() as session:
ingestion = ScholarIngestionService(source=self._source)
try:
run_summary = await ingestion.run_for_user(
return await ingestion.run_for_user(
session,
user_id=candidate.user_id,
trigger_type=RunTriggerType.SCHEDULED,
@ -234,14 +242,8 @@ class SchedulerService:
)
except RunAlreadyInProgressError:
await session.rollback()
logger.info(
"scheduler.run_skipped_locked",
extra={
"event": "scheduler.run_skipped_locked",
"user_id": candidate.user_id,
},
)
return
logger.info("scheduler.run_skipped_locked", extra={"event": "scheduler.run_skipped_locked", "user_id": candidate.user_id})
return None
except RunBlockedBySafetyPolicyError as exc:
await session.rollback()
logger.info(
@ -256,18 +258,16 @@ class SchedulerService:
"metric_value": 1,
},
)
return
return None
except Exception:
await session.rollback()
logger.exception(
"scheduler.run_failed",
extra={
"event": "scheduler.run_failed",
"user_id": candidate.user_id,
},
)
return
logger.exception("scheduler.run_failed", extra={"event": "scheduler.run_failed", "user_id": candidate.user_id})
return None
async def _run_candidate(self, candidate: _AutoRunCandidate) -> None:
run_summary = await self._run_candidate_ingestion(candidate=candidate)
if run_summary is None:
return
logger.info(
"scheduler.run_completed",
extra={
@ -292,41 +292,53 @@ class SchedulerService:
for job in jobs:
await self._run_queue_job(job)
async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
if job.attempt_count >= self._continuation_max_attempts:
session_factory = get_session_factory()
async with session_factory() as session:
dropped = await queue_service.mark_dropped(
session,
job_id=job.id,
reason="max_attempts_reached",
)
await session.commit()
if dropped is not None:
logger.warning(
"scheduler.queue_item_dropped_max_attempts",
extra={
"event": "scheduler.queue_item_dropped_max_attempts",
"queue_item_id": job.id,
"user_id": job.user_id,
"scholar_profile_id": job.scholar_profile_id,
"attempt_count": job.attempt_count,
"max_attempts": self._continuation_max_attempts,
},
)
return
async def _drop_queue_job_if_max_attempts(
self,
job: queue_service.ContinuationQueueJob,
) -> bool:
if job.attempt_count < self._continuation_max_attempts:
return False
session_factory = get_session_factory()
async with session_factory() as session:
dropped = await queue_service.mark_dropped(
session,
job_id=job.id,
reason="max_attempts_reached",
)
await session.commit()
if dropped is not None:
logger.warning(
"scheduler.queue_item_dropped_max_attempts",
extra={
"event": "scheduler.queue_item_dropped_max_attempts",
"queue_item_id": job.id,
"user_id": job.user_id,
"scholar_profile_id": job.scholar_profile_id,
"attempt_count": job.attempt_count,
"max_attempts": self._continuation_max_attempts,
},
)
return True
async def _mark_queue_job_retrying(
self,
job: queue_service.ContinuationQueueJob,
) -> bool:
session_factory = get_session_factory()
async with session_factory() as session:
queue_item = await queue_service.mark_retrying(session, job_id=job.id)
if queue_item is None:
await session.commit()
return
if queue_item.status == QueueItemStatus.DROPPED.value:
await session.commit()
return
await session.commit()
if queue_item is None:
return False
if queue_item.status == QueueItemStatus.DROPPED.value:
return False
return True
async def _queue_job_has_available_scholar(
self,
job: queue_service.ContinuationQueueJob,
) -> bool:
session_factory = get_session_factory()
async with session_factory() as session:
scholar_result = await session.execute(
select(ScholarProfile.id).where(
@ -336,33 +348,124 @@ class SchedulerService:
)
)
scholar_id = scholar_result.scalar_one_or_none()
if scholar_id is None:
dropped = await queue_service.mark_dropped(
session,
job_id=job.id,
reason="scholar_unavailable",
)
await session.commit()
if dropped is not None:
logger.info(
"scheduler.queue_item_dropped_scholar_unavailable",
extra={
"event": "scheduler.queue_item_dropped_scholar_unavailable",
"queue_item_id": job.id,
"user_id": job.user_id,
"scholar_profile_id": job.scholar_profile_id,
},
)
return
async with session_factory() as session:
request_delay_seconds = await self._load_request_delay_for_user(
if scholar_id is not None:
return True
dropped = await queue_service.mark_dropped(
session,
user_id=job.user_id,
job_id=job.id,
reason="scholar_unavailable",
)
await session.commit()
if dropped is not None:
logger.info(
"scheduler.queue_item_dropped_scholar_unavailable",
extra={
"event": "scheduler.queue_item_dropped_scholar_unavailable",
"queue_item_id": job.id,
"user_id": job.user_id,
"scholar_profile_id": job.scholar_profile_id,
},
)
return False
async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None:
session_factory = get_session_factory()
async with session_factory() as recovery_session:
await queue_service.reschedule_job(
recovery_session,
job_id=job.id,
delay_seconds=max(self._tick_seconds, 15),
reason="user_run_lock_active",
error="run_already_in_progress",
)
await recovery_session.commit()
logger.info(
"scheduler.queue_item_deferred_lock",
extra={"event": "scheduler.queue_item_deferred_lock", "queue_item_id": job.id, "user_id": job.user_id},
)
async def _reschedule_queue_job_safety_cooldown(
self,
job: queue_service.ContinuationQueueJob,
exc: RunBlockedBySafetyPolicyError,
) -> None:
cooldown_remaining_seconds = max(
self._tick_seconds,
int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
)
session_factory = get_session_factory()
async with session_factory() as recovery_session:
await queue_service.reschedule_job(
recovery_session,
job_id=job.id,
delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds),
reason="scrape_safety_cooldown",
error=str(exc.message),
)
await recovery_session.commit()
logger.info(
"scheduler.queue_item_deferred_safety_cooldown",
extra={
"event": "scheduler.queue_item_deferred_safety_cooldown",
"queue_item_id": job.id,
"user_id": job.user_id,
"reason": exc.safety_state.get("cooldown_reason"),
"cooldown_remaining_seconds": cooldown_remaining_seconds,
"metric_name": "scheduler_queue_item_deferred_safety_cooldown_total",
"metric_value": 1,
},
)
async def _reschedule_queue_job_after_exception(
self,
job: queue_service.ContinuationQueueJob,
*,
exc: Exception,
) -> None:
session_factory = get_session_factory()
async with session_factory() as recovery_session:
queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id)
if queue_item is None:
await recovery_session.commit()
return
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
await queue_service.mark_dropped(
recovery_session,
job_id=job.id,
reason="scheduler_exception_max_attempts",
error=str(exc),
)
await recovery_session.commit()
logger.warning(
"scheduler.queue_item_dropped_after_exception",
extra={"event": "scheduler.queue_item_dropped_after_exception", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count},
)
return
delay_seconds = queue_service.compute_backoff_seconds(
base_seconds=self._continuation_base_delay_seconds,
attempt_count=int(queue_item.attempt_count),
max_seconds=self._continuation_max_delay_seconds,
)
await queue_service.reschedule_job(
recovery_session,
job_id=job.id,
delay_seconds=delay_seconds,
reason="scheduler_exception",
error=str(exc),
)
await recovery_session.commit()
logger.exception("scheduler.queue_item_run_failed", extra={"event": "scheduler.queue_item_run_failed", "queue_item_id": job.id, "user_id": job.user_id})
async def _run_ingestion_for_queue_job(
self,
job: queue_service.ContinuationQueueJob,
):
session_factory = get_session_factory()
async with session_factory() as session:
request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id)
ingestion = ScholarIngestionService(source=self._source)
try:
run_summary = await ingestion.run_for_user(
return await ingestion.run_for_user(
session,
user_id=job.user_id,
trigger_type=RunTriggerType.SCHEDULED,
@ -372,9 +475,7 @@ class SchedulerService:
max_pages_per_scholar=self._max_pages_per_scholar,
page_size=self._page_size,
scholar_profile_ids={job.scholar_profile_id},
start_cstart_by_scholar_id={
job.scholar_profile_id: job.resume_cstart,
},
start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart},
auto_queue_continuations=self._continuation_queue_enabled,
queue_delay_seconds=self._continuation_base_delay_seconds,
alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold,
@ -383,200 +484,88 @@ class SchedulerService:
)
except RunAlreadyInProgressError:
await session.rollback()
async with session_factory() as recovery_session:
await queue_service.reschedule_job(
recovery_session,
job_id=job.id,
delay_seconds=max(self._tick_seconds, 15),
reason="user_run_lock_active",
error="run_already_in_progress",
)
await recovery_session.commit()
logger.info(
"scheduler.queue_item_deferred_lock",
extra={
"event": "scheduler.queue_item_deferred_lock",
"queue_item_id": job.id,
"user_id": job.user_id,
},
)
return
await self._reschedule_queue_job_lock_active(job)
except RunBlockedBySafetyPolicyError as exc:
await session.rollback()
cooldown_remaining_seconds = max(
self._tick_seconds,
int(exc.safety_state.get("cooldown_remaining_seconds") or 0),
)
async with session_factory() as recovery_session:
await queue_service.reschedule_job(
recovery_session,
job_id=job.id,
delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds),
reason="scrape_safety_cooldown",
error=str(exc.message),
)
await recovery_session.commit()
logger.info(
"scheduler.queue_item_deferred_safety_cooldown",
extra={
"event": "scheduler.queue_item_deferred_safety_cooldown",
"queue_item_id": job.id,
"user_id": job.user_id,
"reason": exc.safety_state.get("cooldown_reason"),
"cooldown_remaining_seconds": cooldown_remaining_seconds,
"metric_name": "scheduler_queue_item_deferred_safety_cooldown_total",
"metric_value": 1,
},
)
return
await self._reschedule_queue_job_safety_cooldown(job, exc)
except Exception as exc:
await session.rollback()
async with session_factory() as recovery_session:
queue_item = await queue_service.increment_attempt_count(
recovery_session,
job_id=job.id,
)
if queue_item is None:
await recovery_session.commit()
return
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
await queue_service.mark_dropped(
recovery_session,
job_id=job.id,
reason="scheduler_exception_max_attempts",
error=str(exc),
)
await recovery_session.commit()
logger.warning(
"scheduler.queue_item_dropped_after_exception",
extra={
"event": "scheduler.queue_item_dropped_after_exception",
"queue_item_id": job.id,
"user_id": job.user_id,
"attempt_count": queue_item.attempt_count,
},
)
return
delay_seconds = queue_service.compute_backoff_seconds(
base_seconds=self._continuation_base_delay_seconds,
attempt_count=int(queue_item.attempt_count),
max_seconds=self._continuation_max_delay_seconds,
)
await queue_service.reschedule_job(
recovery_session,
job_id=job.id,
delay_seconds=delay_seconds,
reason="scheduler_exception",
error=str(exc),
)
await recovery_session.commit()
logger.exception(
"scheduler.queue_item_run_failed",
extra={
"event": "scheduler.queue_item_run_failed",
"queue_item_id": job.id,
"user_id": job.user_id,
},
)
return
await self._reschedule_queue_job_after_exception(job, exc=exc)
return None
@staticmethod
def _log_queue_item_resolved(
*,
event_name: str,
job: queue_service.ContinuationQueueJob,
run_summary,
attempt_count: int | None = None,
delay_seconds: int | None = None,
) -> None:
payload = {
"event": event_name,
"queue_item_id": job.id,
"user_id": job.user_id,
"run_id": run_summary.crawl_run_id,
"status": run_summary.status.value,
}
if attempt_count is not None:
payload["attempt_count"] = attempt_count
if delay_seconds is not None:
payload["delay_seconds"] = delay_seconds
logger.info(event_name, extra=payload)
async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None:
session_factory = get_session_factory()
async with session_factory() as session:
# Failed-attempt budget should advance only when continuation execution fails.
if int(run_summary.failed_count) <= 0:
queue_item = await queue_service.reset_attempt_count(
session,
job_id=job.id,
)
if queue_item is None:
await session.commit()
logger.info(
"scheduler.queue_item_resolved",
extra={
"event": "scheduler.queue_item_resolved",
"queue_item_id": job.id,
"user_id": job.user_id,
"run_id": run_summary.crawl_run_id,
"status": run_summary.status.value,
},
)
return
queue_item = await queue_service.reset_attempt_count(session, job_id=job.id)
await session.commit()
logger.info(
"scheduler.queue_item_progressed",
extra={
"event": "scheduler.queue_item_progressed",
"queue_item_id": job.id,
"user_id": job.user_id,
"attempt_count": int(queue_item.attempt_count),
"run_id": run_summary.crawl_run_id,
"status": run_summary.status.value,
},
if queue_item is None:
self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
return
self._log_queue_item_resolved(
event_name="scheduler.queue_item_progressed",
job=job,
run_summary=run_summary,
attempt_count=int(queue_item.attempt_count),
)
return
queue_item = await queue_service.increment_attempt_count(
session,
job_id=job.id,
)
queue_item = await queue_service.increment_attempt_count(session, job_id=job.id)
if queue_item is None:
await session.commit()
logger.info(
"scheduler.queue_item_resolved",
extra={
"event": "scheduler.queue_item_resolved",
"queue_item_id": job.id,
"user_id": job.user_id,
"run_id": run_summary.crawl_run_id,
"status": run_summary.status.value,
},
)
self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary)
return
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
await queue_service.mark_dropped(
session,
job_id=job.id,
reason="max_attempts_after_run",
)
await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
await session.commit()
logger.warning(
"scheduler.queue_item_dropped_max_attempts_after_run",
extra={
"event": "scheduler.queue_item_dropped_max_attempts_after_run",
"queue_item_id": job.id,
"user_id": job.user_id,
"attempt_count": queue_item.attempt_count,
"run_id": run_summary.crawl_run_id,
"status": run_summary.status.value,
},
extra={"event": "scheduler.queue_item_dropped_max_attempts_after_run", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count, "run_id": run_summary.crawl_run_id, "status": run_summary.status.value},
)
return
delay_seconds = queue_service.compute_backoff_seconds(
base_seconds=self._continuation_base_delay_seconds,
attempt_count=int(queue_item.attempt_count),
max_seconds=self._continuation_max_delay_seconds,
)
await queue_service.reschedule_job(
session,
job_id=job.id,
delay_seconds=delay_seconds,
reason=queue_item.reason,
error=queue_item.last_error,
)
delay_seconds = queue_service.compute_backoff_seconds(base_seconds=self._continuation_base_delay_seconds, attempt_count=int(queue_item.attempt_count), max_seconds=self._continuation_max_delay_seconds)
await queue_service.reschedule_job(session, job_id=job.id, delay_seconds=delay_seconds, reason=queue_item.reason, error=queue_item.last_error)
await session.commit()
logger.info(
"scheduler.queue_item_rescheduled_failed",
extra={
"event": "scheduler.queue_item_rescheduled_failed",
"queue_item_id": job.id,
"user_id": job.user_id,
"attempt_count": queue_item.attempt_count,
"delay_seconds": delay_seconds,
"run_id": run_summary.crawl_run_id,
"status": run_summary.status.value,
},
)
self._log_queue_item_resolved(
event_name="scheduler.queue_item_rescheduled_failed",
job=job,
run_summary=run_summary,
attempt_count=int(queue_item.attempt_count),
delay_seconds=delay_seconds,
)
async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None:
if await self._drop_queue_job_if_max_attempts(job):
return
if not await self._mark_queue_job_retrying(job):
return
if not await self._queue_job_has_available_scholar(job):
return
run_summary = await self._run_ingestion_for_queue_job(job)
if run_summary is None:
return
await self._finalize_queue_job_after_run(job, run_summary)
async def _load_request_delay_for_user(
self,

166
app/services/scholar_parser.py
View file

@ -79,6 +79,18 @@ SHOW_MORE_BUTTON_RE = re.compile(
r"<button\b[^>]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>",
re.I | re.S,
)
PROFILE_ROW_PARSER_DIRECT_MARKERS = (
"gs_ggs",
"gs_ggsd",
"gs_ggsa",
"gs_or_ggsm",
)
PROFILE_ROW_DIRECT_LABEL_TOKENS = (
"pdf",
"[pdf]",
"full text",
"download",
)
class ParseState(StrEnum):
@ -98,6 +110,7 @@ class PublicationCandidate:
citation_count: int | None
authors_text: str | None
venue_text: str | None
pdf_url: str | None
@dataclass(frozen=True)
@ -174,6 +187,7 @@ class ScholarRowParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.title_href: str | None = None
self.direct_download_href: str | None = None
self.title_parts: list[str] = []
self.citation_parts: list[str] = []
self.year_parts: list[str] = []
@ -183,6 +197,13 @@ class ScholarRowParser(HTMLParser):
self._citation_depth = 0
self._year_depth = 0
self._gray_stack: list[dict[str, Any]] = []
self._direct_marker_depth = 0
self._aux_link_stack: list[dict[str, Any]] = []
@staticmethod
def _contains_direct_marker(classes: str) -> bool:
lowered = classes.lower()
return any(marker in lowered for marker in PROFILE_ROW_PARSER_DIRECT_MARKERS)
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if self._title_depth > 0:
@ -193,8 +214,14 @@ class ScholarRowParser(HTMLParser):
self._year_depth += 1
if self._gray_stack:
self._gray_stack[-1]["depth"] += 1
if self._direct_marker_depth > 0:
self._direct_marker_depth += 1
if self._aux_link_stack:
self._aux_link_stack[-1]["depth"] += 1
classes = attr_class(attrs)
if tag in {"div", "span"} and self._contains_direct_marker(classes):
self._direct_marker_depth = 1
if tag == "a" and "gsc_a_at" in classes:
self._title_depth = 1
@ -213,6 +240,16 @@ class ScholarRowParser(HTMLParser):
self._gray_stack.append({"depth": 1, "parts": []})
return
if tag == "a":
self._aux_link_stack.append(
{
"depth": 1,
"href": attr_href(attrs),
"classes": classes,
"parts": [],
}
)
def handle_data(self, data: str) -> None:
if self._title_depth > 0:
self.title_parts.append(data)
@ -222,6 +259,21 @@ class ScholarRowParser(HTMLParser):
self.year_parts.append(data)
if self._gray_stack:
self._gray_stack[-1]["parts"].append(data)
if self._aux_link_stack:
self._aux_link_stack[-1]["parts"].append(data)
def _capture_direct_download_href(self, link: dict[str, Any]) -> None:
if self.direct_download_href:
return
href = link.get("href")
if not isinstance(href, str) or not href.strip():
return
label = normalize_space("".join(link.get("parts", []))).lower()
classes = str(link.get("classes", "")).lower()
label_match = any(token in label for token in PROFILE_ROW_DIRECT_LABEL_TOKENS)
marker_match = self._contains_direct_marker(classes) or self._direct_marker_depth > 0
if label_match or marker_match:
self.direct_download_href = href.strip()
def handle_endtag(self, _tag: str) -> None:
if self._title_depth > 0:
@ -237,6 +289,13 @@ class ScholarRowParser(HTMLParser):
if text:
self.gray_texts.append(text)
self._gray_stack.pop()
if self._aux_link_stack:
self._aux_link_stack[-1]["depth"] -= 1
if self._aux_link_stack[-1]["depth"] == 0:
self._capture_direct_download_href(self._aux_link_stack[-1])
self._aux_link_stack.pop()
if self._direct_marker_depth > 0:
self._direct_marker_depth -= 1
def extract_rows(html: str) -> list[str]:
@ -302,37 +361,60 @@ def parse_citation_count(parts: list[str]) -> int | None:
return int(match.group(0))
def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None, list[str]]:
parser = ScholarRowParser()
parser.feed(row_html)
warnings: list[str] = []
title = normalize_space("".join(parser.title_parts))
if not title:
warnings.append("row_missing_title")
return None, warnings
if not parser.title_href:
warnings.append("row_missing_title_href")
citation_text = normalize_space(" ".join(parser.citation_parts))
citation_count = parse_citation_count(parser.citation_parts)
if citation_text and citation_count is None:
warnings.append("layout_row_citation_unparseable")
year_text = normalize_space(" ".join(parser.year_parts))
year = parse_year(parser.year_parts)
if year_text and year is None:
warnings.append("layout_row_year_unparseable")
authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
return (
PublicationCandidate(
title=title,
title_url=parser.title_href,
cluster_id=parse_cluster_id_from_href(parser.title_href),
year=year,
citation_count=citation_count,
authors_text=authors_text,
venue_text=venue_text,
pdf_url=build_absolute_scholar_url(parser.direct_download_href),
),
warnings,
)
def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]:
rows = extract_rows(html)
warnings: list[str] = []
publications: list[PublicationCandidate] = []
for row_html in rows:
parser = ScholarRowParser()
parser.feed(row_html)
title = normalize_space("".join(parser.title_parts))
if not title:
warnings.append("row_missing_title")
publication, row_warnings = _parse_publication_row(row_html)
warnings.extend(row_warnings)
if publication is None:
continue
authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None
venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None
publications.append(
PublicationCandidate(
title=title,
title_url=parser.title_href,
cluster_id=parse_cluster_id_from_href(parser.title_href),
year=parse_year(parser.year_parts),
citation_count=parse_citation_count(parser.citation_parts),
authors_text=authors_text,
venue_text=venue_text,
)
)
publications.append(publication)
if not rows:
warnings.append("no_rows_detected")
if rows and not publications:
warnings.append("layout_all_rows_unparseable")
return publications, sorted(set(warnings))
@ -635,11 +717,35 @@ def classify_block_or_captcha_reason(
return None
def _warnings_contain(warnings: list[str], code: str) -> bool:
return any(item == code for item in warnings)
def _has_layout_row_failure(marker_counts: dict[str, int], warnings: list[str]) -> bool:
if _warnings_contain(warnings, "layout_all_rows_unparseable"):
return True
if marker_counts.get("gsc_a_tr", 0) <= 0:
return False
if _warnings_contain(warnings, "row_missing_title"):
return True
return marker_counts.get("gsc_a_at", 0) <= 0
def _first_layout_warning(warnings: list[str]) -> str | None:
for warning in warnings:
if warning.startswith("layout_"):
return warning
return None
def detect_state(
fetch_result: FetchResult,
publications: list[PublicationCandidate],
marker_counts: dict[str, int],
*,
warnings: list[str],
has_show_more_button_flag: bool,
articles_range: str | None,
visible_text: str,
) -> tuple[ParseState, str]:
if fetch_result.status_code is None:
@ -660,6 +766,16 @@ def detect_state(
if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS):
return ParseState.NO_RESULTS, "no_results_keyword_detected"
layout_warning = _first_layout_warning(warnings)
if layout_warning is not None:
return ParseState.LAYOUT_CHANGED, layout_warning
if _has_layout_row_failure(marker_counts, warnings):
return ParseState.LAYOUT_CHANGED, "layout_publication_rows_unparseable"
if has_show_more_button_flag and not articles_range:
return ParseState.LAYOUT_CHANGED, "layout_show_more_without_articles_range"
if not publications:
has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0
has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0
@ -699,7 +815,7 @@ def detect_author_search_state(
has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0
if not has_search_markers:
return ParseState.NO_RESULTS, "no_search_candidates_detected"
return ParseState.OK, "search_markers_present_with_empty_results"
return ParseState.LAYOUT_CHANGED, "layout_author_candidates_unparseable"
return ParseState.OK, "author_candidates_extracted"
@ -711,6 +827,7 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
show_more = has_show_more_button(fetch_result.body)
operation_error_banner = has_operation_error_banner(fetch_result.body)
articles_range = extract_articles_range(fetch_result.body)
if show_more:
warnings.append("possible_partial_page_show_more_present")
@ -723,6 +840,9 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
fetch_result,
publications,
marker_counts,
warnings=warnings,
has_show_more_button_flag=show_more,
articles_range=articles_range,
visible_text=visible_text,
)
@ -736,7 +856,7 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage:
warnings=warnings,
has_show_more_button=show_more,
has_operation_error_banner=operation_error_banner,
articles_range=extract_articles_range(fetch_result.body),
articles_range=articles_range,
)

120
app/services/scholar_source.py
View file

@ -123,8 +123,8 @@ class LiveScholarSource:
)
return await asyncio.to_thread(self._fetch_sync, requested_url)
def _fetch_sync(self, requested_url: str) -> FetchResult:
request = Request(
def _build_request(self, requested_url: str) -> Request:
return Request(
requested_url,
headers={
"User-Agent": random.choice(self._user_agents),
@ -134,61 +134,75 @@ class LiveScholarSource:
},
)
@staticmethod
def _http_error_body(exc: HTTPError) -> str:
try:
return exc.read().decode("utf-8", errors="replace")
except Exception:
return ""
@staticmethod
def _network_error_result(requested_url: str, exc: URLError) -> FetchResult:
logger.warning(
"scholar_source.fetch_network_error",
extra={"event": "scholar_source.fetch_network_error", "requested_url": requested_url},
)
return FetchResult(
requested_url=requested_url,
status_code=None,
final_url=None,
body="",
error=str(exc),
)
@staticmethod
def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult:
logger.warning(
"scholar_source.fetch_http_error",
extra={
"event": "scholar_source.fetch_http_error",
"requested_url": requested_url,
"status_code": exc.code,
},
)
return FetchResult(
requested_url=requested_url,
status_code=exc.code,
final_url=exc.geturl(),
body=LiveScholarSource._http_error_body(exc),
error=str(exc),
)
@staticmethod
def _success_result(requested_url: str, response) -> FetchResult:
body = response.read().decode("utf-8", errors="replace")
status_code = getattr(response, "status", 200)
logger.debug(
"scholar_source.fetch_succeeded",
extra={
"event": "scholar_source.fetch_succeeded",
"requested_url": requested_url,
"status_code": status_code,
},
)
return FetchResult(
requested_url=requested_url,
status_code=status_code,
final_url=response.geturl(),
body=body,
error=None,
)
def _fetch_sync(self, requested_url: str) -> FetchResult:
request = self._build_request(requested_url)
try:
with urlopen(request, timeout=self._timeout_seconds) as response:
body = response.read().decode("utf-8", errors="replace")
status_code = getattr(response, "status", 200)
logger.debug(
"scholar_source.fetch_succeeded",
extra={
"event": "scholar_source.fetch_succeeded",
"requested_url": requested_url,
"status_code": status_code,
},
)
return FetchResult(
requested_url=requested_url,
status_code=status_code,
final_url=response.geturl(),
body=body,
error=None,
)
return self._success_result(requested_url, response)
except HTTPError as exc:
body = ""
try:
body = exc.read().decode("utf-8", errors="replace")
except Exception:
body = ""
logger.warning(
"scholar_source.fetch_http_error",
extra={
"event": "scholar_source.fetch_http_error",
"requested_url": requested_url,
"status_code": exc.code,
},
)
return FetchResult(
requested_url=requested_url,
status_code=exc.code,
final_url=exc.geturl(),
body=body,
error=str(exc),
)
return self._http_error_result(requested_url, exc)
except URLError as exc:
logger.warning(
"scholar_source.fetch_network_error",
extra={
"event": "scholar_source.fetch_network_error",
"requested_url": requested_url,
},
)
return FetchResult(
requested_url=requested_url,
status_code=None,
final_url=None,
body="",
error=str(exc),
)
return self._network_error_result(requested_url, exc)
def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:

745
app/services/scholars.py
View file

@ -269,73 +269,103 @@ def _serialize_parsed_author_search_page(parsed: ParsedAuthorSearchPage) -> dict
}
def _payload_state(payload: dict[str, object]) -> ParseState | None:
state_raw = str(payload.get("state", "")).strip()
try:
return ParseState(state_raw)
except ValueError:
return None
def _payload_marker_counts(payload: dict[str, object]) -> dict[str, int]:
marker_counts_payload = payload.get("marker_counts")
if not isinstance(marker_counts_payload, dict):
return {}
parsed: dict[str, int] = {}
for key, value in marker_counts_payload.items():
try:
parsed[str(key)] = int(value)
except (TypeError, ValueError):
continue
return parsed
def _payload_warnings(payload: dict[str, object]) -> list[str]:
warnings_payload = payload.get("warnings")
if not isinstance(warnings_payload, list):
return []
return [str(value) for value in warnings_payload if isinstance(value, str)]
def _parse_optional_string(value: object) -> str | None:
if value is None:
return None
normalized = str(value).strip()
return normalized or None
def _parse_optional_int(value: object) -> int | None:
if isinstance(value, int):
return value
if isinstance(value, str) and value.strip():
try:
return int(value)
except ValueError:
return None
return None
def _normalize_interests(value: object) -> list[str]:
if not isinstance(value, list):
return []
return [str(item) for item in value if isinstance(item, str)]
def _deserialize_candidate_payload(value: object) -> dict[str, object] | None:
if not isinstance(value, dict):
return None
scholar_id = str(value.get("scholar_id", "")).strip()
display_name = str(value.get("display_name", "")).strip()
profile_url = str(value.get("profile_url", "")).strip()
if not scholar_id or not display_name or not profile_url:
return None
return {
"scholar_id": scholar_id,
"display_name": display_name,
"affiliation": _parse_optional_string(value.get("affiliation")),
"email_domain": _parse_optional_string(value.get("email_domain")),
"cited_by_count": _parse_optional_int(value.get("cited_by_count")),
"interests": _normalize_interests(value.get("interests")),
"profile_url": profile_url,
"profile_image_url": _parse_optional_string(value.get("profile_image_url")),
}
def _deserialize_candidates(payload: dict[str, object]) -> list[dict[str, object]]:
candidates_payload = payload.get("candidates")
if not isinstance(candidates_payload, list):
return []
normalized: list[dict[str, object]] = []
for value in candidates_payload:
candidate = _deserialize_candidate_payload(value)
if candidate is not None:
normalized.append(candidate)
return normalized
def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearchPage | None:
if not isinstance(payload, dict):
return None
state_raw = str(payload.get("state", "")).strip()
try:
state = ParseState(state_raw)
except ValueError:
state = _payload_state(payload)
if state is None:
return None
marker_counts_payload = payload.get("marker_counts")
marker_counts = (
{str(key): int(value) for key, value in marker_counts_payload.items()}
if isinstance(marker_counts_payload, dict)
else {}
)
warnings_payload = payload.get("warnings")
warnings = (
[str(value) for value in warnings_payload if isinstance(value, str)]
if isinstance(warnings_payload, list)
else []
)
marker_counts = _payload_marker_counts(payload)
warnings = _payload_warnings(payload)
from app.services.scholar_parser import ScholarSearchCandidate
candidates_payload = payload.get("candidates")
normalized_candidates = []
for value in candidates_payload if isinstance(candidates_payload, list) else []:
if not isinstance(value, dict):
continue
scholar_id = str(value.get("scholar_id", "")).strip()
display_name = str(value.get("display_name", "")).strip()
profile_url = str(value.get("profile_url", "")).strip()
if not scholar_id or not display_name or not profile_url:
continue
interests_payload = value.get("interests")
interests = (
[str(item) for item in interests_payload if isinstance(item, str)]
if isinstance(interests_payload, list)
else []
)
cited_by_count = value.get("cited_by_count")
parsed_cited_by_count: int | None = None
if isinstance(cited_by_count, int):
parsed_cited_by_count = cited_by_count
elif isinstance(cited_by_count, str) and cited_by_count.strip():
try:
parsed_cited_by_count = int(cited_by_count)
except ValueError:
parsed_cited_by_count = None
normalized_candidates.append(
{
"scholar_id": scholar_id,
"display_name": display_name,
"affiliation": str(value.get("affiliation")).strip()
if value.get("affiliation") is not None
else None,
"email_domain": str(value.get("email_domain")).strip()
if value.get("email_domain") is not None
else None,
"cited_by_count": parsed_cited_by_count,
"interests": interests,
"profile_url": profile_url,
"profile_image_url": str(value.get("profile_image_url")).strip()
if value.get("profile_image_url") is not None
else None,
}
)
normalized_candidates = _deserialize_candidates(payload)
return ParsedAuthorSearchPage(
state=state,
@ -552,278 +582,445 @@ async def delete_scholar(
await db_session.commit()
async def search_author_candidates(
*,
source: ScholarSource,
db_session: AsyncSession,
query: str,
limit: int,
network_error_retries: int = 1,
retry_backoff_seconds: float = 1.0,
search_enabled: bool = True,
cache_ttl_seconds: int = 21_600,
blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS,
cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES,
min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS,
interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS,
cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD,
cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS,
retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD,
cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD,
) -> ParsedAuthorSearchPage:
def _normalize_author_search_inputs(query: str, limit: int) -> tuple[str, int, str]:
normalized_query = query.strip()
if len(normalized_query) < 2:
raise ScholarServiceError("Search query must be at least 2 characters.")
bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))
query_key = normalized_query.casefold()
return normalized_query, bounded_limit, normalized_query.casefold()
if not search_enabled:
logger.warning(
"scholar_search.disabled_by_configuration",
extra={
"event": "scholar_search.disabled_by_configuration",
"query": normalized_query,
},
)
return _policy_blocked_author_search_result(
reason=SEARCH_DISABLED_REASON,
warning_codes=["author_search_disabled_by_configuration"],
limit=bounded_limit,
)
await _acquire_author_search_lock(db_session)
runtime_state = await _load_runtime_state_for_update(db_session)
runtime_state_updated = False
now_utc = datetime.now(timezone.utc)
def _disabled_search_result(*, normalized_query: str, bounded_limit: int) -> ParsedAuthorSearchPage:
logger.warning(
"scholar_search.disabled_by_configuration",
extra={"event": "scholar_search.disabled_by_configuration", "query": normalized_query},
)
return _policy_blocked_author_search_result(
reason=SEARCH_DISABLED_REASON,
warning_codes=["author_search_disabled_by_configuration"],
limit=bounded_limit,
)
if runtime_state.cooldown_until is not None:
cooldown_until = runtime_state.cooldown_until
if cooldown_until.tzinfo is None:
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
runtime_state.cooldown_until = cooldown_until
runtime_state_updated = True
if now_utc >= cooldown_until:
logger.info(
"scholar_search.cooldown_expired",
extra={
"event": "scholar_search.cooldown_expired",
"cooldown_until_utc": cooldown_until.isoformat(),
},
)
runtime_state.cooldown_until = None
runtime_state.cooldown_rejection_count = 0
runtime_state.cooldown_alert_emitted = False
runtime_state_updated = True
cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(runtime_state, now_utc)
if cooldown_remaining_seconds > 0:
runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1
bounded_cooldown_rejection_alert_threshold = max(
1,
int(cooldown_rejection_alert_threshold),
)
if (
int(runtime_state.cooldown_rejection_count) >= bounded_cooldown_rejection_alert_threshold
and not bool(runtime_state.cooldown_alert_emitted)
):
logger.error(
"scholar_search.cooldown_rejection_threshold_exceeded",
extra={
"event": "scholar_search.cooldown_rejection_threshold_exceeded",
"query": normalized_query,
"cooldown_rejection_count": int(runtime_state.cooldown_rejection_count),
"threshold": bounded_cooldown_rejection_alert_threshold,
"cooldown_until_utc": runtime_state.cooldown_until.isoformat()
if runtime_state.cooldown_until
else None,
},
)
runtime_state.cooldown_alert_emitted = True
runtime_state_updated = True
def _normalize_runtime_cooldown_state(
runtime_state: AuthorSearchRuntimeState,
*,
now_utc: datetime,
) -> bool:
if runtime_state.cooldown_until is None:
return False
cooldown_until = runtime_state.cooldown_until
updated = False
if cooldown_until.tzinfo is None:
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
runtime_state.cooldown_until = cooldown_until
updated = True
if now_utc < cooldown_until:
return updated
logger.info(
"scholar_search.cooldown_expired",
extra={"event": "scholar_search.cooldown_expired", "cooldown_until_utc": cooldown_until.isoformat()},
)
runtime_state.cooldown_until = None
runtime_state.cooldown_rejection_count = 0
runtime_state.cooldown_alert_emitted = False
return True
logger.warning(
"scholar_search.cooldown_active",
extra={
"event": "scholar_search.cooldown_active",
"query": normalized_query,
"cooldown_remaining_seconds": cooldown_remaining_seconds,
"cooldown_until_utc": runtime_state.cooldown_until.isoformat()
if runtime_state.cooldown_until
else None,
},
)
warning_codes = [
"author_search_cooldown_active",
f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
]
if bool(runtime_state.cooldown_alert_emitted):
warning_codes.append("author_search_cooldown_alert_threshold_exceeded")
if runtime_state_updated:
await db_session.commit()
return _policy_blocked_author_search_result(
reason=SEARCH_COOLDOWN_REASON,
warning_codes=warning_codes,
limit=bounded_limit,
)
def _cooldown_warning_codes(
*,
runtime_state: AuthorSearchRuntimeState,
cooldown_remaining_seconds: int,
) -> list[str]:
warning_codes = [
"author_search_cooldown_active",
f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
]
if bool(runtime_state.cooldown_alert_emitted):
warning_codes.append("author_search_cooldown_alert_threshold_exceeded")
return warning_codes
def _emit_cooldown_threshold_alert(
*,
runtime_state: AuthorSearchRuntimeState,
normalized_query: str,
cooldown_rejection_alert_threshold: int,
) -> bool:
runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1
threshold = max(1, int(cooldown_rejection_alert_threshold))
if int(runtime_state.cooldown_rejection_count) < threshold:
return True
if bool(runtime_state.cooldown_alert_emitted):
return True
logger.error(
"scholar_search.cooldown_rejection_threshold_exceeded",
extra={
"event": "scholar_search.cooldown_rejection_threshold_exceeded",
"query": normalized_query,
"cooldown_rejection_count": int(runtime_state.cooldown_rejection_count),
"threshold": threshold,
"cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
},
)
runtime_state.cooldown_alert_emitted = True
return True
def _cooldown_block_result(
*,
runtime_state: AuthorSearchRuntimeState,
normalized_query: str,
bounded_limit: int,
cooldown_rejection_alert_threshold: int,
cooldown_remaining_seconds: int,
) -> ParsedAuthorSearchPage:
_emit_cooldown_threshold_alert(
runtime_state=runtime_state,
normalized_query=normalized_query,
cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
)
logger.warning(
"scholar_search.cooldown_active",
extra={
"event": "scholar_search.cooldown_active",
"query": normalized_query,
"cooldown_remaining_seconds": cooldown_remaining_seconds,
"cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
},
)
return _policy_blocked_author_search_result(
reason=SEARCH_COOLDOWN_REASON,
warning_codes=_cooldown_warning_codes(
runtime_state=runtime_state,
cooldown_remaining_seconds=cooldown_remaining_seconds,
),
limit=bounded_limit,
)
async def _cache_hit_result(
db_session: AsyncSession,
*,
query_key: str,
now_utc: datetime,
normalized_query: str,
bounded_limit: int,
) -> ParsedAuthorSearchPage | None:
cached = await _cache_get_author_search_result(
db_session,
query_key=query_key,
now_utc=now_utc,
)
if cached is not None:
state_reason_override = (
SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None
)
logger.info(
"scholar_search.cache_hit",
extra={
"event": "scholar_search.cache_hit",
"query": normalized_query,
"state": cached.state.value,
"state_reason": cached.state_reason,
},
)
await db_session.commit()
return _trim_author_search_result(
cached,
limit=bounded_limit,
extra_warnings=["author_search_served_from_cache"],
state_reason_override=state_reason_override,
)
if cached is None:
return None
logger.info(
"scholar_search.cache_hit",
extra={
"event": "scholar_search.cache_hit",
"query": normalized_query,
"state": cached.state.value,
"state_reason": cached.state_reason,
},
)
state_reason_override = SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None
return _trim_author_search_result(
cached,
limit=bounded_limit,
extra_warnings=["author_search_served_from_cache"],
state_reason_override=state_reason_override,
)
if runtime_state.last_live_request_at is not None:
def _throttle_sleep_seconds(
*,
runtime_state: AuthorSearchRuntimeState,
now_utc: datetime,
min_interval_seconds: float,
interval_jitter_seconds: float,
) -> tuple[float, bool]:
updated = False
if runtime_state.last_live_request_at is None:
enforced_wait_seconds = 0.0
else:
last_live_request_at = runtime_state.last_live_request_at
if last_live_request_at.tzinfo is None:
last_live_request_at = last_live_request_at.replace(tzinfo=timezone.utc)
runtime_state.last_live_request_at = last_live_request_at
runtime_state_updated = True
updated = True
enforced_wait_seconds = (
last_live_request_at + timedelta(seconds=max(float(min_interval_seconds), 0.0)) - now_utc
).total_seconds()
else:
enforced_wait_seconds = 0.0
jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0))
sleep_seconds = max(0.0, float(enforced_wait_seconds)) + jitter_seconds
if sleep_seconds > 0.0:
logger.info(
"scholar_search.throttle_wait",
extra={
"event": "scholar_search.throttle_wait",
"query": normalized_query,
"sleep_seconds": round(sleep_seconds, 3),
},
)
await asyncio.sleep(sleep_seconds)
return max(0.0, float(enforced_wait_seconds)) + jitter_seconds, updated
async def _wait_for_author_search_throttle(
*,
runtime_state: AuthorSearchRuntimeState,
normalized_query: str,
now_utc: datetime,
min_interval_seconds: float,
interval_jitter_seconds: float,
) -> bool:
sleep_seconds, updated = _throttle_sleep_seconds(
runtime_state=runtime_state,
now_utc=now_utc,
min_interval_seconds=min_interval_seconds,
interval_jitter_seconds=interval_jitter_seconds,
)
if sleep_seconds <= 0.0:
return updated
logger.info(
"scholar_search.throttle_wait",
extra={"event": "scholar_search.throttle_wait", "query": normalized_query, "sleep_seconds": round(sleep_seconds, 3)},
)
await asyncio.sleep(sleep_seconds)
return True
async def _fetch_author_search_with_retries(
*,
source: ScholarSource,
normalized_query: str,
network_error_retries: int,
retry_backoff_seconds: float,
) -> tuple[ParsedAuthorSearchPage, int, list[str]]:
max_attempts = max(1, int(network_error_retries) + 1)
parsed: ParsedAuthorSearchPage | None = None
retry_warnings: list[str] = []
retry_scheduled_count = 0
for attempt_index in range(max_attempts):
fetch_result = await source.fetch_author_search_html(normalized_query, start=0)
parsed = parse_author_search_page(fetch_result)
if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1:
break
retry_warnings.append("network_retry_scheduled_for_author_search")
retry_scheduled_count += 1
retry_sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index)
if retry_sleep_seconds > 0:
await asyncio.sleep(retry_sleep_seconds)
runtime_state.last_live_request_at = datetime.now(timezone.utc)
runtime_state_updated = True
if parsed is None:
raise ScholarServiceError("Unable to complete scholar author search.")
return parsed, retry_scheduled_count, retry_warnings
merged_parsed = replace(
parsed,
warnings=_merge_warnings(parsed.warnings, retry_warnings),
def _with_retry_warnings(
parsed: ParsedAuthorSearchPage,
*,
retry_warnings: list[str],
retry_scheduled_count: int,
retry_alert_threshold: int,
normalized_query: str,
) -> ParsedAuthorSearchPage:
merged = replace(parsed, warnings=_merge_warnings(parsed.warnings, retry_warnings))
threshold = max(1, int(retry_alert_threshold))
if retry_scheduled_count < threshold:
return merged
logger.warning(
"scholar_search.retry_threshold_exceeded",
extra={
"event": "scholar_search.retry_threshold_exceeded",
"query": normalized_query,
"retry_scheduled_count": retry_scheduled_count,
"threshold": threshold,
"final_state": merged.state.value,
"final_state_reason": merged.state_reason,
},
)
return replace(
merged,
warnings=_merge_warnings(
merged.warnings,
[f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"],
),
)
bounded_retry_alert_threshold = max(1, int(retry_alert_threshold))
if retry_scheduled_count >= bounded_retry_alert_threshold:
logger.warning(
"scholar_search.retry_threshold_exceeded",
extra={
"event": "scholar_search.retry_threshold_exceeded",
"query": normalized_query,
"retry_scheduled_count": retry_scheduled_count,
"threshold": bounded_retry_alert_threshold,
"final_state": merged_parsed.state.value,
"final_state_reason": merged_parsed.state_reason,
},
)
merged_parsed = replace(
merged_parsed,
warnings=_merge_warnings(
merged_parsed.warnings,
[f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"],
),
)
if _is_author_search_block_state(merged_parsed):
runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1
logger.warning(
"scholar_search.block_detected",
extra={
"event": "scholar_search.block_detected",
"query": normalized_query,
"state_reason": merged_parsed.state_reason,
"consecutive_blocked_count": int(runtime_state.consecutive_blocked_count),
},
)
if int(runtime_state.consecutive_blocked_count) >= max(1, int(cooldown_block_threshold)):
runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta(
seconds=max(60, int(cooldown_seconds))
)
runtime_state.consecutive_blocked_count = 0
runtime_state.cooldown_rejection_count = 0
runtime_state.cooldown_alert_emitted = False
merged_parsed = replace(
merged_parsed,
warnings=_merge_warnings(
merged_parsed.warnings,
["author_search_circuit_breaker_armed"],
),
)
logger.error(
"scholar_search.cooldown_activated",
extra={
"event": "scholar_search.cooldown_activated",
"query": normalized_query,
"cooldown_until_utc": runtime_state.cooldown_until.isoformat()
if runtime_state.cooldown_until
else None,
},
)
else:
def _apply_block_circuit_breaker(
*,
runtime_state: AuthorSearchRuntimeState,
merged_parsed: ParsedAuthorSearchPage,
cooldown_block_threshold: int,
cooldown_seconds: int,
normalized_query: str,
) -> ParsedAuthorSearchPage:
if not _is_author_search_block_state(merged_parsed):
runtime_state.consecutive_blocked_count = 0
return merged_parsed
runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1
logger.warning(
"scholar_search.block_detected",
extra={
"event": "scholar_search.block_detected",
"query": normalized_query,
"state_reason": merged_parsed.state_reason,
"consecutive_blocked_count": int(runtime_state.consecutive_blocked_count),
},
)
if int(runtime_state.consecutive_blocked_count) < max(1, int(cooldown_block_threshold)):
return merged_parsed
runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta(seconds=max(60, int(cooldown_seconds)))
runtime_state.consecutive_blocked_count = 0
runtime_state.cooldown_rejection_count = 0
runtime_state.cooldown_alert_emitted = False
logger.error(
"scholar_search.cooldown_activated",
extra={
"event": "scholar_search.cooldown_activated",
"query": normalized_query,
"cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
},
)
return replace(
merged_parsed,
warnings=_merge_warnings(merged_parsed.warnings, ["author_search_circuit_breaker_armed"]),
)
ttl_seconds = (
min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds)))
if _is_author_search_block_state(merged_parsed)
else max(1, int(cache_ttl_seconds))
def _resolve_author_search_cache_ttl_seconds(
*,
merged_parsed: ParsedAuthorSearchPage,
blocked_cache_ttl_seconds: int,
cache_ttl_seconds: int,
) -> int:
if _is_author_search_block_state(merged_parsed):
return min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds)))
return max(1, int(cache_ttl_seconds))
async def _load_locked_runtime_state(
db_session: AsyncSession,
) -> AuthorSearchRuntimeState:
await _acquire_author_search_lock(db_session)
return await _load_runtime_state_for_update(db_session)
async def _cooldown_or_cache_result(
db_session: AsyncSession,
*,
runtime_state: AuthorSearchRuntimeState,
query_key: str,
normalized_query: str,
bounded_limit: int,
cooldown_rejection_alert_threshold: int,
) -> tuple[ParsedAuthorSearchPage | None, bool]:
runtime_state_updated = _normalize_runtime_cooldown_state(
runtime_state,
now_utc=datetime.now(timezone.utc),
)
cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(
runtime_state,
datetime.now(timezone.utc),
)
if cooldown_remaining_seconds > 0:
return (
_cooldown_block_result(
runtime_state=runtime_state,
normalized_query=normalized_query,
bounded_limit=bounded_limit,
cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
cooldown_remaining_seconds=cooldown_remaining_seconds,
),
True,
)
cached_result = await _cache_hit_result(
db_session,
query_key=query_key,
now_utc=datetime.now(timezone.utc),
normalized_query=normalized_query,
bounded_limit=bounded_limit,
)
return cached_result, runtime_state_updated
async def _perform_live_author_search(db_session: AsyncSession, *, source: ScholarSource, runtime_state: AuthorSearchRuntimeState, normalized_query: str, query_key: str, network_error_retries: int, retry_backoff_seconds: float, min_interval_seconds: float, interval_jitter_seconds: float, retry_alert_threshold: int, cooldown_block_threshold: int, cooldown_seconds: int, blocked_cache_ttl_seconds: int, cache_ttl_seconds: int, cache_max_entries: int) -> tuple[ParsedAuthorSearchPage, bool]:
runtime_state_updated = await _wait_for_author_search_throttle(
runtime_state=runtime_state,
normalized_query=normalized_query,
now_utc=datetime.now(timezone.utc),
min_interval_seconds=min_interval_seconds,
interval_jitter_seconds=interval_jitter_seconds,
)
parsed, retry_count, retry_warnings = await _fetch_author_search_with_retries(
source=source,
normalized_query=normalized_query,
network_error_retries=network_error_retries,
retry_backoff_seconds=retry_backoff_seconds,
)
runtime_state.last_live_request_at = datetime.now(timezone.utc)
merged = _with_retry_warnings(
parsed,
retry_warnings=retry_warnings,
retry_scheduled_count=retry_count,
retry_alert_threshold=retry_alert_threshold,
normalized_query=normalized_query,
)
merged = _apply_block_circuit_breaker(
runtime_state=runtime_state,
merged_parsed=merged,
cooldown_block_threshold=cooldown_block_threshold,
cooldown_seconds=cooldown_seconds,
normalized_query=normalized_query,
)
ttl_seconds = _resolve_author_search_cache_ttl_seconds(
merged_parsed=merged,
blocked_cache_ttl_seconds=blocked_cache_ttl_seconds,
cache_ttl_seconds=cache_ttl_seconds,
)
await _cache_set_author_search_result(
db_session,
query_key=query_key,
parsed=merged_parsed,
parsed=merged,
ttl_seconds=float(ttl_seconds),
max_entries=cache_max_entries,
now_utc=datetime.now(timezone.utc),
)
return merged, True
async def search_author_candidates(*, source: ScholarSource, db_session: AsyncSession, query: str, limit: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, search_enabled: bool = True, cache_ttl_seconds: int = 21_600, blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS, cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES, min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS, interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS, cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD, cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS, retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD) -> ParsedAuthorSearchPage:
normalized_query, bounded_limit, query_key = _normalize_author_search_inputs(query, limit)
if not search_enabled:
return _disabled_search_result(
normalized_query=normalized_query,
bounded_limit=bounded_limit,
)
runtime_state = await _load_locked_runtime_state(db_session)
early_result, runtime_state_updated = await _cooldown_or_cache_result(
db_session,
runtime_state=runtime_state,
query_key=query_key,
normalized_query=normalized_query,
bounded_limit=bounded_limit,
cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
)
if early_result is not None:
await db_session.commit()
return early_result
merged_parsed, live_runtime_updated = await _perform_live_author_search(
db_session,
source=source,
runtime_state=runtime_state,
normalized_query=normalized_query,
query_key=query_key,
network_error_retries=network_error_retries,
retry_backoff_seconds=retry_backoff_seconds,
min_interval_seconds=min_interval_seconds,
interval_jitter_seconds=interval_jitter_seconds,
retry_alert_threshold=retry_alert_threshold,
cooldown_block_threshold=cooldown_block_threshold,
cooldown_seconds=cooldown_seconds,
blocked_cache_ttl_seconds=blocked_cache_ttl_seconds,
cache_ttl_seconds=cache_ttl_seconds,
cache_max_entries=cache_max_entries,
)
runtime_state_updated = runtime_state_updated or live_runtime_updated
if runtime_state_updated:
await db_session.commit()
return _trim_author_search_result(
merged_parsed,
limit=bounded_limit,
)
return _trim_author_search_result(merged_parsed, limit=bounded_limit)
async def hydrate_profile_metadata(