diff --git a/README.md b/README.md index 553bf5c..752617c 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,15 @@ docker compose -f docker-compose.yml -f docker-compose.dev.yml down - Scholar tracking is user-scoped: each account can track the same Scholar ID independently. - Publications are shared/global records deduplicated by Scholar cluster ID and normalized fingerprint. - Per-account visibility and read state is stored on scholar-publication links, not on the global publication row. +- Publication records include both canonical Scholar detail URLs (`pub_url`) and direct download targets (`pdf_url`) when available. + +## Backend Architecture (Refactored) + +- `app/services/ingestion.py`: run orchestration, page retry/pagination logic, publication upsert/dedup, safety counters, continuation queue sync. +- `app/services/scholar_parser.py`: strict DOM parsing with explicit layout-state detection and fail-fast warning codes for malformed Scholar markup. +- `app/services/scheduler.py`: automatic/scheduled ingestion runner plus continuation queue draining and retry/drop policy. +- `app/services/publications.py`: user-scoped publication listing/read-state update queries used by dashboard/publication UIs. +- `app/services/import_export.py`: JSON import/export for tracked scholars and publication link state. ## Name Search Status @@ -272,7 +281,13 @@ Scheduled fixture probes run in GitHub Actions via `.github/workflows/scheduled- - `GET /api/v1/publications` supports `mode=all|unread|latest` (plus temporary alias `mode=new`). - `unread` = actionable read-state (`is_read=false`). - `latest` = discovery-state (`first seen in the latest completed check`). +- Publications include `pub_url` and `pdf_url` in API responses. `pdf_url` is a direct-download target extracted from Scholar result-side links when present. - Response counters: - `unread_count`: unread publications in current scope. - `latest_count`: publications discovered in latest completed check. - `new_count`: compatibility alias for `latest_count` (temporary). + +### Scholar Data Portability + +- `GET /api/v1/scholars/export`: exports tracked scholars plus scholar-publication link data as JSON. +- `POST /api/v1/scholars/import`: imports that JSON payload, upserting scholars, global publication records, and per-scholar read state. diff --git a/AGENTS.MD b/agents.md similarity index 69% rename from AGENTS.MD rename to agents.md index 3c72362..69f864c 100644 --- a/AGENTS.MD +++ b/agents.md @@ -24,5 +24,12 @@ These limits prevent IP bans and are not to be optimized away. ## 4. Current Environment & Stack * **Backend:** Python 3.12+, FastAPI, SQLAlchemy (Async/asyncpg), Alembic. -* **Frontend:** TypeScript, React, Vite. -* **Infrastructure:** Multi-stage Docker. \ No newline at end of file +* **Frontend:** TypeScript, Vue 3, Vite. +* **Infrastructure:** Multi-stage Docker. + +## 5. Refactored Service Boundaries (Current) +* **`app/services/scholar_parser.py`:** Parser contract is fail-fast. Layout drift must emit explicit `layout_*` reasons/warnings, never silent partial success. +* **`app/services/ingestion.py`:** Orchestrates ingestion runs; validate parser outputs before persistence; enforce publication candidate constraints before upsert. +* **`app/services/publications.py`:** Publication list/read-state query layer; include both `pub_url` and `pdf_url` for UI consumption. +* **`app/services/import_export.py`:** Handles JSON import/export for user-scoped scholars and scholar-publication link state while preserving global publication dedup rules. +* **`app/services/scheduler.py`:** Owns automatic runs and continuation queue retries/drops; do not bypass safety gate or cooldown logic. diff --git a/app/api/routers/publications.py b/app/api/routers/publications.py index 6e04685..53a8109 100644 --- a/app/api/routers/publications.py +++ b/app/api/routers/publications.py @@ -94,6 +94,7 @@ async def list_publications( "citation_count": item.citation_count, "venue_text": item.venue_text, "pub_url": item.pub_url, + "pdf_url": item.pdf_url, "is_read": item.is_read, "first_seen_at": item.first_seen_at, "is_new_in_latest_run": item.is_new_in_latest_run, diff --git a/app/api/routers/scholars.py b/app/api/routers/scholars.py index 7a4171e..b6a7e47 100644 --- a/app/api/routers/scholars.py +++ b/app/api/routers/scholars.py @@ -13,6 +13,9 @@ from app.api.errors import ApiException from app.api.responses import success_payload from app.api.runtime_deps import get_scholar_source from app.api.schemas import ( + DataExportEnvelope, + DataImportEnvelope, + DataImportRequest, MessageEnvelope, ScholarCreateRequest, ScholarEnvelope, @@ -22,6 +25,7 @@ from app.api.schemas import ( ) from app.db.models import User from app.db.session import get_db_session +from app.services import import_export as import_export_service from app.services import scholars as scholar_service from app.services.scholar_source import ScholarSource from app.settings import settings @@ -81,6 +85,68 @@ async def list_scholars( ) +@router.get( + "/export", + response_model=DataExportEnvelope, +) +async def export_scholars_and_publications( + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + data = await import_export_service.export_user_data( + db_session, + user_id=current_user.id, + ) + return success_payload(request, data=data) + + +@router.post( + "/import", + response_model=DataImportEnvelope, +) +async def import_scholars_and_publications( + payload: DataImportRequest, + request: Request, + db_session: AsyncSession = Depends(get_db_session), + current_user: User = Depends(get_api_current_user), +): + if ( + payload.schema_version is not None + and int(payload.schema_version) != import_export_service.EXPORT_SCHEMA_VERSION + ): + raise ApiException( + status_code=400, + code="invalid_import_schema_version", + message=( + "Import schema version is not supported. " + f"Expected {import_export_service.EXPORT_SCHEMA_VERSION}." + ), + ) + try: + result = await import_export_service.import_user_data( + db_session, + user_id=current_user.id, + scholars=[item.model_dump() for item in payload.scholars], + publications=[item.model_dump() for item in payload.publications], + ) + except import_export_service.ImportExportError as exc: + raise ApiException( + status_code=400, + code="invalid_import_payload", + message=str(exc), + ) from exc + logger.info( + "api.scholars.imported", + extra={ + "event": "api.scholars.imported", + "user_id": current_user.id, + **result, + }, + ) + return success_payload(request, data=result) + + @router.post( "", response_model=ScholarEnvelope, diff --git a/app/api/schemas.py b/app/api/schemas.py index f2f2b0a..c80afa5 100644 --- a/app/api/schemas.py +++ b/app/api/schemas.py @@ -186,6 +186,74 @@ class ScholarImageUrlUpdateRequest(BaseModel): model_config = ConfigDict(extra="forbid") +class ScholarExportItemData(BaseModel): + scholar_id: str + display_name: str | None = None + is_enabled: bool = True + profile_image_override_url: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class PublicationExportItemData(BaseModel): + scholar_id: str + cluster_id: str | None = None + fingerprint_sha256: str | None = None + title: str + year: int | None = None + citation_count: int = 0 + author_text: str | None = None + venue_text: str | None = None + pub_url: str | None = None + pdf_url: str | None = None + is_read: bool = False + + model_config = ConfigDict(extra="forbid") + + +class DataExportData(BaseModel): + schema_version: int + exported_at: str + scholars: list[ScholarExportItemData] + publications: list[PublicationExportItemData] + + model_config = ConfigDict(extra="forbid") + + +class DataExportEnvelope(BaseModel): + data: DataExportData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class DataImportRequest(BaseModel): + schema_version: int | None = None + scholars: list[ScholarExportItemData] = Field(default_factory=list) + publications: list[PublicationExportItemData] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class DataImportResultData(BaseModel): + scholars_created: int + scholars_updated: int + publications_created: int + publications_updated: int + links_created: int + links_updated: int + skipped_records: int + + model_config = ConfigDict(extra="forbid") + + +class DataImportEnvelope(BaseModel): + data: DataImportResultData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + class RunListItemData(BaseModel): id: int trigger_type: str @@ -505,6 +573,7 @@ class PublicationItemData(BaseModel): citation_count: int venue_text: str | None pub_url: str | None + pdf_url: str | None is_read: bool first_seen_at: datetime is_new_in_latest_run: bool diff --git a/app/main.py b/app/main.py index 7fbef93..264a8a2 100644 --- a/app/main.py +++ b/app/main.py @@ -1,6 +1,7 @@ from __future__ import annotations from contextlib import asynccontextmanager +import logging from pathlib import Path from fastapi import FastAPI, HTTPException @@ -23,6 +24,9 @@ from app.security.csrf import CSRFMiddleware from app.services.scheduler import SchedulerService from app.settings import settings +logger = logging.getLogger(__name__) +BUILD_MARKER = "2026-02-19.phase2.direct-pdf-import-export-dashboard-sync" + configure_logging( level=settings.log_level, log_format=settings.log_format, @@ -45,8 +49,22 @@ scheduler_service = SchedulerService( ) +def _log_startup_build_marker() -> None: + logger.info( + "app.startup_build_marker", + extra={ + "event": "app.startup_build_marker", + "build_marker": BUILD_MARKER, + "frontend_enabled": settings.frontend_enabled, + "scheduler_enabled": settings.scheduler_enabled, + "log_format": settings.log_format, + }, + ) + + @asynccontextmanager async def lifespan(_: FastAPI): + _log_startup_build_marker() await scheduler_service.start() yield await scheduler_service.stop() diff --git a/app/services/continuation_queue.py b/app/services/continuation_queue.py index 7d7d93e..7494420 100644 --- a/app/services/continuation_queue.py +++ b/app/services/continuation_queue.py @@ -41,6 +41,70 @@ def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_second return min(seconds, maximum) +async def _get_item_for_user_scholar( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, +) -> IngestionQueueItem | None: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.user_id == user_id, + IngestionQueueItem.scholar_profile_id == scholar_profile_id, + ) + ) + return result.scalar_one_or_none() + + +def _build_queue_item( + *, + now: datetime, + user_id: int, + scholar_profile_id: int, + normalized_cstart: int, + reason: str, + run_id: int | None, + next_attempt_dt: datetime, +) -> IngestionQueueItem: + return IngestionQueueItem( + user_id=user_id, + scholar_profile_id=scholar_profile_id, + resume_cstart=normalized_cstart, + reason=reason, + status=QueueItemStatus.QUEUED.value, + attempt_count=0, + next_attempt_dt=next_attempt_dt, + last_run_id=run_id, + last_error=None, + dropped_reason=None, + dropped_at=None, + created_at=now, + updated_at=now, + ) + + +def _update_existing_queue_item( + *, + item: IngestionQueueItem, + now: datetime, + normalized_cstart: int, + reason: str, + run_id: int | None, + next_attempt_dt: datetime, +) -> None: + item.resume_cstart = normalized_cstart + item.reason = reason + if item.status == QueueItemStatus.DROPPED.value: + item.attempt_count = 0 + item.status = QueueItemStatus.QUEUED.value + item.next_attempt_dt = next_attempt_dt + item.last_run_id = run_id + item.last_error = None + item.dropped_reason = None + item.dropped_at = None + item.updated_at = now + + async def upsert_job( db_session: AsyncSession, *, @@ -53,44 +117,33 @@ async def upsert_job( ) -> IngestionQueueItem: now = datetime.now(timezone.utc) next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds))) - result = await db_session.execute( - select(IngestionQueueItem).where( - IngestionQueueItem.user_id == user_id, - IngestionQueueItem.scholar_profile_id == scholar_profile_id, - ) + item = await _get_item_for_user_scholar( + db_session, + user_id=user_id, + scholar_profile_id=scholar_profile_id, ) - item = result.scalar_one_or_none() normalized_cstart = normalize_cstart(resume_cstart) if item is None: - item = IngestionQueueItem( + item = _build_queue_item( + now=now, user_id=user_id, scholar_profile_id=scholar_profile_id, - resume_cstart=normalized_cstart, + normalized_cstart=normalized_cstart, reason=reason, - status=QueueItemStatus.QUEUED.value, - attempt_count=0, + run_id=run_id, next_attempt_dt=next_attempt_dt, - last_run_id=run_id, - last_error=None, - dropped_reason=None, - dropped_at=None, - created_at=now, - updated_at=now, ) db_session.add(item) return item - item.resume_cstart = normalized_cstart - item.reason = reason - if item.status == QueueItemStatus.DROPPED.value: - item.attempt_count = 0 - item.status = QueueItemStatus.QUEUED.value - item.next_attempt_dt = next_attempt_dt - item.last_run_id = run_id - item.last_error = None - item.dropped_reason = None - item.dropped_at = None - item.updated_at = now + _update_existing_queue_item( + item=item, + now=now, + normalized_cstart=normalized_cstart, + reason=reason, + run_id=run_id, + next_attempt_dt=next_attempt_dt, + ) return item diff --git a/app/services/import_export.py b/app/services/import_export.py new file mode 100644 index 0000000..0c8a84b --- /dev/null +++ b/app/services/import_export.py @@ -0,0 +1,667 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +import hashlib +import re +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import Publication, ScholarProfile, ScholarPublication +from app.services import scholars as scholar_service +from app.services.ingestion import build_publication_url, normalize_title + +EXPORT_SCHEMA_VERSION = 1 +MAX_IMPORT_SCHOLARS = 10_000 +MAX_IMPORT_PUBLICATIONS = 100_000 +WORD_RE = re.compile(r"[a-z0-9]+") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") + + +class ImportExportError(ValueError): + """Raised when import/export payload constraints are violated.""" + + +@dataclass(frozen=True) +class ImportedPublicationInput: + profile: ScholarProfile + title: str + year: int | None + citation_count: int + author_text: str | None + venue_text: str | None + cluster_id: str | None + pub_url: str | None + pdf_url: str | None + fingerprint: str + is_read: bool + + +def _normalize_optional_text(value: Any) -> str | None: + if value is None: + return None + normalized = str(value).strip() + return normalized or None + + +def _normalize_optional_year(value: Any) -> int | None: + if value is None: + return None + try: + year = int(value) + except (TypeError, ValueError): + return None + if year < 1500 or year > 3000: + return None + return year + + +def _normalize_citation_count(value: Any) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + return 0 + return max(0, parsed) + + +def _first_author_last_name(authors_text: str | None) -> str: + if not authors_text: + return "" + first_author = authors_text.split(",", maxsplit=1)[0].strip().lower() + words = WORD_RE.findall(first_author) + if not words: + return "" + return words[-1] + + +def _first_venue_word(venue_text: str | None) -> str: + if not venue_text: + return "" + words = WORD_RE.findall(venue_text.lower()) + if not words: + return "" + return words[0] + + +def _build_fingerprint( + *, + title: str, + year: int | None, + author_text: str | None, + venue_text: str | None, +) -> str: + canonical = "|".join( + [ + normalize_title(title), + str(year) if year is not None else "", + _first_author_last_name(author_text), + _first_venue_word(venue_text), + ] + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _resolve_fingerprint( + *, + title: str, + year: int | None, + author_text: str | None, + venue_text: str | None, + provided_fingerprint: Any, +) -> str: + normalized = _normalize_optional_text(provided_fingerprint) + if normalized and SHA256_RE.fullmatch(normalized.lower()): + return normalized.lower() + return _build_fingerprint( + title=title, + year=year, + author_text=author_text, + venue_text=venue_text, + ) + + +def _validate_import_sizes( + *, + scholars: list[dict[str, Any]], + publications: list[dict[str, Any]], +) -> None: + if len(scholars) > MAX_IMPORT_SCHOLARS: + raise ImportExportError(f"Import exceeds max scholars ({MAX_IMPORT_SCHOLARS}).") + if len(publications) > MAX_IMPORT_PUBLICATIONS: + raise ImportExportError( + f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS})." + ) + + +async def _load_user_scholar_map( + db_session: AsyncSession, + *, + user_id: int, +) -> dict[str, ScholarProfile]: + result = await db_session.execute( + select(ScholarProfile).where(ScholarProfile.user_id == user_id) + ) + profiles = list(result.scalars().all()) + return {profile.scholar_id: profile for profile in profiles} + + +def _apply_imported_scholar_values( + *, + profile: ScholarProfile, + display_name: str | None, + profile_image_override_url: str | None, + is_enabled: bool, +) -> bool: + updated = False + if display_name and profile.display_name != display_name: + profile.display_name = display_name + updated = True + if profile.profile_image_override_url != profile_image_override_url: + profile.profile_image_override_url = profile_image_override_url + updated = True + if bool(profile.is_enabled) != bool(is_enabled): + profile.is_enabled = bool(is_enabled) + updated = True + return updated + + +def _new_scholar_profile( + *, + user_id: int, + scholar_id: str, + display_name: str | None, + profile_image_override_url: str | None, + is_enabled: bool, +) -> ScholarProfile: + return ScholarProfile( + user_id=user_id, + scholar_id=scholar_id, + display_name=display_name, + profile_image_override_url=profile_image_override_url, + is_enabled=bool(is_enabled), + ) + + +async def _upsert_imported_scholars( + db_session: AsyncSession, + *, + user_id: int, + scholars: list[dict[str, Any]], +) -> tuple[dict[str, ScholarProfile], dict[str, int]]: + scholar_map = await _load_user_scholar_map(db_session, user_id=user_id) + counters = {"scholars_created": 0, "scholars_updated": 0, "skipped_records": 0} + for item in scholars: + try: + scholar_id = scholar_service.validate_scholar_id(str(item["scholar_id"])) + display_name = scholar_service.normalize_display_name(str(item.get("display_name") or "")) + override_url = scholar_service.normalize_profile_image_url( + _normalize_optional_text(item.get("profile_image_override_url")) + ) + except (KeyError, scholar_service.ScholarServiceError): + counters["skipped_records"] += 1 + continue + is_enabled = bool(item.get("is_enabled", True)) + existing = scholar_map.get(scholar_id) + if existing is None: + profile = _new_scholar_profile( + user_id=user_id, + scholar_id=scholar_id, + display_name=display_name, + profile_image_override_url=override_url, + is_enabled=is_enabled, + ) + db_session.add(profile) + scholar_map[scholar_id] = profile + counters["scholars_created"] += 1 + continue + if _apply_imported_scholar_values( + profile=existing, + display_name=display_name, + profile_image_override_url=override_url, + is_enabled=is_enabled, + ): + counters["scholars_updated"] += 1 + await db_session.flush() + return scholar_map, counters + + +async def _find_publication_by_cluster( + db_session: AsyncSession, + *, + cluster_id: str, +) -> Publication | None: + result = await db_session.execute( + select(Publication).where(Publication.cluster_id == cluster_id) + ) + return result.scalar_one_or_none() + + +async def _find_publication_by_fingerprint( + db_session: AsyncSession, + *, + fingerprint_sha256: str, +) -> Publication | None: + result = await db_session.execute( + select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256) + ) + return result.scalar_one_or_none() + + +def _apply_imported_publication_values( + *, + publication: Publication, + title: str, + year: int | None, + citation_count: int, + author_text: str | None, + venue_text: str | None, + pub_url: str | None, + pdf_url: str | None, + cluster_id: str | None, +) -> bool: + updated = False + if cluster_id and publication.cluster_id != cluster_id: + publication.cluster_id = cluster_id + updated = True + if publication.title_raw != title: + publication.title_raw = title + publication.title_normalized = normalize_title(title) + updated = True + if publication.year != year: + publication.year = year + updated = True + if int(publication.citation_count or 0) != citation_count: + publication.citation_count = citation_count + updated = True + if publication.author_text != author_text: + publication.author_text = author_text + updated = True + if publication.venue_text != venue_text: + publication.venue_text = venue_text + updated = True + if pub_url and publication.pub_url != pub_url: + publication.pub_url = pub_url + updated = True + if pdf_url and publication.pdf_url != pdf_url: + publication.pdf_url = pdf_url + updated = True + return updated + + +def _new_publication( + *, + cluster_id: str | None, + fingerprint_sha256: str, + title: str, + year: int | None, + citation_count: int, + author_text: str | None, + venue_text: str | None, + pub_url: str | None, + pdf_url: str | None, +) -> Publication: + return Publication( + cluster_id=cluster_id, + fingerprint_sha256=fingerprint_sha256, + title_raw=title, + title_normalized=normalize_title(title), + year=year, + citation_count=citation_count, + author_text=author_text, + venue_text=venue_text, + pub_url=pub_url, + pdf_url=pdf_url, + ) + + +async def _resolve_publication_for_import( + db_session: AsyncSession, + *, + cluster_id: str | None, + fingerprint_sha256: str, + cluster_cache: dict[str, Publication | None], + fingerprint_cache: dict[str, Publication | None], +) -> Publication | None: + if cluster_id: + if cluster_id not in cluster_cache: + cluster_cache[cluster_id] = await _find_publication_by_cluster( + db_session, + cluster_id=cluster_id, + ) + if cluster_cache[cluster_id] is not None: + return cluster_cache[cluster_id] + if fingerprint_sha256 not in fingerprint_cache: + fingerprint_cache[fingerprint_sha256] = await _find_publication_by_fingerprint( + db_session, + fingerprint_sha256=fingerprint_sha256, + ) + return fingerprint_cache[fingerprint_sha256] + + +async def _upsert_scholar_publication_link( + db_session: AsyncSession, + *, + scholar_profile_id: int, + publication_id: int, + is_read: bool, +) -> tuple[bool, bool]: + result = await db_session.execute( + select(ScholarPublication).where( + ScholarPublication.scholar_profile_id == scholar_profile_id, + ScholarPublication.publication_id == publication_id, + ) + ) + link = result.scalar_one_or_none() + if link is None: + db_session.add( + ScholarPublication( + scholar_profile_id=scholar_profile_id, + publication_id=publication_id, + is_read=bool(is_read), + ) + ) + return True, False + if bool(link.is_read) == bool(is_read): + return False, False + link.is_read = bool(is_read) + return False, True + + +def _initialize_import_counters(counters: dict[str, int]) -> None: + counters.update( + { + "publications_created": 0, + "publications_updated": 0, + "links_created": 0, + "links_updated": 0, + } + ) + + +def _build_imported_publication_input( + *, + item: dict[str, Any], + scholar_map: dict[str, ScholarProfile], +) -> ImportedPublicationInput | None: + scholar_id = _normalize_optional_text(item.get("scholar_id")) + title = _normalize_optional_text(item.get("title")) + if not scholar_id or not title: + return None + profile = scholar_map.get(scholar_id) + if profile is None: + return None + year = _normalize_optional_year(item.get("year")) + author_text = _normalize_optional_text(item.get("author_text")) + venue_text = _normalize_optional_text(item.get("venue_text")) + return ImportedPublicationInput( + profile=profile, + title=title, + year=year, + citation_count=_normalize_citation_count(item.get("citation_count")), + author_text=author_text, + venue_text=venue_text, + cluster_id=_normalize_optional_text(item.get("cluster_id")), + pub_url=build_publication_url(_normalize_optional_text(item.get("pub_url"))), + pdf_url=build_publication_url(_normalize_optional_text(item.get("pdf_url"))), + fingerprint=_resolve_fingerprint( + title=title, + year=year, + author_text=author_text, + venue_text=venue_text, + provided_fingerprint=item.get("fingerprint_sha256"), + ), + is_read=bool(item.get("is_read", False)), + ) + + +def _update_link_counters( + *, + counters: dict[str, int], + link_created: bool, + link_updated: bool, +) -> None: + if link_created: + counters["links_created"] += 1 + if link_updated: + counters["links_updated"] += 1 + + +def _cache_resolved_publication( + *, + publication: Publication, + cluster_id: str | None, + fingerprint_sha256: str, + cluster_cache: dict[str, Publication | None], + fingerprint_cache: dict[str, Publication | None], +) -> None: + if cluster_id: + cluster_cache[cluster_id] = publication + fingerprint_cache[fingerprint_sha256] = publication + + +async def _create_import_publication( + db_session: AsyncSession, + *, + payload: ImportedPublicationInput, +) -> Publication: + publication = _new_publication( + cluster_id=payload.cluster_id, + fingerprint_sha256=payload.fingerprint, + title=payload.title, + year=payload.year, + citation_count=payload.citation_count, + author_text=payload.author_text, + venue_text=payload.venue_text, + pub_url=payload.pub_url, + pdf_url=payload.pdf_url, + ) + db_session.add(publication) + await db_session.flush() + return publication + + +def _update_import_publication( + *, + publication: Publication, + payload: ImportedPublicationInput, +) -> bool: + return _apply_imported_publication_values( + publication=publication, + title=payload.title, + year=payload.year, + citation_count=payload.citation_count, + author_text=payload.author_text, + venue_text=payload.venue_text, + pub_url=payload.pub_url, + pdf_url=payload.pdf_url, + cluster_id=payload.cluster_id, + ) + + +async def _upsert_publication_entity( + db_session: AsyncSession, + *, + payload: ImportedPublicationInput, + cluster_cache: dict[str, Publication | None], + fingerprint_cache: dict[str, Publication | None], +) -> tuple[Publication, bool, bool]: + publication = await _resolve_publication_for_import( + db_session, + cluster_id=payload.cluster_id, + fingerprint_sha256=payload.fingerprint, + cluster_cache=cluster_cache, + fingerprint_cache=fingerprint_cache, + ) + created = False + updated = False + if publication is None: + publication = await _create_import_publication( + db_session, + payload=payload, + ) + created = True + else: + updated = _update_import_publication( + publication=publication, + payload=payload, + ) + _cache_resolved_publication( + publication=publication, + cluster_id=payload.cluster_id, + fingerprint_sha256=payload.fingerprint, + cluster_cache=cluster_cache, + fingerprint_cache=fingerprint_cache, + ) + return publication, created, updated + + +async def _upsert_imported_publication( + db_session: AsyncSession, + *, + payload: ImportedPublicationInput, + cluster_cache: dict[str, Publication | None], + fingerprint_cache: dict[str, Publication | None], + counters: dict[str, int], +) -> None: + publication, created, updated = await _upsert_publication_entity( + db_session, + payload=payload, + cluster_cache=cluster_cache, + fingerprint_cache=fingerprint_cache, + ) + if created: + counters["publications_created"] += 1 + if updated: + counters["publications_updated"] += 1 + link_created, link_updated = await _upsert_scholar_publication_link( + db_session, + scholar_profile_id=int(payload.profile.id), + publication_id=int(publication.id), + is_read=payload.is_read, + ) + _update_link_counters( + counters=counters, + link_created=link_created, + link_updated=link_updated, + ) + + +def _exported_at_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]: + return { + "scholar_id": profile.scholar_id, + "display_name": profile.display_name, + "is_enabled": bool(profile.is_enabled), + "profile_image_override_url": profile.profile_image_override_url, + } + + +def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]: + ( + scholar_id, + cluster_id, + fingerprint_sha256, + title_raw, + year, + citation_count, + author_text, + venue_text, + pub_url, + pdf_url, + is_read, + ) = row + return { + "scholar_id": scholar_id, + "cluster_id": cluster_id, + "fingerprint_sha256": fingerprint_sha256, + "title": title_raw, + "year": year, + "citation_count": int(citation_count or 0), + "author_text": author_text, + "venue_text": venue_text, + "pub_url": pub_url, + "pdf_url": pdf_url, + "is_read": bool(is_read), + } + + +async def export_user_data( + db_session: AsyncSession, + *, + user_id: int, +) -> dict[str, Any]: + scholars_result = await db_session.execute( + select(ScholarProfile) + .where(ScholarProfile.user_id == user_id) + .order_by(ScholarProfile.id.asc()) + ) + publication_result = await db_session.execute( + select( + ScholarProfile.scholar_id, + Publication.cluster_id, + Publication.fingerprint_sha256, + Publication.title_raw, + Publication.year, + Publication.citation_count, + Publication.author_text, + Publication.venue_text, + Publication.pub_url, + Publication.pdf_url, + ScholarPublication.is_read, + ) + .join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id) + .join(Publication, Publication.id == ScholarPublication.publication_id) + .where(ScholarProfile.user_id == user_id) + .order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) + ) + scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()] + publications = [ + _serialize_export_publication(row) + for row in publication_result.all() + ] + return { + "schema_version": EXPORT_SCHEMA_VERSION, + "exported_at": _exported_at_iso(), + "scholars": scholars, + "publications": publications, + } + + +async def import_user_data( + db_session: AsyncSession, + *, + user_id: int, + scholars: list[dict[str, Any]], + publications: list[dict[str, Any]], +) -> dict[str, int]: + _validate_import_sizes(scholars=scholars, publications=publications) + scholar_map, counters = await _upsert_imported_scholars( + db_session, + user_id=user_id, + scholars=scholars, + ) + cluster_cache: dict[str, Publication | None] = {} + fingerprint_cache: dict[str, Publication | None] = {} + _initialize_import_counters(counters) + for item in publications: + parsed_item = _build_imported_publication_input( + item=item, + scholar_map=scholar_map, + ) + if parsed_item is None: + counters["skipped_records"] += 1 + continue + await _upsert_imported_publication( + db_session, + payload=parsed_item, + cluster_cache=cluster_cache, + fingerprint_cache=fingerprint_cache, + counters=counters, + ) + await db_session.commit() + return counters diff --git a/app/services/ingestion.py b/app/services/ingestion.py index f8d8b60..91fac24 100644 --- a/app/services/ingestion.py +++ b/app/services/ingestion.py @@ -1,7 +1,7 @@ from __future__ import annotations import asyncio -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timezone import hashlib import json @@ -87,6 +87,59 @@ class PagedParseResult: skipped_no_change: bool +@dataclass +class RunProgress: + succeeded_count: int = 0 + failed_count: int = 0 + partial_count: int = 0 + scholar_results: list[dict[str, Any]] = field(default_factory=list) + + +@dataclass(frozen=True) +class ScholarProcessingOutcome: + result_entry: dict[str, Any] + succeeded_count_delta: int + failed_count_delta: int + partial_count_delta: int + discovered_publication_count: int + + +@dataclass(frozen=True) +class RunFailureSummary: + failed_state_counts: dict[str, int] + failed_reason_counts: dict[str, int] + scrape_failure_counts: dict[str, int] + retries_scheduled_count: int + scholars_with_retries_count: int + retry_exhausted_count: int + + +@dataclass(frozen=True) +class RunAlertSummary: + blocked_failure_count: int + network_failure_count: int + blocked_failure_threshold: int + network_failure_threshold: int + retry_scheduled_threshold: int + alert_flags: dict[str, bool] + + +@dataclass +class PagedLoopState: + fetch_result: FetchResult + parsed_page: ParsedProfilePage + attempt_log: list[dict[str, Any]] + page_logs: list[dict[str, Any]] + publications: list[PublicationCandidate] + pages_fetched: int + pages_attempted: int + current_cstart: int + next_cstart: int + has_more_remaining: bool = False + pagination_truncated_reason: str | None = None + continuation_cstart: int | None = None + + class RunAlreadyInProgressError(RuntimeError): """Raised when a run lock for a user is already held by another process.""" @@ -131,39 +184,36 @@ class ScholarIngestionService: def __init__(self, *, source: ScholarSource) -> None: self._source = source - async def run_for_user( + async def _load_user_settings_for_run( self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, - request_delay_seconds: int, - network_error_retries: int = 1, - retry_backoff_seconds: float = 1.0, - max_pages_per_scholar: int = 30, - page_size: int = 100, - scholar_profile_ids: set[int] | None = None, - start_cstart_by_scholar_id: dict[int, int] | None = None, - auto_queue_continuations: bool = True, - queue_delay_seconds: int = 60, - idempotency_key: str | None = None, - alert_blocked_failure_threshold: int = 1, - alert_network_failure_threshold: int = 2, - alert_retry_scheduled_threshold: int = 3, - ) -> RunExecutionSummary: + ): user_settings = await user_settings_service.get_or_create_settings( db_session, user_id=user_id, ) - now_utc = datetime.now(timezone.utc) - previous_safety_state = run_safety_service.get_safety_event_context( - user_settings, - now_utc=now_utc, + await self._enforce_safety_gate( + db_session, + user_settings=user_settings, + user_id=user_id, + trigger_type=trigger_type, ) - if run_safety_service.clear_expired_cooldown( - user_settings, - now_utc=now_utc, - ): + return user_settings + + async def _enforce_safety_gate( + self, + db_session: AsyncSession, + *, + user_settings, + user_id: int, + trigger_type: RunTriggerType, + ) -> None: + now_utc = datetime.now(timezone.utc) + previous = run_safety_service.get_safety_event_context(user_settings, now_utc=now_utc) + if run_safety_service.clear_expired_cooldown(user_settings, now_utc=now_utc): await db_session.commit() await db_session.refresh(user_settings) logger.info( @@ -171,48 +221,62 @@ class ScholarIngestionService: extra={ "event": "ingestion.safety_cooldown_cleared", "user_id": user_id, - "reason": previous_safety_state.get("cooldown_reason"), - "cooldown_until": previous_safety_state.get("cooldown_until"), + "reason": previous.get("cooldown_reason"), + "cooldown_until": previous.get("cooldown_until"), "metric_name": "ingestion_safety_cooldown_cleared_total", "metric_value": 1, }, ) now_utc = datetime.now(timezone.utc) if run_safety_service.is_cooldown_active(user_settings, now_utc=now_utc): - safety_state = run_safety_service.register_cooldown_blocked_start( - user_settings, + await self._raise_safety_blocked_start( + db_session, + user_settings=user_settings, + user_id=user_id, + trigger_type=trigger_type, now_utc=now_utc, ) - await db_session.commit() - logger.warning( - "ingestion.safety_policy_blocked_run_start", - extra={ - "event": "ingestion.safety_policy_blocked_run_start", - "user_id": user_id, - "trigger_type": trigger_type.value, - "reason": safety_state.get("cooldown_reason"), - "cooldown_until": safety_state.get("cooldown_until"), - "cooldown_remaining_seconds": safety_state.get("cooldown_remaining_seconds"), - "blocked_start_count": ( - (safety_state.get("counters") or {}).get("blocked_start_count") - ), - "metric_name": "ingestion_safety_run_start_blocked_total", - "metric_value": 1, - }, - ) - raise RunBlockedBySafetyPolicyError( - code="scrape_cooldown_active", - message="Scrape safety cooldown is active; run start is temporarily blocked.", - safety_state=safety_state, - ) - lock_acquired = await self._try_acquire_user_lock( - db_session, - user_id=user_id, + async def _raise_safety_blocked_start( + self, + db_session: AsyncSession, + *, + user_settings, + user_id: int, + trigger_type: RunTriggerType, + now_utc: datetime, + ) -> None: + safety_state = run_safety_service.register_cooldown_blocked_start( + user_settings, + now_utc=now_utc, + ) + await db_session.commit() + logger.warning( + "ingestion.safety_policy_blocked_run_start", + extra={ + "event": "ingestion.safety_policy_blocked_run_start", + "user_id": user_id, + "trigger_type": trigger_type.value, + "reason": safety_state.get("cooldown_reason"), + "cooldown_until": safety_state.get("cooldown_until"), + "cooldown_remaining_seconds": safety_state.get("cooldown_remaining_seconds"), + "blocked_start_count": ((safety_state.get("counters") or {}).get("blocked_start_count")), + "metric_name": "ingestion_safety_run_start_blocked_total", + "metric_value": 1, + }, + ) + raise RunBlockedBySafetyPolicyError( + code="scrape_cooldown_active", + message="Scrape safety cooldown is active; run start is temporarily blocked.", + safety_state=safety_state, ) - if not lock_acquired: - raise RunAlreadyInProgressError(f"Run already in progress for user_id={user_id}.") + @staticmethod + def _normalize_run_targets( + *, + scholar_profile_ids: set[int] | None, + start_cstart_by_scholar_id: dict[int, int] | None, + ) -> tuple[set[int] | None, dict[int, int]]: filtered_scholar_ids = ( {int(value) for value in scholar_profile_ids} if scholar_profile_ids is not None @@ -222,41 +286,94 @@ class ScholarIngestionService: int(key): max(0, int(value)) for key, value in (start_cstart_by_scholar_id or {}).items() } + return filtered_scholar_ids, start_cstart_map + async def _load_target_scholars( + self, + db_session: AsyncSession, + *, + user_id: int, + filtered_scholar_ids: set[int] | None, + ) -> list[ScholarProfile]: scholars_stmt = ( select(ScholarProfile) - .where( - ScholarProfile.user_id == user_id, - ScholarProfile.is_enabled.is_(True), - ) + .where(ScholarProfile.user_id == user_id, ScholarProfile.is_enabled.is_(True)) .order_by(ScholarProfile.created_at.asc(), ScholarProfile.id.asc()) ) if filtered_scholar_ids is not None: - scholars_stmt = scholars_stmt.where( - ScholarProfile.id.in_(filtered_scholar_ids) + scholars_stmt = scholars_stmt.where(ScholarProfile.id.in_(filtered_scholar_ids)) + scholars_result = await db_session.execute(scholars_stmt) + scholars = list(scholars_result.scalars().all()) + await self._clear_missing_filtered_jobs( + db_session, + user_id=user_id, + filtered_scholar_ids=filtered_scholar_ids, + scholars=scholars, + ) + return scholars + + async def _clear_missing_filtered_jobs( + self, + db_session: AsyncSession, + *, + user_id: int, + filtered_scholar_ids: set[int] | None, + scholars: list[ScholarProfile], + ) -> None: + if filtered_scholar_ids is None: + return + found_ids = {int(scholar.id) for scholar in scholars} + missing_ids = filtered_scholar_ids - found_ids + for scholar_profile_id in missing_ids: + await queue_service.clear_job_for_scholar( + db_session, + user_id=user_id, + scholar_profile_id=scholar_profile_id, ) - scholars_result = await db_session.execute( - scholars_stmt + @staticmethod + def _create_running_run( + *, + user_id: int, + trigger_type: RunTriggerType, + scholar_count: int, + idempotency_key: str | None, + ) -> CrawlRun: + return CrawlRun( + user_id=user_id, + trigger_type=trigger_type, + status=RunStatus.RUNNING, + scholar_count=scholar_count, + new_pub_count=0, + idempotency_key=idempotency_key, + error_log={}, ) - scholars = list(scholars_result.scalars().all()) - if filtered_scholar_ids is not None: - found_ids = {int(scholar.id) for scholar in scholars} - missing_ids = filtered_scholar_ids - found_ids - for scholar_profile_id in missing_ids: - await queue_service.clear_job_for_scholar( - db_session, - user_id=user_id, - scholar_profile_id=scholar_profile_id, - ) + + @staticmethod + def _log_run_started( + *, + user_id: int, + trigger_type: RunTriggerType, + scholar_count: int, + filtered: bool, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + idempotency_key: str | None, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> None: logger.info( "ingestion.run_started", extra={ "event": "ingestion.run_started", "user_id": user_id, "trigger_type": trigger_type.value, - "scholar_count": len(scholars), - "is_filtered_run": filtered_scholar_ids is not None, + "scholar_count": scholar_count, + "is_filtered_run": filtered, "request_delay_seconds": request_delay_seconds, "network_error_retries": network_error_retries, "retry_backoff_seconds": retry_backoff_seconds, @@ -269,256 +386,538 @@ class ScholarIngestionService: }, ) - run = CrawlRun( - user_id=user_id, - trigger_type=trigger_type, - status=RunStatus.RUNNING, - scholar_count=len(scholars), - new_pub_count=0, - idempotency_key=idempotency_key, - error_log={}, - ) - db_session.add(run) - await db_session.flush() + @staticmethod + async def _wait_between_scholars(*, index: int, request_delay_seconds: int) -> None: + if index <= 0 or request_delay_seconds <= 0: + return + await asyncio.sleep(float(request_delay_seconds)) - succeeded_count = 0 - failed_count = 0 - partial_count = 0 - scholar_results: list[dict[str, Any]] = [] + @staticmethod + def _assert_valid_paged_parse_result( + *, + scholar_id: str, + paged_parse_result: PagedParseResult, + ) -> None: + parsed_page = paged_parse_result.parsed_page + if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS}: + if any(code.startswith("layout_") for code in parsed_page.warnings): + raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.") + for publication in paged_parse_result.publications: + if not publication.title.strip(): + raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.") + if publication.citation_count is not None and int(publication.citation_count) < 0: + raise RuntimeError(f"Negative citation count for scholar_id={scholar_id}.") - for index, scholar in enumerate(scholars): - if index > 0 and request_delay_seconds > 0: - await asyncio.sleep(float(request_delay_seconds)) + @staticmethod + def _apply_first_page_profile_metadata( + *, + scholar: ScholarProfile, + paged_parse_result: PagedParseResult, + run_dt: datetime, + ) -> None: + first_page = paged_parse_result.first_page_parsed_page + if first_page.profile_name and not (scholar.display_name or "").strip(): + scholar.display_name = first_page.profile_name + if first_page.profile_image_url: + scholar.profile_image_url = first_page.profile_image_url + if paged_parse_result.first_page_fingerprint_sha256: + scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256 + scholar.last_initial_page_checked_at = run_dt - run_dt = datetime.now(timezone.utc) - start_cstart = int(start_cstart_map.get(int(scholar.id), 0)) - - paged_parse_result = await self._fetch_and_parse_all_pages_with_retry( - scholar_id=scholar.scholar_id, - start_cstart=start_cstart, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - max_pages=max_pages_per_scholar, - page_size=page_size, - previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256, - ) - fetch_result = paged_parse_result.fetch_result - parsed_page = paged_parse_result.parsed_page - publications = paged_parse_result.publications - attempt_log = paged_parse_result.attempt_log - page_logs = paged_parse_result.page_logs - - first_page = paged_parse_result.first_page_parsed_page - if first_page.profile_name and not (scholar.display_name or "").strip(): - scholar.display_name = first_page.profile_name - if first_page.profile_image_url: - scholar.profile_image_url = first_page.profile_image_url - if paged_parse_result.first_page_fingerprint_sha256: - scholar.last_initial_page_fingerprint_sha256 = ( - paged_parse_result.first_page_fingerprint_sha256 - ) - scholar.last_initial_page_checked_at = run_dt - - logger.info( - "ingestion.scholar_parsed", - extra={ - "event": "ingestion.scholar_parsed", - "user_id": user_id, - "crawl_run_id": run.id, - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - "state": parsed_page.state.value, - "publication_count": len(publications), - "has_show_more_button": parsed_page.has_show_more_button, - "pages_fetched": paged_parse_result.pages_fetched, - "pages_attempted": paged_parse_result.pages_attempted, - "has_more_remaining": paged_parse_result.has_more_remaining, - "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, - "warning_count": len(parsed_page.warnings), - "skipped_no_change": paged_parse_result.skipped_no_change, - }, - ) - - result_entry = { + @staticmethod + def _log_scholar_parsed( + *, + user_id: int, + run_id: int, + scholar: ScholarProfile, + paged_parse_result: PagedParseResult, + ) -> None: + parsed_page = paged_parse_result.parsed_page + logger.info( + "ingestion.scholar_parsed", + extra={ + "event": "ingestion.scholar_parsed", + "user_id": user_id, + "crawl_run_id": run_id, "scholar_profile_id": scholar.id, "scholar_id": scholar.scholar_id, "state": parsed_page.state.value, - "state_reason": parsed_page.state_reason, - "outcome": "failed", - "attempt_count": len(attempt_log), - "publication_count": len(publications), - "start_cstart": start_cstart, - "articles_range": parsed_page.articles_range, - "warnings": parsed_page.warnings, + "publication_count": len(paged_parse_result.publications), "has_show_more_button": parsed_page.has_show_more_button, "pages_fetched": paged_parse_result.pages_fetched, "pages_attempted": paged_parse_result.pages_attempted, "has_more_remaining": paged_parse_result.has_more_remaining, "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, - "continuation_cstart": paged_parse_result.continuation_cstart, + "warning_count": len(parsed_page.warnings), "skipped_no_change": paged_parse_result.skipped_no_change, - "initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, - } - if paged_parse_result.skipped_no_change: - scholar.last_run_status = RunStatus.SUCCESS - scholar.last_run_dt = run_dt - succeeded_count += 1 - result_entry["state"] = first_page.state.value - result_entry["state_reason"] = "no_change_initial_page_signature" - result_entry["outcome"] = "success" - result_entry["publication_count"] = 0 - result_entry["warnings"] = first_page.warnings - result_entry["debug"] = { - "state_reason": "no_change_initial_page_signature", - "first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, - "attempt_log": attempt_log, - "page_logs": page_logs, - } - scholar_results.append(result_entry) - queue_reason, queue_cstart = self._resolve_continuation_queue_target( - outcome=str(result_entry.get("outcome", "")), - state=str(result_entry.get("state", "")), - pagination_truncated_reason=paged_parse_result.pagination_truncated_reason, - continuation_cstart=paged_parse_result.continuation_cstart, - fallback_cstart=start_cstart, - ) - if auto_queue_continuations and queue_reason is not None: - await queue_service.upsert_job( - db_session, - user_id=user_id, - scholar_profile_id=scholar.id, - resume_cstart=queue_cstart, - reason=queue_reason, - run_id=run.id, - delay_seconds=queue_delay_seconds, - ) - result_entry["continuation_enqueued"] = True - result_entry["continuation_reason"] = queue_reason - result_entry["continuation_cstart"] = queue_cstart - else: - cleared = await queue_service.clear_job_for_scholar( - db_session, - user_id=user_id, - scholar_profile_id=scholar.id, - ) - if cleared: - result_entry["continuation_cleared"] = True - continue + }, + ) - had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS} - has_partial_publication_set = len(publications) > 0 and had_page_failure - is_partial_due_to_pagination = ( - paged_parse_result.has_more_remaining - or paged_parse_result.pagination_truncated_reason is not None + @staticmethod + def _build_result_entry( + *, + scholar: ScholarProfile, + start_cstart: int, + paged_parse_result: PagedParseResult, + ) -> dict[str, Any]: + parsed_page = paged_parse_result.parsed_page + return { + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "outcome": "failed", + "attempt_count": len(paged_parse_result.attempt_log), + "publication_count": len(paged_parse_result.publications), + "start_cstart": start_cstart, + "articles_range": parsed_page.articles_range, + "warnings": parsed_page.warnings, + "has_show_more_button": parsed_page.has_show_more_button, + "pages_fetched": paged_parse_result.pages_fetched, + "pages_attempted": paged_parse_result.pages_attempted, + "has_more_remaining": paged_parse_result.has_more_remaining, + "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, + "continuation_cstart": paged_parse_result.continuation_cstart, + "skipped_no_change": paged_parse_result.skipped_no_change, + "initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, + } + + def _skipped_no_change_outcome( + self, + *, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + ) -> ScholarProcessingOutcome: + first_page = paged_parse_result.first_page_parsed_page + scholar.last_run_status = RunStatus.SUCCESS + scholar.last_run_dt = run_dt + result_entry["state"] = first_page.state.value + result_entry["state_reason"] = "no_change_initial_page_signature" + result_entry["outcome"] = "success" + result_entry["publication_count"] = 0 + result_entry["warnings"] = first_page.warnings + result_entry["debug"] = { + "state_reason": "no_change_initial_page_signature", + "first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, + "attempt_log": paged_parse_result.attempt_log, + "page_logs": paged_parse_result.page_logs, + } + return ScholarProcessingOutcome( + result_entry=result_entry, + succeeded_count_delta=1, + failed_count_delta=0, + partial_count_delta=0, + discovered_publication_count=0, + ) + + async def _upsert_publications_outcome( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + ) -> ScholarProcessingOutcome: + parsed_page = paged_parse_result.parsed_page + publications = paged_parse_result.publications + had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS} + has_partial_set = len(publications) > 0 and had_page_failure + if (not had_page_failure) or has_partial_set: + return await self._upsert_success_or_exception( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + has_partial_publication_set=has_partial_set, + ) + return self._parse_failure_outcome( + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + + async def _upsert_success_or_exception( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + has_partial_publication_set: bool, + ) -> ScholarProcessingOutcome: + try: + return await self._upsert_success( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + has_partial_publication_set=has_partial_publication_set, + ) + except Exception as exc: + return self._upsert_exception_outcome( + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + exc=exc, ) - if (not had_page_failure) or has_partial_publication_set: - try: - discovered_publication_count = await self._upsert_profile_publications( - db_session, - run=run, - scholar=scholar, - publications=publications, - ) - run.new_pub_count = int(run.new_pub_count or 0) + discovered_publication_count - - is_partial_scholar = is_partial_due_to_pagination or has_partial_publication_set - scholar.last_run_status = ( - RunStatus.PARTIAL_FAILURE if is_partial_scholar else RunStatus.SUCCESS - ) - scholar.last_run_dt = run_dt - succeeded_count += 1 - result_entry["outcome"] = "partial" if is_partial_scholar else "success" - - if is_partial_scholar: - partial_count += 1 - result_entry["debug"] = self._build_failure_debug_context( - fetch_result=fetch_result, - parsed_page=parsed_page, - attempt_log=attempt_log, - page_logs=page_logs, - ) - except Exception as exc: - failed_count += 1 - scholar.last_run_status = RunStatus.FAILED - scholar.last_run_dt = run_dt - result_entry["state"] = "ingestion_error" - result_entry["state_reason"] = "publication_upsert_exception" - result_entry["outcome"] = "failed" - result_entry["error"] = str(exc) - result_entry["debug"] = self._build_failure_debug_context( - fetch_result=fetch_result, - parsed_page=parsed_page, - attempt_log=attempt_log, - page_logs=page_logs, - exception=exc, - ) - logger.exception( - "ingestion.scholar_failed", - extra={ - "event": "ingestion.scholar_failed", - "user_id": user_id, - "crawl_run_id": run.id, - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - }, - ) - else: - failed_count += 1 - scholar.last_run_status = RunStatus.FAILED - scholar.last_run_dt = run_dt - result_entry["debug"] = self._build_failure_debug_context( - fetch_result=fetch_result, - parsed_page=parsed_page, - attempt_log=attempt_log, - page_logs=page_logs, - ) - logger.warning( - "ingestion.scholar_parse_failed", - extra={ - "event": "ingestion.scholar_parse_failed", - "user_id": user_id, - "crawl_run_id": run.id, - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - "state": parsed_page.state.value, - "state_reason": parsed_page.state_reason, - "status_code": fetch_result.status_code, - }, - ) - - queue_reason, queue_cstart = self._resolve_continuation_queue_target( - outcome=str(result_entry.get("outcome", "")), - state=str(result_entry.get("state", "")), - pagination_truncated_reason=paged_parse_result.pagination_truncated_reason, - continuation_cstart=paged_parse_result.continuation_cstart, - fallback_cstart=start_cstart, + async def _upsert_success( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + has_partial_publication_set: bool, + ) -> ScholarProcessingOutcome: + discovered_count = await self._upsert_profile_publications( + db_session, + run=run, + scholar=scholar, + publications=paged_parse_result.publications, + ) + is_partial = ( + paged_parse_result.has_more_remaining + or paged_parse_result.pagination_truncated_reason is not None + or has_partial_publication_set + ) + scholar.last_run_status = RunStatus.PARTIAL_FAILURE if is_partial else RunStatus.SUCCESS + scholar.last_run_dt = run_dt + result_entry["outcome"] = "partial" if is_partial else "success" + if is_partial: + result_entry["debug"] = self._build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, ) - if auto_queue_continuations and queue_reason is not None: - await queue_service.upsert_job( - db_session, - user_id=user_id, - scholar_profile_id=scholar.id, - resume_cstart=queue_cstart, - reason=queue_reason, - run_id=run.id, - delay_seconds=queue_delay_seconds, - ) - result_entry["continuation_enqueued"] = True - result_entry["continuation_reason"] = queue_reason - result_entry["continuation_cstart"] = queue_cstart - else: - cleared = await queue_service.clear_job_for_scholar( - db_session, - user_id=user_id, - scholar_profile_id=scholar.id, - ) - if cleared: - result_entry["continuation_cleared"] = True + return ScholarProcessingOutcome( + result_entry=result_entry, + succeeded_count_delta=1, + failed_count_delta=0, + partial_count_delta=1 if is_partial else 0, + discovered_publication_count=discovered_count, + ) - scholar_results.append(result_entry) + def _upsert_exception_outcome( + self, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + exc: Exception, + ) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = run_dt + result_entry["state"] = "ingestion_error" + result_entry["state_reason"] = "publication_upsert_exception" + result_entry["outcome"] = "failed" + result_entry["error"] = str(exc) + result_entry["debug"] = self._build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, + exception=exc, + ) + logger.exception( + "ingestion.scholar_failed", + extra={ + "event": "ingestion.scholar_failed", + "crawl_run_id": run.id, + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + }, + ) + return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) + def _parse_failure_outcome( + self, + *, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + ) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = run_dt + result_entry["debug"] = self._build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, + ) + logger.warning( + "ingestion.scholar_parse_failed", + extra={ + "event": "ingestion.scholar_parse_failed", + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": paged_parse_result.parsed_page.state.value, + "state_reason": paged_parse_result.parsed_page.state_reason, + "status_code": paged_parse_result.fetch_result.status_code, + }, + ) + return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) + + async def _sync_continuation_queue( + self, + db_session: AsyncSession, + *, + user_id: int, + scholar: ScholarProfile, + run: CrawlRun, + start_cstart: int, + result_entry: dict[str, Any], + paged_parse_result: PagedParseResult, + auto_queue_continuations: bool, + queue_delay_seconds: int, + ) -> None: + queue_reason, queue_cstart = self._resolve_continuation_queue_target( + outcome=str(result_entry.get("outcome", "")), + state=str(result_entry.get("state", "")), + pagination_truncated_reason=paged_parse_result.pagination_truncated_reason, + continuation_cstart=paged_parse_result.continuation_cstart, + fallback_cstart=start_cstart, + ) + if auto_queue_continuations and queue_reason is not None: + await queue_service.upsert_job( + db_session, + user_id=user_id, + scholar_profile_id=scholar.id, + resume_cstart=queue_cstart, + reason=queue_reason, + run_id=run.id, + delay_seconds=queue_delay_seconds, + ) + result_entry["continuation_enqueued"] = True + result_entry["continuation_reason"] = queue_reason + result_entry["continuation_cstart"] = queue_cstart + return + if await queue_service.clear_job_for_scholar(db_session, user_id=user_id, scholar_profile_id=scholar.id): + result_entry["continuation_cleared"] = True + + async def _process_scholar( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + user_id: int, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + start_cstart: int, + auto_queue_continuations: bool, + queue_delay_seconds: int, + ) -> ScholarProcessingOutcome: + try: + return await self._process_scholar_inner( + db_session, + run=run, + scholar=scholar, + user_id=user_id, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + start_cstart=start_cstart, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + ) + except Exception as exc: + return self._unexpected_scholar_exception_outcome( + run=run, + scholar=scholar, + start_cstart=start_cstart, + exc=exc, + ) + + async def _process_scholar_inner( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + user_id: int, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + start_cstart: int, + auto_queue_continuations: bool, + queue_delay_seconds: int, + ) -> ScholarProcessingOutcome: + run_dt, paged_parse_result, result_entry = await self._fetch_and_prepare_scholar_result( + run=run, + scholar=scholar, + user_id=user_id, + start_cstart=start_cstart, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + ) + outcome = await self._resolve_scholar_outcome( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + await self._sync_continuation_queue( + db_session, + user_id=user_id, + scholar=scholar, + run=run, + start_cstart=start_cstart, + result_entry=outcome.result_entry, + paged_parse_result=paged_parse_result, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + ) + return outcome + + async def _fetch_and_prepare_scholar_result( + self, + *, + run: CrawlRun, + scholar: ScholarProfile, + user_id: int, + start_cstart: int, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + ) -> tuple[datetime, PagedParseResult, dict[str, Any]]: + run_dt = datetime.now(timezone.utc) + paged_parse_result = await self._fetch_and_parse_all_pages_with_retry( + scholar_id=scholar.scholar_id, + start_cstart=start_cstart, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages=max_pages_per_scholar, + page_size=page_size, + previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256, + ) + self._assert_valid_paged_parse_result(scholar_id=scholar.scholar_id, paged_parse_result=paged_parse_result) + self._apply_first_page_profile_metadata(scholar=scholar, paged_parse_result=paged_parse_result, run_dt=run_dt) + self._log_scholar_parsed(user_id=user_id, run_id=run.id, scholar=scholar, paged_parse_result=paged_parse_result) + result_entry = self._build_result_entry( + scholar=scholar, + start_cstart=start_cstart, + paged_parse_result=paged_parse_result, + ) + return run_dt, paged_parse_result, result_entry + + async def _resolve_scholar_outcome( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + ) -> ScholarProcessingOutcome: + if paged_parse_result.skipped_no_change: + return self._skipped_no_change_outcome( + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + return await self._upsert_publications_outcome( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + + @staticmethod + def _apply_outcome_to_progress( + *, + progress: RunProgress, + run: CrawlRun, + outcome: ScholarProcessingOutcome, + ) -> None: + progress.succeeded_count += outcome.succeeded_count_delta + progress.failed_count += outcome.failed_count_delta + progress.partial_count += outcome.partial_count_delta + run.new_pub_count = int(run.new_pub_count or 0) + outcome.discovered_publication_count + progress.scholar_results.append(outcome.result_entry) + + def _unexpected_scholar_exception_outcome( + self, + *, + run: CrawlRun, + scholar: ScholarProfile, + start_cstart: int, + exc: Exception, + ) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = datetime.now(timezone.utc) + logger.exception( + "ingestion.scholar_unexpected_failure", + extra={ + "event": "ingestion.scholar_unexpected_failure", + "crawl_run_id": run.id, + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + }, + ) + return ScholarProcessingOutcome( + result_entry={ + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": "ingestion_error", + "state_reason": "scholar_processing_exception", + "outcome": "failed", + "attempt_count": 0, + "publication_count": 0, + "start_cstart": start_cstart, + "warnings": [], + "error": str(exc), + "debug": {"exception_type": type(exc).__name__, "exception_message": str(exc)}, + }, + succeeded_count_delta=0, + failed_count_delta=1, + partial_count_delta=0, + discovered_publication_count=0, + ) + + @staticmethod + def _summarize_failures( + *, + scholar_results: list[dict[str, Any]], + ) -> RunFailureSummary: failed_state_counts: dict[str, int] = {} failed_reason_counts: dict[str, int] = {} scrape_failure_counts: dict[str, int] = {} @@ -526,115 +925,161 @@ class ScholarIngestionService: scholars_with_retries_count = 0 retry_exhausted_count = 0 for entry in scholar_results: - attempt_count = max(0, _int_or_default(entry.get("attempt_count"), 0)) - retries_for_entry = max(0, attempt_count - 1) + retries_for_entry = max(0, _int_or_default(entry.get("attempt_count"), 0) - 1) if retries_for_entry > 0: retries_scheduled_count += retries_for_entry scholars_with_retries_count += 1 - - outcome = str(entry.get("outcome", "")) - if outcome != "failed": + if str(entry.get("outcome", "")) != "failed": continue - state = str(entry.get("state", "")).strip() if state not in FAILED_STATES: continue failed_state_counts[state] = failed_state_counts.get(state, 0) + 1 - reason = str(entry.get("state_reason", "")).strip() if reason: failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1 - - failure_bucket = _classify_failure_bucket(state=state, state_reason=reason) - scrape_failure_counts[failure_bucket] = ( - scrape_failure_counts.get(failure_bucket, 0) + 1 - ) - - if ( - state == ParseState.NETWORK_ERROR.value - and retries_for_entry > 0 - ): + bucket = _classify_failure_bucket(state=state, state_reason=reason) + scrape_failure_counts[bucket] = scrape_failure_counts.get(bucket, 0) + 1 + if state == ParseState.NETWORK_ERROR.value and retries_for_entry > 0: retry_exhausted_count += 1 + return RunFailureSummary( + failed_state_counts=failed_state_counts, + failed_reason_counts=failed_reason_counts, + scrape_failure_counts=scrape_failure_counts, + retries_scheduled_count=retries_scheduled_count, + scholars_with_retries_count=scholars_with_retries_count, + retry_exhausted_count=retry_exhausted_count, + ) - blocked_failure_count = int(scrape_failure_counts.get(FAILURE_BUCKET_BLOCKED, 0)) - network_failure_count = int(scrape_failure_counts.get(FAILURE_BUCKET_NETWORK, 0)) - - bounded_blocked_threshold = max(1, int(alert_blocked_failure_threshold)) - bounded_network_threshold = max(1, int(alert_network_failure_threshold)) - bounded_retry_threshold = max(1, int(alert_retry_scheduled_threshold)) + @staticmethod + def _build_alert_summary( + *, + failure_summary: RunFailureSummary, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> RunAlertSummary: + blocked_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_BLOCKED, 0)) + network_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_NETWORK, 0)) + blocked_threshold = max(1, int(alert_blocked_failure_threshold)) + network_threshold = max(1, int(alert_network_failure_threshold)) + retry_threshold = max(1, int(alert_retry_scheduled_threshold)) alert_flags = { - "blocked_failure_threshold_exceeded": blocked_failure_count >= bounded_blocked_threshold, - "network_failure_threshold_exceeded": network_failure_count >= bounded_network_threshold, - "retry_scheduled_threshold_exceeded": retries_scheduled_count >= bounded_retry_threshold, + "blocked_failure_threshold_exceeded": blocked_failure_count >= blocked_threshold, + "network_failure_threshold_exceeded": network_failure_count >= network_threshold, + "retry_scheduled_threshold_exceeded": failure_summary.retries_scheduled_count >= retry_threshold, } + return RunAlertSummary( + blocked_failure_count=blocked_failure_count, + network_failure_count=network_failure_count, + blocked_failure_threshold=blocked_threshold, + network_failure_threshold=network_threshold, + retry_scheduled_threshold=retry_threshold, + alert_flags=alert_flags, + ) - if alert_flags["blocked_failure_threshold_exceeded"]: + @staticmethod + def _log_alert_thresholds( + *, + user_id: int, + run_id: int, + failure_summary: RunFailureSummary, + alert_summary: RunAlertSummary, + ) -> None: + if alert_summary.alert_flags["blocked_failure_threshold_exceeded"]: logger.warning( "ingestion.alert_blocked_failure_threshold_exceeded", extra={ "event": "ingestion.alert_blocked_failure_threshold_exceeded", "user_id": user_id, - "crawl_run_id": run.id, - "blocked_failure_count": blocked_failure_count, - "threshold": bounded_blocked_threshold, + "crawl_run_id": run_id, + "blocked_failure_count": alert_summary.blocked_failure_count, + "threshold": alert_summary.blocked_failure_threshold, "metric_name": "ingestion_blocked_failure_threshold_exceeded_total", "metric_value": 1, }, ) - if alert_flags["network_failure_threshold_exceeded"]: + if alert_summary.alert_flags["network_failure_threshold_exceeded"]: logger.warning( "ingestion.alert_network_failure_threshold_exceeded", extra={ "event": "ingestion.alert_network_failure_threshold_exceeded", "user_id": user_id, - "crawl_run_id": run.id, - "network_failure_count": network_failure_count, - "threshold": bounded_network_threshold, + "crawl_run_id": run_id, + "network_failure_count": alert_summary.network_failure_count, + "threshold": alert_summary.network_failure_threshold, "metric_name": "ingestion_network_failure_threshold_exceeded_total", "metric_value": 1, }, ) - if alert_flags["retry_scheduled_threshold_exceeded"]: + if alert_summary.alert_flags["retry_scheduled_threshold_exceeded"]: logger.warning( "ingestion.alert_retry_scheduled_threshold_exceeded", extra={ "event": "ingestion.alert_retry_scheduled_threshold_exceeded", "user_id": user_id, - "crawl_run_id": run.id, - "retries_scheduled_count": retries_scheduled_count, - "threshold": bounded_retry_threshold, + "crawl_run_id": run_id, + "retries_scheduled_count": failure_summary.retries_scheduled_count, + "threshold": alert_summary.retry_scheduled_threshold, "metric_name": "ingestion_retry_scheduled_threshold_exceeded_total", "metric_value": 1, }, ) - pre_apply_safety_state = run_safety_service.get_safety_event_context( + @staticmethod + def _apply_safety_outcome( + *, + user_settings, + run: CrawlRun, + user_id: int, + alert_summary: RunAlertSummary, + ) -> None: + pre_apply_state = run_safety_service.get_safety_event_context( user_settings, now_utc=datetime.now(timezone.utc), ) safety_state, cooldown_trigger_reason = run_safety_service.apply_run_safety_outcome( user_settings, run_id=int(run.id), - blocked_failure_count=blocked_failure_count, - network_failure_count=network_failure_count, - blocked_failure_threshold=bounded_blocked_threshold, - network_failure_threshold=bounded_network_threshold, + blocked_failure_count=alert_summary.blocked_failure_count, + network_failure_count=alert_summary.network_failure_count, + blocked_failure_threshold=alert_summary.blocked_failure_threshold, + network_failure_threshold=alert_summary.network_failure_threshold, blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds, network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds, now_utc=datetime.now(timezone.utc), ) + ScholarIngestionService._log_safety_transition( + user_id=user_id, + run_id=int(run.id), + alert_summary=alert_summary, + pre_apply_state=pre_apply_state, + safety_state=safety_state, + cooldown_trigger_reason=cooldown_trigger_reason, + ) + + @staticmethod + def _log_safety_transition( + *, + user_id: int, + run_id: int, + alert_summary: RunAlertSummary, + pre_apply_state: dict[str, Any], + safety_state: dict[str, Any], + cooldown_trigger_reason: str | None, + ) -> None: if cooldown_trigger_reason is not None: logger.warning( "ingestion.safety_cooldown_entered", extra={ "event": "ingestion.safety_cooldown_entered", "user_id": user_id, - "crawl_run_id": run.id, + "crawl_run_id": run_id, "reason": cooldown_trigger_reason, - "blocked_failure_count": blocked_failure_count, - "network_failure_count": network_failure_count, - "blocked_failure_threshold": bounded_blocked_threshold, - "network_failure_threshold": bounded_network_threshold, + "blocked_failure_count": alert_summary.blocked_failure_count, + "network_failure_count": alert_summary.network_failure_count, + "blocked_failure_threshold": alert_summary.blocked_failure_threshold, + "network_failure_threshold": alert_summary.network_failure_threshold, "cooldown_until": safety_state.get("cooldown_until"), "cooldown_remaining_seconds": safety_state.get("cooldown_remaining_seconds"), "safety_counters": safety_state.get("counters", {}), @@ -642,56 +1087,71 @@ class ScholarIngestionService: "metric_value": 1, }, ) - elif pre_apply_safety_state.get("cooldown_active") and not safety_state.get("cooldown_active"): + elif pre_apply_state.get("cooldown_active") and not safety_state.get("cooldown_active"): logger.info( "ingestion.safety_cooldown_cleared", extra={ "event": "ingestion.safety_cooldown_cleared", "user_id": user_id, - "crawl_run_id": run.id, - "reason": pre_apply_safety_state.get("cooldown_reason"), - "cooldown_until": pre_apply_safety_state.get("cooldown_until"), + "crawl_run_id": run_id, + "reason": pre_apply_state.get("cooldown_reason"), + "cooldown_until": pre_apply_state.get("cooldown_until"), "metric_name": "ingestion_safety_cooldown_cleared_total", "metric_value": 1, }, ) + def _finalize_run_record( + self, + *, + run: CrawlRun, + scholars: list[ScholarProfile], + progress: RunProgress, + failure_summary: RunFailureSummary, + alert_summary: RunAlertSummary, + idempotency_key: str | None, + ) -> None: run.end_dt = datetime.now(timezone.utc) run.status = self._resolve_run_status( scholar_count=len(scholars), - succeeded_count=succeeded_count, - failed_count=failed_count, - partial_count=partial_count, + succeeded_count=progress.succeeded_count, + failed_count=progress.failed_count, + partial_count=progress.partial_count, ) run.error_log = { - "scholar_results": scholar_results, + "scholar_results": progress.scholar_results, "summary": { - "succeeded_count": succeeded_count, - "failed_count": failed_count, - "partial_count": partial_count, - "failed_state_counts": failed_state_counts, - "failed_reason_counts": failed_reason_counts, - "scrape_failure_counts": scrape_failure_counts, + "succeeded_count": progress.succeeded_count, + "failed_count": progress.failed_count, + "partial_count": progress.partial_count, + "failed_state_counts": failure_summary.failed_state_counts, + "failed_reason_counts": failure_summary.failed_reason_counts, + "scrape_failure_counts": failure_summary.scrape_failure_counts, "retry_counts": { - "retries_scheduled_count": retries_scheduled_count, - "scholars_with_retries_count": scholars_with_retries_count, - "retry_exhausted_count": retry_exhausted_count, + "retries_scheduled_count": failure_summary.retries_scheduled_count, + "scholars_with_retries_count": failure_summary.scholars_with_retries_count, + "retry_exhausted_count": failure_summary.retry_exhausted_count, }, "alert_thresholds": { - "blocked_failure_threshold": bounded_blocked_threshold, - "network_failure_threshold": bounded_network_threshold, - "retry_scheduled_threshold": bounded_retry_threshold, + "blocked_failure_threshold": alert_summary.blocked_failure_threshold, + "network_failure_threshold": alert_summary.network_failure_threshold, + "retry_scheduled_threshold": alert_summary.retry_scheduled_threshold, }, - "alert_flags": alert_flags, + "alert_flags": alert_summary.alert_flags, }, - "meta": { - "idempotency_key": idempotency_key, - } - if idempotency_key - else {}, + "meta": {"idempotency_key": idempotency_key} if idempotency_key else {}, } - await db_session.commit() + @staticmethod + def _log_run_completed( + *, + user_id: int, + run: CrawlRun, + scholars: list[ScholarProfile], + progress: RunProgress, + alert_summary: RunAlertSummary, + failure_summary: RunFailureSummary, + ) -> None: logger.info( "ingestion.run_completed", extra={ @@ -700,27 +1160,252 @@ class ScholarIngestionService: "crawl_run_id": run.id, "status": run.status.value, "scholar_count": len(scholars), - "succeeded_count": succeeded_count, - "failed_count": failed_count, - "partial_count": partial_count, + "succeeded_count": progress.succeeded_count, + "failed_count": progress.failed_count, + "partial_count": progress.partial_count, "new_publication_count": run.new_pub_count, - "blocked_failure_count": blocked_failure_count, - "network_failure_count": network_failure_count, - "retries_scheduled_count": retries_scheduled_count, - "alert_flags": alert_flags, + "blocked_failure_count": alert_summary.blocked_failure_count, + "network_failure_count": alert_summary.network_failure_count, + "retries_scheduled_count": failure_summary.retries_scheduled_count, + "alert_flags": alert_summary.alert_flags, }, ) + @staticmethod + def _paging_kwargs( + *, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + ) -> dict[str, Any]: + return { + "request_delay_seconds": request_delay_seconds, + "network_error_retries": network_error_retries, + "retry_backoff_seconds": retry_backoff_seconds, + "max_pages_per_scholar": max_pages_per_scholar, + "page_size": page_size, + } + + @staticmethod + def _threshold_kwargs( + *, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> dict[str, Any]: + return { + "alert_blocked_failure_threshold": alert_blocked_failure_threshold, + "alert_network_failure_threshold": alert_network_failure_threshold, + "alert_retry_scheduled_threshold": alert_retry_scheduled_threshold, + } + + @staticmethod + def _run_execution_summary( + *, + run: CrawlRun, + scholars: list[ScholarProfile], + progress: RunProgress, + ) -> RunExecutionSummary: return RunExecutionSummary( crawl_run_id=run.id, status=run.status, scholar_count=len(scholars), - succeeded_count=succeeded_count, - failed_count=failed_count, - partial_count=partial_count, + succeeded_count=progress.succeeded_count, + failed_count=progress.failed_count, + partial_count=progress.partial_count, new_publication_count=run.new_pub_count, ) + async def _initialize_run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, scholar_profile_ids: set[int] | None, start_cstart_by_scholar_id: dict[int, int] | None, request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, max_pages_per_scholar: int, page_size: int, idempotency_key: str | None, alert_blocked_failure_threshold: int, alert_network_failure_threshold: int, alert_retry_scheduled_threshold: int) -> tuple[Any, CrawlRun, list[ScholarProfile], dict[int, int]]: + user_settings = await self._load_user_settings_for_run( + db_session, + user_id=user_id, + trigger_type=trigger_type, + ) + if not await self._try_acquire_user_lock(db_session, user_id=user_id): + raise RunAlreadyInProgressError(f"Run already in progress for user_id={user_id}.") + filtered_scholar_ids, start_cstart_map = self._normalize_run_targets( + scholar_profile_ids=scholar_profile_ids, + start_cstart_by_scholar_id=start_cstart_by_scholar_id, + ) + scholars = await self._load_target_scholars( + db_session, + user_id=user_id, + filtered_scholar_ids=filtered_scholar_ids, + ) + run = await self._start_run_record_for_targets( + db_session, + user_id=user_id, + trigger_type=trigger_type, + scholars=scholars, + filtered=filtered_scholar_ids is not None, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + idempotency_key=idempotency_key, + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) + return user_settings, run, scholars, start_cstart_map + + async def _start_run_record_for_targets( + self, + db_session: AsyncSession, + *, + user_id: int, + trigger_type: RunTriggerType, + scholars: list[ScholarProfile], + filtered: bool, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + idempotency_key: str | None, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> CrawlRun: + self._log_run_started( + user_id=user_id, + trigger_type=trigger_type, + scholar_count=len(scholars), + filtered=filtered, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + idempotency_key=idempotency_key, + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) + run = self._create_running_run( + user_id=user_id, + trigger_type=trigger_type, + scholar_count=len(scholars), + idempotency_key=idempotency_key, + ) + db_session.add(run) + await db_session.flush() + return run + + async def _run_scholar_iteration( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholars: list[ScholarProfile], + user_id: int, + start_cstart_map: dict[int, int], + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + auto_queue_continuations: bool, + queue_delay_seconds: int, + ) -> RunProgress: + progress = RunProgress() + for index, scholar in enumerate(scholars): + await self._wait_between_scholars(index=index, request_delay_seconds=request_delay_seconds) + start_cstart = int(start_cstart_map.get(int(scholar.id), 0)) + outcome = await self._process_scholar( + db_session, + run=run, + scholar=scholar, + user_id=user_id, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + start_cstart=start_cstart, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + ) + self._apply_outcome_to_progress(progress=progress, run=run, outcome=outcome) + return progress + + def _complete_run_for_user( + self, + *, + user_settings: Any, + run: CrawlRun, + scholars: list[ScholarProfile], + user_id: int, + progress: RunProgress, + idempotency_key: str | None, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> tuple[RunFailureSummary, RunAlertSummary]: + failure_summary = self._summarize_failures(scholar_results=progress.scholar_results) + alert_summary = self._build_alert_summary( + failure_summary=failure_summary, + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) + self._log_alert_thresholds( + user_id=user_id, + run_id=int(run.id), + failure_summary=failure_summary, + alert_summary=alert_summary, + ) + self._apply_safety_outcome(user_settings=user_settings, run=run, user_id=user_id, alert_summary=alert_summary) + self._finalize_run_record( + run=run, + scholars=scholars, + progress=progress, + failure_summary=failure_summary, + alert_summary=alert_summary, + idempotency_key=idempotency_key, + ) + return failure_summary, alert_summary + + async def run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, request_delay_seconds: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, max_pages_per_scholar: int = 30, page_size: int = 100, scholar_profile_ids: set[int] | None = None, start_cstart_by_scholar_id: dict[int, int] | None = None, auto_queue_continuations: bool = True, queue_delay_seconds: int = 60, idempotency_key: str | None = None, alert_blocked_failure_threshold: int = 1, alert_network_failure_threshold: int = 2, alert_retry_scheduled_threshold: int = 3) -> RunExecutionSummary: + paging_kwargs = self._paging_kwargs(request_delay_seconds=request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size) + threshold_kwargs = self._threshold_kwargs(alert_blocked_failure_threshold=alert_blocked_failure_threshold, alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold) + user_settings, run, scholars, start_cstart_map = await self._initialize_run_for_user( + db_session, + user_id=user_id, + trigger_type=trigger_type, + scholar_profile_ids=scholar_profile_ids, + start_cstart_by_scholar_id=start_cstart_by_scholar_id, + idempotency_key=idempotency_key, + **paging_kwargs, + **threshold_kwargs, + ) + progress = await self._run_scholar_iteration( + db_session, + run=run, + scholars=scholars, + user_id=user_id, + start_cstart_map=start_cstart_map, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + **paging_kwargs, + ) + failure_summary, alert_summary = self._complete_run_for_user( + user_settings=user_settings, + run=run, + scholars=scholars, + user_id=user_id, + progress=progress, + idempotency_key=idempotency_key, + **threshold_kwargs, + ) + await db_session.commit() + self._log_run_completed(user_id=user_id, run=run, scholars=scholars, progress=progress, alert_summary=alert_summary, failure_summary=failure_summary) + return self._run_execution_summary(run=run, scholars=scholars, progress=progress) + async def _fetch_profile_page( self, *, @@ -769,6 +1454,57 @@ class ScholarIngestionService: error=str(exc), ) + @staticmethod + def _attempt_log_entry( + *, + attempt: int, + cstart: int, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + ) -> dict[str, Any]: + return { + "attempt": attempt, + "cstart": cstart, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "status_code": fetch_result.status_code, + "fetch_error": fetch_result.error, + } + + @staticmethod + def _should_retry_network_page( + *, + parsed_page: ParsedProfilePage, + attempt_index: int, + max_attempts: int, + ) -> bool: + return parsed_page.state == ParseState.NETWORK_ERROR and attempt_index < max_attempts - 1 + + @staticmethod + async def _sleep_retry_backoff( + *, + scholar_id: str, + cstart: int, + attempt_index: int, + backoff: float, + state_reason: str, + ) -> None: + sleep_seconds = backoff * (2**attempt_index) + logger.warning( + "ingestion.scholar_retry_scheduled", + extra={ + "event": "ingestion.scholar_retry_scheduled", + "scholar_id": scholar_id, + "cstart": cstart, + "attempt": attempt_index + 1, + "next_attempt": attempt_index + 2, + "sleep_seconds": sleep_seconds, + "state_reason": state_reason, + }, + ) + if sleep_seconds > 0: + await asyncio.sleep(sleep_seconds) + async def _fetch_and_parse_page_with_retry( self, *, @@ -792,228 +1528,404 @@ class ScholarIngestionService: ) parsed_page = parse_profile_page(fetch_result) attempt_log.append( - { - "attempt": attempt_index + 1, - "cstart": cstart, - "state": parsed_page.state.value, - "state_reason": parsed_page.state_reason, - "status_code": fetch_result.status_code, - "fetch_error": fetch_result.error, - } + self._attempt_log_entry( + attempt=attempt_index + 1, + cstart=cstart, + fetch_result=fetch_result, + parsed_page=parsed_page, + ) ) - - should_retry = ( - parsed_page.state == ParseState.NETWORK_ERROR - and attempt_index < max_attempts - 1 - ) - if not should_retry: + if not self._should_retry_network_page( + parsed_page=parsed_page, + attempt_index=attempt_index, + max_attempts=max_attempts, + ): break - - sleep_seconds = backoff * (2**attempt_index) - logger.warning( - "ingestion.scholar_retry_scheduled", - extra={ - "event": "ingestion.scholar_retry_scheduled", - "scholar_id": scholar_id, - "cstart": cstart, - "attempt": attempt_index + 1, - "next_attempt": attempt_index + 2, - "sleep_seconds": sleep_seconds, - "state_reason": parsed_page.state_reason, - }, + await self._sleep_retry_backoff( + scholar_id=scholar_id, + cstart=cstart, + attempt_index=attempt_index, + backoff=backoff, + state_reason=parsed_page.state_reason, ) - if sleep_seconds > 0: - await asyncio.sleep(sleep_seconds) if fetch_result is None or parsed_page is None: raise RuntimeError("Fetch-and-parse retry loop produced no result.") return fetch_result, parsed_page, attempt_log - async def _fetch_and_parse_all_pages_with_retry( - self, + @staticmethod + def _page_log_entry( + *, + page_number: int, + cstart: int, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + attempt_count: int, + ) -> dict[str, Any]: + return { + "page": page_number, + "cstart": cstart, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "status_code": fetch_result.status_code, + "publication_count": len(parsed_page.publications), + "articles_range": parsed_page.articles_range, + "has_show_more_button": parsed_page.has_show_more_button, + "warning_codes": parsed_page.warnings, + "attempt_count": attempt_count, + } + + @staticmethod + def _should_skip_no_change( *, - scholar_id: str, start_cstart: int, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - max_pages: int, - page_size: int, - previous_initial_page_fingerprint_sha256: str | None = None, - ) -> PagedParseResult: - bounded_max_pages = max(1, int(max_pages)) - bounded_page_size = max(1, int(page_size)) - - ( - fetch_result, - parsed_page, - first_attempt_log, - ) = await self._fetch_and_parse_page_with_retry( - scholar_id=scholar_id, - cstart=start_cstart, - page_size=bounded_page_size, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - first_page_fetch_result = fetch_result - first_page_parsed_page = parsed_page - first_page_fingerprint_sha256 = build_initial_page_fingerprint(parsed_page) - - attempt_log: list[dict[str, Any]] = list(first_attempt_log) - page_logs: list[dict[str, Any]] = [] - page_logs.append( - { - "page": 1, - "cstart": start_cstart, - "state": parsed_page.state.value, - "state_reason": parsed_page.state_reason, - "status_code": fetch_result.status_code, - "publication_count": len(parsed_page.publications), - "articles_range": parsed_page.articles_range, - "has_show_more_button": parsed_page.has_show_more_button, - "warning_codes": parsed_page.warnings, - "attempt_count": len(first_attempt_log), - } - ) - pages_attempted = 1 - - should_skip_no_change = ( + first_page_fingerprint_sha256: str | None, + previous_initial_page_fingerprint_sha256: str | None, + parsed_page: ParsedProfilePage, + ) -> bool: + return ( start_cstart <= 0 and first_page_fingerprint_sha256 is not None and previous_initial_page_fingerprint_sha256 is not None and first_page_fingerprint_sha256 == previous_initial_page_fingerprint_sha256 and parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} ) - if should_skip_no_change: - return PagedParseResult( - fetch_result=fetch_result, - parsed_page=parsed_page, - first_page_fetch_result=first_page_fetch_result, - first_page_parsed_page=first_page_parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - publications=[], - attempt_log=attempt_log, - page_logs=page_logs, - pages_fetched=1, - pages_attempted=pages_attempted, - has_more_remaining=False, - pagination_truncated_reason=None, - continuation_cstart=None, - skipped_no_change=True, - ) - - # Immediate hard failure: nothing to salvage. - if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: - return PagedParseResult( - fetch_result=fetch_result, - parsed_page=parsed_page, - first_page_fetch_result=first_page_fetch_result, - first_page_parsed_page=first_page_parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - publications=[], - attempt_log=attempt_log, - page_logs=page_logs, - pages_fetched=0, - pages_attempted=pages_attempted, - has_more_remaining=False, - pagination_truncated_reason=None, - continuation_cstart=( - start_cstart if parsed_page.state == ParseState.NETWORK_ERROR else None - ), - skipped_no_change=False, - ) - - publications = list(parsed_page.publications) - pages_fetched = 1 - has_more_remaining = False - pagination_truncated_reason: str | None = None - continuation_cstart: int | None = None - current_cstart = start_cstart - next_cstart = _next_cstart_value( - articles_range=parsed_page.articles_range, - fallback=current_cstart + len(parsed_page.publications), - ) - - while parsed_page.has_show_more_button: - if pages_fetched >= bounded_max_pages: - has_more_remaining = True - pagination_truncated_reason = "max_pages_reached" - continuation_cstart = next_cstart if next_cstart > current_cstart else current_cstart - break - if next_cstart <= current_cstart: - has_more_remaining = True - pagination_truncated_reason = "pagination_cursor_stalled" - continuation_cstart = current_cstart - break - if request_delay_seconds > 0: - await asyncio.sleep(float(request_delay_seconds)) - - current_cstart = next_cstart - ( - page_fetch_result, - page_parsed, - page_attempt_log, - ) = await self._fetch_and_parse_page_with_retry( - scholar_id=scholar_id, - cstart=current_cstart, - page_size=bounded_page_size, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - - pages_attempted += 1 - attempt_log.extend(page_attempt_log) - page_logs.append( - { - "page": pages_attempted, - "cstart": current_cstart, - "state": page_parsed.state.value, - "state_reason": page_parsed.state_reason, - "status_code": page_fetch_result.status_code, - "publication_count": len(page_parsed.publications), - "articles_range": page_parsed.articles_range, - "has_show_more_button": page_parsed.has_show_more_button, - "warning_codes": page_parsed.warnings, - "attempt_count": len(page_attempt_log), - } - ) - - fetch_result = page_fetch_result - parsed_page = page_parsed - if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: - has_more_remaining = True - pagination_truncated_reason = f"page_state_{parsed_page.state.value}" - continuation_cstart = current_cstart - break - - # Google may keep a stale/disabled "show more" marker while returning an empty tail page. - # Treat this as a terminal page to avoid false cursor-stalled partial runs. - if parsed_page.state == ParseState.NO_RESULTS and len(parsed_page.publications) == 0: - pages_fetched += 1 - break - - pages_fetched += 1 - publications.extend(parsed_page.publications) - next_cstart = _next_cstart_value( - articles_range=parsed_page.articles_range, - fallback=current_cstart + len(parsed_page.publications), - ) + @staticmethod + def _skip_no_change_result( + *, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + first_page_fingerprint_sha256: str | None, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]], + ) -> PagedParseResult: return PagedParseResult( fetch_result=fetch_result, parsed_page=parsed_page, + first_page_fetch_result=fetch_result, + first_page_parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + publications=[], + attempt_log=attempt_log, + page_logs=page_logs, + pages_fetched=1, + pages_attempted=1, + has_more_remaining=False, + pagination_truncated_reason=None, + continuation_cstart=None, + skipped_no_change=True, + ) + + @staticmethod + def _initial_failure_result( + *, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + first_page_fingerprint_sha256: str | None, + start_cstart: int, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]], + ) -> PagedParseResult: + continuation_cstart = start_cstart if parsed_page.state == ParseState.NETWORK_ERROR else None + return PagedParseResult( + fetch_result=fetch_result, + parsed_page=parsed_page, + first_page_fetch_result=fetch_result, + first_page_parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + publications=[], + attempt_log=attempt_log, + page_logs=page_logs, + pages_fetched=0, + pages_attempted=1, + has_more_remaining=False, + pagination_truncated_reason=None, + continuation_cstart=continuation_cstart, + skipped_no_change=False, + ) + + @staticmethod + def _build_loop_state( + *, + start_cstart: int, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]], + ) -> PagedLoopState: + next_cstart = _next_cstart_value( + articles_range=parsed_page.articles_range, + fallback=start_cstart + len(parsed_page.publications), + ) + return PagedLoopState( + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_log=attempt_log, + page_logs=page_logs, + publications=list(parsed_page.publications), + pages_fetched=1, + pages_attempted=1, + current_cstart=start_cstart, + next_cstart=next_cstart, + ) + + @staticmethod + def _set_truncated_state( + *, + state: PagedLoopState, + reason: str, + continuation_cstart: int, + ) -> None: + state.has_more_remaining = True + state.pagination_truncated_reason = reason + state.continuation_cstart = continuation_cstart + + def _should_stop_pagination(self, *, state: PagedLoopState, bounded_max_pages: int) -> bool: + if state.pages_fetched >= bounded_max_pages: + self._set_truncated_state( + state=state, + reason="max_pages_reached", + continuation_cstart=( + state.next_cstart if state.next_cstart > state.current_cstart else state.current_cstart + ), + ) + return True + if state.next_cstart <= state.current_cstart: + self._set_truncated_state( + state=state, + reason="pagination_cursor_stalled", + continuation_cstart=state.current_cstart, + ) + return True + return False + + async def _fetch_next_page( + self, + *, + scholar_id: str, + state: PagedLoopState, + request_delay_seconds: int, + bounded_page_size: int, + network_error_retries: int, + retry_backoff_seconds: float, + ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]: + if request_delay_seconds > 0: + await asyncio.sleep(float(request_delay_seconds)) + state.current_cstart = state.next_cstart + return await self._fetch_and_parse_page_with_retry( + scholar_id=scholar_id, + cstart=state.current_cstart, + page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + ) + + @staticmethod + def _record_next_page( + *, + state: PagedLoopState, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + page_attempt_log: list[dict[str, Any]], + ) -> None: + state.pages_attempted += 1 + state.attempt_log.extend(page_attempt_log) + state.page_logs.append( + ScholarIngestionService._page_log_entry( + page_number=state.pages_attempted, + cstart=state.current_cstart, + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_count=len(page_attempt_log), + ) + ) + state.fetch_result = fetch_result + state.parsed_page = parsed_page + + @staticmethod + def _handle_page_state_transition(*, state: PagedLoopState) -> bool: + if state.parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: + ScholarIngestionService._set_truncated_state( + state=state, + reason=f"page_state_{state.parsed_page.state.value}", + continuation_cstart=state.current_cstart, + ) + return True + if state.parsed_page.state == ParseState.NO_RESULTS and len(state.parsed_page.publications) == 0: + state.pages_fetched += 1 + return True + state.pages_fetched += 1 + state.publications.extend(state.parsed_page.publications) + state.next_cstart = _next_cstart_value( + articles_range=state.parsed_page.articles_range, + fallback=state.current_cstart + len(state.parsed_page.publications), + ) + return False + + async def _fetch_initial_page_context( + self, + *, + scholar_id: str, + start_cstart: int, + bounded_page_size: int, + network_error_retries: int, + retry_backoff_seconds: float, + ) -> tuple[FetchResult, ParsedProfilePage, str | None, list[dict[str, Any]], list[dict[str, Any]]]: + fetch_result, parsed_page, first_attempt_log = await self._fetch_and_parse_page_with_retry( + scholar_id=scholar_id, + cstart=start_cstart, + page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + ) + first_page_fingerprint_sha256 = build_initial_page_fingerprint(parsed_page) + attempt_log = list(first_attempt_log) + page_logs = [ + self._page_log_entry( + page_number=1, + cstart=start_cstart, + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_count=len(first_attempt_log), + ) + ] + return fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs + + async def _paginate_loop( + self, + *, + scholar_id: str, + state: PagedLoopState, + bounded_max_pages: int, + request_delay_seconds: int, + bounded_page_size: int, + network_error_retries: int, + retry_backoff_seconds: float, + ) -> None: + while state.parsed_page.has_show_more_button: + if self._should_stop_pagination(state=state, bounded_max_pages=bounded_max_pages): + return + next_fetch_result, next_parsed_page, next_attempt_log = await self._fetch_next_page( + scholar_id=scholar_id, + state=state, + request_delay_seconds=request_delay_seconds, + bounded_page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + ) + self._record_next_page( + state=state, + fetch_result=next_fetch_result, + parsed_page=next_parsed_page, + page_attempt_log=next_attempt_log, + ) + if self._handle_page_state_transition(state=state): + return + + @staticmethod + def _result_from_pagination_state( + *, + state: PagedLoopState, + first_page_fetch_result: FetchResult, + first_page_parsed_page: ParsedProfilePage, + first_page_fingerprint_sha256: str | None, + ) -> PagedParseResult: + return PagedParseResult( + fetch_result=state.fetch_result, + parsed_page=state.parsed_page, first_page_fetch_result=first_page_fetch_result, first_page_parsed_page=first_page_parsed_page, first_page_fingerprint_sha256=first_page_fingerprint_sha256, - publications=_dedupe_publication_candidates(publications), + publications=_dedupe_publication_candidates(state.publications), + attempt_log=state.attempt_log, + page_logs=state.page_logs, + pages_fetched=state.pages_fetched, + pages_attempted=state.pages_attempted, + has_more_remaining=state.has_more_remaining, + pagination_truncated_reason=state.pagination_truncated_reason, + continuation_cstart=state.continuation_cstart, + skipped_no_change=False, + ) + + def _short_circuit_initial_page( + self, + *, + start_cstart: int, + previous_initial_page_fingerprint_sha256: str | None, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + first_page_fingerprint_sha256: str | None, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]], + ) -> PagedParseResult | None: + if self._should_skip_no_change( + start_cstart=start_cstart, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256, + parsed_page=parsed_page, + ): + return self._skip_no_change_result( + fetch_result=fetch_result, + parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + attempt_log=attempt_log, + page_logs=page_logs, + ) + if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: + return self._initial_failure_result( + fetch_result=fetch_result, + parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + start_cstart=start_cstart, + attempt_log=attempt_log, + page_logs=page_logs, + ) + return None + + async def _fetch_and_parse_all_pages_with_retry(self, *, scholar_id: str, start_cstart: int, request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, max_pages: int, page_size: int, previous_initial_page_fingerprint_sha256: str | None = None) -> PagedParseResult: + bounded_max_pages = max(1, int(max_pages)) + bounded_page_size = max(1, int(page_size)) + fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs = ( + await self._fetch_initial_page_context( + scholar_id=scholar_id, + start_cstart=start_cstart, + bounded_page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + )) + shortcut_result = self._short_circuit_initial_page( + start_cstart=start_cstart, + previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256, + fetch_result=fetch_result, + parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, attempt_log=attempt_log, page_logs=page_logs, - pages_fetched=pages_fetched, - pages_attempted=pages_attempted, - has_more_remaining=has_more_remaining, - pagination_truncated_reason=pagination_truncated_reason, - continuation_cstart=continuation_cstart, - skipped_no_change=False, + ) + if shortcut_result is not None: + return shortcut_result + state = self._build_loop_state( + start_cstart=start_cstart, + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_log=attempt_log, + page_logs=page_logs, + ) + await self._paginate_loop( + scholar_id=scholar_id, + state=state, + bounded_max_pages=bounded_max_pages, + request_delay_seconds=request_delay_seconds, + bounded_page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + ) + return self._result_from_pagination_state( + state=state, + first_page_fetch_result=fetch_result, + first_page_parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, ) async def _upsert_profile_publications( @@ -1067,57 +1979,84 @@ class ScholarIngestionService: return discovered_count - async def _resolve_publication( + @staticmethod + def _validate_publication_candidate(candidate: PublicationCandidate) -> None: + if not candidate.title.strip(): + raise RuntimeError("Publication candidate is missing title.") + if candidate.citation_count is not None and int(candidate.citation_count) < 0: + raise RuntimeError("Publication candidate has negative citation_count.") + + async def _find_publication_by_cluster( self, db_session: AsyncSession, + *, + cluster_id: str | None, + ) -> Publication | None: + if not cluster_id: + return None + result = await db_session.execute( + select(Publication).where(Publication.cluster_id == cluster_id) + ) + return result.scalar_one_or_none() + + async def _find_publication_by_fingerprint( + self, + db_session: AsyncSession, + *, + fingerprint: str, + ) -> Publication | None: + result = await db_session.execute( + select(Publication).where(Publication.fingerprint_sha256 == fingerprint) + ) + return result.scalar_one_or_none() + + @staticmethod + def _select_existing_publication( + *, + cluster_publication: Publication | None, + fingerprint_publication: Publication | None, + ) -> Publication | None: + if cluster_publication is not None: + return cluster_publication + return fingerprint_publication + + async def _create_publication( + self, + db_session: AsyncSession, + *, candidate: PublicationCandidate, + fingerprint: str, ) -> Publication: - fingerprint = build_publication_fingerprint(candidate) - - publication: Publication | None = None - cluster_publication: Publication | None = None - - if candidate.cluster_id: - cluster_result = await db_session.execute( - select(Publication).where(Publication.cluster_id == candidate.cluster_id) - ) - cluster_publication = cluster_result.scalar_one_or_none() - publication = cluster_publication - - if publication is None: - fingerprint_result = await db_session.execute( - select(Publication).where(Publication.fingerprint_sha256 == fingerprint) - ) - publication = fingerprint_result.scalar_one_or_none() - - if publication is not None and cluster_publication is not None and publication.id != cluster_publication.id: - publication = cluster_publication - - if publication is None: - publication = Publication( - cluster_id=candidate.cluster_id, - fingerprint_sha256=fingerprint, - title_raw=candidate.title, - title_normalized=normalize_title(candidate.title), - year=candidate.year, - citation_count=int(candidate.citation_count or 0), - author_text=candidate.authors_text, - venue_text=candidate.venue_text, - pub_url=build_publication_url(candidate.title_url), - pdf_url=None, - ) - db_session.add(publication) - await db_session.flush() - logger.debug( - "ingestion.publication_created", - extra={ - "event": "ingestion.publication_created", - "publication_id": publication.id, - "cluster_id": publication.cluster_id, - }, - ) - return publication + publication = Publication( + cluster_id=candidate.cluster_id, + fingerprint_sha256=fingerprint, + title_raw=candidate.title, + title_normalized=normalize_title(candidate.title), + year=candidate.year, + citation_count=int(candidate.citation_count or 0), + author_text=candidate.authors_text, + venue_text=candidate.venue_text, + pub_url=build_publication_url(candidate.title_url), + pdf_url=build_publication_url(candidate.pdf_url), + ) + db_session.add(publication) + await db_session.flush() + logger.debug( + "ingestion.publication_created", + extra={ + "event": "ingestion.publication_created", + "publication_id": publication.id, + "cluster_id": publication.cluster_id, + }, + ) + return publication + @staticmethod + def _update_existing_publication( + *, + publication: Publication, + candidate: PublicationCandidate, + ) -> None: if candidate.cluster_id and publication.cluster_id is None: publication.cluster_id = candidate.cluster_id publication.title_raw = candidate.title @@ -1132,7 +2071,38 @@ class ScholarIngestionService: publication.venue_text = candidate.venue_text if candidate.title_url: publication.pub_url = build_publication_url(candidate.title_url) + if candidate.pdf_url: + publication.pdf_url = build_publication_url(candidate.pdf_url) + async def _resolve_publication( + self, + db_session: AsyncSession, + candidate: PublicationCandidate, + ) -> Publication: + self._validate_publication_candidate(candidate) + fingerprint = build_publication_fingerprint(candidate) + cluster_publication = await self._find_publication_by_cluster( + db_session, + cluster_id=candidate.cluster_id, + ) + fingerprint_publication = await self._find_publication_by_fingerprint( + db_session, + fingerprint=fingerprint, + ) + publication = self._select_existing_publication( + cluster_publication=cluster_publication, + fingerprint_publication=fingerprint_publication, + ) + if publication is None: + return await self._create_publication( + db_session, + candidate=candidate, + fingerprint=fingerprint, + ) + self._update_existing_publication( + publication=publication, + candidate=candidate, + ) return publication def _resolve_run_status( diff --git a/app/services/publications.py b/app/services/publications.py index f4dbaff..4f2948f 100644 --- a/app/services/publications.py +++ b/app/services/publications.py @@ -30,6 +30,7 @@ class PublicationListItem: citation_count: int venue_text: str | None pub_url: str | None + pdf_url: str | None is_read: bool first_seen_at: datetime is_new_in_latest_run: bool @@ -45,6 +46,7 @@ class UnreadPublicationItem: citation_count: int venue_text: str | None pub_url: str | None + pdf_url: str | None def resolve_publication_view_mode(value: str | None) -> str: @@ -95,6 +97,7 @@ def publications_query( Publication.citation_count, Publication.venue_text, Publication.pub_url, + Publication.pdf_url, ScholarPublication.is_read, ScholarPublication.first_seen_run_id, ScholarPublication.created_at, @@ -118,6 +121,44 @@ def publications_query( return stmt +def _publication_list_item_from_row( + row: tuple, + *, + latest_run_id: int | None, +) -> PublicationListItem: + ( + publication_id, + scholar_profile_id, + display_name, + scholar_id, + title_raw, + year, + citation_count, + venue_text, + pub_url, + pdf_url, + is_read, + first_seen_run_id, + created_at, + ) = row + return PublicationListItem( + publication_id=int(publication_id), + scholar_profile_id=int(scholar_profile_id), + scholar_label=(display_name or scholar_id), + title=title_raw, + year=year, + citation_count=int(citation_count or 0), + venue_text=venue_text, + pub_url=pub_url, + pdf_url=pdf_url, + is_read=bool(is_read), + first_seen_at=created_at, + is_new_in_latest_run=( + latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id + ), + ) + + async def list_for_user( db_session: AsyncSession, *, @@ -140,42 +181,10 @@ async def list_for_user( limit=limit, ) ) - - rows = result.all() - items: list[PublicationListItem] = [] - for row in rows: - ( - publication_id, - scholar_profile_id, - display_name, - scholar_id, - title_raw, - year, - citation_count, - venue_text, - pub_url, - is_read, - first_seen_run_id, - created_at, - ) = row - items.append( - PublicationListItem( - publication_id=int(publication_id), - scholar_profile_id=int(scholar_profile_id), - scholar_label=(display_name or scholar_id), - title=title_raw, - year=year, - citation_count=int(citation_count or 0), - venue_text=venue_text, - pub_url=pub_url, - is_read=bool(is_read), - first_seen_at=created_at, - is_new_in_latest_run=( - latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id - ), - ) - ) - return items + return [ + _publication_list_item_from_row(row, latest_run_id=latest_run_id) + for row in result.all() + ] async def list_unread_for_user( @@ -206,6 +215,7 @@ async def list_unread_for_user( citation_count, venue_text, pub_url, + pdf_url, _is_read, _first_seen_run_id, _created_at, @@ -220,6 +230,7 @@ async def list_unread_for_user( citation_count=int(citation_count or 0), venue_text=venue_text, pub_url=pub_url, + pdf_url=pdf_url, ) ) return items @@ -248,6 +259,7 @@ async def list_new_for_latest_run_for_user( citation_count=row.citation_count, venue_text=row.venue_text, pub_url=row.pub_url, + pdf_url=row.pdf_url, ) for row in rows ] diff --git a/app/services/run_safety.py b/app/services/run_safety.py index 210c688..debb8c5 100644 --- a/app/services/run_safety.py +++ b/app/services/run_safety.py @@ -146,6 +146,63 @@ def register_cooldown_blocked_start( return get_safety_state_payload(settings, now_utc=now) +def _update_run_counters( + *, + counters: dict[str, Any], + run_id: int, + blocked_failure_count: int, + network_failure_count: int, +) -> tuple[int, int]: + bounded_blocked_failures = max(0, int(blocked_failure_count)) + bounded_network_failures = max(0, int(network_failure_count)) + counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures + counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures + counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id) + counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = ( + int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1 + if bounded_blocked_failures > 0 + else 0 + ) + counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = ( + int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1 + if bounded_network_failures > 0 + else 0 + ) + return bounded_blocked_failures, bounded_network_failures + + +def _resolve_cooldown_trigger( + *, + blocked_failures: int, + network_failures: int, + blocked_failure_threshold: int, + network_failure_threshold: int, + blocked_cooldown_seconds: int, + network_cooldown_seconds: int, +) -> tuple[str | None, int]: + if blocked_failures >= max(1, int(blocked_failure_threshold)): + return COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD, max(60, int(blocked_cooldown_seconds)) + if network_failures >= max(1, int(network_failure_threshold)): + return COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD, max(60, int(network_cooldown_seconds)) + return None, 0 + + +def _apply_cooldown_decision( + *, + settings: UserSetting, + counters: dict[str, Any], + now: datetime, + reason: str | None, + cooldown_seconds: int, +) -> None: + if reason is None: + clear_expired_cooldown(settings, now_utc=now) + return + settings.scrape_cooldown_reason = reason + settings.scrape_cooldown_until = now + timedelta(seconds=max(60, int(cooldown_seconds))) + counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1 + + def apply_run_safety_outcome( settings: UserSetting, *, @@ -160,41 +217,27 @@ def apply_run_safety_outcome( ) -> tuple[dict[str, Any], str | None]: now = now_utc or _utcnow() counters = _counters_from_state(settings) - - bounded_blocked_failures = max(0, int(blocked_failure_count)) - bounded_network_failures = max(0, int(network_failure_count)) - - counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures - counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures - counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id) - counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = ( - int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1 - if bounded_blocked_failures > 0 - else 0 + blocked_failures, network_failures = _update_run_counters( + counters=counters, + run_id=run_id, + blocked_failure_count=blocked_failure_count, + network_failure_count=network_failure_count, ) - counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = ( - int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1 - if bounded_network_failures > 0 - else 0 + reason, cooldown_seconds = _resolve_cooldown_trigger( + blocked_failures=blocked_failures, + network_failures=network_failures, + blocked_failure_threshold=blocked_failure_threshold, + network_failure_threshold=network_failure_threshold, + blocked_cooldown_seconds=blocked_cooldown_seconds, + network_cooldown_seconds=network_cooldown_seconds, + ) + _apply_cooldown_decision( + settings=settings, + counters=counters, + now=now, + reason=reason, + cooldown_seconds=cooldown_seconds, ) - - reason: str | None = None - cooldown_seconds = 0 - - if bounded_blocked_failures >= max(1, int(blocked_failure_threshold)): - reason = COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD - cooldown_seconds = max(60, int(blocked_cooldown_seconds)) - elif bounded_network_failures >= max(1, int(network_failure_threshold)): - reason = COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD - cooldown_seconds = max(60, int(network_cooldown_seconds)) - - if reason is not None: - settings.scrape_cooldown_reason = reason - settings.scrape_cooldown_until = now + timedelta(seconds=cooldown_seconds) - counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1 - else: - clear_expired_cooldown(settings, now_utc=now) - settings.scrape_safety_state = counters return get_safety_state_payload(settings, now_utc=now), reason diff --git a/app/services/runs.py b/app/services/runs.py index 368815f..3d20ac4 100644 --- a/app/services/runs.py +++ b/app/services/runs.py @@ -129,6 +129,70 @@ def extract_run_summary(error_log: object) -> dict[str, Any]: } +def _queue_item_columns() -> tuple: + return ( + IngestionQueueItem.id, + IngestionQueueItem.scholar_profile_id, + ScholarProfile.display_name, + ScholarProfile.scholar_id, + IngestionQueueItem.status, + IngestionQueueItem.reason, + IngestionQueueItem.dropped_reason, + IngestionQueueItem.attempt_count, + IngestionQueueItem.resume_cstart, + IngestionQueueItem.next_attempt_dt, + IngestionQueueItem.updated_at, + IngestionQueueItem.last_error, + IngestionQueueItem.last_run_id, + ) + + +def _queue_item_select(*, user_id: int): + return ( + select(*_queue_item_columns()) + .join( + ScholarProfile, + and_( + ScholarProfile.id == IngestionQueueItem.scholar_profile_id, + ScholarProfile.user_id == IngestionQueueItem.user_id, + ), + ) + .where(IngestionQueueItem.user_id == user_id) + ) + + +def _queue_list_item_from_row(row: tuple) -> QueueListItem: + ( + item_id, + scholar_profile_id, + display_name, + scholar_id, + status, + reason, + dropped_reason, + attempt_count, + resume_cstart, + next_attempt_dt, + updated_at, + last_error, + last_run_id, + ) = row + return QueueListItem( + id=int(item_id), + scholar_profile_id=int(scholar_profile_id), + scholar_label=(display_name or scholar_id), + status=str(status), + reason=str(reason), + dropped_reason=dropped_reason, + attempt_count=int(attempt_count or 0), + resume_cstart=int(resume_cstart or 0), + next_attempt_dt=next_attempt_dt, + updated_at=updated_at, + last_error=last_error, + last_run_id=int(last_run_id) if last_run_id is not None else None, + ) + + async def list_recent_runs_for_user( db_session: AsyncSession, *, @@ -209,29 +273,7 @@ async def list_queue_items_for_user( limit: int = 200, ) -> list[QueueListItem]: result = await db_session.execute( - select( - IngestionQueueItem.id, - IngestionQueueItem.scholar_profile_id, - ScholarProfile.display_name, - ScholarProfile.scholar_id, - IngestionQueueItem.status, - IngestionQueueItem.reason, - IngestionQueueItem.dropped_reason, - IngestionQueueItem.attempt_count, - IngestionQueueItem.resume_cstart, - IngestionQueueItem.next_attempt_dt, - IngestionQueueItem.updated_at, - IngestionQueueItem.last_error, - IngestionQueueItem.last_run_id, - ) - .join( - ScholarProfile, - and_( - ScholarProfile.id == IngestionQueueItem.scholar_profile_id, - ScholarProfile.user_id == IngestionQueueItem.user_id, - ), - ) - .where(IngestionQueueItem.user_id == user_id) + _queue_item_select(user_id=user_id) .order_by( case((IngestionQueueItem.status == "dropped", 1), else_=0).asc(), IngestionQueueItem.next_attempt_dt.asc(), @@ -239,41 +281,7 @@ async def list_queue_items_for_user( ) .limit(limit) ) - - items: list[QueueListItem] = [] - for row in result.all(): - ( - item_id, - scholar_profile_id, - display_name, - scholar_id, - status, - reason, - dropped_reason, - attempt_count, - resume_cstart, - next_attempt_dt, - updated_at, - last_error, - last_run_id, - ) = row - items.append( - QueueListItem( - id=int(item_id), - scholar_profile_id=int(scholar_profile_id), - scholar_label=(display_name or scholar_id), - status=str(status), - reason=str(reason), - dropped_reason=dropped_reason, - attempt_count=int(attempt_count or 0), - resume_cstart=int(resume_cstart or 0), - next_attempt_dt=next_attempt_dt, - updated_at=updated_at, - last_error=last_error, - last_run_id=int(last_run_id) if last_run_id is not None else None, - ) - ) - return items + return [_queue_list_item_from_row(row) for row in result.all()] async def get_queue_item_for_user( @@ -283,66 +291,14 @@ async def get_queue_item_for_user( queue_item_id: int, ) -> QueueListItem | None: result = await db_session.execute( - select( - IngestionQueueItem.id, - IngestionQueueItem.scholar_profile_id, - ScholarProfile.display_name, - ScholarProfile.scholar_id, - IngestionQueueItem.status, - IngestionQueueItem.reason, - IngestionQueueItem.dropped_reason, - IngestionQueueItem.attempt_count, - IngestionQueueItem.resume_cstart, - IngestionQueueItem.next_attempt_dt, - IngestionQueueItem.updated_at, - IngestionQueueItem.last_error, - IngestionQueueItem.last_run_id, - ) - .join( - ScholarProfile, - and_( - ScholarProfile.id == IngestionQueueItem.scholar_profile_id, - ScholarProfile.user_id == IngestionQueueItem.user_id, - ), - ) - .where( - IngestionQueueItem.user_id == user_id, - IngestionQueueItem.id == queue_item_id, - ) + _queue_item_select(user_id=user_id) + .where(IngestionQueueItem.id == queue_item_id) .limit(1) ) row = result.one_or_none() if row is None: return None - ( - item_id, - scholar_profile_id, - display_name, - scholar_id, - status, - reason, - dropped_reason, - attempt_count, - resume_cstart, - next_attempt_dt, - updated_at, - last_error, - last_run_id, - ) = row - return QueueListItem( - id=int(item_id), - scholar_profile_id=int(scholar_profile_id), - scholar_label=(display_name or scholar_id), - status=str(status), - reason=str(reason), - dropped_reason=dropped_reason, - attempt_count=int(attempt_count or 0), - resume_cstart=int(resume_cstart or 0), - next_attempt_dt=next_attempt_dt, - updated_at=updated_at, - last_error=last_error, - last_run_id=int(last_run_id) if last_run_id is not None else None, - ) + return _queue_list_item_from_row(row) async def retry_queue_item_for_user( diff --git a/app/services/scheduler.py b/app/services/scheduler.py index 96dd5f1..2057663 100644 --- a/app/services/scheduler.py +++ b/app/services/scheduler.py @@ -139,10 +139,7 @@ class SchedulerService: continue await self._run_candidate(candidate) - async def _load_candidates(self) -> list[_AutoRunCandidate]: - if not settings.ingestion_automation_allowed: - return [] - + async def _load_candidate_rows(self) -> list[tuple]: session_factory = get_session_factory() async with session_factory() as session: result = await session.execute( @@ -154,41 +151,48 @@ class SchedulerService: UserSetting.scrape_cooldown_reason, ) .join(User, User.id == UserSetting.user_id) - .where( - User.is_active.is_(True), - UserSetting.auto_run_enabled.is_(True), - ) + .where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True)) .order_by(UserSetting.user_id.asc()) ) - rows = result.all() + return result.all() + + @staticmethod + def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None: + user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row + if cooldown_until is not None and cooldown_until.tzinfo is None: + cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) + if cooldown_until is not None and cooldown_until > now_utc: + logger.info( + "scheduler.run_skipped_safety_cooldown_precheck", + extra={ + "event": "scheduler.run_skipped_safety_cooldown_precheck", + "user_id": int(user_id), + "reason": cooldown_reason, + "cooldown_until": cooldown_until, + "cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()), + "metric_name": "scheduler_run_skipped_safety_cooldown_total", + "metric_value": 1, + }, + ) + return None + return _AutoRunCandidate( + user_id=int(user_id), + run_interval_minutes=int(run_interval_minutes), + request_delay_seconds=int(request_delay_seconds), + cooldown_until=cooldown_until, + cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None), + ) + + async def _load_candidates(self) -> list[_AutoRunCandidate]: + if not settings.ingestion_automation_allowed: + return [] + rows = await self._load_candidate_rows() now_utc = datetime.now(timezone.utc) candidates: list[_AutoRunCandidate] = [] - for user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason in rows: - if cooldown_until is not None and cooldown_until.tzinfo is None: - cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) - if cooldown_until is not None and cooldown_until > now_utc: - logger.info( - "scheduler.run_skipped_safety_cooldown_precheck", - extra={ - "event": "scheduler.run_skipped_safety_cooldown_precheck", - "user_id": int(user_id), - "reason": cooldown_reason, - "cooldown_until": cooldown_until, - "cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()), - "metric_name": "scheduler_run_skipped_safety_cooldown_total", - "metric_value": 1, - }, - ) - continue - candidates.append( - _AutoRunCandidate( - user_id=int(user_id), - run_interval_minutes=int(run_interval_minutes), - request_delay_seconds=int(request_delay_seconds), - cooldown_until=cooldown_until, - cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None), - ) - ) + for row in rows: + candidate = self._candidate_from_row(row, now_utc=now_utc) + if candidate is not None: + candidates.append(candidate) return candidates async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool: @@ -212,12 +216,16 @@ class SchedulerService: ) return now >= next_due_dt - async def _run_candidate(self, candidate: _AutoRunCandidate) -> None: + async def _run_candidate_ingestion( + self, + *, + candidate: _AutoRunCandidate, + ): session_factory = get_session_factory() async with session_factory() as session: ingestion = ScholarIngestionService(source=self._source) try: - run_summary = await ingestion.run_for_user( + return await ingestion.run_for_user( session, user_id=candidate.user_id, trigger_type=RunTriggerType.SCHEDULED, @@ -234,14 +242,8 @@ class SchedulerService: ) except RunAlreadyInProgressError: await session.rollback() - logger.info( - "scheduler.run_skipped_locked", - extra={ - "event": "scheduler.run_skipped_locked", - "user_id": candidate.user_id, - }, - ) - return + logger.info("scheduler.run_skipped_locked", extra={"event": "scheduler.run_skipped_locked", "user_id": candidate.user_id}) + return None except RunBlockedBySafetyPolicyError as exc: await session.rollback() logger.info( @@ -256,18 +258,16 @@ class SchedulerService: "metric_value": 1, }, ) - return + return None except Exception: await session.rollback() - logger.exception( - "scheduler.run_failed", - extra={ - "event": "scheduler.run_failed", - "user_id": candidate.user_id, - }, - ) - return + logger.exception("scheduler.run_failed", extra={"event": "scheduler.run_failed", "user_id": candidate.user_id}) + return None + async def _run_candidate(self, candidate: _AutoRunCandidate) -> None: + run_summary = await self._run_candidate_ingestion(candidate=candidate) + if run_summary is None: + return logger.info( "scheduler.run_completed", extra={ @@ -292,41 +292,53 @@ class SchedulerService: for job in jobs: await self._run_queue_job(job) - async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None: - if job.attempt_count >= self._continuation_max_attempts: - session_factory = get_session_factory() - async with session_factory() as session: - dropped = await queue_service.mark_dropped( - session, - job_id=job.id, - reason="max_attempts_reached", - ) - await session.commit() - if dropped is not None: - logger.warning( - "scheduler.queue_item_dropped_max_attempts", - extra={ - "event": "scheduler.queue_item_dropped_max_attempts", - "queue_item_id": job.id, - "user_id": job.user_id, - "scholar_profile_id": job.scholar_profile_id, - "attempt_count": job.attempt_count, - "max_attempts": self._continuation_max_attempts, - }, - ) - return + async def _drop_queue_job_if_max_attempts( + self, + job: queue_service.ContinuationQueueJob, + ) -> bool: + if job.attempt_count < self._continuation_max_attempts: + return False + session_factory = get_session_factory() + async with session_factory() as session: + dropped = await queue_service.mark_dropped( + session, + job_id=job.id, + reason="max_attempts_reached", + ) + await session.commit() + if dropped is not None: + logger.warning( + "scheduler.queue_item_dropped_max_attempts", + extra={ + "event": "scheduler.queue_item_dropped_max_attempts", + "queue_item_id": job.id, + "user_id": job.user_id, + "scholar_profile_id": job.scholar_profile_id, + "attempt_count": job.attempt_count, + "max_attempts": self._continuation_max_attempts, + }, + ) + return True + async def _mark_queue_job_retrying( + self, + job: queue_service.ContinuationQueueJob, + ) -> bool: session_factory = get_session_factory() async with session_factory() as session: queue_item = await queue_service.mark_retrying(session, job_id=job.id) - if queue_item is None: - await session.commit() - return - if queue_item.status == QueueItemStatus.DROPPED.value: - await session.commit() - return await session.commit() + if queue_item is None: + return False + if queue_item.status == QueueItemStatus.DROPPED.value: + return False + return True + async def _queue_job_has_available_scholar( + self, + job: queue_service.ContinuationQueueJob, + ) -> bool: + session_factory = get_session_factory() async with session_factory() as session: scholar_result = await session.execute( select(ScholarProfile.id).where( @@ -336,33 +348,124 @@ class SchedulerService: ) ) scholar_id = scholar_result.scalar_one_or_none() - if scholar_id is None: - dropped = await queue_service.mark_dropped( - session, - job_id=job.id, - reason="scholar_unavailable", - ) - await session.commit() - if dropped is not None: - logger.info( - "scheduler.queue_item_dropped_scholar_unavailable", - extra={ - "event": "scheduler.queue_item_dropped_scholar_unavailable", - "queue_item_id": job.id, - "user_id": job.user_id, - "scholar_profile_id": job.scholar_profile_id, - }, - ) - return - - async with session_factory() as session: - request_delay_seconds = await self._load_request_delay_for_user( + if scholar_id is not None: + return True + dropped = await queue_service.mark_dropped( session, - user_id=job.user_id, + job_id=job.id, + reason="scholar_unavailable", ) + await session.commit() + if dropped is not None: + logger.info( + "scheduler.queue_item_dropped_scholar_unavailable", + extra={ + "event": "scheduler.queue_item_dropped_scholar_unavailable", + "queue_item_id": job.id, + "user_id": job.user_id, + "scholar_profile_id": job.scholar_profile_id, + }, + ) + return False + + async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None: + session_factory = get_session_factory() + async with session_factory() as recovery_session: + await queue_service.reschedule_job( + recovery_session, + job_id=job.id, + delay_seconds=max(self._tick_seconds, 15), + reason="user_run_lock_active", + error="run_already_in_progress", + ) + await recovery_session.commit() + logger.info( + "scheduler.queue_item_deferred_lock", + extra={"event": "scheduler.queue_item_deferred_lock", "queue_item_id": job.id, "user_id": job.user_id}, + ) + + async def _reschedule_queue_job_safety_cooldown( + self, + job: queue_service.ContinuationQueueJob, + exc: RunBlockedBySafetyPolicyError, + ) -> None: + cooldown_remaining_seconds = max( + self._tick_seconds, + int(exc.safety_state.get("cooldown_remaining_seconds") or 0), + ) + session_factory = get_session_factory() + async with session_factory() as recovery_session: + await queue_service.reschedule_job( + recovery_session, + job_id=job.id, + delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds), + reason="scrape_safety_cooldown", + error=str(exc.message), + ) + await recovery_session.commit() + logger.info( + "scheduler.queue_item_deferred_safety_cooldown", + extra={ + "event": "scheduler.queue_item_deferred_safety_cooldown", + "queue_item_id": job.id, + "user_id": job.user_id, + "reason": exc.safety_state.get("cooldown_reason"), + "cooldown_remaining_seconds": cooldown_remaining_seconds, + "metric_name": "scheduler_queue_item_deferred_safety_cooldown_total", + "metric_value": 1, + }, + ) + + async def _reschedule_queue_job_after_exception( + self, + job: queue_service.ContinuationQueueJob, + *, + exc: Exception, + ) -> None: + session_factory = get_session_factory() + async with session_factory() as recovery_session: + queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id) + if queue_item is None: + await recovery_session.commit() + return + if int(queue_item.attempt_count) >= self._continuation_max_attempts: + await queue_service.mark_dropped( + recovery_session, + job_id=job.id, + reason="scheduler_exception_max_attempts", + error=str(exc), + ) + await recovery_session.commit() + logger.warning( + "scheduler.queue_item_dropped_after_exception", + extra={"event": "scheduler.queue_item_dropped_after_exception", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count}, + ) + return + delay_seconds = queue_service.compute_backoff_seconds( + base_seconds=self._continuation_base_delay_seconds, + attempt_count=int(queue_item.attempt_count), + max_seconds=self._continuation_max_delay_seconds, + ) + await queue_service.reschedule_job( + recovery_session, + job_id=job.id, + delay_seconds=delay_seconds, + reason="scheduler_exception", + error=str(exc), + ) + await recovery_session.commit() + logger.exception("scheduler.queue_item_run_failed", extra={"event": "scheduler.queue_item_run_failed", "queue_item_id": job.id, "user_id": job.user_id}) + + async def _run_ingestion_for_queue_job( + self, + job: queue_service.ContinuationQueueJob, + ): + session_factory = get_session_factory() + async with session_factory() as session: + request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id) ingestion = ScholarIngestionService(source=self._source) try: - run_summary = await ingestion.run_for_user( + return await ingestion.run_for_user( session, user_id=job.user_id, trigger_type=RunTriggerType.SCHEDULED, @@ -372,9 +475,7 @@ class SchedulerService: max_pages_per_scholar=self._max_pages_per_scholar, page_size=self._page_size, scholar_profile_ids={job.scholar_profile_id}, - start_cstart_by_scholar_id={ - job.scholar_profile_id: job.resume_cstart, - }, + start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart}, auto_queue_continuations=self._continuation_queue_enabled, queue_delay_seconds=self._continuation_base_delay_seconds, alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, @@ -383,200 +484,88 @@ class SchedulerService: ) except RunAlreadyInProgressError: await session.rollback() - async with session_factory() as recovery_session: - await queue_service.reschedule_job( - recovery_session, - job_id=job.id, - delay_seconds=max(self._tick_seconds, 15), - reason="user_run_lock_active", - error="run_already_in_progress", - ) - await recovery_session.commit() - logger.info( - "scheduler.queue_item_deferred_lock", - extra={ - "event": "scheduler.queue_item_deferred_lock", - "queue_item_id": job.id, - "user_id": job.user_id, - }, - ) - return + await self._reschedule_queue_job_lock_active(job) except RunBlockedBySafetyPolicyError as exc: await session.rollback() - cooldown_remaining_seconds = max( - self._tick_seconds, - int(exc.safety_state.get("cooldown_remaining_seconds") or 0), - ) - async with session_factory() as recovery_session: - await queue_service.reschedule_job( - recovery_session, - job_id=job.id, - delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds), - reason="scrape_safety_cooldown", - error=str(exc.message), - ) - await recovery_session.commit() - logger.info( - "scheduler.queue_item_deferred_safety_cooldown", - extra={ - "event": "scheduler.queue_item_deferred_safety_cooldown", - "queue_item_id": job.id, - "user_id": job.user_id, - "reason": exc.safety_state.get("cooldown_reason"), - "cooldown_remaining_seconds": cooldown_remaining_seconds, - "metric_name": "scheduler_queue_item_deferred_safety_cooldown_total", - "metric_value": 1, - }, - ) - return + await self._reschedule_queue_job_safety_cooldown(job, exc) except Exception as exc: await session.rollback() - async with session_factory() as recovery_session: - queue_item = await queue_service.increment_attempt_count( - recovery_session, - job_id=job.id, - ) - if queue_item is None: - await recovery_session.commit() - return - if int(queue_item.attempt_count) >= self._continuation_max_attempts: - await queue_service.mark_dropped( - recovery_session, - job_id=job.id, - reason="scheduler_exception_max_attempts", - error=str(exc), - ) - await recovery_session.commit() - logger.warning( - "scheduler.queue_item_dropped_after_exception", - extra={ - "event": "scheduler.queue_item_dropped_after_exception", - "queue_item_id": job.id, - "user_id": job.user_id, - "attempt_count": queue_item.attempt_count, - }, - ) - return - delay_seconds = queue_service.compute_backoff_seconds( - base_seconds=self._continuation_base_delay_seconds, - attempt_count=int(queue_item.attempt_count), - max_seconds=self._continuation_max_delay_seconds, - ) - await queue_service.reschedule_job( - recovery_session, - job_id=job.id, - delay_seconds=delay_seconds, - reason="scheduler_exception", - error=str(exc), - ) - await recovery_session.commit() - logger.exception( - "scheduler.queue_item_run_failed", - extra={ - "event": "scheduler.queue_item_run_failed", - "queue_item_id": job.id, - "user_id": job.user_id, - }, - ) - return + await self._reschedule_queue_job_after_exception(job, exc=exc) + return None + @staticmethod + def _log_queue_item_resolved( + *, + event_name: str, + job: queue_service.ContinuationQueueJob, + run_summary, + attempt_count: int | None = None, + delay_seconds: int | None = None, + ) -> None: + payload = { + "event": event_name, + "queue_item_id": job.id, + "user_id": job.user_id, + "run_id": run_summary.crawl_run_id, + "status": run_summary.status.value, + } + if attempt_count is not None: + payload["attempt_count"] = attempt_count + if delay_seconds is not None: + payload["delay_seconds"] = delay_seconds + logger.info(event_name, extra=payload) + + async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None: + session_factory = get_session_factory() async with session_factory() as session: - # Failed-attempt budget should advance only when continuation execution fails. if int(run_summary.failed_count) <= 0: - queue_item = await queue_service.reset_attempt_count( - session, - job_id=job.id, - ) - if queue_item is None: - await session.commit() - logger.info( - "scheduler.queue_item_resolved", - extra={ - "event": "scheduler.queue_item_resolved", - "queue_item_id": job.id, - "user_id": job.user_id, - "run_id": run_summary.crawl_run_id, - "status": run_summary.status.value, - }, - ) - return + queue_item = await queue_service.reset_attempt_count(session, job_id=job.id) await session.commit() - logger.info( - "scheduler.queue_item_progressed", - extra={ - "event": "scheduler.queue_item_progressed", - "queue_item_id": job.id, - "user_id": job.user_id, - "attempt_count": int(queue_item.attempt_count), - "run_id": run_summary.crawl_run_id, - "status": run_summary.status.value, - }, + if queue_item is None: + self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary) + return + self._log_queue_item_resolved( + event_name="scheduler.queue_item_progressed", + job=job, + run_summary=run_summary, + attempt_count=int(queue_item.attempt_count), ) return - - queue_item = await queue_service.increment_attempt_count( - session, - job_id=job.id, - ) + queue_item = await queue_service.increment_attempt_count(session, job_id=job.id) if queue_item is None: await session.commit() - logger.info( - "scheduler.queue_item_resolved", - extra={ - "event": "scheduler.queue_item_resolved", - "queue_item_id": job.id, - "user_id": job.user_id, - "run_id": run_summary.crawl_run_id, - "status": run_summary.status.value, - }, - ) + self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary) return - if int(queue_item.attempt_count) >= self._continuation_max_attempts: - await queue_service.mark_dropped( - session, - job_id=job.id, - reason="max_attempts_after_run", - ) + await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run") await session.commit() logger.warning( "scheduler.queue_item_dropped_max_attempts_after_run", - extra={ - "event": "scheduler.queue_item_dropped_max_attempts_after_run", - "queue_item_id": job.id, - "user_id": job.user_id, - "attempt_count": queue_item.attempt_count, - "run_id": run_summary.crawl_run_id, - "status": run_summary.status.value, - }, + extra={"event": "scheduler.queue_item_dropped_max_attempts_after_run", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count, "run_id": run_summary.crawl_run_id, "status": run_summary.status.value}, ) return - - delay_seconds = queue_service.compute_backoff_seconds( - base_seconds=self._continuation_base_delay_seconds, - attempt_count=int(queue_item.attempt_count), - max_seconds=self._continuation_max_delay_seconds, - ) - await queue_service.reschedule_job( - session, - job_id=job.id, - delay_seconds=delay_seconds, - reason=queue_item.reason, - error=queue_item.last_error, - ) + delay_seconds = queue_service.compute_backoff_seconds(base_seconds=self._continuation_base_delay_seconds, attempt_count=int(queue_item.attempt_count), max_seconds=self._continuation_max_delay_seconds) + await queue_service.reschedule_job(session, job_id=job.id, delay_seconds=delay_seconds, reason=queue_item.reason, error=queue_item.last_error) await session.commit() - logger.info( - "scheduler.queue_item_rescheduled_failed", - extra={ - "event": "scheduler.queue_item_rescheduled_failed", - "queue_item_id": job.id, - "user_id": job.user_id, - "attempt_count": queue_item.attempt_count, - "delay_seconds": delay_seconds, - "run_id": run_summary.crawl_run_id, - "status": run_summary.status.value, - }, - ) + self._log_queue_item_resolved( + event_name="scheduler.queue_item_rescheduled_failed", + job=job, + run_summary=run_summary, + attempt_count=int(queue_item.attempt_count), + delay_seconds=delay_seconds, + ) + + async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None: + if await self._drop_queue_job_if_max_attempts(job): + return + if not await self._mark_queue_job_retrying(job): + return + if not await self._queue_job_has_available_scholar(job): + return + run_summary = await self._run_ingestion_for_queue_job(job) + if run_summary is None: + return + await self._finalize_queue_job_after_run(job, run_summary) async def _load_request_delay_for_user( self, diff --git a/app/services/scholar_parser.py b/app/services/scholar_parser.py index de2daa5..8f357d0 100644 --- a/app/services/scholar_parser.py +++ b/app/services/scholar_parser.py @@ -79,6 +79,18 @@ SHOW_MORE_BUTTON_RE = re.compile( r"]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>", re.I | re.S, ) +PROFILE_ROW_PARSER_DIRECT_MARKERS = ( + "gs_ggs", + "gs_ggsd", + "gs_ggsa", + "gs_or_ggsm", +) +PROFILE_ROW_DIRECT_LABEL_TOKENS = ( + "pdf", + "[pdf]", + "full text", + "download", +) class ParseState(StrEnum): @@ -98,6 +110,7 @@ class PublicationCandidate: citation_count: int | None authors_text: str | None venue_text: str | None + pdf_url: str | None @dataclass(frozen=True) @@ -174,6 +187,7 @@ class ScholarRowParser(HTMLParser): def __init__(self) -> None: super().__init__(convert_charrefs=True) self.title_href: str | None = None + self.direct_download_href: str | None = None self.title_parts: list[str] = [] self.citation_parts: list[str] = [] self.year_parts: list[str] = [] @@ -183,6 +197,13 @@ class ScholarRowParser(HTMLParser): self._citation_depth = 0 self._year_depth = 0 self._gray_stack: list[dict[str, Any]] = [] + self._direct_marker_depth = 0 + self._aux_link_stack: list[dict[str, Any]] = [] + + @staticmethod + def _contains_direct_marker(classes: str) -> bool: + lowered = classes.lower() + return any(marker in lowered for marker in PROFILE_ROW_PARSER_DIRECT_MARKERS) def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: if self._title_depth > 0: @@ -193,8 +214,14 @@ class ScholarRowParser(HTMLParser): self._year_depth += 1 if self._gray_stack: self._gray_stack[-1]["depth"] += 1 + if self._direct_marker_depth > 0: + self._direct_marker_depth += 1 + if self._aux_link_stack: + self._aux_link_stack[-1]["depth"] += 1 classes = attr_class(attrs) + if tag in {"div", "span"} and self._contains_direct_marker(classes): + self._direct_marker_depth = 1 if tag == "a" and "gsc_a_at" in classes: self._title_depth = 1 @@ -213,6 +240,16 @@ class ScholarRowParser(HTMLParser): self._gray_stack.append({"depth": 1, "parts": []}) return + if tag == "a": + self._aux_link_stack.append( + { + "depth": 1, + "href": attr_href(attrs), + "classes": classes, + "parts": [], + } + ) + def handle_data(self, data: str) -> None: if self._title_depth > 0: self.title_parts.append(data) @@ -222,6 +259,21 @@ class ScholarRowParser(HTMLParser): self.year_parts.append(data) if self._gray_stack: self._gray_stack[-1]["parts"].append(data) + if self._aux_link_stack: + self._aux_link_stack[-1]["parts"].append(data) + + def _capture_direct_download_href(self, link: dict[str, Any]) -> None: + if self.direct_download_href: + return + href = link.get("href") + if not isinstance(href, str) or not href.strip(): + return + label = normalize_space("".join(link.get("parts", []))).lower() + classes = str(link.get("classes", "")).lower() + label_match = any(token in label for token in PROFILE_ROW_DIRECT_LABEL_TOKENS) + marker_match = self._contains_direct_marker(classes) or self._direct_marker_depth > 0 + if label_match or marker_match: + self.direct_download_href = href.strip() def handle_endtag(self, _tag: str) -> None: if self._title_depth > 0: @@ -237,6 +289,13 @@ class ScholarRowParser(HTMLParser): if text: self.gray_texts.append(text) self._gray_stack.pop() + if self._aux_link_stack: + self._aux_link_stack[-1]["depth"] -= 1 + if self._aux_link_stack[-1]["depth"] == 0: + self._capture_direct_download_href(self._aux_link_stack[-1]) + self._aux_link_stack.pop() + if self._direct_marker_depth > 0: + self._direct_marker_depth -= 1 def extract_rows(html: str) -> list[str]: @@ -302,37 +361,60 @@ def parse_citation_count(parts: list[str]) -> int | None: return int(match.group(0)) +def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None, list[str]]: + parser = ScholarRowParser() + parser.feed(row_html) + warnings: list[str] = [] + title = normalize_space("".join(parser.title_parts)) + if not title: + warnings.append("row_missing_title") + return None, warnings + if not parser.title_href: + warnings.append("row_missing_title_href") + + citation_text = normalize_space(" ".join(parser.citation_parts)) + citation_count = parse_citation_count(parser.citation_parts) + if citation_text and citation_count is None: + warnings.append("layout_row_citation_unparseable") + + year_text = normalize_space(" ".join(parser.year_parts)) + year = parse_year(parser.year_parts) + if year_text and year is None: + warnings.append("layout_row_year_unparseable") + + authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None + venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None + return ( + PublicationCandidate( + title=title, + title_url=parser.title_href, + cluster_id=parse_cluster_id_from_href(parser.title_href), + year=year, + citation_count=citation_count, + authors_text=authors_text, + venue_text=venue_text, + pdf_url=build_absolute_scholar_url(parser.direct_download_href), + ), + warnings, + ) + + def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]: rows = extract_rows(html) warnings: list[str] = [] publications: list[PublicationCandidate] = [] for row_html in rows: - parser = ScholarRowParser() - parser.feed(row_html) - - title = normalize_space("".join(parser.title_parts)) - if not title: - warnings.append("row_missing_title") + publication, row_warnings = _parse_publication_row(row_html) + warnings.extend(row_warnings) + if publication is None: continue - - authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None - venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None - - publications.append( - PublicationCandidate( - title=title, - title_url=parser.title_href, - cluster_id=parse_cluster_id_from_href(parser.title_href), - year=parse_year(parser.year_parts), - citation_count=parse_citation_count(parser.citation_parts), - authors_text=authors_text, - venue_text=venue_text, - ) - ) + publications.append(publication) if not rows: warnings.append("no_rows_detected") + if rows and not publications: + warnings.append("layout_all_rows_unparseable") return publications, sorted(set(warnings)) @@ -635,11 +717,35 @@ def classify_block_or_captcha_reason( return None +def _warnings_contain(warnings: list[str], code: str) -> bool: + return any(item == code for item in warnings) + + +def _has_layout_row_failure(marker_counts: dict[str, int], warnings: list[str]) -> bool: + if _warnings_contain(warnings, "layout_all_rows_unparseable"): + return True + if marker_counts.get("gsc_a_tr", 0) <= 0: + return False + if _warnings_contain(warnings, "row_missing_title"): + return True + return marker_counts.get("gsc_a_at", 0) <= 0 + + +def _first_layout_warning(warnings: list[str]) -> str | None: + for warning in warnings: + if warning.startswith("layout_"): + return warning + return None + + def detect_state( fetch_result: FetchResult, publications: list[PublicationCandidate], marker_counts: dict[str, int], *, + warnings: list[str], + has_show_more_button_flag: bool, + articles_range: str | None, visible_text: str, ) -> tuple[ParseState, str]: if fetch_result.status_code is None: @@ -660,6 +766,16 @@ def detect_state( if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS): return ParseState.NO_RESULTS, "no_results_keyword_detected" + layout_warning = _first_layout_warning(warnings) + if layout_warning is not None: + return ParseState.LAYOUT_CHANGED, layout_warning + + if _has_layout_row_failure(marker_counts, warnings): + return ParseState.LAYOUT_CHANGED, "layout_publication_rows_unparseable" + + if has_show_more_button_flag and not articles_range: + return ParseState.LAYOUT_CHANGED, "layout_show_more_without_articles_range" + if not publications: has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0 has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0 @@ -699,7 +815,7 @@ def detect_author_search_state( has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0 if not has_search_markers: return ParseState.NO_RESULTS, "no_search_candidates_detected" - return ParseState.OK, "search_markers_present_with_empty_results" + return ParseState.LAYOUT_CHANGED, "layout_author_candidates_unparseable" return ParseState.OK, "author_candidates_extracted" @@ -711,6 +827,7 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage: show_more = has_show_more_button(fetch_result.body) operation_error_banner = has_operation_error_banner(fetch_result.body) + articles_range = extract_articles_range(fetch_result.body) if show_more: warnings.append("possible_partial_page_show_more_present") @@ -723,6 +840,9 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage: fetch_result, publications, marker_counts, + warnings=warnings, + has_show_more_button_flag=show_more, + articles_range=articles_range, visible_text=visible_text, ) @@ -736,7 +856,7 @@ def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage: warnings=warnings, has_show_more_button=show_more, has_operation_error_banner=operation_error_banner, - articles_range=extract_articles_range(fetch_result.body), + articles_range=articles_range, ) diff --git a/app/services/scholar_source.py b/app/services/scholar_source.py index a1d4b47..83da165 100644 --- a/app/services/scholar_source.py +++ b/app/services/scholar_source.py @@ -123,8 +123,8 @@ class LiveScholarSource: ) return await asyncio.to_thread(self._fetch_sync, requested_url) - def _fetch_sync(self, requested_url: str) -> FetchResult: - request = Request( + def _build_request(self, requested_url: str) -> Request: + return Request( requested_url, headers={ "User-Agent": random.choice(self._user_agents), @@ -134,61 +134,75 @@ class LiveScholarSource: }, ) + @staticmethod + def _http_error_body(exc: HTTPError) -> str: + try: + return exc.read().decode("utf-8", errors="replace") + except Exception: + return "" + + @staticmethod + def _network_error_result(requested_url: str, exc: URLError) -> FetchResult: + logger.warning( + "scholar_source.fetch_network_error", + extra={"event": "scholar_source.fetch_network_error", "requested_url": requested_url}, + ) + return FetchResult( + requested_url=requested_url, + status_code=None, + final_url=None, + body="", + error=str(exc), + ) + + @staticmethod + def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult: + logger.warning( + "scholar_source.fetch_http_error", + extra={ + "event": "scholar_source.fetch_http_error", + "requested_url": requested_url, + "status_code": exc.code, + }, + ) + return FetchResult( + requested_url=requested_url, + status_code=exc.code, + final_url=exc.geturl(), + body=LiveScholarSource._http_error_body(exc), + error=str(exc), + ) + + @staticmethod + def _success_result(requested_url: str, response) -> FetchResult: + body = response.read().decode("utf-8", errors="replace") + status_code = getattr(response, "status", 200) + logger.debug( + "scholar_source.fetch_succeeded", + extra={ + "event": "scholar_source.fetch_succeeded", + "requested_url": requested_url, + "status_code": status_code, + }, + ) + return FetchResult( + requested_url=requested_url, + status_code=status_code, + final_url=response.geturl(), + body=body, + error=None, + ) + + def _fetch_sync(self, requested_url: str) -> FetchResult: + request = self._build_request(requested_url) + try: with urlopen(request, timeout=self._timeout_seconds) as response: - body = response.read().decode("utf-8", errors="replace") - status_code = getattr(response, "status", 200) - logger.debug( - "scholar_source.fetch_succeeded", - extra={ - "event": "scholar_source.fetch_succeeded", - "requested_url": requested_url, - "status_code": status_code, - }, - ) - return FetchResult( - requested_url=requested_url, - status_code=status_code, - final_url=response.geturl(), - body=body, - error=None, - ) + return self._success_result(requested_url, response) except HTTPError as exc: - body = "" - try: - body = exc.read().decode("utf-8", errors="replace") - except Exception: - body = "" - logger.warning( - "scholar_source.fetch_http_error", - extra={ - "event": "scholar_source.fetch_http_error", - "requested_url": requested_url, - "status_code": exc.code, - }, - ) - return FetchResult( - requested_url=requested_url, - status_code=exc.code, - final_url=exc.geturl(), - body=body, - error=str(exc), - ) + return self._http_error_result(requested_url, exc) except URLError as exc: - logger.warning( - "scholar_source.fetch_network_error", - extra={ - "event": "scholar_source.fetch_network_error", - "requested_url": requested_url, - }, - ) - return FetchResult( - requested_url=requested_url, - status_code=None, - final_url=None, - body="", - error=str(exc), - ) + return self._network_error_result(requested_url, exc) def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str: diff --git a/app/services/scholars.py b/app/services/scholars.py index 1abee78..dd99e0c 100644 --- a/app/services/scholars.py +++ b/app/services/scholars.py @@ -269,73 +269,103 @@ def _serialize_parsed_author_search_page(parsed: ParsedAuthorSearchPage) -> dict } +def _payload_state(payload: dict[str, object]) -> ParseState | None: + state_raw = str(payload.get("state", "")).strip() + try: + return ParseState(state_raw) + except ValueError: + return None + + +def _payload_marker_counts(payload: dict[str, object]) -> dict[str, int]: + marker_counts_payload = payload.get("marker_counts") + if not isinstance(marker_counts_payload, dict): + return {} + parsed: dict[str, int] = {} + for key, value in marker_counts_payload.items(): + try: + parsed[str(key)] = int(value) + except (TypeError, ValueError): + continue + return parsed + + +def _payload_warnings(payload: dict[str, object]) -> list[str]: + warnings_payload = payload.get("warnings") + if not isinstance(warnings_payload, list): + return [] + return [str(value) for value in warnings_payload if isinstance(value, str)] + + +def _parse_optional_string(value: object) -> str | None: + if value is None: + return None + normalized = str(value).strip() + return normalized or None + + +def _parse_optional_int(value: object) -> int | None: + if isinstance(value, int): + return value + if isinstance(value, str) and value.strip(): + try: + return int(value) + except ValueError: + return None + return None + + +def _normalize_interests(value: object) -> list[str]: + if not isinstance(value, list): + return [] + return [str(item) for item in value if isinstance(item, str)] + + +def _deserialize_candidate_payload(value: object) -> dict[str, object] | None: + if not isinstance(value, dict): + return None + scholar_id = str(value.get("scholar_id", "")).strip() + display_name = str(value.get("display_name", "")).strip() + profile_url = str(value.get("profile_url", "")).strip() + if not scholar_id or not display_name or not profile_url: + return None + return { + "scholar_id": scholar_id, + "display_name": display_name, + "affiliation": _parse_optional_string(value.get("affiliation")), + "email_domain": _parse_optional_string(value.get("email_domain")), + "cited_by_count": _parse_optional_int(value.get("cited_by_count")), + "interests": _normalize_interests(value.get("interests")), + "profile_url": profile_url, + "profile_image_url": _parse_optional_string(value.get("profile_image_url")), + } + + +def _deserialize_candidates(payload: dict[str, object]) -> list[dict[str, object]]: + candidates_payload = payload.get("candidates") + if not isinstance(candidates_payload, list): + return [] + normalized: list[dict[str, object]] = [] + for value in candidates_payload: + candidate = _deserialize_candidate_payload(value) + if candidate is not None: + normalized.append(candidate) + return normalized + + def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearchPage | None: if not isinstance(payload, dict): return None - state_raw = str(payload.get("state", "")).strip() - try: - state = ParseState(state_raw) - except ValueError: + state = _payload_state(payload) + if state is None: return None - marker_counts_payload = payload.get("marker_counts") - marker_counts = ( - {str(key): int(value) for key, value in marker_counts_payload.items()} - if isinstance(marker_counts_payload, dict) - else {} - ) - warnings_payload = payload.get("warnings") - warnings = ( - [str(value) for value in warnings_payload if isinstance(value, str)] - if isinstance(warnings_payload, list) - else [] - ) + marker_counts = _payload_marker_counts(payload) + warnings = _payload_warnings(payload) from app.services.scholar_parser import ScholarSearchCandidate - candidates_payload = payload.get("candidates") - normalized_candidates = [] - for value in candidates_payload if isinstance(candidates_payload, list) else []: - if not isinstance(value, dict): - continue - scholar_id = str(value.get("scholar_id", "")).strip() - display_name = str(value.get("display_name", "")).strip() - profile_url = str(value.get("profile_url", "")).strip() - if not scholar_id or not display_name or not profile_url: - continue - interests_payload = value.get("interests") - interests = ( - [str(item) for item in interests_payload if isinstance(item, str)] - if isinstance(interests_payload, list) - else [] - ) - cited_by_count = value.get("cited_by_count") - parsed_cited_by_count: int | None = None - if isinstance(cited_by_count, int): - parsed_cited_by_count = cited_by_count - elif isinstance(cited_by_count, str) and cited_by_count.strip(): - try: - parsed_cited_by_count = int(cited_by_count) - except ValueError: - parsed_cited_by_count = None - normalized_candidates.append( - { - "scholar_id": scholar_id, - "display_name": display_name, - "affiliation": str(value.get("affiliation")).strip() - if value.get("affiliation") is not None - else None, - "email_domain": str(value.get("email_domain")).strip() - if value.get("email_domain") is not None - else None, - "cited_by_count": parsed_cited_by_count, - "interests": interests, - "profile_url": profile_url, - "profile_image_url": str(value.get("profile_image_url")).strip() - if value.get("profile_image_url") is not None - else None, - } - ) + normalized_candidates = _deserialize_candidates(payload) return ParsedAuthorSearchPage( state=state, @@ -552,278 +582,445 @@ async def delete_scholar( await db_session.commit() -async def search_author_candidates( - *, - source: ScholarSource, - db_session: AsyncSession, - query: str, - limit: int, - network_error_retries: int = 1, - retry_backoff_seconds: float = 1.0, - search_enabled: bool = True, - cache_ttl_seconds: int = 21_600, - blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS, - cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES, - min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS, - interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS, - cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD, - cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS, - retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, - cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD, -) -> ParsedAuthorSearchPage: +def _normalize_author_search_inputs(query: str, limit: int) -> tuple[str, int, str]: normalized_query = query.strip() if len(normalized_query) < 2: raise ScholarServiceError("Search query must be at least 2 characters.") bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT)) - query_key = normalized_query.casefold() + return normalized_query, bounded_limit, normalized_query.casefold() - if not search_enabled: - logger.warning( - "scholar_search.disabled_by_configuration", - extra={ - "event": "scholar_search.disabled_by_configuration", - "query": normalized_query, - }, - ) - return _policy_blocked_author_search_result( - reason=SEARCH_DISABLED_REASON, - warning_codes=["author_search_disabled_by_configuration"], - limit=bounded_limit, - ) - await _acquire_author_search_lock(db_session) - runtime_state = await _load_runtime_state_for_update(db_session) - runtime_state_updated = False - now_utc = datetime.now(timezone.utc) +def _disabled_search_result(*, normalized_query: str, bounded_limit: int) -> ParsedAuthorSearchPage: + logger.warning( + "scholar_search.disabled_by_configuration", + extra={"event": "scholar_search.disabled_by_configuration", "query": normalized_query}, + ) + return _policy_blocked_author_search_result( + reason=SEARCH_DISABLED_REASON, + warning_codes=["author_search_disabled_by_configuration"], + limit=bounded_limit, + ) - if runtime_state.cooldown_until is not None: - cooldown_until = runtime_state.cooldown_until - if cooldown_until.tzinfo is None: - cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) - runtime_state.cooldown_until = cooldown_until - runtime_state_updated = True - if now_utc >= cooldown_until: - logger.info( - "scholar_search.cooldown_expired", - extra={ - "event": "scholar_search.cooldown_expired", - "cooldown_until_utc": cooldown_until.isoformat(), - }, - ) - runtime_state.cooldown_until = None - runtime_state.cooldown_rejection_count = 0 - runtime_state.cooldown_alert_emitted = False - runtime_state_updated = True - cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(runtime_state, now_utc) - if cooldown_remaining_seconds > 0: - runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1 - bounded_cooldown_rejection_alert_threshold = max( - 1, - int(cooldown_rejection_alert_threshold), - ) - if ( - int(runtime_state.cooldown_rejection_count) >= bounded_cooldown_rejection_alert_threshold - and not bool(runtime_state.cooldown_alert_emitted) - ): - logger.error( - "scholar_search.cooldown_rejection_threshold_exceeded", - extra={ - "event": "scholar_search.cooldown_rejection_threshold_exceeded", - "query": normalized_query, - "cooldown_rejection_count": int(runtime_state.cooldown_rejection_count), - "threshold": bounded_cooldown_rejection_alert_threshold, - "cooldown_until_utc": runtime_state.cooldown_until.isoformat() - if runtime_state.cooldown_until - else None, - }, - ) - runtime_state.cooldown_alert_emitted = True - runtime_state_updated = True +def _normalize_runtime_cooldown_state( + runtime_state: AuthorSearchRuntimeState, + *, + now_utc: datetime, +) -> bool: + if runtime_state.cooldown_until is None: + return False + cooldown_until = runtime_state.cooldown_until + updated = False + if cooldown_until.tzinfo is None: + cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) + runtime_state.cooldown_until = cooldown_until + updated = True + if now_utc < cooldown_until: + return updated + logger.info( + "scholar_search.cooldown_expired", + extra={"event": "scholar_search.cooldown_expired", "cooldown_until_utc": cooldown_until.isoformat()}, + ) + runtime_state.cooldown_until = None + runtime_state.cooldown_rejection_count = 0 + runtime_state.cooldown_alert_emitted = False + return True - logger.warning( - "scholar_search.cooldown_active", - extra={ - "event": "scholar_search.cooldown_active", - "query": normalized_query, - "cooldown_remaining_seconds": cooldown_remaining_seconds, - "cooldown_until_utc": runtime_state.cooldown_until.isoformat() - if runtime_state.cooldown_until - else None, - }, - ) - warning_codes = [ - "author_search_cooldown_active", - f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s", - ] - if bool(runtime_state.cooldown_alert_emitted): - warning_codes.append("author_search_cooldown_alert_threshold_exceeded") - if runtime_state_updated: - await db_session.commit() - return _policy_blocked_author_search_result( - reason=SEARCH_COOLDOWN_REASON, - warning_codes=warning_codes, - limit=bounded_limit, - ) +def _cooldown_warning_codes( + *, + runtime_state: AuthorSearchRuntimeState, + cooldown_remaining_seconds: int, +) -> list[str]: + warning_codes = [ + "author_search_cooldown_active", + f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s", + ] + if bool(runtime_state.cooldown_alert_emitted): + warning_codes.append("author_search_cooldown_alert_threshold_exceeded") + return warning_codes + + +def _emit_cooldown_threshold_alert( + *, + runtime_state: AuthorSearchRuntimeState, + normalized_query: str, + cooldown_rejection_alert_threshold: int, +) -> bool: + runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1 + threshold = max(1, int(cooldown_rejection_alert_threshold)) + if int(runtime_state.cooldown_rejection_count) < threshold: + return True + if bool(runtime_state.cooldown_alert_emitted): + return True + logger.error( + "scholar_search.cooldown_rejection_threshold_exceeded", + extra={ + "event": "scholar_search.cooldown_rejection_threshold_exceeded", + "query": normalized_query, + "cooldown_rejection_count": int(runtime_state.cooldown_rejection_count), + "threshold": threshold, + "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, + }, + ) + runtime_state.cooldown_alert_emitted = True + return True + + +def _cooldown_block_result( + *, + runtime_state: AuthorSearchRuntimeState, + normalized_query: str, + bounded_limit: int, + cooldown_rejection_alert_threshold: int, + cooldown_remaining_seconds: int, +) -> ParsedAuthorSearchPage: + _emit_cooldown_threshold_alert( + runtime_state=runtime_state, + normalized_query=normalized_query, + cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, + ) + logger.warning( + "scholar_search.cooldown_active", + extra={ + "event": "scholar_search.cooldown_active", + "query": normalized_query, + "cooldown_remaining_seconds": cooldown_remaining_seconds, + "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, + }, + ) + return _policy_blocked_author_search_result( + reason=SEARCH_COOLDOWN_REASON, + warning_codes=_cooldown_warning_codes( + runtime_state=runtime_state, + cooldown_remaining_seconds=cooldown_remaining_seconds, + ), + limit=bounded_limit, + ) + + +async def _cache_hit_result( + db_session: AsyncSession, + *, + query_key: str, + now_utc: datetime, + normalized_query: str, + bounded_limit: int, +) -> ParsedAuthorSearchPage | None: cached = await _cache_get_author_search_result( db_session, query_key=query_key, now_utc=now_utc, ) - if cached is not None: - state_reason_override = ( - SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None - ) - logger.info( - "scholar_search.cache_hit", - extra={ - "event": "scholar_search.cache_hit", - "query": normalized_query, - "state": cached.state.value, - "state_reason": cached.state_reason, - }, - ) - await db_session.commit() - return _trim_author_search_result( - cached, - limit=bounded_limit, - extra_warnings=["author_search_served_from_cache"], - state_reason_override=state_reason_override, - ) + if cached is None: + return None + logger.info( + "scholar_search.cache_hit", + extra={ + "event": "scholar_search.cache_hit", + "query": normalized_query, + "state": cached.state.value, + "state_reason": cached.state_reason, + }, + ) + state_reason_override = SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None + return _trim_author_search_result( + cached, + limit=bounded_limit, + extra_warnings=["author_search_served_from_cache"], + state_reason_override=state_reason_override, + ) - if runtime_state.last_live_request_at is not None: + +def _throttle_sleep_seconds( + *, + runtime_state: AuthorSearchRuntimeState, + now_utc: datetime, + min_interval_seconds: float, + interval_jitter_seconds: float, +) -> tuple[float, bool]: + updated = False + if runtime_state.last_live_request_at is None: + enforced_wait_seconds = 0.0 + else: last_live_request_at = runtime_state.last_live_request_at if last_live_request_at.tzinfo is None: last_live_request_at = last_live_request_at.replace(tzinfo=timezone.utc) runtime_state.last_live_request_at = last_live_request_at - runtime_state_updated = True + updated = True enforced_wait_seconds = ( last_live_request_at + timedelta(seconds=max(float(min_interval_seconds), 0.0)) - now_utc ).total_seconds() - else: - enforced_wait_seconds = 0.0 - jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0)) - sleep_seconds = max(0.0, float(enforced_wait_seconds)) + jitter_seconds - if sleep_seconds > 0.0: - logger.info( - "scholar_search.throttle_wait", - extra={ - "event": "scholar_search.throttle_wait", - "query": normalized_query, - "sleep_seconds": round(sleep_seconds, 3), - }, - ) - await asyncio.sleep(sleep_seconds) + return max(0.0, float(enforced_wait_seconds)) + jitter_seconds, updated + +async def _wait_for_author_search_throttle( + *, + runtime_state: AuthorSearchRuntimeState, + normalized_query: str, + now_utc: datetime, + min_interval_seconds: float, + interval_jitter_seconds: float, +) -> bool: + sleep_seconds, updated = _throttle_sleep_seconds( + runtime_state=runtime_state, + now_utc=now_utc, + min_interval_seconds=min_interval_seconds, + interval_jitter_seconds=interval_jitter_seconds, + ) + if sleep_seconds <= 0.0: + return updated + logger.info( + "scholar_search.throttle_wait", + extra={"event": "scholar_search.throttle_wait", "query": normalized_query, "sleep_seconds": round(sleep_seconds, 3)}, + ) + await asyncio.sleep(sleep_seconds) + return True + + +async def _fetch_author_search_with_retries( + *, + source: ScholarSource, + normalized_query: str, + network_error_retries: int, + retry_backoff_seconds: float, +) -> tuple[ParsedAuthorSearchPage, int, list[str]]: max_attempts = max(1, int(network_error_retries) + 1) parsed: ParsedAuthorSearchPage | None = None retry_warnings: list[str] = [] retry_scheduled_count = 0 - for attempt_index in range(max_attempts): fetch_result = await source.fetch_author_search_html(normalized_query, start=0) parsed = parse_author_search_page(fetch_result) if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1: break - retry_warnings.append("network_retry_scheduled_for_author_search") retry_scheduled_count += 1 retry_sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index) if retry_sleep_seconds > 0: await asyncio.sleep(retry_sleep_seconds) - - runtime_state.last_live_request_at = datetime.now(timezone.utc) - runtime_state_updated = True - if parsed is None: raise ScholarServiceError("Unable to complete scholar author search.") + return parsed, retry_scheduled_count, retry_warnings - merged_parsed = replace( - parsed, - warnings=_merge_warnings(parsed.warnings, retry_warnings), + +def _with_retry_warnings( + parsed: ParsedAuthorSearchPage, + *, + retry_warnings: list[str], + retry_scheduled_count: int, + retry_alert_threshold: int, + normalized_query: str, +) -> ParsedAuthorSearchPage: + merged = replace(parsed, warnings=_merge_warnings(parsed.warnings, retry_warnings)) + threshold = max(1, int(retry_alert_threshold)) + if retry_scheduled_count < threshold: + return merged + logger.warning( + "scholar_search.retry_threshold_exceeded", + extra={ + "event": "scholar_search.retry_threshold_exceeded", + "query": normalized_query, + "retry_scheduled_count": retry_scheduled_count, + "threshold": threshold, + "final_state": merged.state.value, + "final_state_reason": merged.state_reason, + }, + ) + return replace( + merged, + warnings=_merge_warnings( + merged.warnings, + [f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"], + ), ) - bounded_retry_alert_threshold = max(1, int(retry_alert_threshold)) - if retry_scheduled_count >= bounded_retry_alert_threshold: - logger.warning( - "scholar_search.retry_threshold_exceeded", - extra={ - "event": "scholar_search.retry_threshold_exceeded", - "query": normalized_query, - "retry_scheduled_count": retry_scheduled_count, - "threshold": bounded_retry_alert_threshold, - "final_state": merged_parsed.state.value, - "final_state_reason": merged_parsed.state_reason, - }, - ) - merged_parsed = replace( - merged_parsed, - warnings=_merge_warnings( - merged_parsed.warnings, - [f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"], - ), - ) - if _is_author_search_block_state(merged_parsed): - runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1 - logger.warning( - "scholar_search.block_detected", - extra={ - "event": "scholar_search.block_detected", - "query": normalized_query, - "state_reason": merged_parsed.state_reason, - "consecutive_blocked_count": int(runtime_state.consecutive_blocked_count), - }, - ) - if int(runtime_state.consecutive_blocked_count) >= max(1, int(cooldown_block_threshold)): - runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta( - seconds=max(60, int(cooldown_seconds)) - ) - runtime_state.consecutive_blocked_count = 0 - runtime_state.cooldown_rejection_count = 0 - runtime_state.cooldown_alert_emitted = False - merged_parsed = replace( - merged_parsed, - warnings=_merge_warnings( - merged_parsed.warnings, - ["author_search_circuit_breaker_armed"], - ), - ) - logger.error( - "scholar_search.cooldown_activated", - extra={ - "event": "scholar_search.cooldown_activated", - "query": normalized_query, - "cooldown_until_utc": runtime_state.cooldown_until.isoformat() - if runtime_state.cooldown_until - else None, - }, - ) - else: +def _apply_block_circuit_breaker( + *, + runtime_state: AuthorSearchRuntimeState, + merged_parsed: ParsedAuthorSearchPage, + cooldown_block_threshold: int, + cooldown_seconds: int, + normalized_query: str, +) -> ParsedAuthorSearchPage: + if not _is_author_search_block_state(merged_parsed): runtime_state.consecutive_blocked_count = 0 + return merged_parsed + runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1 + logger.warning( + "scholar_search.block_detected", + extra={ + "event": "scholar_search.block_detected", + "query": normalized_query, + "state_reason": merged_parsed.state_reason, + "consecutive_blocked_count": int(runtime_state.consecutive_blocked_count), + }, + ) + if int(runtime_state.consecutive_blocked_count) < max(1, int(cooldown_block_threshold)): + return merged_parsed + runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta(seconds=max(60, int(cooldown_seconds))) + runtime_state.consecutive_blocked_count = 0 + runtime_state.cooldown_rejection_count = 0 + runtime_state.cooldown_alert_emitted = False + logger.error( + "scholar_search.cooldown_activated", + extra={ + "event": "scholar_search.cooldown_activated", + "query": normalized_query, + "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, + }, + ) + return replace( + merged_parsed, + warnings=_merge_warnings(merged_parsed.warnings, ["author_search_circuit_breaker_armed"]), + ) - ttl_seconds = ( - min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds))) - if _is_author_search_block_state(merged_parsed) - else max(1, int(cache_ttl_seconds)) + +def _resolve_author_search_cache_ttl_seconds( + *, + merged_parsed: ParsedAuthorSearchPage, + blocked_cache_ttl_seconds: int, + cache_ttl_seconds: int, +) -> int: + if _is_author_search_block_state(merged_parsed): + return min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds))) + return max(1, int(cache_ttl_seconds)) + + +async def _load_locked_runtime_state( + db_session: AsyncSession, +) -> AuthorSearchRuntimeState: + await _acquire_author_search_lock(db_session) + return await _load_runtime_state_for_update(db_session) + + +async def _cooldown_or_cache_result( + db_session: AsyncSession, + *, + runtime_state: AuthorSearchRuntimeState, + query_key: str, + normalized_query: str, + bounded_limit: int, + cooldown_rejection_alert_threshold: int, +) -> tuple[ParsedAuthorSearchPage | None, bool]: + runtime_state_updated = _normalize_runtime_cooldown_state( + runtime_state, + now_utc=datetime.now(timezone.utc), + ) + cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds( + runtime_state, + datetime.now(timezone.utc), + ) + if cooldown_remaining_seconds > 0: + return ( + _cooldown_block_result( + runtime_state=runtime_state, + normalized_query=normalized_query, + bounded_limit=bounded_limit, + cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, + cooldown_remaining_seconds=cooldown_remaining_seconds, + ), + True, + ) + cached_result = await _cache_hit_result( + db_session, + query_key=query_key, + now_utc=datetime.now(timezone.utc), + normalized_query=normalized_query, + bounded_limit=bounded_limit, + ) + return cached_result, runtime_state_updated + + +async def _perform_live_author_search(db_session: AsyncSession, *, source: ScholarSource, runtime_state: AuthorSearchRuntimeState, normalized_query: str, query_key: str, network_error_retries: int, retry_backoff_seconds: float, min_interval_seconds: float, interval_jitter_seconds: float, retry_alert_threshold: int, cooldown_block_threshold: int, cooldown_seconds: int, blocked_cache_ttl_seconds: int, cache_ttl_seconds: int, cache_max_entries: int) -> tuple[ParsedAuthorSearchPage, bool]: + runtime_state_updated = await _wait_for_author_search_throttle( + runtime_state=runtime_state, + normalized_query=normalized_query, + now_utc=datetime.now(timezone.utc), + min_interval_seconds=min_interval_seconds, + interval_jitter_seconds=interval_jitter_seconds, + ) + parsed, retry_count, retry_warnings = await _fetch_author_search_with_retries( + source=source, + normalized_query=normalized_query, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + ) + runtime_state.last_live_request_at = datetime.now(timezone.utc) + merged = _with_retry_warnings( + parsed, + retry_warnings=retry_warnings, + retry_scheduled_count=retry_count, + retry_alert_threshold=retry_alert_threshold, + normalized_query=normalized_query, + ) + merged = _apply_block_circuit_breaker( + runtime_state=runtime_state, + merged_parsed=merged, + cooldown_block_threshold=cooldown_block_threshold, + cooldown_seconds=cooldown_seconds, + normalized_query=normalized_query, + ) + ttl_seconds = _resolve_author_search_cache_ttl_seconds( + merged_parsed=merged, + blocked_cache_ttl_seconds=blocked_cache_ttl_seconds, + cache_ttl_seconds=cache_ttl_seconds, ) await _cache_set_author_search_result( db_session, query_key=query_key, - parsed=merged_parsed, + parsed=merged, ttl_seconds=float(ttl_seconds), max_entries=cache_max_entries, now_utc=datetime.now(timezone.utc), ) + return merged, True + + +async def search_author_candidates(*, source: ScholarSource, db_session: AsyncSession, query: str, limit: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, search_enabled: bool = True, cache_ttl_seconds: int = 21_600, blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS, cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES, min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS, interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS, cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD, cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS, retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD) -> ParsedAuthorSearchPage: + normalized_query, bounded_limit, query_key = _normalize_author_search_inputs(query, limit) + if not search_enabled: + return _disabled_search_result( + normalized_query=normalized_query, + bounded_limit=bounded_limit, + ) + + runtime_state = await _load_locked_runtime_state(db_session) + early_result, runtime_state_updated = await _cooldown_or_cache_result( + db_session, + runtime_state=runtime_state, + query_key=query_key, + normalized_query=normalized_query, + bounded_limit=bounded_limit, + cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, + ) + if early_result is not None: + await db_session.commit() + return early_result + + merged_parsed, live_runtime_updated = await _perform_live_author_search( + db_session, + source=source, + runtime_state=runtime_state, + normalized_query=normalized_query, + query_key=query_key, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + min_interval_seconds=min_interval_seconds, + interval_jitter_seconds=interval_jitter_seconds, + retry_alert_threshold=retry_alert_threshold, + cooldown_block_threshold=cooldown_block_threshold, + cooldown_seconds=cooldown_seconds, + blocked_cache_ttl_seconds=blocked_cache_ttl_seconds, + cache_ttl_seconds=cache_ttl_seconds, + cache_max_entries=cache_max_entries, + ) + runtime_state_updated = runtime_state_updated or live_runtime_updated if runtime_state_updated: await db_session.commit() - - return _trim_author_search_result( - merged_parsed, - limit=bounded_limit, - ) + return _trim_author_search_result(merged_parsed, limit=bounded_limit) async def hydrate_profile_metadata( diff --git a/frontend/src/features/publications/index.ts b/frontend/src/features/publications/index.ts index c5c1977..5c3f7fd 100644 --- a/frontend/src/features/publications/index.ts +++ b/frontend/src/features/publications/index.ts @@ -11,6 +11,7 @@ export interface PublicationItem { citation_count: number; venue_text: string | null; pub_url: string | null; + pdf_url: string | null; is_read: boolean; first_seen_at: string; is_new_in_latest_run: boolean; diff --git a/frontend/src/features/scholars/index.ts b/frontend/src/features/scholars/index.ts index 006082d..5b4edbd 100644 --- a/frontend/src/features/scholars/index.ts +++ b/frontend/src/features/scholars/index.ts @@ -37,6 +37,50 @@ export interface ScholarSearchResult { warnings: string[]; } +export interface ScholarExportItem { + scholar_id: string; + display_name: string | null; + is_enabled: boolean; + profile_image_override_url: string | null; +} + +export interface PublicationExportItem { + scholar_id: string; + cluster_id: string | null; + fingerprint_sha256: string | null; + title: string; + year: number | null; + citation_count: number; + author_text: string | null; + venue_text: string | null; + pub_url: string | null; + pdf_url: string | null; + is_read: boolean; +} + +export interface DataExportPayload { + schema_version: number; + exported_at: string; + scholars: ScholarExportItem[]; + publications: PublicationExportItem[]; +} + +export interface DataImportPayload { + schema_version?: number; + scholars: ScholarExportItem[]; + publications: PublicationExportItem[]; +} + +export interface DataImportResult { + scholars_created: number; + scholars_updated: number; + publications_created: number; + publications_updated: number; + links_created: number; + links_updated: number; + skipped_records: number; +} + interface ScholarsListData { scholars: ScholarProfile[]; } @@ -115,3 +159,20 @@ export async function clearScholarImage( }); return response.data; } + +export async function exportScholarData(): Promise { + const response = await apiRequest("/scholars/export", { + method: "GET", + }); + return response.data; +} + +export async function importScholarData( + payload: DataImportPayload, +): Promise { + const response = await apiRequest("/scholars/import", { + method: "POST", + body: payload, + }); + return response.data; +} diff --git a/frontend/src/pages/DashboardPage.vue b/frontend/src/pages/DashboardPage.vue index 211ce9e..8c26da7 100644 --- a/frontend/src/pages/DashboardPage.vue +++ b/frontend/src/pages/DashboardPage.vue @@ -1,5 +1,5 @@